Index: head/ObsoleteFiles.inc
===================================================================
--- head/ObsoleteFiles.inc	(revision 359373)
+++ head/ObsoleteFiles.inc	(revision 359374)
@@ -1,10795 +1,10800 @@
 #
 # $FreeBSD$
 #
 # This file lists old files (OLD_FILES), libraries (OLD_LIBS) and
 # directories (OLD_DIRS) which should get removed at an update. Recently
 # removed entries first (with the date as a comment). Dynamic libraries are
 # special cased (OLD_LIBS). Static libraries or the generic links to
 # the dynamic libraries (lib*.so) should (if you don't know why to make an
 # exception, make this a "must") be viewed as normal files (OLD_FILES).
 #
 # In case of a complete directory hierarchy the sorting is in depth first
 # order.
 #
 # Before you commit changes to this file please check if any entries in
 # tools/build/mk/OptionalObsoleteFiles.inc can be removed. The following
 # command tells which files are listed more than once regardless of some
 # architecture specific conditionals, so you can not blindly trust the
 # output:
 # ( grep '+=' /usr/src/ObsoleteFiles.inc | sort -u ; \
 # grep '+=' /usr/src/tools/build/mk/OptionalObsoleteFiles.inc | sort -u) | \
 # sort | uniq -d
 #
 # To find regular duplicates not dependent on optional components, you can
 # also use something that will not give you false positives, e.g.:
 # for t in `make -V TARGETS universe`; do
 #   __MAKE_CONF=/dev/null make -f Makefile.inc1 TARGET=$t \
 #   -V OLD_FILES -V OLD_LIBS -V OLD_DIRS check-old | \
 #   xargs -n1 | sort | uniq -d;
 # done
 #
 # For optional components, you can use the following to see if some entries
 # in OptionalObsoleteFiles.inc have been obsoleted by ObsoleteFiles.inc
 # for o in tools/build/options/WITH*; do
 #   __MAKE_CONF=/dev/null make -f Makefile.inc1 -D${o##*/} \
 #   -V OLD_FILES -V OLD_LIBS -V OLD_DIRS check-old | \
 #   xargs -n1 | sort | uniq -d;
 # done
 
+# 20200327: OCF refactoring
+OLD_FILES+=usr/share/man/man9/crypto_find_driver.9
+OLD_FILES+=usr/share/man/man9/crypto_register.9
+OLD_FILES+=usr/share/man/man9/crypto_unregister.9
+
 # 20200323: INTERNALLIB don't install headers anymore
 OLD_FILES+=usr/include/libelftc.h
 OLD_FILES+=usr/include/libifconfig.h
 OLD_FILES+=usr/include/libpmcstat.h
 
 # 20200320: cx and ctau drivers retired
 OLD_FILES+=usr/share/man/man4/ctau.4.gz
 OLD_FILES+=usr/share/man/man4/cx.4.gz
 
 # 20200318: host.conf was deprecated a long time ago.
 OLD_FILES+=etc/host.conf
 OLD_FILES+=etc/rc.d/nsswitch
 
 # 20200310: new clang import which bumps version from 9.0.1 to 10.0.0.
 OLD_FILES+=usr/lib/clang/9.0.1/include/cuda_wrappers/algorithm
 OLD_FILES+=usr/lib/clang/9.0.1/include/cuda_wrappers/complex
 OLD_FILES+=usr/lib/clang/9.0.1/include/cuda_wrappers/new
 OLD_DIRS+=usr/lib/clang/9.0.1/include/cuda_wrappers
 OLD_FILES+=usr/lib/clang/9.0.1/include/openmp_wrappers/__clang_openmp_math.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/openmp_wrappers/__clang_openmp_math_declares.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/openmp_wrappers/cmath
 OLD_FILES+=usr/lib/clang/9.0.1/include/openmp_wrappers/math.h
 OLD_DIRS+=usr/lib/clang/9.0.1/include/openmp_wrappers
 OLD_FILES+=usr/lib/clang/9.0.1/include/ppc_wrappers/emmintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/ppc_wrappers/mm_malloc.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/ppc_wrappers/mmintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/ppc_wrappers/xmmintrin.h
 OLD_DIRS+=usr/lib/clang/9.0.1/include/ppc_wrappers
 OLD_FILES+=usr/lib/clang/9.0.1/include/sanitizer/allocator_interface.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/sanitizer/asan_interface.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/sanitizer/common_interface_defs.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/sanitizer/coverage_interface.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/sanitizer/dfsan_interface.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/sanitizer/hwasan_interface.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/sanitizer/linux_syscall_hooks.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/sanitizer/lsan_interface.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/sanitizer/msan_interface.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/sanitizer/netbsd_syscall_hooks.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/sanitizer/scudo_interface.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/sanitizer/tsan_interface.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/sanitizer/tsan_interface_atomic.h
 OLD_DIRS+=usr/lib/clang/9.0.1/include/sanitizer
 OLD_FILES+=usr/lib/clang/9.0.1/include/__clang_cuda_builtin_vars.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/__clang_cuda_cmath.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/__clang_cuda_complex_builtins.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/__clang_cuda_device_functions.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/__clang_cuda_intrinsics.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/__clang_cuda_libdevice_declares.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/__clang_cuda_math_forward_declares.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/__clang_cuda_runtime_wrapper.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/__stddef_max_align_t.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/__wmmintrin_aes.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/__wmmintrin_pclmul.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/adxintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/altivec.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/ammintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/arm64intr.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/arm_acle.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/arm_fp16.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/arm_neon.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/armintr.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/avx2intrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/avx512bf16intrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/avx512bitalgintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/avx512bwintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/avx512cdintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/avx512dqintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/avx512erintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/avx512fintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/avx512ifmaintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/avx512ifmavlintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/avx512pfintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/avx512vbmi2intrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/avx512vbmiintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/avx512vbmivlintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/avx512vlbf16intrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/avx512vlbitalgintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/avx512vlbwintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/avx512vlcdintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/avx512vldqintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/avx512vlintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/avx512vlvbmi2intrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/avx512vlvnniintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/avx512vlvp2intersectintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/avx512vnniintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/avx512vp2intersectintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/avx512vpopcntdqintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/avx512vpopcntdqvlintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/avxintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/bmi2intrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/bmiintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/cetintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/cldemoteintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/clflushoptintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/clwbintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/clzerointrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/cpuid.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/emmintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/enqcmdintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/f16cintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/fma4intrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/fmaintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/fxsrintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/gfniintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/htmintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/htmxlintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/ia32intrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/immintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/invpcidintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/lwpintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/lzcntintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/mm3dnow.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/mm_malloc.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/mmintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/module.modulemap
 OLD_FILES+=usr/lib/clang/9.0.1/include/movdirintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/msa.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/mwaitxintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/nmmintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/opencl-c-base.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/opencl-c.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/pconfigintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/pkuintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/pmmintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/popcntintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/prfchwintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/ptwriteintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/rdseedintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/rtmintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/s390intrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/sgxintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/shaintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/smmintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/tbmintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/tmmintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/vadefs.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/vaesintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/vecintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/vpclmulqdqintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/waitpkgintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/wbnoinvdintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/wmmintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/x86intrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/xmmintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/xopintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/xsavecintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/xsaveintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/xsaveoptintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/xsavesintrin.h
 OLD_FILES+=usr/lib/clang/9.0.1/include/xtestintrin.h
 OLD_DIRS+=usr/lib/clang/9.0.1/include
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.asan-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.asan-aarch64.so
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.asan-arm.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.asan-arm.so
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.asan-armhf.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.asan-armhf.so
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.asan-i386.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.asan-i386.so
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.asan-preinit-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.asan-preinit-arm.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.asan-preinit-armhf.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.asan-preinit-i386.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.asan-preinit-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.asan-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.asan-x86_64.so
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.asan_cxx-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.asan_cxx-arm.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.asan_cxx-armhf.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.asan_cxx-i386.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.asan_cxx-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.cfi-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.cfi-arm.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.cfi-armhf.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.cfi-i386.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.cfi-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.cfi_diag-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.cfi_diag-arm.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.cfi_diag-armhf.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.cfi_diag-i386.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.cfi_diag-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.dd-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.dd-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.fuzzer-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.fuzzer-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.fuzzer_no_main-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.fuzzer_no_main-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.msan-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.msan-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.msan_cxx-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.msan_cxx-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.profile-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.profile-arm.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.profile-armhf.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.profile-i386.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.profile-powerpc.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.profile-powerpc64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.profile-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.safestack-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.safestack-i386.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.safestack-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.stats-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.stats-arm.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.stats-armhf.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.stats-i386.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.stats-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.stats_client-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.stats_client-arm.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.stats_client-armhf.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.stats_client-i386.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.stats_client-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.tsan-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.tsan-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.tsan_cxx-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.tsan_cxx-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.ubsan_minimal-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.ubsan_minimal-arm.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.ubsan_minimal-armhf.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.ubsan_minimal-i386.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.ubsan_minimal-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.ubsan_standalone-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.ubsan_standalone-arm.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.ubsan_standalone-armhf.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.ubsan_standalone-i386.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.ubsan_standalone-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.ubsan_standalone_cxx-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.ubsan_standalone_cxx-arm.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.ubsan_standalone_cxx-armhf.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.ubsan_standalone_cxx-i386.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.ubsan_standalone_cxx-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.xray-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.xray-arm.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.xray-armhf.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.xray-basic-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.xray-basic-arm.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.xray-basic-armhf.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.xray-basic-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.xray-fdr-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.xray-fdr-arm.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.xray-fdr-armhf.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.xray-fdr-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.xray-profiling-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.xray-profiling-arm.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.xray-profiling-armhf.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.xray-profiling-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.1/lib/freebsd/libclang_rt.xray-x86_64.a
 OLD_DIRS+=usr/lib/clang/9.0.1/lib/freebsd
 OLD_DIRS+=usr/lib/clang/9.0.1/lib
 OLD_DIRS+=usr/lib/clang/9.0.1
 
 # 20200309: amd(8) retired
 OLD_FILES+=etc/amd.map
 OLD_FILES+=etc/newsyslog.conf.d/amd.conf
 OLD_FILES+=etc/rc.d/amd
 OLD_FILES+=usr/bin/pawd
 OLD_FILES+=usr/sbin/amd
 OLD_FILES+=usr/sbin/amq
 OLD_FILES+=usr/sbin/fixmount
 OLD_FILES+=usr/sbin/fsinfo
 OLD_FILES+=usr/sbin/hlfsd
 OLD_FILES+=usr/sbin/mk-amd-map
 OLD_FILES+=usr/sbin/wire-test
 OLD_FILES+=usr/share/examples/etc/amd.map
 OLD_FILES+=usr/share/man/man1/pawd.1.gz
 OLD_FILES+=usr/share/man/man5/amd.conf.5.gz
 OLD_FILES+=usr/share/man/man8/amd.8.gz
 OLD_FILES+=usr/share/man/man8/amq.8.gz
 OLD_FILES+=usr/share/man/man8/fixmount.8.gz
 OLD_FILES+=usr/share/man/man8/fsinfo.8.gz
 OLD_FILES+=usr/share/man/man8/hlfsd.8.gz
 OLD_FILES+=usr/share/man/man8/mk-amd-map.8.gz
 OLD_FILES+=usr/share/man/man8/wire-test.8.gz
 
 # 20200301: bktr removed
 OLD_DIRS+=usr/include/dev/bktr
 OLD_FILES+=usr/include/dev/bktr/ioctl_bktr.h
 OLD_FILES+=usr/include/dev/bktr/ioctl_bt848.h
 OLD_FILES+=usr/include/dev/bktr/ioctl_meteor.h
 .if ${TARGET_ARCH} == "i386"
 OLD_FILES+=usr/include/machine/ioctl_bktr.h
 OLD_FILES+=usr/include/machine/ioctl_meteor.h
 .endif
 OLD_FILES+=usr/share/man/man4/bktr.4.gz
 
 # 20200229: GCC 4.2.1 removed
 .if !defined(WITH_PORT_BASE_GCC)
 OLD_FILES+=usr/bin/g++
 OLD_FILES+=usr/bin/gcc
 OLD_FILES+=usr/share/man/man1/g++.1.gz
 OLD_FILES+=usr/share/man/man1/gcc.1.gz
 .endif
 OLD_FILES+=usr/bin/gcpp
 OLD_FILES+=usr/bin/gperf
 OLD_FILES+=usr/include/c++/4.2/algorithm
 OLD_FILES+=usr/include/c++/4.2/backward/algo.h
 OLD_FILES+=usr/include/c++/4.2/backward/algobase.h
 OLD_FILES+=usr/include/c++/4.2/backward/alloc.h
 OLD_FILES+=usr/include/c++/4.2/backward/backward_warning.h
 OLD_FILES+=usr/include/c++/4.2/backward/bvector.h
 OLD_FILES+=usr/include/c++/4.2/backward/complex.h
 OLD_FILES+=usr/include/c++/4.2/backward/defalloc.h
 OLD_FILES+=usr/include/c++/4.2/backward/deque.h
 OLD_FILES+=usr/include/c++/4.2/backward/fstream.h
 OLD_FILES+=usr/include/c++/4.2/backward/function.h
 OLD_FILES+=usr/include/c++/4.2/backward/hash_map.h
 OLD_FILES+=usr/include/c++/4.2/backward/hash_set.h
 OLD_FILES+=usr/include/c++/4.2/backward/hashtable.h
 OLD_FILES+=usr/include/c++/4.2/backward/heap.h
 OLD_FILES+=usr/include/c++/4.2/backward/iomanip.h
 OLD_FILES+=usr/include/c++/4.2/backward/iostream.h
 OLD_FILES+=usr/include/c++/4.2/backward/istream.h
 OLD_FILES+=usr/include/c++/4.2/backward/iterator.h
 OLD_FILES+=usr/include/c++/4.2/backward/list.h
 OLD_FILES+=usr/include/c++/4.2/backward/map.h
 OLD_FILES+=usr/include/c++/4.2/backward/multimap.h
 OLD_FILES+=usr/include/c++/4.2/backward/multiset.h
 OLD_FILES+=usr/include/c++/4.2/backward/new.h
 OLD_FILES+=usr/include/c++/4.2/backward/ostream.h
 OLD_FILES+=usr/include/c++/4.2/backward/pair.h
 OLD_FILES+=usr/include/c++/4.2/backward/queue.h
 OLD_FILES+=usr/include/c++/4.2/backward/rope.h
 OLD_FILES+=usr/include/c++/4.2/backward/set.h
 OLD_FILES+=usr/include/c++/4.2/backward/slist.h
 OLD_FILES+=usr/include/c++/4.2/backward/stack.h
 OLD_FILES+=usr/include/c++/4.2/backward/stream.h
 OLD_FILES+=usr/include/c++/4.2/backward/streambuf.h
 OLD_FILES+=usr/include/c++/4.2/backward/strstream
 OLD_FILES+=usr/include/c++/4.2/backward/tempbuf.h
 OLD_FILES+=usr/include/c++/4.2/backward/tree.h
 OLD_FILES+=usr/include/c++/4.2/backward/vector.h
 OLD_FILES+=usr/include/c++/4.2/bits/allocator.h
 OLD_FILES+=usr/include/c++/4.2/bits/atomic_word.h
 OLD_FILES+=usr/include/c++/4.2/bits/basic_file.h
 OLD_FILES+=usr/include/c++/4.2/bits/basic_ios.h
 OLD_FILES+=usr/include/c++/4.2/bits/basic_ios.tcc
 OLD_FILES+=usr/include/c++/4.2/bits/basic_string.h
 OLD_FILES+=usr/include/c++/4.2/bits/basic_string.tcc
 OLD_FILES+=usr/include/c++/4.2/bits/boost_concept_check.h
 OLD_FILES+=usr/include/c++/4.2/bits/c++allocator.h
 OLD_FILES+=usr/include/c++/4.2/bits/c++config.h
 OLD_FILES+=usr/include/c++/4.2/bits/c++io.h
 OLD_FILES+=usr/include/c++/4.2/bits/c++locale.h
 OLD_FILES+=usr/include/c++/4.2/bits/c++locale_internal.h
 OLD_FILES+=usr/include/c++/4.2/bits/char_traits.h
 OLD_FILES+=usr/include/c++/4.2/bits/cmath.tcc
 OLD_FILES+=usr/include/c++/4.2/bits/codecvt.h
 OLD_FILES+=usr/include/c++/4.2/bits/compatibility.h
 OLD_FILES+=usr/include/c++/4.2/bits/concept_check.h
 OLD_FILES+=usr/include/c++/4.2/bits/cpp_type_traits.h
 OLD_FILES+=usr/include/c++/4.2/bits/cpu_defines.h
 OLD_FILES+=usr/include/c++/4.2/bits/ctype_base.h
 OLD_FILES+=usr/include/c++/4.2/bits/ctype_inline.h
 OLD_FILES+=usr/include/c++/4.2/bits/ctype_noninline.h
 OLD_FILES+=usr/include/c++/4.2/bits/cxxabi_tweaks.h
 OLD_FILES+=usr/include/c++/4.2/bits/deque.tcc
 OLD_FILES+=usr/include/c++/4.2/bits/fstream.tcc
 OLD_FILES+=usr/include/c++/4.2/bits/functexcept.h
 OLD_FILES+=usr/include/c++/4.2/bits/gslice.h
 OLD_FILES+=usr/include/c++/4.2/bits/gslice_array.h
 OLD_FILES+=usr/include/c++/4.2/bits/gthr-default.h
 OLD_FILES+=usr/include/c++/4.2/bits/gthr-posix.h
 OLD_FILES+=usr/include/c++/4.2/bits/gthr-single.h
 OLD_FILES+=usr/include/c++/4.2/bits/gthr-tpf.h
 OLD_FILES+=usr/include/c++/4.2/bits/gthr.h
 OLD_FILES+=usr/include/c++/4.2/bits/indirect_array.h
 OLD_FILES+=usr/include/c++/4.2/bits/ios_base.h
 OLD_FILES+=usr/include/c++/4.2/bits/istream.tcc
 OLD_FILES+=usr/include/c++/4.2/bits/list.tcc
 OLD_FILES+=usr/include/c++/4.2/bits/locale_classes.h
 OLD_FILES+=usr/include/c++/4.2/bits/locale_facets.h
 OLD_FILES+=usr/include/c++/4.2/bits/locale_facets.tcc
 OLD_FILES+=usr/include/c++/4.2/bits/localefwd.h
 OLD_FILES+=usr/include/c++/4.2/bits/mask_array.h
 OLD_FILES+=usr/include/c++/4.2/bits/messages_members.h
 OLD_FILES+=usr/include/c++/4.2/bits/os_defines.h
 OLD_FILES+=usr/include/c++/4.2/bits/ostream.tcc
 OLD_FILES+=usr/include/c++/4.2/bits/ostream_insert.h
 OLD_FILES+=usr/include/c++/4.2/bits/postypes.h
 OLD_FILES+=usr/include/c++/4.2/bits/slice_array.h
 OLD_FILES+=usr/include/c++/4.2/bits/sstream.tcc
 OLD_FILES+=usr/include/c++/4.2/bits/stl_algo.h
 OLD_FILES+=usr/include/c++/4.2/bits/stl_algobase.h
 OLD_FILES+=usr/include/c++/4.2/bits/stl_bvector.h
 OLD_FILES+=usr/include/c++/4.2/bits/stl_construct.h
 OLD_FILES+=usr/include/c++/4.2/bits/stl_deque.h
 OLD_FILES+=usr/include/c++/4.2/bits/stl_function.h
 OLD_FILES+=usr/include/c++/4.2/bits/stl_heap.h
 OLD_FILES+=usr/include/c++/4.2/bits/stl_iterator.h
 OLD_FILES+=usr/include/c++/4.2/bits/stl_iterator_base_funcs.h
 OLD_FILES+=usr/include/c++/4.2/bits/stl_iterator_base_types.h
 OLD_FILES+=usr/include/c++/4.2/bits/stl_list.h
 OLD_FILES+=usr/include/c++/4.2/bits/stl_map.h
 OLD_FILES+=usr/include/c++/4.2/bits/stl_multimap.h
 OLD_FILES+=usr/include/c++/4.2/bits/stl_multiset.h
 OLD_FILES+=usr/include/c++/4.2/bits/stl_numeric.h
 OLD_FILES+=usr/include/c++/4.2/bits/stl_pair.h
 OLD_FILES+=usr/include/c++/4.2/bits/stl_queue.h
 OLD_FILES+=usr/include/c++/4.2/bits/stl_raw_storage_iter.h
 OLD_FILES+=usr/include/c++/4.2/bits/stl_relops.h
 OLD_FILES+=usr/include/c++/4.2/bits/stl_set.h
 OLD_FILES+=usr/include/c++/4.2/bits/stl_stack.h
 OLD_FILES+=usr/include/c++/4.2/bits/stl_tempbuf.h
 OLD_FILES+=usr/include/c++/4.2/bits/stl_tree.h
 OLD_FILES+=usr/include/c++/4.2/bits/stl_uninitialized.h
 OLD_FILES+=usr/include/c++/4.2/bits/stl_vector.h
 OLD_FILES+=usr/include/c++/4.2/bits/stream_iterator.h
 OLD_FILES+=usr/include/c++/4.2/bits/streambuf.tcc
 OLD_FILES+=usr/include/c++/4.2/bits/streambuf_iterator.h
 OLD_FILES+=usr/include/c++/4.2/bits/stringfwd.h
 OLD_FILES+=usr/include/c++/4.2/bits/time_members.h
 OLD_FILES+=usr/include/c++/4.2/bits/valarray_after.h
 OLD_FILES+=usr/include/c++/4.2/bits/valarray_array.h
 OLD_FILES+=usr/include/c++/4.2/bits/valarray_array.tcc
 OLD_FILES+=usr/include/c++/4.2/bits/valarray_before.h
 OLD_FILES+=usr/include/c++/4.2/bits/vector.tcc
 OLD_FILES+=usr/include/c++/4.2/bitset
 OLD_FILES+=usr/include/c++/4.2/cassert
 OLD_FILES+=usr/include/c++/4.2/cctype
 OLD_FILES+=usr/include/c++/4.2/cerrno
 OLD_FILES+=usr/include/c++/4.2/cfloat
 OLD_FILES+=usr/include/c++/4.2/ciso646
 OLD_FILES+=usr/include/c++/4.2/climits
 OLD_FILES+=usr/include/c++/4.2/clocale
 OLD_FILES+=usr/include/c++/4.2/cmath
 OLD_FILES+=usr/include/c++/4.2/complex
 OLD_FILES+=usr/include/c++/4.2/csetjmp
 OLD_FILES+=usr/include/c++/4.2/csignal
 OLD_FILES+=usr/include/c++/4.2/cstdarg
 OLD_FILES+=usr/include/c++/4.2/cstddef
 OLD_FILES+=usr/include/c++/4.2/cstdio
 OLD_FILES+=usr/include/c++/4.2/cstdlib
 OLD_FILES+=usr/include/c++/4.2/cstring
 OLD_FILES+=usr/include/c++/4.2/ctime
 OLD_FILES+=usr/include/c++/4.2/cwchar
 OLD_FILES+=usr/include/c++/4.2/cwctype
 OLD_FILES+=usr/include/c++/4.2/cxxabi.h
 OLD_FILES+=usr/include/c++/4.2/debug/bitset
 OLD_FILES+=usr/include/c++/4.2/debug/debug.h
 OLD_FILES+=usr/include/c++/4.2/debug/deque
 OLD_FILES+=usr/include/c++/4.2/debug/formatter.h
 OLD_FILES+=usr/include/c++/4.2/debug/functions.h
 OLD_FILES+=usr/include/c++/4.2/debug/hash_map
 OLD_FILES+=usr/include/c++/4.2/debug/hash_map.h
 OLD_FILES+=usr/include/c++/4.2/debug/hash_multimap.h
 OLD_FILES+=usr/include/c++/4.2/debug/hash_multiset.h
 OLD_FILES+=usr/include/c++/4.2/debug/hash_set
 OLD_FILES+=usr/include/c++/4.2/debug/hash_set.h
 OLD_FILES+=usr/include/c++/4.2/debug/list
 OLD_FILES+=usr/include/c++/4.2/debug/macros.h
 OLD_FILES+=usr/include/c++/4.2/debug/map
 OLD_FILES+=usr/include/c++/4.2/debug/map.h
 OLD_FILES+=usr/include/c++/4.2/debug/multimap.h
 OLD_FILES+=usr/include/c++/4.2/debug/multiset.h
 OLD_FILES+=usr/include/c++/4.2/debug/safe_base.h
 OLD_FILES+=usr/include/c++/4.2/debug/safe_iterator.h
 OLD_FILES+=usr/include/c++/4.2/debug/safe_iterator.tcc
 OLD_FILES+=usr/include/c++/4.2/debug/safe_sequence.h
 OLD_FILES+=usr/include/c++/4.2/debug/set
 OLD_FILES+=usr/include/c++/4.2/debug/set.h
 OLD_FILES+=usr/include/c++/4.2/debug/string
 OLD_FILES+=usr/include/c++/4.2/debug/vector
 OLD_FILES+=usr/include/c++/4.2/deque
 OLD_FILES+=usr/include/c++/4.2/exception
 OLD_FILES+=usr/include/c++/4.2/exception_defines.h
 OLD_FILES+=usr/include/c++/4.2/ext/algorithm
 OLD_FILES+=usr/include/c++/4.2/ext/array_allocator.h
 OLD_FILES+=usr/include/c++/4.2/ext/atomicity.h
 OLD_FILES+=usr/include/c++/4.2/ext/bitmap_allocator.h
 OLD_FILES+=usr/include/c++/4.2/ext/codecvt_specializations.h
 OLD_FILES+=usr/include/c++/4.2/ext/concurrence.h
 OLD_FILES+=usr/include/c++/4.2/ext/debug_allocator.h
 OLD_FILES+=usr/include/c++/4.2/ext/functional
 OLD_FILES+=usr/include/c++/4.2/ext/hash_fun.h
 OLD_FILES+=usr/include/c++/4.2/ext/hash_map
 OLD_FILES+=usr/include/c++/4.2/ext/hash_set
 OLD_FILES+=usr/include/c++/4.2/ext/hashtable.h
 OLD_FILES+=usr/include/c++/4.2/ext/iterator
 OLD_FILES+=usr/include/c++/4.2/ext/malloc_allocator.h
 OLD_FILES+=usr/include/c++/4.2/ext/memory
 OLD_FILES+=usr/include/c++/4.2/ext/mt_allocator.h
 OLD_FILES+=usr/include/c++/4.2/ext/new_allocator.h
 OLD_FILES+=usr/include/c++/4.2/ext/numeric
 OLD_FILES+=usr/include/c++/4.2/ext/numeric_traits.h
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/assoc_container.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/basic_tree_policy/basic_tree_policy_base.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/basic_tree_policy/null_node_metadata.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/basic_tree_policy/traits.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/basic_types.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/bin_search_tree_/bin_search_tree_.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/bin_search_tree_/cond_dtor_entry_dealtor.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/bin_search_tree_/cond_key_dtor_entry_dealtor.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/bin_search_tree_/constructors_destructor_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/bin_search_tree_/debug_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/bin_search_tree_/erase_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/bin_search_tree_/find_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/bin_search_tree_/info_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/bin_search_tree_/insert_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/bin_search_tree_/iterators_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/bin_search_tree_/node_iterators.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/bin_search_tree_/point_iterators.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/bin_search_tree_/policy_access_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/bin_search_tree_/r_erase_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/bin_search_tree_/rotate_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/bin_search_tree_/split_join_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/bin_search_tree_/traits.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binary_heap_/binary_heap_.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binary_heap_/const_iterator.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binary_heap_/const_point_iterator.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binary_heap_/constructors_destructor_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binary_heap_/debug_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binary_heap_/entry_cmp.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binary_heap_/entry_pred.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binary_heap_/erase_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binary_heap_/find_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binary_heap_/info_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binary_heap_/insert_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binary_heap_/iterators_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binary_heap_/policy_access_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binary_heap_/resize_policy.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binary_heap_/split_join_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binary_heap_/trace_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binomial_heap_/binomial_heap_.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binomial_heap_/constructors_destructor_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binomial_heap_/debug_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binomial_heap_base_/binomial_heap_base_.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binomial_heap_base_/constructors_destructor_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binomial_heap_base_/debug_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binomial_heap_base_/erase_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binomial_heap_base_/find_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binomial_heap_base_/insert_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/binomial_heap_base_/split_join_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/cc_ht_map_.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/cmp_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/cond_key_dtor_entry_dealtor.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/constructor_destructor_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/constructor_destructor_no_store_hash_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/constructor_destructor_store_hash_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/debug_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/debug_no_store_hash_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/debug_store_hash_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/entry_list_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/erase_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/erase_no_store_hash_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/erase_store_hash_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/find_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/find_store_hash_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/info_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/insert_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/insert_no_store_hash_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/insert_store_hash_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/iterators_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/policy_access_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/resize_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/resize_no_store_hash_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/resize_store_hash_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/size_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/standard_policies.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cc_hash_table_map_/trace_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/cond_dealtor.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/constructors_destructor_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/container_base_dispatch.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/eq_fn/eq_by_less.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/eq_fn/hash_eq_fn.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/constructor_destructor_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/constructor_destructor_no_store_hash_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/constructor_destructor_store_hash_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/debug_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/debug_no_store_hash_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/debug_store_hash_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/erase_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/erase_no_store_hash_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/erase_store_hash_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/find_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/find_no_store_hash_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/find_store_hash_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/gp_ht_map_.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/info_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/insert_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/insert_no_store_hash_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/insert_store_hash_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/iterator_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/policy_access_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/resize_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/resize_no_store_hash_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/resize_store_hash_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/standard_policies.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/gp_hash_table_map_/trace_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/hash_fn/direct_mask_range_hashing_imp.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/hash_fn/direct_mod_range_hashing_imp.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/hash_fn/linear_probe_fn_imp.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/hash_fn/mask_based_range_hashing.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/hash_fn/mod_based_range_hashing.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/hash_fn/probe_fn_base.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/hash_fn/quadratic_probe_fn_imp.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/hash_fn/ranged_hash_fn.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/hash_fn/ranged_probe_fn.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/hash_fn/sample_probe_fn.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/hash_fn/sample_range_hashing.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/hash_fn/sample_ranged_hash_fn.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/hash_fn/sample_ranged_probe_fn.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/left_child_next_sibling_heap_/const_iterator.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/left_child_next_sibling_heap_/const_point_iterator.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/left_child_next_sibling_heap_/constructors_destructor_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/left_child_next_sibling_heap_/debug_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/left_child_next_sibling_heap_/erase_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/left_child_next_sibling_heap_/info_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/left_child_next_sibling_heap_/insert_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/left_child_next_sibling_heap_/iterators_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/left_child_next_sibling_heap_/left_child_next_sibling_heap_.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/left_child_next_sibling_heap_/node.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/left_child_next_sibling_heap_/null_metadata.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/left_child_next_sibling_heap_/policy_access_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/left_child_next_sibling_heap_/trace_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/list_update_map_/constructor_destructor_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/list_update_map_/debug_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/list_update_map_/entry_metadata_base.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/list_update_map_/erase_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/list_update_map_/find_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/list_update_map_/info_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/list_update_map_/insert_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/list_update_map_/iterators_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/list_update_map_/lu_map_.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/list_update_map_/trace_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/list_update_policy/counter_lu_metadata.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/list_update_policy/counter_lu_policy_imp.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/list_update_policy/mtf_lu_policy_imp.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/list_update_policy/sample_update_policy.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/map_debug_base.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/ov_tree_map_/cond_dtor.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/ov_tree_map_/constructors_destructor_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/ov_tree_map_/debug_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/ov_tree_map_/erase_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/ov_tree_map_/info_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/ov_tree_map_/insert_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/ov_tree_map_/iterators_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/ov_tree_map_/node_iterators.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/ov_tree_map_/ov_tree_map_.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/ov_tree_map_/policy_access_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/ov_tree_map_/split_join_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/ov_tree_map_/traits.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pairing_heap_/constructors_destructor_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pairing_heap_/debug_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pairing_heap_/erase_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pairing_heap_/find_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pairing_heap_/insert_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pairing_heap_/pairing_heap_.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pairing_heap_/split_join_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/child_iterator.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/cond_dtor_entry_dealtor.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/const_child_iterator.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/constructors_destructor_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/debug_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/erase_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/find_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/head.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/info_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/insert_join_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/internal_node.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/iterators_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/leaf.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/node_base.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/node_iterators.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/node_metadata_base.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/pat_trie_.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/point_iterators.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/policy_access_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/r_erase_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/rotate_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/split_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/split_join_branch_bag.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/synth_e_access_traits.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/trace_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/traits.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/pat_trie_/update_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/priority_queue_base_dispatch.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/rb_tree_map_/constructors_destructor_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/rb_tree_map_/debug_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/rb_tree_map_/erase_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/rb_tree_map_/find_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/rb_tree_map_/info_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/rb_tree_map_/insert_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/rb_tree_map_/node.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/rb_tree_map_/rb_tree_.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/rb_tree_map_/split_join_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/rb_tree_map_/traits.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/rc_binomial_heap_/constructors_destructor_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/rc_binomial_heap_/debug_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/rc_binomial_heap_/erase_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/rc_binomial_heap_/insert_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/rc_binomial_heap_/rc.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/rc_binomial_heap_/rc_binomial_heap_.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/rc_binomial_heap_/split_join_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/rc_binomial_heap_/trace_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/resize_policy/cc_hash_max_collision_check_resize_trigger_imp.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/resize_policy/hash_exponential_size_policy_imp.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/resize_policy/hash_load_check_resize_trigger_imp.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/resize_policy/hash_load_check_resize_trigger_size_base.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/resize_policy/hash_prime_size_policy_imp.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/resize_policy/hash_standard_resize_policy_imp.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/resize_policy/sample_resize_policy.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/resize_policy/sample_resize_trigger.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/resize_policy/sample_size_policy.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/splay_tree_/constructors_destructor_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/splay_tree_/debug_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/splay_tree_/erase_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/splay_tree_/find_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/splay_tree_/info_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/splay_tree_/insert_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/splay_tree_/node.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/splay_tree_/splay_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/splay_tree_/splay_tree_.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/splay_tree_/split_join_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/splay_tree_/traits.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/standard_policies.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/thin_heap_/constructors_destructor_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/thin_heap_/debug_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/thin_heap_/erase_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/thin_heap_/find_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/thin_heap_/insert_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/thin_heap_/split_join_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/thin_heap_/thin_heap_.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/thin_heap_/trace_fn_imps.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/tree_policy/node_metadata_selector.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/tree_policy/null_node_update_imp.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/tree_policy/order_statistics_imp.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/tree_policy/sample_tree_node_update.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/tree_trace_base.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/trie_policy/node_metadata_selector.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/trie_policy/null_node_update_imp.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/trie_policy/order_statistics_imp.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/trie_policy/prefix_search_node_update_imp.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/trie_policy/sample_trie_e_access_traits.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/trie_policy/sample_trie_node_update.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/trie_policy/string_trie_e_access_traits_imp.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/trie_policy/trie_policy_base.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/type_utils.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/types_traits.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/unordered_iterator/const_iterator.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/unordered_iterator/const_point_iterator.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/unordered_iterator/iterator.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/detail/unordered_iterator/point_iterator.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/exception.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/hash_policy.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/list_update_policy.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/priority_queue.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/tag_and_trait.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/tree_policy.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pb_ds/trie_policy.hpp
 OLD_FILES+=usr/include/c++/4.2/ext/pod_char_traits.h
 OLD_FILES+=usr/include/c++/4.2/ext/pool_allocator.h
 OLD_FILES+=usr/include/c++/4.2/ext/rb_tree
 OLD_FILES+=usr/include/c++/4.2/ext/rc_string_base.h
 OLD_FILES+=usr/include/c++/4.2/ext/rope
 OLD_FILES+=usr/include/c++/4.2/ext/ropeimpl.h
 OLD_FILES+=usr/include/c++/4.2/ext/slist
 OLD_FILES+=usr/include/c++/4.2/ext/sso_string_base.h
 OLD_FILES+=usr/include/c++/4.2/ext/stdio_filebuf.h
 OLD_FILES+=usr/include/c++/4.2/ext/stdio_sync_filebuf.h
 OLD_FILES+=usr/include/c++/4.2/ext/throw_allocator.h
 OLD_FILES+=usr/include/c++/4.2/ext/type_traits.h
 OLD_FILES+=usr/include/c++/4.2/ext/typelist.h
 OLD_FILES+=usr/include/c++/4.2/ext/vstring.h
 OLD_FILES+=usr/include/c++/4.2/ext/vstring.tcc
 OLD_FILES+=usr/include/c++/4.2/ext/vstring_fwd.h
 OLD_FILES+=usr/include/c++/4.2/ext/vstring_util.h
 OLD_FILES+=usr/include/c++/4.2/fstream
 OLD_FILES+=usr/include/c++/4.2/functional
 OLD_FILES+=usr/include/c++/4.2/iomanip
 OLD_FILES+=usr/include/c++/4.2/ios
 OLD_FILES+=usr/include/c++/4.2/iosfwd
 OLD_FILES+=usr/include/c++/4.2/iostream
 OLD_FILES+=usr/include/c++/4.2/istream
 OLD_FILES+=usr/include/c++/4.2/iterator
 OLD_FILES+=usr/include/c++/4.2/limits
 OLD_FILES+=usr/include/c++/4.2/list
 OLD_FILES+=usr/include/c++/4.2/locale
 OLD_FILES+=usr/include/c++/4.2/map
 OLD_FILES+=usr/include/c++/4.2/memory
 OLD_FILES+=usr/include/c++/4.2/new
 OLD_FILES+=usr/include/c++/4.2/numeric
 OLD_FILES+=usr/include/c++/4.2/ostream
 OLD_FILES+=usr/include/c++/4.2/queue
 OLD_FILES+=usr/include/c++/4.2/set
 OLD_FILES+=usr/include/c++/4.2/sstream
 OLD_FILES+=usr/include/c++/4.2/stack
 OLD_FILES+=usr/include/c++/4.2/stdexcept
 OLD_FILES+=usr/include/c++/4.2/streambuf
 OLD_FILES+=usr/include/c++/4.2/string
 OLD_FILES+=usr/include/c++/4.2/tr1/array
 OLD_FILES+=usr/include/c++/4.2/tr1/bind_iterate.h
 OLD_FILES+=usr/include/c++/4.2/tr1/bind_repeat.h
 OLD_FILES+=usr/include/c++/4.2/tr1/boost_shared_ptr.h
 OLD_FILES+=usr/include/c++/4.2/tr1/cctype
 OLD_FILES+=usr/include/c++/4.2/tr1/cfenv
 OLD_FILES+=usr/include/c++/4.2/tr1/cfloat
 OLD_FILES+=usr/include/c++/4.2/tr1/cinttypes
 OLD_FILES+=usr/include/c++/4.2/tr1/climits
 OLD_FILES+=usr/include/c++/4.2/tr1/cmath
 OLD_FILES+=usr/include/c++/4.2/tr1/common.h
 OLD_FILES+=usr/include/c++/4.2/tr1/complex
 OLD_FILES+=usr/include/c++/4.2/tr1/cstdarg
 OLD_FILES+=usr/include/c++/4.2/tr1/cstdbool
 OLD_FILES+=usr/include/c++/4.2/tr1/cstdint
 OLD_FILES+=usr/include/c++/4.2/tr1/cstdio
 OLD_FILES+=usr/include/c++/4.2/tr1/cstdlib
 OLD_FILES+=usr/include/c++/4.2/tr1/ctgmath
 OLD_FILES+=usr/include/c++/4.2/tr1/ctime
 OLD_FILES+=usr/include/c++/4.2/tr1/ctype.h
 OLD_FILES+=usr/include/c++/4.2/tr1/cwchar
 OLD_FILES+=usr/include/c++/4.2/tr1/cwctype
 OLD_FILES+=usr/include/c++/4.2/tr1/fenv.h
 OLD_FILES+=usr/include/c++/4.2/tr1/float.h
 OLD_FILES+=usr/include/c++/4.2/tr1/functional
 OLD_FILES+=usr/include/c++/4.2/tr1/functional_hash.h
 OLD_FILES+=usr/include/c++/4.2/tr1/functional_iterate.h
 OLD_FILES+=usr/include/c++/4.2/tr1/hashtable
 OLD_FILES+=usr/include/c++/4.2/tr1/hashtable_policy.h
 OLD_FILES+=usr/include/c++/4.2/tr1/inttypes.h
 OLD_FILES+=usr/include/c++/4.2/tr1/limits.h
 OLD_FILES+=usr/include/c++/4.2/tr1/math.h
 OLD_FILES+=usr/include/c++/4.2/tr1/memory
 OLD_FILES+=usr/include/c++/4.2/tr1/mu_iterate.h
 OLD_FILES+=usr/include/c++/4.2/tr1/random
 OLD_FILES+=usr/include/c++/4.2/tr1/random.tcc
 OLD_FILES+=usr/include/c++/4.2/tr1/ref_fwd.h
 OLD_FILES+=usr/include/c++/4.2/tr1/ref_wrap_iterate.h
 OLD_FILES+=usr/include/c++/4.2/tr1/repeat.h
 OLD_FILES+=usr/include/c++/4.2/tr1/stdarg.h
 OLD_FILES+=usr/include/c++/4.2/tr1/stdbool.h
 OLD_FILES+=usr/include/c++/4.2/tr1/stdint.h
 OLD_FILES+=usr/include/c++/4.2/tr1/stdio.h
 OLD_FILES+=usr/include/c++/4.2/tr1/stdlib.h
 OLD_FILES+=usr/include/c++/4.2/tr1/tgmath.h
 OLD_FILES+=usr/include/c++/4.2/tr1/tuple
 OLD_FILES+=usr/include/c++/4.2/tr1/tuple_defs.h
 OLD_FILES+=usr/include/c++/4.2/tr1/tuple_iterate.h
 OLD_FILES+=usr/include/c++/4.2/tr1/type_traits
 OLD_FILES+=usr/include/c++/4.2/tr1/type_traits_fwd.h
 OLD_FILES+=usr/include/c++/4.2/tr1/unordered_map
 OLD_FILES+=usr/include/c++/4.2/tr1/unordered_set
 OLD_FILES+=usr/include/c++/4.2/tr1/utility
 OLD_FILES+=usr/include/c++/4.2/tr1/wchar.h
 OLD_FILES+=usr/include/c++/4.2/tr1/wctype.h
 OLD_FILES+=usr/include/c++/4.2/typeinfo
 OLD_FILES+=usr/include/c++/4.2/utility
 OLD_FILES+=usr/include/c++/4.2/valarray
 OLD_FILES+=usr/include/c++/4.2/vector
 .if ${TARGET_ARCH} == "amd64" || ${TARGET_ARCH} == "i386"
 OLD_FILES+=usr/include/gcc/4.2/__wmmintrin_aes.h
 OLD_FILES+=usr/include/gcc/4.2/__wmmintrin_pclmul.h
 OLD_FILES+=usr/include/gcc/4.2/ammintrin.h
 OLD_FILES+=usr/include/gcc/4.2/emmintrin.h
 OLD_FILES+=usr/include/gcc/4.2/mm3dnow.h
 OLD_FILES+=usr/include/gcc/4.2/mm_malloc.h
 OLD_FILES+=usr/include/gcc/4.2/mmintrin.h
 OLD_FILES+=usr/include/gcc/4.2/pmmintrin.h
 OLD_FILES+=usr/include/gcc/4.2/tmmintrin.h
 OLD_FILES+=usr/include/gcc/4.2/wmmintrin.h
 OLD_FILES+=usr/include/gcc/4.2/xmmintrin.h
 .elif ${TARGET_ARCH} == "arm"
 OLD_FILES+=usr/include/gcc/4.2/mmintrin.h
 .elif ${TARGET_ARCH} == "powerpc" || ${TARGET_ARCH} == "powerpc64"
 OLD_FILES+=usr/include/gcc/4.2/altivec.h
 OLD_FILES+=usr/include/gcc/4.2/ppc-asm.h
 OLD_FILES+=usr/include/gcc/4.2/spe.h
 .endif
 OLD_FILES+=usr/lib/libgcov.a
 OLD_FILES+=usr/lib/libgomp.a
 OLD_FILES+=usr/lib/libstdc++.a
 OLD_FILES+=usr/lib/libstdc++.so
 OLD_LIBS+=usr/lib/libstdc++.so.6
 OLD_FILES+=usr/lib/libstdc++_p.a
 OLD_FILES+=usr/lib/libsupc++.a
 OLD_FILES+=usr/lib/libsupc++.so
 OLD_LIBS+=usr/lib/libsupc++.so.1
 OLD_FILES+=usr/lib/libsupc++_p.a
 .if ${TARGET_ARCH} == "amd64" || ${TARGET_ARCH} == "powerpc64"
 OLD_FILES+=usr/lib32/libstdc++.a
 OLD_FILES+=usr/lib32/libstdc++.so
 OLD_LIBS+=usr/lib32/libstdc++.so.6
 OLD_FILES+=usr/lib32/libstdc++_p.a
 OLD_FILES+=usr/lib32/libsupc++.a
 OLD_FILES+=usr/lib32/libsupc++.so
 OLD_LIBS+=usr/lib32/libsupc++.so.1
 OLD_FILES+=usr/lib32/libsupc++_p.a
 .endif
 OLD_LIBS+=usr/lib/libgomp.so.1
 OLD_FILES+=usr/lib/libgomp_p.a
 OLD_FILES+=usr/lib32/libgcov.a
 OLD_FILES+=usr/lib32/libgomp.a
 OLD_LIBS+=usr/lib32/libgomp.so.1
 OLD_FILES+=usr/lib32/libgomp_p.a
 OLD_FILES+=usr/libexec/cc1
 OLD_FILES+=usr/libexec/cc1plus
 OLD_FILES+=usr/share/man/man1/gcpp.1.gz
 OLD_FILES+=usr/share/man/man1/gperf.1.gz
 OLD_FILES+=usr/share/man/man1/gperf.7.gz
 
 # 20200220: Upgrade of ncurses, shlib bumped to version 9
 OLD_LIBS+=lib/libncurses.so.8
 OLD_LIBS+=lib/libncursesw.so.8
 OLD_LIBS+=usr/lib32/libncurses.so.8
 OLD_LIBS+=usr/lib32/libncursesw.so.8
 
 # 20200206: Remove elf2aout
 OLD_FILES+=usr/bin/elf2aout
 OLD_FILES+=usr/share/man/man1/elf2aout.1.gz
 
 # 20200204: simple_httpd removed
 OLD_FILES+=usr/sbin/simple_httpd
 
 # 20200127: vpo removed
 OLD_FILES+=usr/share/man/man4/imm.4.gz
 OLD_FILES+=usr/share/man/man4/vpo.4.gz
 
 # 20200104: gcc libssp removed
 OLD_FILES+=usr/include/ssp/ssp.h
 OLD_FILES+=usr/include/ssp/stdio.h
 OLD_FILES+=usr/include/ssp/string.h
 OLD_FILES+=usr/include/ssp/unistd.h
 OLD_DIRS+=usr/include/ssp
 
 # 20191229: GEOM_SCHED class and gsched tool removed
 OLD_FILES+=sbin/gsched
 OLD_LIBS+=lib/geom/geom_sched.so
 
 # 20191222: new clang import which bumps version from 9.0.0 to 9.0.1.
 OLD_FILES+=usr/lib/clang/9.0.0/include/cuda_wrappers/algorithm
 OLD_FILES+=usr/lib/clang/9.0.0/include/cuda_wrappers/complex
 OLD_FILES+=usr/lib/clang/9.0.0/include/cuda_wrappers/new
 OLD_DIRS+=usr/lib/clang/9.0.0/include/cuda_wrappers
 OLD_FILES+=usr/lib/clang/9.0.0/include/openmp_wrappers/__clang_openmp_math.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/openmp_wrappers/__clang_openmp_math_declares.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/openmp_wrappers/cmath
 OLD_FILES+=usr/lib/clang/9.0.0/include/openmp_wrappers/math.h
 OLD_DIRS+=usr/lib/clang/9.0.0/include/openmp_wrappers
 OLD_FILES+=usr/lib/clang/9.0.0/include/ppc_wrappers/emmintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/ppc_wrappers/mm_malloc.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/ppc_wrappers/mmintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/ppc_wrappers/xmmintrin.h
 OLD_DIRS+=usr/lib/clang/9.0.0/include/ppc_wrappers
 OLD_FILES+=usr/lib/clang/9.0.0/include/sanitizer/allocator_interface.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/sanitizer/asan_interface.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/sanitizer/common_interface_defs.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/sanitizer/coverage_interface.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/sanitizer/dfsan_interface.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/sanitizer/hwasan_interface.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/sanitizer/linux_syscall_hooks.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/sanitizer/lsan_interface.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/sanitizer/msan_interface.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/sanitizer/netbsd_syscall_hooks.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/sanitizer/scudo_interface.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/sanitizer/tsan_interface.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/sanitizer/tsan_interface_atomic.h
 OLD_DIRS+=usr/lib/clang/9.0.0/include/sanitizer
 OLD_FILES+=usr/lib/clang/9.0.0/include/__clang_cuda_builtin_vars.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/__clang_cuda_cmath.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/__clang_cuda_complex_builtins.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/__clang_cuda_device_functions.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/__clang_cuda_intrinsics.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/__clang_cuda_libdevice_declares.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/__clang_cuda_math_forward_declares.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/__clang_cuda_runtime_wrapper.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/__stddef_max_align_t.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/__wmmintrin_aes.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/__wmmintrin_pclmul.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/adxintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/altivec.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/ammintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/arm64intr.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/arm_acle.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/arm_fp16.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/arm_neon.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/armintr.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/avx2intrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/avx512bf16intrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/avx512bitalgintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/avx512bwintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/avx512cdintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/avx512dqintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/avx512erintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/avx512fintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/avx512ifmaintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/avx512ifmavlintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/avx512pfintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/avx512vbmi2intrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/avx512vbmiintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/avx512vbmivlintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/avx512vlbf16intrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/avx512vlbitalgintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/avx512vlbwintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/avx512vlcdintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/avx512vldqintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/avx512vlintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/avx512vlvbmi2intrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/avx512vlvnniintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/avx512vlvp2intersectintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/avx512vnniintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/avx512vp2intersectintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/avx512vpopcntdqintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/avx512vpopcntdqvlintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/avxintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/bmi2intrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/bmiintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/cetintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/cldemoteintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/clflushoptintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/clwbintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/clzerointrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/cpuid.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/emmintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/enqcmdintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/f16cintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/fma4intrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/fmaintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/fxsrintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/gfniintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/htmintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/htmxlintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/ia32intrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/immintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/invpcidintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/lwpintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/lzcntintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/mm3dnow.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/mm_malloc.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/mmintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/module.modulemap
 OLD_FILES+=usr/lib/clang/9.0.0/include/movdirintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/msa.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/mwaitxintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/nmmintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/opencl-c-base.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/opencl-c.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/pconfigintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/pkuintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/pmmintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/popcntintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/prfchwintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/ptwriteintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/rdseedintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/rtmintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/s390intrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/sgxintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/shaintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/smmintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/tbmintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/tmmintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/vadefs.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/vaesintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/vecintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/vpclmulqdqintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/waitpkgintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/wbnoinvdintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/wmmintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/x86intrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/xmmintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/xopintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/xsavecintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/xsaveintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/xsaveoptintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/xsavesintrin.h
 OLD_FILES+=usr/lib/clang/9.0.0/include/xtestintrin.h
 OLD_DIRS+=usr/lib/clang/9.0.0/include
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.asan-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.asan-aarch64.so
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.asan-arm.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.asan-arm.so
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.asan-armhf.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.asan-armhf.so
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.asan-i386.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.asan-i386.so
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.asan-preinit-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.asan-preinit-arm.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.asan-preinit-armhf.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.asan-preinit-i386.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.asan-preinit-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.asan-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.asan-x86_64.so
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.asan_cxx-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.asan_cxx-arm.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.asan_cxx-armhf.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.asan_cxx-i386.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.asan_cxx-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.cfi-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.cfi-arm.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.cfi-armhf.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.cfi-i386.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.cfi-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.cfi_diag-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.cfi_diag-arm.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.cfi_diag-armhf.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.cfi_diag-i386.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.cfi_diag-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.dd-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.dd-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.fuzzer-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.fuzzer-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.fuzzer_no_main-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.fuzzer_no_main-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.msan-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.msan-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.msan_cxx-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.msan_cxx-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.profile-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.profile-arm.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.profile-armhf.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.profile-i386.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.profile-powerpc.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.profile-powerpc64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.profile-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.safestack-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.safestack-i386.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.safestack-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.stats-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.stats-arm.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.stats-armhf.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.stats-i386.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.stats-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.stats_client-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.stats_client-arm.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.stats_client-armhf.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.stats_client-i386.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.stats_client-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.tsan-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.tsan-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.tsan_cxx-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.tsan_cxx-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.ubsan_minimal-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.ubsan_minimal-arm.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.ubsan_minimal-armhf.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.ubsan_minimal-i386.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.ubsan_minimal-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.ubsan_standalone-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.ubsan_standalone-arm.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.ubsan_standalone-armhf.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.ubsan_standalone-i386.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.ubsan_standalone-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-arm.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-armhf.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-i386.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.xray-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.xray-arm.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.xray-armhf.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.xray-basic-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.xray-basic-arm.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.xray-basic-armhf.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.xray-basic-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.xray-fdr-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.xray-fdr-arm.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.xray-fdr-armhf.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.xray-fdr-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.xray-profiling-aarch64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.xray-profiling-arm.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.xray-profiling-armhf.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.xray-profiling-x86_64.a
 OLD_FILES+=usr/lib/clang/9.0.0/lib/freebsd/libclang_rt.xray-x86_64.a
 OLD_DIRS+=usr/lib/clang/9.0.0/lib/freebsd
 OLD_DIRS+=usr/lib/clang/9.0.0/lib
 OLD_DIRS+=usr/lib/clang/9.0.0
 # 20191214: Removal of sranddev(3)
 OLD_FILES+=usr/share/man/man3/sranddev.3.gz
 # 20191213: remove timeout(9)
 OLD_FILES+=usr/share/man/man9/callout_handle_init.9.gz
 OLD_FILES+=usr/share/man/man9/timeout.9.gz
 OLD_FILES+=usr/share/man/man9/untimeout.9.gz
 # 20191128: Removal of trm(4)
 OLD_FILES+=usr/share/man/man4/trm.4.gz
 # 20191121: Removal of sio(4)
 OLD_FILES+=usr/share/man/man4/sio.4.gz
 # 20191105: picobsd(8), et al, removed.
 OLD_FILES+=usr/share/man/man8/picobsd.8.gz
 # 20191009: new clang import which bumps version from 8.0.1 to 9.0.0.
 OLD_FILES+=usr/lib/clang/8.0.1/include/sanitizer/allocator_interface.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/sanitizer/asan_interface.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/sanitizer/common_interface_defs.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/sanitizer/coverage_interface.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/sanitizer/dfsan_interface.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/sanitizer/esan_interface.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/sanitizer/hwasan_interface.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/sanitizer/linux_syscall_hooks.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/sanitizer/lsan_interface.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/sanitizer/msan_interface.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/sanitizer/netbsd_syscall_hooks.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/sanitizer/scudo_interface.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/sanitizer/tsan_interface.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/sanitizer/tsan_interface_atomic.h
 OLD_DIRS+=usr/lib/clang/8.0.1/include/sanitizer
 OLD_FILES+=usr/lib/clang/8.0.1/include/__clang_cuda_builtin_vars.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/__clang_cuda_cmath.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/__clang_cuda_complex_builtins.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/__clang_cuda_device_functions.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/__clang_cuda_intrinsics.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/__clang_cuda_libdevice_declares.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/__clang_cuda_math_forward_declares.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/__clang_cuda_runtime_wrapper.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/__stddef_max_align_t.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/__wmmintrin_aes.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/__wmmintrin_pclmul.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/adxintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/altivec.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/ammintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/arm64intr.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/arm_acle.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/arm_fp16.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/arm_neon.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/armintr.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/avx2intrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/avx512bitalgintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/avx512bwintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/avx512cdintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/avx512dqintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/avx512erintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/avx512fintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/avx512ifmaintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/avx512ifmavlintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/avx512pfintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/avx512vbmi2intrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/avx512vbmiintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/avx512vbmivlintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/avx512vlbitalgintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/avx512vlbwintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/avx512vlcdintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/avx512vldqintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/avx512vlintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/avx512vlvbmi2intrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/avx512vlvnniintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/avx512vnniintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/avx512vpopcntdqintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/avx512vpopcntdqvlintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/avxintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/bmi2intrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/bmiintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/cetintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/cldemoteintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/clflushoptintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/clwbintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/clzerointrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/cpuid.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/emmintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/f16cintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/fma4intrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/fmaintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/fxsrintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/gfniintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/htmintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/htmxlintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/ia32intrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/immintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/invpcidintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/lwpintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/lzcntintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/mm3dnow.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/mm_malloc.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/mmintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/module.modulemap
 OLD_FILES+=usr/lib/clang/8.0.1/include/movdirintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/msa.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/mwaitxintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/nmmintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/opencl-c.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/pconfigintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/pkuintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/pmmintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/popcntintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/prfchwintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/ptwriteintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/rdseedintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/rtmintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/s390intrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/sgxintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/shaintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/smmintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/tbmintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/tmmintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/vadefs.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/vaesintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/vecintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/vpclmulqdqintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/waitpkgintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/wbnoinvdintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/wmmintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/x86intrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/xmmintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/xopintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/xsavecintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/xsaveintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/xsaveoptintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/xsavesintrin.h
 OLD_FILES+=usr/lib/clang/8.0.1/include/xtestintrin.h
 OLD_DIRS+=usr/lib/clang/8.0.1/include
 OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.asan-i386.a
 OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.asan-i386.so
 OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.asan-preinit-i386.a
 OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.asan-preinit-x86_64.a
 OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.asan-x86_64.a
 OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.asan-x86_64.so
 OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.asan_cxx-i386.a
 OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.asan_cxx-x86_64.a
 OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.msan-x86_64.a
 OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.msan_cxx-x86_64.a
 OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.profile-aarch64.a
 OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.profile-arm.a
 OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.profile-armhf.a
 OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.profile-i386.a
 OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.profile-x86_64.a
 OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.safestack-i386.a
 OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.safestack-x86_64.a
 OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.stats-i386.a
 OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.stats-x86_64.a
 OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.stats_client-i386.a
 OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.stats_client-x86_64.a
 OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.tsan-x86_64.a
 OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.tsan_cxx-x86_64.a
 OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.ubsan_minimal-i386.a
 OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.ubsan_minimal-x86_64.a
 OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.ubsan_standalone-i386.a
 OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.ubsan_standalone-x86_64.a
 OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.ubsan_standalone_cxx-i386.a
 OLD_FILES+=usr/lib/clang/8.0.1/lib/freebsd/libclang_rt.ubsan_standalone_cxx-x86_64.a
 OLD_DIRS+=usr/lib/clang/8.0.1/lib/freebsd
 OLD_DIRS+=usr/lib/clang/8.0.1/lib
 OLD_DIRS+=usr/lib/clang/8.0.1
 # 20191009: libc++ 9.0.0 removed some experimental files
 OLD_FILES+=usr/include/c++/v1/experimental/any
 OLD_FILES+=usr/include/c++/v1/experimental/chrono
 OLD_FILES+=usr/include/c++/v1/experimental/numeric
 OLD_FILES+=usr/include/c++/v1/experimental/optional
 OLD_FILES+=usr/include/c++/v1/experimental/ratio
 OLD_FILES+=usr/include/c++/v1/experimental/string_view
 OLD_FILES+=usr/include/c++/v1/experimental/system_error
 OLD_FILES+=usr/include/c++/v1/experimental/tuple
 OLD_FILES+=usr/lib/libc++fs.a
 OLD_FILES+=usr/lib32/libc++fs.a
 # 20191003: Remove useless ZFS tests
 OLD_FILES+=usr/tests/sys/cddl/zfs/tests/cli_root/zpool_create/zpool_create_013_neg.ksh
 OLD_FILES+=usr/tests/sys/cddl/zfs/tests/cli_root/zpool_create/zpool_create_014_neg.ksh
 OLD_FILES+=usr/tests/sys/cddl/zfs/tests/cli_root/zpool_create/zpool_create_016_pos.ksh
 # 20190910: mklocale(1) and colldef(1) removed
 OLD_FILES+=usr/bin/mklocale
 OLD_FILES+=usr/share/man/man1/mklocale.1.gz
 OLD_FILES+=usr/bin/colldef
 OLD_FILES+=usr/share/man/man1/colldef.1.gz
 # 20190904: Remove boot1.efifat
 OLD_FILES+=boot/boot1.efifat
 # 20190903: pc-sysinstall(8) removed
 OLD_FILES+=usr/share/examples/pc-sysinstall/README
 OLD_FILES+=usr/share/examples/pc-sysinstall/pc-autoinstall.conf
 OLD_FILES+=usr/share/examples/pc-sysinstall/pcinstall.cfg.fbsd-netinstall
 OLD_FILES+=usr/share/examples/pc-sysinstall/pcinstall.cfg.geli
 OLD_FILES+=usr/share/examples/pc-sysinstall/pcinstall.cfg.gmirror
 OLD_FILES+=usr/share/examples/pc-sysinstall/pcinstall.cfg.netinstall
 OLD_FILES+=usr/share/examples/pc-sysinstall/pcinstall.cfg.restore
 OLD_FILES+=usr/share/examples/pc-sysinstall/pcinstall.cfg.rsync
 OLD_FILES+=usr/share/examples/pc-sysinstall/pcinstall.cfg.upgrade
 OLD_FILES+=usr/share/examples/pc-sysinstall/pcinstall.cfg.zfs
 OLD_FILES+=usr/share/man/man8/pc-sysinstall.8.gz
 OLD_FILES+=usr/share/pc-sysinstall/backend-partmanager/create-part.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend-partmanager/delete-part.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend-query/detect-emulation.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend-query/detect-laptop.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend-query/detect-nics.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend-query/disk-info.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend-query/disk-list.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend-query/disk-part.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend-query/enable-net.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend-query/get-packages.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend-query/list-components.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend-query/list-config.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend-query/list-mirrors.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend-query/list-packages.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend-query/list-rsync-backups.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend-query/list-tzones.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend-query/query-langs.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend-query/send-logs.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend-query/setup-ssh-keys.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend-query/set-mirror.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend-query/sys-mem.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend-query/test-live.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend-query/test-netup.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend-query/update-part-list.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend-query/xkeyboard-layouts.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend-query/xkeyboard-models.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend-query/xkeyboard-variants.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend/functions-bsdlabel.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend/functions-cleanup.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend/functions-disk.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend/functions-extractimage.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend/functions-ftp.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend/functions-installcomponents.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend/functions-installpackages.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend/functions-localize.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend/functions-mountdisk.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend/functions-mountoptical.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend/functions-networking.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend/functions-newfs.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend/functions-parse.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend/functions-packages.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend/functions-runcommands.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend/functions-unmount.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend/functions-upgrade.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend/functions-users.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend/functions.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend/installimage.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend/parseconfig.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend/startautoinstall.sh
 OLD_FILES+=usr/share/pc-sysinstall/conf/avail-langs
 OLD_FILES+=usr/share/pc-sysinstall/conf/exclude-from-upgrade
 OLD_FILES+=usr/share/pc-sysinstall/conf/license/bsd-en.txt
 OLD_FILES+=usr/share/pc-sysinstall/conf/license/intel-en.txt
 OLD_FILES+=usr/share/pc-sysinstall/conf/license/nvidia-en.txt
 OLD_FILES+=usr/share/pc-sysinstall/conf/pc-sysinstall.conf
 OLD_FILES+=usr/share/pc-sysinstall/doc/help-disk-list
 OLD_FILES+=usr/share/pc-sysinstall/doc/help-disk-size
 OLD_FILES+=usr/share/pc-sysinstall/doc/help-index
 OLD_FILES+=usr/share/pc-sysinstall/doc/help-start-autoinstall
 OLD_FILES+=usr/sbin/pc-sysinstall
 OLD_DIRS+=usr/share/examples/pc-sysinstall
 OLD_DIRS+=usr/share/pc-sysinstall/backend
 OLD_DIRS+=usr/share/pc-sysinstall/backend-partmanager
 OLD_DIRS+=usr/share/pc-sysinstall/backend-query
 OLD_DIRS+=usr/share/pc-sysinstall/conf/license
 OLD_DIRS+=usr/share/pc-sysinstall/conf
 OLD_DIRS+=usr/share/pc-sysinstall/doc
 OLD_DIRS+=usr/share/pc-sysinstall
 # 20190825: zlib 1.0.4 removed from kernel
 OLD_FILES+=usr/include/sys/zlib.h
 OLD_FILES+=usr/include/sys/zutil.h
 # 20190817: pft_ping.py and sniffer.py moved to /usr/tests/sys/netpfil/common
 OLD_FILES+=usr/tests/sys/netpfil/pf/sniffer.py
 OLD_FILES+=usr/tests/sys/netpfil/pf/pft_ping.py
 # 20190816: dir.h removed from POSIX
 OLD_FILES+=usr/include/sys/dir.h
 # 20190729: gzip'ed a.out support removed
 OLD_FILES+=usr/include/sys/inflate.h
 # 20190722: cap_random(3) removed
 OLD_FILES+=lib/casper/libcap_random.so.1
 OLD_FILES+=usr/include/casper/cap_random.h
 OLD_FILES+=usr/share/man/man3/cap_random.3.gz
 OLD_FILES+=usr/share/man/man3/cap_random_buf.3.gz
 # 20190708: vm_page_hold() and _unhold() removed
 OLD_FILES+=usr/share/man/man9/vm_page_hold.9.gz
 # 20190618: sys/capability.h removed (sys/capsicum.h is the one to use)
 OLD_FILES+=usr/include/sys/capability.h
 # 20190615: sys/pwm.h renamed to dev/pwmc.h
 OLD_FILES+=usr/include/sys/pwm.h
 # 20190612: new clang import which bumps version from 8.0.0 to 8.0.1.
 OLD_FILES+=usr/lib/clang/8.0.0/include/sanitizer/allocator_interface.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/sanitizer/asan_interface.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/sanitizer/common_interface_defs.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/sanitizer/coverage_interface.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/sanitizer/dfsan_interface.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/sanitizer/esan_interface.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/sanitizer/hwasan_interface.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/sanitizer/linux_syscall_hooks.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/sanitizer/lsan_interface.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/sanitizer/msan_interface.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/sanitizer/netbsd_syscall_hooks.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/sanitizer/scudo_interface.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/sanitizer/tsan_interface.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/sanitizer/tsan_interface_atomic.h
 OLD_DIRS+=usr/lib/clang/8.0.0/include/sanitizer
 OLD_FILES+=usr/lib/clang/8.0.0/include/__clang_cuda_builtin_vars.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/__clang_cuda_cmath.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/__clang_cuda_complex_builtins.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/__clang_cuda_device_functions.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/__clang_cuda_intrinsics.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/__clang_cuda_libdevice_declares.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/__clang_cuda_math_forward_declares.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/__clang_cuda_runtime_wrapper.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/__stddef_max_align_t.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/__wmmintrin_aes.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/__wmmintrin_pclmul.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/adxintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/altivec.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/ammintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/arm64intr.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/arm_acle.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/arm_fp16.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/arm_neon.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/armintr.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/avx2intrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/avx512bitalgintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/avx512bwintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/avx512cdintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/avx512dqintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/avx512erintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/avx512fintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/avx512ifmaintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/avx512ifmavlintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/avx512pfintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/avx512vbmi2intrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/avx512vbmiintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/avx512vbmivlintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/avx512vlbitalgintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/avx512vlbwintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/avx512vlcdintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/avx512vldqintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/avx512vlintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/avx512vlvbmi2intrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/avx512vlvnniintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/avx512vnniintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/avx512vpopcntdqintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/avx512vpopcntdqvlintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/avxintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/bmi2intrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/bmiintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/cetintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/cldemoteintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/clflushoptintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/clwbintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/clzerointrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/cpuid.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/emmintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/f16cintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/fma4intrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/fmaintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/fxsrintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/gfniintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/htmintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/htmxlintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/ia32intrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/immintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/invpcidintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/lwpintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/lzcntintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/mm3dnow.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/mm_malloc.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/mmintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/module.modulemap
 OLD_FILES+=usr/lib/clang/8.0.0/include/movdirintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/msa.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/mwaitxintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/nmmintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/opencl-c.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/pconfigintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/pkuintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/pmmintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/popcntintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/prfchwintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/ptwriteintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/rdseedintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/rtmintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/s390intrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/sgxintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/shaintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/smmintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/tbmintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/tmmintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/vadefs.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/vaesintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/vecintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/vpclmulqdqintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/waitpkgintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/wbnoinvdintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/wmmintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/x86intrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/xmmintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/xopintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/xsavecintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/xsaveintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/xsaveoptintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/xsavesintrin.h
 OLD_FILES+=usr/lib/clang/8.0.0/include/xtestintrin.h
 OLD_DIRS+=usr/lib/clang/8.0.0/include
 OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.asan-i386.a
 OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.asan-i386.so
 OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.asan-preinit-i386.a
 OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.asan-preinit-x86_64.a
 OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.asan-x86_64.a
 OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.asan-x86_64.so
 OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.asan_cxx-i386.a
 OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.asan_cxx-x86_64.a
 OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.msan-x86_64.a
 OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.msan_cxx-x86_64.a
 OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.profile-arm.a
 OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.profile-armhf.a
 OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.profile-i386.a
 OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.profile-x86_64.a
 OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.safestack-i386.a
 OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.safestack-x86_64.a
 OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.stats-i386.a
 OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.stats-x86_64.a
 OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.stats_client-i386.a
 OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.stats_client-x86_64.a
 OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.tsan-x86_64.a
 OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.tsan_cxx-x86_64.a
 OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.ubsan_minimal-i386.a
 OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.ubsan_minimal-x86_64.a
 OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.ubsan_standalone-i386.a
 OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.ubsan_standalone-x86_64.a
 OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-i386.a
 OLD_FILES+=usr/lib/clang/8.0.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-x86_64.a
 OLD_DIRS+=usr/lib/clang/8.0.0/lib/freebsd
 OLD_DIRS+=usr/lib/clang/8.0.0/lib
 OLD_DIRS+=usr/lib/clang/8.0.0
 # 20190523: Remove obsolete kgzip and support files
 OLD_FILES+=usr/sbin/kgzip
 OLD_FILES+=usr/lib/kgzldr.o
 OLD_FILES+=usr/share/man/man8/kgzip.8.gz
 # 20190517: Remove obsolete 10 and 10/100 ethernet drivers.
 OLD_FILES+=usr/share/man/man4/bm.4.gz
 OLD_FILES+=usr/share/man/man4/cs.4.gz
 OLD_FILES+=usr/share/man/man4/de.4.gz
 OLD_FILES+=usr/share/man/man4/if_de.4.gz
 OLD_FILES+=usr/share/man/man4/ed.4.gz
 OLD_FILES+=usr/share/man/man4/if_ed.4.gz
 OLD_FILES+=usr/share/man/man4/ep.4.gz
 OLD_FILES+=usr/share/man/man4/ex.4.gz
 OLD_FILES+=usr/share/man/man4/fe.4.gz
 OLD_FILES+=usr/share/man/man4/pcn.4.gz
 OLD_FILES+=usr/share/man/man4/if_pcn.4.gz
 OLD_FILES+=usr/share/man/man4/sf.4.gz
 OLD_FILES+=usr/share/man/man4/if_sf.4.gz
 OLD_FILES+=usr/share/man/man4/sn.4.gz
 OLD_FILES+=usr/share/man/man4/if_sn.4.gz
 OLD_FILES+=usr/share/man/man4/tl.4.gz
 OLD_FILES+=usr/share/man/man4/if_tl.4.gz
 OLD_FILES+=usr/share/man/man4/tx.4.gz
 OLD_FILES+=usr/share/man/man4/if_tx.4.gz
 OLD_FILES+=usr/share/man/man4/txp.4.gz
 OLD_FILES+=usr/share/man/man4/if_txp.4.gz
 OLD_FILES+=usr/share/man/man4/vx.4.gz
 OLD_FILES+=usr/share/man/man4/wb.4.gz
 OLD_FILES+=usr/share/man/man4/xe.4.gz
 OLD_FILES+=usr/share/man/man4/if_xe.4.gz
 # 20190513: libcap_sysctl interface change
 OLD_FILES+=lib/casper/libcap_sysctl.so.1
 # 20190509: tests/sys/opencrypto requires the net/py-dpkt package.
 OLD_FILES+=usr/tests/sys/opencrypto/dpkt.py
 OLD_FILES+=usr/tests/sys/opencrypto/dpkt.pyc
 # 20190326: tzdata 2019a import
 OLD_FILES+=usr/share/zoneinfo/Etc/UCT
 # 20190304: new libc++ import which bumps version from 7.0.1 to 8.0.0.
 OLD_FILES+=usr/include/c++/v1/experimental/dynarray
 # 20190304: new clang import which bumps version from 7.0.1 to 8.0.0.
 OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/allocator_interface.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/asan_interface.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/common_interface_defs.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/coverage_interface.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/dfsan_interface.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/esan_interface.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/hwasan_interface.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/linux_syscall_hooks.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/lsan_interface.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/msan_interface.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/netbsd_syscall_hooks.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/scudo_interface.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/tsan_interface.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/sanitizer/tsan_interface_atomic.h
 OLD_DIRS+=usr/lib/clang/7.0.1/include/sanitizer
 OLD_FILES+=usr/lib/clang/7.0.1/include/__clang_cuda_builtin_vars.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/__clang_cuda_cmath.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/__clang_cuda_complex_builtins.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/__clang_cuda_device_functions.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/__clang_cuda_intrinsics.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/__clang_cuda_libdevice_declares.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/__clang_cuda_math_forward_declares.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/__clang_cuda_runtime_wrapper.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/__stddef_max_align_t.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/__wmmintrin_aes.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/__wmmintrin_pclmul.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/adxintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/altivec.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/ammintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/arm64intr.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/arm_acle.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/arm_fp16.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/arm_neon.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/armintr.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/avx2intrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/avx512bitalgintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/avx512bwintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/avx512cdintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/avx512dqintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/avx512erintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/avx512fintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/avx512ifmaintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/avx512ifmavlintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/avx512pfintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/avx512vbmi2intrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/avx512vbmiintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/avx512vbmivlintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/avx512vlbitalgintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/avx512vlbwintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/avx512vlcdintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/avx512vldqintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/avx512vlintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/avx512vlvbmi2intrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/avx512vlvnniintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/avx512vnniintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/avx512vpopcntdqintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/avx512vpopcntdqvlintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/avxintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/bmi2intrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/bmiintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/cetintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/cldemoteintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/clflushoptintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/clwbintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/clzerointrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/cpuid.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/emmintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/f16cintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/fma4intrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/fmaintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/fxsrintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/gfniintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/htmintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/htmxlintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/ia32intrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/immintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/invpcidintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/lwpintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/lzcntintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/mm3dnow.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/mm_malloc.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/mmintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/module.modulemap
 OLD_FILES+=usr/lib/clang/7.0.1/include/movdirintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/msa.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/mwaitxintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/nmmintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/opencl-c.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/pconfigintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/pkuintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/pmmintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/popcntintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/prfchwintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/ptwriteintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/rdseedintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/rtmintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/s390intrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/sgxintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/shaintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/smmintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/tbmintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/tmmintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/vadefs.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/vaesintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/vecintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/vpclmulqdqintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/waitpkgintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/wbnoinvdintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/wmmintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/x86intrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/xmmintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/xopintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/xsavecintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/xsaveintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/xsaveoptintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/xsavesintrin.h
 OLD_FILES+=usr/lib/clang/7.0.1/include/xtestintrin.h
 OLD_DIRS+=usr/lib/clang/7.0.1/include
 OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.asan-i386.a
 OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.asan-i386.so
 OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.asan-preinit-i386.a
 OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.asan-preinit-x86_64.a
 OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.asan-x86_64.a
 OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.asan-x86_64.so
 OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.asan_cxx-i386.a
 OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.asan_cxx-x86_64.a
 OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.msan-x86_64.a
 OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.msan_cxx-x86_64.a
 OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.profile-arm.a
 OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.profile-armhf.a
 OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.profile-i386.a
 OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.profile-x86_64.a
 OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.safestack-i386.a
 OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.safestack-x86_64.a
 OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.stats-i386.a
 OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.stats-x86_64.a
 OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.stats_client-i386.a
 OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.stats_client-x86_64.a
 OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.tsan-x86_64.a
 OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.tsan_cxx-x86_64.a
 OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.ubsan_minimal-i386.a
 OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.ubsan_minimal-x86_64.a
 OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.ubsan_standalone-i386.a
 OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.ubsan_standalone-x86_64.a
 OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.ubsan_standalone_cxx-i386.a
 OLD_FILES+=usr/lib/clang/7.0.1/lib/freebsd/libclang_rt.ubsan_standalone_cxx-x86_64.a
 OLD_DIRS+=usr/lib/clang/7.0.1/lib/freebsd
 OLD_DIRS+=usr/lib/clang/7.0.1/lib
 OLD_DIRS+=usr/lib/clang/7.0.1
 # 20190227: rename seq.h to seqc.h
 OLD_FILES+=usr/include/sys/seq.h
 # 20190222: libifconfig made INTERNALLIB
 OLD_FILES+=usr/lib/libprivateifconfig.a
 OLD_FILES+=usr/lib/libprivateifconfig_p.a
 OLD_FILES+=usr/lib32/libprivateifconfig.a
 OLD_FILES+=usr/lib32/libprivateifconfig_p.a
 # 20190131: pfil(9) changed
 OLD_FILES+=usr/share/man/man9/pfil_hook_get.9.gz
 OLD_FILES+=usr/share/man/man9/pfil_rlock.9.gz
 OLD_FILES+=usr/share/man/man9/pfil_runlock.9.gz
 OLD_FILES+=usr/share/man/man9/pfil_wlock.9.gz
 OLD_FILES+=usr/share/man/man9/pfil_wunlock.9.gz
 # 20190126: adv(4) / adw(4) removal
 OLD_FILES+=usr/share/man/man4/adv.4.gz
 OLD_FILES+=usr/share/man/man4/adw.4.gz
 # 20190123: nonexistant cred_update_thread(9) removed
 OLD_FILES+=usr/share/man/man9/cred_update_thread.9.gz
 # 20190114: old pbuf allocator removed
 OLD_FILES+=usr/share/man/man9/getpbuf.9.gz
 OLD_FILES+=usr/share/man/man9/pbuf.9.gz
 OLD_FILES+=usr/share/man/man9/relpbuf.9.gz
 OLD_FILES+=usr/share/man/man9/trypbuf.9.gz
 # 20181219: ibcs removal
 OLD_FILES+=usr/share/examples/ibcs2/hello.uu
 OLD_FILES+=usr/share/examples/ibcs2/README
 OLD_DIRS+=usr/share/examples/ibcs2
 # 20181215: Migration of CTM to ports
 OLD_FILES+=usr/sbin/ctm
 OLD_FILES+=usr/sbin/ctm_dequeue
 OLD_FILES+=usr/sbin/ctm_rmail
 OLD_FILES+=usr/sbin/ctm_smail
 OLD_FILES+=usr/share/man/man1/ctm.1.gz
 OLD_FILES+=usr/share/man/man1/ctm_dequeue.1.gz
 OLD_FILES+=usr/share/man/man1/ctm_rmail.1.gz
 OLD_FILES+=usr/share/man/man1/ctm_smail.1.gz
 OLD_FILES+=usr/share/man/man5/ctm.5.gz
 # 20181214: Remove timed files
 OLD_FILES+=etc/rc.d/timed
 OLD_FILES+=usr/sbin/timed
 OLD_FILES+=usr/sbin/timedc
 OLD_FILES+=usr/share/man/man8/timed.8.gz
 OLD_FILES+=usr/share/man/man8/timedc.8.gz
 # 20181211: new clang import which bumps version from 6.0.1 to 7.0.1.
 OLD_FILES+=usr/lib/clang/6.0.1/include/sanitizer/allocator_interface.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/sanitizer/asan_interface.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/sanitizer/common_interface_defs.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/sanitizer/coverage_interface.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/sanitizer/dfsan_interface.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/sanitizer/esan_interface.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/sanitizer/hwasan_interface.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/sanitizer/linux_syscall_hooks.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/sanitizer/lsan_interface.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/sanitizer/msan_interface.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/sanitizer/scudo_interface.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/sanitizer/tsan_interface.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/sanitizer/tsan_interface_atomic.h
 OLD_DIRS+=usr/lib/clang/6.0.1/include/sanitizer
 OLD_FILES+=usr/lib/clang/6.0.1/include/__clang_cuda_builtin_vars.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/__clang_cuda_cmath.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/__clang_cuda_complex_builtins.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/__clang_cuda_intrinsics.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/__clang_cuda_math_forward_declares.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/__clang_cuda_runtime_wrapper.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/__stddef_max_align_t.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/__wmmintrin_aes.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/__wmmintrin_pclmul.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/adxintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/altivec.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/ammintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/arm64intr.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/arm_acle.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/arm_neon.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/armintr.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/avx2intrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/avx512bitalgintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/avx512bwintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/avx512cdintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/avx512dqintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/avx512erintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/avx512fintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/avx512ifmaintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/avx512ifmavlintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/avx512pfintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/avx512vbmi2intrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/avx512vbmiintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/avx512vbmivlintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/avx512vlbitalgintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/avx512vlbwintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/avx512vlcdintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/avx512vldqintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/avx512vlintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/avx512vlvbmi2intrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/avx512vlvnniintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/avx512vnniintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/avx512vpopcntdqintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/avx512vpopcntdqvlintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/avxintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/bmi2intrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/bmiintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/cetintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/clflushoptintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/clwbintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/clzerointrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/cpuid.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/emmintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/f16cintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/fma4intrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/fmaintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/fxsrintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/gfniintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/htmintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/htmxlintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/ia32intrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/immintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/lwpintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/lzcntintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/mm3dnow.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/mm_malloc.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/mmintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/module.modulemap
 OLD_FILES+=usr/lib/clang/6.0.1/include/msa.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/mwaitxintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/nmmintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/opencl-c.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/pkuintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/pmmintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/popcntintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/prfchwintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/rdseedintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/rtmintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/s390intrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/shaintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/smmintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/tbmintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/tmmintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/vadefs.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/vaesintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/vecintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/vpclmulqdqintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/wmmintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/x86intrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/xmmintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/xopintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/xsavecintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/xsaveintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/xsaveoptintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/xsavesintrin.h
 OLD_FILES+=usr/lib/clang/6.0.1/include/xtestintrin.h
 OLD_DIRS+=usr/lib/clang/6.0.1/include
 OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.asan-i386.a
 OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.asan-i386.so
 OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.asan-preinit-i386.a
 OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.asan-preinit-x86_64.a
 OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.asan-x86_64.a
 OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.asan-x86_64.so
 OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.asan_cxx-i386.a
 OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.asan_cxx-x86_64.a
 OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.profile-arm.a
 OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.profile-armhf.a
 OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.profile-i386.a
 OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.profile-x86_64.a
 OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.safestack-i386.a
 OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.safestack-x86_64.a
 OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.stats-i386.a
 OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.stats-x86_64.a
 OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.stats_client-i386.a
 OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.stats_client-x86_64.a
 OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.tsan-x86_64.a
 OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.tsan_cxx-x86_64.a
 OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.ubsan_minimal-i386.a
 OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.ubsan_minimal-x86_64.a
 OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.ubsan_standalone-i386.a
 OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.ubsan_standalone-x86_64.a
 OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.ubsan_standalone_cxx-i386.a
 OLD_FILES+=usr/lib/clang/6.0.1/lib/freebsd/libclang_rt.ubsan_standalone_cxx-x86_64.a
 OLD_DIRS+=usr/lib/clang/6.0.1/lib/freebsd
 OLD_DIRS+=usr/lib/clang/6.0.1/lib
 OLD_DIRS+=usr/lib/clang/6.0.1
 # 20181116: Rename test file.
 OLD_FILES+=usr/tests/sys/netinet/reuseport_lb
 # 20181113: libufs version bumped to 7.
 OLD_LIBS+=lib/libufs.so.6
 OLD_LIBS+=usr/lib32/libufs.so.6
 # 20181112: Cleanup old libcap_dns.
 OLD_LIBS+=lib/casper/libcap_dns.so.1
 OLD_LIBS+=usr/lib32/libcap_dns.so.1
 # 20181030: malloc_domain(9) KPI change
 OLD_FILES+=usr/share/man/man9/malloc_domain.9.gz
 # 20181026: joy(4) removal
 OLD_FILES+=usr/share/man/man4/joy.4.gz
 # 20181025: OpenSSL libraries version bump to avoid conflict with ports
 OLD_LIBS+=lib/libcrypto.so.9
 OLD_LIBS+=usr/lib/libssl.so.9
 OLD_LIBS+=usr/lib32/libcrypto.so.9
 OLD_LIBS+=usr/lib32/libssl.so.9
 # 20181021: mse(4) removal
 OLD_FILES+=usr/share/man/man4/mse.4.gz
 # 20181015: Stale libcasper(3) files following r329452
 OLD_LIBS+=lib/casper/libcap_sysctl.so.0
 OLD_LIBS+=lib/casper/libcap_grp.so.0
 OLD_LIBS+=lib/casper/libcap_pwd.so.0
 OLD_LIBS+=lib/casper/libcap_random.so.0
 OLD_LIBS+=lib/casper/libcap_dns.so.0
 OLD_LIBS+=lib/casper/libcap_syslog.so.0
 OLD_LIBS+=usr/lib32/libcap_sysctl.so.0
 OLD_LIBS+=usr/lib32/libcap_grp.so.0
 OLD_LIBS+=usr/lib32/libcap_pwd.so.0
 OLD_LIBS+=usr/lib32/libcap_random.so.0
 OLD_LIBS+=usr/lib32/libcap_dns.so.0
 OLD_LIBS+=usr/lib32/libcap_syslog.so.0
 # 20181012: rename of ixlv(4) to iavf(4)
 OLD_FILES+=usr/share/man/man4/ixlv.4.gz
 # 20181009: OpenSSL 1.1.1
 OLD_FILES+=usr/include/openssl/des_old.h
 OLD_FILES+=usr/include/openssl/dso.h
 OLD_FILES+=usr/include/openssl/krb5_asn.h
 OLD_FILES+=usr/include/openssl/kssl.h
 OLD_FILES+=usr/include/openssl/pqueue.h
 OLD_FILES+=usr/include/openssl/ssl23.h
 OLD_FILES+=usr/include/openssl/ui_compat.h
 OLD_FILES+=usr/lib/debug/usr/lib/engines/lib4758cca.so.debug
 OLD_FILES+=usr/lib/debug/usr/lib/engines/libaep.so.debug
 OLD_FILES+=usr/lib/debug/usr/lib/engines/libatalla.so.debug
 OLD_FILES+=usr/lib/debug/usr/lib/engines/libcapi.so.debug
 OLD_FILES+=usr/lib/debug/usr/lib/engines/libchil.so.debug
 OLD_FILES+=usr/lib/debug/usr/lib/engines/libcswift.so.debug
 OLD_FILES+=usr/lib/debug/usr/lib/engines/libgost.so.debug
 OLD_FILES+=usr/lib/debug/usr/lib/engines/libnuron.so.debug
 OLD_FILES+=usr/lib/debug/usr/lib/engines/libsureware.so.debug
 OLD_FILES+=usr/lib/debug/usr/lib/engines/libubsec.so.debug
 OLD_FILES+=usr/share/openssl/man/man1/dss1.1.gz
 OLD_FILES+=usr/share/openssl/man/man1/md2.1.gz
 OLD_FILES+=usr/share/openssl/man/man1/md4.1.gz
 OLD_FILES+=usr/share/openssl/man/man1/md5.1.gz
 OLD_FILES+=usr/share/openssl/man/man1/mdc2.1.gz
 OLD_FILES+=usr/share/openssl/man/man1/ripemd160.1.gz
 OLD_FILES+=usr/share/openssl/man/man1/sha.1.gz
 OLD_FILES+=usr/share/openssl/man/man1/sha1.1.gz
 OLD_FILES+=usr/share/openssl/man/man1/sha224.1.gz
 OLD_FILES+=usr/share/openssl/man/man1/sha256.1.gz
 OLD_FILES+=usr/share/openssl/man/man1/sha384.1.gz
 OLD_FILES+=usr/share/openssl/man/man1/sha512.1.gz
 OLD_FILES+=usr/share/openssl/man/man1/x509v3_config.1.gz
 OLD_FILES+=usr/share/openssl/man/man3/ASN1_STRING_length_set.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/BIO_get_conn_int_port.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/BIO_get_conn_ip.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/BIO_set.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/BIO_set_conn_int_port.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/BIO_set_conn_ip.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/BN_BLINDING_get_thread_id.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/BN_BLINDING_set_thread_id.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/BN_BLINDING_thread_id.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/BN_CTX_init.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/BN_MONT_CTX_init.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/BN_RECP_CTX_init.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/BN_init.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/BUF_memdup.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/BUF_memdup.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/BUF_strdup.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/BUF_strlcat.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/BUF_strlcpy.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/BUF_strndup.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/CMS_set1_signer_cert.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/CRYPTO_THREADID_cmp.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/CRYPTO_THREADID_cpy.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/CRYPTO_THREADID_current.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/CRYPTO_THREADID_get_callback.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/CRYPTO_THREADID_hash.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/CRYPTO_THREADID_set_callback.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/CRYPTO_destroy_dynlockid.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/CRYPTO_get_new_dynlockid.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/CRYPTO_lock.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/CRYPTO_num_locks.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/CRYPTO_set_dynlock_create_callback.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/CRYPTO_set_dynlock_destroy_callback.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/CRYPTO_set_dynlock_lock_callback.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/CRYPTO_set_locking_callback.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/DES_ede3_cbcm_encrypt.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/DES_enc_read.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/DES_enc_write.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/EC_KEY_get_key_method_data.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/EC_KEY_insert_key_method_data.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/EC_POINT_set_Jprojective_coordinates.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/ERR_load_UI_strings.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/EVP_CIPHER_CTX_cleanup.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/EVP_CIPHER_CTX_init.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/EVP_MAX_MD_SIZE.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/EVP_MD_CTX_cleanup.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/EVP_MD_CTX_create.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/EVP_MD_CTX_destroy.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/EVP_MD_CTX_init.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/EVP_PKEVP_PKEY_CTX_set_app_data.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/EVP_PKEY_CTX_set_rsa_rsa_keygen_bits.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/EVP_PKEY_get_default_digest.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/EVP_dss.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/EVP_dss1.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/EVP_sha.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/HMAC_CTX_cleanup.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/HMAC_CTX_init.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/HMAC_cleanup.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/OPENSSL_ia32cap_loc.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/PEM.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/RAND_SSLeay.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/RSA_PKCS1_SSLeay.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/RSA_null_method.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/SSL.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/SSL_CTX_get_ex_new_index.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/SSL_CTX_need_tmp_rsa.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/SSL_CTX_set_custom_cli_ext.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/SSL_CTX_set_default_read_ahead.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/SSL_CTX_set_ecdh_auto.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/SSL_CTX_set_tmp_rsa.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/SSL_CTX_set_tmp_rsa_callback.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/SSL_SESSION_get_ex_new_index.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/SSL_add_session.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/SSL_flush_sessions.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/SSL_get_accept_state.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/SSL_get_ex_new_index.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/SSL_get_msg_callback_arg.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/SSL_need_tmp_rsa.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/SSL_remove_session.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/SSL_set_ecdh_auto.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/SSL_set_tmp_rsa.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/SSL_set_tmp_rsa_callback.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/SSLeay.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/SSLeay_add_ssl_algorithms.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/SSLeay_version.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/SSLv2_client_method.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/SSLv2_method.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/SSLv2_server_method.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/X509_STORE_CTX_set_chain.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/X509_STORE_CTX_trusted_stack.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/bio.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/blowfish.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/bn.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/bn_add_words.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/bn_check_top.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/bn_cmp_words.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/bn_div_words.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/bn_dump.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/bn_expand.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/bn_expand2.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/bn_fix_top.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/bn_internal.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/bn_mul_add_words.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/bn_mul_comba4.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/bn_mul_comba8.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/bn_mul_high.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/bn_mul_low_normal.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/bn_mul_low_recursive.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/bn_mul_normal.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/bn_mul_part_recursive.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/bn_mul_recursive.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/bn_mul_words.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/bn_print.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/bn_set_high.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/bn_set_low.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/bn_set_max.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/bn_sqr_comba4.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/bn_sqr_comba8.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/bn_sqr_normal.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/bn_sqr_recursive.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/bn_sqr_words.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/bn_sub_words.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/bn_wexpand.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/buffer.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/crypto.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/d2i_ECPKParameters_bio.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/d2i_ECPKParameters_fp.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/d2i_ECPrivate_key.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/d2i_Netscape_RSA.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/d2i_PKCS8PrivateKey.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/d2i_Private_key.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/d2i_X509_bio.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/d2i_X509_fp.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/des.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/des_read_2passwords.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/des_read_password.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/des_read_pw.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/des_read_pw_string.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/dh.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/dsa.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/ec.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/ecdsa.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/engine.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/err.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/evp.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/hmac.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/i2d_ECPKParameters_bio.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/i2d_ECPKParameters_fp.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/i2d_Netscape_RSA.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/i2d_X509_bio.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/i2d_X509_fp.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/lh_delete.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/lh_doall.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/lh_doall_arg.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/lh_error.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/lh_free.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/lh_insert.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/lh_new.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/lh_node_stats.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/lh_node_stats_bio.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/lh_node_usage_stats.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/lh_node_usage_stats_bio.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/lh_retrieve.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/lh_stats.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/lh_stats_bio.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/lhash.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/md5.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/mdc2.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/pem.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/rand.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/rc4.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/ripemd.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/rsa.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/sha.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/ssl.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/threads.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/ui.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/ui_compat.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/x509.3.gz
 OLD_LIBS+=lib/libcrypto.so.8
 OLD_LIBS+=usr/lib/engines/lib4758cca.so
 OLD_LIBS+=usr/lib/engines/libaep.so
 OLD_LIBS+=usr/lib/engines/libatalla.so
 OLD_LIBS+=usr/lib/engines/libcapi.so
 OLD_LIBS+=usr/lib/engines/libchil.so
 OLD_LIBS+=usr/lib/engines/libcswift.so
 OLD_LIBS+=usr/lib/engines/libgost.so
 OLD_LIBS+=usr/lib/engines/libnuron.so
 OLD_LIBS+=usr/lib/engines/libsureware.so
 OLD_LIBS+=usr/lib/engines/libubsec.so
 OLD_LIBS+=usr/lib/libssl.so.8
 OLD_LIBS+=usr/lib32/libcrypto.so.8
 OLD_LIBS+=usr/lib32/lib4758cca.so
 OLD_LIBS+=usr/lib32/libaep.so
 OLD_LIBS+=usr/lib32/libatalla.so
 OLD_LIBS+=usr/lib32/libcapi.so
 OLD_LIBS+=usr/lib32/libchil.so
 OLD_LIBS+=usr/lib32/libcswift.so
 OLD_LIBS+=usr/lib32/libgost.so
 OLD_LIBS+=usr/lib32/libnuron.so
 OLD_LIBS+=usr/lib32/libsureware.so
 OLD_LIBS+=usr/lib32/libubsec.so
 OLD_LIBS+=usr/lib32/libssl.so.8
 # 20180824: libbe(3) SHLIBDIR fixed to reflect correct location
 OLD_LIBS+=usr/lib/libbe.so.1
 # 20180819: Remove deprecated arc4random(3) stir/addrandom interfaces
 OLD_FILES+=usr/share/man/man3/arc4random_addrandom.3.gz
 OLD_FILES+=usr/share/man/man3/arc4random_stir.3.gz
 # 20180819: send-pr(1) placeholder removal
 OLD_FILES+=usr/bin/send-pr
 # 20180725: Cleanup old libcasper.so.0
 OLD_LIBS+=lib/libcasper.so.0
 OLD_LIBS+=usr/lib32/libcasper.so.0
 # 20180722: indent(1) option renamed, test files follow
 OLD_FILES+=usr/bin/indent/tests/nsac.0
 OLD_FILES+=usr/bin/indent/tests/nsac.0.pro
 OLD_FILES+=usr/bin/indent/tests/nsac.0.stdout
 OLD_FILES+=usr/bin/indent/tests/sac.0
 OLD_FILES+=usr/bin/indent/tests/sac.0.pro
 OLD_FILES+=usr/bin/indent/tests/sac.0.stdout
 # 20180721: move of libmlx5.so.1 and libibverbs.so.1
 OLD_LIBS+=usr/lib/libmlx5.so.1
 OLD_LIBS+=usr/lib/libibverbs.so.1
 # 20180710: old numa cleanup
 OLD_FILES+=usr/include/sys/numa.h
 OLD_FILES+=usr/share/man/man2/numa_getaffinity.2.gz
 OLD_FILES+=usr/share/man/man2/numa_setaffinity.2.gz
 OLD_FILES+=usr/share/man/man1/numactl.1.gz
 OLD_FILES+=usr/bin/numactl
 # 20180630: new clang import which bumps version from 6.0.0 to 6.0.1.
 OLD_FILES+=usr/lib/clang/6.0.0/include/sanitizer/allocator_interface.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/sanitizer/asan_interface.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/sanitizer/common_interface_defs.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/sanitizer/coverage_interface.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/sanitizer/dfsan_interface.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/sanitizer/esan_interface.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/sanitizer/hwasan_interface.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/sanitizer/linux_syscall_hooks.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/sanitizer/lsan_interface.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/sanitizer/msan_interface.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/sanitizer/scudo_interface.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/sanitizer/tsan_interface.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/sanitizer/tsan_interface_atomic.h
 OLD_DIRS+=usr/lib/clang/6.0.0/include/sanitizer
 OLD_FILES+=usr/lib/clang/6.0.0/include/__clang_cuda_builtin_vars.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/__clang_cuda_cmath.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/__clang_cuda_complex_builtins.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/__clang_cuda_intrinsics.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/__clang_cuda_math_forward_declares.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/__clang_cuda_runtime_wrapper.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/__stddef_max_align_t.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/__wmmintrin_aes.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/__wmmintrin_pclmul.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/adxintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/altivec.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/ammintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/arm64intr.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/arm_acle.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/arm_neon.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/armintr.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/avx2intrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/avx512bitalgintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/avx512bwintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/avx512cdintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/avx512dqintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/avx512erintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/avx512fintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/avx512ifmaintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/avx512ifmavlintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/avx512pfintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/avx512vbmi2intrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/avx512vbmiintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/avx512vbmivlintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/avx512vlbitalgintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/avx512vlbwintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/avx512vlcdintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/avx512vldqintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/avx512vlintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/avx512vlvbmi2intrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/avx512vlvnniintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/avx512vnniintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/avx512vpopcntdqintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/avx512vpopcntdqvlintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/avxintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/bmi2intrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/bmiintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/cetintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/clflushoptintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/clwbintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/clzerointrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/cpuid.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/emmintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/f16cintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/fma4intrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/fmaintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/fxsrintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/gfniintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/htmintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/htmxlintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/ia32intrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/immintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/lwpintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/lzcntintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/mm3dnow.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/mm_malloc.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/mmintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/module.modulemap
 OLD_FILES+=usr/lib/clang/6.0.0/include/msa.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/mwaitxintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/nmmintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/opencl-c.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/pkuintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/pmmintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/popcntintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/prfchwintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/rdseedintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/rtmintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/s390intrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/shaintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/smmintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/tbmintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/tmmintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/vadefs.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/vaesintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/vecintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/vpclmulqdqintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/wmmintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/x86intrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/xmmintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/xopintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/xsavecintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/xsaveintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/xsaveoptintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/xsavesintrin.h
 OLD_FILES+=usr/lib/clang/6.0.0/include/xtestintrin.h
 OLD_DIRS+=usr/lib/clang/6.0.0/include
 OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.asan-i386.a
 OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.asan-i386.so
 OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.asan-preinit-i386.a
 OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.asan-preinit-x86_64.a
 OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.asan-x86_64.a
 OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.asan-x86_64.so
 OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.asan_cxx-i386.a
 OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.asan_cxx-x86_64.a
 OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.profile-arm.a
 OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.profile-armhf.a
 OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.profile-i386.a
 OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.profile-x86_64.a
 OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.safestack-i386.a
 OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.safestack-x86_64.a
 OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.stats-i386.a
 OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.stats-x86_64.a
 OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.stats_client-i386.a
 OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.stats_client-x86_64.a
 OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.tsan-x86_64.a
 OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.tsan_cxx-x86_64.a
 OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.ubsan_minimal-i386.a
 OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.ubsan_minimal-x86_64.a
 OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.ubsan_standalone-i386.a
 OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.ubsan_standalone-x86_64.a
 OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-i386.a
 OLD_FILES+=usr/lib/clang/6.0.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-x86_64.a
 OLD_DIRS+=usr/lib/clang/6.0.0/lib/freebsd
 OLD_DIRS+=usr/lib/clang/6.0.0/lib
 OLD_DIRS+=usr/lib/clang/6.0.0
 # 20180615: asf(8) removed
 OLD_FILES+=usr/sbin/asf
 OLD_FILES+=usr/share/man/man8/asf.8.gz
 # 20180609: obsolete libc++ files missed from the 5.0.0 import
 OLD_FILES+=usr/include/c++/v1/__refstring
 OLD_FILES+=usr/include/c++/v1/__undef_min_max
 OLD_FILES+=usr/include/c++/v1/tr1/__refstring
 OLD_FILES+=usr/include/c++/v1/tr1/__undef_min_max
 # 20180607: remove nls support from grep
 OLD_FILES+=usr/share/nls/pt_BR.ISO8859-1/grep.cat
 OLD_FILES+=usr/share/nls/hu_HU.ISO8859-2/grep.cat
 OLD_FILES+=usr/share/nls/ja_JP.SJIS/grep.cat
 OLD_FILES+=usr/share/nls/ja_JP.eucJP/grep.cat
 OLD_FILES+=usr/share/nls/gl_ES.ISO8859-1/grep.cat
 OLD_FILES+=usr/share/nls/zh_CN.UTF-8/grep.cat
 OLD_FILES+=usr/share/nls/es_ES.ISO8859-1/grep.cat
 OLD_FILES+=usr/share/nls/ru_RU.KOI8-R/grep.cat
 OLD_FILES+=usr/share/nls/uk_UA.UTF-8/grep.cat
 OLD_FILES+=usr/share/nls/ja_JP.UTF-8/grep.cat
 # 20180528: libpcap update removed header file
 OLD_FILES+=usr/include/pcap/export-defs.h
 # 20180517: retire vxge
 OLD_FILES+=usr/share/man/man4/if_vxge.4.gz
 OLD_FILES+=usr/share/man/man4/vxge.4.gz
 # 20180512: Rename Unbound tools
 OLD_FILES+=usr/sbin/unbound
 OLD_FILES+=usr/sbin/unbound-anchor
 OLD_FILES+=usr/sbin/unbound-checkconf
 OLD_FILES+=usr/sbin/unbound-control
 OLD_FILES+=usr/share/man/man5/unbound.conf.5.gz
 OLD_FILES+=usr/share/man/man8/unbound-anchor.8.gz
 OLD_FILES+=usr/share/man/man8/unbound-checkconf.8.gz
 OLD_FILES+=usr/share/man/man8/unbound-control.8.gz
 OLD_FILES+=usr/share/man/man8/unbound.8.gz
 # 20180508: retire nxge
 OLD_FILES+=usr/share/man/man4/if_nxge.4.gz
 OLD_FILES+=usr/share/man/man4/nxge.4.gz
 # 20180505: rhosts
 OLD_FILES+=usr/share/skel/dot.rhosts
 # 20180502: retire ixgb
 OLD_FILES+=usr/share/man/man4/if_ixgb.4.gz
 OLD_FILES+=usr/share/man/man4/ixgb.4.gz
 # 20180501: retire lmc
 OLD_FILES+=usr/include/dev/lmc/if_lmc.h
 OLD_DIRS+=usr/include/dev/lmc
 OLD_FILES+=usr/sbin/lmcconfig
 OLD_FILES+=usr/share/man/man4/lmc.4.gz
 OLD_FILES+=usr/share/man/man4/if_lmc.4.gz
 OLD_FILES+=usr/share/man/man8/lmcconfig.8.gz
 # 20180417: remove fuswintr and suswintr
 OLD_FILES+=usr/share/man/man9/fuswintr.9.gz
 OLD_FILES+=usr/share/man/man9/suswintr.9.gz
 # 20180413: remove Arcnet support
 OLD_FILES+=usr/include/net/if_arc.h
 OLD_FILES+=usr/share/man/man4/cm.4.gz
 # 20180409: remove FDDI support
 OLD_FILES+=usr/include/net/fddi.h
 OLD_FILES+=usr/share/man/man4/fpa.4.gz
 # 20180319: remove /boot/overlays, replaced by /boot/dtb/overlays
 OLD_DIRS+=boot/overlays
 # 20180311: remove sys/sys/i386/include/pcaudioio.h
 .if ${TARGET_ARCH} == "i386"
 OLD_FILES+=usr/include/machine/pcaudioio.h
 .endif
 # 20180310: remove sys/sys/dataacq.h
 OLD_FILES+=usr/include/sys/dataacq.h
 # 20180306: remove DTrace scripts made obsolete by dwatch(1)
 OLD_FILES+=usr/share/dtrace/watch_execve
 OLD_FILES+=usr/share/dtrace/watch_kill
 OLD_FILES+=usr/share/dtrace/watch_vop_remove
 # 20180212: move devmatch
 OLD_FILES+=usr/sbin/devmatch
 # 20180211: remove usb.conf
 OLD_FILES+=etc/devd/usb.conf
 # 20180208: remove c_rehash(1)
 OLD_FILES+=usr/share/openssl/man/man1/c_rehash.1.gz
 # 20180206: remove gdbtui
 OLD_FILES+=usr/bin/gdbtui
 # 20180201: Obsolete forth files
 OLD_FILES+=boot/pcibios.4th
 # 20180114: new clang import which bumps version from 5.0.1 to 6.0.0.
 OLD_FILES+=usr/lib/clang/5.0.1/include/sanitizer/allocator_interface.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/sanitizer/asan_interface.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/sanitizer/common_interface_defs.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/sanitizer/coverage_interface.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/sanitizer/dfsan_interface.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/sanitizer/esan_interface.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/sanitizer/linux_syscall_hooks.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/sanitizer/lsan_interface.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/sanitizer/msan_interface.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/sanitizer/tsan_interface.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/sanitizer/tsan_interface_atomic.h
 OLD_DIRS+=usr/lib/clang/5.0.1/include/sanitizer
 OLD_FILES+=usr/lib/clang/5.0.1/include/__clang_cuda_builtin_vars.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/__clang_cuda_cmath.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/__clang_cuda_complex_builtins.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/__clang_cuda_intrinsics.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/__clang_cuda_math_forward_declares.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/__clang_cuda_runtime_wrapper.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/__stddef_max_align_t.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/__wmmintrin_aes.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/__wmmintrin_pclmul.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/adxintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/altivec.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/ammintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/arm_acle.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/arm_neon.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/armintr.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/avx2intrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/avx512bwintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/avx512cdintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/avx512dqintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/avx512erintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/avx512fintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/avx512ifmaintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/avx512ifmavlintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/avx512pfintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/avx512vbmiintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/avx512vbmivlintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/avx512vlbwintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/avx512vlcdintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/avx512vldqintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/avx512vlintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/avx512vpopcntdqintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/avxintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/bmi2intrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/bmiintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/clflushoptintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/clzerointrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/cpuid.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/emmintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/f16cintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/fma4intrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/fmaintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/fxsrintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/htmintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/htmxlintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/ia32intrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/immintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/lwpintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/lzcntintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/mm3dnow.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/mm_malloc.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/mmintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/module.modulemap
 OLD_FILES+=usr/lib/clang/5.0.1/include/msa.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/mwaitxintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/nmmintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/opencl-c.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/pkuintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/pmmintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/popcntintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/prfchwintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/rdseedintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/rtmintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/s390intrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/shaintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/smmintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/tbmintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/tmmintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/vadefs.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/vecintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/wmmintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/x86intrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/xmmintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/xopintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/xsavecintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/xsaveintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/xsaveoptintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/xsavesintrin.h
 OLD_FILES+=usr/lib/clang/5.0.1/include/xtestintrin.h
 OLD_DIRS+=usr/lib/clang/5.0.1/include
 OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.asan-i386.a
 OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.asan-i386.so
 OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.asan-preinit-i386.a
 OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.asan-preinit-x86_64.a
 OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.asan-x86_64.a
 OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.asan-x86_64.so
 OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.asan_cxx-i386.a
 OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.asan_cxx-x86_64.a
 OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.profile-arm.a
 OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.profile-armhf.a
 OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.profile-i386.a
 OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.profile-x86_64.a
 OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.safestack-i386.a
 OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.safestack-x86_64.a
 OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.stats-i386.a
 OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.stats-x86_64.a
 OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.stats_client-i386.a
 OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.stats_client-x86_64.a
 OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.ubsan_standalone-i386.a
 OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.ubsan_standalone-x86_64.a
 OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.ubsan_standalone_cxx-i386.a
 OLD_FILES+=usr/lib/clang/5.0.1/lib/freebsd/libclang_rt.ubsan_standalone_cxx-x86_64.a
 OLD_DIRS+=usr/lib/clang/5.0.1/lib/freebsd
 OLD_DIRS+=usr/lib/clang/5.0.1/lib
 OLD_DIRS+=usr/lib/clang/5.0.1
 # 20180109: Remove vestiges of digi(4) driver
 OLD_FILES+=usr/include/sys/digiio.h
 OLD_FILES+=usr/sbin/digictl
 OLD_FILES+=usr/share/man/man8/digictl.8.gz
 # 20180107: Convert remaining geli(8) tests to ATF
 OLD_FILES+=tests/sys/geom/class/eli/nokey_test.sh
 OLD_FILES+=tests/sys/geom/class/eli/readonly_test.sh
 # 20180106: Convert most geli(8) tests to ATF
 OLD_FILES+=tests/sys/geom/class/eli/attach_d_test.sh
 OLD_FILES+=tests/sys/geom/class/eli/configure_b_B_test.sh
 OLD_FILES+=tests/sys/geom/class/eli/detach_l_test.sh
 OLD_FILES+=tests/sys/geom/class/eli/init_B_test.sh
 OLD_FILES+=tests/sys/geom/class/eli/init_J_test.sh
 OLD_FILES+=tests/sys/geom/class/eli/init_a_test.sh
 OLD_FILES+=tests/sys/geom/class/eli/init_alias_test.sh
 OLD_FILES+=tests/sys/geom/class/eli/init_i_P_test.sh
 OLD_FILES+=tests/sys/geom/class/eli/integrity_copy_test.sh
 OLD_FILES+=tests/sys/geom/class/eli/integrity_data_test.sh
 OLD_FILES+=tests/sys/geom/class/eli/integrity_hmac_test.sh
 OLD_FILES+=tests/sys/geom/class/eli/onetime_a_test.sh
 OLD_FILES+=tests/sys/geom/class/eli/onetime_d_test.sh
 # 20171230: Remove /etc/skel from mtree
 OLD_DIRS+=etc/skel
 # 20171208: Remove basename_r(3)
 OLD_FILES+=usr/share/man/man3/basename_r.3.gz
 # 20171204: Move fdformat man page from volume 1 to volume 8.
 OLD_FILES+=usr/share/man/man1/fdformat.1.gz
 # 20171203: libproc version bump
 OLD_LIBS+=usr/lib/libproc.so.4
 OLD_LIBS+=usr/lib32/libproc.so.4
 # 20171203: new clang import which bumps version from 5.0.0 to 5.0.1.
 OLD_FILES+=usr/lib/clang/5.0.0/include/sanitizer/allocator_interface.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/sanitizer/asan_interface.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/sanitizer/common_interface_defs.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/sanitizer/coverage_interface.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/sanitizer/dfsan_interface.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/sanitizer/esan_interface.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/sanitizer/linux_syscall_hooks.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/sanitizer/lsan_interface.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/sanitizer/msan_interface.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/sanitizer/tsan_interface.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/sanitizer/tsan_interface_atomic.h
 OLD_DIRS+=usr/lib/clang/5.0.0/include/sanitizer
 OLD_FILES+=usr/lib/clang/5.0.0/include/__clang_cuda_builtin_vars.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/__clang_cuda_cmath.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/__clang_cuda_complex_builtins.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/__clang_cuda_intrinsics.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/__clang_cuda_math_forward_declares.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/__clang_cuda_runtime_wrapper.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/__stddef_max_align_t.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/__wmmintrin_aes.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/__wmmintrin_pclmul.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/adxintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/altivec.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/ammintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/arm_acle.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/arm_neon.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/armintr.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/avx2intrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/avx512bwintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/avx512cdintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/avx512dqintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/avx512erintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/avx512fintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/avx512ifmaintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/avx512ifmavlintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/avx512pfintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/avx512vbmiintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/avx512vbmivlintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/avx512vlbwintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/avx512vlcdintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/avx512vldqintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/avx512vlintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/avx512vpopcntdqintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/avxintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/bmi2intrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/bmiintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/clflushoptintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/clzerointrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/cpuid.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/emmintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/f16cintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/fma4intrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/fmaintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/fxsrintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/htmintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/htmxlintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/ia32intrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/immintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/lwpintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/lzcntintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/mm3dnow.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/mm_malloc.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/mmintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/module.modulemap
 OLD_FILES+=usr/lib/clang/5.0.0/include/msa.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/mwaitxintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/nmmintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/opencl-c.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/pkuintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/pmmintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/popcntintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/prfchwintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/rdseedintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/rtmintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/s390intrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/shaintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/smmintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/tbmintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/tmmintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/vadefs.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/vecintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/wmmintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/x86intrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/xmmintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/xopintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/xsavecintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/xsaveintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/xsaveoptintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/xsavesintrin.h
 OLD_FILES+=usr/lib/clang/5.0.0/include/xtestintrin.h
 OLD_DIRS+=usr/lib/clang/5.0.0/include
 OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.asan-i386.a
 OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.asan-i386.so
 OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.asan-preinit-i386.a
 OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.asan-preinit-x86_64.a
 OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.asan-x86_64.a
 OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.asan-x86_64.so
 OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.asan_cxx-i386.a
 OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.asan_cxx-x86_64.a
 OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.profile-arm.a
 OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.profile-armhf.a
 OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.profile-i386.a
 OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.profile-x86_64.a
 OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.safestack-i386.a
 OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.safestack-x86_64.a
 OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.stats-i386.a
 OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.stats-x86_64.a
 OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.stats_client-i386.a
 OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.stats_client-x86_64.a
 OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.ubsan_standalone-i386.a
 OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.ubsan_standalone-x86_64.a
 OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-i386.a
 OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-x86_64.a
 OLD_DIRS+=usr/lib/clang/5.0.0/lib/freebsd
 OLD_DIRS+=usr/lib/clang/5.0.0/lib
 OLD_DIRS+=usr/lib/clang/5.0.0
 # 20171118: Remove old etc casper files
 OLD_FILES+=etc/casper/system.dns
 OLD_FILES+=etc/casper/system.grp
 OLD_FILES+=etc/casper/system.pwd
 OLD_FILES+=etc/casper/system.random
 OLD_FILES+=etc/casper/system.sysctl
 OLD_DIRS+=etc/casper
 # 20171116: lint(1) removal
 OLD_FILES+=usr/bin/lint
 OLD_FILES+=usr/libexec/lint1
 OLD_FILES+=usr/libexec/lint2
 OLD_FILES+=usr/libdata/lint/llib-lposix.ln
 OLD_FILES+=usr/libdata/lint/llib-lstdc.ln
 OLD_FILES+=usr/share/man/man1/lint.1.gz
 OLD_FILES+=usr/share/man/man7/lint.7.gz
 OLD_DIRS+=usr/libdata/lint
 # 20171114: Removal of all fortune datfiles other than freebsd-tips
 OLD_FILES+=usr/share/games/fortune/fortunes
 OLD_FILES+=usr/share/games/fortune/fortunes.dat
 OLD_FILES+=usr/share/games/fortune/gerrold.limerick
 OLD_FILES+=usr/share/games/fortune/gerrold.limerick.dat
 OLD_FILES+=usr/share/games/fortune/limerick
 OLD_FILES+=usr/share/games/fortune/limerick.dat
 OLD_FILES+=usr/share/games/fortune/murphy
 OLD_FILES+=usr/share/games/fortune/murphy-o
 OLD_FILES+=usr/share/games/fortune/murphy-o.dat
 OLD_FILES+=usr/share/games/fortune/murphy.dat
 OLD_FILES+=usr/share/games/fortune/startrek
 OLD_FILES+=usr/share/games/fortune/startrek.dat
 OLD_FILES+=usr/share/games/fortune/zippy
 OLD_FILES+=usr/share/games/fortune/zippy.dat
 # 20171112: Removal of eqnchar definition
 OLD_FILES+=usr/share/misc/eqnchar
 # 20171110: Removal of mailaddr man page
 OLD_FILES+=usr/share/man/man7/mailaddr.7.gz
 # 20171108: badsect(8) removal
 OLD_FILES+=sbin/badsect
 OLD_FILES+=rescue/badsect
 OLD_FILES+=usr/share/man/man8/badsect.8.gz
 # 20171105: fixing lib/libclang_rt CRTARCH for arm:armv[67].
 .if ${MACHINE_ARCH:Marmv[67]*} != "" && \
     (!defined(CPUTYPE) || ${CPUTYPE:M*soft*} == "")
 OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.asan-preinit-arm.a
 OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.asan-arm.a
 OLD_LIBS+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.asan-arm.so
 OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.asan_cxx-arm.a
 OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.profile-arm.a
 OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.safestack-arm.a
 OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.stats-arm.a
 OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.stats_client-arm.a
 OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.ubsan_standalone-arm.a
 OLD_FILES+=usr/lib/clang/5.0.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-arm.a
 .endif
 # 20171104: libcap_random should be in /lib not in /usr/lib
 OLD_LIBS+=usr/lib/libcap_random.so.0
 # 20171104: Casper can work only as shared library
 OLD_FILES+=usr/lib/libcap_dns.a
 OLD_FILES+=usr/lib/libcap_dns_p.a
 OLD_FILES+=usr/lib/libcap_grp.a
 OLD_FILES+=usr/lib/libcap_grp_p.a
 OLD_FILES+=usr/lib/libcap_pwd.a
 OLD_FILES+=usr/lib/libcap_pwd_p.a
 OLD_FILES+=usr/lib/libcap_random.a
 OLD_FILES+=usr/lib/libcap_random_p.a
 OLD_FILES+=usr/lib/libcap_sysctl.a
 OLD_FILES+=usr/lib/libcap_sysctl_p.a
 OLD_FILES+=usr/lib/libcasper.a
 OLD_FILES+=usr/lib/libcasper_p.a
 OLD_FILES+=usr/lib32/libcap_dns.a
 OLD_FILES+=usr/lib32/libcap_dns_p.a
 OLD_FILES+=usr/lib32/libcap_grp.a
 OLD_FILES+=usr/lib32/libcap_grp_p.a
 OLD_FILES+=usr/lib32/libcap_pwd.a
 OLD_FILES+=usr/lib32/libcap_pwd_p.a
 OLD_FILES+=usr/lib32/libcap_random.a
 OLD_FILES+=usr/lib32/libcap_random_p.a
 OLD_FILES+=usr/lib32/libcap_sysctl.a
 OLD_FILES+=usr/lib32/libcap_sysctl_p.a
 OLD_FILES+=usr/lib32/libcasper.a
 OLD_FILES+=usr/lib32/libcasper_p.a
 # 20171031: Removal of adding_user man page
 OLD_FILES+=usr/share/man/man7/adding_user.7.gz
 # 20171031: Disconnected libpathconv tests
 OLD_DIRS+=usr/tests/lib/libpathconv
 # 20171017: Removal of mbpool(9)
 OLD_FILES+=usr/include/sys/mbpool.h
 OLD_FILES+=usr/share/man/man9/mbpool.9.gz
 OLD_FILES+=usr/share/man/man9/mbp_destroy.9.gz
 OLD_FILES+=usr/share/man/man9/mbp_alloc.9.gz
 OLD_FILES+=usr/share/man/man9/mbp_ext_free.9.gz
 OLD_FILES+=usr/share/man/man9/mbp_count.9.gz
 OLD_FILES+=usr/share/man/man9/mbp_card_free.9.gz
 OLD_FILES+=usr/share/man/man9/mbp_get_keep.9.gz
 OLD_FILES+=usr/share/man/man9/mbp_free.9.gz
 OLD_FILES+=usr/share/man/man9/mbp_get.9.gz
 OLD_FILES+=usr/share/man/man9/mbp_create.9.gz
 OLD_FILES+=usr/share/man/man9/mbp_sync.9.gz
 # 20171010: Remove libstand
 OLD_FILES+=usr/lib/libstand.a
 OLD_FILES+=usr/lib/libstand_p.a
 OLD_FILES+=usr/lib32/libstand.a
 OLD_FILES+=usr/lib32/libstand_p.a
 OLD_FILES+=usr/include/stand.h
 OLD_FILES+=usr/share/man/man3/libstand.3.gz
 # 20171003: remove RCMDS
 OLD_FILES+=bin/rcp
 OLD_FILES+=rescue/rcp
 OLD_FILES+=usr/bin/rlogin
 OLD_FILES+=usr/bin/rsh
 OLD_FILES+=usr/libexec/rlogind
 OLD_FILES+=usr/libexec/rshd
 OLD_FILES+=usr/share/man/man1/rcp.1.gz
 OLD_FILES+=usr/share/man/man1/rlogin.1.gz
 OLD_FILES+=usr/share/man/man1/rsh.1.gz
 OLD_FILES+=usr/share/man/man8/rlogind.8.gz
 OLD_FILES+=usr/share/man/man8/rshd.8.gz
 # 20170927: crshared
 OLD_FILES+=usr/share/man/man9/crshared.9.gz
 # 20170927: procctl
 OLD_FILES+=usr/share/man/man8/procctl.8.gz
 OLD_FILES+=usr/sbin/procctl
 # 20170926: remove unneeded man aliases and locales directory
 OLD_FILES+=usr/share/man/en.ISO8859-1/man1
 OLD_FILES+=usr/share/man/en.ISO8859-1/man2
 OLD_FILES+=usr/share/man/en.ISO8859-1/man3
 OLD_FILES+=usr/share/man/en.ISO8859-1/man4
 OLD_FILES+=usr/share/man/en.ISO8859-1/man5
 OLD_FILES+=usr/share/man/en.ISO8859-1/man6
 OLD_FILES+=usr/share/man/en.ISO8859-1/man7
 OLD_FILES+=usr/share/man/en.ISO8859-1/man8
 OLD_FILES+=usr/share/man/en.ISO8859-1/man9
 OLD_DIRS+=usr/share/man/en.ISO8859-1
 OLD_FILES+=usr/share/man/en.ISO8859-1/mandoc.db
 OLD_FILES+=usr/share/man/en.UTF-8/man1
 OLD_FILES+=usr/share/man/en.UTF-8/man2
 OLD_FILES+=usr/share/man/en.UTF-8/man3
 OLD_FILES+=usr/share/man/en.UTF-8/man4
 OLD_FILES+=usr/share/man/en.UTF-8/man5
 OLD_FILES+=usr/share/man/en.UTF-8/man6
 OLD_FILES+=usr/share/man/en.UTF-8/man7
 OLD_FILES+=usr/share/man/en.UTF-8/man8
 OLD_FILES+=usr/share/man/en.UTF-8/man9
 OLD_FILES+=usr/share/man/en.UTF-8/mandoc.db
 OLD_DIRS+=usr/share/man/en.UTF-8
 OLD_FILES+=usr/share/man/en.ISO8859-15
 OLD_FILES+=usr/share/openssl/man/en.ISO8859-1/man1
 OLD_FILES+=usr/share/openssl/man/en.ISO8859-1/man3
 OLD_FILES+=usr/share/openssl/man/en.ISO8859-1/mandoc.db
 OLD_DIRS+=usr/share/openssl/man/en.ISO8859-1
 OLD_FILES+=usr/share/openssl/man/en.ISO8859-15
 OLD_DIRS+=usr/share/man/ja/man1
 OLD_DIRS+=usr/share/man/ja/man2
 OLD_DIRS+=usr/share/man/ja/man3
 OLD_DIRS+=usr/share/man/ja/man4
 OLD_DIRS+=usr/share/man/ja/man5
 OLD_DIRS+=usr/share/man/ja/man6
 OLD_DIRS+=usr/share/man/ja/man7
 OLD_DIRS+=usr/share/man/ja/man8
 OLD_DIRS+=usr/share/man/ja/man9
 OLD_DIRS+=usr/share/man/ja
 # 20170913: remove unneeded catman utility
 OLD_FILES+=etc/periodic/weekly/330.catman
 OLD_FILES+=usr/bin/catman
 OLD_FILES+=usr/libexec/catman.local
 OLD_FILES+=usr/share/man/man1/catman.1.gz
 OLD_FILES+=usr/share/man/man8/catman.local.8.gz
 OLD_DIRS+=usr/share/man/cat1
 OLD_DIRS+=usr/share/man/cat2
 OLD_DIRS+=usr/share/man/cat3
 OLD_DIRS+=usr/share/man/cat4/amd64
 OLD_DIRS+=usr/share/man/cat4/arm
 OLD_DIRS+=usr/share/man/cat4/i386
 OLD_DIRS+=usr/share/man/cat4/powerpc
 OLD_DIRS+=usr/share/man/cat4/sparc64
 OLD_DIRS+=usr/share/man/cat4
 OLD_DIRS+=usr/share/man/cat5
 OLD_DIRS+=usr/share/man/cat6
 OLD_DIRS+=usr/share/man/cat7
 OLD_DIRS+=usr/share/man/cat8/amd64
 OLD_DIRS+=usr/share/man/cat8/arm
 OLD_DIRS+=usr/share/man/cat8/i386
 OLD_DIRS+=usr/share/man/cat8/powerpc
 OLD_DIRS+=usr/share/man/cat8/sparc64
 OLD_DIRS+=usr/share/man/cat8
 OLD_DIRS+=usr/share/man/cat9
 OLD_DIRS+=usr/share/man/en.ISO8859-1/cat1
 OLD_DIRS+=usr/share/man/en.ISO8859-1/cat2
 OLD_DIRS+=usr/share/man/en.ISO8859-1/cat3
 OLD_DIRS+=usr/share/man/en.ISO8859-1/cat4/amd64
 OLD_DIRS+=usr/share/man/en.ISO8859-1/cat4/arm
 OLD_DIRS+=usr/share/man/en.ISO8859-1/cat4/i386
 OLD_DIRS+=usr/share/man/en.ISO8859-1/cat4/powerpc
 OLD_DIRS+=usr/share/man/en.ISO8859-1/cat4/sparc64
 OLD_DIRS+=usr/share/man/en.ISO8859-1/cat4
 OLD_DIRS+=usr/share/man/en.ISO8859-1/cat5
 OLD_DIRS+=usr/share/man/en.ISO8859-1/cat6
 OLD_DIRS+=usr/share/man/en.ISO8859-1/cat7
 OLD_DIRS+=usr/share/man/en.ISO8859-1/cat8/amd64
 OLD_DIRS+=usr/share/man/en.ISO8859-1/cat8/arm
 OLD_DIRS+=usr/share/man/en.ISO8859-1/cat8/i386
 OLD_DIRS+=usr/share/man/en.ISO8859-1/cat8/powerpc
 OLD_DIRS+=usr/share/man/en.ISO8859-1/cat8/sparc64
 OLD_DIRS+=usr/share/man/en.ISO8859-1/cat8
 OLD_DIRS+=usr/share/man/en.ISO8859-1/cat9
 OLD_DIRS+=usr/share/man/en.UTF-8/cat1
 OLD_DIRS+=usr/share/man/en.UTF-8/cat2
 OLD_DIRS+=usr/share/man/en.UTF-8/cat3
 OLD_DIRS+=usr/share/man/en.UTF-8/cat4/amd64
 OLD_DIRS+=usr/share/man/en.UTF-8/cat4/arm
 OLD_DIRS+=usr/share/man/en.UTF-8/cat4/i386
 OLD_DIRS+=usr/share/man/en.UTF-8/cat4/powerpc
 OLD_DIRS+=usr/share/man/en.UTF-8/cat4/sparc64
 OLD_DIRS+=usr/share/man/en.UTF-8/cat4
 OLD_DIRS+=usr/share/man/en.UTF-8/cat5
 OLD_DIRS+=usr/share/man/en.UTF-8/cat6
 OLD_DIRS+=usr/share/man/en.UTF-8/cat7
 OLD_DIRS+=usr/share/man/en.UTF-8/cat8/amd64
 OLD_DIRS+=usr/share/man/en.UTF-8/cat8/arm
 OLD_DIRS+=usr/share/man/en.UTF-8/cat8/i386
 OLD_DIRS+=usr/share/man/en.UTF-8/cat8/powerpc
 OLD_DIRS+=usr/share/man/en.UTF-8/cat8/sparc64
 OLD_DIRS+=usr/share/man/en.UTF-8/cat8
 OLD_DIRS+=usr/share/man/en.UTF-8/cat9
 OLD_DIRS+=usr/share/man/ja/cat1
 OLD_DIRS+=usr/share/man/ja/cat2
 OLD_DIRS+=usr/share/man/ja/cat3
 OLD_DIRS+=usr/share/man/ja/cat4/amd64
 OLD_DIRS+=usr/share/man/ja/cat4/arm
 OLD_DIRS+=usr/share/man/ja/cat4/i386
 OLD_DIRS+=usr/share/man/ja/cat4/powerpc
 OLD_DIRS+=usr/share/man/ja/cat4/sparc64
 OLD_DIRS+=usr/share/man/ja/cat4
 OLD_DIRS+=usr/share/man/ja/cat5
 OLD_DIRS+=usr/share/man/ja/cat6
 OLD_DIRS+=usr/share/man/ja/cat7
 OLD_DIRS+=usr/share/man/ja/cat8/amd64
 OLD_DIRS+=usr/share/man/ja/cat8/arm
 OLD_DIRS+=usr/share/man/ja/cat8/powerpc
 OLD_DIRS+=usr/share/man/ja/cat8/sparc64
 OLD_DIRS+=usr/share/man/ja/cat8
 OLD_DIRS+=usr/share/man/ja/cat9
 OLD_DIRS+=usr/share/openssl/man/cat1
 OLD_DIRS+=usr/share/openssl/man/cat3
 OLD_DIRS+=usr/share/openssl/man/en.ISO8859-1/cat1
 OLD_DIRS+=usr/share/openssl/man/en.ISO8859-1/cat3
 # 20170802: ksyms(4) ioctl interface was removed
 OLD_FILES+=usr/include/sys/ksyms.h
 # 20170729: the iicbus/pcf8563 driver is replaced with iicbus/nxprtc
 OLD_FILES+=usr/include/dev/iicbus/pcf8563reg.h
 # 20170722: new clang import which bumps version from 4.0.0 to 5.0.0.
 OLD_FILES+=usr/lib/clang/4.0.0/include/sanitizer/allocator_interface.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/sanitizer/asan_interface.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/sanitizer/common_interface_defs.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/sanitizer/coverage_interface.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/sanitizer/dfsan_interface.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/sanitizer/esan_interface.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/sanitizer/linux_syscall_hooks.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/sanitizer/lsan_interface.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/sanitizer/msan_interface.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/sanitizer/tsan_interface_atomic.h
 OLD_DIRS+=usr/lib/clang/4.0.0/include/sanitizer
 OLD_FILES+=usr/lib/clang/4.0.0/include/__clang_cuda_builtin_vars.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/__clang_cuda_cmath.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/__clang_cuda_complex_builtins.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/__clang_cuda_intrinsics.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/__clang_cuda_math_forward_declares.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/__clang_cuda_runtime_wrapper.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/__stddef_max_align_t.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/__wmmintrin_aes.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/__wmmintrin_pclmul.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/adxintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/altivec.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/ammintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/arm_acle.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/arm_neon.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/armintr.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/avx2intrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/avx512bwintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/avx512cdintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/avx512dqintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/avx512erintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/avx512fintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/avx512ifmaintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/avx512ifmavlintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/avx512pfintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/avx512vbmiintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/avx512vbmivlintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/avx512vlbwintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/avx512vlcdintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/avx512vldqintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/avx512vlintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/avxintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/bmi2intrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/bmiintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/clflushoptintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/cpuid.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/emmintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/f16cintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/fma4intrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/fmaintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/fxsrintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/htmintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/htmxlintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/ia32intrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/immintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/lzcntintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/mm3dnow.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/mm_malloc.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/mmintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/module.modulemap
 OLD_FILES+=usr/lib/clang/4.0.0/include/msa.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/mwaitxintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/nmmintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/opencl-c.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/pkuintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/pmmintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/popcntintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/prfchwintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/rdseedintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/rtmintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/s390intrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/shaintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/smmintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/tbmintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/tmmintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/vadefs.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/vecintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/wmmintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/x86intrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/xmmintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/xopintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/xsavecintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/xsaveintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/xsaveoptintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/xsavesintrin.h
 OLD_FILES+=usr/lib/clang/4.0.0/include/xtestintrin.h
 OLD_DIRS+=usr/lib/clang/4.0.0/include
 OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.asan-i386.a
 OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.asan-i386.so
 OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.asan-preinit-i386.a
 OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.asan-preinit-x86_64.a
 OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.asan-x86_64.a
 OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.asan-x86_64.so
 OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.asan_cxx-i386.a
 OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.asan_cxx-x86_64.a
 OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.profile-arm.a
 OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.profile-i386.a
 OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.profile-x86_64.a
 OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.safestack-i386.a
 OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.safestack-x86_64.a
 OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.stats-i386.a
 OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.stats-x86_64.a
 OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.stats_client-i386.a
 OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.stats_client-x86_64.a
 OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.ubsan_standalone-i386.a
 OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.ubsan_standalone-x86_64.a
 OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-i386.a
 OLD_FILES+=usr/lib/clang/4.0.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-x86_64.a
 OLD_DIRS+=usr/lib/clang/4.0.0/lib/freebsd
 OLD_DIRS+=usr/lib/clang/4.0.0/lib
 OLD_DIRS+=usr/lib/clang/4.0.0
 OLD_FILES+=usr/bin/llvm-pdbdump
 # 20170610: chown-f_test replaced by chown_test
 OLD_FILES+=usr/tests/usr.sbin/chown/chown-f_test
 # 20170609: drop obsolete manpage link (if_rtwn.ko -> rtwn.ko)
 OLD_FILES+=usr/share/man/man4/if_rtwn.4.gz
 # 20170531: removal of groff
 OLD_FILES+=usr/bin/addftinfo
 OLD_FILES+=usr/bin/afmtodit
 OLD_FILES+=usr/bin/checknr
 OLD_FILES+=usr/bin/colcrt
 OLD_FILES+=usr/bin/eqn
 OLD_FILES+=usr/bin/grn
 OLD_FILES+=usr/bin/grodvi
 OLD_FILES+=usr/bin/groff
 OLD_FILES+=usr/bin/grog
 OLD_FILES+=usr/bin/grolbp
 OLD_FILES+=usr/bin/grolj4
 OLD_FILES+=usr/bin/grops
 OLD_FILES+=usr/bin/grotty
 OLD_FILES+=usr/bin/hpftodit
 OLD_FILES+=usr/bin/indxbib
 OLD_FILES+=usr/bin/lkbib
 OLD_FILES+=usr/bin/lookbib
 OLD_FILES+=usr/bin/mmroff
 OLD_FILES+=usr/bin/neqn
 OLD_FILES+=usr/bin/nroff
 OLD_FILES+=usr/bin/pfbtops
 OLD_FILES+=usr/bin/pic
 OLD_FILES+=usr/bin/post-grohtml
 OLD_FILES+=usr/bin/pre-grohtml
 OLD_FILES+=usr/bin/psroff
 OLD_FILES+=usr/bin/refer
 OLD_FILES+=usr/bin/tbl
 OLD_FILES+=usr/bin/tfmtodit
 OLD_FILES+=usr/bin/troff
 OLD_FILES+=usr/bin/vgrind
 OLD_FILES+=usr/libexec/vfontedpr
 OLD_FILES+=usr/share/dict/eign
 OLD_FILES+=usr/share/groff_font/devX100-12/CB
 OLD_FILES+=usr/share/groff_font/devX100-12/CBI
 OLD_FILES+=usr/share/groff_font/devX100-12/CI
 OLD_FILES+=usr/share/groff_font/devX100-12/CR
 OLD_FILES+=usr/share/groff_font/devX100-12/DESC
 OLD_FILES+=usr/share/groff_font/devX100-12/HB
 OLD_FILES+=usr/share/groff_font/devX100-12/HBI
 OLD_FILES+=usr/share/groff_font/devX100-12/HI
 OLD_FILES+=usr/share/groff_font/devX100-12/HR
 OLD_FILES+=usr/share/groff_font/devX100-12/NB
 OLD_FILES+=usr/share/groff_font/devX100-12/NBI
 OLD_FILES+=usr/share/groff_font/devX100-12/NI
 OLD_FILES+=usr/share/groff_font/devX100-12/NR
 OLD_FILES+=usr/share/groff_font/devX100-12/S
 OLD_FILES+=usr/share/groff_font/devX100-12/TB
 OLD_FILES+=usr/share/groff_font/devX100-12/TBI
 OLD_FILES+=usr/share/groff_font/devX100-12/TI
 OLD_FILES+=usr/share/groff_font/devX100-12/TR
 OLD_DIRS+=usr/share/groff_font/devX100-12
 OLD_FILES+=usr/share/groff_font/devX100/CB
 OLD_FILES+=usr/share/groff_font/devX100/CBI
 OLD_FILES+=usr/share/groff_font/devX100/CI
 OLD_FILES+=usr/share/groff_font/devX100/CR
 OLD_FILES+=usr/share/groff_font/devX100/DESC
 OLD_FILES+=usr/share/groff_font/devX100/HB
 OLD_FILES+=usr/share/groff_font/devX100/HBI
 OLD_FILES+=usr/share/groff_font/devX100/HI
 OLD_FILES+=usr/share/groff_font/devX100/HR
 OLD_FILES+=usr/share/groff_font/devX100/NB
 OLD_FILES+=usr/share/groff_font/devX100/NBI
 OLD_FILES+=usr/share/groff_font/devX100/NI
 OLD_FILES+=usr/share/groff_font/devX100/NR
 OLD_FILES+=usr/share/groff_font/devX100/S
 OLD_FILES+=usr/share/groff_font/devX100/TB
 OLD_FILES+=usr/share/groff_font/devX100/TBI
 OLD_FILES+=usr/share/groff_font/devX100/TI
 OLD_FILES+=usr/share/groff_font/devX100/TR
 OLD_DIRS+=usr/share/groff_font/devX100
 OLD_FILES+=usr/share/groff_font/devX75-12/CB
 OLD_FILES+=usr/share/groff_font/devX75-12/CBI
 OLD_FILES+=usr/share/groff_font/devX75-12/CI
 OLD_FILES+=usr/share/groff_font/devX75-12/CR
 OLD_FILES+=usr/share/groff_font/devX75-12/DESC
 OLD_FILES+=usr/share/groff_font/devX75-12/HB
 OLD_FILES+=usr/share/groff_font/devX75-12/HBI
 OLD_FILES+=usr/share/groff_font/devX75-12/HI
 OLD_FILES+=usr/share/groff_font/devX75-12/HR
 OLD_FILES+=usr/share/groff_font/devX75-12/NB
 OLD_FILES+=usr/share/groff_font/devX75-12/NBI
 OLD_FILES+=usr/share/groff_font/devX75-12/NI
 OLD_FILES+=usr/share/groff_font/devX75-12/NR
 OLD_FILES+=usr/share/groff_font/devX75-12/S
 OLD_FILES+=usr/share/groff_font/devX75-12/TB
 OLD_FILES+=usr/share/groff_font/devX75-12/TBI
 OLD_FILES+=usr/share/groff_font/devX75-12/TI
 OLD_FILES+=usr/share/groff_font/devX75-12/TR
 OLD_DIRS+=usr/share/groff_font/devX75-12
 OLD_FILES+=usr/share/groff_font/devX75/CB
 OLD_FILES+=usr/share/groff_font/devX75/CBI
 OLD_FILES+=usr/share/groff_font/devX75/CI
 OLD_FILES+=usr/share/groff_font/devX75/CR
 OLD_FILES+=usr/share/groff_font/devX75/DESC
 OLD_FILES+=usr/share/groff_font/devX75/HB
 OLD_FILES+=usr/share/groff_font/devX75/HBI
 OLD_FILES+=usr/share/groff_font/devX75/HI
 OLD_FILES+=usr/share/groff_font/devX75/HR
 OLD_FILES+=usr/share/groff_font/devX75/NB
 OLD_FILES+=usr/share/groff_font/devX75/NBI
 OLD_FILES+=usr/share/groff_font/devX75/NI
 OLD_FILES+=usr/share/groff_font/devX75/NR
 OLD_FILES+=usr/share/groff_font/devX75/S
 OLD_FILES+=usr/share/groff_font/devX75/TB
 OLD_FILES+=usr/share/groff_font/devX75/TBI
 OLD_FILES+=usr/share/groff_font/devX75/TI
 OLD_FILES+=usr/share/groff_font/devX75/TR
 OLD_DIRS+=usr/share/groff_font/devX75
 OLD_FILES+=usr/share/groff_font/devascii/B
 OLD_FILES+=usr/share/groff_font/devascii/BI
 OLD_FILES+=usr/share/groff_font/devascii/CW
 OLD_FILES+=usr/share/groff_font/devascii/DESC
 OLD_FILES+=usr/share/groff_font/devascii/I
 OLD_FILES+=usr/share/groff_font/devascii/L
 OLD_FILES+=usr/share/groff_font/devascii/R
 OLD_FILES+=usr/share/groff_font/devascii/S
 OLD_DIRS+=usr/share/groff_font/devascii
 OLD_FILES+=usr/share/groff_font/devcp1047/B
 OLD_FILES+=usr/share/groff_font/devcp1047/BI
 OLD_FILES+=usr/share/groff_font/devcp1047/CW
 OLD_FILES+=usr/share/groff_font/devcp1047/DESC
 OLD_FILES+=usr/share/groff_font/devcp1047/I
 OLD_FILES+=usr/share/groff_font/devcp1047/L
 OLD_FILES+=usr/share/groff_font/devcp1047/R
 OLD_FILES+=usr/share/groff_font/devcp1047/S
 OLD_DIRS+=usr/share/groff_font/devcp1047
 OLD_FILES+=usr/share/groff_font/devdvi/CW
 OLD_FILES+=usr/share/groff_font/devdvi/CWEC
 OLD_FILES+=usr/share/groff_font/devdvi/CWI
 OLD_FILES+=usr/share/groff_font/devdvi/CWIEC
 OLD_FILES+=usr/share/groff_font/devdvi/CWITC
 OLD_FILES+=usr/share/groff_font/devdvi/CWTC
 OLD_FILES+=usr/share/groff_font/devdvi/CompileFonts
 OLD_FILES+=usr/share/groff_font/devdvi/DESC
 OLD_FILES+=usr/share/groff_font/devdvi/EX
 OLD_FILES+=usr/share/groff_font/devdvi/HB
 OLD_FILES+=usr/share/groff_font/devdvi/HBEC
 OLD_FILES+=usr/share/groff_font/devdvi/HBI
 OLD_FILES+=usr/share/groff_font/devdvi/HBIEC
 OLD_FILES+=usr/share/groff_font/devdvi/HBITC
 OLD_FILES+=usr/share/groff_font/devdvi/HBTC
 OLD_FILES+=usr/share/groff_font/devdvi/HI
 OLD_FILES+=usr/share/groff_font/devdvi/HIEC
 OLD_FILES+=usr/share/groff_font/devdvi/HITC
 OLD_FILES+=usr/share/groff_font/devdvi/HR
 OLD_FILES+=usr/share/groff_font/devdvi/HREC
 OLD_FILES+=usr/share/groff_font/devdvi/HRTC
 OLD_FILES+=usr/share/groff_font/devdvi/MI
 OLD_FILES+=usr/share/groff_font/devdvi/Makefile
 OLD_FILES+=usr/share/groff_font/devdvi/S
 OLD_FILES+=usr/share/groff_font/devdvi/SA
 OLD_FILES+=usr/share/groff_font/devdvi/SB
 OLD_FILES+=usr/share/groff_font/devdvi/SC
 OLD_FILES+=usr/share/groff_font/devdvi/TB
 OLD_FILES+=usr/share/groff_font/devdvi/TBEC
 OLD_FILES+=usr/share/groff_font/devdvi/TBI
 OLD_FILES+=usr/share/groff_font/devdvi/TBIEC
 OLD_FILES+=usr/share/groff_font/devdvi/TBITC
 OLD_FILES+=usr/share/groff_font/devdvi/TBTC
 OLD_FILES+=usr/share/groff_font/devdvi/TI
 OLD_FILES+=usr/share/groff_font/devdvi/TIEC
 OLD_FILES+=usr/share/groff_font/devdvi/TITC
 OLD_FILES+=usr/share/groff_font/devdvi/TR
 OLD_FILES+=usr/share/groff_font/devdvi/TREC
 OLD_FILES+=usr/share/groff_font/devdvi/TRTC
 OLD_FILES+=usr/share/groff_font/devdvi/ec.map
 OLD_FILES+=usr/share/groff_font/devdvi/msam.map
 OLD_FILES+=usr/share/groff_font/devdvi/msbm.map
 OLD_FILES+=usr/share/groff_font/devdvi/tc.map
 OLD_FILES+=usr/share/groff_font/devdvi/texb.map
 OLD_FILES+=usr/share/groff_font/devdvi/texex.map
 OLD_FILES+=usr/share/groff_font/devdvi/texi.map
 OLD_FILES+=usr/share/groff_font/devdvi/texmi.map
 OLD_FILES+=usr/share/groff_font/devdvi/texr.map
 OLD_FILES+=usr/share/groff_font/devdvi/texsy.map
 OLD_FILES+=usr/share/groff_font/devdvi/textex.map
 OLD_FILES+=usr/share/groff_font/devdvi/textt.map
 OLD_DIRS+=usr/share/groff_font/devdvi
 OLD_FILES+=usr/share/groff_font/devhtml/B
 OLD_FILES+=usr/share/groff_font/devhtml/BI
 OLD_FILES+=usr/share/groff_font/devhtml/CB
 OLD_FILES+=usr/share/groff_font/devhtml/CBI
 OLD_FILES+=usr/share/groff_font/devhtml/CI
 OLD_FILES+=usr/share/groff_font/devhtml/CR
 OLD_FILES+=usr/share/groff_font/devhtml/DESC
 OLD_FILES+=usr/share/groff_font/devhtml/I
 OLD_FILES+=usr/share/groff_font/devhtml/R
 OLD_FILES+=usr/share/groff_font/devhtml/S
 OLD_DIRS+=usr/share/groff_font/devhtml
 OLD_FILES+=usr/share/groff_font/devkoi8-r/B
 OLD_FILES+=usr/share/groff_font/devkoi8-r/BI
 OLD_FILES+=usr/share/groff_font/devkoi8-r/CW
 OLD_FILES+=usr/share/groff_font/devkoi8-r/DESC
 OLD_FILES+=usr/share/groff_font/devkoi8-r/I
 OLD_FILES+=usr/share/groff_font/devkoi8-r/L
 OLD_FILES+=usr/share/groff_font/devkoi8-r/R
 OLD_FILES+=usr/share/groff_font/devkoi8-r/S
 OLD_DIRS+=usr/share/groff_font/devkoi8-r
 OLD_FILES+=usr/share/groff_font/devlatin1/B
 OLD_FILES+=usr/share/groff_font/devlatin1/BI
 OLD_FILES+=usr/share/groff_font/devlatin1/CW
 OLD_FILES+=usr/share/groff_font/devlatin1/DESC
 OLD_FILES+=usr/share/groff_font/devlatin1/I
 OLD_FILES+=usr/share/groff_font/devlatin1/L
 OLD_FILES+=usr/share/groff_font/devlatin1/R
 OLD_FILES+=usr/share/groff_font/devlatin1/S
 OLD_DIRS+=usr/share/groff_font/devlatin1
 OLD_FILES+=usr/share/groff_font/devlbp/CB
 OLD_FILES+=usr/share/groff_font/devlbp/CI
 OLD_FILES+=usr/share/groff_font/devlbp/CR
 OLD_FILES+=usr/share/groff_font/devlbp/DESC
 OLD_FILES+=usr/share/groff_font/devlbp/EB
 OLD_FILES+=usr/share/groff_font/devlbp/EI
 OLD_FILES+=usr/share/groff_font/devlbp/ER
 OLD_FILES+=usr/share/groff_font/devlbp/HB
 OLD_FILES+=usr/share/groff_font/devlbp/HBI
 OLD_FILES+=usr/share/groff_font/devlbp/HI
 OLD_FILES+=usr/share/groff_font/devlbp/HNB
 OLD_FILES+=usr/share/groff_font/devlbp/HNBI
 OLD_FILES+=usr/share/groff_font/devlbp/HNI
 OLD_FILES+=usr/share/groff_font/devlbp/HNR
 OLD_FILES+=usr/share/groff_font/devlbp/HR
 OLD_FILES+=usr/share/groff_font/devlbp/TB
 OLD_FILES+=usr/share/groff_font/devlbp/TBI
 OLD_FILES+=usr/share/groff_font/devlbp/TI
 OLD_FILES+=usr/share/groff_font/devlbp/TR
 OLD_DIRS+=usr/share/groff_font/devlbp
 OLD_FILES+=usr/share/groff_font/devlj4/AB
 OLD_FILES+=usr/share/groff_font/devlj4/ABI
 OLD_FILES+=usr/share/groff_font/devlj4/AI
 OLD_FILES+=usr/share/groff_font/devlj4/ALBB
 OLD_FILES+=usr/share/groff_font/devlj4/ALBR
 OLD_FILES+=usr/share/groff_font/devlj4/AOB
 OLD_FILES+=usr/share/groff_font/devlj4/AOI
 OLD_FILES+=usr/share/groff_font/devlj4/AOR
 OLD_FILES+=usr/share/groff_font/devlj4/AR
 OLD_FILES+=usr/share/groff_font/devlj4/CB
 OLD_FILES+=usr/share/groff_font/devlj4/CBI
 OLD_FILES+=usr/share/groff_font/devlj4/CI
 OLD_FILES+=usr/share/groff_font/devlj4/CLARENDON
 OLD_FILES+=usr/share/groff_font/devlj4/CORONET
 OLD_FILES+=usr/share/groff_font/devlj4/CR
 OLD_FILES+=usr/share/groff_font/devlj4/DESC
 OLD_FILES+=usr/share/groff_font/devlj4/GB
 OLD_FILES+=usr/share/groff_font/devlj4/GBI
 OLD_FILES+=usr/share/groff_font/devlj4/GI
 OLD_FILES+=usr/share/groff_font/devlj4/GR
 OLD_FILES+=usr/share/groff_font/devlj4/LGB
 OLD_FILES+=usr/share/groff_font/devlj4/LGI
 OLD_FILES+=usr/share/groff_font/devlj4/LGR
 OLD_FILES+=usr/share/groff_font/devlj4/MARIGOLD
 OLD_FILES+=usr/share/groff_font/devlj4/OB
 OLD_FILES+=usr/share/groff_font/devlj4/OBI
 OLD_FILES+=usr/share/groff_font/devlj4/OI
 OLD_FILES+=usr/share/groff_font/devlj4/OR
 OLD_FILES+=usr/share/groff_font/devlj4/S
 OLD_FILES+=usr/share/groff_font/devlj4/SYMBOL
 OLD_FILES+=usr/share/groff_font/devlj4/TB
 OLD_FILES+=usr/share/groff_font/devlj4/TBI
 OLD_FILES+=usr/share/groff_font/devlj4/TI
 OLD_FILES+=usr/share/groff_font/devlj4/TNRB
 OLD_FILES+=usr/share/groff_font/devlj4/TNRBI
 OLD_FILES+=usr/share/groff_font/devlj4/TNRI
 OLD_FILES+=usr/share/groff_font/devlj4/TNRR
 OLD_FILES+=usr/share/groff_font/devlj4/TR
 OLD_FILES+=usr/share/groff_font/devlj4/UB
 OLD_FILES+=usr/share/groff_font/devlj4/UBI
 OLD_FILES+=usr/share/groff_font/devlj4/UCB
 OLD_FILES+=usr/share/groff_font/devlj4/UCBI
 OLD_FILES+=usr/share/groff_font/devlj4/UCI
 OLD_FILES+=usr/share/groff_font/devlj4/UCR
 OLD_FILES+=usr/share/groff_font/devlj4/UI
 OLD_FILES+=usr/share/groff_font/devlj4/UR
 OLD_FILES+=usr/share/groff_font/devlj4/WINGDINGS
 OLD_DIRS+=usr/share/groff_font/devlj4
 OLD_FILES+=usr/share/groff_font/devps/AB
 OLD_FILES+=usr/share/groff_font/devps/ABI
 OLD_FILES+=usr/share/groff_font/devps/AI
 OLD_FILES+=usr/share/groff_font/devps/AR
 OLD_FILES+=usr/share/groff_font/devps/BMB
 OLD_FILES+=usr/share/groff_font/devps/BMBI
 OLD_FILES+=usr/share/groff_font/devps/BMI
 OLD_FILES+=usr/share/groff_font/devps/BMR
 OLD_FILES+=usr/share/groff_font/devps/CB
 OLD_FILES+=usr/share/groff_font/devps/CBI
 OLD_FILES+=usr/share/groff_font/devps/CI
 OLD_FILES+=usr/share/groff_font/devps/CR
 OLD_FILES+=usr/share/groff_font/devps/DESC
 OLD_FILES+=usr/share/groff_font/devps/EURO
 OLD_FILES+=usr/share/groff_font/devps/HB
 OLD_FILES+=usr/share/groff_font/devps/HBI
 OLD_FILES+=usr/share/groff_font/devps/HI
 OLD_FILES+=usr/share/groff_font/devps/HNB
 OLD_FILES+=usr/share/groff_font/devps/HNBI
 OLD_FILES+=usr/share/groff_font/devps/HNI
 OLD_FILES+=usr/share/groff_font/devps/HNR
 OLD_FILES+=usr/share/groff_font/devps/HR
 OLD_FILES+=usr/share/groff_font/devps/Makefile
 OLD_FILES+=usr/share/groff_font/devps/NB
 OLD_FILES+=usr/share/groff_font/devps/NBI
 OLD_FILES+=usr/share/groff_font/devps/NI
 OLD_FILES+=usr/share/groff_font/devps/NR
 OLD_FILES+=usr/share/groff_font/devps/PB
 OLD_FILES+=usr/share/groff_font/devps/PBI
 OLD_FILES+=usr/share/groff_font/devps/PI
 OLD_FILES+=usr/share/groff_font/devps/PR
 OLD_FILES+=usr/share/groff_font/devps/S
 OLD_FILES+=usr/share/groff_font/devps/SS
 OLD_FILES+=usr/share/groff_font/devps/TB
 OLD_FILES+=usr/share/groff_font/devps/TBI
 OLD_FILES+=usr/share/groff_font/devps/TI
 OLD_FILES+=usr/share/groff_font/devps/TR
 OLD_FILES+=usr/share/groff_font/devps/ZCMI
 OLD_FILES+=usr/share/groff_font/devps/ZD
 OLD_FILES+=usr/share/groff_font/devps/ZDR
 OLD_FILES+=usr/share/groff_font/devps/afmname
 OLD_FILES+=usr/share/groff_font/devps/dingbats.map
 OLD_FILES+=usr/share/groff_font/devps/dingbats.rmap
 OLD_FILES+=usr/share/groff_font/devps/download
 OLD_FILES+=usr/share/groff_font/devps/freeeuro.pfa
 OLD_FILES+=usr/share/groff_font/devps/lgreekmap
 OLD_FILES+=usr/share/groff_font/devps/prologue
 OLD_FILES+=usr/share/groff_font/devps/symbol.sed
 OLD_FILES+=usr/share/groff_font/devps/symbolchars
 OLD_FILES+=usr/share/groff_font/devps/symbolsl.afm
 OLD_FILES+=usr/share/groff_font/devps/symbolsl.pfa
 OLD_FILES+=usr/share/groff_font/devps/text.enc
 OLD_FILES+=usr/share/groff_font/devps/textmap
 OLD_FILES+=usr/share/groff_font/devps/zapfdr.pfa
 OLD_DIRS+=usr/share/groff_font/devps
 OLD_FILES+=usr/share/groff_font/devutf8/B
 OLD_FILES+=usr/share/groff_font/devutf8/BI
 OLD_FILES+=usr/share/groff_font/devutf8/CW
 OLD_FILES+=usr/share/groff_font/devutf8/DESC
 OLD_FILES+=usr/share/groff_font/devutf8/I
 OLD_FILES+=usr/share/groff_font/devutf8/L
 OLD_FILES+=usr/share/groff_font/devutf8/R
 OLD_FILES+=usr/share/groff_font/devutf8/S
 OLD_DIRS+=usr/share/groff_font/devutf8
 OLD_DIRS+=usr/share/groff_font
 OLD_FILES+=usr/share/man/man1/addftinfo.1.gz
 OLD_FILES+=usr/share/man/man1/afmtodit.1.gz
 OLD_FILES+=usr/share/man/man1/checknr.1.gz
 OLD_FILES+=usr/share/man/man1/colcrt.1.gz
 OLD_FILES+=usr/share/man/man1/eqn.1.gz
 OLD_FILES+=usr/share/man/man1/grn.1.gz
 OLD_FILES+=usr/share/man/man1/grodvi.1.gz
 OLD_FILES+=usr/share/man/man1/groff.1.gz
 OLD_FILES+=usr/share/man/man1/grog.1.gz
 OLD_FILES+=usr/share/man/man1/grolbp.1.gz
 OLD_FILES+=usr/share/man/man1/grolj4.1.gz
 OLD_FILES+=usr/share/man/man1/grops.1.gz
 OLD_FILES+=usr/share/man/man1/grotty.1.gz
 OLD_FILES+=usr/share/man/man1/hpftodit.1.gz
 OLD_FILES+=usr/share/man/man1/indxbib.1.gz
 OLD_FILES+=usr/share/man/man1/lkbib.1.gz
 OLD_FILES+=usr/share/man/man1/lookbib.1.gz
 OLD_FILES+=usr/share/man/man1/mmroff.1.gz
 OLD_FILES+=usr/share/man/man1/neqn.1.gz
 OLD_FILES+=usr/share/man/man1/nroff.1.gz
 OLD_FILES+=usr/share/man/man1/pfbtops.1.gz
 OLD_FILES+=usr/share/man/man1/pic.1.gz
 OLD_FILES+=usr/share/man/man1/psroff.1.gz
 OLD_FILES+=usr/share/man/man1/refer.1.gz
 OLD_FILES+=usr/share/man/man1/tbl.1.gz
 OLD_FILES+=usr/share/man/man1/tfmtodit.1.gz
 OLD_FILES+=usr/share/man/man1/troff.1.gz
 OLD_FILES+=usr/share/man/man1/vgrind.1.gz
 OLD_FILES+=usr/share/man/man5/groff_font.5.gz
 OLD_FILES+=usr/share/man/man5/groff_out.5.gz
 OLD_FILES+=usr/share/man/man5/groff_tmac.5.gz
 OLD_FILES+=usr/share/man/man5/lj4_font.5.gz
 OLD_FILES+=usr/share/man/man5/tmac.5.gz
 OLD_FILES+=usr/share/man/man5/vgrindefs.5.gz
 OLD_FILES+=usr/share/man/man7/ditroff.7.gz
 OLD_FILES+=usr/share/man/man7/groff.7.gz
 OLD_FILES+=usr/share/man/man7/groff_char.7.gz
 OLD_FILES+=usr/share/man/man7/groff_diff.7.gz
 OLD_FILES+=usr/share/man/man7/groff_man.7.gz
 OLD_FILES+=usr/share/man/man7/groff_mdoc.7.gz
 OLD_FILES+=usr/share/man/man7/groff_me.7.gz
 OLD_FILES+=usr/share/man/man7/groff_mm.7.gz
 OLD_FILES+=usr/share/man/man7/groff_mmse.7.gz
 OLD_FILES+=usr/share/man/man7/groff_ms.7.gz
 OLD_FILES+=usr/share/man/man7/groff_trace.7.gz
 OLD_FILES+=usr/share/man/man7/groff_www.7.gz
 OLD_FILES+=usr/share/man/man7/mdoc.samples.7.gz
 OLD_FILES+=usr/share/man/man7/me.7.gz
 OLD_FILES+=usr/share/man/man7/mm.7.gz
 OLD_FILES+=usr/share/man/man7/mmse.7.gz
 OLD_FILES+=usr/share/man/man7/ms.7.gz
 OLD_FILES+=usr/share/man/man7/orig_me.7.gz
 OLD_FILES+=usr/share/me/acm.me
 OLD_FILES+=usr/share/me/chars.me
 OLD_FILES+=usr/share/me/deltext.me
 OLD_FILES+=usr/share/me/eqn.me
 OLD_FILES+=usr/share/me/float.me
 OLD_FILES+=usr/share/me/footnote.me
 OLD_FILES+=usr/share/me/index.me
 OLD_FILES+=usr/share/me/letterhead.me
 OLD_FILES+=usr/share/me/local.me
 OLD_FILES+=usr/share/me/null.me
 OLD_FILES+=usr/share/me/refer.me
 OLD_FILES+=usr/share/me/revisions
 OLD_FILES+=usr/share/me/sh.me
 OLD_FILES+=usr/share/me/tbl.me
 OLD_FILES+=usr/share/me/thesis.me
 OLD_DIRS+=usr/share/me
 OLD_FILES+=usr/share/misc/vgrindefs
 OLD_FILES+=usr/share/misc/vgrindefs.db
 OLD_FILES+=usr/share/tmac/X.tmac
 OLD_FILES+=usr/share/tmac/Xps.tmac
 OLD_FILES+=usr/share/tmac/a4.tmac
 OLD_FILES+=usr/share/tmac/an-old.tmac
 OLD_FILES+=usr/share/tmac/an.tmac
 OLD_FILES+=usr/share/tmac/andoc.tmac
 OLD_FILES+=usr/share/tmac/composite.tmac
 OLD_FILES+=usr/share/tmac/cp1047.tmac
 OLD_FILES+=usr/share/tmac/devtag.tmac
 OLD_FILES+=usr/share/tmac/doc.tmac
 OLD_FILES+=usr/share/tmac/dvi.tmac
 OLD_FILES+=usr/share/tmac/e.tmac
 OLD_FILES+=usr/share/tmac/ec.tmac
 OLD_FILES+=usr/share/tmac/eqnrc
 OLD_FILES+=usr/share/tmac/europs.tmac
 OLD_FILES+=usr/share/tmac/html-end.tmac
 OLD_FILES+=usr/share/tmac/html.tmac
 OLD_FILES+=usr/share/tmac/hyphen.ru
 OLD_FILES+=usr/share/tmac/hyphen.us
 OLD_FILES+=usr/share/tmac/hyphenex.us
 OLD_FILES+=usr/share/tmac/koi8-r.tmac
 OLD_FILES+=usr/share/tmac/latin1.tmac
 OLD_FILES+=usr/share/tmac/latin2.tmac
 OLD_FILES+=usr/share/tmac/latin9.tmac
 OLD_FILES+=usr/share/tmac/lbp.tmac
 OLD_FILES+=usr/share/tmac/lj4.tmac
 OLD_FILES+=usr/share/tmac/m.tmac
 OLD_FILES+=usr/share/tmac/man.local
 OLD_FILES+=usr/share/tmac/man.tmac
 OLD_FILES+=usr/share/tmac/mandoc.tmac
 OLD_FILES+=usr/share/tmac/mdoc.local
 OLD_FILES+=usr/share/tmac/mdoc.tmac
 OLD_FILES+=usr/share/tmac/mdoc/doc-common
 OLD_FILES+=usr/share/tmac/mdoc/doc-ditroff
 OLD_FILES+=usr/share/tmac/mdoc/doc-nroff
 OLD_FILES+=usr/share/tmac/mdoc/doc-syms
 OLD_FILES+=usr/share/tmac/mdoc/fr.ISO8859-1
 OLD_FILES+=usr/share/tmac/mdoc/ru.KOI8-R
 OLD_DIRS+=usr/share/tmac/mdoc
 OLD_FILES+=usr/share/tmac/me.tmac
 OLD_FILES+=usr/share/tmac/mm/0.MT
 OLD_FILES+=usr/share/tmac/mm/4.MT
 OLD_FILES+=usr/share/tmac/mm/5.MT
 OLD_FILES+=usr/share/tmac/mm/locale
 OLD_FILES+=usr/share/tmac/mm/mm.tmac
 OLD_FILES+=usr/share/tmac/mm/mmse.tmac
 OLD_FILES+=usr/share/tmac/mm/ms.cov
 OLD_FILES+=usr/share/tmac/mm/se_locale
 OLD_FILES+=usr/share/tmac/mm/se_ms.cov
 OLD_DIRS+=usr/share/tmac/mm
 OLD_FILES+=usr/share/tmac/ms.tmac
 OLD_FILES+=usr/share/tmac/mse.tmac
 OLD_FILES+=usr/share/tmac/papersize.tmac
 OLD_FILES+=usr/share/tmac/pic.tmac
 OLD_FILES+=usr/share/tmac/ps.tmac
 OLD_FILES+=usr/share/tmac/psatk.tmac
 OLD_FILES+=usr/share/tmac/psold.tmac
 OLD_FILES+=usr/share/tmac/pspic.tmac
 OLD_FILES+=usr/share/tmac/s.tmac
 OLD_FILES+=usr/share/tmac/safer.tmac
 OLD_FILES+=usr/share/tmac/tmac.orig_me
 OLD_FILES+=usr/share/tmac/tmac.vgrind
 OLD_FILES+=usr/share/tmac/trace.tmac
 OLD_FILES+=usr/share/tmac/troffrc
 OLD_FILES+=usr/share/tmac/troffrc-end
 OLD_FILES+=usr/share/tmac/tty-char.tmac
 OLD_FILES+=usr/share/tmac/tty.tmac
 OLD_FILES+=usr/share/tmac/unicode.tmac
 OLD_FILES+=usr/share/tmac/www.tmac
 OLD_DIRS+=usr/share/tmac
 # 20170607: remove incorrect atf_check(1) manpage link
 OLD_FILES+=usr/share/man/man1/atf_check.1.gz
 # 20170601: remove stale manpage
 OLD_FILES+=usr/share/man/man2/cap_rights_get.2.gz
 # 20170601: old libifconfig and libifc
 OLD_FILES+=usr/lib/libifc.a
 OLD_FILES+=usr/lib/libifc_p.a
 OLD_FILES+=usr/lib/libifconfig.a
 OLD_FILES+=usr/lib/libifconfig_p.a
 OLD_FILES+=usr/lib32/libifc.a
 OLD_FILES+=usr/lib32/libifc_p.a
 OLD_FILES+=usr/lib32/libifconfig.a
 OLD_FILES+=usr/lib32/libifconfig_p.a
 # 20170529: mount.conf(8) -> mount.conf(5)
 OLD_FILES+=usr/share/man/man8/mount.conf.8.gz
 # 20170525: remove misleading template
 OLD_FILES+=usr/share/misc/man.template
 # 20170525: disconnect the roff docs from the build
 OLD_FILES+=usr/share/doc/papers/beyond43.ascii.gz
 OLD_FILES+=usr/share/doc/papers/bio.ascii.gz
 OLD_FILES+=usr/share/doc/papers/contents.ascii.gz
 OLD_FILES+=usr/share/doc/papers/devfs.ascii.gz
 OLD_FILES+=usr/share/doc/papers/diskperf.ascii.gz
 OLD_FILES+=usr/share/doc/papers/fsinterface.ascii.gz
 OLD_FILES+=usr/share/doc/papers/hwpmc.ascii.gz
 OLD_FILES+=usr/share/doc/papers/jail.ascii.gz
 OLD_FILES+=usr/share/doc/papers/kernmalloc.ascii.gz
 OLD_FILES+=usr/share/doc/papers/kerntune.ascii.gz
 OLD_FILES+=usr/share/doc/papers/malloc.ascii.gz
 OLD_FILES+=usr/share/doc/papers/newvm.ascii.gz
 OLD_FILES+=usr/share/doc/papers/releng.ascii.gz
 OLD_FILES+=usr/share/doc/papers/sysperf.ascii.gz
 OLD_FILES+=usr/share/doc/papers/timecounter.ascii.gz
 OLD_DIRS+=usr/share/doc/papers
 OLD_FILES+=usr/share/doc/psd/01.cacm/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/psd/01.cacm
 OLD_FILES+=usr/share/doc/psd/02.implement/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/psd/02.implement
 OLD_FILES+=usr/share/doc/psd/03.iosys/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/psd/03.iosys
 OLD_FILES+=usr/share/doc/psd/04.uprog/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/psd/04.uprog
 OLD_FILES+=usr/share/doc/psd/05.sysman/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/psd/05.sysman
 OLD_FILES+=usr/share/doc/psd/06.Clang/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/psd/06.Clang
 OLD_FILES+=usr/share/doc/psd/12.make/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/psd/12.make
 OLD_FILES+=usr/share/doc/psd/13.rcs/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/psd/13.rcs
 OLD_FILES+=usr/share/doc/psd/13.rcs/rcs_func.ascii.gz
 OLD_DIRS+=usr/share/doc/psd/13.rcs
 OLD_FILES+=usr/share/doc/psd/15.yacc/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/psd/15.yacc
 OLD_FILES+=usr/share/doc/psd/16.lex/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/psd/16.lex
 OLD_FILES+=usr/share/doc/psd/17.m4/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/psd/17.m4
 OLD_FILES+=usr/share/doc/psd/18.gprof/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/psd/18.gprof
 OLD_FILES+=usr/share/doc/psd/20.ipctut/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/psd/20.ipctut
 OLD_FILES+=usr/share/doc/psd/21.ipc/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/psd/21.ipc
 OLD_FILES+=usr/share/doc/psd/22.rpcgen/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/psd/22.rpcgen
 OLD_FILES+=usr/share/doc/psd/23.rpc/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/psd/23.rpc
 OLD_FILES+=usr/share/doc/psd/24.xdr/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/psd/24.xdr
 OLD_FILES+=usr/share/doc/psd/25.xdrrfc/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/psd/25.xdrrfc
 OLD_FILES+=usr/share/doc/psd/26.rpcrfc/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/psd/26.rpcrfc
 OLD_FILES+=usr/share/doc/psd/27.nfsrfc/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/psd/27.nfsrfc
 OLD_FILES+=usr/share/doc/psd/Title.ascii.gz
 OLD_FILES+=usr/share/doc/psd/contents.ascii.gz
 OLD_DIRS+=usr/share/doc/psd/
 OLD_FILES+=usr/share/doc/smm/01.setup/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/smm/01.setup
 OLD_FILES+=usr/share/doc/smm/02.config/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/smm/02.config
 OLD_FILES+=usr/share/doc/smm/03.fsck/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/smm/03.fsck
 OLD_FILES+=usr/share/doc/smm/04.quotas/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/smm/04.quotas
 OLD_FILES+=usr/share/doc/smm/05.fastfs/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/smm/05.fastfs
 OLD_FILES+=usr/share/doc/smm/06.nfs/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/smm/06.nfs
 OLD_FILES+=usr/share/doc/smm/07.lpd/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/smm/07.lpd
 OLD_FILES+=usr/share/doc/smm/08.sendmailop/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/smm/08.sendmailop
 OLD_FILES+=usr/share/doc/smm/11.timedop/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/smm/11.timedop
 OLD_FILES+=usr/share/doc/smm/12.timed/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/smm/12.timed
 OLD_FILES+=usr/share/doc/smm/18.net/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/smm/18.net
 OLD_FILES+=usr/share/doc/smm/Title.ascii.gz
 OLD_FILES+=usr/share/doc/smm/contents.ascii.gz
 OLD_DIRS+=usr/share/doc/smm
 OLD_FILES+=usr/share/doc/usd/04.csh/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/usd/04.csh
 OLD_FILES+=usr/share/doc/usd/05.dc/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/usd/05.dc
 OLD_FILES+=usr/share/doc/usd/06.bc/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/usd/06.bc
 OLD_FILES+=usr/share/doc/usd/07.mail/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/usd/07.mail
 OLD_FILES+=usr/share/doc/usd/10.exref/paper.ascii.gz
 OLD_FILES+=usr/share/doc/usd/10.exref/summary.ascii.gz
 OLD_DIRS+=usr/share/doc/usd/10.exref
 OLD_FILES+=usr/share/doc/usd/11.edit/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/usd/11.edit
 OLD_FILES+=usr/share/doc/usd/12.vi/paper.ascii.gz
 OLD_FILES+=usr/share/doc/usd/12.vi/summary.ascii.gz
 OLD_FILES+=usr/share/doc/usd/12.vi/viapwh.ascii.gz
 OLD_DIRS+=usr/share/doc/usd/12.vi
 OLD_FILES+=usr/share/doc/usd/13.viref/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/usd/13.viref
 OLD_FILES+=usr/share/doc/usd/18.msdiffs/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/usd/18.msdiffs
 OLD_FILES+=usr/share/doc/usd/19.memacros/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/usd/19.memacros
 OLD_FILES+=usr/share/doc/usd/20.meref/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/usd/20.meref
 OLD_FILES+=usr/share/doc/usd/21.troff/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/usd/21.troff
 OLD_FILES+=usr/share/doc/usd/22.trofftut/paper.ascii.gz
 OLD_DIRS+=usr/share/doc/usd/22.trofftut
 OLD_FILES+=usr/share/doc/usd/Title.ascii.gz
 OLD_FILES+=usr/share/doc/usd/contents.ascii.gz
 OLD_DIRS+=usr/share/doc/usd
 # 20170523: 64-bit inode support, library version bumps
 OLD_LIBS+=lib/libzfs.so.2
 OLD_LIBS+=usr/lib/libarchive.so.6
 OLD_LIBS+=usr/lib/libmilter.so.5
 OLD_LIBS+=usr/lib32/libzfs.so.2
 OLD_LIBS+=usr/lib32/libarchive.so.6
 OLD_LIBS+=usr/lib32/libmilter.so.5
 # 20170427: NATM configuration support removed
 OLD_FILES+=etc/rc.d/atm1
 OLD_FILES+=etc/rc.d/atm2
 OLD_FILES+=etc/rc.d/atm3
 # 20170424: NATM support removed
 OLD_FILES+=rescue/atmconfig
 OLD_FILES+=sbin/atmconfig
 OLD_FILES+=usr/include/bsnmp/snmp_atm.h
 OLD_FILES+=usr/include/dev/utopia/idtphy.h
 OLD_FILES+=usr/include/dev/utopia/suni.h
 OLD_FILES+=usr/include/dev/utopia/utopia.h
 OLD_FILES+=usr/include/dev/utopia/utopia_priv.h
 OLD_DIRS+=usr/include/dev/utopia
 OLD_FILES+=usr/include/net/if_atm.h
 OLD_FILES+=usr/include/netgraph/atm/ng_atm.h
 OLD_FILES+=usr/include/netinet/if_atm.h
 OLD_FILES+=usr/include/netnatm/natm.h
 OLD_FILES+=usr/lib/debug/sbin/atmconfig.debug
 OLD_FILES+=usr/lib/debug/usr/lib/snmp_atm.so.6.debug
 OLD_FILES+=usr/lib/snmp_atm.so
 OLD_FILES+=usr/lib/snmp_atm.so.6
 OLD_FILES+=usr/share/doc/atm/atmconfig.help
 OLD_FILES+=usr/share/doc/atm/atmconfig_device.help
 OLD_DIRS+=usr/share/doc/atm
 OLD_FILES+=usr/share/man/man3/snmp_atm.3.gz
 OLD_FILES+=usr/share/man/man4/en.4.gz
 OLD_FILES+=usr/share/man/man4/fatm.4.gz
 OLD_FILES+=usr/share/man/man4/hatm.4.gz
 OLD_FILES+=usr/share/man/man4/if_en.4.gz
 OLD_FILES+=usr/share/man/man4/if_fatm.4.gz
 OLD_FILES+=usr/share/man/man4/if_hatm.4.gz
 OLD_FILES+=usr/share/man/man4/if_patm.4.gz
 OLD_FILES+=usr/share/man/man4/natm.4.gz
 OLD_FILES+=usr/share/man/man4/natmip.4.gz
 OLD_FILES+=usr/share/man/man4/ng_atm.4.gz
 OLD_FILES+=usr/share/man/man4/patm.4.gz
 OLD_FILES+=usr/share/man/man4/utopia.4.gz
 OLD_FILES+=usr/share/man/man8/atmconfig.8.gz
 OLD_FILES+=usr/share/man/man9/utopia.9.gz
 OLD_FILES+=usr/share/snmp/defs/atm_freebsd.def
 OLD_FILES+=usr/share/snmp/defs/atm_tree.def
 OLD_FILES+=usr/share/snmp/mibs/BEGEMOT-ATM-FREEBSD-MIB.txt
 OLD_FILES+=usr/share/snmp/mibs/BEGEMOT-ATM.txt
 # 20170420: remove GNU diff
 OLD_FILES+=usr/share/man/man7/diff.7.gz
 # 20170322: rename <x> to <x>_test to match the FreeBSD test suite name scheme
 OLD_FILES+=usr/tests/usr.bin/col/col
 OLD_FILES+=usr/tests/usr.bin/diff/diff
 OLD_FILES+=usr/tests/usr.bin/ident/ident
 OLD_FILES+=usr/tests/usr.bin/mkimg/mkimg
 OLD_FILES+=usr/tests/usr.bin/sdiff/sdiff
 OLD_FILES+=usr/tests/usr.bin/soelim/soelim
 OLD_FILES+=usr/tests/usr.sbin/pw/pw_config
 OLD_FILES+=usr/tests/usr.sbin/pw/pw_etcdir
 OLD_FILES+=usr/tests/usr.sbin/pw/pw_groupadd
 OLD_FILES+=usr/tests/usr.sbin/pw/pw_groupdel
 OLD_FILES+=usr/tests/usr.sbin/pw/pw_groupmod
 OLD_FILES+=usr/tests/usr.sbin/pw/pw_lock
 OLD_FILES+=usr/tests/usr.sbin/pw/pw_useradd
 OLD_FILES+=usr/tests/usr.sbin/pw/pw_userdel
 OLD_FILES+=usr/tests/usr.sbin/pw/pw_usermod
 OLD_FILES+=usr/tests/usr.sbin/pw/pw_usernext
 # 20170319: io_test requires zh_TW.Big5 locale.
 OLD_FILES+=usr/tests/lib/libc/locale/io_test
 # 20170319: remove nls for non supported Big5* locales
 OLD_DIRS+=usr/share/nls/zh_HK.Big5HKSCS
 OLD_DIRS+=usr/share/nls/zh_TW.Big5
 # 20170313: move .../sys/geom/eli/... to .../sys/geom/class/eli/...
 OLD_FILES+=usr/tests/sys/geom/eli/pbkdf2/pbkdf2
 OLD_FILES+=usr/tests/sys/geom/eli/pbkdf2/Kyuafile
 OLD_FILES+=usr/tests/sys/geom/eli/Kyuafile
 OLD_DIRS+=usr/tests/sys/geom/eli/pbkdf2
 OLD_DIRS+=usr/tests/sys/geom/eli
 # 20170313: sbin/ipftest and ipresend temporarily disconnected.
 OLD_FILES+=sbin/ipftest
 OLD_FILES+=sbin/ipresend
 # 20170311: Remove WITHOUT_MANDOCDB option
 OLD_FILES+=usr/share/man/man1/makewhatis.1.gz
 # 20170308: rename some tests
 OLD_FILES+=usr/tests/bin/pwait/pwait
 OLD_FILES+=usr/tests/usr.bin/timeout/timeout
 # 20170307: remove pcap-int.h
 OLD_FILES+=usr/include/pcap-int.h
 # 20170302: new libc++ import which bumps version from 3.9.1 to 4.0.0.
 OLD_FILES+=usr/include/c++/v1/__undef___deallocate
 OLD_FILES+=usr/include/c++/v1/tr1/__undef___deallocate
 # 20170302: new clang import which bumps version from 3.9.1 to 4.0.0.
 OLD_FILES+=usr/lib/clang/3.9.1/include/sanitizer/allocator_interface.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/sanitizer/asan_interface.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/sanitizer/common_interface_defs.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/sanitizer/coverage_interface.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/sanitizer/dfsan_interface.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/sanitizer/esan_interface.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/sanitizer/linux_syscall_hooks.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/sanitizer/lsan_interface.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/sanitizer/msan_interface.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/sanitizer/tsan_interface_atomic.h
 OLD_DIRS+=usr/lib/clang/3.9.1/include/sanitizer
 OLD_FILES+=usr/lib/clang/3.9.1/include/__clang_cuda_cmath.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/__clang_cuda_intrinsics.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/__clang_cuda_math_forward_declares.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/__clang_cuda_runtime_wrapper.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/__stddef_max_align_t.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/__wmmintrin_aes.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/__wmmintrin_pclmul.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/adxintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/altivec.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/ammintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/arm_acle.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/arm_neon.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/avx2intrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/avx512bwintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/avx512cdintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/avx512dqintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/avx512erintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/avx512fintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/avx512ifmaintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/avx512ifmavlintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/avx512pfintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/avx512vbmiintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/avx512vbmivlintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/avx512vlbwintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/avx512vlcdintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/avx512vldqintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/avx512vlintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/avxintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/bmi2intrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/bmiintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/clflushoptintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/cpuid.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/cuda_builtin_vars.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/emmintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/f16cintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/fma4intrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/fmaintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/fxsrintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/htmintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/htmxlintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/ia32intrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/immintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/lzcntintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/mm3dnow.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/mm_malloc.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/mmintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/module.modulemap
 OLD_FILES+=usr/lib/clang/3.9.1/include/msa.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/mwaitxintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/nmmintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/opencl-c.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/pkuintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/pmmintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/popcntintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/prfchwintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/rdseedintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/rtmintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/s390intrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/shaintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/smmintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/tbmintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/tmmintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/vadefs.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/vecintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/wmmintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/x86intrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/xmmintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/xopintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/xsavecintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/xsaveintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/xsaveoptintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/xsavesintrin.h
 OLD_FILES+=usr/lib/clang/3.9.1/include/xtestintrin.h
 OLD_DIRS+=usr/lib/clang/3.9.1/include
 OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.asan-i386.a
 OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.asan-i386.so
 OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.asan-preinit-i386.a
 OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.asan-preinit-x86_64.a
 OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.asan-x86_64.a
 OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.asan-x86_64.so
 OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.asan_cxx-i386.a
 OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.asan_cxx-x86_64.a
 OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.profile-arm.a
 OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.profile-i386.a
 OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.profile-x86_64.a
 OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.safestack-i386.a
 OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.safestack-x86_64.a
 OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.stats-i386.a
 OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.stats-x86_64.a
 OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.stats_client-i386.a
 OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.stats_client-x86_64.a
 OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.ubsan_standalone-i386.a
 OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.ubsan_standalone-x86_64.a
 OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.ubsan_standalone_cxx-i386.a
 OLD_FILES+=usr/lib/clang/3.9.1/lib/freebsd/libclang_rt.ubsan_standalone_cxx-x86_64.a
 OLD_DIRS+=usr/lib/clang/3.9.1/lib/freebsd
 OLD_DIRS+=usr/lib/clang/3.9.1/lib
 OLD_DIRS+=usr/lib/clang/3.9.1
 # 20170226: SVR4 compatibility removed
 .if ${TARGET_ARCH} == "amd64" || ${TARGET_ARCH} == "i386"
 OLD_FILES+=usr/share/man/man4/streams.4
 OLD_FILES+=usr/share/man/man4/svr4.4
 .endif
 # 20170219: OpenPAM RADULA upgrade removed the libpam tests
 OLD_FILES+=usr/tests/lib/libpam/Kyuafile
 OLD_FILES+=usr/tests/lib/libpam/t_openpam_ctype
 OLD_FILES+=usr/tests/lib/libpam/t_openpam_readlinev
 OLD_FILES+=usr/tests/lib/libpam/t_openpam_readword
 OLD_DIRS+=usr/test/lib/libpam
 # 20170206: remove bdes(1)
 OLD_FILES+=usr/bin/bdes
 OLD_FILES+=usr/lib/debug/usr/bin/bdes.debug
 OLD_FILES+=usr/share/man/man1/bdes.1.gz
 # 20170206: merged projects/ipsec
 OLD_FILES+=usr/include/netinet/ip_ipsec.h
 OLD_FILES+=usr/include/netinet6/ip6_ipsec.h
 # 20170128: remove pc98 support
 OLD_FILES+=usr/include/dev/ic/i8251.h
 OLD_FILES+=usr/include/dev/ic/i8255.h
 OLD_FILES+=usr/include/dev/ic/rsa.h
 OLD_FILES+=usr/include/dev/ic/wd33c93reg.h
 OLD_FILES+=usr/include/sys/disk/pc98.h
 OLD_FILES+=usr/include/sys/diskpc98.h
 OLD_FILES+=usr/share/man/man4/i386/ct.4.gz
 OLD_FILES+=usr/share/man/man4/i386/snc.4.gz
 OLD_FILES+=usr/share/syscons/keymaps/jp.pc98.iso.kbd
 OLD_FILES+=usr/share/syscons/keymaps/jp.pc98.kbd
 OLD_FILES+=usr/share/vt/keymaps/jp.pc98.iso.kbd
 OLD_FILES+=usr/share/vt/keymaps/jp.pc98.kbd
 # 20170110: Four files from ggate tests consolidated into one
 OLD_FILES+=usr/tests/sys/geom/class/gate/1_test
 OLD_FILES+=usr/tests/sys/geom/class/gate/2_test
 OLD_FILES+=usr/tests/sys/geom/class/gate/3_test
 OLD_FILES+=usr/tests/sys/geom/class/gate/conf.sh
 # 20170103: libbsnmptools.so made into an INTERNALLIB
 OLD_FILES+=usr/lib/libbsnmptools.a
 OLD_FILES+=usr/lib/libbsnmptools_p.a
 OLD_LIBS+=usr/lib/libbsnmptools.so.0
 OLD_LIBS+=usr/lib/libbsnmptools.so
 # 20170102: sysdecode_getfsstat_flags() renamed to sysdecode_getfsstat_mode()
 OLD_FILES+=usr/share/man/man3/sysdecode_getfsstat_flags.3.gz
 # 20161230: libarchive ACL pax test renamed to test_acl_pax_posix1e.tar.uu
 OLD_FILES+=usr/tests/lib/libarchive/test_acl_pax.tar.uu
 # 20161229: Three files from gnop tests consolidated into one
 OLD_FILES+=usr/tests/sys/geom/class/nop/1_test
 OLD_FILES+=usr/tests/sys/geom/class/nop/2_test
 OLD_FILES+=usr/tests/sys/geom/class/nop/conf.sh
 # 20161217: new clang import which bumps version from 3.9.0 to 3.9.1.
 OLD_FILES+=usr/lib/clang/3.9.0/include/sanitizer/allocator_interface.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/sanitizer/asan_interface.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/sanitizer/common_interface_defs.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/sanitizer/coverage_interface.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/sanitizer/dfsan_interface.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/sanitizer/esan_interface.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/sanitizer/linux_syscall_hooks.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/sanitizer/lsan_interface.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/sanitizer/msan_interface.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/sanitizer/tsan_interface_atomic.h
 OLD_DIRS+=usr/lib/clang/3.9.0/include/sanitizer
 OLD_FILES+=usr/lib/clang/3.9.0/include/__clang_cuda_cmath.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/__clang_cuda_intrinsics.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/__clang_cuda_math_forward_declares.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/__clang_cuda_runtime_wrapper.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/__stddef_max_align_t.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/__wmmintrin_aes.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/__wmmintrin_pclmul.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/adxintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/altivec.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/ammintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/arm_acle.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/arm_neon.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/avx2intrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/avx512bwintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/avx512cdintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/avx512dqintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/avx512erintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/avx512fintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/avx512ifmaintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/avx512ifmavlintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/avx512pfintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/avx512vbmiintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/avx512vbmivlintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/avx512vlbwintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/avx512vlcdintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/avx512vldqintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/avx512vlintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/avxintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/bmi2intrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/bmiintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/clflushoptintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/cpuid.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/cuda_builtin_vars.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/emmintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/f16cintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/fma4intrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/fmaintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/fxsrintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/htmintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/htmxlintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/ia32intrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/immintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/lzcntintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/mm3dnow.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/mm_malloc.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/mmintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/module.modulemap
 OLD_FILES+=usr/lib/clang/3.9.0/include/mwaitxintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/nmmintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/opencl-c.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/pkuintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/pmmintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/popcntintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/prfchwintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/rdseedintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/rtmintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/s390intrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/shaintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/smmintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/tbmintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/tmmintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/vadefs.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/vecintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/wmmintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/x86intrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/xmmintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/xopintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/xsavecintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/xsaveintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/xsaveoptintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/xsavesintrin.h
 OLD_FILES+=usr/lib/clang/3.9.0/include/xtestintrin.h
 OLD_DIRS+=usr/lib/clang/3.9.0/include
 OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.asan-i386.a
 OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.asan-i386.so
 OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.asan-preinit-i386.a
 OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.asan-preinit-x86_64.a
 OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.asan-x86_64.a
 OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.asan-x86_64.so
 OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.asan_cxx-i386.a
 OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.asan_cxx-x86_64.a
 OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.profile-arm.a
 OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.profile-i386.a
 OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.profile-x86_64.a
 OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.safestack-i386.a
 OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.safestack-x86_64.a
 OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.stats-i386.a
 OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.stats-x86_64.a
 OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.stats_client-i386.a
 OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.stats_client-x86_64.a
 OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.ubsan_standalone-i386.a
 OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.ubsan_standalone-x86_64.a
 OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-i386.a
 OLD_FILES+=usr/lib/clang/3.9.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-x86_64.a
 OLD_DIRS+=usr/lib/clang/3.9.0/lib/freebsd
 OLD_DIRS+=usr/lib/clang/3.9.0/lib
 OLD_DIRS+=usr/lib/clang/3.9.0
 # 20161205: libproc version bump
 OLD_LIBS+=usr/lib/libproc.so.3
 OLD_LIBS+=usr/lib32/libproc.so.3
 # 20161127: Remove vm_page_cache(9)
 OLD_FILES+=usr/share/man/man9/vm_page_cache.9.gz
 # 20161124: new clang import which bumps version from 3.8.0 to 3.9.0.
 OLD_FILES+=usr/lib/clang/3.8.0/include/sanitizer/allocator_interface.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/sanitizer/asan_interface.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/sanitizer/common_interface_defs.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/sanitizer/coverage_interface.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/sanitizer/dfsan_interface.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/sanitizer/linux_syscall_hooks.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/sanitizer/lsan_interface.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/sanitizer/msan_interface.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/sanitizer/tsan_interface_atomic.h
 OLD_DIRS+=usr/lib/clang/3.8.0/include/sanitizer
 OLD_FILES+=usr/lib/clang/3.8.0/include/__clang_cuda_runtime_wrapper.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/__stddef_max_align_t.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/__wmmintrin_aes.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/__wmmintrin_pclmul.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/adxintrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/altivec.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/ammintrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/arm_acle.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/arm_neon.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/avx2intrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/avx512bwintrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/avx512cdintrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/avx512dqintrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/avx512erintrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/avx512fintrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/avx512vlbwintrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/avx512vldqintrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/avx512vlintrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/avxintrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/bmi2intrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/bmiintrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/cpuid.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/cuda_builtin_vars.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/emmintrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/f16cintrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/fma4intrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/fmaintrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/fxsrintrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/htmintrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/htmxlintrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/ia32intrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/immintrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/lzcntintrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/mm3dnow.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/mm_malloc.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/mmintrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/module.modulemap
 OLD_FILES+=usr/lib/clang/3.8.0/include/nmmintrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/pkuintrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/pmmintrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/popcntintrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/prfchwintrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/rdseedintrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/rtmintrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/s390intrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/shaintrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/smmintrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/tbmintrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/tmmintrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/vadefs.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/vecintrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/wmmintrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/x86intrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/xmmintrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/xopintrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/xsavecintrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/xsaveintrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/xsaveoptintrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/xsavesintrin.h
 OLD_FILES+=usr/lib/clang/3.8.0/include/xtestintrin.h
 OLD_DIRS+=usr/lib/clang/3.8.0/include
 OLD_FILES+=usr/lib/clang/3.8.0/lib/freebsd/libclang_rt.asan-i386.a
 OLD_FILES+=usr/lib/clang/3.8.0/lib/freebsd/libclang_rt.asan-i386.so
 OLD_FILES+=usr/lib/clang/3.8.0/lib/freebsd/libclang_rt.asan-preinit-i386.a
 OLD_FILES+=usr/lib/clang/3.8.0/lib/freebsd/libclang_rt.asan-preinit-x86_64.a
 OLD_FILES+=usr/lib/clang/3.8.0/lib/freebsd/libclang_rt.asan-x86_64.a
 OLD_FILES+=usr/lib/clang/3.8.0/lib/freebsd/libclang_rt.asan-x86_64.so
 OLD_FILES+=usr/lib/clang/3.8.0/lib/freebsd/libclang_rt.asan_cxx-i386.a
 OLD_FILES+=usr/lib/clang/3.8.0/lib/freebsd/libclang_rt.asan_cxx-x86_64.a
 OLD_FILES+=usr/lib/clang/3.8.0/lib/freebsd/libclang_rt.profile-arm.a
 OLD_FILES+=usr/lib/clang/3.8.0/lib/freebsd/libclang_rt.profile-i386.a
 OLD_FILES+=usr/lib/clang/3.8.0/lib/freebsd/libclang_rt.profile-x86_64.a
 OLD_FILES+=usr/lib/clang/3.8.0/lib/freebsd/libclang_rt.safestack-i386.a
 OLD_FILES+=usr/lib/clang/3.8.0/lib/freebsd/libclang_rt.safestack-x86_64.a
 OLD_FILES+=usr/lib/clang/3.8.0/lib/freebsd/libclang_rt.ubsan_standalone-i386.a
 OLD_FILES+=usr/lib/clang/3.8.0/lib/freebsd/libclang_rt.ubsan_standalone-x86_64.a
 OLD_FILES+=usr/lib/clang/3.8.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-i386.a
 OLD_FILES+=usr/lib/clang/3.8.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-x86_64.a
 OLD_DIRS+=usr/lib/clang/3.8.0/lib/freebsd
 OLD_DIRS+=usr/lib/clang/3.8.0/lib
 OLD_DIRS+=usr/lib/clang/3.8.0
 # 20161121: Hyper-V manuals only apply to amd64 and i386.
 .if ${TARGET_ARCH} != "amd64" && ${TARGET_ARCH} != "i386"
 OLD_FILES+=usr/share/man/man4/hv_kvp.4.gz
 OLD_FILES+=usr/share/man/man4/hv_netvsc.4.gz
 OLD_FILES+=usr/share/man/man4/hv_storvsc.4.gz
 OLD_FILES+=usr/share/man/man4/hv_utils.4.gz
 OLD_FILES+=usr/share/man/man4/hv_vmbus.4.gz
 OLD_FILES+=usr/share/man/man4/hv_vss.4.gz
 .endif
 # 20161118: Remove hv_ata_pci_disengage(4)
 OLD_FILES+=usr/share/man/man4/hv_ata_pci_disengage.4.gz
 # 20161017: urtwn(4) was merged into rtwn(4)
 OLD_FILES+=usr/share/man/man4/urtwn.4.gz
 OLD_FILES+=usr/share/man/man4/urtwnfw.4.gz
 # 20161015: Remove GNU rcs
 OLD_FILES+=usr/bin/ci
 OLD_FILES+=usr/bin/co
 OLD_FILES+=usr/bin/merge
 OLD_FILES+=usr/bin/rcs
 OLD_FILES+=usr/bin/rcsclean
 OLD_FILES+=usr/bin/rcsdiff
 OLD_FILES+=usr/bin/rcsfreeze
 OLD_FILES+=usr/bin/rcsmerge
 OLD_FILES+=usr/bin/rlog
 OLD_FILES+=usr/share/doc/psd/13.rcs/paper.ascii.gz
 OLD_FILES+=usr/share/doc/psd/13.rcs/rcs_func.ascii.gz
 OLD_DIRS+=usr/share/doc/psd/13.rcs
 OLD_FILES+=usr/share/man/man1/ci.1.gz
 OLD_FILES+=usr/share/man/man1/co.1.gz
 OLD_FILES+=usr/share/man/man1/merge.1.gz
 OLD_FILES+=usr/share/man/man1/rcs.1.gz
 OLD_FILES+=usr/share/man/man1/rcsclean.1.gz
 OLD_FILES+=usr/share/man/man1/rcsdiff.1.gz
 OLD_FILES+=usr/share/man/man1/rcsfreeze.1.gz
 OLD_FILES+=usr/share/man/man1/rcsintro.1.gz
 OLD_FILES+=usr/share/man/man1/rcsmerge.1.gz
 OLD_FILES+=usr/share/man/man1/rlog.1.gz
 OLD_FILES+=usr/share/man/man5/rcsfile.5.gz
 # 20161010: remove link to removed m_getclr(9) macro
 OLD_FILES+=usr/share/man/man9/m_getclr.9.gz
 # 20161003: MK_ELFCOPY_AS_OBJCOPY option retired
 OLD_FILES+=usr/bin/elfcopy
 OLD_FILES+=usr/share/man/man1/elfcopy.1.gz
 # 20160906: libkqueue tests moved to /usr/tests/sys/kqueue/libkqueue
 OLD_FILES+=usr/tests/sys/kqueue/kqtest
 OLD_FILES+=usr/tests/sys/kqueue/kqueue_test
 # 20160903: idle page zeroing support removed
 OLD_FILES+=usr/share/man/man9/pmap_zero_idle.9.gz
 # 20160901: Remove digi(4)
 OLD_FILES+=usr/share/man/man4/digi.4.gz
 # 20160819: Remove ie(4)
 OLD_FILES+=usr/share/man/man4/i386/ie.4.gz
 # 20160819: Remove spic(4)
 OLD_FILES+=usr/share/man/man4/spic.4.gz
 # 20160819: Remove wl(4) and wlconfig(8)
 OLD_FILES+=usr/share/man/man4/i386/wl.4.gz
 OLD_FILES+=usr/sbin/wlconfig
 OLD_FILES+=usr/share/man/man8/i386/wlconfig.8.gz
 # 20160819: Remove si(4) and sicontrol(8)
 OLD_FILES+=usr/share/man/man4/si.4.gz
 OLD_FILES+=usr/sbin/sicontrol
 OLD_FILES+=usr/share/man/man8/sicontrol.8.gz
 # 20160819: Remove scd(4)
 OLD_FILES+=usr/share/man/man4/scd.4.gz
 # 20160815: Remove mcd(4)
 OLD_FILES+=usr/share/man/man4/mcd.4.gz
 # 20160805: lockmgr_waiters(9) removed
 OLD_FILES+=usr/share/man/man9/lockmgr_waiters.9.gz
 # 20160703: POSIXify locales with variants
 OLD_FILES+=usr/share/locale/zh_Hant_TW.UTF-8/LC_COLLATE
 OLD_FILES+=usr/share/locale/zh_Hant_TW.UTF-8/LC_CTYPE
 OLD_FILES+=usr/share/locale/zh_Hant_TW.UTF-8/LC_MESSAGES
 OLD_FILES+=usr/share/locale/zh_Hant_TW.UTF-8/LC_MONETARY
 OLD_FILES+=usr/share/locale/zh_Hant_TW.UTF-8/LC_NUMERIC
 OLD_FILES+=usr/share/locale/zh_Hant_TW.UTF-8/LC_TIME
 OLD_DIRS+=usr/share/locale/zh_Hant_TW.UTF-8
 OLD_FILES+=usr/share/locale/zh_Hant_TW.Big5/LC_COLLATE
 OLD_FILES+=usr/share/locale/zh_Hant_TW.Big5/LC_CTYPE
 OLD_FILES+=usr/share/locale/zh_Hant_TW.Big5/LC_MESSAGES
 OLD_FILES+=usr/share/locale/zh_Hant_TW.Big5/LC_MONETARY
 OLD_FILES+=usr/share/locale/zh_Hant_TW.Big5/LC_NUMERIC
 OLD_FILES+=usr/share/locale/zh_Hant_TW.Big5/LC_TIME
 OLD_DIRS+=usr/share/locale/zh_Hant_TW.Big5
 OLD_FILES+=usr/share/locale/zh_Hant_HK.UTF-8/LC_COLLATE
 OLD_FILES+=usr/share/locale/zh_Hant_HK.UTF-8/LC_CTYPE
 OLD_FILES+=usr/share/locale/zh_Hant_HK.UTF-8/LC_MESSAGES
 OLD_FILES+=usr/share/locale/zh_Hant_HK.UTF-8/LC_MONETARY
 OLD_FILES+=usr/share/locale/zh_Hant_HK.UTF-8/LC_NUMERIC
 OLD_FILES+=usr/share/locale/zh_Hant_HK.UTF-8/LC_TIME
 OLD_DIRS+=usr/share/locale/zh_Hant_HK.UTF-8
 OLD_FILES+=usr/share/locale/zh_Hans_CN.eucCN/LC_COLLATE
 OLD_FILES+=usr/share/locale/zh_Hans_CN.eucCN/LC_CTYPE
 OLD_FILES+=usr/share/locale/zh_Hans_CN.eucCN/LC_MESSAGES
 OLD_FILES+=usr/share/locale/zh_Hans_CN.eucCN/LC_MONETARY
 OLD_FILES+=usr/share/locale/zh_Hans_CN.eucCN/LC_NUMERIC
 OLD_FILES+=usr/share/locale/zh_Hans_CN.eucCN/LC_TIME
 OLD_DIRS+=usr/share/locale/zh_Hans_CN.eucCN
 OLD_FILES+=usr/share/locale/zh_Hans_CN.UTF-8/LC_COLLATE
 OLD_FILES+=usr/share/locale/zh_Hans_CN.UTF-8/LC_CTYPE
 OLD_FILES+=usr/share/locale/zh_Hans_CN.UTF-8/LC_MESSAGES
 OLD_FILES+=usr/share/locale/zh_Hans_CN.UTF-8/LC_MONETARY
 OLD_FILES+=usr/share/locale/zh_Hans_CN.UTF-8/LC_NUMERIC
 OLD_FILES+=usr/share/locale/zh_Hans_CN.UTF-8/LC_TIME
 OLD_DIRS+=usr/share/locale/zh_Hans_CN.UTF-8
 OLD_FILES+=usr/share/locale/zh_Hans_CN.GBK/LC_COLLATE
 OLD_FILES+=usr/share/locale/zh_Hans_CN.GBK/LC_CTYPE
 OLD_FILES+=usr/share/locale/zh_Hans_CN.GBK/LC_MESSAGES
 OLD_FILES+=usr/share/locale/zh_Hans_CN.GBK/LC_MONETARY
 OLD_FILES+=usr/share/locale/zh_Hans_CN.GBK/LC_NUMERIC
 OLD_FILES+=usr/share/locale/zh_Hans_CN.GBK/LC_TIME
 OLD_DIRS+=usr/share/locale/zh_Hans_CN.GBK
 OLD_FILES+=usr/share/locale/zh_Hans_CN.GB2312/LC_COLLATE
 OLD_FILES+=usr/share/locale/zh_Hans_CN.GB2312/LC_CTYPE
 OLD_FILES+=usr/share/locale/zh_Hans_CN.GB2312/LC_MESSAGES
 OLD_FILES+=usr/share/locale/zh_Hans_CN.GB2312/LC_MONETARY
 OLD_FILES+=usr/share/locale/zh_Hans_CN.GB2312/LC_NUMERIC
 OLD_FILES+=usr/share/locale/zh_Hans_CN.GB2312/LC_TIME
 OLD_DIRS+=usr/share/locale/zh_Hans_CN.GB2312
 OLD_FILES+=usr/share/locale/zh_Hans_CN.GB18030/LC_COLLATE
 OLD_FILES+=usr/share/locale/zh_Hans_CN.GB18030/LC_CTYPE
 OLD_FILES+=usr/share/locale/zh_Hans_CN.GB18030/LC_MESSAGES
 OLD_FILES+=usr/share/locale/zh_Hans_CN.GB18030/LC_MONETARY
 OLD_FILES+=usr/share/locale/zh_Hans_CN.GB18030/LC_NUMERIC
 OLD_FILES+=usr/share/locale/zh_Hans_CN.GB18030/LC_TIME
 OLD_DIRS+=usr/share/locale/zh_Hans_CN.GB18030
 OLD_FILES+=usr/share/locale/sr_Latn_RS.UTF-8/LC_COLLATE
 OLD_FILES+=usr/share/locale/sr_Latn_RS.UTF-8/LC_CTYPE
 OLD_FILES+=usr/share/locale/sr_Latn_RS.UTF-8/LC_MESSAGES
 OLD_FILES+=usr/share/locale/sr_Latn_RS.UTF-8/LC_MONETARY
 OLD_FILES+=usr/share/locale/sr_Latn_RS.UTF-8/LC_NUMERIC
 OLD_FILES+=usr/share/locale/sr_Latn_RS.UTF-8/LC_TIME
 OLD_DIRS+=usr/share/locale/sr_Latn_RS.UTF-8
 OLD_FILES+=usr/share/locale/sr_Latn_RS.ISO8859-2/LC_COLLATE
 OLD_FILES+=usr/share/locale/sr_Latn_RS.ISO8859-2/LC_CTYPE
 OLD_FILES+=usr/share/locale/sr_Latn_RS.ISO8859-2/LC_MESSAGES
 OLD_FILES+=usr/share/locale/sr_Latn_RS.ISO8859-2/LC_MONETARY
 OLD_FILES+=usr/share/locale/sr_Latn_RS.ISO8859-2/LC_NUMERIC
 OLD_FILES+=usr/share/locale/sr_Latn_RS.ISO8859-2/LC_TIME
 OLD_DIRS+=usr/share/locale/sr_Latn_RS.ISO8859-2
 OLD_FILES+=usr/share/locale/sr_Cyrl_RS.UTF-8/LC_COLLATE
 OLD_FILES+=usr/share/locale/sr_Cyrl_RS.UTF-8/LC_CTYPE
 OLD_FILES+=usr/share/locale/sr_Cyrl_RS.UTF-8/LC_MESSAGES
 OLD_FILES+=usr/share/locale/sr_Cyrl_RS.UTF-8/LC_MONETARY
 OLD_FILES+=usr/share/locale/sr_Cyrl_RS.UTF-8/LC_NUMERIC
 OLD_FILES+=usr/share/locale/sr_Cyrl_RS.UTF-8/LC_TIME
 OLD_DIRS+=usr/share/locale/sr_Cyrl_RS.UTF-8
 OLD_FILES+=usr/share/locale/sr_Cyrl_RS.ISO8859-5/LC_COLLATE
 OLD_FILES+=usr/share/locale/sr_Cyrl_RS.ISO8859-5/LC_CTYPE
 OLD_FILES+=usr/share/locale/sr_Cyrl_RS.ISO8859-5/LC_MESSAGES
 OLD_FILES+=usr/share/locale/sr_Cyrl_RS.ISO8859-5/LC_MONETARY
 OLD_FILES+=usr/share/locale/sr_Cyrl_RS.ISO8859-5/LC_NUMERIC
 OLD_FILES+=usr/share/locale/sr_Cyrl_RS.ISO8859-5/LC_TIME
 OLD_DIRS+=usr/share/locale/sr_Cyrl_RS.ISO8859-5
 OLD_FILES+=usr/share/locale/mn_Cyrl_MN.UTF-8/LC_COLLATE
 OLD_FILES+=usr/share/locale/mn_Cyrl_MN.UTF-8/LC_CTYPE
 OLD_FILES+=usr/share/locale/mn_Cyrl_MN.UTF-8/LC_MESSAGES
 OLD_FILES+=usr/share/locale/mn_Cyrl_MN.UTF-8/LC_MONETARY
 OLD_FILES+=usr/share/locale/mn_Cyrl_MN.UTF-8/LC_NUMERIC
 OLD_FILES+=usr/share/locale/mn_Cyrl_MN.UTF-8/LC_TIME
 OLD_DIRS+=usr/share/locale/mn_Cyrl_MN.UTF-8
 OLD_FILES+=usr/share/locale/kk_Cyrl_KZ.UTF-8/LC_COLLATE
 OLD_FILES+=usr/share/locale/kk_Cyrl_KZ.UTF-8/LC_CTYPE
 OLD_FILES+=usr/share/locale/kk_Cyrl_KZ.UTF-8/LC_MESSAGES
 OLD_FILES+=usr/share/locale/kk_Cyrl_KZ.UTF-8/LC_MONETARY
 OLD_FILES+=usr/share/locale/kk_Cyrl_KZ.UTF-8/LC_NUMERIC
 OLD_FILES+=usr/share/locale/kk_Cyrl_KZ.UTF-8/LC_TIME
 OLD_DIRS+=usr/share/locale/kk_Cyrl_KZ.UTF-8
 # 20160608: removed pam_verbose_error
 OLD_LIBS+=usr/lib/libpam.so.5
 OLD_LIBS+=usr/lib/pam_chroot.so.5
 OLD_LIBS+=usr/lib/pam_deny.so.5
 OLD_LIBS+=usr/lib/pam_echo.so.5
 OLD_LIBS+=usr/lib/pam_exec.so.5
 OLD_LIBS+=usr/lib/pam_ftpusers.so.5
 OLD_LIBS+=usr/lib/pam_group.so.5
 OLD_LIBS+=usr/lib/pam_guest.so.5
 OLD_LIBS+=usr/lib/pam_krb5.so.5
 OLD_LIBS+=usr/lib/pam_ksu.so.5
 OLD_LIBS+=usr/lib/pam_lastlog.so.5
 OLD_LIBS+=usr/lib/pam_login_access.so.5
 OLD_LIBS+=usr/lib/pam_nologin.so.5
 OLD_LIBS+=usr/lib/pam_opie.so.5
 OLD_LIBS+=usr/lib/pam_opieaccess.so.5
 OLD_LIBS+=usr/lib/pam_passwdqc.so.5
 OLD_LIBS+=usr/lib/pam_permit.so.5
 OLD_LIBS+=usr/lib/pam_radius.so.5
 OLD_LIBS+=usr/lib/pam_rhosts.so.5
 OLD_LIBS+=usr/lib/pam_rootok.so.5
 OLD_LIBS+=usr/lib/pam_securetty.so.5
 OLD_LIBS+=usr/lib/pam_self.so.5
 OLD_LIBS+=usr/lib/pam_ssh.so.5
 OLD_LIBS+=usr/lib/pam_tacplus.so.5
 OLD_LIBS+=usr/lib/pam_unix.so.5
 OLD_LIBS+=usr/lib32/libpam.so.5
 OLD_LIBS+=usr/lib32/pam_chroot.so.5
 OLD_LIBS+=usr/lib32/pam_deny.so.5
 OLD_LIBS+=usr/lib32/pam_echo.so.5
 OLD_LIBS+=usr/lib32/pam_exec.so.5
 OLD_LIBS+=usr/lib32/pam_ftpusers.so.5
 OLD_LIBS+=usr/lib32/pam_group.so.5
 OLD_LIBS+=usr/lib32/pam_guest.so.5
 OLD_LIBS+=usr/lib32/pam_krb5.so.5
 OLD_LIBS+=usr/lib32/pam_ksu.so.5
 OLD_LIBS+=usr/lib32/pam_lastlog.so.5
 OLD_LIBS+=usr/lib32/pam_login_access.so.5
 OLD_LIBS+=usr/lib32/pam_nologin.so.5
 OLD_LIBS+=usr/lib32/pam_opie.so.5
 OLD_LIBS+=usr/lib32/pam_opieaccess.so.5
 OLD_LIBS+=usr/lib32/pam_passwdqc.so.5
 OLD_LIBS+=usr/lib32/pam_permit.so.5
 OLD_LIBS+=usr/lib32/pam_radius.so.5
 OLD_LIBS+=usr/lib32/pam_rhosts.so.5
 OLD_LIBS+=usr/lib32/pam_rootok.so.5
 OLD_LIBS+=usr/lib32/pam_securetty.so.5
 OLD_LIBS+=usr/lib32/pam_self.so.5
 OLD_LIBS+=usr/lib32/pam_ssh.so.5
 OLD_LIBS+=usr/lib32/pam_tacplus.so.5
 OLD_LIBS+=usr/lib32/pam_unix.so.5
 # 20160523: remove extranous ALTQ files
 OLD_FILES+=usr/include/altq/altq_codel.h
 OLD_FILES+=usr/include/altq/altq_fairq.h
 # 20160519: remove DTrace Toolkit from base
 OLD_FILES+=usr/sbin/dtruss
 OLD_FILES+=usr/share/dtrace/toolkit/execsnoop
 OLD_FILES+=usr/share/dtrace/toolkit/hotkernel
 OLD_FILES+=usr/share/dtrace/toolkit/hotuser
 OLD_FILES+=usr/share/dtrace/toolkit/opensnoop
 OLD_FILES+=usr/share/dtrace/toolkit/procsystime
 OLD_DIRS+=usr/share/dtrace/toolkit
 OLD_FILES+=usr/share/man/man1/dtruss.1.gz
 # 20160519: stale MLINK removed
 OLD_FILES+=usr/share/man/man9/rman_await_resource.9.gz
 # 20160517: ReiserFS removed
 OLD_FILES+=usr/share/man/man5/reiserfs.5.gz
 # 20160504: tests rework
 OLD_FILES+=usr/tests/lib/libc/regex/data/README
 # 20160430: kvm_getfiles(3) removed from kvm(3)
 OLD_LIBS+=lib/libkvm.so.6
 OLD_LIBS+=usr/lib32/libkvm.so.6
 OLD_FILES+=usr/share/man/man3/kvm_getfiles.3.gz
 # 20160423: remove mroute6d
 OLD_FILES+=etc/rc.d/mroute6d
 # 20160419: rename units.lib -> definitions.units
 OLD_FILES+=usr/share/misc/units.lib
 # 20160419: remove Big5HKSCS locales
 OLD_FILES+=usr/share/locale/zh_HK.Big5HKSCS/LC_COLLATE
 OLD_FILES+=usr/share/locale/zh_HK.Big5HKSCS/LC_CTYPE
 OLD_FILES+=usr/share/locale/zh_HK.Big5HKSCS/LC_MESSAGES
 OLD_FILES+=usr/share/locale/zh_HK.Big5HKSCS/LC_MONETARY
 OLD_FILES+=usr/share/locale/zh_HK.Big5HKSCS/LC_NUMERIC
 OLD_FILES+=usr/share/locale/zh_HK.Big5HKSCS/LC_TIME
 OLD_DIRS+=usr/share/locale/zh_HK.Big5HKSCS
 OLD_FILES+=usr/share/locale/zh_Hant_HK.Big5HKSCS/LC_COLLATE
 OLD_FILES+=usr/share/locale/zh_Hant_HK.Big5HKSCS/LC_CTYPE
 OLD_FILES+=usr/share/locale/zh_Hant_HK.Big5HKSCS/LC_MESSAGES
 OLD_FILES+=usr/share/locale/zh_Hant_HK.Big5HKSCS/LC_MONETARY
 OLD_FILES+=usr/share/locale/zh_Hant_HK.Big5HKSCS/LC_NUMERIC
 OLD_FILES+=usr/share/locale/zh_Hant_HK.Big5HKSCS/LC_TIME
 OLD_DIRS+=usr/share/locale/zh_Hant_HK.Big5HKSCS
 # 20160317: rman_res_t size bump to uintmax_t
 OLD_LIBS+=usr/lib/libdevinfo.so.5
 OLD_LIBS+=usr/lib32/libdevinfo.so.5
 # 20160305: new clang import which bumps version from 3.7.1 to 3.8.0.
 OLD_FILES+=usr/bin/macho-dump
 OLD_FILES+=usr/bin/tblgen
 OLD_FILES+=usr/lib/clang/3.7.1/include/sanitizer/allocator_interface.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/sanitizer/asan_interface.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/sanitizer/common_interface_defs.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/sanitizer/coverage_interface.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/sanitizer/dfsan_interface.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/sanitizer/linux_syscall_hooks.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/sanitizer/lsan_interface.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/sanitizer/msan_interface.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/sanitizer/tsan_interface_atomic.h
 OLD_DIRS+=usr/lib/clang/3.7.1/include/sanitizer
 OLD_FILES+=usr/lib/clang/3.7.1/include/__stddef_max_align_t.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/__wmmintrin_aes.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/__wmmintrin_pclmul.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/adxintrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/altivec.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/ammintrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/arm_acle.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/arm_neon.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/avx2intrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/avx512bwintrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/avx512cdintrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/avx512dqintrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/avx512erintrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/avx512fintrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/avx512vlbwintrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/avx512vldqintrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/avx512vlintrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/avxintrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/bmi2intrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/bmiintrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/cpuid.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/cuda_builtin_vars.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/emmintrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/f16cintrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/fma4intrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/fmaintrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/fxsrintrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/htmintrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/htmxlintrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/ia32intrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/immintrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/lzcntintrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/mm3dnow.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/mm_malloc.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/mmintrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/module.modulemap
 OLD_FILES+=usr/lib/clang/3.7.1/include/nmmintrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/pmmintrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/popcntintrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/prfchwintrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/rdseedintrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/rtmintrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/s390intrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/shaintrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/smmintrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/tbmintrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/tmmintrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/vadefs.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/vecintrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/wmmintrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/x86intrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/xmmintrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/xopintrin.h
 OLD_FILES+=usr/lib/clang/3.7.1/include/xtestintrin.h
 OLD_DIRS+=usr/lib/clang/3.7.1/include
 OLD_FILES+=usr/lib/clang/3.7.1/lib/freebsd/libclang_rt.asan-i386.a
 OLD_FILES+=usr/lib/clang/3.7.1/lib/freebsd/libclang_rt.asan-preinit-i386.a
 OLD_FILES+=usr/lib/clang/3.7.1/lib/freebsd/libclang_rt.asan-preinit-x86_64.a
 OLD_FILES+=usr/lib/clang/3.7.1/lib/freebsd/libclang_rt.asan-x86_64.a
 OLD_FILES+=usr/lib/clang/3.7.1/lib/freebsd/libclang_rt.asan_cxx-i386.a
 OLD_FILES+=usr/lib/clang/3.7.1/lib/freebsd/libclang_rt.asan_cxx-x86_64.a
 OLD_FILES+=usr/lib/clang/3.7.1/lib/freebsd/libclang_rt.profile-arm.a
 OLD_FILES+=usr/lib/clang/3.7.1/lib/freebsd/libclang_rt.profile-i386.a
 OLD_FILES+=usr/lib/clang/3.7.1/lib/freebsd/libclang_rt.profile-x86_64.a
 OLD_FILES+=usr/lib/clang/3.7.1/lib/freebsd/libclang_rt.safestack-i386.a
 OLD_FILES+=usr/lib/clang/3.7.1/lib/freebsd/libclang_rt.safestack-x86_64.a
 OLD_FILES+=usr/lib/clang/3.7.1/lib/freebsd/libclang_rt.ubsan_standalone-i386.a
 OLD_FILES+=usr/lib/clang/3.7.1/lib/freebsd/libclang_rt.ubsan_standalone-x86_64.a
 OLD_FILES+=usr/lib/clang/3.7.1/lib/freebsd/libclang_rt.ubsan_standalone_cxx-i386.a
 OLD_FILES+=usr/lib/clang/3.7.1/lib/freebsd/libclang_rt.ubsan_standalone_cxx-x86_64.a
 OLD_DIRS+=usr/lib/clang/3.7.1/lib/freebsd
 OLD_DIRS+=usr/lib/clang/3.7.1/lib
 OLD_DIRS+=usr/lib/clang/3.7.1
 # 20160301: Remove taskqueue_enqueue_fast
 OLD_FILES+=usr/share/man/man9/taskqueue_enqueue_fast.9.gz
 # 20160225: Remove casperd and libcapsicum.
 OLD_FILES+=sbin/casperd
 OLD_FILES+=etc/rc.d/casperd
 OLD_FILES+=usr/share/man/man8/casperd.8.gz
 OLD_FILES+=usr/include/libcapsicum.h
 OLD_FILES+=usr/include/libcapsicum_service.h
 OLD_FILES+=usr/include/libcapsicum.h
 OLD_FILES+=usr/share/man/man3/libcapsicum.3.gz
 OLD_FILES+=usr/include/libcapsicum_dns.h
 OLD_FILES+=usr/include/libcapsicum_grp.h
 OLD_FILES+=usr/include/libcapsicum_impl.h
 OLD_FILES+=usr/include/libcapsicum_pwd.h
 OLD_FILES+=usr/include/libcapsicum_random.h
 OLD_FILES+=usr/include/libcapsicum_sysctl.h
 OLD_FILES+=libexec/casper/dns
 OLD_FILES+=libexec/casper/grp
 OLD_FILES+=libexec/casper/pwd
 OLD_FILES+=libexec/casper/random
 OLD_FILES+=libexec/casper/sysctl
 OLD_FILES+=libexec/casper/.debug/random.debug
 OLD_FILES+=libexec/casper/.debug/dns.debug
 OLD_FILES+=libexec/casper/.debug/sysctl.debug
 OLD_FILES+=libexec/casper/.debug/pwd.debug
 OLD_FILES+=libexec/casper/.debug/grp.debug
 OLD_DIRS+=libexec/casper/.debug
 OLD_DIRS+=libexec/casper
 OLD_FILES+=usr/lib/libcapsicum.a
 OLD_FILES+=usr/lib/libcapsicum.so
 OLD_LIBS+=lib/libcapsicum.so.0
 OLD_FILES+=usr/lib/libcapsicum_p.a
 OLD_FILES+=usr/lib32/libcapsicum.a
 OLD_FILES+=usr/lib32/libcapsicum.so
 OLD_LIBS+=usr/lib32/libcapsicum.so.0
 OLD_FILES+=usr/lib32/libcapsicum_p.a
 # 20160223: functionality from mkulzma(1) merged into mkuzip(1)
 OLD_FILES+=usr/bin/mkulzma
 OLD_FILES+=usr/share/man/man4/geom_uncompress.4.gz
 OLD_FILES+=usr/share/man/man8/mkulzma.8.gz
 # 20160211: Remove obsolete unbound-control-setup
 OLD_FILES+=usr/sbin/unbound-control-setup
 # 20160121: cc.h moved
 OLD_FILES+=usr/include/netinet/cc.h
 # 20160116: Update mandoc to cvs snapshot 20160116
 OLD_FILES+=usr/share/mdocml/example.style.css
 OLD_FILES+=usr/share/mdocml/style.css
 OLD_DIRS+=usr/share/mdocml
 # 20160114: SA-16:06.snmpd
 OLD_FILES+=usr/share/examples/etc/snmpd.config
 # 20160107: GNU ld installed as ld.bfd and linked as ld
 OLD_FILES+=usr/lib/debug/usr/bin/ld.debug
 # 20151225: new clang import which bumps version from 3.7.0 to 3.7.1.
 OLD_FILES+=usr/lib/clang/3.7.0/include/sanitizer/allocator_interface.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/sanitizer/asan_interface.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/sanitizer/common_interface_defs.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/sanitizer/coverage_interface.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/sanitizer/dfsan_interface.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/sanitizer/linux_syscall_hooks.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/sanitizer/lsan_interface.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/sanitizer/msan_interface.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/sanitizer/tsan_interface_atomic.h
 OLD_DIRS+=usr/lib/clang/3.7.0/include/sanitizer
 OLD_FILES+=usr/lib/clang/3.7.0/include/__stddef_max_align_t.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/__wmmintrin_aes.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/__wmmintrin_pclmul.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/adxintrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/altivec.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/ammintrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/arm_acle.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/arm_neon.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/avx2intrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/avx512bwintrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/avx512cdintrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/avx512dqintrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/avx512erintrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/avx512fintrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/avx512vlbwintrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/avx512vldqintrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/avx512vlintrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/avxintrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/bmi2intrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/bmiintrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/cpuid.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/cuda_builtin_vars.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/emmintrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/f16cintrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/fma4intrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/fmaintrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/fxsrintrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/htmintrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/htmxlintrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/ia32intrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/immintrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/lzcntintrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/mm3dnow.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/mm_malloc.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/mmintrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/module.modulemap
 OLD_FILES+=usr/lib/clang/3.7.0/include/nmmintrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/pmmintrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/popcntintrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/prfchwintrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/rdseedintrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/rtmintrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/s390intrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/shaintrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/smmintrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/tbmintrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/tmmintrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/vadefs.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/vecintrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/wmmintrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/x86intrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/xmmintrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/xopintrin.h
 OLD_FILES+=usr/lib/clang/3.7.0/include/xtestintrin.h
 OLD_DIRS+=usr/lib/clang/3.7.0/include
 OLD_FILES+=usr/lib/clang/3.7.0/lib/freebsd/libclang_rt.asan-i386.a
 OLD_FILES+=usr/lib/clang/3.7.0/lib/freebsd/libclang_rt.asan-preinit-i386.a
 OLD_FILES+=usr/lib/clang/3.7.0/lib/freebsd/libclang_rt.asan-preinit-x86_64.a
 OLD_FILES+=usr/lib/clang/3.7.0/lib/freebsd/libclang_rt.asan-x86_64.a
 OLD_FILES+=usr/lib/clang/3.7.0/lib/freebsd/libclang_rt.asan_cxx-i386.a
 OLD_FILES+=usr/lib/clang/3.7.0/lib/freebsd/libclang_rt.asan_cxx-x86_64.a
 OLD_FILES+=usr/lib/clang/3.7.0/lib/freebsd/libclang_rt.profile-arm.a
 OLD_FILES+=usr/lib/clang/3.7.0/lib/freebsd/libclang_rt.profile-i386.a
 OLD_FILES+=usr/lib/clang/3.7.0/lib/freebsd/libclang_rt.profile-x86_64.a
 OLD_FILES+=usr/lib/clang/3.7.0/lib/freebsd/libclang_rt.safestack-i386.a
 OLD_FILES+=usr/lib/clang/3.7.0/lib/freebsd/libclang_rt.safestack-x86_64.a
 OLD_FILES+=usr/lib/clang/3.7.0/lib/freebsd/libclang_rt.ubsan_standalone-i386.a
 OLD_FILES+=usr/lib/clang/3.7.0/lib/freebsd/libclang_rt.ubsan_standalone-x86_64.a
 OLD_FILES+=usr/lib/clang/3.7.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-i386.a
 OLD_FILES+=usr/lib/clang/3.7.0/lib/freebsd/libclang_rt.ubsan_standalone_cxx-x86_64.a
 OLD_DIRS+=usr/lib/clang/3.7.0/lib/freebsd
 OLD_DIRS+=usr/lib/clang/3.7.0/lib
 OLD_DIRS+=usr/lib/clang/3.7.0
 # 20151130: libelf moved from /usr/lib to /lib (libkvm dependency in r291406)
 OLD_LIBS+=usr/lib/libelf.so.2
 # 20151115: Fox bad upgrade scheme
 OLD_FILES+=usr/share/locale/zh_CN.GB18030/zh_Hans_CN.GB18030
 OLD_FILES+=usr/share/locale/zh_CN.GB2312/zh_Hans_CN.GB2312
 OLD_FILES+=usr/share/locale/zh_CN.GBK/zh_Hans_CN.GBK
 OLD_FILES+=usr/share/locale/zh_CN.UTF-8/zh_Hans_CN.UTF-8
 OLD_FILES+=usr/share/locale/zh_CN.eucCN/zh_Hans_CN.eucCN
 OLD_FILES+=usr/share/locale/zh_TW.Big5/zh_Hant_TW.Big5
 OLD_FILES+=usr/share/locale/zh_TW.UTF-8/zh_Hant_TW.UTF-8
 # 20151107: String collation improvements
 OLD_FILES+=usr/share/locale/UTF-8/LC_CTYPE
 OLD_DIRS+=usr/share/locale/UTF-8
 OLD_FILES+=usr/share/locale/kk_KZ.PT154/LC_COLLATE
 OLD_FILES+=usr/share/locale/kk_KZ.PT154/LC_CTYPE
 OLD_FILES+=usr/share/locale/kk_KZ.PT154/LC_MESSAGES
 OLD_FILES+=usr/share/locale/kk_KZ.PT154/LC_MONETARY
 OLD_FILES+=usr/share/locale/kk_KZ.PT154/LC_NUMERIC
 OLD_FILES+=usr/share/locale/kk_KZ.PT154/LC_TIME
 OLD_DIRS+=usr/share/locale/kk_KZ.PT154/
 OLD_FILES+=usr/share/locale/la_LN.ISO8859-1/LC_COLLATE
 OLD_FILES+=usr/share/locale/la_LN.ISO8859-1/LC_CTYPE
 OLD_FILES+=usr/share/locale/la_LN.ISO8859-1/LC_TIME
 OLD_DIRS+=usr/share/locale/la_LN.ISO8859-1
 OLD_FILES+=usr/share/locale/la_LN.ISO8859-13/LC_COLLATE
 OLD_FILES+=usr/share/locale/la_LN.ISO8859-13/LC_CTYPE
 OLD_DIRS+=usr/share/locale/la_LN.ISO8859-13
 OLD_FILES+=usr/share/locale/la_LN.ISO8859-15/LC_COLLATE
 OLD_FILES+=usr/share/locale/la_LN.ISO8859-15/LC_CTYPE
 OLD_FILES+=usr/share/locale/la_LN.ISO8859-15/LC_TIME
 OLD_DIRS+=usr/share/locale/la_LN.ISO8859-15
 OLD_FILES+=usr/share/locale/la_LN.ISO8859-2/LC_COLLATE
 OLD_FILES+=usr/share/locale/la_LN.ISO8859-2/LC_CTYPE
 OLD_FILES+=usr/share/locale/la_LN.ISO8859-2/LC_TIME
 OLD_DIRS+=usr/share/locale/la_LN.ISO8859-2
 OLD_FILES+=usr/share/locale/la_LN.ISO8859-4/LC_COLLATE
 OLD_FILES+=usr/share/locale/la_LN.ISO8859-4/LC_CTYPE
 OLD_FILES+=usr/share/locale/la_LN.ISO8859-4/LC_TIME
 OLD_DIRS+=usr/share/locale/la_LN.ISO8859-4
 OLD_FILES+=usr/share/locale/la_LN.US-ASCII/LC_COLLATE
 OLD_FILES+=usr/share/locale/la_LN.US-ASCII/LC_CTYPE
 OLD_FILES+=usr/share/locale/la_LN.US-ASCII/LC_TIME
 OLD_DIRS+=usr/share/locale/la_LN.US-ASCII
 OLD_FILES+=usr/share/locale/lt_LT.ISO8859-4/LC_MESSAGES
 OLD_FILES+=usr/share/locale/lt_LT.ISO8859-4/LC_TIME
 OLD_FILES+=usr/share/locale/lt_LT.ISO8859-4/LC_COLLATE
 OLD_FILES+=usr/share/locale/lt_LT.ISO8859-4/LC_MONETARY
 OLD_FILES+=usr/share/locale/lt_LT.ISO8859-4/LC_CTYPE
 OLD_FILES+=usr/share/locale/lt_LT.ISO8859-4/LC_NUMERIC
 OLD_DIRS+=usr/share/locale/lt_LT.ISO8859-4
 OLD_FILES+=usr/share/locale/no_NO.ISO8859-1/LC_COLLATE
 OLD_FILES+=usr/share/locale/no_NO.ISO8859-1/LC_CTYPE
 OLD_FILES+=usr/share/locale/no_NO.ISO8859-1/LC_MESSAGES
 OLD_FILES+=usr/share/locale/no_NO.ISO8859-1/LC_MONETARY
 OLD_FILES+=usr/share/locale/no_NO.ISO8859-1/LC_NUMERIC
 OLD_FILES+=usr/share/locale/no_NO.ISO8859-1/LC_TIME
 OLD_DIRS+=usr/share/locale/no_NO.ISO8859-1
 OLD_FILES+=usr/share/locale/no_NO.ISO8859-15/LC_COLLATE
 OLD_FILES+=usr/share/locale/no_NO.ISO8859-15/LC_CTYPE
 OLD_FILES+=usr/share/locale/no_NO.ISO8859-15/LC_MESSAGES
 OLD_FILES+=usr/share/locale/no_NO.ISO8859-15/LC_MONETARY
 OLD_FILES+=usr/share/locale/no_NO.ISO8859-15/LC_NUMERIC
 OLD_FILES+=usr/share/locale/no_NO.ISO8859-15/LC_TIME
 OLD_DIRS+=usr/share/locale/no_NO.ISO8859-15
 OLD_FILES+=usr/share/locale/no_NO.UTF-8/LC_COLLATE
 OLD_FILES+=usr/share/locale/no_NO.UTF-8/LC_CTYPE
 OLD_FILES+=usr/share/locale/no_NO.UTF-8/LC_MESSAGES
 OLD_FILES+=usr/share/locale/no_NO.UTF-8/LC_MONETARY
 OLD_FILES+=usr/share/locale/no_NO.UTF-8/LC_NUMERIC
 OLD_FILES+=usr/share/locale/no_NO.UTF-8/LC_TIME
 OLD_DIRS+=usr/share/locale/no_NO.UTF-8
 OLD_FILES+=usr/share/locale/sr_YU.ISO8859-2/LC_COLLATE
 OLD_FILES+=usr/share/locale/sr_YU.ISO8859-2/LC_TIME
 OLD_FILES+=usr/share/locale/sr_YU.ISO8859-2/LC_CTYPE
 OLD_FILES+=usr/share/locale/sr_YU.ISO8859-2/LC_MESSAGES
 OLD_FILES+=usr/share/locale/sr_YU.ISO8859-2/LC_NUMERIC
 OLD_FILES+=usr/share/locale/sr_YU.ISO8859-2/LC_MONETARY
 OLD_DIRS+=usr/share/locale/sr_YU.ISO8859-2
 OLD_FILES+=usr/share/locale/sr_YU.ISO8859-5/LC_COLLATE
 OLD_FILES+=usr/share/locale/sr_YU.ISO8859-5/LC_MONETARY
 OLD_FILES+=usr/share/locale/sr_YU.ISO8859-5/LC_NUMERIC
 OLD_FILES+=usr/share/locale/sr_YU.ISO8859-5/LC_CTYPE
 OLD_FILES+=usr/share/locale/sr_YU.ISO8859-5/LC_TIME
 OLD_FILES+=usr/share/locale/sr_YU.ISO8859-5/LC_MESSAGES
 OLD_DIRS+=usr/share/locale/sr_YU.ISO8859-5
 OLD_FILES+=usr/share/locale/sr_YU.UTF-8/LC_COLLATE
 OLD_FILES+=usr/share/locale/sr_YU.UTF-8/LC_MONETARY
 OLD_FILES+=usr/share/locale/sr_YU.UTF-8/LC_CTYPE
 OLD_FILES+=usr/share/locale/sr_YU.UTF-8/LC_TIME
 OLD_FILES+=usr/share/locale/sr_YU.UTF-8/LC_NUMERIC
 OLD_FILES+=usr/share/locale/sr_YU.UTF-8/LC_MESSAGES
 OLD_DIRS+=usr/share/locale/sr_YU.UTF-8
 # 20151101: added missing _test suffix on multiple tests in lib/libc
 OLD_FILES+=usr/tests/lib/libc/c063/faccessat
 OLD_FILES+=usr/tests/lib/libc/c063/fchmodat
 OLD_FILES+=usr/tests/lib/libc/c063/fchownat
 OLD_FILES+=usr/tests/lib/libc/c063/fexecve
 OLD_FILES+=usr/tests/lib/libc/c063/fstatat
 OLD_FILES+=usr/tests/lib/libc/c063/linkat
 OLD_FILES+=usr/tests/lib/libc/c063/mkdirat
 OLD_FILES+=usr/tests/lib/libc/c063/mkfifoat
 OLD_FILES+=usr/tests/lib/libc/c063/mknodat
 OLD_FILES+=usr/tests/lib/libc/c063/openat
 OLD_FILES+=usr/tests/lib/libc/c063/readlinkat
 OLD_FILES+=usr/tests/lib/libc/c063/renameat
 OLD_FILES+=usr/tests/lib/libc/c063/symlinkat
 OLD_FILES+=usr/tests/lib/libc/c063/unlinkat
 OLD_FILES+=usr/tests/lib/libc/c063/utimensat
 OLD_FILES+=usr/tests/lib/libc/string/memchr
 OLD_FILES+=usr/tests/lib/libc/string/memcpy
 OLD_FILES+=usr/tests/lib/libc/string/memmem
 OLD_FILES+=usr/tests/lib/libc/string/memset
 OLD_FILES+=usr/tests/lib/libc/string/strcat
 OLD_FILES+=usr/tests/lib/libc/string/strchr
 OLD_FILES+=usr/tests/lib/libc/string/strcmp
 OLD_FILES+=usr/tests/lib/libc/string/strcpy
 OLD_FILES+=usr/tests/lib/libc/string/strcspn
 OLD_FILES+=usr/tests/lib/libc/string/strerror
 OLD_FILES+=usr/tests/lib/libc/string/strlen
 OLD_FILES+=usr/tests/lib/libc/string/strpbrk
 OLD_FILES+=usr/tests/lib/libc/string/strrchr
 OLD_FILES+=usr/tests/lib/libc/string/strspn
 OLD_FILES+=usr/tests/lib/libc/string/swab
 # 20151101: 430.status-rwho was renamed to 430.status-uptime
 OLD_FILES+=etc/periodic/daily/430.status-rwho
 # 20151030: OpenSSL 1.0.2d import
 OLD_FILES+=usr/share/openssl/man/man3/CMS_set1_signer_certs.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/EVP_PKEY_ctrl.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/EVP_PKEY_ctrl_str.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/d2i_509_CRL_fp.3.gz
 OLD_LIBS+=lib/libcrypto.so.7
 OLD_LIBS+=usr/lib/libssl.so.7
 OLD_LIBS+=usr/lib32/libcrypto.so.7
 OLD_LIBS+=usr/lib32/libssl.so.7
 # 20151029: LinuxKPI moved to sys/compat/linuxkpi
 OLD_FILES+=usr/include/dev/usb/usb_compat_linux.h
 # 20151015: test symbols moved to /usr/lib/debug
 OLD_DIRS+=usr/tests/lib/atf/libatf-c++/.debug
 OLD_FILES+=usr/tests/lib/atf/libatf-c++/.debug/atf_c++_test.debug
 OLD_FILES+=usr/tests/lib/atf/libatf-c++/.debug/build_test.debug
 OLD_FILES+=usr/tests/lib/atf/libatf-c++/.debug/check_test.debug
 OLD_FILES+=usr/tests/lib/atf/libatf-c++/.debug/config_test.debug
 OLD_FILES+=usr/tests/lib/atf/libatf-c++/.debug/macros_test.debug
 OLD_FILES+=usr/tests/lib/atf/libatf-c++/.debug/tests_test.debug
 OLD_FILES+=usr/tests/lib/atf/libatf-c++/.debug/utils_test.debug
 OLD_DIRS+=usr/tests/lib/atf/libatf-c++/detail/.debug
 OLD_FILES+=usr/tests/lib/atf/libatf-c++/detail/.debug/application_test.debug
 OLD_FILES+=usr/tests/lib/atf/libatf-c++/detail/.debug/env_test.debug
 OLD_FILES+=usr/tests/lib/atf/libatf-c++/detail/.debug/exceptions_test.debug
 OLD_FILES+=usr/tests/lib/atf/libatf-c++/detail/.debug/fs_test.debug
 OLD_FILES+=usr/tests/lib/atf/libatf-c++/detail/.debug/process_test.debug
 OLD_FILES+=usr/tests/lib/atf/libatf-c++/detail/.debug/sanity_test.debug
 OLD_FILES+=usr/tests/lib/atf/libatf-c++/detail/.debug/text_test.debug
 OLD_FILES+=usr/tests/lib/atf/libatf-c++/detail/.debug/version_helper.debug
 OLD_DIRS+=usr/tests/lib/atf/libatf-c/.debug
 OLD_FILES+=usr/tests/lib/atf/libatf-c/.debug/atf_c_test.debug
 OLD_FILES+=usr/tests/lib/atf/libatf-c/.debug/build_test.debug
 OLD_FILES+=usr/tests/lib/atf/libatf-c/.debug/check_test.debug
 OLD_FILES+=usr/tests/lib/atf/libatf-c/.debug/config_test.debug
 OLD_FILES+=usr/tests/lib/atf/libatf-c/.debug/error_test.debug
 OLD_FILES+=usr/tests/lib/atf/libatf-c/.debug/macros_test.debug
 OLD_FILES+=usr/tests/lib/atf/libatf-c/.debug/tc_test.debug
 OLD_FILES+=usr/tests/lib/atf/libatf-c/.debug/tp_test.debug
 OLD_FILES+=usr/tests/lib/atf/libatf-c/.debug/utils_test.debug
 OLD_DIRS+=usr/tests/lib/atf/libatf-c/detail/.debug
 OLD_FILES+=usr/tests/lib/atf/libatf-c/detail/.debug/dynstr_test.debug
 OLD_FILES+=usr/tests/lib/atf/libatf-c/detail/.debug/env_test.debug
 OLD_FILES+=usr/tests/lib/atf/libatf-c/detail/.debug/fs_test.debug
 OLD_FILES+=usr/tests/lib/atf/libatf-c/detail/.debug/list_test.debug
 OLD_FILES+=usr/tests/lib/atf/libatf-c/detail/.debug/map_test.debug
 OLD_FILES+=usr/tests/lib/atf/libatf-c/detail/.debug/process_helpers.debug
 OLD_FILES+=usr/tests/lib/atf/libatf-c/detail/.debug/process_test.debug
 OLD_FILES+=usr/tests/lib/atf/libatf-c/detail/.debug/sanity_test.debug
 OLD_FILES+=usr/tests/lib/atf/libatf-c/detail/.debug/text_test.debug
 OLD_FILES+=usr/tests/lib/atf/libatf-c/detail/.debug/user_test.debug
 OLD_FILES+=usr/tests/lib/atf/libatf-c/detail/.debug/version_helper.debug
 OLD_DIRS+=usr/tests/lib/atf/test-programs/.debug
 OLD_FILES+=usr/tests/lib/atf/test-programs/.debug/c_helpers.debug
 OLD_FILES+=usr/tests/lib/atf/test-programs/.debug/cpp_helpers.debug
 OLD_DIRS+=usr/tests/lib/libc/c063/.debug
 OLD_FILES+=usr/tests/lib/libc/c063/.debug/faccessat.debug
 OLD_FILES+=usr/tests/lib/libc/c063/.debug/fchmodat.debug
 OLD_FILES+=usr/tests/lib/libc/c063/.debug/fchownat.debug
 OLD_FILES+=usr/tests/lib/libc/c063/.debug/fexecve.debug
 OLD_FILES+=usr/tests/lib/libc/c063/.debug/fstatat.debug
 OLD_FILES+=usr/tests/lib/libc/c063/.debug/linkat.debug
 OLD_FILES+=usr/tests/lib/libc/c063/.debug/mkdirat.debug
 OLD_FILES+=usr/tests/lib/libc/c063/.debug/mkfifoat.debug
 OLD_FILES+=usr/tests/lib/libc/c063/.debug/mknodat.debug
 OLD_FILES+=usr/tests/lib/libc/c063/.debug/openat.debug
 OLD_FILES+=usr/tests/lib/libc/c063/.debug/readlinkat.debug
 OLD_FILES+=usr/tests/lib/libc/c063/.debug/renameat.debug
 OLD_FILES+=usr/tests/lib/libc/c063/.debug/symlinkat.debug
 OLD_FILES+=usr/tests/lib/libc/c063/.debug/unlinkat.debug
 OLD_FILES+=usr/tests/lib/libc/c063/.debug/utimensat.debug
 OLD_DIRS+=usr/tests/lib/libc/db/.debug
 OLD_FILES+=usr/tests/lib/libc/db/.debug/h_db.debug
 OLD_DIRS+=usr/tests/lib/libc/gen/.debug
 OLD_FILES+=usr/tests/lib/libc/gen/.debug/alarm_test.debug
 OLD_FILES+=usr/tests/lib/libc/gen/.debug/arc4random_test.debug
 OLD_FILES+=usr/tests/lib/libc/gen/.debug/assert_test.debug
 OLD_FILES+=usr/tests/lib/libc/gen/.debug/basedirname_test.debug
 OLD_FILES+=usr/tests/lib/libc/gen/.debug/dir_test.debug
 OLD_FILES+=usr/tests/lib/libc/gen/.debug/floatunditf_test.debug
 OLD_FILES+=usr/tests/lib/libc/gen/.debug/fnmatch_test.debug
 OLD_FILES+=usr/tests/lib/libc/gen/.debug/fpclassify2_test.debug
 OLD_FILES+=usr/tests/lib/libc/gen/.debug/fpclassify_test.debug
 OLD_FILES+=usr/tests/lib/libc/gen/.debug/fpsetmask_test.debug
 OLD_FILES+=usr/tests/lib/libc/gen/.debug/fpsetround_test.debug
 OLD_FILES+=usr/tests/lib/libc/gen/.debug/ftok_test.debug
 OLD_FILES+=usr/tests/lib/libc/gen/.debug/getcwd_test.debug
 OLD_FILES+=usr/tests/lib/libc/gen/.debug/getgrent_test.debug
 OLD_FILES+=usr/tests/lib/libc/gen/.debug/glob_test.debug
 OLD_FILES+=usr/tests/lib/libc/gen/.debug/humanize_number_test.debug
 OLD_FILES+=usr/tests/lib/libc/gen/.debug/isnan_test.debug
 OLD_FILES+=usr/tests/lib/libc/gen/.debug/nice_test.debug
 OLD_FILES+=usr/tests/lib/libc/gen/.debug/pause_test.debug
 OLD_FILES+=usr/tests/lib/libc/gen/.debug/raise_test.debug
 OLD_FILES+=usr/tests/lib/libc/gen/.debug/realpath_test.debug
 OLD_FILES+=usr/tests/lib/libc/gen/.debug/setdomainname_test.debug
 OLD_FILES+=usr/tests/lib/libc/gen/.debug/sethostname_test.debug
 OLD_FILES+=usr/tests/lib/libc/gen/.debug/sleep_test.debug
 OLD_FILES+=usr/tests/lib/libc/gen/.debug/syslog_test.debug
 OLD_FILES+=usr/tests/lib/libc/gen/.debug/time_test.debug
 OLD_FILES+=usr/tests/lib/libc/gen/.debug/ttyname_test.debug
 OLD_FILES+=usr/tests/lib/libc/gen/.debug/vis_test.debug
 OLD_DIRS+=usr/tests/lib/libc/gen/execve/.debug
 OLD_FILES+=usr/tests/lib/libc/gen/execve/.debug/execve_test.debug
 OLD_DIRS+=usr/tests/lib/libc/gen/posix_spawn/.debug
 OLD_FILES+=usr/tests/lib/libc/gen/posix_spawn/.debug/fileactions_test.debug
 OLD_FILES+=usr/tests/lib/libc/gen/posix_spawn/.debug/h_fileactions.debug
 OLD_FILES+=usr/tests/lib/libc/gen/posix_spawn/.debug/h_spawn.debug
 OLD_FILES+=usr/tests/lib/libc/gen/posix_spawn/.debug/h_spawnattr.debug
 OLD_FILES+=usr/tests/lib/libc/gen/posix_spawn/.debug/spawn_test.debug
 OLD_FILES+=usr/tests/lib/libc/gen/posix_spawn/.debug/spawnattr_test.debug
 OLD_DIRS+=usr/tests/lib/libc/hash/.debug
 OLD_FILES+=usr/tests/lib/libc/hash/.debug/h_hash.debug
 OLD_FILES+=usr/tests/lib/libc/hash/.debug/sha2_test.debug
 OLD_DIRS+=usr/tests/lib/libc/inet/.debug
 OLD_FILES+=usr/tests/lib/libc/inet/.debug/inet_network_test.debug
 OLD_DIRS+=usr/tests/lib/libc/locale/.debug
 OLD_FILES+=usr/tests/lib/libc/locale/.debug/io_test.debug
 OLD_FILES+=usr/tests/lib/libc/locale/.debug/mbrtowc_test.debug
 OLD_FILES+=usr/tests/lib/libc/locale/.debug/mbsnrtowcs_test.debug
 OLD_FILES+=usr/tests/lib/libc/locale/.debug/mbstowcs_test.debug
 OLD_FILES+=usr/tests/lib/libc/locale/.debug/mbtowc_test.debug
 OLD_FILES+=usr/tests/lib/libc/locale/.debug/wcscspn_test.debug
 OLD_FILES+=usr/tests/lib/libc/locale/.debug/wcspbrk_test.debug
 OLD_FILES+=usr/tests/lib/libc/locale/.debug/wcsspn_test.debug
 OLD_FILES+=usr/tests/lib/libc/locale/.debug/wcstod_test.debug
 OLD_FILES+=usr/tests/lib/libc/locale/.debug/wctomb_test.debug
 OLD_DIRS+=usr/tests/lib/libc/net/.debug
 OLD_FILES+=usr/tests/lib/libc/net/.debug/ether_aton_test.debug
 OLD_FILES+=usr/tests/lib/libc/net/.debug/getprotoent_test.debug
 OLD_FILES+=usr/tests/lib/libc/net/.debug/h_dns_server.debug
 OLD_FILES+=usr/tests/lib/libc/net/.debug/h_nsd_recurse.debug
 OLD_FILES+=usr/tests/lib/libc/net/.debug/h_protoent.debug
 OLD_FILES+=usr/tests/lib/libc/net/.debug/h_servent.debug
 OLD_DIRS+=usr/tests/lib/libc/regex/.debug
 OLD_FILES+=usr/tests/lib/libc/regex/.debug/exhaust_test.debug
 OLD_FILES+=usr/tests/lib/libc/regex/.debug/h_regex.debug
 OLD_FILES+=usr/tests/lib/libc/regex/.debug/regex_att_test.debug
 OLD_DIRS+=usr/tests/lib/libc/ssp/.debug
 OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_fgets.debug
 OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_getcwd.debug
 OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_gets.debug
 OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_memcpy.debug
 OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_memmove.debug
 OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_memset.debug
 OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_raw.debug
 OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_read.debug
 OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_readlink.debug
 OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_snprintf.debug
 OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_sprintf.debug
 OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_stpcpy.debug
 OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_stpncpy.debug
 OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_strcat.debug
 OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_strcpy.debug
 OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_strncat.debug
 OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_strncpy.debug
 OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_vsnprintf.debug
 OLD_FILES+=usr/tests/lib/libc/ssp/.debug/h_vsprintf.debug
 OLD_DIRS+=usr/tests/lib/libc/stdio/.debug
 OLD_FILES+=usr/tests/lib/libc/stdio/.debug/clearerr_test.debug
 OLD_FILES+=usr/tests/lib/libc/stdio/.debug/fflush_test.debug
 OLD_FILES+=usr/tests/lib/libc/stdio/.debug/fmemopen2_test.debug
 OLD_FILES+=usr/tests/lib/libc/stdio/.debug/fmemopen_test.debug
 OLD_FILES+=usr/tests/lib/libc/stdio/.debug/fopen_test.debug
 OLD_FILES+=usr/tests/lib/libc/stdio/.debug/fputc_test.debug
 OLD_FILES+=usr/tests/lib/libc/stdio/.debug/mktemp_test.debug
 OLD_FILES+=usr/tests/lib/libc/stdio/.debug/popen_test.debug
 OLD_FILES+=usr/tests/lib/libc/stdio/.debug/printf_test.debug
 OLD_FILES+=usr/tests/lib/libc/stdio/.debug/scanf_test.debug
 OLD_DIRS+=usr/tests/lib/libc/stdlib/.debug
 OLD_FILES+=usr/tests/lib/libc/stdlib/.debug/abs_test.debug
 OLD_FILES+=usr/tests/lib/libc/stdlib/.debug/atoi_test.debug
 OLD_FILES+=usr/tests/lib/libc/stdlib/.debug/div_test.debug
 OLD_FILES+=usr/tests/lib/libc/stdlib/.debug/exit_test.debug
 OLD_FILES+=usr/tests/lib/libc/stdlib/.debug/getenv_test.debug
 OLD_FILES+=usr/tests/lib/libc/stdlib/.debug/h_getopt.debug
 OLD_FILES+=usr/tests/lib/libc/stdlib/.debug/h_getopt_long.debug
 OLD_FILES+=usr/tests/lib/libc/stdlib/.debug/hsearch_test.debug
 OLD_FILES+=usr/tests/lib/libc/stdlib/.debug/posix_memalign_test.debug
 OLD_FILES+=usr/tests/lib/libc/stdlib/.debug/random_test.debug
 OLD_FILES+=usr/tests/lib/libc/stdlib/.debug/strtod_test.debug
 OLD_FILES+=usr/tests/lib/libc/stdlib/.debug/strtol_test.debug
 OLD_FILES+=usr/tests/lib/libc/stdlib/.debug/system_test.debug
 OLD_DIRS+=usr/tests/lib/libc/string/.debug
 OLD_FILES+=usr/tests/lib/libc/string/.debug/memchr.debug
 OLD_FILES+=usr/tests/lib/libc/string/.debug/memcpy.debug
 OLD_FILES+=usr/tests/lib/libc/string/.debug/memmem.debug
 OLD_FILES+=usr/tests/lib/libc/string/.debug/memset.debug
 OLD_FILES+=usr/tests/lib/libc/string/.debug/strcat.debug
 OLD_FILES+=usr/tests/lib/libc/string/.debug/strchr.debug
 OLD_FILES+=usr/tests/lib/libc/string/.debug/strcmp.debug
 OLD_FILES+=usr/tests/lib/libc/string/.debug/strcpy.debug
 OLD_FILES+=usr/tests/lib/libc/string/.debug/strcspn.debug
 OLD_FILES+=usr/tests/lib/libc/string/.debug/strerror.debug
 OLD_FILES+=usr/tests/lib/libc/string/.debug/strlen.debug
 OLD_FILES+=usr/tests/lib/libc/string/.debug/strpbrk.debug
 OLD_FILES+=usr/tests/lib/libc/string/.debug/strrchr.debug
 OLD_FILES+=usr/tests/lib/libc/string/.debug/strspn.debug
 OLD_FILES+=usr/tests/lib/libc/string/.debug/swab.debug
 OLD_DIRS+=usr/tests/lib/libc/sys/.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/access_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/chroot_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/clock_gettime_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/connect_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/dup_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/fsync_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/getcontext_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/getgroups_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/getitimer_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/getlogin_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/getpid_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/getrusage_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/getsid_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/gettimeofday_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/issetugid_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/kevent_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/kill_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/link_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/listen_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/mincore_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/mkdir_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/mkfifo_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/mknod_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/mlock_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/mmap_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/mprotect_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/msgctl_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/msgget_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/msgrcv_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/msgsnd_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/msync_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/nanosleep_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/pipe2_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/pipe_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/poll_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/revoke_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/select_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/setrlimit_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/setuid_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/sigaction_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/sigqueue_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/sigtimedwait_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/socketpair_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/stat_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/timer_create_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/truncate_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/ucontext_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/umask_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/unlink_test.debug
 OLD_FILES+=usr/tests/lib/libc/sys/.debug/write_test.debug
 OLD_DIRS+=usr/tests/lib/libc/termios/.debug
 OLD_FILES+=usr/tests/lib/libc/termios/.debug/tcsetpgrp_test.debug
 OLD_DIRS+=usr/tests/lib/libc/tls/.debug
 OLD_FILES+=usr/tests/lib/libc/tls/.debug/h_tls_dlopen.so.debug
 OLD_FILES+=usr/tests/lib/libc/tls/.debug/libh_tls_dynamic.so.1.debug
 OLD_FILES+=usr/tests/lib/libc/tls/.debug/tls_dlopen_test.debug
 OLD_FILES+=usr/tests/lib/libc/tls/.debug/tls_dynamic_test.debug
 OLD_DIRS+=usr/tests/lib/libc/ttyio/.debug
 OLD_FILES+=usr/tests/lib/libc/ttyio/.debug/ttyio_test.debug
 OLD_DIRS+=usr/tests/lib/libcrypt/.debug
 OLD_FILES+=usr/tests/lib/libcrypt/.debug/crypt_tests.debug
 OLD_DIRS+=usr/tests/lib/libmp/.debug
 OLD_FILES+=usr/tests/lib/libmp/.debug/legacy_test.debug
 OLD_DIRS+=usr/tests/lib/libnv/.debug
 OLD_FILES+=usr/tests/lib/libnv/.debug/dnv_tests.debug
 OLD_FILES+=usr/tests/lib/libnv/.debug/nv_array_tests.debug
 OLD_FILES+=usr/tests/lib/libnv/.debug/nv_tests.debug
 OLD_FILES+=usr/tests/lib/libnv/.debug/nvlist_add_test.debug
 OLD_FILES+=usr/tests/lib/libnv/.debug/nvlist_exists_test.debug
 OLD_FILES+=usr/tests/lib/libnv/.debug/nvlist_free_test.debug
 OLD_FILES+=usr/tests/lib/libnv/.debug/nvlist_get_test.debug
 OLD_FILES+=usr/tests/lib/libnv/.debug/nvlist_move_test.debug
 OLD_FILES+=usr/tests/lib/libnv/.debug/nvlist_send_recv_test.debug
 OLD_DIRS+=usr/tests/lib/libpam/.debug
 OLD_FILES+=usr/tests/lib/libpam/.debug/t_openpam_ctype.debug
 OLD_FILES+=usr/tests/lib/libpam/.debug/t_openpam_readlinev.debug
 OLD_FILES+=usr/tests/lib/libpam/.debug/t_openpam_readword.debug
 OLD_DIRS+=usr/tests/lib/libproc/.debug
 OLD_FILES+=usr/tests/lib/libproc/.debug/proc_test.debug
 OLD_FILES+=usr/tests/lib/libproc/.debug/target_prog.debug
 OLD_DIRS+=usr/tests/lib/librt/.debug
 OLD_FILES+=usr/tests/lib/librt/.debug/sched_test.debug
 OLD_FILES+=usr/tests/lib/librt/.debug/sem_test.debug
 OLD_DIRS+=usr/tests/lib/libthr/.debug
 OLD_FILES+=usr/tests/lib/libthr/.debug/barrier_test.debug
 OLD_FILES+=usr/tests/lib/libthr/.debug/cond_test.debug
 OLD_FILES+=usr/tests/lib/libthr/.debug/condwait_test.debug
 OLD_FILES+=usr/tests/lib/libthr/.debug/detach_test.debug
 OLD_FILES+=usr/tests/lib/libthr/.debug/equal_test.debug
 OLD_FILES+=usr/tests/lib/libthr/.debug/fork_test.debug
 OLD_FILES+=usr/tests/lib/libthr/.debug/fpu_test.debug
 OLD_FILES+=usr/tests/lib/libthr/.debug/h_atexit.debug
 OLD_FILES+=usr/tests/lib/libthr/.debug/h_cancel.debug
 OLD_FILES+=usr/tests/lib/libthr/.debug/h_exit.debug
 OLD_FILES+=usr/tests/lib/libthr/.debug/h_resolv.debug
 OLD_FILES+=usr/tests/lib/libthr/.debug/join_test.debug
 OLD_FILES+=usr/tests/lib/libthr/.debug/kill_test.debug
 OLD_FILES+=usr/tests/lib/libthr/.debug/mutex_test.debug
 OLD_FILES+=usr/tests/lib/libthr/.debug/once_test.debug
 OLD_FILES+=usr/tests/lib/libthr/.debug/preempt_test.debug
 OLD_FILES+=usr/tests/lib/libthr/.debug/rwlock_test.debug
 OLD_FILES+=usr/tests/lib/libthr/.debug/sem_test.debug
 OLD_FILES+=usr/tests/lib/libthr/.debug/siglongjmp_test.debug
 OLD_FILES+=usr/tests/lib/libthr/.debug/sigmask_test.debug
 OLD_FILES+=usr/tests/lib/libthr/.debug/sigsuspend_test.debug
 OLD_FILES+=usr/tests/lib/libthr/.debug/sleep_test.debug
 OLD_FILES+=usr/tests/lib/libthr/.debug/swapcontext_test.debug
 OLD_DIRS+=usr/tests/lib/libthr/dlopen/.debug
 OLD_FILES+=usr/tests/lib/libthr/dlopen/.debug/dlopen_test.debug
 OLD_FILES+=usr/tests/lib/libthr/dlopen/.debug/h_pthread_dlopen.so.1.debug
 OLD_FILES+=usr/tests/lib/libthr/dlopen/.debug/main_pthread_create_test.debug
 OLD_DIRS+=usr/tests/lib/libutil/.debug
 OLD_FILES+=usr/tests/lib/libutil/.debug/flopen_test.debug
 OLD_FILES+=usr/tests/lib/libutil/.debug/grp_test.debug
 OLD_FILES+=usr/tests/lib/libutil/.debug/humanize_number_test.debug
 OLD_FILES+=usr/tests/lib/libutil/.debug/pidfile_test.debug
 OLD_FILES+=usr/tests/lib/libutil/.debug/trimdomain-nodomain_test.debug
 OLD_FILES+=usr/tests/lib/libutil/.debug/trimdomain_test.debug
 OLD_DIRS+=usr/tests/lib/libxo/.debug
 OLD_FILES+=usr/tests/lib/libxo/.debug/libenc_test.so.debug
 OLD_FILES+=usr/tests/lib/libxo/.debug/test_01.debug
 OLD_FILES+=usr/tests/lib/libxo/.debug/test_02.debug
 OLD_FILES+=usr/tests/lib/libxo/.debug/test_03.debug
 OLD_FILES+=usr/tests/lib/libxo/.debug/test_04.debug
 OLD_FILES+=usr/tests/lib/libxo/.debug/test_05.debug
 OLD_FILES+=usr/tests/lib/libxo/.debug/test_06.debug
 OLD_FILES+=usr/tests/lib/libxo/.debug/test_07.debug
 OLD_FILES+=usr/tests/lib/libxo/.debug/test_08.debug
 OLD_FILES+=usr/tests/lib/libxo/.debug/test_09.debug
 OLD_FILES+=usr/tests/lib/libxo/.debug/test_10.debug
 OLD_FILES+=usr/tests/lib/libxo/.debug/test_11.debug
 OLD_DIRS+=usr/tests/lib/msun/.debug
 OLD_FILES+=usr/tests/lib/msun/.debug/acos_test.debug
 OLD_FILES+=usr/tests/lib/msun/.debug/asin_test.debug
 OLD_FILES+=usr/tests/lib/msun/.debug/atan_test.debug
 OLD_FILES+=usr/tests/lib/msun/.debug/cbrt_test.debug
 OLD_FILES+=usr/tests/lib/msun/.debug/ceil_test.debug
 OLD_FILES+=usr/tests/lib/msun/.debug/cos_test.debug
 OLD_FILES+=usr/tests/lib/msun/.debug/cosh_test.debug
 OLD_FILES+=usr/tests/lib/msun/.debug/erf_test.debug
 OLD_FILES+=usr/tests/lib/msun/.debug/exp_test.debug
 OLD_FILES+=usr/tests/lib/msun/.debug/fmod_test.debug
 OLD_FILES+=usr/tests/lib/msun/.debug/infinity_test.debug
 OLD_FILES+=usr/tests/lib/msun/.debug/ldexp_test.debug
 OLD_FILES+=usr/tests/lib/msun/.debug/log_test.debug
 OLD_FILES+=usr/tests/lib/msun/.debug/pow_test.debug
 OLD_FILES+=usr/tests/lib/msun/.debug/precision_test.debug
 OLD_FILES+=usr/tests/lib/msun/.debug/round_test.debug
 OLD_FILES+=usr/tests/lib/msun/.debug/scalbn_test.debug
 OLD_FILES+=usr/tests/lib/msun/.debug/sin_test.debug
 OLD_FILES+=usr/tests/lib/msun/.debug/sinh_test.debug
 OLD_FILES+=usr/tests/lib/msun/.debug/sqrt_test.debug
 OLD_FILES+=usr/tests/lib/msun/.debug/tan_test.debug
 OLD_FILES+=usr/tests/lib/msun/.debug/tanh_test.debug
 OLD_DIRS+=usr/tests/libexec/rtld-elf/.debug
 OLD_FILES+=usr/tests/libexec/rtld-elf/.debug/ld_library_pathfds.debug
 OLD_FILES+=usr/tests/libexec/rtld-elf/.debug/libpythagoras.so.0.debug
 OLD_FILES+=usr/tests/libexec/rtld-elf/.debug/target.debug
 OLD_DIRS+=usr/tests/sbin/devd/.debug
 OLD_FILES+=usr/tests/sbin/devd/.debug/client_test.debug
 OLD_DIRS+=usr/tests/sbin/dhclient/.debug
 OLD_FILES+=usr/tests/sbin/dhclient/.debug/option-domain-search_test.debug
 OLD_DIRS+=usr/tests/share/examples/tests/atf/.debug
 OLD_FILES+=usr/tests/share/examples/tests/atf/.debug/printf_test.debug
 OLD_DIRS+=usr/tests/share/examples/tests/plain/.debug
 OLD_FILES+=usr/tests/share/examples/tests/plain/.debug/printf_test.debug
 OLD_DIRS+=usr/tests/sys/aio/.debug
 OLD_FILES+=usr/tests/sys/aio/.debug/aio_kqueue_test.debug
 OLD_FILES+=usr/tests/sys/aio/.debug/aio_test.debug
 OLD_FILES+=usr/tests/sys/aio/.debug/lio_kqueue_test.debug
 OLD_DIRS+=usr/tests/sys/fifo/.debug
 OLD_FILES+=usr/tests/sys/fifo/.debug/fifo_create.debug
 OLD_FILES+=usr/tests/sys/fifo/.debug/fifo_io.debug
 OLD_FILES+=usr/tests/sys/fifo/.debug/fifo_misc.debug
 OLD_FILES+=usr/tests/sys/fifo/.debug/fifo_open.debug
 OLD_DIRS+=usr/tests/sys/file/.debug
 OLD_FILES+=usr/tests/sys/file/.debug/closefrom_test.debug
 OLD_FILES+=usr/tests/sys/file/.debug/dup_test.debug
 OLD_FILES+=usr/tests/sys/file/.debug/fcntlflags_test.debug
 OLD_FILES+=usr/tests/sys/file/.debug/flock_helper.debug
 OLD_FILES+=usr/tests/sys/file/.debug/ftruncate_test.debug
 OLD_FILES+=usr/tests/sys/file/.debug/newfileops_on_fork_test.debug
 OLD_DIRS+=usr/tests/sys/kern/.debug
 OLD_FILES+=usr/tests/sys/kern/.debug/kern_descrip_test.debug
 OLD_FILES+=usr/tests/sys/kern/.debug/ptrace_test.debug
 OLD_FILES+=usr/tests/sys/kern/.debug/unix_seqpacket_test.debug
 OLD_DIRS+=usr/tests/sys/kern/execve/.debug
 OLD_FILES+=usr/tests/sys/kern/execve/.debug/execve_helper.debug
 OLD_FILES+=usr/tests/sys/kern/execve/.debug/good_aout.debug
 OLD_DIRS+=usr/tests/sys/kqueue/.debug
 OLD_FILES+=usr/tests/sys/kqueue/.debug/kqtest.debug
 OLD_DIRS+=usr/tests/sys/mqueue/.debug
 OLD_FILES+=usr/tests/sys/mqueue/.debug/mqtest1.debug
 OLD_FILES+=usr/tests/sys/mqueue/.debug/mqtest2.debug
 OLD_FILES+=usr/tests/sys/mqueue/.debug/mqtest3.debug
 OLD_FILES+=usr/tests/sys/mqueue/.debug/mqtest4.debug
 OLD_FILES+=usr/tests/sys/mqueue/.debug/mqtest5.debug
 OLD_DIRS+=usr/tests/sys/netinet/.debug
 OLD_FILES+=usr/tests/sys/netinet/.debug/udp_dontroute.debug
 OLD_DIRS+=usr/tests/sys/pjdfstest/.debug
 OLD_FILES+=usr/tests/sys/pjdfstest/.debug/pjdfstest.debug
 OLD_DIRS+=usr/tests/sys/vm/.debug
 OLD_FILES+=usr/tests/sys/vm/.debug/mmap_test.debug
 # 20151015: Rename files due to file-installed-as-dir bug
 OLD_FILES+=usr/share/doc/legal/realtek
 OLD_FILES+=usr/share/doc/legal/realtek/LICENSE
 OLD_DIRS+=usr/share/doc/legal/realtek
 OLD_DIRS+=usr/share/doc/legal/intel_ipw
 OLD_FILES+=usr/share/doc/legal/intel_ipw/LICENSE
 OLD_FILES+=usr/share/doc/legal/intel_iwn
 OLD_FILES+=usr/share/doc/legal/intel_iwn/LICENSE
 OLD_DIRS+=usr/share/doc/legal/intel_iwn
 OLD_DIRS+=usr/share/doc/legal/intel_iwi
 OLD_FILES+=usr/share/doc/legal/intel_iwi/LICENSE
 OLD_DIRS+=usr/share/doc/legal/intel_wpi
 OLD_FILES+=usr/share/doc/legal/intel_wpi/LICENSE
 # 20151006: new libc++ import
 OLD_FILES+=usr/include/c++/__tuple_03
 OLD_FILES+=usr/include/c++/v1/__tuple_03
 OLD_FILES+=usr/include/c++/v1/tr1/__tuple_03
 # 20151006: new clang import which bumps version from 3.6.1 to 3.7.0.
 OLD_FILES+=usr/lib/clang/3.6.1/include/__stddef_max_align_t.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/__wmmintrin_aes.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/__wmmintrin_pclmul.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/adxintrin.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/altivec.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/ammintrin.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/arm_acle.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/arm_neon.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/avx2intrin.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/avx512bwintrin.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/avx512erintrin.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/avx512fintrin.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/avx512vlbwintrin.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/avx512vlintrin.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/avxintrin.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/bmi2intrin.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/bmiintrin.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/cpuid.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/emmintrin.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/f16cintrin.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/fma4intrin.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/fmaintrin.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/ia32intrin.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/immintrin.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/lzcntintrin.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/mm3dnow.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/mm_malloc.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/mmintrin.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/module.modulemap
 OLD_FILES+=usr/lib/clang/3.6.1/include/nmmintrin.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/pmmintrin.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/popcntintrin.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/prfchwintrin.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/rdseedintrin.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/rtmintrin.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/shaintrin.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/smmintrin.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/tbmintrin.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/tmmintrin.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/wmmintrin.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/x86intrin.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/xmmintrin.h
 OLD_FILES+=usr/lib/clang/3.6.1/include/xopintrin.h
 OLD_DIRS+=usr/lib/clang/3.6.1/include
 OLD_FILES+=usr/lib/clang/3.6.1/lib/freebsd/libclang_rt.asan-i386.a
 OLD_FILES+=usr/lib/clang/3.6.1/lib/freebsd/libclang_rt.asan-x86_64.a
 OLD_FILES+=usr/lib/clang/3.6.1/lib/freebsd/libclang_rt.asan_cxx-i386.a
 OLD_FILES+=usr/lib/clang/3.6.1/lib/freebsd/libclang_rt.asan_cxx-x86_64.a
 OLD_FILES+=usr/lib/clang/3.6.1/lib/freebsd/libclang_rt.profile-arm.a
 OLD_FILES+=usr/lib/clang/3.6.1/lib/freebsd/libclang_rt.profile-i386.a
 OLD_FILES+=usr/lib/clang/3.6.1/lib/freebsd/libclang_rt.profile-x86_64.a
 OLD_FILES+=usr/lib/clang/3.6.1/lib/freebsd/libclang_rt.san-i386.a
 OLD_FILES+=usr/lib/clang/3.6.1/lib/freebsd/libclang_rt.san-x86_64.a
 OLD_FILES+=usr/lib/clang/3.6.1/lib/freebsd/libclang_rt.ubsan-i386.a
 OLD_FILES+=usr/lib/clang/3.6.1/lib/freebsd/libclang_rt.ubsan-x86_64.a
 OLD_FILES+=usr/lib/clang/3.6.1/lib/freebsd/libclang_rt.ubsan_cxx-i386.a
 OLD_FILES+=usr/lib/clang/3.6.1/lib/freebsd/libclang_rt.ubsan_cxx-x86_64.a
 OLD_DIRS+=usr/lib/clang/3.6.1/lib/freebsd
 OLD_DIRS+=usr/lib/clang/3.6.1/lib
 OLD_DIRS+=usr/lib/clang/3.6.1
 # 20150928: unused sgsmsg utility is removed
 OLD_FILES+=usr/bin/sgsmsg
 # 20150926: remove links to removed/unimplemented mbuf(9) macros
 OLD_FILES+=usr/share/man/man9/MEXT_ADD_REF.9.gz
 OLD_FILES+=usr/share/man/man9/MEXTFREE.9.gz
 OLD_FILES+=usr/share/man/man9/MEXT_IS_REF.9.gz
 OLD_FILES+=usr/share/man/man9/MEXT_REM_REF.9.gz
 OLD_FILES+=usr/share/man/man9/MFREE.9.gz
 # 20150818: *allocm() are gone in jemalloc 4.0.0
 OLD_FILES+=usr/share/man/man3/allocm.3.gz
 OLD_FILES+=usr/share/man/man3/dallocm.3.gz
 OLD_FILES+=usr/share/man/man3/nallocm.3.gz
 OLD_FILES+=usr/share/man/man3/rallocm.3.gz
 OLD_FILES+=usr/share/man/man3/sallocm.3.gz
 # 20150802: Remove netbsd's test on pw(8)
 OLD_FILES+=usr/tests/usr.sbin/pw/pw_test
 # 20150719: Remove libarchive.pc
 OLD_FILES+=usr/libdata/pkgconfig/libarchive.pc
 # 20150705: Rename DTrace provider man pages.
 OLD_FILES+=usr/share/man/man4/dtrace-io.4.gz
 OLD_FILES+=usr/share/man/man4/dtrace-ip.4.gz
 OLD_FILES+=usr/share/man/man4/dtrace-proc.4.gz
 OLD_FILES+=usr/share/man/man4/dtrace-sched.4.gz
 OLD_FILES+=usr/share/man/man4/dtrace-tcp.4.gz
 OLD_FILES+=usr/share/man/man4/dtrace-udp.4.gz
 # 20150704: nvlist private headers no longer installed
 OLD_FILES+=usr/include/sys/nv_impl.h
 OLD_FILES+=usr/include/sys/nvlist_impl.h
 OLD_FILES+=usr/include/sys/nvpair_impl.h
 # 20150624
 OLD_LIBS+=usr/lib/libugidfw.so.4
 OLD_LIBS+=usr/lib32/libugidfw.so.4
 # 20150604: Move nvlist man pages to section 9.
 OLD_FILES+=usr/share/man/man3/libnv.3.gz
 OLD_FILES+=usr/share/man/man3/nv.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_add_binary.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_add_bool.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_add_descriptor.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_add_null.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_add_number.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_add_nvlist.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_add_string.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_add_stringf.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_add_stringv.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_clone.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_create.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_destroy.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_dump.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_empty.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_error.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_exists.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_exists_binary.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_exists_bool.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_exists_descriptor.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_exists_null.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_exists_number.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_exists_nvlist.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_exists_string.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_exists_type.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_fdump.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_flags.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_free.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_free_binary.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_free_bool.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_free_descriptor.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_free_null.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_free_number.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_free_nvlist.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_free_string.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_free_type.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_get_binary.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_get_bool.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_get_descriptor.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_get_number.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_get_nvlist.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_get_parent.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_get_string.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_move_binary.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_move_descriptor.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_move_nvlist.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_move_string.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_next.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_pack.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_recv.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_send.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_set_error.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_size.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_take_binary.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_take_bool.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_take_descriptor.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_take_number.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_take_nvlist.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_take_string.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_unpack.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_xfer.3.gz
 # 20150702: Remove duplicated nvlist includes.
 OLD_FILES+=usr/include/dnv.h
 OLD_FILES+=usr/include/nv.h
 # 20150528: PCI IOV device driver methods moved to a separate kobj interface.
 OLD_FILES+=usr/share/man/man9/PCI_ADD_VF.9.gz
 OLD_FILES+=usr/share/man/man9/PCI_INIT_IOV.9.gz
 OLD_FILES+=usr/share/man/man9/PCI_UNINIT_IOV.9.gz
 # 20150525: new clang import which bumps version from 3.6.0 to 3.6.1.
 OLD_FILES+=usr/lib/clang/3.6.0/include/__stddef_max_align_t.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/__wmmintrin_aes.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/__wmmintrin_pclmul.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/adxintrin.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/altivec.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/ammintrin.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/arm_acle.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/arm_neon.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/avx2intrin.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/avx512bwintrin.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/avx512erintrin.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/avx512fintrin.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/avx512vlbwintrin.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/avx512vlintrin.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/avxintrin.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/bmi2intrin.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/bmiintrin.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/cpuid.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/emmintrin.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/f16cintrin.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/fma4intrin.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/fmaintrin.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/ia32intrin.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/immintrin.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/lzcntintrin.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/mm3dnow.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/mm_malloc.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/mmintrin.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/module.modulemap
 OLD_FILES+=usr/lib/clang/3.6.0/include/nmmintrin.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/pmmintrin.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/popcntintrin.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/prfchwintrin.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/rdseedintrin.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/rtmintrin.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/shaintrin.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/smmintrin.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/tbmintrin.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/tmmintrin.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/wmmintrin.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/x86intrin.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/xmmintrin.h
 OLD_FILES+=usr/lib/clang/3.6.0/include/xopintrin.h
 OLD_DIRS+=usr/lib/clang/3.6.0/include
 OLD_FILES+=usr/lib/clang/3.6.0/lib/freebsd/libclang_rt.asan-i386.a
 OLD_FILES+=usr/lib/clang/3.6.0/lib/freebsd/libclang_rt.asan-x86_64.a
 OLD_FILES+=usr/lib/clang/3.6.0/lib/freebsd/libclang_rt.asan_cxx-i386.a
 OLD_FILES+=usr/lib/clang/3.6.0/lib/freebsd/libclang_rt.asan_cxx-x86_64.a
 OLD_FILES+=usr/lib/clang/3.6.0/lib/freebsd/libclang_rt.profile-arm.a
 OLD_FILES+=usr/lib/clang/3.6.0/lib/freebsd/libclang_rt.profile-i386.a
 OLD_FILES+=usr/lib/clang/3.6.0/lib/freebsd/libclang_rt.profile-x86_64.a
 OLD_FILES+=usr/lib/clang/3.6.0/lib/freebsd/libclang_rt.san-i386.a
 OLD_FILES+=usr/lib/clang/3.6.0/lib/freebsd/libclang_rt.san-x86_64.a
 OLD_FILES+=usr/lib/clang/3.6.0/lib/freebsd/libclang_rt.ubsan-i386.a
 OLD_FILES+=usr/lib/clang/3.6.0/lib/freebsd/libclang_rt.ubsan-x86_64.a
 OLD_FILES+=usr/lib/clang/3.6.0/lib/freebsd/libclang_rt.ubsan_cxx-i386.a
 OLD_FILES+=usr/lib/clang/3.6.0/lib/freebsd/libclang_rt.ubsan_cxx-x86_64.a
 OLD_DIRS+=usr/lib/clang/3.6.0/lib/freebsd
 OLD_DIRS+=usr/lib/clang/3.6.0/lib
 OLD_DIRS+=usr/lib/clang/3.6.0
 # 20150521
 OLD_FILES+=usr/bin/demandoc
 OLD_FILES+=usr/share/man/man1/demandoc.1.gz
 OLD_FILES+=usr/share/man/man3/mandoc.3.gz
 OLD_FILES+=usr/share/man/man3/mandoc_headers.3.gz
 # 20150520
 OLD_FILES+=usr/lib/libheimsqlite.a
 OLD_FILES+=usr/lib/libheimsqlite.so
 OLD_LIBS+=usr/lib/libheimsqlite.so.11
 OLD_FILES+=usr/lib/libheimsqlite_p.a
 OLD_FILES+=usr/lib32/libheimsqlite.a
 OLD_FILES+=usr/lib32/libheimsqlite.so
 OLD_LIBS+=usr/lib32/libheimsqlite.so.11
 OLD_FILES+=usr/lib32/libheimsqlite_p.a
 # 20150518: tzdata2015c update
 OLD_FILES+=usr/share/zoneinfo/America/Montreal
 # 20150506
 OLD_FILES+=usr/share/man/man9/NDHASGIANT.9.gz
 # 20150504
 OLD_FILES+=usr/share/examples/etc/libmap32.conf
 OLD_FILES+=usr/include/bsdstat.h
 OLD_LIBS+=usr/lib32/private/libatf-c++.so.2
 OLD_LIBS+=usr/lib32/private/libbsdstat.so.1
 OLD_LIBS+=usr/lib32/private/libheimipcs.so.11
 OLD_LIBS+=usr/lib32/private/libsqlite3.so.0
 OLD_LIBS+=usr/lib32/private/libunbound.so.5
 OLD_LIBS+=usr/lib32/private/libatf-c.so.1
 OLD_LIBS+=usr/lib32/private/libheimipcc.so.11
 OLD_LIBS+=usr/lib32/private/libldns.so.5
 OLD_LIBS+=usr/lib32/private/libssh.so.5
 OLD_LIBS+=usr/lib32/private/libucl.so.1
 OLD_DIRS+=usr/lib32/private
 OLD_LIBS+=usr/lib/private/libatf-c++.so.2
 OLD_LIBS+=usr/lib/private/libbsdstat.so.1
 OLD_LIBS+=usr/lib/private/libheimipcs.so.11
 OLD_LIBS+=usr/lib/private/libsqlite3.so.0
 OLD_LIBS+=usr/lib/private/libunbound.so.5
 OLD_LIBS+=usr/lib/private/libatf-c.so.1
 OLD_LIBS+=usr/lib/private/libheimipcc.so.11
 OLD_LIBS+=usr/lib/private/libldns.so.5
 OLD_LIBS+=usr/lib/private/libssh.so.5
 OLD_LIBS+=usr/lib/private/libucl.so.1
 OLD_DIRS+=usr/lib/private
 # 20150501
 OLD_FILES+=usr/bin/soeliminate
 OLD_FILES+=usr/share/man/man1/soeliminate.1.gz
 # 20150501: Remove the nvlist_.*[vf] functions manpages.
 OLD_FILES+=usr/share/man/man3/nvlist_addf_binary.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_addf_bool.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_addf_descriptor.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_addf_null.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_addf_number.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_addf_nvlist.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_addf_string.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_addv_binary.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_addv_bool.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_addv_descriptor.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_addv_null.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_addv_number.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_addv_nvlist.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_addv_string.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_existsf.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_existsf_binary.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_existsf_bool.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_existsf_descriptor.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_existsf_null.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_existsf_number.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_existsf_nvlist.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_existsf_string.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_existsf_type.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_existsv.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_existsv_binary.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_existsv_bool.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_existsv_descriptor.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_existsv_null.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_existsv_number.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_existsv_nvlist.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_existsv_string.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_existsv_type.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_freef.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_freef_binary.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_freef_bool.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_freef_descriptor.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_freef_null.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_freef_number.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_freef_nvlist.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_freef_string.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_freef_type.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_freev.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_freev_binary.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_freev_bool.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_freev_descriptor.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_freev_null.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_freev_number.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_freev_nvlist.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_freev_string.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_freev_type.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_getf_binary.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_getf_bool.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_getf_descriptor.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_getf_number.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_getf_nvlist.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_getf_string.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_getv_binary.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_getv_bool.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_getv_descriptor.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_getv_number.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_getv_nvlist.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_getv_string.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_movef_binary.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_movef_descriptor.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_movef_nvlist.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_movef_string.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_movev_binary.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_movev_descriptor.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_movev_nvlist.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_movev_string.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_takef_binary.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_takef_bool.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_takef_descriptor.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_takef_number.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_takef_nvlist.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_takef_string.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_takev_binary.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_takev_bool.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_takev_descriptor.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_takev_number.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_takev_nvlist.3.gz
 OLD_FILES+=usr/share/man/man3/nvlist_takev_string.3.gz
 # 20150429: remove never written documentation
 OLD_FILES+=usr/share/doc/papers/hwpmc.ascii.gz
 # 20150427: test/sys/kern/mmap_test moved to test/sys/vm/mmap_test
 OLD_FILES+=usr/tests/sys/kern/mmap_test
 # 20150422: zlib.c moved from net to libkern
 OLD_FILES+=usr/include/net/zlib.h
 OLD_FILES+=usr/include/net/zutil.h
 # 20150418
 OLD_FILES+=sbin/mount_oldnfs
 OLD_FILES+=usr/share/man/man8/mount_oldnfs.8.gz
 # 20150416: ALTQ moved to net/altq
 OLD_FILES+=usr/include/altq/altq_rmclass_debug.h
 OLD_FILES+=usr/include/altq/altq.h
 OLD_FILES+=usr/include/altq/altq_cdnr.h
 OLD_FILES+=usr/include/altq/altq_hfsc.h
 OLD_FILES+=usr/include/altq/altq_priq.h
 OLD_FILES+=usr/include/altq/altqconf.h
 OLD_FILES+=usr/include/altq/altq_classq.h
 OLD_FILES+=usr/include/altq/altq_red.h
 OLD_FILES+=usr/include/altq/if_altq.h
 OLD_FILES+=usr/include/altq/altq_var.h
 OLD_FILES+=usr/include/altq/altq_rmclass.h
 OLD_FILES+=usr/include/altq/altq_cbq.h
 OLD_FILES+=usr/include/altq/altq_rio.h
 OLD_DIRS+=usr/include/altq
 # 20150330: ntp 4.2.8p1
 OLD_FILES+=usr/share/doc/ntp/driver1.html
 OLD_FILES+=usr/share/doc/ntp/driver10.html
 OLD_FILES+=usr/share/doc/ntp/driver11.html
 OLD_FILES+=usr/share/doc/ntp/driver12.html
 OLD_FILES+=usr/share/doc/ntp/driver16.html
 OLD_FILES+=usr/share/doc/ntp/driver18.html
 OLD_FILES+=usr/share/doc/ntp/driver19.html
 OLD_FILES+=usr/share/doc/ntp/driver2.html
 OLD_FILES+=usr/share/doc/ntp/driver20.html
 OLD_FILES+=usr/share/doc/ntp/driver22.html
 OLD_FILES+=usr/share/doc/ntp/driver26.html
 OLD_FILES+=usr/share/doc/ntp/driver27.html
 OLD_FILES+=usr/share/doc/ntp/driver28.html
 OLD_FILES+=usr/share/doc/ntp/driver29.html
 OLD_FILES+=usr/share/doc/ntp/driver3.html
 OLD_FILES+=usr/share/doc/ntp/driver30.html
 OLD_FILES+=usr/share/doc/ntp/driver32.html
 OLD_FILES+=usr/share/doc/ntp/driver33.html
 OLD_FILES+=usr/share/doc/ntp/driver34.html
 OLD_FILES+=usr/share/doc/ntp/driver35.html
 OLD_FILES+=usr/share/doc/ntp/driver36.html
 OLD_FILES+=usr/share/doc/ntp/driver37.html
 OLD_FILES+=usr/share/doc/ntp/driver4.html
 OLD_FILES+=usr/share/doc/ntp/driver5.html
 OLD_FILES+=usr/share/doc/ntp/driver6.html
 OLD_FILES+=usr/share/doc/ntp/driver7.html
 OLD_FILES+=usr/share/doc/ntp/driver8.html
 OLD_FILES+=usr/share/doc/ntp/driver9.html
 OLD_FILES+=usr/share/doc/ntp/ldisc.html
 OLD_FILES+=usr/share/doc/ntp/measure.html
 OLD_FILES+=usr/share/doc/ntp/mx4200data.html
 OLD_FILES+=usr/share/doc/ntp/notes.html
 OLD_FILES+=usr/share/doc/ntp/patches.html
 OLD_FILES+=usr/share/doc/ntp/porting.html
 OLD_FILES+=usr/share/man/man1/sntp.1.gz
 # 20150329
 .if ${TARGET_ARCH} == "arm"
 OLD_FILES+=usr/include/bootconfig.h
 .endif
 # 20150326
 OLD_FILES+=usr/share/man/man1/pmcstudy.1.gz
 # 20150315: new clang import which bumps version from 3.5.1 to 3.6.0.
 OLD_FILES+=usr/include/clang/3.5.1/__wmmintrin_aes.h
 OLD_FILES+=usr/include/clang/3.5.1/__wmmintrin_pclmul.h
 OLD_FILES+=usr/include/clang/3.5.1/altivec.h
 OLD_FILES+=usr/include/clang/3.5.1/ammintrin.h
 OLD_FILES+=usr/include/clang/3.5.1/arm_acle.h
 OLD_FILES+=usr/include/clang/3.5.1/arm_neon.h
 OLD_FILES+=usr/include/clang/3.5.1/avx2intrin.h
 OLD_FILES+=usr/include/clang/3.5.1/avxintrin.h
 OLD_FILES+=usr/include/clang/3.5.1/bmi2intrin.h
 OLD_FILES+=usr/include/clang/3.5.1/bmiintrin.h
 OLD_FILES+=usr/include/clang/3.5.1/cpuid.h
 OLD_FILES+=usr/include/clang/3.5.1/emmintrin.h
 OLD_FILES+=usr/include/clang/3.5.1/f16cintrin.h
 OLD_FILES+=usr/include/clang/3.5.1/fma4intrin.h
 OLD_FILES+=usr/include/clang/3.5.1/fmaintrin.h
 OLD_FILES+=usr/include/clang/3.5.1/ia32intrin.h
 OLD_FILES+=usr/include/clang/3.5.1/immintrin.h
 OLD_FILES+=usr/include/clang/3.5.1/lzcntintrin.h
 OLD_FILES+=usr/include/clang/3.5.1/mm3dnow.h
 OLD_FILES+=usr/include/clang/3.5.1/mm_malloc.h
 OLD_FILES+=usr/include/clang/3.5.1/mmintrin.h
 OLD_FILES+=usr/include/clang/3.5.1/module.modulemap
 OLD_FILES+=usr/include/clang/3.5.1/nmmintrin.h
 OLD_FILES+=usr/include/clang/3.5.1/pmmintrin.h
 OLD_FILES+=usr/include/clang/3.5.1/popcntintrin.h
 OLD_FILES+=usr/include/clang/3.5.1/prfchwintrin.h
 OLD_FILES+=usr/include/clang/3.5.1/rdseedintrin.h
 OLD_FILES+=usr/include/clang/3.5.1/rtmintrin.h
 OLD_FILES+=usr/include/clang/3.5.1/shaintrin.h
 OLD_FILES+=usr/include/clang/3.5.1/smmintrin.h
 OLD_FILES+=usr/include/clang/3.5.1/tbmintrin.h
 OLD_FILES+=usr/include/clang/3.5.1/tmmintrin.h
 OLD_FILES+=usr/include/clang/3.5.1/wmmintrin.h
 OLD_FILES+=usr/include/clang/3.5.1/x86intrin.h
 OLD_FILES+=usr/include/clang/3.5.1/xmmintrin.h
 OLD_FILES+=usr/include/clang/3.5.1/xopintrin.h
 OLD_DIRS+=usr/include/clang/3.5.1
 OLD_DIRS+=usr/include/clang
 OLD_FILES+=usr/lib/clang/3.5.1/lib/freebsd/libclang_rt.asan-i386.a
 OLD_FILES+=usr/lib/clang/3.5.1/lib/freebsd/libclang_rt.asan-x86_64.a
 OLD_FILES+=usr/lib/clang/3.5.1/lib/freebsd/libclang_rt.asan_cxx-i386.a
 OLD_FILES+=usr/lib/clang/3.5.1/lib/freebsd/libclang_rt.asan_cxx-x86_64.a
 OLD_FILES+=usr/lib/clang/3.5.1/lib/freebsd/libclang_rt.profile-arm.a
 OLD_FILES+=usr/lib/clang/3.5.1/lib/freebsd/libclang_rt.profile-i386.a
 OLD_FILES+=usr/lib/clang/3.5.1/lib/freebsd/libclang_rt.profile-x86_64.a
 OLD_FILES+=usr/lib/clang/3.5.1/lib/freebsd/libclang_rt.san-i386.a
 OLD_FILES+=usr/lib/clang/3.5.1/lib/freebsd/libclang_rt.san-x86_64.a
 OLD_FILES+=usr/lib/clang/3.5.1/lib/freebsd/libclang_rt.ubsan-i386.a
 OLD_FILES+=usr/lib/clang/3.5.1/lib/freebsd/libclang_rt.ubsan-x86_64.a
 OLD_FILES+=usr/lib/clang/3.5.1/lib/freebsd/libclang_rt.ubsan_cxx-i386.a
 OLD_FILES+=usr/lib/clang/3.5.1/lib/freebsd/libclang_rt.ubsan_cxx-x86_64.a
 OLD_DIRS+=usr/lib/clang/3.5.1/lib/freebsd
 OLD_DIRS+=usr/lib/clang/3.5.1/lib
 OLD_DIRS+=usr/lib/clang/3.5.1
 # 20150302: binutils documentation distributed as a manpage
 OLD_FILES+=usr/share/doc/binutils/as.txt
 OLD_FILES+=usr/share/doc/binutils/ld.txt
 OLD_DIRS+=usr/share/doc/binutils
 # 20150222: Removed bcd(6) and ppt(6)
 OLD_FILES+=usr/bin/bcd
 OLD_FILES+=usr/bin/ppt
 OLD_FILES+=usr/share/man/man6/bcd.6.gz
 OLD_FILES+=usr/share/man/man6/ppt.6.gz
 # 20150217: Removed remnants of ar(4) driver
 OLD_FILES+=usr/include/dev/ic/hd64570.h
 # 20150212: /usr/games moving into /usr/bin
 OLD_FILES+=usr/games/bcd
 OLD_FILES+=usr/games/caesar
 OLD_FILES+=usr/games/factor
 OLD_FILES+=usr/games/fortune
 OLD_FILES+=usr/games/grdc
 OLD_FILES+=usr/games/morse
 OLD_FILES+=usr/games/number
 OLD_FILES+=usr/games/pom
 OLD_FILES+=usr/games/ppt
 OLD_FILES+=usr/games/primes
 OLD_FILES+=usr/games/random
 OLD_FILES+=usr/games/rot13
 OLD_FILES+=usr/games/strfile
 OLD_FILES+=usr/games/unstr
 OLD_DIRS+=usr/games
 # 20150209: liblzma header
 OLD_FILES+=usr/include/lzma/lzma.h
 # 20150124: spl.9 and friends
 OLD_FILES+=usr/share/man/man9/spl.9.gz
 OLD_FILES+=usr/share/man/man9/spl0.9.gz
 OLD_FILES+=usr/share/man/man9/splbio.9.gz
 OLD_FILES+=usr/share/man/man9/splclock.9.gz
 OLD_FILES+=usr/share/man/man9/splhigh.9.gz
 OLD_FILES+=usr/share/man/man9/splimp.9.gz
 OLD_FILES+=usr/share/man/man9/splnet.9.gz
 OLD_FILES+=usr/share/man/man9/splsoftclock.9.gz
 OLD_FILES+=usr/share/man/man9/splsofttty.9.gz
 OLD_FILES+=usr/share/man/man9/splstatclock.9.gz
 OLD_FILES+=usr/share/man/man9/spltty.9.gz
 OLD_FILES+=usr/share/man/man9/splvm.9.gz
 OLD_FILES+=usr/share/man/man9/splx.9.gz
 # 20150118: toeplitz.c moved from netinet to net
 OLD_FILES+=usr/include/netinet/toeplitz.h
 # 20150118: new clang import which bumps version from 3.5.0 to 3.5.1.
 OLD_FILES+=usr/include/clang/3.5.0/__wmmintrin_aes.h
 OLD_FILES+=usr/include/clang/3.5.0/__wmmintrin_pclmul.h
 OLD_FILES+=usr/include/clang/3.5.0/altivec.h
 OLD_FILES+=usr/include/clang/3.5.0/ammintrin.h
 OLD_FILES+=usr/include/clang/3.5.0/arm_acle.h
 OLD_FILES+=usr/include/clang/3.5.0/arm_neon.h
 OLD_FILES+=usr/include/clang/3.5.0/avx2intrin.h
 OLD_FILES+=usr/include/clang/3.5.0/avxintrin.h
 OLD_FILES+=usr/include/clang/3.5.0/bmi2intrin.h
 OLD_FILES+=usr/include/clang/3.5.0/bmiintrin.h
 OLD_FILES+=usr/include/clang/3.5.0/cpuid.h
 OLD_FILES+=usr/include/clang/3.5.0/emmintrin.h
 OLD_FILES+=usr/include/clang/3.5.0/f16cintrin.h
 OLD_FILES+=usr/include/clang/3.5.0/fma4intrin.h
 OLD_FILES+=usr/include/clang/3.5.0/fmaintrin.h
 OLD_FILES+=usr/include/clang/3.5.0/ia32intrin.h
 OLD_FILES+=usr/include/clang/3.5.0/immintrin.h
 OLD_FILES+=usr/include/clang/3.5.0/lzcntintrin.h
 OLD_FILES+=usr/include/clang/3.5.0/mm3dnow.h
 OLD_FILES+=usr/include/clang/3.5.0/mm_malloc.h
 OLD_FILES+=usr/include/clang/3.5.0/mmintrin.h
 OLD_FILES+=usr/include/clang/3.5.0/module.modulemap
 OLD_FILES+=usr/include/clang/3.5.0/nmmintrin.h
 OLD_FILES+=usr/include/clang/3.5.0/pmmintrin.h
 OLD_FILES+=usr/include/clang/3.5.0/popcntintrin.h
 OLD_FILES+=usr/include/clang/3.5.0/prfchwintrin.h
 OLD_FILES+=usr/include/clang/3.5.0/rdseedintrin.h
 OLD_FILES+=usr/include/clang/3.5.0/rtmintrin.h
 OLD_FILES+=usr/include/clang/3.5.0/shaintrin.h
 OLD_FILES+=usr/include/clang/3.5.0/smmintrin.h
 OLD_FILES+=usr/include/clang/3.5.0/tbmintrin.h
 OLD_FILES+=usr/include/clang/3.5.0/tmmintrin.h
 OLD_FILES+=usr/include/clang/3.5.0/wmmintrin.h
 OLD_FILES+=usr/include/clang/3.5.0/x86intrin.h
 OLD_FILES+=usr/include/clang/3.5.0/xmmintrin.h
 OLD_FILES+=usr/include/clang/3.5.0/xopintrin.h
 OLD_DIRS+=usr/include/clang/3.5.0
 OLD_FILES+=usr/lib/clang/3.5.0/lib/freebsd/libclang_rt.asan-i386.a
 OLD_FILES+=usr/lib/clang/3.5.0/lib/freebsd/libclang_rt.asan-x86_64.a
 OLD_FILES+=usr/lib/clang/3.5.0/lib/freebsd/libclang_rt.asan_cxx-i386.a
 OLD_FILES+=usr/lib/clang/3.5.0/lib/freebsd/libclang_rt.asan_cxx-x86_64.a
 OLD_FILES+=usr/lib/clang/3.5.0/lib/freebsd/libclang_rt.profile-arm.a
 OLD_FILES+=usr/lib/clang/3.5.0/lib/freebsd/libclang_rt.profile-i386.a
 OLD_FILES+=usr/lib/clang/3.5.0/lib/freebsd/libclang_rt.profile-x86_64.a
 OLD_FILES+=usr/lib/clang/3.5.0/lib/freebsd/libclang_rt.san-i386.a
 OLD_FILES+=usr/lib/clang/3.5.0/lib/freebsd/libclang_rt.san-x86_64.a
 OLD_FILES+=usr/lib/clang/3.5.0/lib/freebsd/libclang_rt.ubsan-i386.a
 OLD_FILES+=usr/lib/clang/3.5.0/lib/freebsd/libclang_rt.ubsan-x86_64.a
 OLD_FILES+=usr/lib/clang/3.5.0/lib/freebsd/libclang_rt.ubsan_cxx-i386.a
 OLD_FILES+=usr/lib/clang/3.5.0/lib/freebsd/libclang_rt.ubsan_cxx-x86_64.a
 OLD_DIRS+=usr/lib/clang/3.5.0/lib/freebsd
 OLD_DIRS+=usr/lib/clang/3.5.0/lib
 OLD_DIRS+=usr/lib/clang/3.5.0
 # 20150102: removal of asr(4)
 OLD_FILES+=usr/share/man/man4/asr.4.gz
 # 20150102: removal of texinfo
 OLD_FILES+=usr/bin/info
 OLD_FILES+=usr/bin/infokey
 OLD_FILES+=usr/bin/install-info
 OLD_FILES+=usr/bin/makeinfo
 OLD_FILES+=usr/bin/texindex
 OLD_FILES+=usr/share/info/am-utils.info.gz
 OLD_FILES+=usr/share/info/as.info.gz
 OLD_FILES+=usr/share/info/binutils.info.gz
 OLD_FILES+=usr/share/info/com_err.info.gz
 OLD_FILES+=usr/share/info/cpp.info.gz
 OLD_FILES+=usr/share/info/cppinternals.info.gz
 OLD_FILES+=usr/share/info/diff.info.gz
 OLD_FILES+=usr/share/info/dir
 OLD_FILES+=usr/share/info/gcc.info.gz
 OLD_FILES+=usr/share/info/gccint.info.gz
 OLD_FILES+=usr/share/info/gdb.info.gz
 OLD_FILES+=usr/share/info/gdbint.info.gz
 OLD_FILES+=usr/share/info/gperf.info.gz
 OLD_FILES+=usr/share/info/grep.info.gz
 OLD_FILES+=usr/share/info/groff.info.gz
 OLD_FILES+=usr/share/info/heimdal.info.gz
 OLD_FILES+=usr/share/info/history.info.gz
 OLD_FILES+=usr/share/info/info-stnd.info.gz
 OLD_FILES+=usr/share/info/info.info.gz
 OLD_FILES+=usr/share/info/ld.info.gz
 OLD_FILES+=usr/share/info/regex.info.gz
 OLD_FILES+=usr/share/info/rluserman.info.gz
 OLD_FILES+=usr/share/info/stabs.info.gz
 OLD_FILES+=usr/share/info/texinfo.info.gz
 OLD_FILES+=usr/share/man/man1/info.1.gz
 OLD_FILES+=usr/share/man/man1/infokey.1.gz
 OLD_FILES+=usr/share/man/man1/install-info.1.gz
 OLD_FILES+=usr/share/man/man1/makeinfo.1.gz
 OLD_FILES+=usr/share/man/man1/texindex.1.gz
 OLD_FILES+=usr/share/man/man5/info.5.gz
 OLD_FILES+=usr/share/man/man5/texinfo.5.gz
 OLD_DIRS+=usr/share/info
 # 20141231: new clang import which bumps version from 3.4.1 to 3.5.0.
 OLD_FILES+=usr/include/clang/3.4.1/__wmmintrin_aes.h
 OLD_FILES+=usr/include/clang/3.4.1/__wmmintrin_pclmul.h
 OLD_FILES+=usr/include/clang/3.4.1/altivec.h
 OLD_FILES+=usr/include/clang/3.4.1/ammintrin.h
 OLD_FILES+=usr/include/clang/3.4.1/arm_neon.h
 OLD_FILES+=usr/include/clang/3.4.1/avx2intrin.h
 OLD_FILES+=usr/include/clang/3.4.1/avxintrin.h
 OLD_FILES+=usr/include/clang/3.4.1/bmi2intrin.h
 OLD_FILES+=usr/include/clang/3.4.1/bmiintrin.h
 OLD_FILES+=usr/include/clang/3.4.1/cpuid.h
 OLD_FILES+=usr/include/clang/3.4.1/emmintrin.h
 OLD_FILES+=usr/include/clang/3.4.1/f16cintrin.h
 OLD_FILES+=usr/include/clang/3.4.1/fma4intrin.h
 OLD_FILES+=usr/include/clang/3.4.1/fmaintrin.h
 OLD_FILES+=usr/include/clang/3.4.1/immintrin.h
 OLD_FILES+=usr/include/clang/3.4.1/lzcntintrin.h
 OLD_FILES+=usr/include/clang/3.4.1/mm3dnow.h
 OLD_FILES+=usr/include/clang/3.4.1/mm_malloc.h
 OLD_FILES+=usr/include/clang/3.4.1/mmintrin.h
 OLD_FILES+=usr/include/clang/3.4.1/module.map
 OLD_FILES+=usr/include/clang/3.4.1/nmmintrin.h
 OLD_FILES+=usr/include/clang/3.4.1/pmmintrin.h
 OLD_FILES+=usr/include/clang/3.4.1/popcntintrin.h
 OLD_FILES+=usr/include/clang/3.4.1/prfchwintrin.h
 OLD_FILES+=usr/include/clang/3.4.1/rdseedintrin.h
 OLD_FILES+=usr/include/clang/3.4.1/rtmintrin.h
 OLD_FILES+=usr/include/clang/3.4.1/shaintrin.h
 OLD_FILES+=usr/include/clang/3.4.1/smmintrin.h
 OLD_FILES+=usr/include/clang/3.4.1/tbmintrin.h
 OLD_FILES+=usr/include/clang/3.4.1/tmmintrin.h
 OLD_FILES+=usr/include/clang/3.4.1/wmmintrin.h
 OLD_FILES+=usr/include/clang/3.4.1/x86intrin.h
 OLD_FILES+=usr/include/clang/3.4.1/xmmintrin.h
 OLD_FILES+=usr/include/clang/3.4.1/xopintrin.h
 OLD_DIRS+=usr/include/clang/3.4.1
 # 20141225: Remove gpib/ieee488
 OLD_FILES+=usr/include/dev/ieee488/ibfoo_int.h
 OLD_FILES+=usr/include/dev/ieee488/tnt4882.h
 OLD_FILES+=usr/include/dev/ieee488/ugpib.h
 OLD_FILES+=usr/include/dev/ieee488/upd7210.h
 OLD_DIRS+=usr/include/dev/ieee488
 OLD_FILES+=usr/include/gpib/gpib.h
 OLD_DIRS+=usr/include/gpib
 OLD_FILES+=usr/lib/libgpib.a
 OLD_FILES+=usr/lib/libgpib_p.a
 OLD_FILES+=usr/lib/libgpib.so
 OLD_LIBS+=usr/lib/libgpib.so.3
 OLD_FILES+=usr/lib/libgpib_p.a
 OLD_FILES+=usr/lib32/libgpib.a
 OLD_FILES+=usr/lib32/libgpib_p.a
 OLD_FILES+=usr/lib32/libgpib.so
 OLD_LIBS+=usr/lib32/libgpib.so.3
 OLD_FILES+=usr/share/man/man3/gpib.3.gz
 OLD_FILES+=usr/share/man/man3/ibclr.3.gz
 OLD_FILES+=usr/share/man/man3/ibdev.3.gz
 OLD_FILES+=usr/share/man/man3/ibdma.3.gz
 OLD_FILES+=usr/share/man/man3/ibeos.3.gz
 OLD_FILES+=usr/share/man/man3/ibeot.3.gz
 OLD_FILES+=usr/share/man/man3/ibloc.3.gz
 OLD_FILES+=usr/share/man/man3/ibonl.3.gz
 OLD_FILES+=usr/share/man/man3/ibpad.3.gz
 OLD_FILES+=usr/share/man/man3/ibrd.3.gz
 OLD_FILES+=usr/share/man/man3/ibsad.3.gz
 OLD_FILES+=usr/share/man/man3/ibsic.3.gz
 OLD_FILES+=usr/share/man/man3/ibtmo.3.gz
 OLD_FILES+=usr/share/man/man3/ibtrg.3.gz
 OLD_FILES+=usr/share/man/man3/ibwrt.3.gz
 OLD_FILES+=usr/share/man/man4/gpib.4.gz
 OLD_FILES+=usr/share/man/man4/pcii.4.gz
 OLD_FILES+=usr/share/man/man4/tnt4882.4.gz
 # 20141224: libxo moved to /lib
 OLD_LIBS+=usr/lib/libxo.so.0
 # 20141223: remove in6_gif.h, in_gif.h and if_stf.h
 OLD_FILES+=usr/include/net/if_stf.h
 OLD_FILES+=usr/include/netinet/in_gif.h
 OLD_FILES+=usr/include/netinet6/in6_gif.h
 # 20141209: pw tests broken into a file per command
 OLD_FILES+=usr/tests/usr.sbin/pw/pw_delete
 OLD_FILES+=usr/tests/usr.sbin/pw/pw_modify
 # 20141202: update to mandoc CVS 20141201
 OLD_FILES+=usr.bin/preconv
 OLD_FILES+=share/man/man1/preconv.1.gz
 # 20141129: mrouted rc.d scripts removed from base
 OLD_FILES+=etc/rc.d/mrouted
 # 20141126: convert sbin/mdconfig/tests to ATF format tests
 OLD_FILES+=usr/tests/sbin/mdconfig/legacy_test
 OLD_FILES+=usr/tests/sbin/mdconfig/mdconfig.test
 OLD_FILES+=usr/tests/sbin/mdconfig/run.pl
 # 20141126: remove xform_ipip decapsulation fallback
 OLD_FILES+=usr/include/netipsec/ipip_var.h
 # 20141122: mandoc updated to 1.13.1
 OLD_FILES+=usr/share/mdocml/external.png
 # 20141111: SF_KQUEUE code removed
 OLD_FILES+=usr/include/sys/sf_base.h
 OLD_FILES+=usr/include/sys/sf_sync.h
 # 20141109: faith/faithd removal
 OLD_FILES+=etc/rc.d/faith
 OLD_FILES+=usr/share/man/man4/faith.4.gz
 OLD_FILES+=usr/share/man/man4/if_faith.4.gz
 OLD_FILES+=usr/sbin/faithd
 OLD_FILES+=usr/share/man/man8/faithd.8.gz
 # 20141107: overhaul if_gre(4)
 OLD_FILES+=usr/include/netinet/ip_gre.h
 # 20141102: postrandom obsoleted by new /dev/random code
 OLD_FILES+=etc/rc.d/postrandom
 # 20141031: initrandom obsoleted by new /dev/random code
 OLD_FILES+=etc/rc.d/initrandom
 # 20141030: atf 0.21 import
 OLD_FILES+=usr/share/man/man3/atf-c++-api.3.gz
 # 20141028: debug files accidentally installed as directory name
 OLD_FILES+=usr/lib/debug/usr/lib/i18n
 OLD_FILES+=usr/lib/debug/usr/lib/private
 OLD_FILES+=usr/lib/debug/usr/lib32/i18n
 OLD_FILES+=usr/lib/debug/usr/lib32/private
 # 20141015: OpenSSL 1.0.1j import
 OLD_FILES+=usr/share/openssl/man/man3/CMS_sign_add1_signer.3.gz
 # 20141003: libproc version bump
 OLD_LIBS+=usr/lib/libproc.so.2
 OLD_LIBS+=usr/lib32/libproc.so.2
 # 20140922: sleepq_calc_signal_retval.9 and sleepq_catch_signals.9 removed
 OLD_FILES+=usr/share/man/man9/sleepq_calc_signal_retval.9.gz
 OLD_FILES+=usr/share/man/man9/sleepq_catch_signals.9.gz
 # 20140917: hv_kvpd rc.d script removed in favor of devd configuration
 OLD_FILES+=etc/rc.d/hv_kvpd
 # 20140917: libnv was accidentally being installed to /usr/lib instead of /lib
 OLD_LIBS+=usr/lib/libnv.so.0
 # 20140829: rc.d/kerberos removed
 OLD_FILES+=etc/rc.d/kerberos
 # 20140827: tzdata2014f import
 OLD_FILES+=usr/share/zoneinfo/Asia/Chongqing
 OLD_FILES+=usr/share/zoneinfo/Asia/Harbin
 OLD_FILES+=usr/share/zoneinfo/Asia/Kashgar
 # 20140814: libopie version bump
 OLD_LIBS+=usr/lib/libopie.so.7
 OLD_LIBS+=usr/lib32/libopie.so.7
 # 20140811: otp-sha renamed to otp-sha1
 OLD_FILES+=usr/bin/otp-sha
 OLD_FILES+=usr/share/man/man1/otp-sha.1.gz
 # 20140807: Remove private lib files that should not be installed.
 OLD_FILES+=usr/lib32/private/libatf-c.a
 OLD_FILES+=usr/lib32/private/libatf-c.so
 OLD_FILES+=usr/lib32/private/libatf-c_p.a
 OLD_FILES+=usr/lib32/private/libatf-c++.a
 OLD_FILES+=usr/lib32/private/libatf-c++.so
 OLD_FILES+=usr/lib32/private/libatf-c++_p.a
 OLD_FILES+=usr/lib32/private/libbsdstat.a
 OLD_FILES+=usr/lib32/private/libbsdstat.so
 OLD_FILES+=usr/lib32/private/libbsdstat_p.a
 OLD_FILES+=usr/lib32/private/libheimipcc.a
 OLD_FILES+=usr/lib32/private/libheimipcc.so
 OLD_FILES+=usr/lib32/private/libheimipcc_p.a
 OLD_FILES+=usr/lib32/private/libheimipcs.a
 OLD_FILES+=usr/lib32/private/libheimipcs.so
 OLD_FILES+=usr/lib32/private/libheimipcs_p.a
 OLD_FILES+=usr/lib32/private/libldns.a
 OLD_FILES+=usr/lib32/private/libldns.so
 OLD_FILES+=usr/lib32/private/libldns_p.a
 OLD_FILES+=usr/lib32/private/libssh.a
 OLD_FILES+=usr/lib32/private/libssh.so
 OLD_FILES+=usr/lib32/private/libssh_p.a
 OLD_FILES+=usr/lib32/private/libunbound.a
 OLD_FILES+=usr/lib32/private/libunbound.so
 OLD_FILES+=usr/lib32/private/libunbound_p.a
 OLD_FILES+=usr/lib32/private/libucl.a
 OLD_FILES+=usr/lib32/private/libucl.so
 OLD_FILES+=usr/lib32/private/libucl_p.a
 OLD_FILES+=usr/lib/private/libatf-c.a
 OLD_FILES+=usr/lib/private/libatf-c.so
 OLD_FILES+=usr/lib/private/libatf-c_p.a
 OLD_FILES+=usr/lib/private/libatf-c++.a
 OLD_FILES+=usr/lib/private/libatf-c++.so
 OLD_FILES+=usr/lib/private/libatf-c++_p.a
 OLD_FILES+=usr/lib/private/libbsdstat.a
 OLD_FILES+=usr/lib/private/libbsdstat.so
 OLD_FILES+=usr/lib/private/libbsdstat_p.a
 OLD_FILES+=usr/lib/private/libheimipcc.a
 OLD_FILES+=usr/lib/private/libheimipcc.so
 OLD_FILES+=usr/lib/private/libheimipcc_p.a
 OLD_FILES+=usr/lib/private/libheimipcs.a
 OLD_FILES+=usr/lib/private/libheimipcs.so
 OLD_FILES+=usr/lib/private/libheimipcs_p.a
 OLD_FILES+=usr/lib/private/libldns.a
 OLD_FILES+=usr/lib/private/libldns.so
 OLD_FILES+=usr/lib/private/libldns_p.a
 OLD_FILES+=usr/lib/private/libssh.a
 OLD_FILES+=usr/lib/private/libssh.so
 OLD_FILES+=usr/lib/private/libssh_p.a
 OLD_FILES+=usr/lib/private/libunbound.a
 OLD_FILES+=usr/lib/private/libunbound.so
 OLD_FILES+=usr/lib/private/libunbound_p.a
 OLD_FILES+=usr/lib/private/libucl.a
 OLD_FILES+=usr/lib/private/libucl.so
 OLD_FILES+=usr/lib/private/libucl_p.a
 # 20140803: Remove an obsolete man page
 OLD_FILES+=usr/share/man/man9/pmap_change_wiring.9.gz
 # 20140731
 OLD_FILES+=usr/share/man/man9/SYSCTL_ADD_OID.9.gz
 # 20140728: libsbuf restored to old version.
 OLD_LIBS+=lib/libsbuf.so.7
 OLD_LIBS+=usr/lib32/libsbuf.so.7
 # 20140728: Remove an obsolete man page
 OLD_FILES+=usr/share/man/man9/VOP_GETVOBJECT.9.gz
 OLD_FILES+=usr/share/man/man9/VOP_CREATEVOBJECT.9.gz
 OLD_FILES+=usr/share/man/man9/VOP_DESTROYVOBJECT.9.gz
 # 20140723: renamed to PCBGROUP.9
 OLD_FILES+=usr/share/man/man9/PCBGROUPS.9.gz
 # 20140722: browse_packages_ftp.sh removed
 OLD_FILES+=usr/share/examples/bsdconfig/browse_packages_ftp.sh
 # 20140718: Remove obsolete man pages
 OLD_FILES+=usr/share/man/man9/zero_copy.9.gz
 OLD_FILES+=usr/share/man/man9/zero_copy_sockets.9.gz
 # 20140718: Remove an obsolete man page
 OLD_FILES+=usr/share/man/man9/pmap_page_protect.9.gz
 # 20140717: Remove an obsolete man page
 OLD_FILES+=usr/share/man/man9/pmap_clear_reference.9.gz
 # 20140716: Remove an incorrectly named man page
 OLD_FILES+=usr/share/man/man9/pmap_ts_modified.9.gz
 # 20140712: Removal of bsd.dtrace.mk
 OLD_FILES+=usr/share/mk/bsd.dtrace.mk
 # 20140705: turn libreadline into an internal lib
 OLD_LIBS+=lib/libreadline.so.8
 OLD_FILES+=usr/lib/libreadline.a
 OLD_FILES+=usr/lib/libreadline_p.a
 OLD_FILES+=usr/lib/libreadline.so
 OLD_FILES+=usr/lib/libhistory.a
 OLD_FILES+=usr/lib/libhistory_p.a
 OLD_FILES+=usr/lib/libhistory.so
 OLD_LIBS+=usr/lib/libhistory.so.8
 OLD_FILES+=usr/lib32/libhistory.a
 OLD_FILES+=usr/lib32/libhistory.so
 OLD_LIBS+=usr/lib32/libhistory.so.8
 OLD_FILES+=usr/lib32/libhistory_p.a
 OLD_FILES+=usr/lib32/libreadline.a
 OLD_FILES+=usr/lib32/libreadline.so
 OLD_LIBS+=usr/lib32/libreadline.so.8
 OLD_FILES+=usr/lib32/libreadline_p.a
 OLD_FILES+=usr/include/readline/chardefs.h
 OLD_FILES+=usr/include/readline/history.h
 OLD_FILES+=usr/include/readline/keymaps.h
 OLD_FILES+=usr/include/readline/readline.h
 OLD_FILES+=usr/include/readline/tilde.h
 OLD_FILES+=usr/include/readline/rlconf.h
 OLD_FILES+=usr/include/readline/rlstdc.h
 OLD_FILES+=usr/include/readline/rltypedefs.h
 OLD_FILES+=usr/include/readline/rltypedefs.h
 OLD_DIRS+=usr/include/readline
 OLD_FILES+=usr/share/info/readline.info.gz
 OLD_FILES+=usr/share/man/man3/readline.3.gz
 OLD_FILES+=usr/share/man/man3/rlhistory.3.gz
 # 20140625: csup removal
 OLD_FILES+=usr/bin/csup
 OLD_FILES+=usr/bin/cpasswd
 OLD_FILES+=usr/share/man/man1/csup.1.gz
 OLD_FILES+=usr/share/man/man1/cpasswd.1.gz
 OLD_FILES+=usr/share/examples/cvsup/README
 OLD_FILES+=usr/share/examples/cvsup/cvs-supfile
 OLD_FILES+=usr/share/examples/cvsup/stable-supfile
 OLD_FILES+=usr/share/examples/cvsup/standard-supfile
 OLD_DIRS+=usr/share/examples/cvsup
 # 20140614: send-pr removal
 OLD_FILES+=usr/bin/sendbug
 OLD_FILES+=usr/share/info/send-pr.info.gz
 OLD_FILES+=usr/share/man/man1/send-pr.1.gz
 OLD_FILES+=usr/share/man/man1/sendbug.1.gz
 OLD_FILES+=etc/gnats/freefall
 OLD_DIRS+=etc/gnats
 # 20140512: new clang import which bumps version from 3.4 to 3.4.1.
 OLD_FILES+=usr/include/clang/3.4/__wmmintrin_aes.h
 OLD_FILES+=usr/include/clang/3.4/__wmmintrin_pclmul.h
 OLD_FILES+=usr/include/clang/3.4/altivec.h
 OLD_FILES+=usr/include/clang/3.4/ammintrin.h
 OLD_FILES+=usr/include/clang/3.4/avx2intrin.h
 OLD_FILES+=usr/include/clang/3.4/avxintrin.h
 OLD_FILES+=usr/include/clang/3.4/bmi2intrin.h
 OLD_FILES+=usr/include/clang/3.4/bmiintrin.h
 OLD_FILES+=usr/include/clang/3.4/cpuid.h
 OLD_FILES+=usr/include/clang/3.4/emmintrin.h
 OLD_FILES+=usr/include/clang/3.4/f16cintrin.h
 OLD_FILES+=usr/include/clang/3.4/fma4intrin.h
 OLD_FILES+=usr/include/clang/3.4/fmaintrin.h
 OLD_FILES+=usr/include/clang/3.4/immintrin.h
 OLD_FILES+=usr/include/clang/3.4/lzcntintrin.h
 OLD_FILES+=usr/include/clang/3.4/mm3dnow.h
 OLD_FILES+=usr/include/clang/3.4/mm_malloc.h
 OLD_FILES+=usr/include/clang/3.4/mmintrin.h
 OLD_FILES+=usr/include/clang/3.4/module.map
 OLD_FILES+=usr/include/clang/3.4/nmmintrin.h
 OLD_FILES+=usr/include/clang/3.4/pmmintrin.h
 OLD_FILES+=usr/include/clang/3.4/popcntintrin.h
 OLD_FILES+=usr/include/clang/3.4/prfchwintrin.h
 OLD_FILES+=usr/include/clang/3.4/rdseedintrin.h
 OLD_FILES+=usr/include/clang/3.4/rtmintrin.h
 OLD_FILES+=usr/include/clang/3.4/shaintrin.h
 OLD_FILES+=usr/include/clang/3.4/smmintrin.h
 OLD_FILES+=usr/include/clang/3.4/tbmintrin.h
 OLD_FILES+=usr/include/clang/3.4/tmmintrin.h
 OLD_FILES+=usr/include/clang/3.4/wmmintrin.h
 OLD_FILES+=usr/include/clang/3.4/x86intrin.h
 OLD_FILES+=usr/include/clang/3.4/xmmintrin.h
 OLD_FILES+=usr/include/clang/3.4/xopintrin.h
 OLD_FILES+=usr/include/clang/3.4/arm_neon.h
 OLD_FILES+=usr/include/clang/3.4/module.map
 OLD_DIRS+=usr/include/clang/3.4
 # 20140505: Bogusly installing src.opts.mk
 OLD_FILES+=usr/share/mk/src.opts.mk
 # 20140505: Reject PR kern/187551
 OLD_FILES+=usr/tests/sbin/ifconfig/fibs_test
 # 20140502: Removal of lindev(4)
 OLD_FILES+=usr/share/man/man4/lindev.4.gz
 # 20140425
 OLD_FILES+=usr/lib/libssp_p.a
 OLD_FILES+=usr/lib/libstand_p.a
 OLD_FILES+=usr/lib32/libssp_p.a
 OLD_FILES+=usr/lib32/libstand_p.a
 # 20140314: AppleTalk
 OLD_DIRS+=usr/include/netatalk
 OLD_FILES+=usr/include/netatalk/aarp.h
 OLD_FILES+=usr/include/netatalk/at.h
 OLD_FILES+=usr/include/netatalk/at_extern.h
 OLD_FILES+=usr/include/netatalk/at_var.h
 OLD_FILES+=usr/include/netatalk/ddp.h
 OLD_FILES+=usr/include/netatalk/ddp_pcb.h
 OLD_FILES+=usr/include/netatalk/ddp_var.h
 OLD_FILES+=usr/include/netatalk/endian.h
 OLD_FILES+=usr/include/netatalk/phase2.h
 # 20140314: Remove IPX/SPX
 OLD_LIBS+=lib/libipx.so.5
 OLD_FILES+=usr/include/netipx/ipx.h
 OLD_FILES+=usr/include/netipx/ipx_if.h
 OLD_FILES+=usr/include/netipx/ipx_pcb.h
 OLD_FILES+=usr/include/netipx/ipx_var.h
 OLD_FILES+=usr/include/netipx/spx.h
 OLD_FILES+=usr/include/netipx/spx_debug.h
 OLD_FILES+=usr/include/netipx/spx_timer.h
 OLD_FILES+=usr/include/netipx/spx_var.h
 OLD_DIRS+=usr/include/netipx
 OLD_FILES+=usr/lib/libipx.a
 OLD_FILES+=usr/lib/libipx.so
 OLD_FILES+=usr/lib/libipx_p.a
 OLD_FILES+=usr/lib32/libipx.a
 OLD_FILES+=usr/lib32/libipx.so
 OLD_LIBS+=usr/lib32/libipx.so.5
 OLD_FILES+=usr/lib32/libipx_p.a
 OLD_FILES+=usr/sbin/IPXrouted
 OLD_FILES+=usr/share/man/man3/ipx.3.gz
 OLD_FILES+=usr/share/man/man3/ipx_addr.3.gz
 OLD_FILES+=usr/share/man/man3/ipx_ntoa.3.gz
 OLD_FILES+=usr/share/man/man4/ef.4.gz
 OLD_FILES+=usr/share/man/man4/if_ef.4.gz
 OLD_FILES+=usr/share/man/man8/IPXrouted.8.gz
 # 20140314: bsdconfig usermgmt rewrite
 OLD_FILES+=usr/libexec/bsdconfig/070.usermgmt/userinput
 # 20140307: bsdconfig groupmgmt rewrite
 OLD_FILES+=usr/libexec/bsdconfig/070.usermgmt/groupinput
 # 20140223: Remove libyaml
 OLD_FILES+=usr/lib/private/libyaml.a
 OLD_FILES+=usr/lib/private/libyaml.so
 OLD_LIBS+=usr/lib/private/libyaml.so.1
 OLD_FILES+=usr/lib/private/libyaml_p.a
 OLD_FILES+=usr/lib32/private/libyaml.a
 OLD_FILES+=usr/lib32/private/libyaml.so
 OLD_LIBS+=usr/lib32/private/libyaml.so.1
 OLD_FILES+=usr/lib32/private/libyaml_p.a
 # 20140216: new clang import which bumps version from 3.3 to 3.4.
 OLD_FILES+=usr/bin/llvm-prof
 OLD_FILES+=usr/include/clang/3.3/__wmmintrin_aes.h
 OLD_FILES+=usr/include/clang/3.3/__wmmintrin_pclmul.h
 OLD_FILES+=usr/include/clang/3.3/altivec.h
 OLD_FILES+=usr/include/clang/3.3/ammintrin.h
 OLD_FILES+=usr/include/clang/3.3/avx2intrin.h
 OLD_FILES+=usr/include/clang/3.3/avxintrin.h
 OLD_FILES+=usr/include/clang/3.3/bmi2intrin.h
 OLD_FILES+=usr/include/clang/3.3/bmiintrin.h
 OLD_FILES+=usr/include/clang/3.3/cpuid.h
 OLD_FILES+=usr/include/clang/3.3/emmintrin.h
 OLD_FILES+=usr/include/clang/3.3/f16cintrin.h
 OLD_FILES+=usr/include/clang/3.3/fma4intrin.h
 OLD_FILES+=usr/include/clang/3.3/fmaintrin.h
 OLD_FILES+=usr/include/clang/3.3/immintrin.h
 OLD_FILES+=usr/include/clang/3.3/lzcntintrin.h
 OLD_FILES+=usr/include/clang/3.3/mm3dnow.h
 OLD_FILES+=usr/include/clang/3.3/mm_malloc.h
 OLD_FILES+=usr/include/clang/3.3/mmintrin.h
 OLD_FILES+=usr/include/clang/3.3/module.map
 OLD_FILES+=usr/include/clang/3.3/nmmintrin.h
 OLD_FILES+=usr/include/clang/3.3/pmmintrin.h
 OLD_FILES+=usr/include/clang/3.3/popcntintrin.h
 OLD_FILES+=usr/include/clang/3.3/prfchwintrin.h
 OLD_FILES+=usr/include/clang/3.3/rdseedintrin.h
 OLD_FILES+=usr/include/clang/3.3/rtmintrin.h
 OLD_FILES+=usr/include/clang/3.3/smmintrin.h
 OLD_FILES+=usr/include/clang/3.3/tmmintrin.h
 OLD_FILES+=usr/include/clang/3.3/wmmintrin.h
 OLD_FILES+=usr/include/clang/3.3/x86intrin.h
 OLD_FILES+=usr/include/clang/3.3/xmmintrin.h
 OLD_FILES+=usr/include/clang/3.3/xopintrin.h
 OLD_FILES+=usr/share/man/man1/llvm-prof.1.gz
 OLD_FILES+=usr/share/man/man1/llvm-ranlib.1.gz
 OLD_DIRS+=usr/include/clang/3.3
 # 20140216: nve(4) removed
 OLD_FILES+=usr/share/man/man4/if_nve.4.gz
 OLD_FILES+=usr/share/man/man4/nve.4.gz
 # 20140205: Open Firmware device moved
 OLD_FILES+=usr/include/dev/ofw/ofw_nexus.h
 # 20140128: libelf and libdwarf import
 OLD_LIBS+=usr/lib/libelf.so.1
 OLD_LIBS+=usr/lib32/libelf.so.1
 OLD_LIBS+=usr/lib/libdwarf.so.3
 OLD_LIBS+=usr/lib32/libdwarf.so.3
 # 20140123: apicvar header moved to x86
 OLD_FILES+=usr/include/machine/apicvar.h
 # 20131215: libcam version bumped
 OLD_LIBS+=lib/libcam.so.6 usr/lib32/libcam.so.6
 # 20131202: libcapsicum and libcasper moved to /lib/
 OLD_LIBS+=usr/lib/libcapsicum.so.0
 OLD_LIBS+=usr/lib/libcasper.so.0
 # 20131109: extattr(2) mlinks fixed
 OLD_FILES+=usr/share/man/man2/extattr_delete_list.2.gz
 OLD_FILES+=usr/share/man/man2/extattr_get_list.2.gz
 # 20131107: example files removed
 OLD_FILES+=usr/share/examples/libusb20/aux.c
 OLD_FILES+=usr/share/examples/libusb20/aux.h
 # 20131105: tzdata 2013h import
 OLD_FILES+=usr/share/zoneinfo/America/Shiprock
 OLD_FILES+=usr/share/zoneinfo/Antarctica/South_Pole
 # 20131103: WITH_LIBICONV_COMPAT removal
 OLD_FILES+=usr/include/_libiconv_compat.h
 OLD_FILES+=usr/lib/libiconv.a
 OLD_FILES+=usr/lib/libiconv.so
 OLD_FILES+=usr/lib/libiconv.so.3
 OLD_FILES+=usr/lib/libiconv_p.a
 OLD_FILES+=usr/lib32/libiconv.a
 OLD_FILES+=usr/lib32/libiconv.so
 OLD_FILES+=usr/lib32/libiconv.so.3
 OLD_FILES+=usr/lib32/libiconv_p.a
 # 20131103: removal of utxrm(8), use 'utx rm' instead.
 OLD_FILES+=usr/sbin/utxrm
 OLD_FILES+=usr/share/man/man8/utxrm.8.gz
 # 20131031: pkg_install has been removed
 OLD_FILES+=etc/periodic/daily/220.backup-pkgdb
 OLD_FILES+=etc/periodic/daily/490.status-pkg-changes
 OLD_FILES+=etc/periodic/security/460.chkportsum
 OLD_FILES+=etc/periodic/weekly/400.status-pkg
 OLD_FILES+=usr/sbin/pkg_add
 OLD_FILES+=usr/sbin/pkg_create
 OLD_FILES+=usr/sbin/pkg_delete
 OLD_FILES+=usr/sbin/pkg_info
 OLD_FILES+=usr/sbin/pkg_updating
 OLD_FILES+=usr/sbin/pkg_version
 OLD_FILES+=usr/share/man/man1/pkg_add.1.gz
 OLD_FILES+=usr/share/man/man1/pkg_create.1.gz
 OLD_FILES+=usr/share/man/man1/pkg_delete.1.gz
 OLD_FILES+=usr/share/man/man1/pkg_info.1.gz
 OLD_FILES+=usr/share/man/man1/pkg_updating.1.gz
 OLD_FILES+=usr/share/man/man1/pkg_version.1.gz
 # 20131030: /etc/keys moved to /usr/share/keys
 OLD_DIRS+=etc/keys
 OLD_DIRS+=etc/keys/pkg
 OLD_DIRS+=etc/keys/pkg/revoked
 OLD_DIRS+=etc/keys/pkg/trusted
 OLD_FILES+=etc/keys/pkg/trusted/pkg.freebsd.org.2013102301
 # 20131028: ng_fec(4) removed
 OLD_FILES+=usr/include/netgraph/ng_fec.h
 OLD_FILES+=usr/share/man/man4/ng_fec.4.gz
 # 20131027: header moved
 OLD_FILES+=usr/include/net/pf_mtag.h
 # 20131023: remove never used iscsi directory
 OLD_DIRS+=usr/share/examples/iscsi
 # 20131021: isf(4) removed
 OLD_FILES+=usr/sbin/isfctl
 OLD_FILES+=usr/share/man/man4/isf.4.gz
 OLD_FILES+=usr/share/man/man8/isfctl.8.gz
 # 20131014: libbsdyml becomes private
 OLD_FILES+=usr/lib/libbsdyml.a
 OLD_FILES+=usr/lib/libbsdyml.so
 OLD_LIBS+=usr/lib/libbsdyml.so.0
 OLD_FILES+=usr/lib/libbsdyml_p.a
 OLD_FILES+=usr/lib32/libbsdyml.a
 OLD_FILES+=usr/lib32/libbsdyml.so
 OLD_LIBS+=usr/lib32/libbsdyml.so.0
 OLD_FILES+=usr/lib32/libbsdyml_p.a
 OLD_FILES+=usr/share/man/man3/libbsdyml.3.gz
 OLD_FILES+=usr/include/bsdyml.h
 # 20131013: Removal of the ATF tools
 OLD_FILES+=etc/atf/FreeBSD.conf
 OLD_FILES+=etc/atf/atf-run.hooks
 OLD_FILES+=etc/atf/common.conf
 OLD_FILES+=usr/bin/atf-config
 OLD_FILES+=usr/bin/atf-report
 OLD_FILES+=usr/bin/atf-run
 OLD_FILES+=usr/bin/atf-version
 OLD_FILES+=usr/share/atf/atf-run.hooks
 OLD_FILES+=usr/share/examples/atf/atf-run.hooks
 OLD_FILES+=usr/share/examples/atf/tests-results.css
 OLD_FILES+=usr/share/man/man1/atf-config.1.gz
 OLD_FILES+=usr/share/man/man1/atf-report.1.gz
 OLD_FILES+=usr/share/man/man1/atf-run.1.gz
 OLD_FILES+=usr/share/man/man1/atf-version.1.gz
 OLD_FILES+=usr/share/man/man5/atf-formats.5.gz
 OLD_FILES+=usr/share/xml/atf/tests-results.dtd
 OLD_FILES+=usr/share/xsl/atf/tests-results.xsl
 OLD_DIRS+=etc/atf
 OLD_DIRS+=usr/share/examples/atf
 OLD_DIRS+=usr/share/xml/atf
 OLD_DIRS+=usr/share/xml
 OLD_DIRS+=usr/share/xsl/atf
 OLD_DIRS+=usr/share/xsl
 # 20131009: freebsd-version moved from /libexec to /bin
 OLD_FILES+=libexec/freebsd-version
 # 20131001: ar and ranlib from binutils not used
 OLD_FILES+=usr/bin/gnu-ar
 OLD_FILES+=usr/bin/gnu-ranlib
 OLD_FILES+=usr/share/man/man1/gnu-ar.1.gz
 OLD_FILES+=usr/share/man/man1/gnu-ranlib.1.gz
 # 20130930: BIND removed from base
 OLD_FILES+=etc/mtree/BIND.chroot.dist
 OLD_FILES+=etc/namedb
 OLD_FILES+=etc/periodic/daily/470.status-named
 OLD_FILES+=usr/bin/dig
 OLD_FILES+=usr/bin/nslookup
 OLD_FILES+=usr/bin/nsupdate
 OLD_DIRS+=usr/include/lwres
 OLD_FILES+=usr/include/lwres/context.h
 OLD_FILES+=usr/include/lwres/int.h
 OLD_FILES+=usr/include/lwres/ipv6.h
 OLD_FILES+=usr/include/lwres/lang.h
 OLD_FILES+=usr/include/lwres/list.h
 OLD_FILES+=usr/include/lwres/lwbuffer.h
 OLD_FILES+=usr/include/lwres/lwpacket.h
 OLD_FILES+=usr/include/lwres/lwres.h
 OLD_FILES+=usr/include/lwres/net.h
 OLD_FILES+=usr/include/lwres/netdb.h
 OLD_FILES+=usr/include/lwres/platform.h
 OLD_FILES+=usr/include/lwres/result.h
 OLD_FILES+=usr/include/lwres/string.h
 OLD_FILES+=usr/include/lwres/version.h
 OLD_FILES+=usr/lib/liblwres.a
 OLD_FILES+=usr/lib/liblwres.so
 OLD_LIBS+=usr/lib/liblwres.so.90
 OLD_FILES+=usr/lib/liblwres_p.a
 OLD_FILES+=usr/sbin/arpaname
 OLD_FILES+=usr/sbin/ddns-confgen
 OLD_FILES+=usr/sbin/dnssec-dsfromkey
 OLD_FILES+=usr/sbin/dnssec-keyfromlabel
 OLD_FILES+=usr/sbin/dnssec-keygen
 OLD_FILES+=usr/sbin/dnssec-revoke
 OLD_FILES+=usr/sbin/dnssec-settime
 OLD_FILES+=usr/sbin/dnssec-signzone
 OLD_FILES+=usr/sbin/dnssec-verify
 OLD_FILES+=usr/sbin/genrandom
 OLD_FILES+=usr/sbin/isc-hmac-fixup
 OLD_FILES+=usr/sbin/lwresd
 OLD_FILES+=usr/sbin/named
 OLD_FILES+=usr/sbin/named-checkconf
 OLD_FILES+=usr/sbin/named-checkzone
 OLD_FILES+=usr/sbin/named-compilezone
 OLD_FILES+=usr/sbin/named-journalprint
 OLD_FILES+=usr/sbin/named.reconfig
 OLD_FILES+=usr/sbin/named.reload
 OLD_FILES+=usr/sbin/nsec3hash
 OLD_FILES+=usr/sbin/rndc
 OLD_FILES+=usr/sbin/rndc-confgen
 OLD_DIRS+=usr/share/doc/bind9
 OLD_FILES+=usr/share/doc/bind9/CHANGES
 OLD_FILES+=usr/share/doc/bind9/COPYRIGHT
 OLD_FILES+=usr/share/doc/bind9/FAQ
 OLD_FILES+=usr/share/doc/bind9/HISTORY
 OLD_FILES+=usr/share/doc/bind9/README
 OLD_DIRS+=usr/share/doc/bind9/arm
 OLD_FILES+=usr/share/doc/bind9/arm/Bv9ARM.ch01.html
 OLD_FILES+=usr/share/doc/bind9/arm/Bv9ARM.ch02.html
 OLD_FILES+=usr/share/doc/bind9/arm/Bv9ARM.ch03.html
 OLD_FILES+=usr/share/doc/bind9/arm/Bv9ARM.ch04.html
 OLD_FILES+=usr/share/doc/bind9/arm/Bv9ARM.ch05.html
 OLD_FILES+=usr/share/doc/bind9/arm/Bv9ARM.ch06.html
 OLD_FILES+=usr/share/doc/bind9/arm/Bv9ARM.ch07.html
 OLD_FILES+=usr/share/doc/bind9/arm/Bv9ARM.ch08.html
 OLD_FILES+=usr/share/doc/bind9/arm/Bv9ARM.ch09.html
 OLD_FILES+=usr/share/doc/bind9/arm/Bv9ARM.ch10.html
 OLD_FILES+=usr/share/doc/bind9/arm/Bv9ARM.html
 OLD_FILES+=usr/share/doc/bind9/arm/Bv9ARM.pdf
 OLD_FILES+=usr/share/doc/bind9/arm/man.arpaname.html
 OLD_FILES+=usr/share/doc/bind9/arm/man.ddns-confgen.html
 OLD_FILES+=usr/share/doc/bind9/arm/man.dig.html
 OLD_FILES+=usr/share/doc/bind9/arm/man.dnssec-dsfromkey.html
 OLD_FILES+=usr/share/doc/bind9/arm/man.dnssec-keyfromlabel.html
 OLD_FILES+=usr/share/doc/bind9/arm/man.dnssec-keygen.html
 OLD_FILES+=usr/share/doc/bind9/arm/man.dnssec-revoke.html
 OLD_FILES+=usr/share/doc/bind9/arm/man.dnssec-settime.html
 OLD_FILES+=usr/share/doc/bind9/arm/man.dnssec-signzone.html
 OLD_FILES+=usr/share/doc/bind9/arm/man.dnssec-verify.html
 OLD_FILES+=usr/share/doc/bind9/arm/man.genrandom.html
 OLD_FILES+=usr/share/doc/bind9/arm/man.host.html
 OLD_FILES+=usr/share/doc/bind9/arm/man.isc-hmac-fixup.html
 OLD_FILES+=usr/share/doc/bind9/arm/man.named-checkconf.html
 OLD_FILES+=usr/share/doc/bind9/arm/man.named-checkzone.html
 OLD_FILES+=usr/share/doc/bind9/arm/man.named-journalprint.html
 OLD_FILES+=usr/share/doc/bind9/arm/man.named.html
 OLD_FILES+=usr/share/doc/bind9/arm/man.nsec3hash.html
 OLD_FILES+=usr/share/doc/bind9/arm/man.nsupdate.html
 OLD_FILES+=usr/share/doc/bind9/arm/man.rndc-confgen.html
 OLD_FILES+=usr/share/doc/bind9/arm/man.rndc.conf.html
 OLD_FILES+=usr/share/doc/bind9/arm/man.rndc.html
 OLD_DIRS+=usr/share/doc/bind9/misc
 OLD_FILES+=usr/share/doc/bind9/misc/dnssec
 OLD_FILES+=usr/share/doc/bind9/misc/format-options.pl
 OLD_FILES+=usr/share/doc/bind9/misc/ipv6
 OLD_FILES+=usr/share/doc/bind9/misc/migration
 OLD_FILES+=usr/share/doc/bind9/misc/migration-4to9
 OLD_FILES+=usr/share/doc/bind9/misc/options
 OLD_FILES+=usr/share/doc/bind9/misc/rfc-compliance
 OLD_FILES+=usr/share/doc/bind9/misc/roadmap
 OLD_FILES+=usr/share/doc/bind9/misc/sdb
 OLD_FILES+=usr/share/doc/bind9/misc/sort-options.pl
 OLD_FILES+=usr/share/man/man1/arpaname.1.gz
 OLD_FILES+=usr/share/man/man1/dig.1.gz
 OLD_FILES+=usr/share/man/man1/nslookup.1.gz
 OLD_FILES+=usr/share/man/man1/nsupdate.1.gz
 OLD_FILES+=usr/share/man/man3/lwres.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_addr_parse.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_buffer.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_buffer_add.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_buffer_back.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_buffer_clear.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_buffer_first.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_buffer_forward.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_buffer_getmem.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_buffer_getuint16.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_buffer_getuint32.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_buffer_getuint8.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_buffer_init.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_buffer_invalidate.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_buffer_putmem.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_buffer_putuint16.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_buffer_putuint32.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_buffer_putuint8.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_buffer_subtract.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_conf_clear.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_conf_get.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_conf_init.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_conf_parse.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_conf_print.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_config.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_context.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_context_allocmem.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_context_create.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_context_destroy.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_context_freemem.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_context_initserial.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_context_nextserial.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_context_sendrecv.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_endhostent.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_endhostent_r.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_freeaddrinfo.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_freehostent.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_gabn.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_gabnrequest_free.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_gabnrequest_parse.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_gabnrequest_render.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_gabnresponse_free.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_gabnresponse_parse.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_gabnresponse_render.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_gai_strerror.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_getaddrinfo.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_getaddrsbyname.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_gethostbyaddr.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_gethostbyaddr_r.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_gethostbyname.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_gethostbyname2.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_gethostbyname_r.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_gethostent.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_gethostent_r.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_getipnode.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_getipnodebyaddr.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_getipnodebyname.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_getnamebyaddr.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_getnameinfo.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_getrrsetbyname.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_gnba.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_gnbarequest_free.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_gnbarequest_parse.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_gnbarequest_render.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_gnbaresponse_free.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_gnbaresponse_parse.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_gnbaresponse_render.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_herror.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_hstrerror.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_inetntop.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_lwpacket_parseheader.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_lwpacket_renderheader.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_net_ntop.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_noop.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_nooprequest_free.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_nooprequest_parse.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_nooprequest_render.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_noopresponse_free.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_noopresponse_parse.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_noopresponse_render.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_packet.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_resutil.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_sethostent.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_sethostent_r.3.gz
 OLD_FILES+=usr/share/man/man3/lwres_string_parse.3.gz
 OLD_FILES+=usr/share/man/man5/named.conf.5.gz
 OLD_FILES+=usr/share/man/man5/rndc.conf.5.gz
 OLD_FILES+=usr/share/man/man8/ddns-confgen.8.gz
 OLD_FILES+=usr/share/man/man8/dnssec-dsfromkey.8.gz
 OLD_FILES+=usr/share/man/man8/dnssec-keyfromlabel.8.gz
 OLD_FILES+=usr/share/man/man8/dnssec-keygen.8.gz
 OLD_FILES+=usr/share/man/man8/dnssec-revoke.8.gz
 OLD_FILES+=usr/share/man/man8/dnssec-settime.8.gz
 OLD_FILES+=usr/share/man/man8/dnssec-signzone.8.gz
 OLD_FILES+=usr/share/man/man8/dnssec-verify.8.gz
 OLD_FILES+=usr/share/man/man8/genrandom.8.gz
 OLD_FILES+=usr/share/man/man8/isc-hmac-fixup.8.gz
 OLD_FILES+=usr/share/man/man8/lwresd.8.gz
 OLD_FILES+=usr/share/man/man8/named-checkconf.8.gz
 OLD_FILES+=usr/share/man/man8/named-checkzone.8.gz
 OLD_FILES+=usr/share/man/man8/named-compilezone.8.gz
 OLD_FILES+=usr/share/man/man8/named-journalprint.8.gz
 OLD_FILES+=usr/share/man/man8/named.8.gz
 OLD_FILES+=usr/share/man/man8/named.reconfig.8.gz
 OLD_FILES+=usr/share/man/man8/named.reload.8.gz
 OLD_FILES+=usr/share/man/man8/nsec3hash.8.gz
 OLD_FILES+=usr/share/man/man8/rndc-confgen.8.gz
 OLD_FILES+=usr/share/man/man8/rndc.8.gz
 OLD_DIRS+=var/named/dev
 OLD_DIRS+=var/named/etc
 OLD_DIRS+=var/named/etc/namedb
 OLD_FILES+=var/named/etc/namedb/PROTO.localhost-v6.rev
 OLD_FILES+=var/named/etc/namedb/PROTO.localhost.rev
 OLD_DIRS+=var/named/etc/namedb/dynamic
 OLD_FILES+=var/named/etc/namedb/make-localhost
 OLD_DIRS+=var/named/etc/namedb/master
 OLD_FILES+=var/named/etc/namedb/master/empty.db
 OLD_FILES+=var/named/etc/namedb/master/localhost-forward.db
 OLD_FILES+=var/named/etc/namedb/master/localhost-reverse.db
 #OLD_FILES+=var/named/etc/namedb/named.conf	# intentionally left out
 OLD_FILES+=var/named/etc/namedb/named.root
 OLD_DIRS+=var/named/etc/namedb/working
 OLD_DIRS+=var/named/etc/namedb/slave
 OLD_DIRS+=var/named/var
 OLD_DIRS+=var/named/var/dump
 OLD_DIRS+=var/named/var/log
 OLD_DIRS+=var/named/var/run
 OLD_DIRS+=var/named/var/run/named
 OLD_DIRS+=var/named/var/stats
 OLD_DIRS+=var/run/named
 # 20130923: example moved
 OLD_FILES+=usr/share/examples/bsdconfig/browse_packages.sh
 # 20130908: libssh becomes private
 OLD_FILES+=usr/lib/libssh.a
 OLD_FILES+=usr/lib/libssh.so
 OLD_LIBS+=usr/lib/libssh.so.5
 OLD_FILES+=usr/lib/libssh_p.a
 OLD_FILES+=usr/lib32/libssh.a
 OLD_FILES+=usr/lib32/libssh.so
 OLD_LIBS+=usr/lib32/libssh.so.5
 OLD_FILES+=usr/lib32/libssh_p.a
 # 20130903: gnupatch is no more
 OLD_FILES+=usr/bin/gnupatch
 OLD_FILES+=usr/share/man/man1/gnupatch.1.gz
 # 20130829: bsdpatch is patch unconditionally
 OLD_FILES+=usr/bin/bsdpatch
 OLD_FILES+=usr/share/man/man1/bsdpatch.1.gz
 # 20130822: bind 9.9.3-P2 import
 OLD_LIBS+=usr/lib/liblwres.so.80
 # 20130814: vm_page_busy(9)
 OLD_FILES+=usr/share/man/man9/vm_page_flash.9.gz
 OLD_FILES+=usr/share/man/man9/vm_page_io.9.gz
 OLD_FILES+=usr/share/man/man9/vm_page_io_finish.9.gz
 OLD_FILES+=usr/share/man/man9/vm_page_io_start.9.gz
 OLD_FILES+=usr/share/man/man9/vm_page_wakeup.9.gz
 # 20130710: libkvm version bump 
 OLD_LIBS+=lib/libkvm.so.5
 OLD_LIBS+=usr/lib32/libkvm.so.5
 # 20130623: dialog update from 1.1 to 1.2
 OLD_LIBS+=usr/lib/libdialog.so.7
 OLD_LIBS+=usr/lib32/libdialog.so.7
 # 20130616: vfs_mount.9 removed
 OLD_FILES+=usr/share/man/man9/vfs_mount.9.gz
 # 20130614: remove CVS from base
 OLD_FILES+=usr/bin/cvs
 OLD_FILES+=usr/bin/cvsbug
 OLD_FILES+=usr/share/doc/psd/28.cvs/paper.ascii.gz
 OLD_FILES+=usr/share/doc/psd/28.cvs/paper.ps.gz
 OLD_DIRS+=usr/share/doc/psd/28.cvs
 OLD_FILES+=usr/share/examples/cvs/contrib/README
 OLD_FILES+=usr/share/examples/cvs/contrib/clmerge
 OLD_FILES+=usr/share/examples/cvs/contrib/cln_hist
 OLD_FILES+=usr/share/examples/cvs/contrib/commit_prep
 OLD_FILES+=usr/share/examples/cvs/contrib/cvs2vendor
 OLD_FILES+=usr/share/examples/cvs/contrib/cvs_acls
 OLD_FILES+=usr/share/examples/cvs/contrib/cvscheck
 OLD_FILES+=usr/share/examples/cvs/contrib/cvscheck.man
 OLD_FILES+=usr/share/examples/cvs/contrib/cvshelp.man
 OLD_FILES+=usr/share/examples/cvs/contrib/descend.man
 OLD_FILES+=usr/share/examples/cvs/contrib/easy-import
 OLD_FILES+=usr/share/examples/cvs/contrib/intro.doc
 OLD_FILES+=usr/share/examples/cvs/contrib/log
 OLD_FILES+=usr/share/examples/cvs/contrib/log_accum
 OLD_FILES+=usr/share/examples/cvs/contrib/mfpipe
 OLD_FILES+=usr/share/examples/cvs/contrib/rcs-to-cvs
 OLD_FILES+=usr/share/examples/cvs/contrib/rcs2log
 OLD_FILES+=usr/share/examples/cvs/contrib/rcslock
 OLD_FILES+=usr/share/examples/cvs/contrib/sccs2rcs
 OLD_DIRS+=usr/share/examples/cvs/contrib
 OLD_DIRS+=usr/share/examples/cvs
 OLD_FILES+=usr/share/info/cvs.info.gz
 OLD_FILES+=usr/share/info/cvsclient.info.gz
 OLD_FILES+=usr/share/man/man1/cvs.1.gz
 OLD_FILES+=usr/share/man/man5/cvs.5.gz
 OLD_FILES+=usr/share/man/man8/cvsbug.8.gz
 # 20130607: WITH_DEBUG_FILES added
 OLD_FILES+=lib/libufs.so.6.symbols
 OLD_FILES+=usr/lib32/libufs.so.6.symbols
 # 20130417: nfs fha moved from nfsserver to nfs
 OLD_FILES+=usr/include/nfsserver/nfs_fha.h
 # 20130411: new clang import which bumps version from 3.2 to 3.3.
 OLD_FILES+=usr/include/clang/3.2/__wmmintrin_aes.h
 OLD_FILES+=usr/include/clang/3.2/__wmmintrin_pclmul.h
 OLD_FILES+=usr/include/clang/3.2/altivec.h
 OLD_FILES+=usr/include/clang/3.2/ammintrin.h
 OLD_FILES+=usr/include/clang/3.2/avx2intrin.h
 OLD_FILES+=usr/include/clang/3.2/avxintrin.h
 OLD_FILES+=usr/include/clang/3.2/bmi2intrin.h
 OLD_FILES+=usr/include/clang/3.2/bmiintrin.h
 OLD_FILES+=usr/include/clang/3.2/cpuid.h
 OLD_FILES+=usr/include/clang/3.2/emmintrin.h
 OLD_FILES+=usr/include/clang/3.2/f16cintrin.h
 OLD_FILES+=usr/include/clang/3.2/fma4intrin.h
 OLD_FILES+=usr/include/clang/3.2/fmaintrin.h
 OLD_FILES+=usr/include/clang/3.2/immintrin.h
 OLD_FILES+=usr/include/clang/3.2/lzcntintrin.h
 OLD_FILES+=usr/include/clang/3.2/mm3dnow.h
 OLD_FILES+=usr/include/clang/3.2/mm_malloc.h
 OLD_FILES+=usr/include/clang/3.2/mmintrin.h
 OLD_FILES+=usr/include/clang/3.2/module.map
 OLD_FILES+=usr/include/clang/3.2/nmmintrin.h
 OLD_FILES+=usr/include/clang/3.2/pmmintrin.h
 OLD_FILES+=usr/include/clang/3.2/popcntintrin.h
 OLD_FILES+=usr/include/clang/3.2/rtmintrin.h
 OLD_FILES+=usr/include/clang/3.2/smmintrin.h
 OLD_FILES+=usr/include/clang/3.2/tmmintrin.h
 OLD_FILES+=usr/include/clang/3.2/wmmintrin.h
 OLD_FILES+=usr/include/clang/3.2/x86intrin.h
 OLD_FILES+=usr/include/clang/3.2/xmmintrin.h
 OLD_FILES+=usr/include/clang/3.2/xopintrin.h
 OLD_DIRS+=usr/include/clang/3.2
 # 20130404: legacy ATA stack removed
 OLD_FILES+=etc/periodic/daily/405.status-ata-raid
 OLD_FILES+=rescue/atacontrol
 OLD_FILES+=sbin/atacontrol
 OLD_FILES+=usr/share/man/man8/atacontrol.8.gz
 OLD_FILES+=usr/share/man/man4/atapicam.4.gz
 OLD_FILES+=usr/share/man/man4/ataraid.4.gz
 OLD_FILES+=usr/sbin/burncd
 OLD_FILES+=usr/share/man/man8/burncd.8.gz
 # 20130316: vinum.4 removed
 OLD_FILES+=usr/share/man/man4/vinum.4.gz
 # 20130312: fortunes-o removed
 OLD_FILES+=usr/share/games/fortune/fortunes-o
 OLD_FILES+=usr/share/games/fortune/fortunes-o.dat
 # 20130311: Ports are no more available via cvsup
 OLD_FILES+=usr/share/examples/cvsup/ports-supfile
 OLD_FILES+=usr/share/examples/cvsup/refuse
 OLD_FILES+=usr/share/examples/cvsup/refuse.README
 # 20130309: NWFS and NCP supports removed
 OLD_FILES+=usr/bin/ncplist
 OLD_FILES+=usr/bin/ncplogin
 OLD_FILES+=usr/bin/ncplogout
 OLD_FILES+=usr/include/fs/nwfs/nwfs.h
 OLD_FILES+=usr/include/fs/nwfs/nwfs_mount.h
 OLD_FILES+=usr/include/fs/nwfs/nwfs_node.h
 OLD_FILES+=usr/include/fs/nwfs/nwfs_subr.h
 OLD_DIRS+=usr/include/fs/nwfs
 OLD_FILES+=usr/include/netncp/ncp.h
 OLD_FILES+=usr/include/netncp/ncp_cfg.h
 OLD_FILES+=usr/include/netncp/ncp_conn.h
 OLD_FILES+=usr/include/netncp/ncp_file.h
 OLD_FILES+=usr/include/netncp/ncp_lib.h
 OLD_FILES+=usr/include/netncp/ncp_ncp.h
 OLD_FILES+=usr/include/netncp/ncp_nls.h
 OLD_FILES+=usr/include/netncp/ncp_rcfile.h
 OLD_FILES+=usr/include/netncp/ncp_rq.h
 OLD_FILES+=usr/include/netncp/ncp_sock.h
 OLD_FILES+=usr/include/netncp/ncp_subr.h
 OLD_FILES+=usr/include/netncp/ncp_user.h
 OLD_FILES+=usr/include/netncp/ncpio.h
 OLD_FILES+=usr/include/netncp/nwerror.h
 OLD_DIRS+=usr/include/netncp
 OLD_FILES+=usr/lib/libncp.a
 OLD_FILES+=usr/lib/libncp.so
 OLD_LIBS+=usr/lib/libncp.so.4
 OLD_FILES+=usr/lib/libncp_p.a
 OLD_FILES+=usr/lib32/libncp.a
 OLD_FILES+=usr/lib32/libncp.so
 OLD_LIBS+=usr/lib32/libncp.so.4
 OLD_FILES+=usr/lib32/libncp_p.a
 OLD_FILES+=usr/sbin/mount_nwfs
 OLD_FILES+=usr/share/examples/nwclient/dot.nwfsrc
 OLD_FILES+=usr/share/examples/nwclient/nwfs.sh.sample
 OLD_DIRS+=usr/share/examples/nwclient
 OLD_FILES+=usr/share/man/man1/ncplist.1.gz
 OLD_FILES+=usr/share/man/man1/ncplogin.1.gz
 OLD_FILES+=usr/share/man/man1/ncplogout.1.gz
 OLD_FILES+=usr/share/man/man8/mount_nwfs.8.gz
 # 20130302: NTFS support removed
 OLD_FILES+=rescue/mount_ntfs
 OLD_FILES+=sbin/mount_ntfs
 OLD_FILES+=usr/include/fs/ntfs/ntfs.h
 OLD_FILES+=usr/include/fs/ntfs/ntfs_compr.h
 OLD_FILES+=usr/include/fs/ntfs/ntfs_ihash.h
 OLD_FILES+=usr/include/fs/ntfs/ntfs_inode.h
 OLD_FILES+=usr/include/fs/ntfs/ntfs_subr.h
 OLD_FILES+=usr/include/fs/ntfs/ntfs_vfsops.h
 OLD_FILES+=usr/include/fs/ntfs/ntfsmount.h
 OLD_DIRS+=usr/include/fs/ntfs
 OLD_FILES+=usr/share/man/man8/mount_ntfs.8.gz
 # 20130302: PORTALFS support removed
 OLD_FILES+=usr/include/fs/portalfs/portal.h
 OLD_DIRS+=usr/include/fs/portalfs
 OLD_FILES+=usr/sbin/mount_portalfs
 OLD_FILES+=usr/share/examples/portal/README
 OLD_FILES+=usr/share/examples/portal/portal.conf
 OLD_DIRS+=usr/share/examples/portal
 OLD_FILES+=usr/share/man/man8/mount_portalfs.8.gz
 # 20130302: CODAFS support removed
 OLD_FILES+=usr/share/man/man4/coda.4.gz
 # 20130302: XFS support removed
 OLD_FILES+=usr/share/man/man5/xfs.5.gz
 # 20130302: Capsicum overhaul
 OLD_FILES+=usr/share/man/man2/cap_getrights.2.gz
 OLD_FILES+=usr/share/man/man2/cap_new.2.gz
 # 20130213: OpenSSL 1.0.1e import
 OLD_FILES+=usr/share/openssl/man/man3/EVP_PKEY_verifyrecover.3.gz
 OLD_FILES+=usr/share/openssl/man/man3/EVP_PKEY_verifyrecover_init.3.gz
 # 20130116: removed long unused directories for .1aout section manpages
 OLD_FILES+=usr/share/man/en.ISO8859-1/man1aout
 OLD_FILES+=usr/share/man/en.UTF-8/man1aout
 OLD_DIRS+=usr/share/man/man1aout
 OLD_DIRS+=usr/share/man/cat1aout
 OLD_DIRS+=usr/share/man/en.ISO8859-1/cat1aout
 OLD_DIRS+=usr/share/man/en.UTF-8/cat1aout
 # 20130103: gnats-supfile removed
 OLD_FILES+=usr/share/examples/cvsup/gnats-supfile
 # 20121230: libdisk removed
 OLD_FILES+=usr/share/man/man3/libdisk.3.gz usr/include/libdisk.h
 OLD_FILES+=usr/lib/libdisk.a usr/lib32/libdisk.a
 # 20121230: remove wrongly created directories for auditdistd
 OLD_DIRS+=var/dist
 OLD_DIRS+=var/remote
 # 20121114: zpool-features manual page moved from section 5 to 7
 OLD_FILES+=usr/share/man/man5/zpool-features.5.gz
 # 20121022: remove harp, hfa and idt man page
 OLD_FILES+=usr/share/man/man4/harp.4.gz
 OLD_FILES+=usr/share/man/man4/hfa.4.gz
 OLD_FILES+=usr/share/man/man4/idt.4.gz
 OLD_FILES+=usr/share/man/man4/if_idt.4.gz
 # 20121022: VFS_LOCK_GIANT elimination
 OLD_FILES+=usr/share/man/man9/VFS_LOCK_GIANT.9.gz
 OLD_FILES+=usr/share/man/man9/VFS_UNLOCK_GIANT.9.gz
 # 20121004: remove incomplete unwind.h
 OLD_FILES+=usr/include/clang/3.2/unwind.h
 # 20120910: NetBSD compat shims removed
 OLD_FILES+=usr/include/cam/scsi/scsi_low_pisa.h
 OLD_FILES+=usr/include/sys/device_port.h
 # 20120909: doc and www supfiles removed
 OLD_FILES+=usr/share/examples/cvsup/doc-supfile
 OLD_FILES+=usr/share/examples/cvsup/www-supfile
 # 20120908: pf cleanup
 OLD_FILES+=usr/include/net/if_pflow.h
 # 20120816: new clang import which bumps version from 3.1 to 3.2
 OLD_FILES+=usr/bin/llvm-ld
 OLD_FILES+=usr/bin/llvm-stub
 OLD_FILES+=usr/include/clang/3.1/altivec.h
 OLD_FILES+=usr/include/clang/3.1/avx2intrin.h
 OLD_FILES+=usr/include/clang/3.1/avxintrin.h
 OLD_FILES+=usr/include/clang/3.1/bmi2intrin.h
 OLD_FILES+=usr/include/clang/3.1/bmiintrin.h
 OLD_FILES+=usr/include/clang/3.1/cpuid.h
 OLD_FILES+=usr/include/clang/3.1/emmintrin.h
 OLD_FILES+=usr/include/clang/3.1/fma4intrin.h
 OLD_FILES+=usr/include/clang/3.1/immintrin.h
 OLD_FILES+=usr/include/clang/3.1/lzcntintrin.h
 OLD_FILES+=usr/include/clang/3.1/mm3dnow.h
 OLD_FILES+=usr/include/clang/3.1/mm_malloc.h
 OLD_FILES+=usr/include/clang/3.1/mmintrin.h
 OLD_FILES+=usr/include/clang/3.1/module.map
 OLD_FILES+=usr/include/clang/3.1/nmmintrin.h
 OLD_FILES+=usr/include/clang/3.1/pmmintrin.h
 OLD_FILES+=usr/include/clang/3.1/popcntintrin.h
 OLD_FILES+=usr/include/clang/3.1/smmintrin.h
 OLD_FILES+=usr/include/clang/3.1/tmmintrin.h
 OLD_FILES+=usr/include/clang/3.1/unwind.h
 OLD_FILES+=usr/include/clang/3.1/wmmintrin.h
 OLD_FILES+=usr/include/clang/3.1/x86intrin.h
 OLD_FILES+=usr/include/clang/3.1/xmmintrin.h
 OLD_DIRS+=usr/include/clang/3.1
 OLD_FILES+=usr/share/man/man1/llvm-ld.1.gz
 # 20120712: OpenSSL 1.0.1c import
 OLD_LIBS+=lib/libcrypto.so.6
 OLD_LIBS+=usr/lib/libssl.so.6
 OLD_LIBS+=usr/lib32/libcrypto.so.6
 OLD_LIBS+=usr/lib32/libssl.so.6
 OLD_FILES+=usr/include/openssl/aes_locl.h
 OLD_FILES+=usr/include/openssl/bio_lcl.h
 OLD_FILES+=usr/include/openssl/e_os.h
 OLD_FILES+=usr/include/openssl/fips.h
 OLD_FILES+=usr/include/openssl/fips_rand.h
 OLD_FILES+=usr/include/openssl/pq_compat.h
 OLD_FILES+=usr/include/openssl/tmdiff.h
 OLD_FILES+=usr/include/openssl/ui_locl.h
 OLD_FILES+=usr/share/openssl/man/man3/CRYPTO_set_id_callback.3.gz
 # 20120621: remove old man page
 OLD_FILES+=usr/share/man/man8/vnconfig.8.gz
 # 20120619: TOE support updated
 OLD_FILES+=usr/include/netinet/toedev.h
 # 20120613: auth.conf removed
 OLD_FILES+=etc/auth.conf
 OLD_FILES+=usr/share/examples/etc/auth.conf
 OLD_FILES+=usr/share/man/man3/auth.3.gz
 OLD_FILES+=usr/share/man/man3/auth_getval.3.gz
 OLD_FILES+=usr/share/man/man5/auth.conf.5.gz
 # 20120530: kde pam lives now in ports
 OLD_FILES+=etc/pam.d/kde
 # 20120521: byacc import
 OLD_FILES+=usr/bin/yyfix
 OLD_FILES+=usr/share/man/man1/yyfix.1.gz
 # 20120505: new clang import installed a redundant internal header
 OLD_FILES+=usr/include/clang/3.1/stdalign.h
 # 20120428: MD2 removed from libmd
 OLD_LIBS+=lib/libmd.so.5
 OLD_FILES+=usr/include/md2.h
 OLD_LIBS+=usr/lib32/libmd.so.5
 OLD_FILES+=usr/share/man/man3/MD2Data.3.gz
 OLD_FILES+=usr/share/man/man3/MD2End.3.gz
 OLD_FILES+=usr/share/man/man3/MD2File.3.gz
 OLD_FILES+=usr/share/man/man3/MD2FileChunk.3.gz
 OLD_FILES+=usr/share/man/man3/MD2Final.3.gz
 OLD_FILES+=usr/share/man/man3/MD2Init.3.gz
 OLD_FILES+=usr/share/man/man3/MD2Update.3.gz
 OLD_FILES+=usr/share/man/man3/md2.3.gz
 # 20120425: libusb version bump (r234684)
 OLD_LIBS+=usr/lib/libusb.so.2
 OLD_LIBS+=usr/lib32/libusb.so.2
 OLD_FILES+=usr/share/man/man3/libsub_get_active_config_descriptor.3.gz
 # 20120415: new clang import which bumps version from 3.0 to 3.1
 OLD_FILES+=usr/include/clang/3.0/altivec.h
 OLD_FILES+=usr/include/clang/3.0/avxintrin.h
 OLD_FILES+=usr/include/clang/3.0/emmintrin.h
 OLD_FILES+=usr/include/clang/3.0/immintrin.h
 OLD_FILES+=usr/include/clang/3.0/mm3dnow.h
 OLD_FILES+=usr/include/clang/3.0/mm_malloc.h
 OLD_FILES+=usr/include/clang/3.0/mmintrin.h
 OLD_FILES+=usr/include/clang/3.0/nmmintrin.h
 OLD_FILES+=usr/include/clang/3.0/pmmintrin.h
 OLD_FILES+=usr/include/clang/3.0/smmintrin.h
 OLD_FILES+=usr/include/clang/3.0/tmmintrin.h
 OLD_FILES+=usr/include/clang/3.0/wmmintrin.h
 OLD_FILES+=usr/include/clang/3.0/x86intrin.h
 OLD_FILES+=usr/include/clang/3.0/xmmintrin.h
 OLD_DIRS+=usr/include/clang/3.0
 # 20120412: BIND 9.8.1 release notes removed
 OLD_FILES+=usr/share/doc/bind9/RELEASE-NOTES-BIND-9.8.1.pdf
 OLD_FILES+=usr/share/doc/bind9/RELEASE-NOTES-BIND-9.8.1.txt
 OLD_FILES+=usr/share/doc/bind9/RELEASE-NOTES-BIND-9.8.1.html
 OLD_FILES+=usr/share/doc/bind9/release-notes.css
 # 20120330: legacy(4) moved to x86
 OLD_FILES+=usr/include/machine/legacyvar.h
 # 20120324: MPI headers updated
 OLD_FILES+=usr/include/dev/mpt/mpilib/mpi_inb.h
 # 20120322: hwpmc_mips24k.h removed
 OLD_FILES+=usr/include/dev/hwpmc/hwpmc_mips24k.h
 # 20120322: Update heimdal to 1.5.1.
 OLD_FILES+=usr/include/krb5-v4compat.h \
 	usr/include/krb_err.h \
 	usr/include/hdb-private.h \
         usr/share/man/man3/krb5_addresses.3.gz \
         usr/share/man/man3/krb5_cc_cursor.3.gz \
         usr/share/man/man3/krb5_cc_ops.3.gz \
         usr/share/man/man3/krb5_config.3.gz \
         usr/share/man/man3/krb5_config_get_int_default.3.gz \
         usr/share/man/man3/krb5_context.3.gz \
         usr/share/man/man3/krb5_data.3.gz \
         usr/share/man/man3/krb5_err.3.gz \
         usr/share/man/man3/krb5_errx.3.gz \
         usr/share/man/man3/krb5_keyblock.3.gz \
         usr/share/man/man3/krb5_keytab_entry.3.gz \
         usr/share/man/man3/krb5_kt_cursor.3.gz \
         usr/share/man/man3/krb5_kt_ops.3.gz \
         usr/share/man/man3/krb5_set_warn_dest.3.gz \
         usr/share/man/man3/krb5_verr.3.gz \
         usr/share/man/man3/krb5_verrx.3.gz \
         usr/share/man/man3/krb5_vwarnx.3.gz \
         usr/share/man/man3/krb5_warn.3.gz \
         usr/share/man/man3/krb5_warnx.3.gz
 OLD_LIBS+=usr/lib/libasn1.so.10 \
 	usr/lib/libhdb.so.10 \
 	usr/lib/libheimntlm.so.10 \
 	usr/lib/libhx509.so.10 \
 	usr/lib/libkadm5clnt.so.10 \
 	usr/lib/libkadm5srv.so.10 \
 	usr/lib/libkafs5.so.10 \
 	usr/lib/libkrb5.so.10 \
 	usr/lib/libroken.so.10 \
 	usr/lib32/libasn1.so.10 \
 	usr/lib32/libhdb.so.10 \
 	usr/lib32/libheimntlm.so.10 \
 	usr/lib32/libhx509.so.10 \
 	usr/lib32/libkadm5clnt.so.10 \
 	usr/lib32/libkadm5srv.so.10 \
 	usr/lib32/libkafs5.so.10 \
 	usr/lib32/libkrb5.so.10 \
 	usr/lib32/libroken.so.10
 # 20120309: Remove fifofs header files.
 OLD_FILES+=usr/include/fs/fifofs/fifo.h
 OLD_DIRS+=usr/include/fs/fifofs
 # 20120304: xlocale cleanup
 OLD_FILES+=usr/include/_xlocale_ctype.h
 # 20120225: libarchive 3.0.3
 OLD_FILES+=usr/share/man/man3/archive_read_data_into_buffer.3.gz \
 	usr/share/man/man3/archive_read_support_compression_all.3.gz \
 	usr/share/man/man3/archive_read_support_compression_bzip2.3.gz \
 	usr/share/man/man3/archive_read_support_compression_compress.3.gz \
 	usr/share/man/man3/archive_read_support_compression_gzip.3.gz \
 	usr/share/man/man3/archive_read_support_compression_lzma.3.gz \
 	usr/share/man/man3/archive_read_support_compression_none.3.gz \
 	usr/share/man/man3/archive_read_support_compression_program.3.gz \
 	usr/share/man/man3/archive_read_support_compression_program_signature.3.gz \
 	usr/share/man/man3/archive_read_support_compression_xz.3.gz \
 	usr/share/man/man3/archive_write_set_callbacks.3.gz \
 	usr/share/man/man3/archive_write_set_compression_bzip2.3.gz \
 	usr/share/man/man3/archive_write_set_compression_compress.3.gz \
 	usr/share/man/man3/archive_write_set_compression_gzip.3.gz \
 	usr/share/man/man3/archive_write_set_compression_none.3.gz \
 	usr/share/man/man3/archive_write_set_compression_program.3.gz
 OLD_LIBS+=usr/lib/libarchive.so.5
 OLD_LIBS+=usr/lib32/libarchive.so.5
 # 20120113: removal of wtmpcvt(1)
 OLD_FILES+=usr/bin/wtmpcvt
 OLD_FILES+=usr/share/man/man1/wtmpcvt.1.gz
 # 20111214: eventtimers(7) moved to eventtimers(4)
 OLD_FILES+=usr/share/man/man7/eventtimers.7.gz
 # 20111125: amd(4) removed
 OLD_FILES+=usr/share/man/man4/amd.4.gz
 # 20111125: libodialog removed
 OLD_FILES+=usr/lib/libodialog.a
 OLD_FILES+=usr/lib/libodialog.so
 OLD_LIBS+=usr/lib/libodialog.so.7
 OLD_FILES+=usr/lib/libodialog_p.a
 OLD_FILES+=usr/lib32/libodialog.a
 OLD_FILES+=usr/lib32/libodialog.so
 OLD_LIBS+=usr/lib32/libodialog.so.7
 OLD_FILES+=usr/lib32/libodialog_p.a
 # 20110930: sysinstall removed
 OLD_FILES+=usr/sbin/sysinstall
 OLD_FILES+=usr/share/man/man8/sysinstall.8.gz
 OLD_FILES+=usr/lib/libftpio.a
 OLD_FILES+=usr/lib/libftpio.so
 OLD_LIBS+=usr/lib/libftpio.so.8
 OLD_FILES+=usr/lib/libftpio_p.a
 OLD_FILES+=usr/lib32/libftpio.a
 OLD_FILES+=usr/lib32/libftpio.so
 OLD_LIBS+=usr/lib32/libftpio.so.8
 OLD_FILES+=usr/lib32/libftpio_p.a
 OLD_FILES+=usr/include/ftpio.h
 OLD_FILES+=usr/share/man/man3/ftpio.3.gz
 # 20110915: rename congestion control manpages
 OLD_FILES+=usr/share/man/man9/cc.9.gz
 # 20110831: atomic page flags operations
 OLD_FILES+=usr/share/man/man9/vm_page_flag.9.gz
 OLD_FILES+=usr/share/man/man9/vm_page_flag_clear.9.gz
 OLD_FILES+=usr/share/man/man9/vm_page_flag_set.9.gz
 # 20110828: library version bump for 9.0
 OLD_LIBS+=lib/libcam.so.5
 OLD_LIBS+=lib/libpcap.so.7
 OLD_LIBS+=lib/libufs.so.5
 OLD_LIBS+=usr/lib/libbsnmp.so.5
 OLD_LIBS+=usr/lib/libdwarf.so.2
 OLD_LIBS+=usr/lib/libopie.so.6
 OLD_LIBS+=usr/lib/librtld_db.so.1
 OLD_LIBS+=usr/lib/libtacplus.so.4
 OLD_LIBS+=usr/lib32/libcam.so.5
 OLD_LIBS+=usr/lib32/libpcap.so.7
 OLD_LIBS+=usr/lib32/libufs.so.5
 OLD_LIBS+=usr/lib32/libbsnmp.so.5
 OLD_LIBS+=usr/lib32/libdwarf.so.2
 OLD_LIBS+=usr/lib32/libopie.so.6
 OLD_LIBS+=usr/lib32/librtld_db.so.1
 OLD_LIBS+=usr/lib32/libtacplus.so.4
 # 20110817: no more acd.4, ad.4, afd.4 and ast.4
 OLD_FILES+=usr/share/man/man4/acd.4.gz
 OLD_FILES+=usr/share/man/man4/ad.4.gz
 OLD_FILES+=usr/share/man/man4/afd.4.gz
 OLD_FILES+=usr/share/man/man4/ast.4.gz
 # 20110718: no longer useful in the age of rc.d
 OLD_FILES+=usr/sbin/named.reconfig
 OLD_FILES+=usr/sbin/named.reload
 OLD_FILES+=usr/share/man/man8/named.reconfig.8.gz
 OLD_FILES+=usr/share/man/man8/named.reload.8.gz
 # 20110716: bind 9.8.0 import
 OLD_LIBS+=usr/lib/liblwres.so.50
 OLD_FILES+=usr/share/doc/bind9/KNOWN-DEFECTS
 OLD_FILES+=usr/share/doc/bind9/NSEC3-NOTES
 OLD_FILES+=usr/share/doc/bind9/README.idnkit
 OLD_FILES+=usr/share/doc/bind9/README.pkcs11
 # 20110709: vm_map_clean.9 -> vm_map_sync.9
 OLD_FILES+=usr/share/man/man9/vm_map_clean.9.gz
 # 20110709: Catch up with removal of these functions.
 OLD_FILES+=usr/share/man/man9/vm_page_copy.9.gz
 OLD_FILES+=usr/share/man/man9/vm_page_protect.9.gz
 OLD_FILES+=usr/share/man/man9/vm_page_zero_fill.9.gz
 # 20110707: script no longer needed by /etc/rc.d/nfsd
 OLD_FILES+=etc/rc.d/nfsserver
 # 20110705: files moved so both NFS clients can share them
 OLD_FILES+=usr/include/nfsclient/krpc.h
 OLD_FILES+=usr/include/nfsclient/nfsdiskless.h
 # 20110705: the switch of default NFS client to the new one
 OLD_FILES+=sbin/mount_newnfs
 OLD_FILES+=usr/share/man/man8/mount_newnfs.8.gz
 OLD_FILES+=usr/include/nfsclient/nfs_kdtrace.h
 # 20110628: calendar.msk removed
 OLD_FILES+=usr/share/calendar/ru_RU.KOI8-R/calendar.msk
 # 20110517: libpkg removed
 OLD_FILES+=usr/include/pkg.h
 OLD_FILES+=usr/lib/libpkg.a
 OLD_FILES+=usr/lib/libpkg.so
 OLD_LIBS+=usr/lib/libpkg.so.0
 OLD_FILES+=usr/lib/libpkg_p.a
 OLD_FILES+=usr/lib32/libpkg.a
 OLD_FILES+=usr/lib32/libpkg.so
 OLD_LIBS+=usr/lib32/libpkg.so.0
 OLD_FILES+=usr/lib32/libpkg_p.a
 # 20110517: libsbuf version bump
 OLD_LIBS+=lib/libsbuf.so.5
 OLD_LIBS+=usr/lib32/libsbuf.so.5
 # 20110502: new clang import which bumps version from 2.9 to 3.0
 OLD_FILES+=usr/include/clang/2.9/emmintrin.h
 OLD_FILES+=usr/include/clang/2.9/mm_malloc.h
 OLD_FILES+=usr/include/clang/2.9/mmintrin.h
 OLD_FILES+=usr/include/clang/2.9/pmmintrin.h
 OLD_FILES+=usr/include/clang/2.9/tmmintrin.h
 OLD_FILES+=usr/include/clang/2.9/xmmintrin.h
 OLD_DIRS+=usr/include/clang/2.9
 # 20110417: removal of Objective-C support
 OLD_FILES+=usr/include/objc/encoding.h
 OLD_FILES+=usr/include/objc/hash.h
 OLD_FILES+=usr/include/objc/NXConstStr.h
 OLD_FILES+=usr/include/objc/objc-api.h
 OLD_FILES+=usr/include/objc/objc-decls.h
 OLD_FILES+=usr/include/objc/objc-list.h
 OLD_FILES+=usr/include/objc/objc.h
 OLD_FILES+=usr/include/objc/Object.h
 OLD_FILES+=usr/include/objc/Protocol.h
 OLD_FILES+=usr/include/objc/runtime.h
 OLD_FILES+=usr/include/objc/sarray.h
 OLD_FILES+=usr/include/objc/thr.h
 OLD_FILES+=usr/include/objc/typedstream.h
 OLD_FILES+=usr/lib/libobjc.a
 OLD_FILES+=usr/lib/libobjc.so
 OLD_FILES+=usr/lib/libobjc_p.a
 OLD_FILES+=usr/libexec/cc1obj
 OLD_LIBS+=usr/lib/libobjc.so.4
 OLD_DIRS+=usr/include/objc
 OLD_FILES+=usr/lib32/libobjc.a
 OLD_FILES+=usr/lib32/libobjc.so
 OLD_FILES+=usr/lib32/libobjc_p.a
 OLD_LIBS+=usr/lib32/libobjc.so.4
 # 20110331: firmware.img created at build time
 OLD_FILES+=usr/share/examples/kld/firmware/fwimage/firmware.img
 # 20110224: sticky.8 -> sticky.7
 OLD_FILES+=usr/share/man/man8/sticky.8.gz
 # 20110220: new clang import which bumps version from 2.8 to 2.9
 OLD_FILES+=usr/include/clang/2.8/emmintrin.h
 OLD_FILES+=usr/include/clang/2.8/mm_malloc.h
 OLD_FILES+=usr/include/clang/2.8/mmintrin.h
 OLD_FILES+=usr/include/clang/2.8/pmmintrin.h
 OLD_FILES+=usr/include/clang/2.8/tmmintrin.h
 OLD_FILES+=usr/include/clang/2.8/xmmintrin.h
 OLD_DIRS+=usr/include/clang/2.8
 # 20110119: netinet/sctp_cc_functions.h removed
 OLD_FILES+=usr/include/netinet/sctp_cc_functions.h
 # 20110119: Remove SYSCTL_*X* sysctl additions.
 OLD_FILES+=usr/share/man/man9/SYSCTL_XINT.9.gz \
     usr/share/man/man9/SYSCTL_XLONG.9.gz
 
 # 20110112: Update dialog to new version, rename old libdialog to libodialog,
 #     removing associated man pages and header files.
 OLD_FILES+=usr/share/man/man3/draw_shadow.3.gz \
     usr/share/man/man3/draw_box.3.gz usr/share/man/man3/line_edit.3.gz \
     usr/share/man/man3/strheight.3.gz usr/share/man/man3/strwidth.3.gz \
     usr/share/man/man3/dialog_create_rc.3.gz \
     usr/share/man/man3/dialog_yesno.3.gz usr/share/man/man3/dialog_noyes.3.gz \
     usr/share/man/man3/dialog_prgbox.3.gz \
     usr/share/man/man3/dialog_textbox.3.gz usr/share/man/man3/dialog_menu.3.gz \
     usr/share/man/man3/dialog_checklist.3.gz \
     usr/share/man/man3/dialog_radiolist.3.gz \
     usr/share/man/man3/dialog_inputbox.3.gz \
     usr/share/man/man3/dialog_clear_norefresh.3.gz \
     usr/share/man/man3/dialog_clear.3.gz usr/share/man/man3/dialog_update.3.gz \
     usr/share/man/man3/dialog_fselect.3.gz \
     usr/share/man/man3/dialog_notify.3.gz \
     usr/share/man/man3/dialog_mesgbox.3.gz \
     usr/share/man/man3/dialog_gauge.3.gz usr/share/man/man3/init_dialog.3.gz \
     usr/share/man/man3/end_dialog.3.gz usr/share/man/man3/use_helpfile.3.gz \
     usr/share/man/man3/use_helpline.3.gz usr/share/man/man3/get_helpline.3.gz \
     usr/share/man/man3/restore_helpline.3.gz \
     usr/share/man/man3/dialog_msgbox.3.gz \
     usr/share/man/man3/dialog_ftree.3.gz usr/share/man/man3/dialog_tree.3.gz \
     usr/share/examples/dialog/README usr/share/examples/dialog/checklist \
     usr/share/examples/dialog/ftreebox usr/share/examples/dialog/infobox \
     usr/share/examples/dialog/inputbox usr/share/examples/dialog/menubox \
     usr/share/examples/dialog/msgbox usr/share/examples/dialog/prgbox \
     usr/share/examples/dialog/radiolist usr/share/examples/dialog/textbox \
     usr/share/examples/dialog/treebox usr/share/examples/dialog/yesno \
     usr/share/examples/libdialog/Makefile usr/share/examples/libdialog/check1.c\
     usr/share/examples/libdialog/check2.c usr/share/examples/libdialog/check3.c\
     usr/share/examples/libdialog/dselect.c \
     usr/share/examples/libdialog/fselect.c \
     usr/share/examples/libdialog/ftree1.c \
     usr/share/examples/libdialog/ftree1.test \
     usr/share/examples/libdialog/ftree2.c \
     usr/share/examples/libdialog/ftree2.test \
     usr/share/examples/libdialog/gauge.c usr/share/examples/libdialog/input1.c \
     usr/share/examples/libdialog/input2.c usr/share/examples/libdialog/menu1.c \
     usr/share/examples/libdialog/menu2.c usr/share/examples/libdialog/menu3.c \
     usr/share/examples/libdialog/msg.c usr/share/examples/libdialog/prgbox.c \
     usr/share/examples/libdialog/radio1.c usr/share/examples/libdialog/radio2.c\
     usr/share/examples/libdialog/radio3.c usr/share/examples/libdialog/text.c \
     usr/share/examples/libdialog/tree.c usr/share/examples/libdialog/yesno.c
 OLD_DIRS+=usr/share/examples/libdialog usr/share/examples/dialog
 # 20101114: Remove long-obsolete MAKEDEV.8
 OLD_FILES+=usr/share/man/man8/MAKEDEV.8.gz
 # 20101112: vgonel(9) has gone to private API a while ago
 OLD_FILES+=usr/share/man/man9/vgonel.9.gz
 # 20101112: removed gasp.info
 OLD_FILES+=usr/share/info/gasp.info.gz
 # 20101109: machine/mutex.h removed
 OLD_FILES+=usr/include/machine/mutex.h
 # 20101109: headers moved from machine/ to x86/
 .if ${TARGET_ARCH} == "amd64" || ${TARGET_ARCH} == "i386"
 OLD_FILES+=usr/include/machine/mptable.h
 .endif
 # 20101101: headers moved from machine/ to x86/
 .if ${TARGET_ARCH} == "amd64" || ${TARGET_ARCH} == "i386"
 OLD_FILES+=usr/include/machine/apicreg.h
 OLD_FILES+=usr/include/machine/mca.h
 .endif
 # 20101020: catch up with vm_page_sleep_if_busy rename
 OLD_FILES+=usr/share/man/man9/vm_page_sleep_busy.9.gz
 # 20101018: taskqueue(9) updates
 OLD_FILES+=usr/share/man/man9/taskqueue_find.9.gz
 # 20101011: removed subblock.h from liblzma
 OLD_FILES+=usr/include/lzma/subblock.h
 # 20101002: removed manpath.config
 OLD_FILES+=etc/manpath.config
 OLD_FILES+=usr/share/examples/etc/manpath.config
 # 20100910: renamed sbuf_overflowed to sbuf_error
 OLD_FILES+=usr/share/man/man9/sbuf_overflowed.9.gz
 # 20100815: retired last traces of chooseproc(9)
 OLD_FILES+=usr/share/man/man9/chooseproc.9.gz
 # 20100806: removal of unused libcompat routines
 OLD_FILES+=usr/share/man/man3/ascftime.3.gz
 OLD_FILES+=usr/share/man/man3/cfree.3.gz
 OLD_FILES+=usr/share/man/man3/cftime.3.gz
 OLD_FILES+=usr/share/man/man3/getpw.3.gz
 # 20100801: tzdata2010k import
 OLD_FILES+=usr/share/zoneinfo/Pacific/Ponape
 OLD_FILES+=usr/share/zoneinfo/Pacific/Truk
 # 20100725: acpi_aiboost(4) removal.
 OLD_FILES+=usr/share/man/man4/acpi_aiboost.4.gz
 # 20100724: nfsclient/nfs_lock.h moved to nfs/nfs_lock.h
 OLD_FILES+=usr/include/nfsclient/nfs_lock.h
 # 20100720: new clang import which bumps version from 2.0 to 2.8
 OLD_FILES+=usr/include/clang/2.0/emmintrin.h
 OLD_FILES+=usr/include/clang/2.0/mm_malloc.h
 OLD_FILES+=usr/include/clang/2.0/mmintrin.h
 OLD_FILES+=usr/include/clang/2.0/pmmintrin.h
 OLD_FILES+=usr/include/clang/2.0/tmmintrin.h
 OLD_FILES+=usr/include/clang/2.0/xmmintrin.h
 OLD_DIRS+=usr/include/clang/2.0
 # 20100706: removed pc-sysinstall's detect-vmware.sh
 OLD_FILES+=usr/share/pc-sysinstall/backend-query/detect-vmware.sh
 # 20100701: [powerpc] removed <machine/intr.h>
 .if ${TARGET_ARCH} == "powerpc"
 OLD_FILES+=usr/include/machine/intr.h
 .endif
 # 20100514: library version bump for versioned symbols for liblzma
 OLD_LIBS+=usr/lib/liblzma.so.0
 OLD_LIBS+=usr/lib32/liblzma.so.0
 # 20100511: move GCC-specific headers to /usr/include/gcc
 .if ${TARGET_ARCH} == "amd64" || ${TARGET_ARCH} == "i386"
 OLD_FILES+=usr/include/emmintrin.h
 OLD_FILES+=usr/include/mm_malloc.h
 OLD_FILES+=usr/include/pmmintrin.h
 OLD_FILES+=usr/include/xmmintrin.h
 .endif
 .if ${TARGET_ARCH} == "amd64" || ${TARGET_ARCH} == "i386" || ${TARGET_ARCH} == "arm"
 OLD_FILES+=usr/include/mmintrin.h
 .endif
 .if ${TARGET_ARCH} == "powerpc"
 OLD_FILES+=usr/include/altivec.h
 OLD_FILES+=usr/include/ppc-asm.h
 OLD_FILES+=usr/include/spe.h
 .endif
 # 20100416: [mips] removed <machine/psl.h>
 .if ${TARGET_ARCH} == "mips"
 OLD_FILES+=usr/include/machine/psl.h
 .endif
 # 20100415: [mips] removed unused headers
 .if ${TARGET_ARCH} == "mips"
 OLD_FILES+=usr/include/machine/archtype.h
 OLD_FILES+=usr/include/machine/segments.h
 OLD_FILES+=usr/include/machine/rm7000.h
 OLD_FILES+=usr/include/machine/defs.h
 OLD_FILES+=usr/include/machine/queue.h
 .endif
 # 20100326: gcpio removal
 OLD_FILES+=usr/bin/gcpio
 OLD_FILES+=usr/share/info/cpio.info.gz
 OLD_FILES+=usr/share/man/man1/gcpio.1.gz
 # 20100322: libz update
 OLD_LIBS+=lib/libz.so.5
 OLD_LIBS+=usr/lib32/libz.so.5
 # 20100314: removal of regexp.h
 OLD_FILES+=usr/include/regexp.h
 OLD_FILES+=usr/share/man/man3/regexp.3.gz
 OLD_FILES+=usr/share/man/man3/regsub.3.gz
 # 20100303: actual removal of utmp.h
 OLD_FILES+=usr/include/utmp.h
 # 20100208: man pages moved
 .if ${TARGET_ARCH} == "i386"
 OLD_FILES+=usr/share/man/man4/i386/alpm.4.gz
 OLD_FILES+=usr/share/man/man4/i386/amdpm.4.gz
 OLD_FILES+=usr/share/man/man4/i386/mcd.4.gz
 OLD_FILES+=usr/share/man/man4/i386/padlock.4.gz
 OLD_FILES+=usr/share/man/man4/i386/pcf.4.gz
 OLD_FILES+=usr/share/man/man4/i386/scd.4.gz
 OLD_FILES+=usr/share/man/man4/i386/viapm.4.gz
 .endif
 # 20100122: move BSDL bc/dc USD documents to /usr/share/doc/usd
 OLD_FILES+=usr/share/doc/papers/bc.ascii.gz
 OLD_FILES+=usr/share/doc/papers/dc.ascii.gz
 # 20100120: replacing GNU bc/dc with BSDL versions
 OLD_FILES+=usr/share/examples/bc/ckbook.b
 OLD_FILES+=usr/share/examples/bc/pi.b
 OLD_FILES+=usr/share/examples/bc/primes.b
 OLD_FILES+=usr/share/examples/bc/twins.b
 OLD_FILES+=usr/share/info/dc.info.gz
 OLD_DIRS+=usr/share/examples/bc
 # 20100114: removal of ttyslot(3)
 OLD_FILES+=usr/share/man/man3/ttyslot.3.gz
 # 20100113: remove utmp.h, replace it by utmpx.h
 OLD_FILES+=usr/share/man/man3/login.3.gz
 OLD_FILES+=usr/share/man/man3/logout.3.gz
 OLD_FILES+=usr/share/man/man3/logwtmp.3.gz
 OLD_FILES+=usr/share/man/man3/ulog_endutxent.3.gz
 OLD_FILES+=usr/share/man/man3/ulog_getutxent.3.gz
 OLD_FILES+=usr/share/man/man3/ulog_getutxline.3.gz
 OLD_FILES+=usr/share/man/man3/ulog_getutxuser.3.gz
 OLD_FILES+=usr/share/man/man3/ulog_pututxline.3.gz
 OLD_FILES+=usr/share/man/man3/ulog_setutxent.3.gz
 OLD_FILES+=usr/share/man/man3/ulog_setutxfile.3.gz
 OLD_FILES+=usr/share/man/man5/lastlog.5.gz
 OLD_FILES+=usr/share/man/man5/utmp.5.gz
 OLD_FILES+=usr/share/man/man5/wtmp.5.gz
 OLD_LIBS+=lib/libutil.so.8
 OLD_LIBS+=usr/lib32/libutil.so.8
 # 20100105: new userland semaphore implementation
 OLD_FILES+=usr/include/sys/semaphore.h
 # 20100103: ntptrace(8) removed
 OLD_FILES+=usr/sbin/ntptrace
 OLD_FILES+=usr/share/man/man8/ntptrace.8.gz
 # 20091229: remove no longer relevant examples
 OLD_FILES+=usr/share/examples/pppd/auth-down.sample
 OLD_FILES+=usr/share/examples/pppd/auth-up.sample
 OLD_FILES+=usr/share/examples/pppd/chap-secrets.sample
 OLD_FILES+=usr/share/examples/pppd/chat.sh.sample
 OLD_FILES+=usr/share/examples/pppd/ip-down.sample
 OLD_FILES+=usr/share/examples/pppd/ip-up.sample
 OLD_FILES+=usr/share/examples/pppd/options.sample
 OLD_FILES+=usr/share/examples/pppd/pap-secrets.sample
 OLD_FILES+=usr/share/examples/pppd/ppp.deny.sample
 OLD_FILES+=usr/share/examples/pppd/ppp.shells.sample
 OLD_DIRS+=usr/share/examples/pppd
 OLD_FILES+=usr/share/examples/slattach/unit-command.sh
 OLD_DIRS+=usr/share/examples/slattach
 OLD_FILES+=usr/share/examples/sliplogin/slip.hosts
 OLD_FILES+=usr/share/examples/sliplogin/slip.login
 OLD_FILES+=usr/share/examples/sliplogin/slip.logout
 OLD_FILES+=usr/share/examples/sliplogin/slip.slparms
 OLD_DIRS+=usr/share/examples/sliplogin
 OLD_FILES+=usr/share/examples/startslip/sldown.sh
 OLD_FILES+=usr/share/examples/startslip/slip.sh
 OLD_FILES+=usr/share/examples/startslip/slup.sh
 OLD_DIRS+=usr/share/examples/startslip
 # 20091202: unify rc.firewall and rc.firewall6.
 OLD_FILES+=etc/rc.d/ip6fw
 OLD_FILES+=etc/rc.firewall6
 OLD_FILES+=usr/share/examples/etc/rc.firewall6
 # 20091117: removal of rc.early(8) link
 OLD_FILES+=usr/share/man/man8/rc.early.8.gz
 # 20091117: usr/share/zoneinfo/GMT link removed
 OLD_FILES+=usr/share/zoneinfo/GMT
 # 20091027: pselect.3 implemented as syscall
 OLD_FILES+=usr/share/man/man3/pselect.3.gz
 # 20091005: fusword.9 and susword.9 removed
 OLD_FILES+=usr/share/man/man9/fusword.9.gz
 OLD_FILES+=usr/share/man/man9/susword.9.gz
 # 20090909: vesa and dpms promoted to be i386/amd64 common
 OLD_FILES+=usr/include/machine/pc/vesa.h
 OLD_FILES+=usr/share/man/man4/i386/dpms.4.gz
 # 20090904: remove lukemftpd
 OLD_FILES+=usr/libexec/lukemftpd
 OLD_FILES+=usr/share/man/man5/ftpd.conf.5.gz
 OLD_FILES+=usr/share/man/man5/ftpusers.5.gz
 OLD_FILES+=usr/share/man/man8/lukemftpd.8.gz
 # 20090902: BSD.{x11,x11-4}.dist are dead and BSD.local.dist lives in ports/
 OLD_FILES+=etc/mtree/BSD.local.dist
 OLD_FILES+=etc/mtree/BSD.x11.dist
 OLD_FILES+=etc/mtree/BSD.x11-4.dist
 # 20090812: net80211 documentation overhaul
 OLD_FILES+=usr/share/man/man9/ieee80211_add_rates.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_add_xrates.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_alloc_node.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_attach.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_begin_scan.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_cfgget.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_cfgset.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_chan2ieee.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_chan2mode.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_create_ibss.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_crypto_attach.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_crypto_detach.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_decap.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_dump_pkt.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_dup_bss.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_encap.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_end_scan.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_find_node.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_fix_rate.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_free_allnodes.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_ieee2mhz.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_ioctl.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_lookup_node.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_media2rate.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_media_change.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_media_init.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_media_status.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_mhz2ieee.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_next_scan.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_node_attach.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_node_detach.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_node_lateattach.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_print_essid.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_proto_attach.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_proto_detach.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_rate2media.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_recv_mgmt.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_send_mgmt.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_setmode.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_timeout_nodes.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_watchdog.9.gz
 OLD_FILES+=usr/share/man/man9/ieee80211_wep_crypt.9.gz
 # 20090801: vimage.h removed in favour of vnet.h
 OLD_FILES+=usr/include/sys/vimage.h
 # 20101208: libbsnmp was moved to usr/lib
 OLD_LIBS+=lib/libbsnmp.so.5
 # 20090719: library version bump for 8.0
 OLD_LIBS+=lib/libalias.so.6
 OLD_LIBS+=lib/libavl.so.1
 OLD_LIBS+=lib/libbegemot.so.3
 OLD_LIBS+=lib/libbsdxml.so.3
 OLD_LIBS+=lib/libbsnmp.so.4
 OLD_LIBS+=lib/libcam.so.4
 OLD_LIBS+=lib/libcrypt.so.4
 OLD_LIBS+=lib/libcrypto.so.5
 OLD_LIBS+=lib/libctf.so.1
 OLD_LIBS+=lib/libdevstat.so.6
 OLD_LIBS+=lib/libdtrace.so.1
 OLD_LIBS+=lib/libedit.so.6
 OLD_LIBS+=lib/libgeom.so.4
 OLD_LIBS+=lib/libipsec.so.3
 OLD_LIBS+=lib/libipx.so.4
 OLD_LIBS+=lib/libkiconv.so.3
 OLD_LIBS+=lib/libkvm.so.4
 OLD_LIBS+=lib/libmd.so.4
 OLD_LIBS+=lib/libncurses.so.7
 OLD_LIBS+=lib/libncursesw.so.7
 OLD_LIBS+=lib/libnvpair.so.1
 OLD_LIBS+=lib/libpcap.so.6
 OLD_LIBS+=lib/libreadline.so.7
 OLD_LIBS+=lib/libsbuf.so.4
 OLD_LIBS+=lib/libufs.so.4
 OLD_LIBS+=lib/libumem.so.1
 OLD_LIBS+=lib/libutil.so.7
 OLD_LIBS+=lib/libuutil.so.1
 OLD_LIBS+=lib/libz.so.4
 OLD_LIBS+=lib/libzfs.so.1
 OLD_LIBS+=lib/libzpool.so.1
 OLD_LIBS+=usr/lib/libarchive.so.4
 OLD_LIBS+=usr/lib/libauditd.so.4
 OLD_LIBS+=usr/lib/libbluetooth.so.3
 OLD_LIBS+=usr/lib/libbsm.so.2
 OLD_LIBS+=usr/lib/libbz2.so.3
 OLD_LIBS+=usr/lib/libcalendar.so.4
 OLD_LIBS+=usr/lib/libcom_err.so.4
 OLD_LIBS+=usr/lib/libdevinfo.so.4
 OLD_LIBS+=usr/lib/libdialog.so.6
 OLD_LIBS+=usr/lib/libdwarf.so.1
 OLD_LIBS+=usr/lib/libfetch.so.5
 OLD_LIBS+=usr/lib/libform.so.4
 OLD_LIBS+=usr/lib/libformw.so.4
 OLD_LIBS+=usr/lib/libftpio.so.7
 OLD_LIBS+=usr/lib/libgnuregex.so.4
 OLD_LIBS+=usr/lib/libgpib.so.2
 OLD_LIBS+=usr/lib/libhistory.so.7
 OLD_LIBS+=usr/lib/libmagic.so.3
 OLD_LIBS+=usr/lib/libmemstat.so.2
 OLD_LIBS+=usr/lib/libmenu.so.4
 OLD_LIBS+=usr/lib/libmenuw.so.4
 OLD_LIBS+=usr/lib/libmilter.so.4
 OLD_LIBS+=usr/lib/libncp.so.3
 OLD_LIBS+=usr/lib/libnetgraph.so.3
 OLD_LIBS+=usr/lib/libngatm.so.3
 OLD_LIBS+=usr/lib/libobjc.so.3
 OLD_LIBS+=usr/lib/libopie.so.5
 OLD_LIBS+=usr/lib/libpam.so.4
 OLD_LIBS+=usr/lib/libpanel.so.4
 OLD_LIBS+=usr/lib/libpanelw.so.4
 OLD_LIBS+=usr/lib/libpmc.so.4
 OLD_LIBS+=usr/lib/libproc.so.1
 OLD_LIBS+=usr/lib/libradius.so.3
 OLD_LIBS+=usr/lib/librpcsvc.so.4
 OLD_LIBS+=usr/lib/libsdp.so.3
 OLD_LIBS+=usr/lib/libsmb.so.3
 OLD_LIBS+=usr/lib/libssh.so.4
 OLD_LIBS+=usr/lib/libssl.so.5
 OLD_LIBS+=usr/lib/libtacplus.so.3
 OLD_LIBS+=usr/lib/libugidfw.so.3
 OLD_LIBS+=usr/lib/libusb.so.1
 OLD_LIBS+=usr/lib/libusbhid.so.3
 OLD_LIBS+=usr/lib/libvgl.so.5
 OLD_LIBS+=usr/lib/libwrap.so.5
 OLD_LIBS+=usr/lib/libypclnt.so.3
 OLD_LIBS+=usr/lib/pam_chroot.so.4
 OLD_LIBS+=usr/lib/pam_deny.so.4
 OLD_LIBS+=usr/lib/pam_echo.so.4
 OLD_LIBS+=usr/lib/pam_exec.so.4
 OLD_LIBS+=usr/lib/pam_ftpusers.so.4
 OLD_LIBS+=usr/lib/pam_group.so.4
 OLD_LIBS+=usr/lib/pam_guest.so.4
 OLD_LIBS+=usr/lib/pam_krb5.so.4
 OLD_LIBS+=usr/lib/pam_ksu.so.4
 OLD_LIBS+=usr/lib/pam_lastlog.so.4
 OLD_LIBS+=usr/lib/pam_login_access.so.4
 OLD_LIBS+=usr/lib/pam_nologin.so.4
 OLD_LIBS+=usr/lib/pam_opie.so.4
 OLD_LIBS+=usr/lib/pam_opieaccess.so.4
 OLD_LIBS+=usr/lib/pam_passwdqc.so.4
 OLD_LIBS+=usr/lib/pam_permit.so.4
 OLD_LIBS+=usr/lib/pam_radius.so.4
 OLD_LIBS+=usr/lib/pam_rhosts.so.4
 OLD_LIBS+=usr/lib/pam_rootok.so.4
 OLD_LIBS+=usr/lib/pam_securetty.so.4
 OLD_LIBS+=usr/lib/pam_self.so.4
 OLD_LIBS+=usr/lib/pam_ssh.so.4
 OLD_LIBS+=usr/lib/pam_tacplus.so.4
 OLD_LIBS+=usr/lib/pam_unix.so.4
 OLD_LIBS+=usr/lib/snmp_atm.so.5
 OLD_LIBS+=usr/lib/snmp_bridge.so.5
 OLD_LIBS+=usr/lib/snmp_hostres.so.5
 OLD_LIBS+=usr/lib/snmp_mibII.so.5
 OLD_LIBS+=usr/lib/snmp_netgraph.so.5
 OLD_LIBS+=usr/lib/snmp_pf.so.5
 OLD_LIBS+=usr/lib32/libalias.so.6
 OLD_LIBS+=usr/lib32/libarchive.so.4
 OLD_LIBS+=usr/lib32/libauditd.so.4
 OLD_LIBS+=usr/lib32/libavl.so.1
 OLD_LIBS+=usr/lib32/libbegemot.so.3
 OLD_LIBS+=usr/lib32/libbluetooth.so.3
 OLD_LIBS+=usr/lib32/libbsdxml.so.3
 OLD_LIBS+=usr/lib32/libbsm.so.2
 OLD_LIBS+=usr/lib32/libbsnmp.so.4
 OLD_LIBS+=usr/lib32/libbz2.so.3
 OLD_LIBS+=usr/lib32/libcalendar.so.4
 OLD_LIBS+=usr/lib32/libcam.so.4
 OLD_LIBS+=usr/lib32/libcom_err.so.4
 OLD_LIBS+=usr/lib32/libcrypt.so.4
 OLD_LIBS+=usr/lib32/libcrypto.so.5
 OLD_LIBS+=usr/lib32/libctf.so.1
 OLD_LIBS+=usr/lib32/libdevinfo.so.4
 OLD_LIBS+=usr/lib32/libdevstat.so.6
 OLD_LIBS+=usr/lib32/libdialog.so.6
 OLD_LIBS+=usr/lib32/libdtrace.so.1
 OLD_LIBS+=usr/lib32/libdwarf.so.1
 OLD_LIBS+=usr/lib32/libedit.so.6
 OLD_LIBS+=usr/lib32/libfetch.so.5
 OLD_LIBS+=usr/lib32/libform.so.4
 OLD_LIBS+=usr/lib32/libformw.so.4
 OLD_LIBS+=usr/lib32/libftpio.so.7
 OLD_LIBS+=usr/lib32/libgeom.so.4
 OLD_LIBS+=usr/lib32/libgnuregex.so.4
 OLD_LIBS+=usr/lib32/libgpib.so.2
 OLD_LIBS+=usr/lib32/libhistory.so.7
 OLD_LIBS+=usr/lib32/libipsec.so.3
 OLD_LIBS+=usr/lib32/libipx.so.4
 OLD_LIBS+=usr/lib32/libkiconv.so.3
 OLD_LIBS+=usr/lib32/libkvm.so.4
 OLD_LIBS+=usr/lib32/libmagic.so.3
 OLD_LIBS+=usr/lib32/libmd.so.4
 OLD_LIBS+=usr/lib32/libmemstat.so.2
 OLD_LIBS+=usr/lib32/libmenu.so.4
 OLD_LIBS+=usr/lib32/libmenuw.so.4
 OLD_LIBS+=usr/lib32/libmilter.so.4
 OLD_LIBS+=usr/lib32/libncp.so.3
 OLD_LIBS+=usr/lib32/libncurses.so.7
 OLD_LIBS+=usr/lib32/libncursesw.so.7
 OLD_LIBS+=usr/lib32/libnetgraph.so.3
 OLD_LIBS+=usr/lib32/libngatm.so.3
 OLD_LIBS+=usr/lib32/libnvpair.so.1
 OLD_LIBS+=usr/lib32/libobjc.so.3
 OLD_LIBS+=usr/lib32/libopie.so.5
 OLD_LIBS+=usr/lib32/libpam.so.4
 OLD_LIBS+=usr/lib32/libpanel.so.4
 OLD_LIBS+=usr/lib32/libpanelw.so.4
 OLD_LIBS+=usr/lib32/libpcap.so.6
 OLD_LIBS+=usr/lib32/libpmc.so.4
 OLD_LIBS+=usr/lib32/libproc.so.1
 OLD_LIBS+=usr/lib32/libradius.so.3
 OLD_LIBS+=usr/lib32/libreadline.so.7
 OLD_LIBS+=usr/lib32/librpcsvc.so.4
 OLD_LIBS+=usr/lib32/libsbuf.so.4
 OLD_LIBS+=usr/lib32/libsdp.so.3
 OLD_LIBS+=usr/lib32/libsmb.so.3
 OLD_LIBS+=usr/lib32/libssh.so.4
 OLD_LIBS+=usr/lib32/libssl.so.5
 OLD_LIBS+=usr/lib32/libtacplus.so.3
 OLD_LIBS+=usr/lib32/libufs.so.4
 OLD_LIBS+=usr/lib32/libugidfw.so.3
 OLD_LIBS+=usr/lib32/libumem.so.1
 OLD_LIBS+=usr/lib32/libusb.so.1
 OLD_LIBS+=usr/lib32/libusbhid.so.3
 OLD_LIBS+=usr/lib32/libutil.so.7
 OLD_LIBS+=usr/lib32/libuutil.so.1
 OLD_LIBS+=usr/lib32/libvgl.so.5
 OLD_LIBS+=usr/lib32/libwrap.so.5
 OLD_LIBS+=usr/lib32/libypclnt.so.3
 OLD_LIBS+=usr/lib32/libz.so.4
 OLD_LIBS+=usr/lib32/libzfs.so.1
 OLD_LIBS+=usr/lib32/libzpool.so.1
 OLD_LIBS+=usr/lib32/pam_chroot.so.4
 OLD_LIBS+=usr/lib32/pam_deny.so.4
 OLD_LIBS+=usr/lib32/pam_echo.so.4
 OLD_LIBS+=usr/lib32/pam_exec.so.4
 OLD_LIBS+=usr/lib32/pam_ftpusers.so.4
 OLD_LIBS+=usr/lib32/pam_group.so.4
 OLD_LIBS+=usr/lib32/pam_guest.so.4
 OLD_LIBS+=usr/lib32/pam_krb5.so.4
 OLD_LIBS+=usr/lib32/pam_ksu.so.4
 OLD_LIBS+=usr/lib32/pam_lastlog.so.4
 OLD_LIBS+=usr/lib32/pam_login_access.so.4
 OLD_LIBS+=usr/lib32/pam_nologin.so.4
 OLD_LIBS+=usr/lib32/pam_opie.so.4
 OLD_LIBS+=usr/lib32/pam_opieaccess.so.4
 OLD_LIBS+=usr/lib32/pam_passwdqc.so.4
 OLD_LIBS+=usr/lib32/pam_permit.so.4
 OLD_LIBS+=usr/lib32/pam_radius.so.4
 OLD_LIBS+=usr/lib32/pam_rhosts.so.4
 OLD_LIBS+=usr/lib32/pam_rootok.so.4
 OLD_LIBS+=usr/lib32/pam_securetty.so.4
 OLD_LIBS+=usr/lib32/pam_self.so.4
 OLD_LIBS+=usr/lib32/pam_ssh.so.4
 OLD_LIBS+=usr/lib32/pam_tacplus.so.4
 OLD_LIBS+=usr/lib32/pam_unix.so.4
 # 20090718: the gdm pam.d file is no longer required.
 OLD_FILES+=etc/pam.d/gdm
 # 20090714: net_add_domain(9) renamed to domain_add(9)
 OLD_FILES+=usr/share/man/man9/net_add_domain.9.gz
 # 20090713: vimage container structs removed.
 OLD_FILES+=usr/include/netinet/vinet.h
 OLD_FILES+=usr/include/netinet6/vinet6.h
 OLD_FILES+=usr/include/netipsec/vipsec.h
 # 20090712: ieee80211.4 -> net80211.4
 OLD_FILES+=usr/share/man/man4/ieee80211.4.gz
 # 20090711: typo fixed, kproc_resume,.9 -> kproc_resume.9
 OLD_FILES+=usr/share/man/man9/kproc_resume,.9.gz
 # 20090709: msgctl.3 msgget.3 msgrcv.3 msgsnd.3 manual pages moved
 OLD_FILES+=usr/share/man/man3/msgctl.3.gz
 OLD_FILES+=usr/share/man/man3/msgget.3.gz
 OLD_FILES+=usr/share/man/man3/msgrcv.3.gz
 OLD_FILES+=usr/share/man/man3/msgsnd.3.gz
 # 20090630: old kernel RPC implementation removal
 OLD_FILES+=usr/include/nfs/rpcv2.h
 # 20090624: update usbdi(9)
 OLD_FILES+=usr/share/man/man9/usbd_abort_default_pipe.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_abort_pipe.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_alloc_buffer.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_alloc_xfer.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_clear_endpoint_stall.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_clear_endpoint_stall_async.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_clear_endpoint_toggle.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_close_pipe.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_device2interface_handle.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_do_request_async.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_do_request_flags_pipe.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_endpoint_count.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_find_edesc.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_find_idesc.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_free_buffer.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_free_xfer.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_get_buffer.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_get_config.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_get_config_desc.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_get_config_desc_full.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_get_config_descriptor.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_get_device_descriptor.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_get_endpoint_descriptor.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_get_interface_altindex.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_get_interface_descriptor.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_get_no_alts.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_get_quirks.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_get_speed.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_get_string.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_get_string_desc.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_get_xfer_status.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_interface2device_handle.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_interface2endpoint_descriptor.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_interface_count.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_open_pipe.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_open_pipe_intr.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_pipe2device_handle.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_set_config_index.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_set_config_no.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_set_interface.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_setup_default_xfer.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_setup_isoc_xfer.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_setup_xfer.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_sync_transfer.9.gz
 OLD_FILES+=usr/share/man/man9/usbd_transfer.9.gz
 OLD_FILES+=usr/share/man/man9/usb_find_desc.9.gz
 # 20090623: number of headers needed for a usb driver reduced
 OLD_FILES+=usr/include/dev/usb/usb_defs.h
 OLD_FILES+=usr/include/dev/usb/usb_error.h
 OLD_FILES+=usr/include/dev/usb/usb_handle_request.h
 OLD_FILES+=usr/include/dev/usb/usb_hid.h
 OLD_FILES+=usr/include/dev/usb/usb_lookup.h
 OLD_FILES+=usr/include/dev/usb/usb_mfunc.h
 OLD_FILES+=usr/include/dev/usb/usb_parse.h
 OLD_FILES+=usr/include/dev/usb/usb_revision.h
 # 20090609: devclass_add_driver is no longer public
 OLD_FILES+=usr/share/man/man9/devclass_add_driver.9.gz
 OLD_FILES+=usr/share/man/man9/devclass_delete_driver.9.gz
 OLD_FILES+=usr/share/man/man9/devclass_find_driver.9.gz
 # 20090605: removal of clists
 OLD_FILES+=usr/include/sys/clist.h
 # 20090602: removal of window(1)
 OLD_FILES+=usr/bin/window
 OLD_FILES+=usr/share/man/man1/window.1.gz
 # 20090531: bind 9.6.1rc1 import
 OLD_LIBS+=usr/lib/liblwres.so.30
 # 20090530: removal of early.sh
 OLD_FILES+=etc/rc.d/early.sh
 # 20090527: renaming of S{LIST,TAILQ}_REMOVE_NEXT() to _REMOVE_AFTER()
 OLD_FILES+=usr/share/man/man3/SLIST_REMOVE_NEXT.3.gz
 OLD_FILES+=usr/share/man/man3/STAILQ_REMOVE_NEXT.3.gz
 # 20090527: removal of legacy USB stack
 OLD_FILES+=usr/include/legacy/dev/usb/dsbr100io.h
 OLD_FILES+=usr/include/legacy/dev/usb/ehcireg.h
 OLD_FILES+=usr/include/legacy/dev/usb/ehcivar.h
 OLD_FILES+=usr/include/legacy/dev/usb/hid.h
 OLD_FILES+=usr/include/legacy/dev/usb/if_urtwreg.h
 OLD_FILES+=usr/include/legacy/dev/usb/if_urtwvar.h
 OLD_FILES+=usr/include/legacy/dev/usb/ohcireg.h
 OLD_FILES+=usr/include/legacy/dev/usb/ohcivar.h
 OLD_FILES+=usr/include/legacy/dev/usb/rio500_usb.h
 OLD_FILES+=usr/include/legacy/dev/usb/rt2573_ucode.h
 OLD_FILES+=usr/include/legacy/dev/usb/sl811hsreg.h
 OLD_FILES+=usr/include/legacy/dev/usb/sl811hsvar.h
 OLD_FILES+=usr/include/legacy/dev/usb/ubser.h
 OLD_FILES+=usr/include/legacy/dev/usb/ucomvar.h
 OLD_FILES+=usr/include/legacy/dev/usb/udbp.h
 OLD_FILES+=usr/include/legacy/dev/usb/uftdireg.h
 OLD_FILES+=usr/include/legacy/dev/usb/ugraphire_rdesc.h
 OLD_FILES+=usr/include/legacy/dev/usb/uhcireg.h
 OLD_FILES+=usr/include/legacy/dev/usb/uhcivar.h
 OLD_FILES+=usr/include/legacy/dev/usb/usb.h
 OLD_FILES+=usr/include/legacy/dev/usb/usb_mem.h
 OLD_FILES+=usr/include/legacy/dev/usb/usb_port.h
 OLD_FILES+=usr/include/legacy/dev/usb/usb_quirks.h
 OLD_FILES+=usr/include/legacy/dev/usb/usbcdc.h
 OLD_FILES+=usr/include/legacy/dev/usb/usbdi.h
 OLD_FILES+=usr/include/legacy/dev/usb/usbdi_util.h
 OLD_FILES+=usr/include/legacy/dev/usb/usbdivar.h
 OLD_FILES+=usr/include/legacy/dev/usb/usbhid.h
 OLD_FILES+=usr/include/legacy/dev/usb/uxb360gp_rdesc.h
 OLD_DIRS+=usr/include/legacy/dev/usb
 OLD_DIRS+=usr/include/legacy/dev
 OLD_DIRS+=usr/include/legacy
 # 20090526: removal of makekey(8)
 OLD_FILES+=usr/libexec/makekey
 OLD_FILES+=usr/share/man/man8/makekey.8.gz
 # 20090522: removal of University of Michigan NFSv4 client
 OLD_FILES+=etc/rc.d/idmapd
 OLD_FILES+=sbin/idmapd
 OLD_FILES+=sbin/mount_nfs4
 OLD_FILES+=usr/share/man/man8/idmapd.8.gz
 OLD_FILES+=usr/share/man/man8/mount_nfs4.8.gz
 # 20090513: removal of legacy versions of USB network interface drivers
 OLD_FILES+=usr/include/legacy/dev/usb/if_upgtvar.h
 OLD_FILES+=usr/include/legacy/dev/usb/usb_ethersubr.h
 # 20090417: removal of legacy versions of USB network interface drivers
 OLD_FILES+=usr/include/legacy/dev/usb/if_auereg.h
 OLD_FILES+=usr/include/legacy/dev/usb/if_axereg.h
 OLD_FILES+=usr/include/legacy/dev/usb/if_cdcereg.h
 OLD_FILES+=usr/include/legacy/dev/usb/if_cuereg.h
 OLD_FILES+=usr/include/legacy/dev/usb/if_kuereg.h
 OLD_FILES+=usr/include/legacy/dev/usb/if_ruereg.h
 OLD_FILES+=usr/include/legacy/dev/usb/if_rumreg.h
 OLD_FILES+=usr/include/legacy/dev/usb/if_rumvar.h
 OLD_FILES+=usr/include/legacy/dev/usb/if_udavreg.h
 OLD_FILES+=usr/include/legacy/dev/usb/if_uralreg.h
 OLD_FILES+=usr/include/legacy/dev/usb/if_uralvar.h
 OLD_FILES+=usr/include/legacy/dev/usb/if_zydfw.h
 OLD_FILES+=usr/include/legacy/dev/usb/if_zydreg.h
 OLD_FILES+=usr/include/legacy/dev/usb/kue_fw.h
 # 20090416: removal of ar(4), ray(4), sr(4), raycontrol(8)
 OLD_FILES+=usr/sbin/raycontrol
 OLD_FILES+=usr/share/man/man4/i386/ar.4.gz
 OLD_FILES+=usr/share/man/man4/i386/ray.4.gz
 OLD_FILES+=usr/share/man/man4/i386/sr.4.gz
 OLD_FILES+=usr/share/man/man8/raycontrol.8.gz
 # 20090410: VOP_LEASE.9 removed
 OLD_FILES+=usr/share/man/man9/VOP_LEASE.9.gz
 # 20090406: usb_sw_transfer.h removed
 OLD_FILES+=usr/include/dev/usb/usb_sw_transfer.h
 # 20090405: removal of if_ppp(4) and if_sl(4)
 OLD_FILES+=sbin/slattach rescue/slattach
 OLD_FILES+=sbin/startslip rescue/startslip
 OLD_FILES+=usr/include/net/if_ppp.h
 OLD_FILES+=usr/include/net/if_pppvar.h
 OLD_FILES+=usr/include/net/if_slvar.h
 OLD_FILES+=usr/include/net/ppp_comp.h
 OLD_FILES+=usr/include/net/slip.h
 OLD_FILES+=usr/sbin/sliplogin
 OLD_FILES+=usr/sbin/slstat
 OLD_FILES+=usr/sbin/pppd
 OLD_FILES+=usr/sbin/pppstats
 OLD_FILES+=usr/share/man/man1/startslip.1.gz
 OLD_FILES+=usr/share/man/man4/if_ppp.4.gz
 OLD_FILES+=usr/share/man/man4/if_sl.4.gz
 OLD_FILES+=usr/share/man/man4/ppp.4.gz
 OLD_FILES+=usr/share/man/man4/sl.4.gz
 OLD_FILES+=usr/share/man/man8/pppd.8.gz
 OLD_FILES+=usr/share/man/man8/pppstats.8.gz
 OLD_FILES+=usr/share/man/man8/slattach.8.gz
 OLD_FILES+=usr/share/man/man8/slip.8.gz
 OLD_FILES+=usr/share/man/man8/sliplogin.8.gz
 OLD_FILES+=usr/share/man/man8/slstat.8.gz
 # 20090321: libpcap upgraded to 1.0.0
 OLD_LIBS+=lib/libpcap.so.5
 OLD_LIBS+=usr/lib32/libpcap.so.5
 # 20090319: uscanner(4) has been removed
 OLD_FILES+=usr/share/man/man4/uscanner.4.gz
 # 20090313: k8temp(4) renamed to amdtemp(4)
 OLD_FILES+=usr/share/man/man4/k8temp.4.gz
 # 20090308: libusb.so.1 renamed
 OLD_LIBS+=usr/lib/libusb20.so.1
 OLD_FILES+=usr/lib/libusb20.a
 OLD_FILES+=usr/lib/libusb20.so
 OLD_FILES+=usr/lib/libusb20_p.a
 OLD_FILES+=usr/include/libusb20_compat01.h
 OLD_FILES+=usr/include/libusb20_compat10.h
 OLD_LIBS+=usr/lib32/libusb20.so.1
 OLD_FILES+=usr/lib32/libusb20.a
 OLD_FILES+=usr/lib32/libusb20.so
 OLD_FILES+=usr/lib32/libusb20_p.a
 # 20090226: libmp(3) functions renamed
 OLD_LIBS+=usr/lib/libmp.so.6
 OLD_LIBS+=usr/lib32/libmp.so.6
 # 20090223: changeover of USB stacks
 OLD_FILES+=usr/include/dev/usb2/include/ufm2_ioctl.h
 OLD_FILES+=usr/include/dev/usb2/include/urio2_ioctl.h
 OLD_FILES+=usr/include/dev/usb2/include/usb2_cdc.h
 OLD_FILES+=usr/include/dev/usb2/include/usb2_defs.h
 OLD_FILES+=usr/include/dev/usb2/include/usb2_devid.h
 OLD_FILES+=usr/include/dev/usb2/include/usb2_devtable.h
 OLD_FILES+=usr/include/dev/usb2/include/usb2_endian.h
 OLD_FILES+=usr/include/dev/usb2/include/usb2_error.h
 OLD_FILES+=usr/include/dev/usb2/include/usb2_hid.h
 OLD_FILES+=usr/include/dev/usb2/include/usb2_ioctl.h
 OLD_FILES+=usr/include/dev/usb2/include/usb2_mfunc.h
 OLD_FILES+=usr/include/dev/usb2/include/usb2_revision.h
 OLD_FILES+=usr/include/dev/usb2/include/usb2_standard.h
 OLD_DIRS+=usr/include/dev/usb2/include
 OLD_DIRS+=usr/include/dev/usb2
 OLD_FILES+=usr/include/dev/usb/dsbr100io.h
 OLD_FILES+=usr/include/dev/usb/ehcireg.h
 OLD_FILES+=usr/include/dev/usb/ehcivar.h
 OLD_FILES+=usr/include/dev/usb/hid.h
 OLD_FILES+=usr/include/dev/usb/if_auereg.h
 OLD_FILES+=usr/include/dev/usb/if_axereg.h
 OLD_FILES+=usr/include/dev/usb/if_cdcereg.h
 OLD_FILES+=usr/include/dev/usb/if_cuereg.h
 OLD_FILES+=usr/include/dev/usb/if_kuereg.h
 OLD_FILES+=usr/include/dev/usb/if_ruereg.h
 OLD_FILES+=usr/include/dev/usb/if_rumreg.h
 OLD_FILES+=usr/include/dev/usb/if_rumvar.h
 OLD_FILES+=usr/include/dev/usb/if_udavreg.h
 OLD_FILES+=usr/include/dev/usb/if_upgtvar.h
 OLD_FILES+=usr/include/dev/usb/if_uralreg.h
 OLD_FILES+=usr/include/dev/usb/if_uralvar.h
 OLD_FILES+=usr/include/dev/usb/if_urtwreg.h
 OLD_FILES+=usr/include/dev/usb/if_urtwvar.h
 OLD_FILES+=usr/include/dev/usb/if_zydfw.h
 OLD_FILES+=usr/include/dev/usb/if_zydreg.h
 OLD_FILES+=usr/include/dev/usb/kue_fw.h
 OLD_FILES+=usr/include/dev/usb/ohcireg.h
 OLD_FILES+=usr/include/dev/usb/ohcivar.h
 OLD_FILES+=usr/include/dev/usb/rio500_usb.h
 OLD_FILES+=usr/include/dev/usb/rt2573_ucode.h
 OLD_FILES+=usr/include/dev/usb/sl811hsreg.h
 OLD_FILES+=usr/include/dev/usb/sl811hsvar.h
 OLD_FILES+=usr/include/dev/usb/ubser.h
 OLD_FILES+=usr/include/dev/usb/ucomvar.h
 OLD_FILES+=usr/include/dev/usb/udbp.h
 OLD_FILES+=usr/include/dev/usb/uftdireg.h
 OLD_FILES+=usr/include/dev/usb/ugraphire_rdesc.h
 OLD_FILES+=usr/include/dev/usb/uhcireg.h
 OLD_FILES+=usr/include/dev/usb/uhcivar.h
 OLD_FILES+=usr/include/dev/usb/usb_ethersubr.h
 OLD_FILES+=usr/include/dev/usb/usb_mem.h
 OLD_FILES+=usr/include/dev/usb/usb_port.h
 OLD_FILES+=usr/include/dev/usb/usb_quirks.h
 OLD_FILES+=usr/include/dev/usb/usbcdc.h
 OLD_FILES+=usr/include/dev/usb/usbdivar.h
 OLD_FILES+=usr/include/dev/usb/uxb360gp_rdesc.h
 OLD_FILES+=usr/sbin/usbdevs
 OLD_FILES+=usr/share/man/man8/usbdevs.8.gz
 # 20090203: removal of pccard header files
 OLD_FILES+=usr/include/pccard/cardinfo.h
 OLD_FILES+=usr/include/pccard/cis.h
 OLD_DIRS+=usr/include/pccard
 # 20090203: adding_user.8 moved to adding_user.7
 OLD_FILES+=usr/share/man/man8/adding_user.8.gz
 # 20090122: tzdata2009a import
 OLD_FILES+=usr/share/zoneinfo/Asia/Katmandu
 # 20090102: file 4.26 import
 OLD_FILES+=usr/share/misc/magic.mime
 OLD_FILES+=usr/share/misc/magic.mime.mgc
 # 20081223: bind 9.4.3 import, nsupdate.8 moved to nsupdate.1
 OLD_FILES+=usr/share/man/man8/nsupdate.8.gz
 # 20081223: ipprotosw.h removed
 OLD_FILES+=usr/include/netinet/ipprotosw.h
 # 20081123: vfs_mountedon.9 removed
 OLD_FILES+=usr/share/man/man9/vfs_mountedon.9.gz
 # 20081023: FREE.9 and MALLOC.9 removed
 OLD_FILES+=usr/share/man/man9/FREE.9.gz
 OLD_FILES+=usr/share/man/man9/MALLOC.9.gz
 # 20080928: removal of inaccurate device_ids(9) manual page
 OLD_FILES+=usr/share/man/man9/device_ids.9.gz
 OLD_FILES+=usr/share/man/man9/major.9.gz
 OLD_FILES+=usr/share/man/man9/minor.9.gz
 OLD_FILES+=usr/share/man/man9/umajor.9.gz
 OLD_FILES+=usr/share/man/man9/uminor.9.gz
 # 20080917: removal of manpage for axed kernel primitive suser(9)
 OLD_FILES+=usr/share/man/man9/suser.9.gz
 OLD_FILES+=usr/share/man/man9/suser_cred.9.gz
 # 20080913: pax removed from rescue
 OLD_FILES+=rescue/pax
 # 20080823: removal of unneeded pt_chown, to implement grantpt(3)
 OLD_FILES+=usr/libexec/pt_chown
 # 20080822: ntp 4.2.4p5 import
 OLD_FILES+=usr/share/doc/ntp/driver23.html
 OLD_FILES+=usr/share/doc/ntp/driver24.html
 # 20080821: several man pages moved from man4.i386 to man4
 .if ${TARGET_ARCH} == "i386"
 OLD_FILES+=usr/share/man/man4/i386/acpi_aiboost.4.gz
 OLD_FILES+=usr/share/man/man4/i386/acpi_asus.4.gz
 OLD_FILES+=usr/share/man/man4/i386/acpi_fujitsu.4.gz
 OLD_FILES+=usr/share/man/man4/i386/acpi_ibm.4.gz
 OLD_FILES+=usr/share/man/man4/i386/acpi_panasonic.4.gz
 OLD_FILES+=usr/share/man/man4/i386/acpi_sony.4.gz
 OLD_FILES+=usr/share/man/man4/i386/acpi_toshiba.4.gz
 OLD_FILES+=usr/share/man/man4/i386/ichwd.4.gz
 OLD_FILES+=usr/share/man/man4/i386/if_ndis.4.gz
 OLD_FILES+=usr/share/man/man4/i386/io.4.gz
 OLD_FILES+=usr/share/man/man4/i386/linux.4.gz
 OLD_FILES+=usr/share/man/man4/i386/ndis.4.gz
 .endif
 # 20080820: MPSAFE TTY layer integrated
 OLD_FILES+=usr/include/sys/linedisc.h
 OLD_FILES+=usr/share/man/man3/posix_openpt.3.gz
 # 20080725: sgtty.h removed
 OLD_FILES+=usr/include/sgtty.h
 # 20080706: bsdlabel(8) removed on powerpc
 .if ${TARGET_ARCH} == "powerpc"
 OLD_FILES+=sbin/bsdlabel
 OLD_FILES+=usr/share/man/man8/bsdlabel.8.gz
 .endif
 # 20080704: sbsh(4) removed
 OLD_FILES+=usr/share/man/man4/if_sbsh.4.gz
 OLD_FILES+=usr/share/man/man4/sbsh.4.gz
 # 20080704: cnw(4) removed
 OLD_FILES+=usr/share/man/man4/if_cnw.4.gz
 OLD_FILES+=usr/share/man/man4/cnw.4.gz
 # 20080704: oltr(4) removed
 .if ${TARGET_ARCH} == "i386"
 OLD_FILES+=usr/share/man/man4/i386/if_oltr.4.gz
 OLD_FILES+=usr/share/man/man4/i386/oltr.4.gz
 .endif
 # 20080704: arl(4) removed
 .if ${TARGET_ARCH} == "i386"
 OLD_FILES+=usr/sbin/arlcontrol
 OLD_FILES+=usr/share/man/man4/i386/arl.4.gz
 OLD_FILES+=usr/share/man/man8/arlcontrol.8.gz
 .endif
 # 20080703: sunlabel only for sparc64
 .if ${TARGET_ARCH} != "sparc64"
 OLD_FILES+=sbin/sunlabel
 OLD_FILES+=usr/share/man/man8/sunlabel.8.gz
 .endif
 # 20080701: wpa_supplicant.conf moved to share/examples/etc/
 OLD_FILES+=usr/share/examples/wpa_supplicant/wpa_supplicant.conf
 OLD_DIRS+=usr/share/examples/wpa_supplicant
 # 20080614: pecoff image activator removed
 .if ${TARGET_ARCH} == "i386"
 OLD_FILES+=usr/include/machine/pecoff_machdep.h
 .endif
 # 20080614: sgtty removed
 OLD_FILES+=usr/include/sys/ttychars.h
 OLD_FILES+=usr/include/sys/ttydev.h
 OLD_FILES+=usr/share/man/man3/gtty.3.gz
 OLD_FILES+=usr/share/man/man3/stty.3.gz
 # 20080609: gpt(8) removed
 OLD_FILES+=sbin/gpt
 OLD_FILES+=usr/share/man/man8/gpt.8.gz
 # 20080525: I4B removed
 OLD_FILES+=etc/isdn/answer
 OLD_FILES+=etc/isdn/isdntel
 OLD_FILES+=etc/isdn/record
 OLD_FILES+=etc/isdn/tell
 OLD_FILES+=etc/isdn/tell-record
 OLD_FILES+=etc/isdn/unknown_incoming
 OLD_FILES+=etc/isdn/holidays.D
 OLD_FILES+=etc/isdn/isdnd.rates.A
 OLD_FILES+=etc/isdn/isdnd.rates.D
 OLD_FILES+=etc/isdn/isdnd.rates.F
 OLD_FILES+=etc/isdn/isdnd.rates.L
 OLD_FILES+=etc/isdn/isdnd.rates.UK.BT
 OLD_FILES+=etc/isdn/isdnd.rc.sample
 OLD_FILES+=etc/isdn/isdntel.alias.sample
 OLD_DIRS+=etc/isdn
 OLD_FILES+=etc/rc.d/isdnd
 OLD_FILES+=usr/include/i4b/i4b_cause.h
 OLD_FILES+=usr/include/i4b/i4b_debug.h
 OLD_FILES+=usr/include/i4b/i4b_ioctl.h
 OLD_FILES+=usr/include/i4b/i4b_rbch_ioctl.h
 OLD_FILES+=usr/include/i4b/i4b_tel_ioctl.h
 OLD_FILES+=usr/include/i4b/i4b_trace.h
 OLD_DIRS+=usr/include/i4b
 OLD_FILES+=usr/sbin/dtmfdecode
 OLD_FILES+=usr/sbin/g711conv
 OLD_FILES+=usr/sbin/isdnd
 OLD_FILES+=usr/sbin/isdndebug
 OLD_FILES+=usr/sbin/isdndecode
 OLD_FILES+=usr/sbin/isdnmonitor
 OLD_FILES+=usr/sbin/isdnphone
 OLD_FILES+=usr/sbin/isdntel
 OLD_FILES+=usr/sbin/isdntelctl
 OLD_FILES+=usr/sbin/isdntrace
 OLD_FILES+=usr/share/isdn/0.al
 OLD_FILES+=usr/share/isdn/1.al
 OLD_FILES+=usr/share/isdn/2.al
 OLD_FILES+=usr/share/isdn/3.al
 OLD_FILES+=usr/share/isdn/4.al
 OLD_FILES+=usr/share/isdn/5.al
 OLD_FILES+=usr/share/isdn/6.al
 OLD_FILES+=usr/share/isdn/7.al
 OLD_FILES+=usr/share/isdn/8.al
 OLD_FILES+=usr/share/isdn/9.al
 OLD_FILES+=usr/share/isdn/beep.al
 OLD_FILES+=usr/share/isdn/msg.al
 OLD_DIRS+=usr/share/isdn
 OLD_FILES+=usr/share/man/man1/dtmfdecode.1.gz
 OLD_FILES+=usr/share/man/man1/g711conv.1.gz
 OLD_FILES+=usr/share/man/man4/i4b.4.gz
 OLD_FILES+=usr/share/man/man4/i4bcapi.4.gz
 OLD_FILES+=usr/share/man/man4/i4bctl.4.gz
 OLD_FILES+=usr/share/man/man4/i4bing.4.gz
 OLD_FILES+=usr/share/man/man4/i4bipr.4.gz
 OLD_FILES+=usr/share/man/man4/i4bisppp.4.gz
 OLD_FILES+=usr/share/man/man4/i4bq921.4.gz
 OLD_FILES+=usr/share/man/man4/i4bq931.4.gz
 OLD_FILES+=usr/share/man/man4/i4brbch.4.gz
 OLD_FILES+=usr/share/man/man4/i4btel.4.gz
 OLD_FILES+=usr/share/man/man4/i4btrc.4.gz
 OLD_FILES+=usr/share/man/man4/iavc.4.gz
 OLD_FILES+=usr/share/man/man4/isic.4.gz
 OLD_FILES+=usr/share/man/man4/ifpi.4.gz
 OLD_FILES+=usr/share/man/man4/ifpi2.4.gz
 OLD_FILES+=usr/share/man/man4/ifpnp.4.gz
 OLD_FILES+=usr/share/man/man4/ihfc.4.gz
 OLD_FILES+=usr/share/man/man4/itjc.4.gz
 OLD_FILES+=usr/share/man/man4/iwic.4.gz
 OLD_FILES+=usr/share/man/man5/isdnd.rc.5.gz
 OLD_FILES+=usr/share/man/man5/isdnd.rates.5.gz
 OLD_FILES+=usr/share/man/man5/isdnd.acct.5.gz
 OLD_FILES+=usr/share/man/man8/isdnd.8.gz
 OLD_FILES+=usr/share/man/man8/isdndebug.8.gz
 OLD_FILES+=usr/share/man/man8/isdndecode.8.gz
 OLD_FILES+=usr/share/man/man8/isdnmonitor.8.gz
 OLD_FILES+=usr/share/man/man8/isdnphone.8.gz
 OLD_FILES+=usr/share/man/man8/isdntel.8.gz
 OLD_FILES+=usr/share/man/man8/isdntelctl.8.gz
 OLD_FILES+=usr/share/man/man8/isdntrace.8.gz
 OLD_FILES+=usr/share/examples/isdn/contrib/README
 OLD_FILES+=usr/share/examples/isdn/contrib/anleitung.ppp
 OLD_FILES+=usr/share/examples/isdn/contrib/answer.c
 OLD_FILES+=usr/share/examples/isdn/contrib/answer.sh
 OLD_FILES+=usr/share/examples/isdn/contrib/convert.sh
 OLD_FILES+=usr/share/examples/isdn/contrib/hplay.c
 OLD_FILES+=usr/share/examples/isdn/contrib/i4b-ppp-newbie.txt
 OLD_FILES+=usr/share/examples/isdn/contrib/isdnctl
 OLD_FILES+=usr/share/examples/isdn/contrib/isdnd_acct
 OLD_FILES+=usr/share/examples/isdn/contrib/isdnd_acct.pl
 OLD_FILES+=usr/share/examples/isdn/contrib/isdntelmux.c
 OLD_FILES+=usr/share/examples/isdn/contrib/mrtg-isp0.sh
 OLD_FILES+=usr/share/examples/isdn/i4brunppp/Makefile
 OLD_FILES+=usr/share/examples/isdn/i4brunppp/README
 OLD_FILES+=usr/share/examples/isdn/i4brunppp/i4brunppp-isdnd.rc
 OLD_FILES+=usr/share/examples/isdn/i4brunppp/i4brunppp.8
 OLD_FILES+=usr/share/examples/isdn/i4brunppp/i4brunppp.c
 OLD_FILES+=usr/share/examples/isdn/v21/Makefile
 OLD_FILES+=usr/share/examples/isdn/v21/README
 OLD_FILES+=usr/share/examples/isdn/v21/v21modem.c
 OLD_FILES+=usr/share/examples/isdn/FAQ
 OLD_FILES+=usr/share/examples/isdn/KERNEL
 OLD_FILES+=usr/share/examples/isdn/Overview
 OLD_FILES+=usr/share/examples/isdn/README
 OLD_FILES+=usr/share/examples/isdn/ROADMAP
 OLD_FILES+=usr/share/examples/isdn/ReleaseNotes
 OLD_FILES+=usr/share/examples/isdn/Resources
 OLD_FILES+=usr/share/examples/isdn/SupportedCards
 OLD_FILES+=usr/share/examples/isdn/ThankYou
 OLD_DIRS+=usr/share/examples/isdn/contrib
 OLD_DIRS+=usr/share/examples/isdn/i4brunppp
 OLD_DIRS+=usr/share/examples/isdn/v21
 OLD_DIRS+=usr/share/examples/isdn
 OLD_FILES+=usr/share/examples/ppp/isdnd.rc
 OLD_FILES+=usr/share/examples/ppp/ppp.conf.isdn
 # 20080525: ng_atmpif removed
 OLD_FILES+=usr/include/netgraph/atm/ng_atmpif.h
 OLD_FILES+=usr/share/man/man4/ng_atmpif.4.gz
 # 20080522: pmap_addr_hint removed
 OLD_FILES+=usr/share/man/man9/pmap_addr_hint.9.gz
 # 20080517: ipsec_osdep.h removed
 OLD_FILES+=usr/include/netipsec/ipsec_osdep.h
 # 20080507: heimdal 1.1 import
 OLD_LIBS+=usr/lib/libasn1.so.9
 OLD_LIBS+=usr/lib/libgssapi.so.9
 OLD_LIBS+=usr/lib/libgssapi_krb5.so.9
 OLD_LIBS+=usr/lib/libhdb.so.9
 OLD_LIBS+=usr/lib/libkadm5clnt.so.9
 OLD_LIBS+=usr/lib/libkadm5srv.so.9
 OLD_LIBS+=usr/lib/libkafs5.so.9
 OLD_LIBS+=usr/lib/libkrb5.so.9
 OLD_LIBS+=usr/lib/libroken.so.9
 OLD_LIBS+=usr/lib32/libgssapi.so.9
 # 20080420: Symbol card support dropped
 OLD_FILES+=usr/include/dev/wi/spectrum24t_cf.h
 # 20080420: awi removal
 OLD_FILES+=usr/share/man/man4/awi.4.gz
 OLD_FILES+=usr/share/man/man4/if_awi.4.gz
 # 20080331: pkg_sign has been removed
 OLD_FILES+=usr/sbin/pkg_check
 OLD_FILES+=usr/sbin/pkg_sign
 OLD_FILES+=usr/share/man/man1/pkg_check.1.gz
 OLD_FILES+=usr/share/man/man1/pkg_sign.1.gz
 # 20080325: tzdata2008b import
 OLD_FILES+=usr/share/zoneinfo/Asia/Calcutta
 OLD_FILES+=usr/share/zoneinfo/Asia/Saigon
 # 20080314: stack_print(9) mlink fixed
 OLD_FILES+=usr/share/man/man9/stack_printf.9.gz
 # 20080312: libkse removal
 OLD_FILES+=usr/include/sys/kse.h
 OLD_FILES+=usr/lib/libkse.so
 OLD_LIBS+=usr/lib/libkse.so.3
 OLD_FILES+=usr/share/man/man2/kse.2.gz
 OLD_FILES+=usr/share/man/man2/kse_create.2.gz
 OLD_FILES+=usr/share/man/man2/kse_exit.2.gz
 OLD_FILES+=usr/share/man/man2/kse_release.2.gz
 OLD_FILES+=usr/share/man/man2/kse_switchin.2.gz
 OLD_FILES+=usr/share/man/man2/kse_thr_interrupt.2.gz
 OLD_FILES+=usr/share/man/man2/kse_wakeup.2.gz
 OLD_FILES+=usr/lib32/libkse.so
 OLD_LIBS+=usr/lib32/libkse.so.3
 # 20080225: bsdar/bsdranlib rename to ar/ranlib
 OLD_FILES+=usr/bin/bsdar
 OLD_FILES+=usr/bin/bsdranlib
 OLD_FILES+=usr/share/man/man1/bsdar.1.gz
 OLD_FILES+=usr/share/man/man1/bsdranlib.1.gz
 # 20080220: geom_lvm rename to geom_linux_lvm
 OLD_FILES+=usr/share/man/man4/geom_lvm.4.gz
 # 20080126: oldcard.4 removal
 OLD_FILES+=usr/share/man/man4/card.4.gz
 OLD_FILES+=usr/share/man/man4/oldcard.4.gz
 # 20080122: Removed from the tree
 OLD_FILES+=usr/share/man/man9/BUF_REFCNT.9.gz
 # 20080108: Moved to section 2
 OLD_FILES+=usr/share/man/man3/shm_open.3.gz
 OLD_FILES+=usr/share/man/man3/shm_unlink.3.gz
 # 20071207: Merged with fortunes-o.real
 OLD_FILES+=usr/share/games/fortune/fortunes2-o
 OLD_FILES+=usr/share/games/fortune/fortunes2-o.dat
 # 20071201: Removal of XRPU driver
 OLD_FILES+=usr/include/sys/xrpuio.h
 # 20071129: Disabled static versions of libkse by default
 OLD_FILES+=usr/lib/libkse.a
 OLD_FILES+=usr/lib/libkse_p.a
 OLD_FILES+=usr/lib/libkse_pic.a
 OLD_FILES+=usr/lib32/libkse.a
 OLD_FILES+=usr/lib32/libkse_p.a
 OLD_FILES+=usr/lib32/libkse_pic.a
 # 20071129: Removed a Solaris compatibility header
 OLD_FILES+=usr/include/sys/_elf_solaris.h
 # 20071125: Renamed to pmc_get_msr()
 OLD_FILES+=usr/share/man/man3/pmc_x86_get_msr.3.gz
 # 20071108: Removed very crunch OLDCARD support file
 OLD_FILES+=etc/defaults/pccard.conf
 # 20071025: rc.d/nfslocking superseded by rc.d/lockd and rc.d/statd
 OLD_FILES+=etc/rc.d/nfslocking
 # 20070930: rename of cached to nscd
 OLD_FILES+=etc/cached.conf
 OLD_FILES+=etc/rc.d/cached
 OLD_FILES+=usr/sbin/cached
 OLD_FILES+=usr/share/man/man5/cached.conf.5.gz
 OLD_FILES+=usr/share/man/man8/cached.8.gz
 # 20070807: removal of PowerPC specific header file.
 .if ${TARGET_ARCH} == "powerpc"
 OLD_FILES+=usr/include/machine/interruptvar.h
 .endif
 # 20070801: fast_ipsec.4 gone
 OLD_FILES+=usr/share/man/man4/fast_ipsec.4.gz
 # 20070715: netatm temporarily disconnected (removed 20080525)
 OLD_FILES+=rescue/atm
 OLD_FILES+=rescue/fore_dnld
 OLD_FILES+=rescue/ilmid
 OLD_FILES+=sbin/atm
 OLD_FILES+=sbin/fore_dnld
 OLD_FILES+=sbin/ilmid
 OLD_FILES+=usr/include/libatm.h
 OLD_FILES+=usr/include/netatm/atm.h
 OLD_FILES+=usr/include/netatm/atm_cm.h
 OLD_FILES+=usr/include/netatm/atm_if.h
 OLD_FILES+=usr/include/netatm/atm_ioctl.h
 OLD_FILES+=usr/include/netatm/atm_pcb.h
 OLD_FILES+=usr/include/netatm/atm_sap.h
 OLD_FILES+=usr/include/netatm/atm_sigmgr.h
 OLD_FILES+=usr/include/netatm/atm_stack.h
 OLD_FILES+=usr/include/netatm/atm_sys.h
 OLD_FILES+=usr/include/netatm/atm_var.h
 OLD_FILES+=usr/include/netatm/atm_vc.h
 OLD_FILES+=usr/include/netatm/ipatm/ipatm.h
 OLD_FILES+=usr/include/netatm/ipatm/ipatm_serv.h
 OLD_FILES+=usr/include/netatm/ipatm/ipatm_var.h
 OLD_FILES+=usr/include/netatm/port.h
 OLD_FILES+=usr/include/netatm/queue.h
 OLD_FILES+=usr/include/netatm/sigpvc/sigpvc_var.h
 OLD_FILES+=usr/include/netatm/spans/spans_cls.h
 OLD_FILES+=usr/include/netatm/spans/spans_kxdr.h
 OLD_FILES+=usr/include/netatm/spans/spans_var.h
 OLD_FILES+=usr/include/netatm/uni/sscf_uni.h
 OLD_FILES+=usr/include/netatm/uni/sscf_uni_var.h
 OLD_FILES+=usr/include/netatm/uni/sscop.h
 OLD_FILES+=usr/include/netatm/uni/sscop_misc.h
 OLD_FILES+=usr/include/netatm/uni/sscop_pdu.h
 OLD_FILES+=usr/include/netatm/uni/sscop_var.h
 OLD_FILES+=usr/include/netatm/uni/uni.h
 OLD_FILES+=usr/include/netatm/uni/uniip_var.h
 OLD_FILES+=usr/include/netatm/uni/unisig.h
 OLD_FILES+=usr/include/netatm/uni/unisig_decode.h
 OLD_FILES+=usr/include/netatm/uni/unisig_mbuf.h
 OLD_FILES+=usr/include/netatm/uni/unisig_msg.h
 OLD_FILES+=usr/include/netatm/uni/unisig_print.h
 OLD_FILES+=usr/include/netatm/uni/unisig_var.h
 OLD_FILES+=usr/lib/libatm.a
 OLD_FILES+=usr/lib/libatm_p.a
 OLD_FILES+=usr/sbin/atmarpd
 OLD_FILES+=usr/sbin/scspd
 OLD_FILES+=usr/share/man/en.ISO8859-1/man8/atm.8.gz
 OLD_FILES+=usr/share/man/en.ISO8859-1/man8/atmarpd.8.gz
 OLD_FILES+=usr/share/man/en.ISO8859-1/man8/fore_dnld.8.gz
 OLD_FILES+=usr/share/man/en.ISO8859-1/man8/ilmid.8.gz
 OLD_FILES+=usr/share/man/en.ISO8859-1/man8/scspd.8.gz
 OLD_FILES+=usr/share/man/man8/atm.8.gz
 OLD_FILES+=usr/share/man/man8/atmarpd.8.gz
 OLD_FILES+=usr/share/man/man8/fore_dnld.8.gz
 OLD_FILES+=usr/share/man/man8/ilmid.8.gz
 OLD_FILES+=usr/share/man/man8/scspd.8.gz
 OLD_FILES+=usr/share/examples/atm/NOTES
 OLD_FILES+=usr/share/examples/atm/README
 OLD_FILES+=usr/share/examples/atm/Startup
 OLD_FILES+=usr/share/examples/atm/atm-config.sh
 OLD_FILES+=usr/share/examples/atm/atm-sockets.txt
 OLD_FILES+=usr/share/examples/atm/cpcs-design.txt
 OLD_FILES+=usr/share/examples/atm/fore-microcode.txt
 OLD_FILES+=usr/share/examples/atm/sscf-design.txt
 OLD_FILES+=usr/share/examples/atm/sscop-design.txt
 OLD_LIBS+=lib/libatm.so.5
 OLD_LIBS+=usr/lib/libatm.so
 OLD_DIRS+=usr/include/netatm/sigpvc
 OLD_DIRS+=usr/include/netatm/spans
 OLD_DIRS+=usr/include/netatm/ipatm
 OLD_DIRS+=usr/include/netatm/uni
 OLD_DIRS+=usr/include/netatm
 OLD_DIRS+=usr/share/examples/atm
 OLD_FILES+=usr/lib32/libatm.a
 OLD_FILES+=usr/lib32/libatm.so
 OLD_LIBS+=usr/lib32/libatm.so.5
 OLD_FILES+=usr/lib32/libatm_p.a
 # 20070705: I4B headers repo-copied to include/i4b/
 .if ${TARGET_ARCH} == "i386"
 OLD_FILES+=usr/include/machine/i4b_cause.h
 OLD_FILES+=usr/include/machine/i4b_debug.h
 OLD_FILES+=usr/include/machine/i4b_ioctl.h
 OLD_FILES+=usr/include/machine/i4b_rbch_ioctl.h
 OLD_FILES+=usr/include/machine/i4b_tel_ioctl.h
 OLD_FILES+=usr/include/machine/i4b_trace.h
 .endif
 # 20070703: pf 4.1 import
 OLD_FILES+=usr/libexec/ftp-proxy
 # 20070701: KAME IPSec removal
 OLD_FILES+=usr/include/netinet6/ah.h
 OLD_FILES+=usr/include/netinet6/ah6.h
 OLD_FILES+=usr/include/netinet6/ah_aesxcbcmac.h
 OLD_FILES+=usr/include/netinet6/esp.h
 OLD_FILES+=usr/include/netinet6/esp6.h
 OLD_FILES+=usr/include/netinet6/esp_aesctr.h
 OLD_FILES+=usr/include/netinet6/esp_camellia.h
 OLD_FILES+=usr/include/netinet6/esp_rijndael.h
 OLD_FILES+=usr/include/netinet6/ipsec.h
 OLD_FILES+=usr/include/netinet6/ipsec6.h
 OLD_FILES+=usr/include/netinet6/ipcomp.h
 OLD_FILES+=usr/include/netinet6/ipcomp6.h
 OLD_FILES+=usr/include/netkey/key.h
 OLD_FILES+=usr/include/netkey/key_debug.h
 OLD_FILES+=usr/include/netkey/key_var.h
 OLD_FILES+=usr/include/netkey/keydb.h
 OLD_FILES+=usr/include/netkey/keysock.h
 OLD_DIRS+=usr/include/netkey
 # 20070701: remove wicontrol
 OLD_FILES+=usr/sbin/wicontrol
 OLD_FILES+=usr/share/man/man8/wicontrol.8.gz
 # 20070625: umapfs removal
 OLD_FILES+=rescue/mount_umapfs
 OLD_FILES+=sbin/mount_umapfs
 OLD_FILES+=usr/include/fs/umapfs/umap.h
 OLD_FILES+=usr/share/man/man8/mount_umapfs.8.gz
 OLD_DIRS+=usr/include/fs/umapfs
 # 20070618: Removal of the PROTO.localhost* files
 OLD_FILES+=etc/namedb/PROTO.localhost-v6.rev
 OLD_FILES+=etc/namedb/PROTO.localhost.rev
 OLD_FILES+=etc/namedb/make-localhost
 # 20070618: shared library version bump
 OLD_LIBS+=lib/libalias.so.5
 OLD_LIBS+=lib/libbsnmp.so.3
 OLD_LIBS+=lib/libncurses.so.6
 OLD_LIBS+=lib/libncursesw.so.6
 OLD_LIBS+=lib/libreadline.so.6
 OLD_LIBS+=usr/lib/libdialog.so.5
 OLD_LIBS+=usr/lib/libgnuregex.so.3
 OLD_LIBS+=usr/lib/libhistory.so.6
 OLD_LIBS+=usr/lib/libpam.so.3
 OLD_LIBS+=usr/lib/libssh.so.3
 OLD_LIBS+=usr/lib/pam_chroot.so.3
 OLD_LIBS+=usr/lib/pam_deny.so.3
 OLD_LIBS+=usr/lib/pam_echo.so.3
 OLD_LIBS+=usr/lib/pam_exec.so.3
 OLD_LIBS+=usr/lib/pam_ftpusers.so.3
 OLD_LIBS+=usr/lib/pam_group.so.3
 OLD_LIBS+=usr/lib/pam_guest.so.3
 OLD_LIBS+=usr/lib/pam_krb5.so.3
 OLD_LIBS+=usr/lib/pam_ksu.so.3
 OLD_LIBS+=usr/lib/pam_lastlog.so.3
 OLD_LIBS+=usr/lib/pam_login_access.so.3
 OLD_LIBS+=usr/lib/pam_nologin.so.3
 OLD_LIBS+=usr/lib/pam_opie.so.3
 OLD_LIBS+=usr/lib/pam_opieaccess.so.3
 OLD_LIBS+=usr/lib/pam_passwdqc.so.3
 OLD_LIBS+=usr/lib/pam_permit.so.3
 OLD_LIBS+=usr/lib/pam_radius.so.3
 OLD_LIBS+=usr/lib/pam_rhosts.so.3
 OLD_LIBS+=usr/lib/pam_rootok.so.3
 OLD_LIBS+=usr/lib/pam_securetty.so.3
 OLD_LIBS+=usr/lib/pam_self.so.3
 OLD_LIBS+=usr/lib/pam_ssh.so.3
 OLD_LIBS+=usr/lib/pam_tacplus.so.3
 OLD_LIBS+=usr/lib/pam_unix.so.3
 OLD_LIBS+=usr/lib/snmp_atm.so.4
 OLD_LIBS+=usr/lib/snmp_bridge.so.4
 OLD_LIBS+=usr/lib/snmp_hostres.so.4
 OLD_LIBS+=usr/lib/snmp_mibII.so.4
 OLD_LIBS+=usr/lib/snmp_netgraph.so.4
 OLD_LIBS+=usr/lib/snmp_pf.so.4
 OLD_LIBS+=usr/lib32/libalias.so.5
 OLD_LIBS+=usr/lib32/libbsnmp.so.3
 OLD_LIBS+=usr/lib32/libdialog.so.5
 OLD_LIBS+=usr/lib32/libgnuregex.so.3
 OLD_LIBS+=usr/lib32/libhistory.so.6
 OLD_LIBS+=usr/lib32/libncurses.so.6
 OLD_LIBS+=usr/lib32/libncursesw.so.6
 OLD_LIBS+=usr/lib32/libpam.so.3
 OLD_LIBS+=usr/lib32/libreadline.so.6
 OLD_LIBS+=usr/lib32/libssh.so.3
 OLD_LIBS+=usr/lib32/pam_chroot.so.3
 OLD_LIBS+=usr/lib32/pam_deny.so.3
 OLD_LIBS+=usr/lib32/pam_echo.so.3
 OLD_LIBS+=usr/lib32/pam_exec.so.3
 OLD_LIBS+=usr/lib32/pam_ftpusers.so.3
 OLD_LIBS+=usr/lib32/pam_group.so.3
 OLD_LIBS+=usr/lib32/pam_guest.so.3
 OLD_LIBS+=usr/lib32/pam_krb5.so.3
 OLD_LIBS+=usr/lib32/pam_ksu.so.3
 OLD_LIBS+=usr/lib32/pam_lastlog.so.3
 OLD_LIBS+=usr/lib32/pam_login_access.so.3
 OLD_LIBS+=usr/lib32/pam_nologin.so.3
 OLD_LIBS+=usr/lib32/pam_opie.so.3
 OLD_LIBS+=usr/lib32/pam_opieaccess.so.3
 OLD_LIBS+=usr/lib32/pam_passwdqc.so.3
 OLD_LIBS+=usr/lib32/pam_permit.so.3
 OLD_LIBS+=usr/lib32/pam_radius.so.3
 OLD_LIBS+=usr/lib32/pam_rhosts.so.3
 OLD_LIBS+=usr/lib32/pam_rootok.so.3
 OLD_LIBS+=usr/lib32/pam_securetty.so.3
 OLD_LIBS+=usr/lib32/pam_self.so.3
 OLD_LIBS+=usr/lib32/pam_ssh.so.3
 OLD_LIBS+=usr/lib32/pam_tacplus.so.3
 OLD_LIBS+=usr/lib32/pam_unix.so.3
 # 20070613: IPX over IP tunnel removal
 OLD_FILES+=usr/include/netipx/ipx_ip.h
 # 20070605: sched_core removal
 OLD_FILES+=usr/share/man/man4/sched_core.4.gz
 # 20070603: BIND 9.4.1 import
 OLD_LIBS+=usr/lib/liblwres.so.10
 # 20070521: shared library version bump
 OLD_LIBS+=lib/libatm.so.4
 OLD_LIBS+=lib/libbegemot.so.2
 OLD_LIBS+=lib/libbsdxml.so.2
 OLD_LIBS+=lib/libcam.so.3
 OLD_LIBS+=lib/libcrypt.so.3
 OLD_LIBS+=lib/libdevstat.so.5
 OLD_LIBS+=lib/libedit.so.5
 OLD_LIBS+=lib/libgeom.so.3
 OLD_LIBS+=lib/libipsec.so.2
 OLD_LIBS+=lib/libipx.so.3
 OLD_LIBS+=lib/libkiconv.so.2
 OLD_LIBS+=lib/libkse.so.2
 OLD_LIBS+=lib/libkvm.so.3
 OLD_LIBS+=lib/libm.so.4
 OLD_LIBS+=lib/libmd.so.3
 OLD_LIBS+=lib/libpcap.so.4
 OLD_LIBS+=lib/libpthread.so.2
 OLD_LIBS+=lib/libsbuf.so.3
 OLD_LIBS+=lib/libthr.so.2
 OLD_LIBS+=lib/libufs.so.3
 OLD_LIBS+=lib/libutil.so.6
 OLD_LIBS+=lib/libz.so.3
 OLD_LIBS+=usr/lib/libbluetooth.so.2
 OLD_LIBS+=usr/lib/libbsm.so.1
 OLD_LIBS+=usr/lib/libbz2.so.2
 OLD_LIBS+=usr/lib/libcalendar.so.3
 OLD_LIBS+=usr/lib/libcom_err.so.3
 OLD_LIBS+=usr/lib/libdevinfo.so.3
 OLD_LIBS+=usr/lib/libfetch.so.4
 OLD_LIBS+=usr/lib/libform.so.3
 OLD_LIBS+=usr/lib/libformw.so.3
 OLD_LIBS+=usr/lib/libftpio.so.6
 OLD_LIBS+=usr/lib/libgpib.so.1
 OLD_LIBS+=usr/lib/libkse.so.2
 OLD_LIBS+=usr/lib/libmagic.so.2
 OLD_LIBS+=usr/lib/libmemstat.so.1
 OLD_LIBS+=usr/lib/libmenu.so.3
 OLD_LIBS+=usr/lib/libmenuw.so.3
 OLD_LIBS+=usr/lib/libmilter.so.3
 OLD_LIBS+=usr/lib/libmp.so.5
 OLD_LIBS+=usr/lib/libncp.so.2
 OLD_LIBS+=usr/lib/libnetgraph.so.2
 OLD_LIBS+=usr/lib/libngatm.so.2
 OLD_LIBS+=usr/lib/libopie.so.4
 OLD_LIBS+=usr/lib/libpanel.so.3
 OLD_LIBS+=usr/lib/libpanelw.so.3
 OLD_LIBS+=usr/lib/libpmc.so.3
 OLD_LIBS+=usr/lib/libradius.so.2
 OLD_LIBS+=usr/lib/librpcsvc.so.3
 OLD_LIBS+=usr/lib/libsdp.so.2
 OLD_LIBS+=usr/lib/libsmb.so.2
 OLD_LIBS+=usr/lib/libstdc++.so.5
 OLD_LIBS+=usr/lib/libtacplus.so.2
 OLD_LIBS+=usr/lib/libthr.so.2
 OLD_LIBS+=usr/lib/libthread_db.so.2
 OLD_LIBS+=usr/lib/libugidfw.so.2
 OLD_LIBS+=usr/lib/libusbhid.so.2
 OLD_LIBS+=usr/lib/libvgl.so.4
 OLD_LIBS+=usr/lib/libwrap.so.4
 OLD_LIBS+=usr/lib/libypclnt.so.2
 OLD_LIBS+=usr/lib/snmp_bridge.so.3
 OLD_LIBS+=usr/lib/snmp_hostres.so.3
 OLD_LIBS+=usr/lib32/libatm.so.4
 OLD_LIBS+=usr/lib32/libbegemot.so.2
 OLD_LIBS+=usr/lib32/libbluetooth.so.2
 OLD_LIBS+=usr/lib32/libbsdxml.so.2
 OLD_LIBS+=usr/lib32/libbsm.so.1
 OLD_LIBS+=usr/lib32/libbz2.so.2
 OLD_LIBS+=usr/lib32/libcalendar.so.3
 OLD_LIBS+=usr/lib32/libcam.so.3
 OLD_LIBS+=usr/lib32/libcom_err.so.3
 OLD_LIBS+=usr/lib32/libcrypt.so.3
 OLD_LIBS+=usr/lib32/libdevinfo.so.3
 OLD_LIBS+=usr/lib32/libdevstat.so.5
 OLD_LIBS+=usr/lib32/libedit.so.5
 OLD_LIBS+=usr/lib32/libfetch.so.4
 OLD_LIBS+=usr/lib32/libform.so.3
 OLD_LIBS+=usr/lib32/libformw.so.3
 OLD_LIBS+=usr/lib32/libftpio.so.6
 OLD_LIBS+=usr/lib32/libgeom.so.3
 OLD_LIBS+=usr/lib32/libgpib.so.1
 OLD_LIBS+=usr/lib32/libipsec.so.2
 OLD_LIBS+=usr/lib32/libipx.so.3
 OLD_LIBS+=usr/lib32/libkiconv.so.2
 OLD_LIBS+=usr/lib32/libkse.so.2
 OLD_LIBS+=usr/lib32/libkvm.so.3
 OLD_LIBS+=usr/lib32/libm.so.4
 OLD_LIBS+=usr/lib32/libmagic.so.2
 OLD_LIBS+=usr/lib32/libmd.so.3
 OLD_LIBS+=usr/lib32/libmemstat.so.1
 OLD_LIBS+=usr/lib32/libmenu.so.3
 OLD_LIBS+=usr/lib32/libmenuw.so.3
 OLD_LIBS+=usr/lib32/libmilter.so.3
 OLD_LIBS+=usr/lib32/libmp.so.5
 OLD_LIBS+=usr/lib32/libncp.so.2
 OLD_LIBS+=usr/lib32/libnetgraph.so.2
 OLD_LIBS+=usr/lib32/libngatm.so.2
 OLD_LIBS+=usr/lib32/libopie.so.4
 OLD_LIBS+=usr/lib32/libpanel.so.3
 OLD_LIBS+=usr/lib32/libpanelw.so.3
 OLD_LIBS+=usr/lib32/libpcap.so.4
 OLD_LIBS+=usr/lib32/libpmc.so.3
 OLD_LIBS+=usr/lib32/libpthread.so.2
 OLD_LIBS+=usr/lib32/libradius.so.2
 OLD_LIBS+=usr/lib32/librpcsvc.so.3
 OLD_LIBS+=usr/lib32/libsbuf.so.3
 OLD_LIBS+=usr/lib32/libsdp.so.2
 OLD_LIBS+=usr/lib32/libsmb.so.2
 OLD_LIBS+=usr/lib32/libstdc++.so.5
 OLD_LIBS+=usr/lib32/libtacplus.so.2
 OLD_LIBS+=usr/lib32/libthr.so.2
 OLD_LIBS+=usr/lib32/libthread_db.so.2
 OLD_LIBS+=usr/lib32/libufs.so.3
 OLD_LIBS+=usr/lib32/libugidfw.so.2
 OLD_LIBS+=usr/lib32/libusbhid.so.2
 OLD_LIBS+=usr/lib32/libutil.so.6
 OLD_LIBS+=usr/lib32/libvgl.so.4
 OLD_LIBS+=usr/lib32/libwrap.so.4
 OLD_LIBS+=usr/lib32/libypclnt.so.2
 OLD_LIBS+=usr/lib32/libz.so.3
 # 20070519: GCC 4.2
 OLD_FILES+=usr/bin/f77
 OLD_FILES+=usr/bin/protoize
 OLD_FILES+=usr/include/g2c.h
 OLD_FILES+=usr/libexec/f771
 OLD_FILES+=usr/share/info/g77.info.gz
 OLD_FILES+=usr/share/man/man1/f77.1.gz
 OLD_FILES+=usr/include/c++/3.4/algorithm
 OLD_FILES+=usr/include/c++/3.4/backward/algo.h
 OLD_FILES+=usr/include/c++/3.4/backward/algobase.h
 OLD_FILES+=usr/include/c++/3.4/backward/alloc.h
 OLD_FILES+=usr/include/c++/3.4/backward/backward_warning.h
 OLD_FILES+=usr/include/c++/3.4/backward/bvector.h
 OLD_FILES+=usr/include/c++/3.4/backward/complex.h
 OLD_FILES+=usr/include/c++/3.4/backward/defalloc.h
 OLD_FILES+=usr/include/c++/3.4/backward/deque.h
 OLD_FILES+=usr/include/c++/3.4/backward/fstream.h
 OLD_FILES+=usr/include/c++/3.4/backward/function.h
 OLD_FILES+=usr/include/c++/3.4/backward/hash_map.h
 OLD_FILES+=usr/include/c++/3.4/backward/hash_set.h
 OLD_FILES+=usr/include/c++/3.4/backward/hashtable.h
 OLD_FILES+=usr/include/c++/3.4/backward/heap.h
 OLD_FILES+=usr/include/c++/3.4/backward/iomanip.h
 OLD_FILES+=usr/include/c++/3.4/backward/iostream.h
 OLD_FILES+=usr/include/c++/3.4/backward/istream.h
 OLD_FILES+=usr/include/c++/3.4/backward/iterator.h
 OLD_FILES+=usr/include/c++/3.4/backward/list.h
 OLD_FILES+=usr/include/c++/3.4/backward/map.h
 OLD_FILES+=usr/include/c++/3.4/backward/multimap.h
 OLD_FILES+=usr/include/c++/3.4/backward/multiset.h
 OLD_FILES+=usr/include/c++/3.4/backward/new.h
 OLD_FILES+=usr/include/c++/3.4/backward/ostream.h
 OLD_FILES+=usr/include/c++/3.4/backward/pair.h
 OLD_FILES+=usr/include/c++/3.4/backward/queue.h
 OLD_FILES+=usr/include/c++/3.4/backward/rope.h
 OLD_FILES+=usr/include/c++/3.4/backward/set.h
 OLD_FILES+=usr/include/c++/3.4/backward/slist.h
 OLD_FILES+=usr/include/c++/3.4/backward/stack.h
 OLD_FILES+=usr/include/c++/3.4/backward/stream.h
 OLD_FILES+=usr/include/c++/3.4/backward/streambuf.h
 OLD_FILES+=usr/include/c++/3.4/backward/strstream
 OLD_FILES+=usr/include/c++/3.4/backward/tempbuf.h
 OLD_FILES+=usr/include/c++/3.4/backward/tree.h
 OLD_FILES+=usr/include/c++/3.4/backward/vector.h
 OLD_FILES+=usr/include/c++/3.4/bits/allocator.h
 OLD_FILES+=usr/include/c++/3.4/bits/atomic_word.h
 OLD_FILES+=usr/include/c++/3.4/bits/atomicity.h
 OLD_FILES+=usr/include/c++/3.4/bits/basic_file.h
 OLD_FILES+=usr/include/c++/3.4/bits/basic_ios.h
 OLD_FILES+=usr/include/c++/3.4/bits/basic_ios.tcc
 OLD_FILES+=usr/include/c++/3.4/bits/basic_string.h
 OLD_FILES+=usr/include/c++/3.4/bits/basic_string.tcc
 OLD_FILES+=usr/include/c++/3.4/bits/boost_concept_check.h
 OLD_FILES+=usr/include/c++/3.4/bits/c++allocator.h
 OLD_FILES+=usr/include/c++/3.4/bits/c++config.h
 OLD_FILES+=usr/include/c++/3.4/bits/c++io.h
 OLD_FILES+=usr/include/c++/3.4/bits/c++locale.h
 OLD_FILES+=usr/include/c++/3.4/bits/c++locale_internal.h
 OLD_FILES+=usr/include/c++/3.4/bits/char_traits.h
 OLD_FILES+=usr/include/c++/3.4/bits/cmath.tcc
 OLD_FILES+=usr/include/c++/3.4/bits/codecvt.h
 OLD_FILES+=usr/include/c++/3.4/bits/codecvt_specializations.h
 OLD_FILES+=usr/include/c++/3.4/bits/concept_check.h
 OLD_FILES+=usr/include/c++/3.4/bits/concurrence.h
 OLD_FILES+=usr/include/c++/3.4/bits/cpp_type_traits.h
 OLD_FILES+=usr/include/c++/3.4/bits/ctype_base.h
 OLD_FILES+=usr/include/c++/3.4/bits/ctype_inline.h
 OLD_FILES+=usr/include/c++/3.4/bits/ctype_noninline.h
 OLD_FILES+=usr/include/c++/3.4/bits/deque.tcc
 OLD_FILES+=usr/include/c++/3.4/bits/fstream.tcc
 OLD_FILES+=usr/include/c++/3.4/bits/functexcept.h
 OLD_FILES+=usr/include/c++/3.4/bits/gslice.h
 OLD_FILES+=usr/include/c++/3.4/bits/gslice_array.h
 OLD_FILES+=usr/include/c++/3.4/bits/gthr-default.h
 OLD_FILES+=usr/include/c++/3.4/bits/gthr-posix.h
 OLD_FILES+=usr/include/c++/3.4/bits/gthr-single.h
 OLD_FILES+=usr/include/c++/3.4/bits/gthr.h
 OLD_FILES+=usr/include/c++/3.4/bits/indirect_array.h
 OLD_FILES+=usr/include/c++/3.4/bits/ios_base.h
 OLD_FILES+=usr/include/c++/3.4/bits/istream.tcc
 OLD_FILES+=usr/include/c++/3.4/bits/list.tcc
 OLD_FILES+=usr/include/c++/3.4/bits/locale_classes.h
 OLD_FILES+=usr/include/c++/3.4/bits/locale_facets.h
 OLD_FILES+=usr/include/c++/3.4/bits/locale_facets.tcc
 OLD_FILES+=usr/include/c++/3.4/bits/localefwd.h
 OLD_FILES+=usr/include/c++/3.4/bits/mask_array.h
 OLD_FILES+=usr/include/c++/3.4/bits/messages_members.h
 OLD_FILES+=usr/include/c++/3.4/bits/os_defines.h
 OLD_FILES+=usr/include/c++/3.4/bits/ostream.tcc
 OLD_FILES+=usr/include/c++/3.4/bits/postypes.h
 OLD_FILES+=usr/include/c++/3.4/bits/slice_array.h
 OLD_FILES+=usr/include/c++/3.4/bits/sstream.tcc
 OLD_FILES+=usr/include/c++/3.4/bits/stl_algo.h
 OLD_FILES+=usr/include/c++/3.4/bits/stl_algobase.h
 OLD_FILES+=usr/include/c++/3.4/bits/stl_bvector.h
 OLD_FILES+=usr/include/c++/3.4/bits/stl_construct.h
 OLD_FILES+=usr/include/c++/3.4/bits/stl_deque.h
 OLD_FILES+=usr/include/c++/3.4/bits/stl_function.h
 OLD_FILES+=usr/include/c++/3.4/bits/stl_heap.h
 OLD_FILES+=usr/include/c++/3.4/bits/stl_iterator.h
 OLD_FILES+=usr/include/c++/3.4/bits/stl_iterator_base_funcs.h
 OLD_FILES+=usr/include/c++/3.4/bits/stl_iterator_base_types.h
 OLD_FILES+=usr/include/c++/3.4/bits/stl_list.h
 OLD_FILES+=usr/include/c++/3.4/bits/stl_map.h
 OLD_FILES+=usr/include/c++/3.4/bits/stl_multimap.h
 OLD_FILES+=usr/include/c++/3.4/bits/stl_multiset.h
 OLD_FILES+=usr/include/c++/3.4/bits/stl_numeric.h
 OLD_FILES+=usr/include/c++/3.4/bits/stl_pair.h
 OLD_FILES+=usr/include/c++/3.4/bits/stl_queue.h
 OLD_FILES+=usr/include/c++/3.4/bits/stl_raw_storage_iter.h
 OLD_FILES+=usr/include/c++/3.4/bits/stl_relops.h
 OLD_FILES+=usr/include/c++/3.4/bits/stl_set.h
 OLD_FILES+=usr/include/c++/3.4/bits/stl_stack.h
 OLD_FILES+=usr/include/c++/3.4/bits/stl_tempbuf.h
 OLD_FILES+=usr/include/c++/3.4/bits/stl_threads.h
 OLD_FILES+=usr/include/c++/3.4/bits/stl_tree.h
 OLD_FILES+=usr/include/c++/3.4/bits/stl_uninitialized.h
 OLD_FILES+=usr/include/c++/3.4/bits/stl_vector.h
 OLD_FILES+=usr/include/c++/3.4/bits/stream_iterator.h
 OLD_FILES+=usr/include/c++/3.4/bits/streambuf.tcc
 OLD_FILES+=usr/include/c++/3.4/bits/streambuf_iterator.h
 OLD_FILES+=usr/include/c++/3.4/bits/stringfwd.h
 OLD_FILES+=usr/include/c++/3.4/bits/time_members.h
 OLD_FILES+=usr/include/c++/3.4/bits/type_traits.h
 OLD_FILES+=usr/include/c++/3.4/bits/valarray_after.h
 OLD_FILES+=usr/include/c++/3.4/bits/valarray_array.h
 OLD_FILES+=usr/include/c++/3.4/bits/valarray_array.tcc
 OLD_FILES+=usr/include/c++/3.4/bits/valarray_before.h
 OLD_FILES+=usr/include/c++/3.4/bits/vector.tcc
 OLD_FILES+=usr/include/c++/3.4/bitset
 OLD_FILES+=usr/include/c++/3.4/cassert
 OLD_FILES+=usr/include/c++/3.4/cctype
 OLD_FILES+=usr/include/c++/3.4/cerrno
 OLD_FILES+=usr/include/c++/3.4/cfloat
 OLD_FILES+=usr/include/c++/3.4/ciso646
 OLD_FILES+=usr/include/c++/3.4/climits
 OLD_FILES+=usr/include/c++/3.4/clocale
 OLD_FILES+=usr/include/c++/3.4/cmath
 OLD_FILES+=usr/include/c++/3.4/complex
 OLD_FILES+=usr/include/c++/3.4/csetjmp
 OLD_FILES+=usr/include/c++/3.4/csignal
 OLD_FILES+=usr/include/c++/3.4/cstdarg
 OLD_FILES+=usr/include/c++/3.4/cstddef
 OLD_FILES+=usr/include/c++/3.4/cstdio
 OLD_FILES+=usr/include/c++/3.4/cstdlib
 OLD_FILES+=usr/include/c++/3.4/cstring
 OLD_FILES+=usr/include/c++/3.4/ctime
 OLD_FILES+=usr/include/c++/3.4/cwchar
 OLD_FILES+=usr/include/c++/3.4/cwctype
 OLD_FILES+=usr/include/c++/3.4/cxxabi.h
 OLD_FILES+=usr/include/c++/3.4/debug/bitset
 OLD_FILES+=usr/include/c++/3.4/debug/debug.h
 OLD_FILES+=usr/include/c++/3.4/debug/deque
 OLD_FILES+=usr/include/c++/3.4/debug/formatter.h
 OLD_FILES+=usr/include/c++/3.4/debug/hash_map
 OLD_FILES+=usr/include/c++/3.4/debug/hash_map.h
 OLD_FILES+=usr/include/c++/3.4/debug/hash_multimap.h
 OLD_FILES+=usr/include/c++/3.4/debug/hash_multiset.h
 OLD_FILES+=usr/include/c++/3.4/debug/hash_set
 OLD_FILES+=usr/include/c++/3.4/debug/hash_set.h
 OLD_FILES+=usr/include/c++/3.4/debug/list
 OLD_FILES+=usr/include/c++/3.4/debug/map
 OLD_FILES+=usr/include/c++/3.4/debug/map.h
 OLD_FILES+=usr/include/c++/3.4/debug/multimap.h
 OLD_FILES+=usr/include/c++/3.4/debug/multiset.h
 OLD_FILES+=usr/include/c++/3.4/debug/safe_base.h
 OLD_FILES+=usr/include/c++/3.4/debug/safe_iterator.h
 OLD_FILES+=usr/include/c++/3.4/debug/safe_iterator.tcc
 OLD_FILES+=usr/include/c++/3.4/debug/safe_sequence.h
 OLD_FILES+=usr/include/c++/3.4/debug/set
 OLD_FILES+=usr/include/c++/3.4/debug/set.h
 OLD_FILES+=usr/include/c++/3.4/debug/string
 OLD_FILES+=usr/include/c++/3.4/debug/vector
 OLD_FILES+=usr/include/c++/3.4/deque
 OLD_FILES+=usr/include/c++/3.4/exception
 OLD_FILES+=usr/include/c++/3.4/exception_defines.h
 OLD_FILES+=usr/include/c++/3.4/ext/algorithm
 OLD_FILES+=usr/include/c++/3.4/ext/bitmap_allocator.h
 OLD_FILES+=usr/include/c++/3.4/ext/debug_allocator.h
 OLD_FILES+=usr/include/c++/3.4/ext/enc_filebuf.h
 OLD_FILES+=usr/include/c++/3.4/ext/functional
 OLD_FILES+=usr/include/c++/3.4/ext/hash_fun.h
 OLD_FILES+=usr/include/c++/3.4/ext/hash_map
 OLD_FILES+=usr/include/c++/3.4/ext/hash_set
 OLD_FILES+=usr/include/c++/3.4/ext/hashtable.h
 OLD_FILES+=usr/include/c++/3.4/ext/iterator
 OLD_FILES+=usr/include/c++/3.4/ext/malloc_allocator.h
 OLD_FILES+=usr/include/c++/3.4/ext/memory
 OLD_FILES+=usr/include/c++/3.4/ext/mt_allocator.h
 OLD_FILES+=usr/include/c++/3.4/ext/new_allocator.h
 OLD_FILES+=usr/include/c++/3.4/ext/numeric
 OLD_FILES+=usr/include/c++/3.4/ext/pod_char_traits.h
 OLD_FILES+=usr/include/c++/3.4/ext/pool_allocator.h
 OLD_FILES+=usr/include/c++/3.4/ext/rb_tree
 OLD_FILES+=usr/include/c++/3.4/ext/rope
 OLD_FILES+=usr/include/c++/3.4/ext/ropeimpl.h
 OLD_FILES+=usr/include/c++/3.4/ext/slist
 OLD_FILES+=usr/include/c++/3.4/ext/stdio_filebuf.h
 OLD_FILES+=usr/include/c++/3.4/ext/stdio_sync_filebuf.h
 OLD_FILES+=usr/include/c++/3.4/fstream
 OLD_FILES+=usr/include/c++/3.4/functional
 OLD_FILES+=usr/include/c++/3.4/iomanip
 OLD_FILES+=usr/include/c++/3.4/ios
 OLD_FILES+=usr/include/c++/3.4/iosfwd
 OLD_FILES+=usr/include/c++/3.4/iostream
 OLD_FILES+=usr/include/c++/3.4/istream
 OLD_FILES+=usr/include/c++/3.4/iterator
 OLD_FILES+=usr/include/c++/3.4/limits
 OLD_FILES+=usr/include/c++/3.4/list
 OLD_FILES+=usr/include/c++/3.4/locale
 OLD_FILES+=usr/include/c++/3.4/map
 OLD_FILES+=usr/include/c++/3.4/memory
 OLD_FILES+=usr/include/c++/3.4/new
 OLD_FILES+=usr/include/c++/3.4/numeric
 OLD_FILES+=usr/include/c++/3.4/ostream
 OLD_FILES+=usr/include/c++/3.4/queue
 OLD_FILES+=usr/include/c++/3.4/set
 OLD_FILES+=usr/include/c++/3.4/sstream
 OLD_FILES+=usr/include/c++/3.4/stack
 OLD_FILES+=usr/include/c++/3.4/stdexcept
 OLD_FILES+=usr/include/c++/3.4/streambuf
 OLD_FILES+=usr/include/c++/3.4/string
 OLD_FILES+=usr/include/c++/3.4/typeinfo
 OLD_FILES+=usr/include/c++/3.4/utility
 OLD_FILES+=usr/include/c++/3.4/valarray
 OLD_FILES+=usr/include/c++/3.4/vector
 OLD_DIRS+=usr/include/c++/3.4/backward
 OLD_DIRS+=usr/include/c++/3.4/bits
 OLD_DIRS+=usr/include/c++/3.4/debug
 OLD_DIRS+=usr/include/c++/3.4/ext
 OLD_DIRS+=usr/include/c++/3.4
 # 20070510: zpool/zfs moved to /sbin
 OLD_FILES+=usr/sbin/zfs
 OLD_FILES+=usr/sbin/zpool
 # 20070423: rc.bluetooth (examples) removed
 OLD_FILES+=usr/share/examples/netgraph/bluetooth/rc.bluetooth
 OLD_DIRS+=usr/share/examples/netgraph/bluetooth
 # 20070421: worm.4 removed
 OLD_FILES+=usr/share/man/man4/worm.4.gz
 # 20070417: trunk(4) renamed to lagg(4)
 OLD_FILES+=usr/include/net/if_trunk.h
 # 20070409: uuidgen moved to /bin/
 OLD_FILES+=usr/bin/uuidgen
 # 20070328: bzip2 1.0.4
 OLD_FILES+=usr/share/info/bzip2.info.gz
 # 20070303: libarchive 2.0
 OLD_LIBS+=usr/lib/libarchive.so.3
 OLD_LIBS+=usr/lib32/libarchive.so.3
 # 20070301: remove addr2ascii and ascii2addr
 OLD_FILES+=usr/share/man/man3/addr2ascii.3.gz
 OLD_FILES+=usr/share/man/man3/ascii2addr.3.gz
 # 20070225: vm_page_unmanage() removed
 OLD_FILES+=usr/share/man/man9/vm_page_unmanage.9.gz
 # 20070216: VFS_VPTOFH(9) -> VOP_VPTOFH(9)
 OLD_FILES+=usr/share/man/man9/VFS_VPTOFH.9.gz
 # 20070212: kame.4 removed
 OLD_FILES+=usr/share/man/man4/kame.4.gz
 # 20070201: remove libmytinfo link
 OLD_FILES+=usr/lib/libmytinfo.a
 OLD_FILES+=usr/lib/libmytinfo.so
 OLD_FILES+=usr/lib/libmytinfo_p.a
 OLD_FILES+=usr/lib/libmytinfow.a
 OLD_FILES+=usr/lib/libmytinfow.so
 OLD_FILES+=usr/lib/libmytinfow_p.a
 OLD_FILES+=usr/lib32/libmytinfo.a
 OLD_FILES+=usr/lib32/libmytinfo.so
 OLD_FILES+=usr/lib32/libmytinfo_p.a
 OLD_FILES+=usr/lib32/libmytinfow.a
 OLD_FILES+=usr/lib32/libmytinfow.so
 OLD_FILES+=usr/lib32/libmytinfow_p.a
 # 20070128: remove vnconfig
 OLD_FILES+=usr/sbin/vnconfig
 # 20070127: remove bpf_compat.h
 OLD_FILES+=usr/include/net/bpf_compat.h
 # 20070125: objformat bites the dust
 OLD_FILES+=usr/bin/objformat
 OLD_FILES+=usr/share/man/man1/objformat.1.gz
 OLD_FILES+=usr/include/objformat.h
 OLD_FILES+=usr/share/man/man3/getobjformat.3.gz
 # 20061201: remove symlink to *.so.4 libalias modules
 OLD_FILES+=usr/lib/libalias_cuseeme.so
 OLD_FILES+=usr/lib/libalias_dummy.so
 OLD_FILES+=usr/lib/libalias_ftp.so
 OLD_FILES+=usr/lib/libalias_irc.so
 OLD_FILES+=usr/lib/libalias_nbt.so
 OLD_FILES+=usr/lib/libalias_pptp.so
 OLD_FILES+=usr/lib/libalias_skinny.so
 OLD_FILES+=usr/lib/libalias_smedia.so
 # 20061201: remove old *.so.4 libalias modules
 OLD_FILES+=lib/libalias_cuseeme.so.4
 OLD_FILES+=lib/libalias_dummy.so.4
 OLD_FILES+=lib/libalias_ftp.so.4
 OLD_FILES+=lib/libalias_irc.so.4
 OLD_FILES+=lib/libalias_nbt.so.4
 OLD_FILES+=lib/libalias_pptp.so.4
 OLD_FILES+=lib/libalias_skinny.so.4
 OLD_FILES+=lib/libalias_smedia.so.4
 # 20061126: remove old man page
 OLD_FILES+=usr/share/man/man3/archive_read_set_bytes_per_block.3.gz
 # 20061125: remove old man page
 OLD_FILES+=usr/share/man/man9/devsw.9.gz
 # 20061122: remove obsolete mount programs
 OLD_FILES+=sbin/mount_devfs
 OLD_FILES+=sbin/mount_ext2fs
 OLD_FILES+=sbin/mount_fdescfs
 OLD_FILES+=sbin/mount_linprocfs
 OLD_FILES+=sbin/mount_procfs
 OLD_FILES+=sbin/mount_std
 OLD_FILES+=rescue/mount_devfs
 OLD_FILES+=rescue/mount_ext2fs
 OLD_FILES+=rescue/mount_fdescfs
 OLD_FILES+=rescue/mount_linprocfs
 OLD_FILES+=rescue/mount_procfs
 OLD_FILES+=rescue/mount_std
 OLD_FILES+=usr/share/man/man8/mount_devfs.8.gz
 OLD_FILES+=usr/share/man/man8/mount_ext2fs.8.gz
 OLD_FILES+=usr/share/man/man8/mount_fdescfs.8.gz
 OLD_FILES+=usr/share/man/man8/mount_linprocfs.8.gz
 OLD_FILES+=usr/share/man/man8/mount_procfs.8.gz
 OLD_FILES+=usr/share/man/man8/mount_std.8.gz
 # 20061116: uhidev.4 removed
 OLD_FILES+=usr/share/man/man4/uhidev.4.gz
 # 20061106: archive_write_prepare.3 removed
 OLD_FILES+=usr/share/man/man3/archive_write_prepare.3.gz
 # 20061018: pccardc removed
 OLD_FILES+=usr/sbin/pccardc usr/share/man/man8/pccardc.8.gz
 # 20060930: demangle.h from contrib/libstdc++/include/ext/
 OLD_FILES+=usr/include/c++/3.4/ext/demangle.h
 # 20060929: mrouted removed
 OLD_FILES+=usr/sbin/map-mbone
 OLD_FILES+=usr/sbin/mrinfo
 OLD_FILES+=usr/sbin/mrouted
 OLD_FILES+=usr/sbin/mtrace
 OLD_FILES+=usr/share/man/man8/map-mbone.8.gz
 OLD_FILES+=usr/share/man/man8/mrinfo.8.gz
 OLD_FILES+=usr/share/man/man8/mrouted.8.gz
 OLD_FILES+=usr/share/man/man8/mtrace.8.gz
 # 20060924: tcpslice removed
 OLD_FILES+=usr/sbin/tcpslice
 OLD_FILES+=usr/share/man/man1/tcpslice.1.gz
 # 20060829: kvmdb cleanup script removed
 OLD_FILES+=etc/periodic/weekly/120.clean-kvmdb
 # 20060822: ramdisk{,-own} have been replaced by mdconfig{,2}
 OLD_FILES+=etc/rc.d/ramdisk
 OLD_FILES+=etc/rc.d/ramdisk-own
 # 20060729: OpenSSL 0.9.7e -> 0.9.8b upgrade
 OLD_FILES+=usr/include/openssl/eng_int.h
 OLD_FILES+=usr/include/openssl/hw_4758_cca_err.h
 OLD_FILES+=usr/include/openssl/hw_aep_err.h
 OLD_FILES+=usr/include/openssl/hw_atalla_err.h
 OLD_FILES+=usr/include/openssl/hw_cswift_err.h
 OLD_FILES+=usr/include/openssl/hw_ncipher_err.h
 OLD_FILES+=usr/include/openssl/hw_nuron_err.h
 OLD_FILES+=usr/include/openssl/hw_sureware_err.h
 OLD_FILES+=usr/include/openssl/hw_ubsec_err.h
 # 20060713: mount_linsysfs(8) never existed in 7.x
 OLD_FILES+=sbin/mount_linsysfs
 OLD_FILES+=usr/share/man/man8/mount_linsysfs.8.gz
 # 20060704: KAME compat file net_osdep.h removed
 OLD_FILES+=usr/include/net/net_osdep.h
 # 20060605: man page links removed by OpenBSM 1.0 alpha 6 import
 OLD_FILES+=usr/share/man/man3/au_to_socket.3.gz
 OLD_FILES+=usr/share/man/man3/au_to_socket_ex_128.3.gz
 OLD_FILES+=usr/share/man/man3/au_to_socket_ex_32.3.gz
 # 20060517: pcvt removed
 OLD_FILES+=usr/share/pcvt/README.FIRST
 OLD_FILES+=usr/share/pcvt/Etc/xmodmap-german
 OLD_FILES+=usr/share/pcvt/Etc/pcvt.sh
 OLD_FILES+=usr/share/pcvt/Etc/pcvt.el
 OLD_FILES+=usr/share/pcvt/Etc/Terminfo
 OLD_FILES+=usr/share/pcvt/Etc/Termcap
 OLD_DIRS+=usr/share/pcvt/Etc
 OLD_FILES+=usr/share/pcvt/Doc/NotesAndHints
 OLD_FILES+=usr/share/pcvt/Doc/Keyboard.VT
 OLD_FILES+=usr/share/pcvt/Doc/Keyboard.HP
 OLD_FILES+=usr/share/pcvt/Doc/EscapeSequences
 OLD_FILES+=usr/share/pcvt/Doc/Charsets
 OLD_FILES+=usr/share/pcvt/Doc/CharGen
 OLD_FILES+=usr/share/pcvt/Doc/Bibliography
 OLD_FILES+=usr/share/pcvt/Doc/Acknowledgements
 OLD_DIRS+=usr/share/pcvt/Doc
 OLD_DIRS+=usr/share/pcvt
 OLD_FILES+=usr/share/misc/pcvtfonts/vt220l.816
 OLD_FILES+=usr/share/misc/pcvtfonts/vt220l.814
 OLD_FILES+=usr/share/misc/pcvtfonts/vt220l.810
 OLD_FILES+=usr/share/misc/pcvtfonts/vt220l.808
 OLD_FILES+=usr/share/misc/pcvtfonts/vt220h.816
 OLD_FILES+=usr/share/misc/pcvtfonts/vt220h.814
 OLD_FILES+=usr/share/misc/pcvtfonts/vt220h.810
 OLD_FILES+=usr/share/misc/pcvtfonts/vt220h.808
 OLD_DIRS+=usr/share/misc/pcvtfonts
 OLD_FILES+=usr/share/misc/keycap.pcvt
 OLD_FILES+=usr/share/man/man8/ispcvt.8.gz
 OLD_FILES+=usr/share/man/man5/keycap.5.gz
 OLD_FILES+=usr/share/man/man4/pcvt.4.gz
 OLD_FILES+=usr/share/man/man3/kgetstr.3.gz
 OLD_FILES+=usr/share/man/man3/kgetnum.3.gz
 OLD_FILES+=usr/share/man/man3/kgetflag.3.gz
 OLD_FILES+=usr/share/man/man3/kgetent.3.gz
 OLD_FILES+=usr/share/man/man3/keycap.3.gz
 OLD_FILES+=usr/share/man/man1/vt220keys.1.gz
 OLD_FILES+=usr/share/man/man1/scon.1.gz
 OLD_FILES+=usr/share/man/man1/loadfont.1.gz
 OLD_FILES+=usr/share/man/man1/kcon.1.gz
 OLD_FILES+=usr/share/man/man1/fontedit.1.gz
 OLD_FILES+=usr/share/man/man1/cursor.1.gz
 OLD_FILES+=usr/sbin/vt220keys
 OLD_FILES+=usr/sbin/scon
 OLD_FILES+=usr/sbin/loadfont
 OLD_FILES+=usr/sbin/kcon
 OLD_FILES+=usr/sbin/ispcvt
 OLD_FILES+=usr/sbin/fontedit
 OLD_FILES+=usr/sbin/cursor
 OLD_FILES+=usr/lib/libkeycap_p.a
 OLD_FILES+=usr/lib/libkeycap.a
 OLD_FILES+=usr/include/machine/pcvt_ioctl.h
 # 20060514: lnc(4) replaced by le(4)
 OLD_FILES+=usr/share/man/man4/i386/lnc.4.gz
 # 20060512: remove ip6fw
 OLD_FILES+=etc/periodic/security/600.ip6fwdenied
 OLD_FILES+=etc/periodic/security/650.ip6fwlimit
 OLD_FILES+=sbin/ip6fw
 OLD_FILES+=usr/include/netinet6/ip6_fw.h
 OLD_FILES+=usr/share/man/man8/ip6fw.8.gz
 # 20060424: sab(4) removed
 OLD_FILES+=usr/share/man/man4/sab.4.gz
 # 20060328: remove redundant rc.d script
 OLD_FILES+=etc/rc.d/ike
 # 20060127: revert libdisk to static-only
 OLD_FILES+=usr/lib/libdisk.so
 # 20060115: sys/pccard includes cleanup
 OLD_FILES+=usr/include/pccard/driver.h
 OLD_FILES+=usr/include/pccard/i82365.h
 OLD_FILES+=usr/include/pccard/meciareg.h
 OLD_FILES+=usr/include/pccard/pccard_nbk.h
 OLD_FILES+=usr/include/pccard/pcic_pci.h
 OLD_FILES+=usr/include/pccard/pcicvar.h
 OLD_FILES+=usr/include/pccard/slot.h
 # 20051215: rescue/nextboot.sh renamed to rescue/nextboot
 OLD_FILES+=rescue/nextboot.sh
 # 20051214: usbd(8) removed
 OLD_FILES+=etc/rc.d/usbd
 OLD_FILES+=etc/usbd.conf
 OLD_FILES+=usr/sbin/usbd
 OLD_FILES+=usr/share/man/man8/usbd.8.gz
 # 20051029: rc.d/ppp-user renamed to rc.d/ppp for convenience
 OLD_FILES+=etc/rc.d/ppp-user
 # 20051012: setkey(8) moved to /sbin/
 OLD_FILES+=usr/sbin/setkey
 # 20050930: pccardd(8) removed
 OLD_FILES+=usr/sbin/pccardd
 OLD_FILES+=usr/share/man/man5/pccard.conf.5.gz
 OLD_FILES+=usr/share/man/man8/pccardd.8.gz
 # 20050927: bridge(4) replaced by if_bridge(4)
 OLD_FILES+=usr/include/net/bridge.h
 # 20050831: not implemented
 OLD_FILES+=usr/share/man/man3/getino.3.gz
 OLD_FILES+=usr/share/man/man3/putino.3.gz
 # 20050825: T/TCP retired several months ago
 OLD_FILES+=usr/share/man/man4/ttcp.4.gz
 # 20050805 tn3270 retired long ago
 OLD_FILES+=usr/share/misc/map3270
 # 20050801: too old to be interesting here
 OLD_FILES+=usr/share/doc/papers/px.ps.gz
 # 20050721: moved to ports
 OLD_FILES+=usr/sbin/vttest
 OLD_FILES+=usr/share/man/man1/vttest.1.gz
 # 20050617: wpa man pages moved to section 8
 OLD_FILES+=usr/share/man/man1/hostapd.1.gz
 OLD_FILES+=usr/share/man/man1/hostapd_cli.1.gz
 OLD_FILES+=usr/share/man/man1/wpa_cli.1.gz
 OLD_FILES+=usr/share/man/man1/wpa_supplicant.1.gz
 # 20050610: rexecd (insecure by design)
 OLD_FILES+=etc/pam.d/rexecd
 OLD_FILES+=usr/share/man/man8/rexecd.8.gz
 OLD_FILES+=usr/libexec/rexecd
 # 20050606: OpenBSD dhclient replaces ISC one
 OLD_FILES+=bin/omshell
 OLD_FILES+=sbin/omshell
 OLD_FILES+=usr/share/man/man1/omshell.1.gz
 OLD_FILES+=usr/share/man/man5/dhcp-eval.5.gz
 # 200504XX: ipf tools moved from /usr to /
 OLD_FILES+=rescue/ipfs
 OLD_FILES+=rescue/ipfstat
 OLD_FILES+=rescue/ipmon
 OLD_FILES+=rescue/ipnat
 OLD_FILES+=usr/sbin/ipftest
 OLD_FILES+=usr/sbin/ipresend
 OLD_FILES+=usr/sbin/ipsend
 OLD_FILES+=usr/sbin/iptest
 OLD_FILES+=usr/share/man/man1/ipnat.1.gz
 OLD_FILES+=usr/share/man/man1/ipsend.1.gz
 OLD_FILES+=usr/share/man/man1/iptest.1.gz
 OLD_FILES+=usr/share/man/man5/ipsend.5.gz
 # 200503XX: bsdtar takes over gtar
 OLD_FILES+=usr/bin/gtar
 OLD_FILES+=usr/share/man/man1/gtar.1.gz
 # 200503XX
 OLD_FILES+=usr/share/man/man3/exp10.3.gz
 OLD_FILES+=usr/share/man/man3/exp10f.3.gz
 OLD_FILES+=usr/share/man/man3/fpsetsticky.3.gz
 # 20050324: updated release infrastructure
 OLD_FILES+=usr/share/man/man5/drivers.conf.5.gz
 # 20050317: removed from BIND 9 distribution
 OLD_FILES+=usr/share/doc/bind9/KNOWN_DEFECTS
 # 2005XXXX:
 OLD_FILES+=sbin/mount_autofs
 OLD_FILES+=usr/lib/libautofs.a
 OLD_FILES+=usr/lib/libautofs.so
 OLD_FILES+=usr/share/man/man8/mount_autofs.8.gz
 # 20050203: Merged with fortunes
 OLD_FILES+=usr/share/games/fortune/fortunes2
 OLD_FILES+=usr/share/games/fortune/fortunes2.dat
 # 200501XX:
 OLD_FILES+=usr/libexec/getNAME
 # 200411XX: gvinum replaces vinum
 OLD_FILES+=bin/vinum
 OLD_FILES+=rescue/vinum
 OLD_FILES+=sbin/vinum
 OLD_FILES+=usr/share/man/man8/vinum.8.gz
 # 200411XX: libxpg4 removal
 OLD_FILES+=usr/lib/libxpg4.a
 OLD_FILES+=usr/lib/libxpg4.so
 OLD_FILES+=usr/lib/libxpg4_p.a
 # 20041109: replaced by em(4)
 OLD_FILES+=usr/share/man/man4/gx.4.gz
 OLD_FILES+=usr/share/man/man4/if_gx.4.gz
 # 20041017: rune interface removed
 OLD_FILES+=usr/include/rune.h
 OLD_FILES+=usr/share/man/man3/fgetrune.3.gz
 OLD_FILES+=usr/share/man/man3/fputrune.3.gz
 OLD_FILES+=usr/share/man/man3/fungetrune.3.gz
 OLD_FILES+=usr/share/man/man3/mbrrune.3.gz
 OLD_FILES+=usr/share/man/man3/mbrune.3.gz
 OLD_FILES+=usr/share/man/man3/rune.3.gz
 OLD_FILES+=usr/share/man/man3/setinvalidrune.3.gz
 OLD_FILES+=usr/share/man/man3/sgetrune.3.gz
 OLD_FILES+=usr/share/man/man3/sputrune.3.gz
 # 20040925: bind9 import
 OLD_FILES+=usr/bin/dnskeygen
 OLD_FILES+=usr/bin/dnsquery
 OLD_FILES+=usr/lib/libisc.a
 OLD_FILES+=usr/lib/libisc.so
 OLD_FILES+=usr/lib/libisc_p.a
 OLD_FILES+=usr/libexec/named-xfer
 OLD_FILES+=usr/sbin/named.restart
 OLD_FILES+=usr/sbin/ndc
 OLD_FILES+=usr/sbin/nslookup
 OLD_FILES+=usr/sbin/nsupdate
 OLD_FILES+=usr/share/doc/bind/html/acl.html
 OLD_FILES+=usr/share/doc/bind/html/address_list.html
 OLD_FILES+=usr/share/doc/bind/html/comments.html
 OLD_FILES+=usr/share/doc/bind/html/config.html
 OLD_FILES+=usr/share/doc/bind/html/controls.html
 OLD_FILES+=usr/share/doc/bind/html/docdef.html
 OLD_FILES+=usr/share/doc/bind/html/example.html
 OLD_FILES+=usr/share/doc/bind/html/include.html
 OLD_FILES+=usr/share/doc/bind/html/index.html
 OLD_FILES+=usr/share/doc/bind/html/key.html
 OLD_FILES+=usr/share/doc/bind/html/logging.html
 OLD_FILES+=usr/share/doc/bind/html/master.html
 OLD_FILES+=usr/share/doc/bind/html/options.html
 OLD_FILES+=usr/share/doc/bind/html/server.html
 OLD_FILES+=usr/share/doc/bind/html/trusted-keys.html
 OLD_FILES+=usr/share/doc/bind/html/zone.html
 OLD_FILES+=usr/share/doc/bind/misc/DynamicUpdate
 OLD_FILES+=usr/share/doc/bind/misc/FAQ.1of2
 OLD_FILES+=usr/share/doc/bind/misc/FAQ.2of2
 OLD_FILES+=usr/share/doc/bind/misc/rfc2317-notes.txt
 OLD_FILES+=usr/share/doc/bind/misc/style.txt
 OLD_FILES+=usr/share/man/man1/dnskeygen.1.gz
 OLD_FILES+=usr/share/man/man1/dnsquery.1.gz
 OLD_FILES+=usr/share/man/man8/named-bootconf.8.gz
 OLD_FILES+=usr/share/man/man8/named-xfer.8.gz
 OLD_FILES+=usr/share/man/man8/named.restart.8.gz
 OLD_FILES+=usr/share/man/man8/ndc.8.gz
 OLD_FILES+=usr/share/man/man8/nslookup.8.gz
 # 200409XX
 OLD_FILES+=usr/share/man/man3/ENSURE.3.gz
 OLD_FILES+=usr/share/man/man3/ENSURE_ERR.3.gz
 OLD_FILES+=usr/share/man/man3/INSIST.3.gz
 OLD_FILES+=usr/share/man/man3/INSIST_ERR.3.gz
 OLD_FILES+=usr/share/man/man3/INVARIANT.3.gz
 OLD_FILES+=usr/share/man/man3/INVARIANT_ERR.3.gz
 OLD_FILES+=usr/share/man/man3/REQUIRE.3.gz
 OLD_FILES+=usr/share/man/man3/REQUIRE_ERR.3.gz
 OLD_FILES+=usr/share/man/man3/assertion_type_to_text.3.gz
 OLD_FILES+=usr/share/man/man3/assertions.3.gz
 OLD_FILES+=usr/share/man/man3/bitncmp.3.gz
 OLD_FILES+=usr/share/man/man3/evAddTime.3.gz
 OLD_FILES+=usr/share/man/man3/evCancelConn.3.gz
 OLD_FILES+=usr/share/man/man3/evCancelRW.3.gz
 OLD_FILES+=usr/share/man/man3/evClearIdleTimer.3.gz
 OLD_FILES+=usr/share/man/man3/evClearTimer.3.gz
 OLD_FILES+=usr/share/man/man3/evCmpTime.3.gz
 OLD_FILES+=usr/share/man/man3/evConnFunc.3.gz
 OLD_FILES+=usr/share/man/man3/evConnect.3.gz
 OLD_FILES+=usr/share/man/man3/evConsIovec.3.gz
 OLD_FILES+=usr/share/man/man3/evConsTime.3.gz
 OLD_FILES+=usr/share/man/man3/evCreate.3.gz
 OLD_FILES+=usr/share/man/man3/evDefer.3.gz
 OLD_FILES+=usr/share/man/man3/evDeselectFD.3.gz
 OLD_FILES+=usr/share/man/man3/evDestroy.3.gz
 OLD_FILES+=usr/share/man/man3/evDispatch.3.gz
 OLD_FILES+=usr/share/man/man3/evDo.3.gz
 OLD_FILES+=usr/share/man/man3/evDrop.3.gz
 OLD_FILES+=usr/share/man/man3/evFileFunc.3.gz
 OLD_FILES+=usr/share/man/man3/evGetNext.3.gz
 OLD_FILES+=usr/share/man/man3/evHold.3.gz
 OLD_FILES+=usr/share/man/man3/evInitID.3.gz
 OLD_FILES+=usr/share/man/man3/evLastEventTime.3.gz
 OLD_FILES+=usr/share/man/man3/evListen.3.gz
 OLD_FILES+=usr/share/man/man3/evMainLoop.3.gz
 OLD_FILES+=usr/share/man/man3/evNowTime.3.gz
 OLD_FILES+=usr/share/man/man3/evPrintf.3.gz
 OLD_FILES+=usr/share/man/man3/evRead.3.gz
 OLD_FILES+=usr/share/man/man3/evResetTimer.3.gz
 OLD_FILES+=usr/share/man/man3/evSelectFD.3.gz
 OLD_FILES+=usr/share/man/man3/evSetDebug.3.gz
 OLD_FILES+=usr/share/man/man3/evSetIdleTimer.3.gz
 OLD_FILES+=usr/share/man/man3/evSetTimer.3.gz
 OLD_FILES+=usr/share/man/man3/evStreamFunc.3.gz
 OLD_FILES+=usr/share/man/man3/evSubTime.3.gz
 OLD_FILES+=usr/share/man/man3/evTestID.3.gz
 OLD_FILES+=usr/share/man/man3/evTimeRW.3.gz
 OLD_FILES+=usr/share/man/man3/evTimeSpec.3.gz
 OLD_FILES+=usr/share/man/man3/evTimeVal.3.gz
 OLD_FILES+=usr/share/man/man3/evTimerFunc.3.gz
 OLD_FILES+=usr/share/man/man3/evTouchIdleTimer.3.gz
 OLD_FILES+=usr/share/man/man3/evTryAccept.3.gz
 OLD_FILES+=usr/share/man/man3/evUnhold.3.gz
 OLD_FILES+=usr/share/man/man3/evUntimeRW.3.gz
 OLD_FILES+=usr/share/man/man3/evUnwait.3.gz
 OLD_FILES+=usr/share/man/man3/evWaitFor.3.gz
 OLD_FILES+=usr/share/man/man3/evWaitFunc.3.gz
 OLD_FILES+=usr/share/man/man3/evWrite.3.gz
 OLD_FILES+=usr/share/man/man3/eventlib.3.gz
 OLD_FILES+=usr/share/man/man3/heap.3.gz
 OLD_FILES+=usr/share/man/man3/heap_decreased.3.gz
 OLD_FILES+=usr/share/man/man3/heap_delete.3.gz
 OLD_FILES+=usr/share/man/man3/heap_element.3.gz
 OLD_FILES+=usr/share/man/man3/heap_for_each.3.gz
 OLD_FILES+=usr/share/man/man3/heap_free.3.gz
 OLD_FILES+=usr/share/man/man3/heap_increased.3.gz
 OLD_FILES+=usr/share/man/man3/heap_insert.3.gz
 OLD_FILES+=usr/share/man/man3/heap_new.3.gz
 OLD_FILES+=usr/share/man/man3/log_add_channel.3.gz
 OLD_FILES+=usr/share/man/man3/log_category_is_active.3.gz
 OLD_FILES+=usr/share/man/man3/log_close_stream.3.gz
 OLD_FILES+=usr/share/man/man3/log_dec_references.3.gz
 OLD_FILES+=usr/share/man/man3/log_free_channel.3.gz
 OLD_FILES+=usr/share/man/man3/log_free_context.3.gz
 OLD_FILES+=usr/share/man/man3/log_get_filename.3.gz
 OLD_FILES+=usr/share/man/man3/log_get_stream.3.gz
 OLD_FILES+=usr/share/man/man3/log_inc_references.3.gz
 OLD_FILES+=usr/share/man/man3/log_new_context.3.gz
 OLD_FILES+=usr/share/man/man3/log_new_file_channel.3.gz
 OLD_FILES+=usr/share/man/man3/log_new_null_channel.3.gz
 OLD_FILES+=usr/share/man/man3/log_new_syslog_channel.3.gz
 OLD_FILES+=usr/share/man/man3/log_open_stream.3.gz
 OLD_FILES+=usr/share/man/man3/log_option.3.gz
 OLD_FILES+=usr/share/man/man3/log_remove_channel.3.gz
 OLD_FILES+=usr/share/man/man3/log_set_file_owner.3.gz
 OLD_FILES+=usr/share/man/man3/log_vwrite.3.gz
 OLD_FILES+=usr/share/man/man3/log_write.3.gz
 OLD_FILES+=usr/share/man/man3/logging.3.gz
 OLD_FILES+=usr/share/man/man3/memcluster.3.gz
 OLD_FILES+=usr/share/man/man3/memget.3.gz
 OLD_FILES+=usr/share/man/man3/memput.3.gz
 OLD_FILES+=usr/share/man/man3/memstats.3.gz
 OLD_FILES+=usr/share/man/man3/set_assertion_failure_callback.3.
 OLD_FILES+=usr/share/man/man3/sigwait.3.gz
 OLD_FILES+=usr/share/man/man3/tree_add.3.gz
 OLD_FILES+=usr/share/man/man3/tree_delete.3.gz
 OLD_FILES+=usr/share/man/man3/tree_init.3.gz
 OLD_FILES+=usr/share/man/man3/tree_mung.3.gz
 OLD_FILES+=usr/share/man/man3/tree_srch.3.gz
 OLD_FILES+=usr/share/man/man3/tree_trav.3.gz
 # 2004XXYY: OS internal libs, no ports use them, no need to use OLD_LIBS
 OLD_FILES+=lib/geom/geom_concat.so.1
 OLD_FILES+=lib/geom/geom_label.so.1
 OLD_FILES+=lib/geom/geom_nop.so.1
 OLD_FILES+=lib/geom/geom_stripe.so.1
 # 20040728: GCC 3.4.2
 OLD_DIRS+=usr/include/c++/3.3
 OLD_FILES+=usr/include/c++/3.3/FlexLexer.h
 OLD_FILES+=usr/include/c++/3.3/algorithm
 OLD_FILES+=usr/include/c++/3.3/backward/algo.h
 OLD_FILES+=usr/include/c++/3.3/backward/algobase.h
 OLD_FILES+=usr/include/c++/3.3/backward/alloc.h
 OLD_FILES+=usr/include/c++/3.3/backward/backward_warning.h
 OLD_FILES+=usr/include/c++/3.3/backward/bvector.h
 OLD_FILES+=usr/include/c++/3.3/backward/complex.h
 OLD_FILES+=usr/include/c++/3.3/backward/defalloc.h
 OLD_FILES+=usr/include/c++/3.3/backward/deque.h
 OLD_FILES+=usr/include/c++/3.3/backward/fstream.h
 OLD_FILES+=usr/include/c++/3.3/backward/function.h
 OLD_FILES+=usr/include/c++/3.3/backward/hash_map.h
 OLD_FILES+=usr/include/c++/3.3/backward/hash_set.h
 OLD_FILES+=usr/include/c++/3.3/backward/hashtable.h
 OLD_FILES+=usr/include/c++/3.3/backward/heap.h
 OLD_FILES+=usr/include/c++/3.3/backward/iomanip.h
 OLD_FILES+=usr/include/c++/3.3/backward/iostream.h
 OLD_FILES+=usr/include/c++/3.3/backward/istream.h
 OLD_FILES+=usr/include/c++/3.3/backward/iterator.h
 OLD_FILES+=usr/include/c++/3.3/backward/list.h
 OLD_FILES+=usr/include/c++/3.3/backward/map.h
 OLD_FILES+=usr/include/c++/3.3/backward/multimap.h
 OLD_FILES+=usr/include/c++/3.3/backward/multiset.h
 OLD_FILES+=usr/include/c++/3.3/backward/new.h
 OLD_FILES+=usr/include/c++/3.3/backward/ostream.h
 OLD_FILES+=usr/include/c++/3.3/backward/pair.h
 OLD_FILES+=usr/include/c++/3.3/backward/queue.h
 OLD_FILES+=usr/include/c++/3.3/backward/rope.h
 OLD_FILES+=usr/include/c++/3.3/backward/set.h
 OLD_FILES+=usr/include/c++/3.3/backward/slist.h
 OLD_FILES+=usr/include/c++/3.3/backward/stack.h
 OLD_FILES+=usr/include/c++/3.3/backward/stream.h
 OLD_FILES+=usr/include/c++/3.3/backward/streambuf.h
 OLD_FILES+=usr/include/c++/3.3/backward/strstream
 OLD_FILES+=usr/include/c++/3.3/backward/strstream.h
 OLD_FILES+=usr/include/c++/3.3/backward/tempbuf.h
 OLD_FILES+=usr/include/c++/3.3/backward/tree.h
 OLD_FILES+=usr/include/c++/3.3/backward/vector.h
 OLD_DIRS+=usr/include/c++/3.3/backward
 OLD_FILES+=usr/include/c++/3.3/bits/atomicity.h
 OLD_FILES+=usr/include/c++/3.3/bits/basic_file.h
 OLD_FILES+=usr/include/c++/3.3/bits/basic_ios.h
 OLD_FILES+=usr/include/c++/3.3/bits/basic_ios.tcc
 OLD_FILES+=usr/include/c++/3.3/bits/basic_string.h
 OLD_FILES+=usr/include/c++/3.3/bits/basic_string.tcc
 OLD_FILES+=usr/include/c++/3.3/bits/boost_concept_check.h
 OLD_FILES+=usr/include/c++/3.3/bits/c++config.h
 OLD_FILES+=usr/include/c++/3.3/bits/c++io.h
 OLD_FILES+=usr/include/c++/3.3/bits/c++locale.h
 OLD_FILES+=usr/include/c++/3.3/bits/c++locale_internal.h
 OLD_FILES+=usr/include/c++/3.3/bits/char_traits.h
 OLD_FILES+=usr/include/c++/3.3/bits/cmath.tcc
 OLD_FILES+=usr/include/c++/3.3/bits/codecvt.h
 OLD_FILES+=usr/include/c++/3.3/bits/codecvt_specializations.h
 OLD_FILES+=usr/include/c++/3.3/bits/concept_check.h
 OLD_FILES+=usr/include/c++/3.3/bits/cpp_type_traits.h
 OLD_FILES+=usr/include/c++/3.3/bits/ctype_base.h
 OLD_FILES+=usr/include/c++/3.3/bits/ctype_inline.h
 OLD_FILES+=usr/include/c++/3.3/bits/ctype_noninline.h
 OLD_FILES+=usr/include/c++/3.3/bits/deque.tcc
 OLD_FILES+=usr/include/c++/3.3/bits/fpos.h
 OLD_FILES+=usr/include/c++/3.3/bits/fstream.tcc
 OLD_FILES+=usr/include/c++/3.3/bits/functexcept.h
 OLD_FILES+=usr/include/c++/3.3/bits/generic_shadow.h
 OLD_FILES+=usr/include/c++/3.3/bits/gslice.h
 OLD_FILES+=usr/include/c++/3.3/bits/gslice_array.h
 OLD_FILES+=usr/include/c++/3.3/bits/gthr-default.h
 OLD_FILES+=usr/include/c++/3.3/bits/gthr-posix.h
 OLD_FILES+=usr/include/c++/3.3/bits/gthr-single.h
 OLD_FILES+=usr/include/c++/3.3/bits/gthr.h
 OLD_FILES+=usr/include/c++/3.3/bits/indirect_array.h
 OLD_FILES+=usr/include/c++/3.3/bits/ios_base.h
 OLD_FILES+=usr/include/c++/3.3/bits/istream.tcc
 OLD_FILES+=usr/include/c++/3.3/bits/list.tcc
 OLD_FILES+=usr/include/c++/3.3/bits/locale_classes.h
 OLD_FILES+=usr/include/c++/3.3/bits/locale_facets.h
 OLD_FILES+=usr/include/c++/3.3/bits/locale_facets.tcc
 OLD_FILES+=usr/include/c++/3.3/bits/localefwd.h
 OLD_FILES+=usr/include/c++/3.3/bits/mask_array.h
 OLD_FILES+=usr/include/c++/3.3/bits/messages_members.h
 OLD_FILES+=usr/include/c++/3.3/bits/os_defines.h
 OLD_FILES+=usr/include/c++/3.3/bits/ostream.tcc
 OLD_FILES+=usr/include/c++/3.3/bits/pthread_allocimpl.h
 OLD_FILES+=usr/include/c++/3.3/bits/slice.h
 OLD_FILES+=usr/include/c++/3.3/bits/slice_array.h
 OLD_FILES+=usr/include/c++/3.3/bits/sstream.tcc
 OLD_FILES+=usr/include/c++/3.3/bits/stl_algo.h
 OLD_FILES+=usr/include/c++/3.3/bits/stl_algobase.h
 OLD_FILES+=usr/include/c++/3.3/bits/stl_alloc.h
 OLD_FILES+=usr/include/c++/3.3/bits/stl_bvector.h
 OLD_FILES+=usr/include/c++/3.3/bits/stl_construct.h
 OLD_FILES+=usr/include/c++/3.3/bits/stl_deque.h
 OLD_FILES+=usr/include/c++/3.3/bits/stl_function.h
 OLD_FILES+=usr/include/c++/3.3/bits/stl_heap.h
 OLD_FILES+=usr/include/c++/3.3/bits/stl_iterator.h
 OLD_FILES+=usr/include/c++/3.3/bits/stl_iterator_base_funcs.h
 OLD_FILES+=usr/include/c++/3.3/bits/stl_iterator_base_types.h
 OLD_FILES+=usr/include/c++/3.3/bits/stl_list.h
 OLD_FILES+=usr/include/c++/3.3/bits/stl_map.h
 OLD_FILES+=usr/include/c++/3.3/bits/stl_multimap.h
 OLD_FILES+=usr/include/c++/3.3/bits/stl_multiset.h
 OLD_FILES+=usr/include/c++/3.3/bits/stl_numeric.h
 OLD_FILES+=usr/include/c++/3.3/bits/stl_pair.h
 OLD_FILES+=usr/include/c++/3.3/bits/stl_pthread_alloc.h
 OLD_FILES+=usr/include/c++/3.3/bits/stl_queue.h
 OLD_FILES+=usr/include/c++/3.3/bits/stl_raw_storage_iter.h
 OLD_FILES+=usr/include/c++/3.3/bits/stl_relops.h
 OLD_FILES+=usr/include/c++/3.3/bits/stl_set.h
 OLD_FILES+=usr/include/c++/3.3/bits/stl_stack.h
 OLD_FILES+=usr/include/c++/3.3/bits/stl_tempbuf.h
 OLD_FILES+=usr/include/c++/3.3/bits/stl_threads.h
 OLD_FILES+=usr/include/c++/3.3/bits/stl_tree.h
 OLD_FILES+=usr/include/c++/3.3/bits/stl_uninitialized.h
 OLD_FILES+=usr/include/c++/3.3/bits/stl_vector.h
 OLD_FILES+=usr/include/c++/3.3/bits/stream_iterator.h
 OLD_FILES+=usr/include/c++/3.3/bits/streambuf.tcc
 OLD_FILES+=usr/include/c++/3.3/bits/streambuf_iterator.h
 OLD_FILES+=usr/include/c++/3.3/bits/stringfwd.h
 OLD_FILES+=usr/include/c++/3.3/bits/time_members.h
 OLD_FILES+=usr/include/c++/3.3/bits/type_traits.h
 OLD_FILES+=usr/include/c++/3.3/bits/valarray_array.h
 OLD_FILES+=usr/include/c++/3.3/bits/valarray_array.tcc
 OLD_FILES+=usr/include/c++/3.3/bits/valarray_meta.h
 OLD_FILES+=usr/include/c++/3.3/bits/vector.tcc
 OLD_DIRS+=usr/include/c++/3.3/bits
 OLD_FILES+=usr/include/c++/3.3/bitset
 OLD_FILES+=usr/include/c++/3.3/cassert
 OLD_FILES+=usr/include/c++/3.3/cctype
 OLD_FILES+=usr/include/c++/3.3/cerrno
 OLD_FILES+=usr/include/c++/3.3/cfloat
 OLD_FILES+=usr/include/c++/3.3/ciso646
 OLD_FILES+=usr/include/c++/3.3/climits
 OLD_FILES+=usr/include/c++/3.3/clocale
 OLD_FILES+=usr/include/c++/3.3/cmath
 OLD_FILES+=usr/include/c++/3.3/complex
 OLD_FILES+=usr/include/c++/3.3/csetjmp
 OLD_FILES+=usr/include/c++/3.3/csignal
 OLD_FILES+=usr/include/c++/3.3/cstdarg
 OLD_FILES+=usr/include/c++/3.3/cstddef
 OLD_FILES+=usr/include/c++/3.3/cstdio
 OLD_FILES+=usr/include/c++/3.3/cstdlib
 OLD_FILES+=usr/include/c++/3.3/cstring
 OLD_FILES+=usr/include/c++/3.3/ctime
 OLD_FILES+=usr/include/c++/3.3/cwchar
 OLD_FILES+=usr/include/c++/3.3/cwctype
 OLD_FILES+=usr/include/c++/3.3/cxxabi.h
 OLD_FILES+=usr/include/c++/3.3/deque
 OLD_FILES+=usr/include/c++/3.3/exception
 OLD_FILES+=usr/include/c++/3.3/exception_defines.h
 OLD_FILES+=usr/include/c++/3.3/ext/algorithm
 OLD_FILES+=usr/include/c++/3.3/ext/enc_filebuf.h
 OLD_FILES+=usr/include/c++/3.3/ext/functional
 OLD_FILES+=usr/include/c++/3.3/ext/hash_map
 OLD_FILES+=usr/include/c++/3.3/ext/hash_set
 OLD_FILES+=usr/include/c++/3.3/ext/iterator
 OLD_FILES+=usr/include/c++/3.3/ext/memory
 OLD_FILES+=usr/include/c++/3.3/ext/numeric
 OLD_FILES+=usr/include/c++/3.3/ext/rb_tree
 OLD_FILES+=usr/include/c++/3.3/ext/rope
 OLD_FILES+=usr/include/c++/3.3/ext/ropeimpl.h
 OLD_FILES+=usr/include/c++/3.3/ext/slist
 OLD_FILES+=usr/include/c++/3.3/ext/stdio_filebuf.h
 OLD_FILES+=usr/include/c++/3.3/ext/stl_hash_fun.h
 OLD_FILES+=usr/include/c++/3.3/ext/stl_hashtable.h
 OLD_FILES+=usr/include/c++/3.3/ext/stl_rope.h
 OLD_DIRS+=usr/include/c++/3.3/ext
 OLD_FILES+=usr/include/c++/3.3/fstream
 OLD_FILES+=usr/include/c++/3.3/functional
 OLD_FILES+=usr/include/c++/3.3/iomanip
 OLD_FILES+=usr/include/c++/3.3/ios
 OLD_FILES+=usr/include/c++/3.3/iosfwd
 OLD_FILES+=usr/include/c++/3.3/iostream
 OLD_FILES+=usr/include/c++/3.3/istream
 OLD_FILES+=usr/include/c++/3.3/iterator
 OLD_FILES+=usr/include/c++/3.3/limits
 OLD_FILES+=usr/include/c++/3.3/list
 OLD_FILES+=usr/include/c++/3.3/locale
 OLD_FILES+=usr/include/c++/3.3/map
 OLD_FILES+=usr/include/c++/3.3/memory
 OLD_FILES+=usr/include/c++/3.3/new
 OLD_FILES+=usr/include/c++/3.3/numeric
 OLD_FILES+=usr/include/c++/3.3/ostream
 OLD_FILES+=usr/include/c++/3.3/queue
 OLD_FILES+=usr/include/c++/3.3/set
 OLD_FILES+=usr/include/c++/3.3/sstream
 OLD_FILES+=usr/include/c++/3.3/stack
 OLD_FILES+=usr/include/c++/3.3/stdexcept
 OLD_FILES+=usr/include/c++/3.3/streambuf
 OLD_FILES+=usr/include/c++/3.3/string
 OLD_FILES+=usr/include/c++/3.3/typeinfo
 OLD_FILES+=usr/include/c++/3.3/utility
 OLD_FILES+=usr/include/c++/3.3/valarray
 OLD_FILES+=usr/include/c++/3.3/vector
 # 20040713: fla(4) removed.
 OLD_FILES+=usr/share/man/man4/fla.4.gz
 # 200407XX
 OLD_FILES+=usr/sbin/kernbb
 OLD_FILES+=usr/sbin/ntp-genkeys
 OLD_FILES+=usr/sbin/ntptimeset
 OLD_FILES+=usr/share/man/man8/kernbb.8.gz
 OLD_FILES+=usr/share/man/man8/ntp-genkeys.8.gz
 # 20040627: usbdevs.h and usbdevs_data.h removal
 OLD_FILES+=usr/include/dev/usb/usbdevs.h
 OLD_FILES+=usr/include/dev/usb/usbdevs_data.h
 # 200406XX
 OLD_FILES+=usr/bin/gasp
 OLD_FILES+=usr/bin/gdbreplay
 OLD_FILES+=usr/share/man/man1/gasp.1.gz
 OLD_FILES+=sbin/mountd
 OLD_FILES+=sbin/mount_fdesc
 OLD_FILES+=sbin/mount_umap
 OLD_FILES+=sbin/mount_union
 OLD_FILES+=sbin/mount_msdos
 OLD_FILES+=sbin/mount_null
 OLD_FILES+=sbin/mount_kernfs
 # 200405XX: arl
 OLD_FILES+=usr/sbin/arlconfig
 OLD_FILES+=usr/share/man/man8/arlconfig.8.gz
 # 200403XX
 OLD_FILES+=bin/raidctl
 OLD_FILES+=sbin/raidctl
 OLD_FILES+=usr/bin/sasc
 OLD_FILES+=usr/sbin/sgsc
 OLD_FILES+=usr/sbin/stlload
 OLD_FILES+=usr/sbin/stlstats
 OLD_FILES+=usr/share/man/man1/sasc.1.gz
 OLD_FILES+=usr/share/man/man1/sgsc.1.gz
 OLD_FILES+=usr/share/man/man4/i386/stl.4.gz
 OLD_FILES+=usr/share/man/man8/raidctl.8.gz
 # 20040229: clean_environment() was removed after 3 days
 OLD_FILES+=usr/share/man/man3/clean_environment.3.gz
 # 20040119: installed as `isdntel' in newer systems
 OLD_FILES+=etc/isdn/isdntel.sh
 # 200XYYZZ: /lib transition clitches
 OLD_FILES+=lib/libalias.so
 OLD_FILES+=lib/libatm.so
 OLD_FILES+=lib/libbsdxml.so
 OLD_FILES+=lib/libc.so
 OLD_FILES+=lib/libcam.so
 OLD_FILES+=lib/libcrypt.so
 OLD_FILES+=lib/libcrypto.so
 OLD_FILES+=lib/libdevstat.so
 OLD_FILES+=lib/libedit.so
 OLD_FILES+=lib/libgeom.so
 OLD_FILES+=lib/libipsec.so
 OLD_FILES+=lib/libipx.so
 OLD_FILES+=lib/libkvm.so
 OLD_FILES+=lib/libm.so
 OLD_FILES+=lib/libmd.so
 OLD_FILES+=lib/libncurses.so
 OLD_FILES+=lib/libreadline.so
 OLD_FILES+=lib/libsbuf.so
 OLD_FILES+=lib/libufs.so
 OLD_FILES+=lib/libz.so
 # 200312XX
 OLD_FILES+=bin/cxconfig
 OLD_FILES+=sbin/cxconfig
 OLD_FILES+=usr/share/man/man8/cxconfig.8.gz
 # 20031016: MULTI_DRIVER_MODULE macro removed
 OLD_FILES+=usr/share/man/man9/MULTI_DRIVER_MODULE.9.gz
 # 200309XX
 OLD_FILES+=usr/bin/symorder
 OLD_FILES+=usr/share/man/man1/symorder.1.gz
 # 200308XX
 OLD_FILES+=usr/sbin/amldb
 OLD_FILES+=usr/share/man/man8/amldb.8.gz
 # 200307XX
 OLD_FILES+=sbin/mount_nwfs
 OLD_FILES+=sbin/mount_portalfs
 OLD_FILES+=sbin/mount_smbfs
 # 200306XX
 OLD_FILES+=usr/sbin/dev_mkdb
 OLD_FILES+=usr/share/man/man8/dev_mkdb.8.gz
 # 200304XX
 OLD_FILES+=usr/lib/libcipher.a
 OLD_FILES+=usr/lib/libcipher.so
 OLD_FILES+=usr/lib/libcipher_p.a
 OLD_FILES+=usr/lib/libgmp.a
 OLD_FILES+=usr/lib/libgmp.so
 OLD_FILES+=usr/lib/libgmp_p.a
 OLD_FILES+=usr/lib/libperl.a
 OLD_FILES+=usr/lib/libperl.so
 OLD_FILES+=usr/lib/libperl_p.a
 OLD_FILES+=usr/lib/libposix1e.a
 OLD_FILES+=usr/lib/libposix1e.so
 OLD_FILES+=usr/lib/libposix1e_p.a
 OLD_FILES+=usr/lib/libskey.a
 OLD_FILES+=usr/lib/libskey.so
 OLD_FILES+=usr/lib/libskey_p.a
 OLD_FILES+=usr/libexec/tradcpp0
 OLD_FILES+=usr/libexec/cpp0
 # 200304XX: removal of xten
 OLD_FILES+=usr/libexec/xtend
 OLD_FILES+=usr/sbin/xten
 OLD_FILES+=usr/share/man/man1/xten.1.gz
 OLD_FILES+=usr/share/man/man8/xtend.8.gz
 # 200303XX
 OLD_FILES+=usr/lib/libacl.so
 OLD_FILES+=usr/lib/libdescrypt.so
 OLD_FILES+=usr/lib/libf2c.so
 OLD_FILES+=usr/lib/libg++.so
 OLD_FILES+=usr/lib/libkdb.so
 OLD_FILES+=usr/lib/librsaINTL.so
 OLD_FILES+=usr/lib/libscrypt.so
 OLD_FILES+=usr/lib/libss.so
 # 200302XX
 OLD_FILES+=usr/lib/libacl.a
 OLD_FILES+=usr/lib/libacl_p.a
 OLD_FILES+=usr/lib/libkadm.a
 OLD_FILES+=usr/lib/libkadm.so
 OLD_FILES+=usr/lib/libkadm_p.a
 OLD_FILES+=usr/lib/libkafs.a
 OLD_FILES+=usr/lib/libkafs.so
 OLD_FILES+=usr/lib/libkafs_p.a
 OLD_FILES+=usr/lib/libkdb.a
 OLD_FILES+=usr/lib/libkdb_p.a
 OLD_FILES+=usr/lib/libkrb.a
 OLD_FILES+=usr/lib/libkrb.so
 OLD_FILES+=usr/lib/libkrb_p.a
 OLD_FILES+=usr/share/man/man3/SSL_CIPHER_get_name.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_COMP_add_compression_method.3
 OLD_FILES+=usr/share/man/man3/SSL_CTX_add_extra_chain_cert.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_CTX_add_session.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_CTX_ctrl.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_CTX_flush_sessions.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_CTX_free.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_CTX_get_verify_mode.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_CTX_load_verify_locations.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_CTX_new.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_CTX_sess_number.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_CTX_sess_set_cache_size.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_CTX_sess_set_get_cb.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_CTX_sessions.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_CTX_set_cert_store.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_CTX_set_cert_verify_callback.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_CTX_set_cipher_list.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_CTX_set_client_CA_list.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_CTX_set_client_cert_cb.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_CTX_set_default_passwd_cb.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_CTX_set_generate_session_id.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_CTX_set_info_callback.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_CTX_set_max_cert_list.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_CTX_set_mode.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_CTX_set_msg_callback.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_CTX_set_options.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_CTX_set_quiet_shutdown.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_CTX_set_session_cache_mode.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_CTX_set_session_id_context.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_CTX_set_ssl_version.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_CTX_set_timeout.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_CTX_set_tmp_dh_callback.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_CTX_set_tmp_rsa_callback.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_CTX_set_verify.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_CTX_use_certificate.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_SESSION_free.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_SESSION_get_ex_new_index.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_SESSION_get_time.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_accept.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_alert_type_string.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_clear.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_connect.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_do_handshake.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_free.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_get_SSL_CTX.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_get_ciphers.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_get_client_CA_list.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_get_current_cipher.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_get_default_timeout.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_get_error.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_get_ex_data_X509_STORE_CTX_idx.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_get_ex_new_index.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_get_fd.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_get_peer_cert_chain.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_get_peer_certificate.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_get_rbio.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_get_session.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_get_verify_result.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_get_version.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_library_init.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_load_client_CA_file.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_new.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_pending.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_read.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_rstate_string.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_session_reused.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_set_bio.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_set_connect_state.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_set_fd.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_set_session.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_set_shutdown.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_set_verify_result.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_shutdown.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_state_string.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_want.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_write.3.gz
 OLD_FILES+=usr/share/man/man3/d2i_SSL_SESSION.3.gz
 # 200301XX
 OLD_FILES+=usr/share/man/man3/des_3cbc_encrypt.3.gz
 OLD_FILES+=usr/share/man/man3/des_3ecb_encrypt.3.gz
 OLD_FILES+=usr/share/man/man3/des_cbc_cksum.3.gz
 OLD_FILES+=usr/share/man/man3/des_cbc_encrypt.3.gz
 OLD_FILES+=usr/share/man/man3/des_cfb_encrypt.3.gz
 OLD_FILES+=usr/share/man/man3/des_ecb_encrypt.3.gz
 OLD_FILES+=usr/share/man/man3/des_enc_read.3.gz
 OLD_FILES+=usr/share/man/man3/des_enc_write.3.gz
 OLD_FILES+=usr/share/man/man3/des_is_weak_key.3.gz
 OLD_FILES+=usr/share/man/man3/des_key_sched.3.gz
 OLD_FILES+=usr/share/man/man3/des_ofb_encrypt.3.gz
 OLD_FILES+=usr/share/man/man3/des_pcbc_encrypt.3.gz
 OLD_FILES+=usr/share/man/man3/des_quad_cksum.3.gz
 OLD_FILES+=usr/share/man/man3/des_random_key.3.gz
 OLD_FILES+=usr/share/man/man3/des_read_2password.3.gz
 OLD_FILES+=usr/share/man/man3/des_read_password.3.gz
 OLD_FILES+=usr/share/man/man3/des_read_pw_string.3.gz
 OLD_FILES+=usr/share/man/man3/des_set_key.3.gz
 OLD_FILES+=usr/share/man/man3/des_set_odd_parity.3.gz
 OLD_FILES+=usr/share/man/man3/des_string_to_2key.3.gz
 OLD_FILES+=usr/share/man/man3/des_string_to_key.3.gz
 # 200212XX
 OLD_FILES+=usr/sbin/kenv
 OLD_FILES+=usr/bin/kenv
 OLD_FILES+=usr/sbin/elf2aout
 # 200210XX
 OLD_FILES+=usr/include/libusbhid.h
 OLD_FILES+=usr/share/man/man3/All_FreeBSD.3.gz
 OLD_FILES+=usr/share/man/man3/CheckRules.3.gz
 OLD_FILES+=usr/share/man/man3/ChunkCanBeRoot.3.gz
 OLD_FILES+=usr/share/man/man3/Clone_Disk.3.gz
 OLD_FILES+=usr/share/man/man3/Collapse_Chunk.3.gz
 OLD_FILES+=usr/share/man/man3/Collapse_Disk.3.gz
 OLD_FILES+=usr/share/man/man3/Create_Chunk.3.gz
 OLD_FILES+=usr/share/man/man3/Create_Chunk_DWIM.3.gz
 OLD_FILES+=usr/share/man/man3/Cyl_Aligned.3.gz
 OLD_FILES+=usr/share/man/man3/Debug_Disk.3.gz
 OLD_FILES+=usr/share/man/man3/Delete_Chunk.3.gz
 OLD_FILES+=usr/share/man/man3/Disk_Names.3.gz
 OLD_FILES+=usr/share/man/man3/Free_Disk.3.gz
 OLD_FILES+=usr/share/man/man3/MakeDev.3.gz
 OLD_FILES+=usr/share/man/man3/MakeDevDisk.3.gz
 OLD_FILES+=usr/share/man/man3/Next_Cyl_Aligned.3.gz
 OLD_FILES+=usr/share/man/man3/Next_Track_Aligned.3.gz
 OLD_FILES+=usr/share/man/man3/Open_Disk.3.gz
 OLD_FILES+=usr/share/man/man3/Prev_Cyl_Aligned.3.gz
 OLD_FILES+=usr/share/man/man3/Prev_Track_Aligned.3.gz
 OLD_FILES+=usr/share/man/man3/Set_Bios_Geom.3.gz
 OLD_FILES+=usr/share/man/man3/Set_Boot_Blocks.3.gz
 OLD_FILES+=usr/share/man/man3/Set_Boot_Mgr.3.gz
 OLD_FILES+=usr/share/man/man3/ShowChunkFlags.3.gz
 OLD_FILES+=usr/share/man/man3/Track_Aligned.3.gz
 OLD_FILES+=usr/share/man/man3/Write_Disk.3.gz
 OLD_FILES+=usr/share/man/man3/slice_type_name.3.gz
 # 200210XX: most games moved to ports
 OLD_FILES+=usr/share/man/man6/adventure.6.gz
 OLD_FILES+=usr/share/man/man6/arithmetic.6.gz
 OLD_FILES+=usr/share/man/man6/atc.6.gz
 OLD_FILES+=usr/share/man/man6/backgammon.6.gz
 OLD_FILES+=usr/share/man/man6/battlestar.6.gz
 OLD_FILES+=usr/share/man/man6/bs.6.gz
 OLD_FILES+=usr/share/man/man6/canfield.6.gz
 OLD_FILES+=usr/share/man/man6/cfscores.6.gz
 OLD_FILES+=usr/share/man/man6/cribbage.6.gz
 OLD_FILES+=usr/share/man/man6/fish.6.gz
 OLD_FILES+=usr/share/man/man6/hack.6.gz
 OLD_FILES+=usr/share/man/man6/hangman.6.gz
 OLD_FILES+=usr/share/man/man6/larn.6.gz
 OLD_FILES+=usr/share/man/man6/mille.6.gz
 OLD_FILES+=usr/share/man/man6/phantasia.6.gz
 OLD_FILES+=usr/share/man/man6/piano.6.gz
 OLD_FILES+=usr/share/man/man6/pig.6.gz
 OLD_FILES+=usr/share/man/man6/quiz.6.gz
 OLD_FILES+=usr/share/man/man6/rain.6.gz
 OLD_FILES+=usr/share/man/man6/robots.6.gz
 OLD_FILES+=usr/share/man/man6/rogue.6.gz
 OLD_FILES+=usr/share/man/man6/sail.6.gz
 OLD_FILES+=usr/share/man/man6/snake.6.gz
 OLD_FILES+=usr/share/man/man6/snscore.6.gz
 OLD_FILES+=usr/share/man/man6/trek.6.gz
 OLD_FILES+=usr/share/man/man6/wargames.6.gz
 OLD_FILES+=usr/share/man/man6/worm.6.gz
 OLD_FILES+=usr/share/man/man6/worms.6.gz
 OLD_FILES+=usr/share/man/man6/wump.6.gz
 # 200207XX
 OLD_FILES+=usr/share/man/man1aout/ar.1aout.gz
 OLD_FILES+=usr/share/man/man1aout/as.1aout.gz
 OLD_FILES+=usr/share/man/man1aout/ld.1aout.gz
 OLD_FILES+=usr/share/man/man1aout/nm.1aout.gz
 OLD_FILES+=usr/share/man/man1aout/ranlib.1aout.gz
 OLD_FILES+=usr/share/man/man1aout/size.1aout.gz
 OLD_FILES+=usr/share/man/man1aout/strings.1aout.gz
 OLD_FILES+=usr/share/man/man1aout/strip.1aout.gz
 OLD_FILES+=bin/mountd
 OLD_FILES+=bin/nfsd
 # 20020707 sbin/nfsd -> usr.sbin/nfsd
 OLD_FILES+=sbin/nfsd
 # 200206XX
 OLD_FILES+=usr/lib/libpam_ssh.a
 OLD_FILES+=usr/lib/libpam_ssh_p.a
 OLD_FILES+=usr/bin/help
 OLD_FILES+=usr/bin/sccs
 .if ${TARGET_ARCH} != "amd64" && ${TARGET} != "arm" && ${TARGET_ARCH} != "i386" && ${TARGET} != "powerpc"
 OLD_FILES+=usr/bin/gdbserver
 .endif
 OLD_FILES+=usr/bin/ssh-keysign
 OLD_FILES+=usr/sbin/gifconfig
 OLD_FILES+=usr/sbin/prefix
 # 200205XX
 OLD_FILES+=usr/bin/doscmd
 # 200204XX
 OLD_FILES+=usr/bin/a2p
 OLD_FILES+=usr/bin/ptx
 OLD_FILES+=usr/bin/pod2text
 OLD_FILES+=usr/bin/pod2man
 OLD_FILES+=usr/bin/pod2latex
 OLD_FILES+=usr/bin/pod2html
 OLD_FILES+=usr/bin/h2ph
 OLD_FILES+=usr/bin/dprofpp
 OLD_FILES+=usr/bin/c2ph
 OLD_FILES+=usr/bin/h2xs
 OLD_FILES+=usr/bin/pl2pm
 OLD_FILES+=usr/bin/splain
 OLD_FILES+=usr/bin/s2p
 OLD_FILES+=usr/bin/find2perl
 OLD_FILES+=usr/sbin/pkg_update
 OLD_FILES+=usr/sbin/scriptdump
 # 20020409 GC kget(1), userconfig is long dead.
 OLD_FILES+=sbin/kget
 OLD_FILES+=usr/share/man/man8/kget.8.gz
 # 200203XX
 OLD_FILES+=usr/lib/libss.a
 OLD_FILES+=usr/lib/libss_p.a
 OLD_FILES+=usr/lib/libtelnet.a
 OLD_FILES+=usr/lib/libtelnet_p.a
 OLD_FILES+=usr/sbin/diskpart
 # 200202XX
 OLD_FILES+=usr/bin/gprof4
 # 200201XX
 OLD_FILES+=usr/sbin/linux
 # 2001XXXX
 OLD_FILES+=usr/bin/joy
 OLD_FILES+=usr/sbin/ibcs2
 OLD_FILES+=usr/sbin/svr4
 OLD_FILES+=usr/bin/chflags
 OLD_FILES+=usr/sbin/uuconv
 OLD_FILES+=usr/sbin/uuchk
 OLD_FILES+=usr/sbin/portmap
 OLD_FILES+=usr/sbin/pmap_set
 OLD_FILES+=usr/sbin/pmap_dump
 OLD_FILES+=usr/sbin/mcon
 OLD_FILES+=usr/sbin/stlstty
 OLD_FILES+=usr/sbin/ispppcontrol
 OLD_FILES+=usr/sbin/rndcontrol
 # 20011001: UUCP migration to ports
 OLD_FILES+=usr/bin/uucp
 OLD_FILES+=usr/bin/uulog
 OLD_FILES+=usr/bin/uuname
 OLD_FILES+=usr/bin/uupick
 OLD_FILES+=usr/bin/uusched
 OLD_FILES+=usr/bin/uustat
 OLD_FILES+=usr/bin/uuto
 OLD_FILES+=usr/bin/uux
 OLD_FILES+=usr/libexec/uucp/uucico
 OLD_FILES+=usr/libexec/uucp/uuxqt
 OLD_FILES+=usr/libexec/uucpd
 OLD_FILES+=usr/share/man/man1/uuconv.1.gz
 OLD_FILES+=usr/share/man/man1/uucp.1.gz
 OLD_FILES+=usr/share/man/man1/uulog.1.gz
 OLD_FILES+=usr/share/man/man1/uuname.1.gz
 OLD_FILES+=usr/share/man/man1/uupick.1.gz
 OLD_FILES+=usr/share/man/man1/uustat.1.gz
 OLD_FILES+=usr/share/man/man1/uuto.1.gz
 OLD_FILES+=usr/share/man/man1/uux.1.gz
 OLD_FILES+=usr/share/man/man8/uuchk.8.gz
 OLD_FILES+=usr/share/man/man8/uucico.8.gz
 OLD_FILES+=usr/share/man/man8/uucpd.8.gz
 OLD_FILES+=usr/share/man/man8/uusched.8.gz
 OLD_FILES+=usr/share/man/man8/uuxqt.8.gz
 # 20010523 mount_portal -> mount_portalfs
 OLD_FILES+=sbin/mount_portal
 OLD_FILES+=usr/share/man/man8/mount_portal.8.gz
 # 200104XX
 OLD_FILES+=usr/lib/libdescrypt.a
 OLD_FILES+=usr/lib/libscrypt.a
 OLD_FILES+=usr/lib/libscrypt_p.a
 OLD_FILES+=usr/sbin/pim6stat
 OLD_FILES+=usr/sbin/pim6sd
 OLD_FILES+=usr/sbin/pim6dd
 # 20010217
 OLD_FILES+=usr/share/doc/bind/misc/dns-setup
 # 20001200
 OLD_FILES+=usr/lib/libgcc_r_pic.a
 # 200009XX
 OLD_FILES+=usr/lib/libRSAglue.a
 OLD_FILES+=usr/lib/libRSAglue.so
 OLD_FILES+=usr/lib/librsaINTL.a
 OLD_FILES+=usr/lib/librsaUSA.a
 OLD_FILES+=usr/lib/librsaUSA.so
 # 200002XX ?
 OLD_FILES+=usr/lib/libf2c.a
 OLD_FILES+=usr/lib/libf2c_p.a
 OLD_FILES+=usr/lib/libg++.a
 OLD_FILES+=usr/lib/libg++_p.a
 # 20001006
 OLD_FILES+=usr/bin/miniperl
 # 20000810
 OLD_FILES+=usr/bin/sperl
 # 200001XX
 OLD_FILES+=usr/sbin/apmconf
 ## unsorted
 # do we still support aout builds?
 #OLD_FILES+=usr/lib/aout/c++rt0.o
 #OLD_FILES+=usr/lib/aout/crt0.o
 #OLD_FILES+=usr/lib/aout/gcrt0.o
 #OLD_FILES+=usr/lib/aout/scrt0.o
 #OLD_FILES+=usr/lib/aout/sgcrt0.o
 OLD_FILES+=usr/lib/pam_ftp.so
 OLD_FILES+=usr/share/man/man1/CA.pl.1.gz
 OLD_FILES+=usr/share/man/man1/asn1parse.1.gz
 OLD_FILES+=usr/share/man/man1/ca.1.gz
 OLD_FILES+=usr/share/man/man1/ciphers.1.gz
 OLD_FILES+=usr/share/man/man1/config.1.gz
 OLD_FILES+=usr/share/man/man1/crl.1.gz
 OLD_FILES+=usr/share/man/man1/crl2pkcs7.1.gz
 OLD_FILES+=usr/share/man/man1/dgst.1.gz
 OLD_FILES+=usr/share/man/man1/dhparam.1.gz
 OLD_FILES+=usr/share/man/man1/doscmd.1.gz
 OLD_FILES+=usr/share/man/man1/dsa.1.gz
 OLD_FILES+=usr/share/man/man1/dsaparam.1.gz
 OLD_FILES+=usr/share/man/man1/enc.1.gz
 OLD_FILES+=usr/share/man/man1/gendsa.1.gz
 OLD_FILES+=usr/share/man/man1/genrsa.1.gz
 OLD_FILES+=usr/share/man/man1/getNAME.1.gz
 OLD_FILES+=usr/share/man/man1/nseq.1.gz
 OLD_FILES+=usr/share/man/man1/ocsp.1.gz
 OLD_FILES+=usr/share/man/man1/openssl.1.gz
 OLD_FILES+=usr/share/man/man1/pkcs12.1.gz
 OLD_FILES+=usr/share/man/man1/pkcs7.1.gz
 OLD_FILES+=usr/share/man/man1/pkcs8.1.gz
 OLD_FILES+=usr/share/man/man1/rand.1.gz
 OLD_FILES+=usr/share/man/man1/req.1.gz
 OLD_FILES+=usr/share/man/man1/rsa.1.gz
 OLD_FILES+=usr/share/man/man1/rsautl.1.gz
 OLD_FILES+=usr/share/man/man1/s_client.1.gz
 OLD_FILES+=usr/share/man/man1/s_server.1.gz
 OLD_FILES+=usr/share/man/man1/sess_id.1.gz
 OLD_FILES+=usr/share/man/man1/smime.1.gz
 OLD_FILES+=usr/share/man/man1/speed.1.gz
 OLD_FILES+=usr/share/man/man1/spkac.1.gz
 OLD_FILES+=usr/share/man/man1/verify.1.gz
 OLD_FILES+=usr/share/man/man1/version.1.gz
 OLD_FILES+=usr/share/man/man1/x509.1.gz
 OLD_FILES+=usr/share/man/man3/SSL_COMP_add_compression_method.3.gz
 OLD_FILES+=usr/share/man/man3/SSL_CTX_get_ex_new_index.3.gz
 OLD_FILES+=usr/share/man/man3/archive_entry_dup.3.gz
 OLD_FILES+=usr/share/man/man3/archive_entry_set_tartype.3.gz
 OLD_FILES+=usr/share/man/man3/archive_entry_tartype.3.gz
 OLD_FILES+=usr/share/man/man3/archive_read_data_into_file.3.gz
 OLD_FILES+=usr/share/man/man3/archive_read_open_tar.3.gz
 OLD_FILES+=usr/share/man/man3/archive_read_support_format_gnutar.3.gz
 OLD_FILES+=usr/share/man/man3/cipher.3.gz
 OLD_FILES+=usr/share/man/man3/des_cipher.3.gz
 OLD_FILES+=usr/share/man/man3/des_setkey.3.gz
 OLD_FILES+=usr/share/man/man3/encrypt.3.gz
 OLD_FILES+=usr/share/man/man3/endvfsent.3.gz
 OLD_FILES+=usr/share/man/man3/getvfsbytype.3.gz
 OLD_FILES+=usr/share/man/man3/getvfsent.3.gz
 OLD_FILES+=usr/share/man/man3/isnanf.3.gz
 OLD_FILES+=usr/share/man/man3/libautofs.3.gz
 OLD_FILES+=usr/share/man/man3/pthread_attr_setsstack.3.gz
 OLD_FILES+=usr/share/man/man3/pthread_getcancelstate.3.gz
 OLD_FILES+=usr/share/man/man3/pthread_mutexattr_getpshared.3.gz
 OLD_FILES+=usr/share/man/man3/pthread_mutexattr_setpshared.3.gz
 OLD_FILES+=usr/share/man/man3/set_assertion_failure_callback.3.gz
 OLD_FILES+=usr/share/man/man3/setkey.3.gz
 OLD_FILES+=usr/share/man/man3/setvfsent.3.gz
 OLD_FILES+=usr/share/man/man3/ssl.3.gz
 OLD_FILES+=usr/share/man/man3/vfsisloadable.3.gz
 OLD_FILES+=usr/share/man/man3/vfsload.3.gz
 OLD_FILES+=usr/share/man/man4/als4000.4.gz
 OLD_FILES+=usr/share/man/man4/csa.4.gz
 OLD_FILES+=usr/share/man/man4/emu10k1.4.gz
 OLD_FILES+=usr/share/man/man4/euc.4.gz
 OLD_FILES+=usr/share/man/man4/gusc.4.gz
 OLD_FILES+=usr/share/man/man4/if_fwp.4.gz
 OLD_FILES+=usr/share/man/man4/lomac.4.gz
 OLD_FILES+=usr/share/man/man4/maestro3.4.gz
 OLD_FILES+=usr/share/man/man4/raid.4.gz
 OLD_FILES+=usr/share/man/man4/sbc.4.gz
 OLD_FILES+=usr/share/man/man4/sd.4.gz
 OLD_FILES+=usr/share/man/man4/snc.4.gz
 OLD_FILES+=usr/share/man/man4/st.4.gz
 OLD_FILES+=usr/share/man/man4/uaudio.4.gz
 OLD_FILES+=usr/share/man/man4/utf2.4.gz
 OLD_FILES+=usr/share/man/man4/vinumdebug.4.gz
 OLD_FILES+=usr/share/man/man5/disklabel.5.gz
 OLD_FILES+=usr/share/man/man5/dm.conf.5.gz
 OLD_FILES+=usr/share/man/man5/ranlib.5.gz
 OLD_FILES+=usr/share/man/man5/utf2.5.gz
 OLD_FILES+=usr/share/man/man7/groff_mwww.7.gz
 OLD_FILES+=usr/share/man/man7/mmroff.7.gz
 OLD_FILES+=usr/share/man/man7/mwww.7.gz
 OLD_FILES+=usr/share/man/man8/apm.8.gz
 OLD_FILES+=usr/share/man/man8/apmconf.8.gz
 OLD_FILES+=usr/share/man/man8/apmd.8.gz
 OLD_FILES+=usr/share/man/man8/dm.8.gz
 OLD_FILES+=usr/share/man/man8/pam_ftp.8.gz
 OLD_FILES+=usr/share/man/man8/pam_wheel.8.gz
 OLD_FILES+=usr/share/man/man8/sconfig.8.gz
 OLD_FILES+=usr/share/man/man8/ssl.8.gz
 OLD_FILES+=usr/share/man/man8/wlconfig.8.gz
 OLD_FILES+=usr/share/man/man9/CURSIG.9.gz
 OLD_FILES+=usr/share/man/man9/VFS_INIT.9.gz
 OLD_FILES+=usr/share/man/man9/at_exit.9.gz
 OLD_FILES+=usr/share/man/man9/at_fork.9.gz
 OLD_FILES+=usr/share/man/man9/cdevsw_add.9.gz
 OLD_FILES+=usr/share/man/man9/cdevsw_remove.9.gz
 OLD_FILES+=usr/share/man/man9/cv_waitq_empty.9.gz
 OLD_FILES+=usr/share/man/man9/cv_waitq_remove.9.gz
 OLD_FILES+=usr/share/man/man9/endtsleep.9.gz
 OLD_FILES+=usr/share/man/man9/jumbo.9.gz
 OLD_FILES+=usr/share/man/man9/jumbo_freem.9.gz
 OLD_FILES+=usr/share/man/man9/jumbo_pg_alloc.9.gz
 OLD_FILES+=usr/share/man/man9/jumbo_pg_free.9.gz
 OLD_FILES+=usr/share/man/man9/jumbo_pg_steal.9.gz
 OLD_FILES+=usr/share/man/man9/jumbo_phys_to_kva.9.gz
 OLD_FILES+=usr/share/man/man9/jumbo_vm_init.9.gz
 OLD_FILES+=usr/share/man/man9/mac_biba.9.gz
 OLD_FILES+=usr/share/man/man9/mac_bsdextended.9.gz
 OLD_FILES+=usr/share/man/man9/mono_time.9.gz
 OLD_FILES+=usr/share/man/man9/p1003_1b.9.gz
 OLD_FILES+=usr/share/man/man9/pmap_prefault.9.gz
 OLD_FILES+=usr/share/man/man9/posix4.9.gz
 OLD_FILES+=usr/share/man/man9/resource_query_name.9.gz
 OLD_FILES+=usr/share/man/man9/resource_query_string.9.gz
 OLD_FILES+=usr/share/man/man9/resource_query_unit.9.gz
 OLD_FILES+=usr/share/man/man9/rm_at_exit.9.gz
 OLD_FILES+=usr/share/man/man9/rm_at_fork.9.gz
 OLD_FILES+=usr/share/man/man9/runtime.9.gz
 OLD_FILES+=usr/share/man/man9/sleepinit.9.gz
 OLD_FILES+=usr/share/man/man9/unsleep.9.gz
 OLD_FILES+=usr/share/games/atc/Game_List
 OLD_FILES+=usr/share/games/atc/Killer
 OLD_FILES+=usr/share/games/atc/crossover
 OLD_FILES+=usr/share/games/atc/default
 OLD_FILES+=usr/share/games/atc/easy
 OLD_FILES+=usr/share/games/atc/game_2
 OLD_FILES+=usr/share/games/larn/larnmaze
 OLD_FILES+=usr/share/games/larn/larnopts
 OLD_FILES+=usr/share/games/larn/larn.help
 OLD_FILES+=usr/share/games/quiz.db/africa
 OLD_FILES+=usr/share/games/quiz.db/america
 OLD_FILES+=usr/share/games/quiz.db/areas
 OLD_FILES+=usr/share/games/quiz.db/arith
 OLD_FILES+=usr/share/games/quiz.db/asia
 OLD_FILES+=usr/share/games/quiz.db/babies
 OLD_FILES+=usr/share/games/quiz.db/bard
 OLD_FILES+=usr/share/games/quiz.db/chinese
 OLD_FILES+=usr/share/games/quiz.db/collectives
 OLD_FILES+=usr/share/games/quiz.db/ed
 OLD_FILES+=usr/share/games/quiz.db/elements
 OLD_FILES+=usr/share/games/quiz.db/europe
 OLD_FILES+=usr/share/games/quiz.db/flowers
 OLD_FILES+=usr/share/games/quiz.db/greek
 OLD_FILES+=usr/share/games/quiz.db/inca
 OLD_FILES+=usr/share/games/quiz.db/index
 OLD_FILES+=usr/share/games/quiz.db/latin
 OLD_FILES+=usr/share/games/quiz.db/locomotive
 OLD_FILES+=usr/share/games/quiz.db/midearth
 OLD_FILES+=usr/share/games/quiz.db/morse
 OLD_FILES+=usr/share/games/quiz.db/murders
 OLD_FILES+=usr/share/games/quiz.db/poetry
 OLD_FILES+=usr/share/games/quiz.db/posneg
 OLD_FILES+=usr/share/games/quiz.db/pres
 OLD_FILES+=usr/share/games/quiz.db/province
 OLD_FILES+=usr/share/games/quiz.db/seq-easy
 OLD_FILES+=usr/share/games/quiz.db/seq-hard
 OLD_FILES+=usr/share/games/quiz.db/sexes
 OLD_FILES+=usr/share/games/quiz.db/sov
 OLD_FILES+=usr/share/games/quiz.db/spell
 OLD_FILES+=usr/share/games/quiz.db/state
 OLD_FILES+=usr/share/games/quiz.db/trek
 OLD_FILES+=usr/share/games/quiz.db/ucc
 OLD_FILES+=usr/share/games/cribbage.instr
 OLD_FILES+=usr/share/games/fish.instr
 OLD_FILES+=usr/share/games/wump.info
 OLD_FILES+=usr/games/hide/adventure
 OLD_FILES+=usr/games/hide/arithmetic
 OLD_FILES+=usr/games/hide/atc
 OLD_FILES+=usr/games/hide/backgammon
 OLD_FILES+=usr/games/hide/teachgammon
 OLD_FILES+=usr/games/hide/battlestar
 OLD_FILES+=usr/games/hide/bs
 OLD_FILES+=usr/games/hide/canfield
 OLD_FILES+=usr/games/hide/cribbage
 OLD_FILES+=usr/games/hide/fish
 OLD_FILES+=usr/games/hide/hack
 OLD_FILES+=usr/games/hide/hangman
 OLD_FILES+=usr/games/hide/larn
 OLD_FILES+=usr/games/hide/mille
 OLD_FILES+=usr/games/hide/phantasia
 OLD_FILES+=usr/games/hide/quiz
 OLD_FILES+=usr/games/hide/robots
 OLD_FILES+=usr/games/hide/rogue
 OLD_FILES+=usr/games/hide/sail
 OLD_FILES+=usr/games/hide/snake
 OLD_FILES+=usr/games/hide/trek
 OLD_FILES+=usr/games/hide/worm
 OLD_FILES+=usr/games/hide/wump
 OLD_FILES+=usr/games/adventure
 OLD_FILES+=usr/games/arithmetic
 OLD_FILES+=usr/games/atc
 OLD_FILES+=usr/games/backgammon
 OLD_FILES+=usr/games/teachgammon
 OLD_FILES+=usr/games/battlestar
 OLD_FILES+=usr/games/bs
 OLD_FILES+=usr/games/canfield
 OLD_FILES+=usr/games/cfscores
 OLD_FILES+=usr/games/cribbage
 OLD_FILES+=usr/games/dm
 OLD_FILES+=usr/games/fish
 OLD_FILES+=usr/games/hack
 OLD_FILES+=usr/games/hangman
 OLD_FILES+=usr/games/larn
 OLD_FILES+=usr/games/mille
 OLD_FILES+=usr/games/phantasia
 OLD_FILES+=usr/games/piano
 OLD_FILES+=usr/games/pig
 OLD_FILES+=usr/games/quiz
 OLD_FILES+=usr/games/rain
 OLD_FILES+=usr/games/robots
 OLD_FILES+=usr/games/rogue
 OLD_FILES+=usr/games/sail
 OLD_FILES+=usr/games/snake
 OLD_FILES+=usr/games/snscore
 OLD_FILES+=usr/games/trek
 OLD_FILES+=usr/games/wargames
 OLD_FILES+=usr/games/worm
 OLD_FILES+=usr/games/worms
 OLD_FILES+=usr/games/wump
 OLD_FILES+=sbin/mount_reiserfs
 OLD_FILES+=usr/include/cam/cam_extend.h
 OLD_FILES+=usr/include/dev/wi/wi_hostap.h
 OLD_FILES+=usr/include/disktab.h
 OLD_FILES+=usr/include/g++/FlexLexer.h
 OLD_FILES+=usr/include/g++/PlotFile.h
 OLD_FILES+=usr/include/g++/SFile.h
 OLD_FILES+=usr/include/g++/_G_config.h
 OLD_FILES+=usr/include/g++/algo.h
 OLD_FILES+=usr/include/g++/algobase.h
 OLD_FILES+=usr/include/g++/algorithm
 OLD_FILES+=usr/include/g++/alloc.h
 OLD_FILES+=usr/include/g++/bitset
 OLD_FILES+=usr/include/g++/builtinbuf.h
 OLD_FILES+=usr/include/g++/bvector.h
 OLD_FILES+=usr/include/g++/cassert
 OLD_FILES+=usr/include/g++/cctype
 OLD_FILES+=usr/include/g++/cerrno
 OLD_FILES+=usr/include/g++/cfloat
 OLD_FILES+=usr/include/g++/ciso646
 OLD_FILES+=usr/include/g++/climits
 OLD_FILES+=usr/include/g++/clocale
 OLD_FILES+=usr/include/g++/cmath
 OLD_FILES+=usr/include/g++/complex
 OLD_FILES+=usr/include/g++/complex.h
 OLD_FILES+=usr/include/g++/csetjmp
 OLD_FILES+=usr/include/g++/csignal
 OLD_FILES+=usr/include/g++/cstdarg
 OLD_FILES+=usr/include/g++/cstddef
 OLD_FILES+=usr/include/g++/cstdio
 OLD_FILES+=usr/include/g++/cstdlib
 OLD_FILES+=usr/include/g++/cstring
 OLD_FILES+=usr/include/g++/ctime
 OLD_FILES+=usr/include/g++/cwchar
 OLD_FILES+=usr/include/g++/cwctype
 OLD_FILES+=usr/include/g++/defalloc.h
 OLD_FILES+=usr/include/g++/deque
 OLD_FILES+=usr/include/g++/deque.h
 OLD_FILES+=usr/include/g++/editbuf.h
 OLD_FILES+=usr/include/g++/exception
 OLD_FILES+=usr/include/g++/floatio.h
 OLD_FILES+=usr/include/g++/fstream
 OLD_FILES+=usr/include/g++/fstream.h
 OLD_FILES+=usr/include/g++/function.h
 OLD_FILES+=usr/include/g++/functional
 OLD_FILES+=usr/include/g++/hash_map
 OLD_FILES+=usr/include/g++/hash_map.h
 OLD_FILES+=usr/include/g++/hash_set
 OLD_FILES+=usr/include/g++/hash_set.h
 OLD_FILES+=usr/include/g++/hashtable.h
 OLD_FILES+=usr/include/g++/heap.h
 OLD_FILES+=usr/include/g++/indstream.h
 OLD_FILES+=usr/include/g++/iolibio.h
 OLD_FILES+=usr/include/g++/iomanip
 OLD_FILES+=usr/include/g++/iomanip.h
 OLD_FILES+=usr/include/g++/iosfwd
 OLD_FILES+=usr/include/g++/iostdio.h
 OLD_FILES+=usr/include/g++/iostream
 OLD_FILES+=usr/include/g++/iostream.h
 OLD_FILES+=usr/include/g++/iostreamP.h
 OLD_FILES+=usr/include/g++/istream.h
 OLD_FILES+=usr/include/g++/iterator
 OLD_FILES+=usr/include/g++/iterator.h
 OLD_FILES+=usr/include/g++/libio.h
 OLD_FILES+=usr/include/g++/libioP.h
 OLD_FILES+=usr/include/g++/list
 OLD_FILES+=usr/include/g++/list.h
 OLD_FILES+=usr/include/g++/map
 OLD_FILES+=usr/include/g++/map.h
 OLD_FILES+=usr/include/g++/memory
 OLD_FILES+=usr/include/g++/multimap.h
 OLD_FILES+=usr/include/g++/multiset.h
 OLD_FILES+=usr/include/g++/new
 OLD_FILES+=usr/include/g++/new.h
 OLD_FILES+=usr/include/g++/numeric
 OLD_FILES+=usr/include/g++/ostream.h
 OLD_FILES+=usr/include/g++/pair.h
 OLD_FILES+=usr/include/g++/parsestream.h
 OLD_FILES+=usr/include/g++/pfstream.h
 OLD_FILES+=usr/include/g++/procbuf.h
 OLD_FILES+=usr/include/g++/pthread_alloc
 OLD_FILES+=usr/include/g++/pthread_alloc.h
 OLD_FILES+=usr/include/g++/queue
 OLD_FILES+=usr/include/g++/rope
 OLD_FILES+=usr/include/g++/rope.h
 OLD_FILES+=usr/include/g++/ropeimpl.h
 OLD_FILES+=usr/include/g++/set
 OLD_FILES+=usr/include/g++/set.h
 OLD_FILES+=usr/include/g++/slist
 OLD_FILES+=usr/include/g++/slist.h
 OLD_FILES+=usr/include/g++/sstream
 OLD_FILES+=usr/include/g++/stack
 OLD_FILES+=usr/include/g++/stack.h
 OLD_FILES+=usr/include/g++/std/bastring.cc
 OLD_FILES+=usr/include/g++/std/bastring.h
 OLD_FILES+=usr/include/g++/std/complext.cc
 OLD_FILES+=usr/include/g++/std/complext.h
 OLD_FILES+=usr/include/g++/std/dcomplex.h
 OLD_FILES+=usr/include/g++/std/fcomplex.h
 OLD_FILES+=usr/include/g++/std/gslice.h
 OLD_FILES+=usr/include/g++/std/gslice_array.h
 OLD_FILES+=usr/include/g++/std/indirect_array.h
 OLD_FILES+=usr/include/g++/std/ldcomplex.h
 OLD_FILES+=usr/include/g++/std/mask_array.h
 OLD_FILES+=usr/include/g++/std/slice.h
 OLD_FILES+=usr/include/g++/std/slice_array.h
 OLD_FILES+=usr/include/g++/std/std_valarray.h
 OLD_FILES+=usr/include/g++/std/straits.h
 OLD_FILES+=usr/include/g++/std/valarray_array.h
 OLD_FILES+=usr/include/g++/std/valarray_array.tcc
 OLD_FILES+=usr/include/g++/std/valarray_meta.h
 OLD_FILES+=usr/include/g++/stdexcept
 OLD_FILES+=usr/include/g++/stdiostream.h
 OLD_FILES+=usr/include/g++/stl.h
 OLD_FILES+=usr/include/g++/stl_algo.h
 OLD_FILES+=usr/include/g++/stl_algobase.h
 OLD_FILES+=usr/include/g++/stl_alloc.h
 OLD_FILES+=usr/include/g++/stl_bvector.h
 OLD_FILES+=usr/include/g++/stl_config.h
 OLD_FILES+=usr/include/g++/stl_construct.h
 OLD_FILES+=usr/include/g++/stl_deque.h
 OLD_FILES+=usr/include/g++/stl_function.h
 OLD_FILES+=usr/include/g++/stl_hash_fun.h
 OLD_FILES+=usr/include/g++/stl_hash_map.h
 OLD_FILES+=usr/include/g++/stl_hash_set.h
 OLD_FILES+=usr/include/g++/stl_hashtable.h
 OLD_FILES+=usr/include/g++/stl_heap.h
 OLD_FILES+=usr/include/g++/stl_iterator.h
 OLD_FILES+=usr/include/g++/stl_list.h
 OLD_FILES+=usr/include/g++/stl_map.h
 OLD_FILES+=usr/include/g++/stl_multimap.h
 OLD_FILES+=usr/include/g++/stl_multiset.h
 OLD_FILES+=usr/include/g++/stl_numeric.h
 OLD_FILES+=usr/include/g++/stl_pair.h
 OLD_FILES+=usr/include/g++/stl_queue.h
 OLD_FILES+=usr/include/g++/stl_raw_storage_iter.h
 OLD_FILES+=usr/include/g++/stl_relops.h
 OLD_FILES+=usr/include/g++/stl_rope.h
 OLD_FILES+=usr/include/g++/stl_set.h
 OLD_FILES+=usr/include/g++/stl_slist.h
 OLD_FILES+=usr/include/g++/stl_stack.h
 OLD_FILES+=usr/include/g++/stl_tempbuf.h
 OLD_FILES+=usr/include/g++/stl_tree.h
 OLD_FILES+=usr/include/g++/stl_uninitialized.h
 OLD_FILES+=usr/include/g++/stl_vector.h
 OLD_FILES+=usr/include/g++/stream.h
 OLD_FILES+=usr/include/g++/streambuf.h
 OLD_FILES+=usr/include/g++/strfile.h
 OLD_FILES+=usr/include/g++/string
 OLD_FILES+=usr/include/g++/strstream
 OLD_FILES+=usr/include/g++/strstream.h
 OLD_FILES+=usr/include/g++/tempbuf.h
 OLD_FILES+=usr/include/g++/tree.h
 OLD_FILES+=usr/include/g++/type_traits.h
 OLD_FILES+=usr/include/g++/typeinfo
 OLD_FILES+=usr/include/g++/utility
 OLD_FILES+=usr/include/g++/valarray
 OLD_FILES+=usr/include/g++/vector
 OLD_FILES+=usr/include/g++/vector.h
 OLD_FILES+=usr/include/gmp.h
 OLD_FILES+=usr/include/isc/assertions.h
 OLD_FILES+=usr/include/isc/ctl.h
 OLD_FILES+=usr/include/isc/dst.h
 OLD_FILES+=usr/include/isc/eventlib.h
 OLD_FILES+=usr/include/isc/heap.h
 OLD_FILES+=usr/include/isc/irpmarshall.h
 OLD_FILES+=usr/include/isc/list.h
 OLD_FILES+=usr/include/isc/logging.h
 OLD_FILES+=usr/include/isc/memcluster.h
 OLD_FILES+=usr/include/isc/misc.h
 OLD_FILES+=usr/include/isc/tree.h
 OLD_FILES+=usr/include/machine/ansi.h
 OLD_FILES+=usr/include/machine/apic.h
 OLD_FILES+=usr/include/machine/asc_ioctl.h
 OLD_FILES+=usr/include/machine/asnames.h
 OLD_FILES+=usr/include/machine/bus_at386.h
 OLD_FILES+=usr/include/machine/bus_memio.h
 OLD_FILES+=usr/include/machine/bus_pc98.h
 OLD_FILES+=usr/include/machine/bus_pio.h
 OLD_FILES+=usr/include/machine/cdk.h
 OLD_FILES+=usr/include/machine/comstats.h
 OLD_FILES+=usr/include/machine/console.h
 OLD_FILES+=usr/include/machine/critical.h
 OLD_FILES+=usr/include/machine/cronyx.h
 OLD_FILES+=usr/include/machine/dvcfg.h
 OLD_FILES+=usr/include/machine/globaldata.h
 OLD_FILES+=usr/include/machine/globals.h
 OLD_FILES+=usr/include/machine/gsc.h
 OLD_FILES+=usr/include/machine/i4b_isppp.h
 OLD_FILES+=usr/include/machine/if_wavelan_ieee.h
 OLD_FILES+=usr/include/machine/iic.h
 OLD_FILES+=usr/include/machine/ioctl_ctx.h
 OLD_FILES+=usr/include/machine/ioctl_fd.h
 OLD_FILES+=usr/include/machine/ipl.h
 OLD_FILES+=usr/include/machine/lock.h
 OLD_FILES+=usr/include/machine/mouse.h
 OLD_FILES+=usr/include/machine/mpapic.h
 OLD_FILES+=usr/include/machine/mtpr.h
 OLD_FILES+=usr/include/machine/pc/msdos.h
 OLD_FILES+=usr/include/machine/physio_proc.h
 OLD_FILES+=usr/include/machine/smb.h
 OLD_FILES+=usr/include/machine/spigot.h
 OLD_FILES+=usr/include/machine/types.h
 OLD_FILES+=usr/include/machine/uc_device.h
 OLD_FILES+=usr/include/machine/ultrasound.h
 OLD_FILES+=usr/include/machine/wtio.h
 OLD_FILES+=usr/include/msdosfs/bootsect.h
 OLD_FILES+=usr/include/msdosfs/bpb.h
 OLD_FILES+=usr/include/msdosfs/denode.h
 OLD_FILES+=usr/include/msdosfs/direntry.h
 OLD_FILES+=usr/include/msdosfs/fat.h
 OLD_FILES+=usr/include/msdosfs/msdosfsmount.h
 OLD_FILES+=usr/include/net/hostcache.h
 OLD_FILES+=usr/include/net/if_faith.h
 OLD_FILES+=usr/include/net/if_ieee80211.h
 OLD_FILES+=usr/include/net/if_tunvar.h
 OLD_FILES+=usr/include/net/intrq.h
 OLD_FILES+=usr/include/netatm/kern_include.h
 OLD_FILES+=usr/include/netinet/if_fddi.h
 OLD_FILES+=usr/include/netinet/in_hostcache.h
 OLD_FILES+=usr/include/netinet/ip_flow.h
 OLD_FILES+=usr/include/netinet/ip_fw2.h
 OLD_FILES+=usr/include/netinet6/in6_prefix.h
 OLD_FILES+=usr/include/netns/idp.h
 OLD_FILES+=usr/include/netns/idp_var.h
 OLD_FILES+=usr/include/netns/ns.h
 OLD_FILES+=usr/include/netns/ns_error.h
 OLD_FILES+=usr/include/netns/ns_if.h
 OLD_FILES+=usr/include/netns/ns_pcb.h
 OLD_FILES+=usr/include/netns/sp.h
 OLD_FILES+=usr/include/netns/spidp.h
 OLD_FILES+=usr/include/netns/spp_debug.h
 OLD_FILES+=usr/include/netns/spp_timer.h
 OLD_FILES+=usr/include/netns/spp_var.h
 OLD_FILES+=usr/include/nfs/nfs.h
 OLD_FILES+=usr/include/nfs/nfsm_subs.h
 OLD_FILES+=usr/include/nfs/nfsmount.h
 OLD_FILES+=usr/include/nfs/nfsnode.h
 OLD_FILES+=usr/include/nfs/nfsrtt.h
 OLD_FILES+=usr/include/nfs/nfsrvcache.h
 OLD_FILES+=usr/include/nfs/nfsv2.h
 OLD_FILES+=usr/include/nfs/nqnfs.h
 OLD_FILES+=usr/include/ntfs/ntfs.h
 OLD_FILES+=usr/include/ntfs/ntfs_compr.h
 OLD_FILES+=usr/include/ntfs/ntfs_ihash.h
 OLD_FILES+=usr/include/ntfs/ntfs_inode.h
 OLD_FILES+=usr/include/ntfs/ntfs_subr.h
 OLD_FILES+=usr/include/ntfs/ntfs_vfsops.h
 OLD_FILES+=usr/include/ntfs/ntfsmount.h
 OLD_FILES+=usr/include/nwfs/nwfs.h
 OLD_FILES+=usr/include/nwfs/nwfs_mount.h
 OLD_FILES+=usr/include/nwfs/nwfs_node.h
 OLD_FILES+=usr/include/nwfs/nwfs_subr.h
 OLD_FILES+=usr/include/posix4/_semaphore.h
 OLD_FILES+=usr/include/posix4/aio.h
 OLD_FILES+=usr/include/posix4/ksem.h
 OLD_FILES+=usr/include/posix4/mqueue.h
 OLD_FILES+=usr/include/posix4/posix4.h
 OLD_FILES+=usr/include/posix4/sched.h
 OLD_FILES+=usr/include/posix4/semaphore.h
 OLD_DIRS+=usr/include/posix4
 OLD_FILES+=usr/include/security/_pam_compat.h
 OLD_FILES+=usr/include/security/_pam_macros.h
 OLD_FILES+=usr/include/security/_pam_types.h
 OLD_FILES+=usr/include/security/pam_malloc.h
 OLD_FILES+=usr/include/security/pam_misc.h
 OLD_FILES+=usr/include/skey.h
 OLD_FILES+=usr/include/strhash.h
 OLD_FILES+=usr/include/struct.h
 OLD_FILES+=usr/include/sys/_label.h
 OLD_FILES+=usr/include/sys/_posix.h
 OLD_FILES+=usr/include/sys/bus_private.h
 OLD_FILES+=usr/include/sys/ccdvar.h
 OLD_FILES+=usr/include/sys/diskslice.h
 OLD_FILES+=usr/include/sys/dmap.h
 OLD_FILES+=usr/include/sys/inttypes.h
 OLD_FILES+=usr/include/sys/jumbo.h
 OLD_FILES+=usr/include/sys/mac_policy.h
 OLD_FILES+=usr/include/sys/pbioio.h
 OLD_FILES+=usr/include/sys/syscall-hide.h
 OLD_FILES+=usr/include/sys/tprintf.h
 OLD_FILES+=usr/include/sys/vnioctl.h
 OLD_FILES+=usr/include/sys/wormio.h
 OLD_FILES+=usr/include/telnet.h
 OLD_FILES+=usr/include/ufs/mfs/mfs_extern.h
 OLD_FILES+=usr/include/ufs/mfs/mfsnode.h
 OLD_FILES+=usr/include/values.h
 OLD_FILES+=usr/include/vm/vm_zone.h
 OLD_FILES+=usr/share/examples/etc/usbd.conf
 OLD_FILES+=usr/share/examples/meteor/README
 OLD_FILES+=usr/share/examples/meteor/rgb16.c
 OLD_FILES+=usr/share/examples/meteor/rgb24.c
 OLD_FILES+=usr/share/examples/meteor/test-n.c
 OLD_FILES+=usr/share/examples/meteor/yuvpk.c
 OLD_FILES+=usr/share/examples/meteor/yuvpl.c
 OLD_FILES+=usr/share/examples/worm/README
 OLD_FILES+=usr/share/examples/worm/makecdfs.sh
 OLD_FILES+=usr/share/groff_font/devlj4/Makefile
 OLD_FILES+=usr/share/groff_font/devlj4/text.map
 OLD_FILES+=usr/share/groff_font/devlj4/special.map
 OLD_FILES+=usr/share/misc/nslookup.help
 OLD_FILES+=usr/share/sendmail/cf/feature/nodns.m4
 OLD_FILES+=usr/share/syscons/keymaps/lat-amer.kbd
 OLD_FILES+=usr/share/vi/catalog/ru_SU.KOI8-R
 OLD_FILES+=usr/share/zoneinfo/Africa/Timbuktu
 OLD_FILES+=usr/share/zoneinfo/Africa/Asmera
 OLD_FILES+=usr/share/zoneinfo/America/Buenos_Aires
 OLD_FILES+=usr/share/zoneinfo/America/Cordoba
 OLD_FILES+=usr/share/zoneinfo/America/Jujuy
 OLD_FILES+=usr/share/zoneinfo/America/Catamarca
 OLD_FILES+=usr/share/zoneinfo/America/Mendoza
 OLD_FILES+=usr/share/zoneinfo/America/Indianapolis
 OLD_FILES+=usr/share/zoneinfo/America/Louisville
 OLD_FILES+=usr/share/zoneinfo/America/Argentina/ComodRivadavia
 OLD_FILES+=usr/share/zoneinfo/Atlantic/Faeroe
 OLD_FILES+=usr/share/zoneinfo/Europe/Belfast
 OLD_FILES+=usr/share/zoneinfo/Pacific/Yap
 OLD_FILES+=usr/share/zoneinfo/SystemV/YST9
 OLD_FILES+=usr/share/zoneinfo/SystemV/PST8
 OLD_FILES+=usr/share/zoneinfo/SystemV/EST5EDT
 OLD_FILES+=usr/share/zoneinfo/SystemV/CST6CDT
 OLD_FILES+=usr/share/zoneinfo/SystemV/MST7MDT
 OLD_FILES+=usr/share/zoneinfo/SystemV/PST8PDT
 OLD_FILES+=usr/share/zoneinfo/SystemV/YST9YDT
 OLD_FILES+=usr/share/zoneinfo/SystemV/HST10
 OLD_FILES+=usr/share/zoneinfo/SystemV/MST7
 OLD_FILES+=usr/share/zoneinfo/SystemV/EST5
 OLD_FILES+=usr/share/zoneinfo/SystemV/AST4ADT
 OLD_FILES+=usr/share/zoneinfo/SystemV/CST6
 OLD_FILES+=usr/share/zoneinfo/SystemV/AST4
 OLD_FILES+=usr/share/doc/ntp/accopt.htm
 OLD_FILES+=usr/share/doc/ntp/assoc.htm
 OLD_FILES+=usr/share/doc/ntp/audio.htm
 OLD_FILES+=usr/share/doc/ntp/authopt.htm
 OLD_FILES+=usr/share/doc/ntp/biblio.htm
 OLD_FILES+=usr/share/doc/ntp/build.htm
 OLD_FILES+=usr/share/doc/ntp/clockopt.htm
 OLD_FILES+=usr/share/doc/ntp/config.htm
 OLD_FILES+=usr/share/doc/ntp/confopt.htm
 OLD_FILES+=usr/share/doc/ntp/copyright.htm
 OLD_FILES+=usr/share/doc/ntp/debug.htm
 OLD_FILES+=usr/share/doc/ntp/driver1.htm
 OLD_FILES+=usr/share/doc/ntp/driver10.htm
 OLD_FILES+=usr/share/doc/ntp/driver11.htm
 OLD_FILES+=usr/share/doc/ntp/driver12.htm
 OLD_FILES+=usr/share/doc/ntp/driver16.htm
 OLD_FILES+=usr/share/doc/ntp/driver18.htm
 OLD_FILES+=usr/share/doc/ntp/driver19.htm
 OLD_FILES+=usr/share/doc/ntp/driver2.htm
 OLD_FILES+=usr/share/doc/ntp/driver20.htm
 OLD_FILES+=usr/share/doc/ntp/driver22.htm
 OLD_FILES+=usr/share/doc/ntp/driver23.htm
 OLD_FILES+=usr/share/doc/ntp/driver24.htm
 OLD_FILES+=usr/share/doc/ntp/driver26.htm
 OLD_FILES+=usr/share/doc/ntp/driver27.htm
 OLD_FILES+=usr/share/doc/ntp/driver28.htm
 OLD_FILES+=usr/share/doc/ntp/driver29.htm
 OLD_FILES+=usr/share/doc/ntp/driver3.htm
 OLD_FILES+=usr/share/doc/ntp/driver30.htm
 OLD_FILES+=usr/share/doc/ntp/driver32.htm
 OLD_FILES+=usr/share/doc/ntp/driver33.htm
 OLD_FILES+=usr/share/doc/ntp/driver34.htm
 OLD_FILES+=usr/share/doc/ntp/driver35.htm
 OLD_FILES+=usr/share/doc/ntp/driver36.htm
 OLD_FILES+=usr/share/doc/ntp/driver37.htm
 OLD_FILES+=usr/share/doc/ntp/driver4.htm
 OLD_FILES+=usr/share/doc/ntp/driver5.htm
 OLD_FILES+=usr/share/doc/ntp/driver6.htm
 OLD_FILES+=usr/share/doc/ntp/driver7.htm
 OLD_FILES+=usr/share/doc/ntp/driver8.htm
 OLD_FILES+=usr/share/doc/ntp/driver9.htm
 OLD_FILES+=usr/share/doc/ntp/exec.htm
 OLD_FILES+=usr/share/doc/ntp/extern.htm
 OLD_FILES+=usr/share/doc/ntp/gadget.htm
 OLD_FILES+=usr/share/doc/ntp/hints.htm
 OLD_FILES+=usr/share/doc/ntp/howto.htm
 OLD_FILES+=usr/share/doc/ntp/htmlprimer.htm
 OLD_FILES+=usr/share/doc/ntp/index.htm
 OLD_FILES+=usr/share/doc/ntp/kern.htm
 OLD_FILES+=usr/share/doc/ntp/kernpps.htm
 OLD_FILES+=usr/share/doc/ntp/ldisc.htm
 OLD_FILES+=usr/share/doc/ntp/measure.htm
 OLD_FILES+=usr/share/doc/ntp/miscopt.htm
 OLD_FILES+=usr/share/doc/ntp/monopt.htm
 OLD_FILES+=usr/share/doc/ntp/mx4200data.htm
 OLD_FILES+=usr/share/doc/ntp/notes.htm
 OLD_FILES+=usr/share/doc/ntp/ntpd.htm
 OLD_FILES+=usr/share/doc/ntp/ntpdate.htm
 OLD_FILES+=usr/share/doc/ntp/ntpdc.htm
 OLD_FILES+=usr/share/doc/ntp/ntpq.htm
 OLD_FILES+=usr/share/doc/ntp/ntptime.htm
 OLD_FILES+=usr/share/doc/ntp/ntptrace.htm
 OLD_FILES+=usr/share/doc/ntp/parsedata.htm
 OLD_FILES+=usr/share/doc/ntp/parsenew.htm
 OLD_FILES+=usr/share/doc/ntp/patches.htm
 OLD_FILES+=usr/share/doc/ntp/porting.htm
 OLD_FILES+=usr/share/doc/ntp/pps.htm
 OLD_FILES+=usr/share/doc/ntp/prefer.htm
 OLD_FILES+=usr/share/doc/ntp/qth.htm
 OLD_FILES+=usr/share/doc/ntp/quick.htm
 OLD_FILES+=usr/share/doc/ntp/rdebug.htm
 OLD_FILES+=usr/share/doc/ntp/refclock.htm
 OLD_FILES+=usr/share/doc/ntp/release.htm
 OLD_FILES+=usr/share/doc/ntp/tickadj.htm
 OLD_FILES+=usr/share/doc/papers/nqnfs.ascii.gz
 OLD_FILES+=usr/share/doc/papers/px.ascii.gz
 OLD_FILES+=usr/share/man/man3/exp10.3.gz
 OLD_FILES+=usr/share/man/man3/exp10f.3.gz
 OLD_FILES+=usr/share/man/man3/fpsetsticky.3.gz
 OLD_FILES+=usr/share/man/man3/gss_krb5_compat_des3_mic.3.gz
 OLD_FILES+=usr/share/man/man3/gss_krb5_copy_ccache.3.gz
 OLD_FILES+=usr/share/man/man3/mac_is_present_np.3.gz
 OLD_FILES+=usr/share/man/man3/mbmb.3.gz
 OLD_FILES+=usr/share/man/man3/setrunelocale.3.gz
 OLD_FILES+=usr/share/man/man5/usbd.conf.5.gz
 .if ${TARGET_ARCH} != "i386" && ${TARGET_ARCH} != "amd64"
 OLD_FILES+=usr/share/man/man8/boot_i386.8.gz
 .endif
 .if ${TARGET_ARCH} != "aarch64" && ${TARGET} != "arm" && \
     ${TARGET_ARCH} != "powerpc" && ${TARGET_ARCH} != "powerpc64" && \
     ${TARGET_ARCH} != "sparc64" && ${TARGET} != "mips"
 OLD_FILES+=usr/share/man/man8/ofwdump.8.gz
 .endif
 OLD_FILES+=usr/share/man/man8/mount_reiserfs.8.gz
 OLD_FILES+=usr/share/man/man9/VFS_START.9.gz
 OLD_FILES+=usr/share/man/man9/cpu_critical_exit.9.gz
 OLD_FILES+=usr/share/man/man9/cpu_critical_enter.9.gz
 OLD_FILES+=usr/share/info/annotate.info.gz
 OLD_FILES+=usr/share/info/tar.info.gz
 OLD_FILES+=usr/share/bsnmp/defs/tree.def
 OLD_FILES+=usr/share/bsnmp/defs/mibII_tree.def
 OLD_FILES+=usr/share/bsnmp/defs/netgraph_tree.def
 OLD_FILES+=usr/share/bsnmp/mibs/FOKUS-MIB.txt
 OLD_FILES+=usr/share/bsnmp/mibs/BEGEMOT-MIB.txt
 OLD_FILES+=usr/share/bsnmp/mibs/BEGEMOT-SNMPD.txt
 OLD_FILES+=usr/share/bsnmp/mibs/BEGEMOT-NETGRAPH.txt
 OLD_FILES+=usr/libdata/msdosfs/iso22dos
 OLD_FILES+=usr/libdata/msdosfs/iso72dos
 OLD_FILES+=usr/libdata/msdosfs/koi2dos
 OLD_FILES+=usr/libdata/msdosfs/koi8u2dos
 # The following files are *not* obsolete, they just don't get touched at
 # install, so don't add them:
 #  - boot/loader.rc
 #  - usr/share/tmac/man.local
 #  - usr/share/tmac/mm/locale
 #  - usr/share/tmac/mm/se_locale
 #  - var/yp/Makefile
 
 # Early entries split OLD_FILES, OLD_LIBS, and OLD_DIRS into separate sections
 # in this file, but this practice was abandoned in the mid-2000s.
 #
 # 20071120: shared library version bump
 OLD_LIBS+=usr/lib/libasn1.so.8
 OLD_LIBS+=usr/lib/libgssapi.so.8
 OLD_LIBS+=usr/lib/libgssapi_krb5.so.8
 OLD_LIBS+=usr/lib/libhdb.so.8
 OLD_LIBS+=usr/lib/libkadm5clnt.so.8
 OLD_LIBS+=usr/lib/libkadm5srv.so.8
 OLD_LIBS+=usr/lib/libkafs5.so.8
 OLD_LIBS+=usr/lib/libkrb5.so.8
 OLD_LIBS+=usr/lib/libobjc.so.2
 OLD_LIBS+=usr/lib32/libgssapi.so.8
 OLD_LIBS+=usr/lib32/libobjc.so.2
 # 20070519: GCC 4.2
 OLD_LIBS+=usr/lib/libg2c.a
 OLD_LIBS+=usr/lib/libg2c.so
 OLD_LIBS+=usr/lib/libg2c.so.2
 OLD_LIBS+=usr/lib/libg2c_p.a
 OLD_LIBS+=usr/lib/libgcc_pic.a
 OLD_LIBS+=usr/lib32/libg2c.a
 OLD_LIBS+=usr/lib32/libg2c.so
 OLD_LIBS+=usr/lib32/libg2c.so.2
 OLD_LIBS+=usr/lib32/libg2c_p.a
 OLD_LIBS+=usr/lib32/libgcc_pic.a
 # 20060729: OpenSSL 0.9.7e -> 0.9.8b upgrade
 OLD_LIBS+=lib/libcrypto.so.4
 OLD_LIBS+=usr/lib/libssl.so.4
 OLD_LIBS+=usr/lib32/libcrypto.so.4
 OLD_LIBS+=usr/lib32/libssl.so.4
 # 20060521: gethostbyaddr(3) ABI change
 OLD_LIBS+=usr/lib/libroken.so.8
 OLD_LIBS+=lib/libatm.so.3
 OLD_LIBS+=lib/libc.so.6
 OLD_LIBS+=lib/libutil.so.5
 OLD_LIBS+=usr/lib32/libatm.so.3
 OLD_LIBS+=usr/lib32/libc.so.6
 OLD_LIBS+=usr/lib32/libutil.so.5
 # 20060413: shared library moved to /usr/lib
 OLD_LIBS+=lib/libgpib.so.1
 # 20060413: libpcap.so.4 moved to /lib/
 OLD_LIBS+=usr/lib/libpcap.so.4
 # 20060412: libpthread.so.2 moved to /lib/
 OLD_LIBS+=usr/lib/libpthread.so.2
 # 20060127: revert libdisk to static-only
 OLD_LIBS+=usr/lib/libdisk.so.3
 # 20051027: libc_r discontinued (removed 20101113)
 OLD_LIBS+=usr/lib/libc_r.a
 OLD_LIBS+=usr/lib/libc_r.so
 OLD_LIBS+=usr/lib/libc_r.so.7
 OLD_LIBS+=usr/lib/libc_r_p.a
 OLD_LIBS+=usr/lib32/libc_r.a
 OLD_LIBS+=usr/lib32/libc_r.so
 OLD_LIBS+=usr/lib32/libc_r.so.7
 OLD_LIBS+=usr/lib32/libc_r_p.a
 # 20050722: bump for 6.0-RELEASE
 OLD_LIBS+=lib/libalias.so.4
 OLD_LIBS+=lib/libatm.so.2
 OLD_LIBS+=lib/libbegemot.so.1
 OLD_LIBS+=lib/libbsdxml.so.1
 OLD_LIBS+=lib/libbsnmp.so.2
 OLD_LIBS+=lib/libc.so.5
 OLD_LIBS+=lib/libcam.so.2
 OLD_LIBS+=lib/libcrypt.so.2
 OLD_LIBS+=lib/libcrypto.so.3
 OLD_LIBS+=lib/libdevstat.so.4
 OLD_LIBS+=lib/libedit.so.4
 OLD_LIBS+=lib/libgeom.so.2
 OLD_LIBS+=lib/libgpib.so.0
 OLD_LIBS+=lib/libipsec.so.1
 OLD_LIBS+=lib/libipx.so.2
 OLD_LIBS+=lib/libkiconv.so.1
 OLD_LIBS+=lib/libkvm.so.2
 OLD_LIBS+=lib/libm.so.3
 OLD_LIBS+=lib/libmd.so.2
 OLD_LIBS+=lib/libncurses.so.5
 OLD_LIBS+=lib/libreadline.so.5
 OLD_LIBS+=lib/libsbuf.so.2
 OLD_LIBS+=lib/libufs.so.2
 OLD_LIBS+=lib/libutil.so.4
 OLD_LIBS+=lib/libz.so.2
 OLD_LIBS+=usr/lib/libarchive.so.1
 OLD_LIBS+=usr/lib/libasn1.so.7
 OLD_LIBS+=usr/lib/libbluetooth.so.1
 OLD_LIBS+=usr/lib/libbz2.so.1
 OLD_LIBS+=usr/lib/libc_r.so.5
 OLD_LIBS+=usr/lib/libcalendar.so.2
 OLD_LIBS+=usr/lib/libcom_err.so.2
 OLD_LIBS+=usr/lib/libdevinfo.so.2
 OLD_LIBS+=usr/lib/libdialog.so.4
 OLD_LIBS+=usr/lib/libfetch.so.3
 OLD_LIBS+=usr/lib/libform.so.2
 OLD_LIBS+=usr/lib/libftpio.so.5
 OLD_LIBS+=usr/lib/libg2c.so.1
 OLD_LIBS+=usr/lib/libgnuregex.so.2
 OLD_LIBS+=usr/lib/libgssapi.so.7
 OLD_LIBS+=usr/lib/libhdb.so.7
 OLD_LIBS+=usr/lib/libhistory.so.5
 OLD_LIBS+=usr/lib/libkadm5clnt.so.7
 OLD_LIBS+=usr/lib/libkadm5srv.so.7
 OLD_LIBS+=usr/lib/libkafs5.so.7
 OLD_LIBS+=usr/lib/libkrb5.so.7
 OLD_LIBS+=usr/lib/libmagic.so.1
 OLD_LIBS+=usr/lib/libmenu.so.2
 OLD_LIBS+=usr/lib/libmilter.so.2
 OLD_LIBS+=usr/lib/libmp.so.4
 OLD_LIBS+=usr/lib/libncp.so.1
 OLD_LIBS+=usr/lib/libnetgraph.so.1
 OLD_LIBS+=usr/lib/libngatm.so.1
 OLD_LIBS+=usr/lib/libobjc.so.1
 OLD_LIBS+=usr/lib/libopie.so.3
 OLD_LIBS+=usr/lib/libpam.so.2
 OLD_LIBS+=usr/lib/libpanel.so.2
 OLD_LIBS+=usr/lib/libpcap.so.3
 OLD_LIBS+=usr/lib/libpmc.so.2
 OLD_LIBS+=usr/lib/libpthread.so.1
 OLD_LIBS+=usr/lib/libradius.so.1
 OLD_LIBS+=usr/lib/libroken.so.7
 OLD_LIBS+=usr/lib/librpcsvc.so.2
 OLD_LIBS+=usr/lib/libsdp.so.1
 OLD_LIBS+=usr/lib/libsmb.so.1
 OLD_LIBS+=usr/lib/libssh.so.2
 OLD_LIBS+=usr/lib/libssl.so.3
 OLD_LIBS+=usr/lib/libstdc++.so.4
 OLD_LIBS+=usr/lib/libtacplus.so.1
 OLD_LIBS+=usr/lib/libthr.so.1
 OLD_LIBS+=usr/lib/libthread_db.so.1
 OLD_LIBS+=usr/lib/libugidfw.so.1
 OLD_LIBS+=usr/lib/libusbhid.so.1
 OLD_LIBS+=usr/lib/libvgl.so.3
 OLD_LIBS+=usr/lib/libwrap.so.3
 OLD_LIBS+=usr/lib/libypclnt.so.1
 OLD_LIBS+=usr/lib/pam_chroot.so.2
 OLD_LIBS+=usr/lib/pam_deny.so.2
 OLD_LIBS+=usr/lib/pam_echo.so.2
 OLD_LIBS+=usr/lib/pam_exec.so.2
 OLD_LIBS+=usr/lib/pam_ftpusers.so.2
 OLD_LIBS+=usr/lib/pam_group.so.2
 OLD_LIBS+=usr/lib/pam_guest.so.2
 OLD_LIBS+=usr/lib/pam_krb5.so.2
 OLD_LIBS+=usr/lib/pam_ksu.so.2
 OLD_LIBS+=usr/lib/pam_lastlog.so.2
 OLD_LIBS+=usr/lib/pam_login_access.so.2
 OLD_LIBS+=usr/lib/pam_nologin.so.2
 OLD_LIBS+=usr/lib/pam_opie.so.2
 OLD_LIBS+=usr/lib/pam_opieaccess.so.2
 OLD_LIBS+=usr/lib/pam_passwdqc.so.2
 OLD_LIBS+=usr/lib/pam_permit.so.2
 OLD_LIBS+=usr/lib/pam_radius.so.2
 OLD_LIBS+=usr/lib/pam_rhosts.so.2
 OLD_LIBS+=usr/lib/pam_rootok.so.2
 OLD_LIBS+=usr/lib/pam_securetty.so.2
 OLD_LIBS+=usr/lib/pam_self.so.2
 OLD_LIBS+=usr/lib/pam_ssh.so.2
 OLD_LIBS+=usr/lib/pam_tacplus.so.2
 OLD_LIBS+=usr/lib/pam_unix.so.2
 OLD_LIBS+=usr/lib/snmp_atm.so.3
 OLD_LIBS+=usr/lib/snmp_mibII.so.3
 OLD_LIBS+=usr/lib/snmp_netgraph.so.3
 OLD_LIBS+=usr/lib/snmp_pf.so.3
 # 200505XX: ?
 OLD_LIBS+=usr/lib/snmp_atm.so.2
 OLD_LIBS+=usr/lib/snmp_mibII.so.2
 OLD_LIBS+=usr/lib/snmp_netgraph.so.2
 OLD_LIBS+=usr/lib/snmp_pf.so.2
 # 2005XXXX: not ready for primetime yet
 OLD_LIBS+=usr/lib/libautofs.so.1
 # 200411XX: libxpg4 removal
 OLD_LIBS+=usr/lib/libxpg4.so.3
 # 200410XX: libm compatibility fix
 OLD_LIBS+=lib/libm.so.2
 # 20041001: version bump
 OLD_LIBS+=lib/libreadline.so.4
 OLD_LIBS+=usr/lib/libhistory.so.4
 OLD_LIBS+=usr/lib/libopie.so.2
 OLD_LIBS+=usr/lib/libpcap.so.2
 # 20040925: bind9 import
 OLD_LIBS+=usr/lib/libisc.so.1
 # 200408XX
 OLD_LIBS+=usr/lib/snmp_netgraph.so.1
 # 200404XX
 OLD_LIBS+=usr/lib/libsnmp.so.1
 OLD_LIBS+=usr/lib/snmp_mibII.so.1
 # 200309XX
 OLD_LIBS+=usr/lib/libasn1.so.6
 OLD_LIBS+=usr/lib/libhdb.so.6
 OLD_LIBS+=usr/lib/libkadm5clnt.so.6
 OLD_LIBS+=usr/lib/libkadm5srv.so.6
 OLD_LIBS+=usr/lib/libkrb5.so.6
 OLD_LIBS+=usr/lib/libroken.so.6
 # 200304XX
 OLD_LIBS+=usr/lib/libc.so.4
 OLD_LIBS+=usr/lib/libc_r.so.4
 OLD_LIBS+=usr/lib/libdevstat.so.2
 OLD_LIBS+=usr/lib/libedit.so.3
 OLD_LIBS+=usr/lib/libgmp.so.3
 OLD_LIBS+=usr/lib/libmp.so.3
 OLD_LIBS+=usr/lib/libpam.so.1
 OLD_LIBS+=usr/lib/libposix1e.so.2
 OLD_LIBS+=usr/lib/libskey.so.2
 OLD_LIBS+=usr/lib/libusbhid.so.0
 OLD_LIBS+=usr/lib/libvgl.so.2
 # 20030218: OpenSSL 0.9.7 import
 OLD_FILES+=usr/include/des.h
 OLD_FILES+=usr/lib/libdes.a
 OLD_FILES+=usr/lib/libdes.so
 OLD_LIBS+=usr/lib/libdes.so.3
 OLD_FILES+=usr/lib/libdes_p.a
 # 200302XX
 OLD_LIBS+=usr/lib/libacl.so.3
 OLD_LIBS+=usr/lib/libasn1.so.5
 OLD_LIBS+=usr/lib/libcrypto.so.2
 OLD_LIBS+=usr/lib/libgssapi.so.5
 OLD_LIBS+=usr/lib/libhdb.so.5
 OLD_LIBS+=usr/lib/libkadm.so.3
 OLD_LIBS+=usr/lib/libkadm5clnt.so.5
 OLD_LIBS+=usr/lib/libkadm5srv.so.5
 OLD_LIBS+=usr/lib/libkafs.so.3
 OLD_LIBS+=usr/lib/libkafs5.so.5
 OLD_LIBS+=usr/lib/libkdb.so.3
 OLD_LIBS+=usr/lib/libkrb.so.3
 OLD_LIBS+=usr/lib/libroken.so.
 OLD_LIBS+=usr/lib/libssl.so.2
 OLD_LIBS+=usr/lib/pam_kerberosIV.so
 # 200208XX
 OLD_LIBS+=usr/lib/libgssapi.so.4
 # 200203XX
 OLD_LIBS+=usr/lib/libss.so.3
 OLD_LIBS+=usr/lib/libusb.so.0
 # 200112XX
 OLD_LIBS+=usr/lib/libfetch.so.2
 # 200110XX
 OLD_LIBS+=usr/lib/libgssapi.so.3
 # 200104XX
 OLD_LIBS+=usr/lib/libdescrypt.so.2
 OLD_LIBS+=usr/lib/libscrypt.so.2
 # 200102XX
 OLD_LIBS+=usr/lib/libcrypto.so.1
 OLD_LIBS+=usr/lib/libssl.so.1
 # 200009XX
 OLD_LIBS+=usr/lib/libRSAglue.so.1
 OLD_LIBS+=usr/lib/librsaINTL.so.1
 OLD_LIBS+=usr/lib/librsaUSA.so.1
 # 200006XX
 OLD_LIBS+=usr/lib/libalias.so.3
 OLD_LIBS+=usr/lib/libfetch.so.1
 OLD_LIBS+=usr/lib/libipsec.so.0
 # 200005XX
 OLD_LIBS+=usr/lib/libxpg4.so.2
 # 200002XX
 OLD_LIBS+=usr/lib/libc.so.3
 OLD_LIBS+=usr/lib/libcurses.so.2
 OLD_LIBS+=usr/lib/libdialog.so.3
 OLD_LIBS+=usr/lib/libedit.so.2
 OLD_LIBS+=usr/lib/libf2c.so.2
 OLD_LIBS+=usr/lib/libftpio.so.4
 OLD_LIBS+=usr/lib/libg++.so.4
 OLD_LIBS+=usr/lib/libhistory.so.3
 OLD_LIBS+=usr/lib/libmytinfo.so.2
 OLD_LIBS+=usr/lib/libncurses.so.3
 OLD_LIBS+=usr/lib/libreadline.so.3
 OLD_LIBS+=usr/lib/libss.so.2
 OLD_LIBS+=usr/lib/libtermcap.so.2
 OLD_LIBS+=usr/lib/libutil.so.2
 OLD_LIBS+=usr/lib/libvgl.so.1
 OLD_LIBS+=usr/lib/libwrap.so.2
 # ???
 OLD_LIBS+=usr/lib/libarchive.so.2
 OLD_LIBS+=usr/lib/libbsnmp.so.1
 OLD_LIBS+=usr/lib/libc_r.so.6
 OLD_LIBS+=usr/lib32/libarchive.so.2
 OLD_LIBS+=usr/lib32/libc_r.so.6
 OLD_LIBS+=usr/lib/libcipher.so.2
 OLD_LIBS+=usr/lib/libgssapi.so.6
 OLD_LIBS+=usr/lib/libkse.so.1
 OLD_LIBS+=usr/lib/liblwres.so.3
 OLD_LIBS+=usr/lib/pam_ftp.so.2
 
 # 20040925: bind9 import
 OLD_DIRS+=usr/share/doc/bind/html
 OLD_DIRS+=usr/share/doc/bind/misc
 OLD_DIRS+=usr/share/doc/bind/
 # ???
 OLD_DIRS+=usr/include/g++/std
 OLD_DIRS+=usr/include/msdosfs
 OLD_DIRS+=usr/include/ntfs
 OLD_DIRS+=usr/include/nwfs
 OLD_DIRS+=usr/include/ufs/mfs
 # 20011001: UUCP migration to ports
 OLD_DIRS+=usr/libexec/uucp
 
 
 .include "tools/build/mk/OptionalObsoleteFiles.inc"
Index: head/share/man/man4/crypto.4
===================================================================
--- head/share/man/man4/crypto.4	(revision 359373)
+++ head/share/man/man4/crypto.4	(revision 359374)
@@ -1,464 +1,462 @@
 .\"	$NetBSD: crypto.4,v 1.24 2014/01/27 21:23:59 pgoyette Exp $
 .\"
 .\" Copyright (c) 2008 The NetBSD Foundation, Inc.
 .\" Copyright (c) 2014 The FreeBSD Foundation
 .\" All rights reserved.
 .\"
 .\" Portions of this documentation were written by John-Mark Gurney
 .\" under sponsorship of the FreeBSD Foundation and
 .\" Rubicon Communications, LLC (Netgate).
 .\"
 .\" This code is derived from software contributed to The NetBSD Foundation
 .\" by Coyote Point Systems, Inc.
 .\"
 .\" Redistribution and use in source and binary forms, with or without
 .\" modification, are permitted provided that the following conditions
 .\" are met:
 .\" 1. Redistributions of source code must retain the above copyright
 .\"    notice, this list of conditions and the following disclaimer.
 .\" 2. Redistributions in binary form must reproduce the above copyright
 .\"    notice, this list of conditions and the following disclaimer in the
 .\"    documentation and/or other materials provided with the distribution.
 .\"
 .\" THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 .\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 .\" TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 .\" PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 .\" BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 .\" CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 .\" SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 .\" INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 .\" CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 .\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 .\" POSSIBILITY OF SUCH DAMAGE.
 .\"
 .\"
 .\"
 .\" Copyright (c) 2004
 .\"	Jonathan Stone <jonathan@dsg.stanford.edu>. All rights reserved.
 .\"
 .\" Redistribution and use in source and binary forms, with or without
 .\" modification, are permitted provided that the following conditions
 .\" are met:
 .\" 1. Redistributions of source code must retain the above copyright
 .\"    notice, this list of conditions and the following disclaimer.
 .\" 2. Redistributions in binary form must reproduce the above copyright
 .\"    notice, this list of conditions and the following disclaimer in the
 .\"    documentation and/or other materials provided with the distribution.
 .\"
 .\" THIS SOFTWARE IS PROVIDED BY Jonathan Stone AND CONTRIBUTORS ``AS IS'' AND
 .\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 .\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 .\" ARE DISCLAIMED.  IN NO EVENT SHALL Jonathan Stone OR THE VOICES IN HIS HEAD
 .\" BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 .\" CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 .\" SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 .\" INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 .\" CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 .\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
 .\" THE POSSIBILITY OF SUCH DAMAGE.
 .\"
 .\" $FreeBSD$
 .\"
-.Dd December 17, 2019
+.Dd March 27, 2020
 .Dt CRYPTO 4
 .Os
 .Sh NAME
 .Nm crypto ,
 .Nm cryptodev
 .Nd user-mode access to hardware-accelerated cryptography
 .Sh SYNOPSIS
 .Cd device crypto
 .Cd device cryptodev
 .Pp
 .In sys/ioctl.h
 .In sys/time.h
 .In crypto/cryptodev.h
 .Sh DESCRIPTION
 The
 .Nm
 driver gives user-mode applications access to hardware-accelerated
 cryptographic transforms as implemented by the
 .Xr crypto 9
 in-kernel interface.
 .Pp
 The
 .Pa /dev/crypto
 special device provides an
 .Xr ioctl 2
 based interface.
 User-mode applications open the special device and
 then issue
 .Xr ioctl 2
 calls on the descriptor.
 User-mode access to
 .Pa /dev/crypto
 is controlled by two
 .Xr sysctl 8
 variables:
 .Ic kern.userasymcrypto
 and
 .Ic kern.cryptodevallowsoft .
 .Pp
 The
 .Nm
 device provides two distinct modes of operation: one mode for
 symmetric-keyed cryptographic requests and digests, and a second mode for
 both asymmetric-key (public-key/private-key) requests and
 modular arithmetic (for Diffie-Hellman key exchange and other
 cryptographic protocols).
 The two modes are described separately below.
 .Sh THEORY OF OPERATION
 Regardless of whether symmetric-key or asymmetric-key operations are
 to be performed, use of the device requires a basic series of steps:
 .Bl -enum
 .It
 Open the
 .Pa /dev/crypto
 device.
 .It
 Create a new cryptography file descriptor via
 .Dv CRIOGET
 to use for all subsequent
 .Xr ioctl 2
 commands.
 .It
 Close the
 .Pa /dev/crypto
 device.
 .It
 If any symmetric-keyed cryptographic or digest operations will be performed,
 create a session with
 .Dv CIOCGSESSION .
 Most applications will require at least one symmetric session.
 Since cipher and MAC keys are tied to sessions, many
 applications will require more.
 Asymmetric operations do not use sessions.
 .It
 Submit requests, synchronously with
 .Dv CIOCCRYPT
 (symmetric),
 .Dv CIOCCRYPTAEAD
 (symmetric),
 or
 .Dv CIOCKEY
 (asymmetric).
 .It
 Optionally destroy a session with
 .Dv CIOCFSESSION .
 .It
 Close the cryptography file descriptor with
 .Xr close 2 .
 This will automatically close any remaining sessions associated with the
 file desriptor.
 .El
 .Sh SYMMETRIC-KEY OPERATION
 The symmetric-key operation mode provides a context-based API
 to traditional symmetric-key encryption (or privacy) algorithms,
 or to keyed and unkeyed one-way hash (HMAC and MAC) algorithms.
-The symmetric-key mode also permits fused operation,
+The symmetric-key mode also permits encrypt-then-authenticate fused operation,
 where the hardware performs both a privacy algorithm and an integrity-check
 algorithm in a single pass over the data: either a fused
 encrypt/HMAC-generate operation, or a fused HMAC-verify/decrypt operation.
 .Pp
 To use symmetric mode, you must first create a session specifying
 the algorithm(s) and key(s) to use; then issue encrypt or decrypt
 requests against the session.
 .Ss Algorithms
 For a list of supported algorithms, see
 .Xr crypto 7
 and
 .Xr crypto 9 .
 .Ss IOCTL Request Descriptions
 .\"
 .Bl -tag -width CIOCGSESSION
 .\"
 .It Dv CRIOGET Fa int *fd
 Clone the fd argument to
 .Xr ioctl 2 ,
 yielding a new file descriptor for the creation of sessions.
 .\"
 .It Dv CIOCFINDDEV Fa struct crypt_find_op *fop
 .Bd -literal
 struct crypt_find_op {
     int     crid;       /* driver id + flags */
     char    name[32];   /* device/driver name */
 };
 
 .Ed
 If
 .Fa crid
 is -1, then find the driver named
 .Fa name
 and return the id in
 .Fa crid .
 If
 .Fa crid
 is not -1, return the name of the driver with
 .Fa crid
 in
 .Fa name .
 In either case, if the driver is not found,
 .Dv ENOENT
 is returned.
 .It Dv CIOCGSESSION Fa struct session_op *sessp
 .Bd -literal
 struct session_op {
     u_int32_t cipher;	/* e.g. CRYPTO_DES_CBC */
     u_int32_t mac;	/* e.g. CRYPTO_MD5_HMAC */
 
     u_int32_t keylen;	/* cipher key */
     const void *key;
     int mackeylen;	/* mac key */
     const void *mackey;
 
     u_int32_t ses;	/* returns: ses # */
 };
 
 .Ed
 Create a new cryptographic session on a file descriptor for the device;
 that is, a persistent object specific to the chosen
 privacy algorithm, integrity algorithm, and keys specified in
 .Fa sessp .
 The special value 0 for either privacy or integrity
 is reserved to indicate that the indicated operation (privacy or integrity)
 is not desired for this session.
 .Pp
 Multiple sessions may be bound to a single file descriptor.
 The session ID returned in
 .Fa sessp-\*[Gt]ses
 is supplied as a required field in the symmetric-operation structure
 .Fa crypt_op
 for future encryption or hashing requests.
 .\" .Pp
 .\" This implementation will never return a session ID of 0 for a successful
 .\" creation of a session, which is a
 .\" .Nx
 .\" extension.
 .Pp
 For non-zero symmetric-key privacy algorithms, the privacy algorithm
 must be specified in
 .Fa sessp-\*[Gt]cipher ,
 the key length in
 .Fa sessp-\*[Gt]keylen ,
 and the key value in the octets addressed by
 .Fa sessp-\*[Gt]key .
 .Pp
 For keyed one-way hash algorithms, the one-way hash must be specified
 in
 .Fa sessp-\*[Gt]mac ,
 the key length in
 .Fa sessp-\*[Gt]mackey ,
 and the key value in the octets addressed by
 .Fa sessp-\*[Gt]mackeylen .
 .\"
 .Pp
 Support for a specific combination of fused privacy and
 integrity-check algorithms depends on whether the underlying
 hardware supports that combination.
 Not all combinations are supported
 by all hardware, even if the hardware supports each operation as a
 stand-alone non-fused operation.
 .It Dv CIOCGSESSION2 Fa struct session2_op *sessp
 .Bd -literal
 struct session2_op {
     u_int32_t cipher;	/* e.g. CRYPTO_DES_CBC */
     u_int32_t mac;	/* e.g. CRYPTO_MD5_HMAC */
 
     u_int32_t keylen;	/* cipher key */
     const void *key;
     int mackeylen;	/* mac key */
     const void *mackey;
 
     u_int32_t ses;	/* returns: ses # */
     int	crid;		/* driver id + flags (rw) */
     int	pad[4];		/* for future expansion */
 };
 
 .Ed
 This request is similar to CIOGSESSION except that
 .Fa sessp-\*[Gt]crid
 requests either a specific crypto device or a class of devices (software vs
 hardware).
 The
 .Fa sessp-\*[Gt]pad
 field must be initialized to zero.
 .It Dv CIOCCRYPT Fa struct crypt_op *cr_op
 .Bd -literal
 struct crypt_op {
     u_int32_t ses;
     u_int16_t op;	/* e.g. COP_ENCRYPT */
     u_int16_t flags;
     u_int len;
     caddr_t src, dst;
     caddr_t mac;		/* must be large enough for result */
     caddr_t iv;
 };
 
 .Ed
 Request a symmetric-key (or hash) operation.
 To encrypt, set
 .Fa cr_op-\*[Gt]op
 to
 .Dv COP_ENCRYPT .
 To decrypt, set
 .Fa cr_op-\*[Gt]op
 to
 .Dv COP_DECRYPT .
 The field
 .Fa cr_op-\*[Gt]len
 supplies the length of the input buffer; the fields
 .Fa cr_op-\*[Gt]src ,
 .Fa cr_op-\*[Gt]dst ,
 .Fa cr_op-\*[Gt]mac ,
 .Fa cr_op-\*[Gt]iv
 supply the addresses of the input buffer, output buffer,
 one-way hash, and initialization vector, respectively.
-If a session is using both a privacy algorithm and a hash algorithm,
-the request will generate a hash of the input buffer before
-generating the output buffer by default.
-If the
-.Dv COP_F_CIPHER_FIRST
-flag is included in the
-.Fa cr_op-\*[Gt]flags
-field,
-then the request will generate a hash of the output buffer after
-executing the privacy algorithm.
+.Pp
+If a session is using either fused encrypt-then-authenticate or
+an AEAD algorithm,
+decryption operations require the associated hash as an input.
+If the hash is incorrect, the
+operation will fail with
+.Dv EBADMSG
+and the output buffer will remain unchanged.
 .It Dv CIOCCRYPTAEAD Fa struct crypt_aead *cr_aead
 .Bd -literal
 struct crypt_aead {
     u_int32_t ses;
     u_int16_t op;	/* e.g. COP_ENCRYPT */
     u_int16_t flags;
     u_int len;
     u_int aadlen;
     u_int ivlen;
     caddr_t src, dst;
     caddr_t aad;
     caddr_t tag;		/* must be large enough for result */
     caddr_t iv;
 };
 
 .Ed
 The
 .Dv CIOCCRYPTAEAD
 is similar to the
 .Dv CIOCCRYPT
 but provides additional data in
 .Fa cr_aead-\*[Gt]aad
 to include in the authentication mode.
 .It Dv CIOCFSESSION Fa u_int32_t ses_id
 Destroys the session identified by
 .Fa ses_id .
 .El
 .\"
 .Sh ASYMMETRIC-KEY OPERATION
 .Ss Asymmetric-key algorithms
 Contingent upon hardware support, the following asymmetric
 (public-key/private-key; or key-exchange subroutine) operations may
 also be available:
 .Pp
 .Bl -column "CRK_DH_COMPUTE_KEY" "Input parameter" "Output parameter" -offset indent -compact
 .It Em "Algorithm" Ta "Input parameter" Ta "Output parameter"
 .It Em " " Ta "Count" Ta "Count"
 .It Dv CRK_MOD_EXP Ta 3 Ta 1
 .It Dv CRK_MOD_EXP_CRT Ta 6 Ta 1
 .It Dv CRK_DSA_SIGN Ta 5 Ta 2
 .It Dv CRK_DSA_VERIFY Ta 7 Ta 0
 .It Dv CRK_DH_COMPUTE_KEY Ta 3 Ta 1
 .El
 .Pp
 See below for discussion of the input and output parameter counts.
 .Ss Asymmetric-key commands
 .Bl -tag -width CIOCKEY
 .It Dv CIOCASYMFEAT Fa int *feature_mask
 Returns a bitmask of supported asymmetric-key operations.
 Each of the above-listed asymmetric operations is present
 if and only if the bit position numbered by the code for that operation
 is set.
 For example,
 .Dv CRK_MOD_EXP
 is available if and only if the bit
 .Pq 1 \*[Lt]\*[Lt] Dv CRK_MOD_EXP
 is set.
 .It Dv CIOCKEY Fa struct crypt_kop *kop
 .Bd -literal
 struct crypt_kop {
     u_int crk_op;		/* e.g. CRK_MOD_EXP */
     u_int crk_status;		/* return status */
     u_short crk_iparams;	/* # of input params */
     u_short crk_oparams;	/* # of output params */
     u_int crk_pad1;
     struct crparam crk_param[CRK_MAXPARAM];
 };
 
 /* Bignum parameter, in packed bytes. */
 struct crparam {
     void * crp_p;
     u_int crp_nbits;
 };
 
 .Ed
 Performs an asymmetric-key operation from the list above.
 The specific operation is supplied in
 .Fa kop-\*[Gt]crk_op ;
 final status for the operation is returned in
 .Fa kop-\*[Gt]crk_status .
 The number of input arguments and the number of output arguments
 is specified in
 .Fa kop-\*[Gt]crk_iparams
 and
 .Fa kop-\*[Gt]crk_iparams ,
 respectively.
 The field
 .Fa crk_param[]
 must be filled in with exactly
 .Fa kop-\*[Gt]crk_iparams + kop-\*[Gt]crk_oparams
 arguments, each encoded as a
 .Fa struct crparam
 (address, bitlength) pair.
 .Pp
 The semantics of these arguments are currently undocumented.
 .El
 .Sh SEE ALSO
 .Xr aesni 4 ,
 .Xr hifn 4 ,
 .Xr ipsec 4 ,
 .Xr padlock 4 ,
 .Xr safe 4 ,
 .Xr ubsec 4 ,
 .Xr crypto 7 ,
 .Xr geli 8 ,
 .Xr crypto 9
 .Sh HISTORY
 The
 .Nm
 driver first appeared in
 .Ox 3.0 .
 The
 .Nm
 driver was imported to
 .Fx 5.0 .
 .Sh BUGS
 Error checking and reporting is weak.
 .Pp
 The values specified for symmetric-key key sizes to
 .Dv CIOCGSESSION
 must exactly match the values expected by
 .Xr opencrypto 9 .
 The output buffer and MAC buffers supplied to
 .Dv CIOCCRYPT
 must follow whether privacy or integrity algorithms were specified for
 session: if you request a
 .No non- Ns Dv NULL
 algorithm, you must supply a suitably-sized buffer.
 .Pp
 The scheme for passing arguments for asymmetric requests is baroque.
 .Pp
 .Dv CRIOGET
 should not exist.
 It should be possible to use the
 .Dv CIOC Ns \&*
 commands directly on a
 .Pa /dev/crypto
 file descriptor.
Index: head/share/man/man7/crypto.7
===================================================================
--- head/share/man/man7/crypto.7	(revision 359373)
+++ head/share/man/man7/crypto.7	(revision 359374)
@@ -1,141 +1,119 @@
 .\" Copyright (c) 2014 The FreeBSD Foundation
 .\" All rights reserved.
 .\"
 .\" This documentation was written by John-Mark Gurney under
 .\" the sponsorship of the FreeBSD Foundation and
 .\" Rubicon Communications, LLC (Netgate).
 .\" Redistribution and use in source and binary forms, with or without
 .\" modification, are permitted provided that the following conditions
 .\" are met:
 .\" 1.  Redistributions of source code must retain the above copyright
 .\"     notice, this list of conditions and the following disclaimer.
 .\" 2.  Redistributions in binary form must reproduce the above copyright
 .\"     notice, this list of conditions and the following disclaimer in the
 .\"     documentation and/or other materials provided with the distribution.
 .\"
 .\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 .\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 .\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 .\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 .\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 .\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 .\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 .\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 .\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 .\" SUCH DAMAGE.
 .\"
 .\" $FreeBSD$
 .\"
-.Dd January 2, 2015
+.Dd March 27, 2020
 .Dt CRYPTO 7
 .Os
 .Sh NAME
 .Nm crypto
 .Nd OpenCrypto algorithms
 .Sh SYNOPSIS
 In the kernel configuration file:
 .Cd "device crypto"
 .Pp
 Or load the crypto.ko module.
 .Sh DESCRIPTION
 The following cryptographic algorithms that are part of the OpenCrypto
 framework have the following requirements.
 .Pp
 Cipher algorithms:
 .Bl -tag -width ".Dv CRYPTO_AES_CBC"
 .It Dv CRYPTO_AES_CBC
 .Bl -tag -width "Block size :" -compact -offset indent
 .It IV size :
 16
 .It Block size :
 16
 .It Key size :
 16, 24 or 32
 .El
 .Pp
 This algorithm implements Cipher-block chaining.
 .It Dv CRYPTO_AES_NIST_GCM_16
 .Bl -tag -width "Block size :" -compact -offset indent
 .It IV size :
 12
 .It Block size :
 1
 .It Key size :
 16, 24 or 32
 .It Digest size :
 16
 .El
 .Pp
 This algorithm implements Galois/Counter Mode.
-This is the cipher part of an AEAD
+This cipher uses AEAD
 .Pq Authenticated Encryption with Associated Data
 mode.
-This requires use of the use of a proper authentication mode, one of
-.Dv CRYPTO_AES_128_NIST_GMAC ,
-.Dv CRYPTO_AES_192_NIST_GMAC
-or
-.Dv CRYPTO_AES_256_NIST_GMAC ,
-that corresponds with the number of bits in the key that you are using.
 .Pp
-The associated data (if any) must be provided by the authentication mode op.
-The authentication tag will be read/written from/to the offset crd_inject
-specified in the descriptor for the authentication mode.
+The authentication tag will be read/written from/to the offset
+.Va crp_digest_start
+specified in the request.
 .Pp
 Note: You must provide an IV on every call.
 .It Dv CRYPTO_AES_ICM
 .Bl -tag -width "Block size :" -compact -offset indent
 .It IV size :
 16
 .It Block size :
 1 (aesni), 16 (software)
 .It Key size :
 16, 24 or 32
 .El
 .Pp
 This algorithm implements Integer Counter Mode.
 This is similar to what most people call counter mode, but instead of the
 counter being split into a nonce and a counter part, then entire nonce is
 used as the initial counter.
 This does mean that if a counter is required that rolls over at 32 bits,
 the transaction need to be split into two parts where the counter rolls over.
 The counter incremented as a 128-bit big endian number.
 .Pp
 Note: You must provide an IV on every call.
 .It Dv CRYPTO_AES_XTS
 .Bl -tag -width "Block size :" -compact -offset indent
 .It IV size :
 8
 .It Block size :
 16
 .It Key size :
 32 or 64
 .El
 .Pp
 This algorithm implements XEX Tweakable Block Cipher with Ciphertext Stealing
 as defined in NIST SP 800-38E.
 .Pp
 NOTE: The ciphertext stealing part is not implemented which is why this cipher
 is listed as having a block size of 16 instead of 1.
-.El
-.Pp
-Authentication algorithms:
-.Bl -tag -width ".Dv CRYPTO_AES_256_NIST_GMAC"
-.It CRYPTO_AES_128_NIST_GMAC
-See
-.Dv CRYPTO_AES_NIST_GCM_16
-in the cipher mode section.
-.It CRYPTO_AES_192_NIST_GMAC
-See
-.Dv CRYPTO_AES_NIST_GCM_16
-in the cipher mode section.
-.It CRYPTO_AES_256_NIST_GMAC
-See
-.Dv CRYPTO_AES_NIST_GCM_16
-in the cipher mode section.
 .El
 .Sh SEE ALSO
 .Xr crypto 4 ,
 .Xr crypto 9
 .Sh BUGS
 Not all the implemented algorithms are listed.
Index: head/share/man/man9/Makefile
===================================================================
--- head/share/man/man9/Makefile	(revision 359373)
+++ head/share/man/man9/Makefile	(revision 359374)
@@ -1,2335 +1,2352 @@
 # $FreeBSD$
 
 .include <src.opts.mk>
 
 MAN=	accept_filter.9 \
 	accf_data.9 \
 	accf_dns.9 \
 	accf_http.9 \
 	acl.9 \
 	alq.9 \
 	altq.9 \
 	atomic.9 \
 	bhnd.9 \
 	bhnd_erom.9 \
 	bios.9 \
 	bitset.9 \
 	boot.9 \
 	bpf.9 \
 	buf.9 \
 	buf_ring.9 \
 	BUF_ISLOCKED.9 \
 	BUF_LOCK.9 \
 	BUF_LOCKFREE.9 \
 	BUF_LOCKINIT.9 \
 	BUF_RECURSED.9 \
 	BUF_TIMELOCK.9 \
 	BUF_UNLOCK.9 \
 	bus_activate_resource.9 \
 	BUS_ADD_CHILD.9 \
 	bus_adjust_resource.9 \
 	bus_alloc_resource.9 \
 	BUS_BIND_INTR.9 \
 	bus_child_present.9 \
 	BUS_CHILD_DELETED.9 \
 	BUS_CHILD_DETACHED.9 \
 	BUS_CONFIG_INTR.9 \
 	bus_delayed_attach_children.9 \
 	BUS_DESCRIBE_INTR.9 \
 	bus_dma.9 \
 	bus_generic_attach.9 \
 	bus_generic_detach.9 \
 	bus_generic_new_pass.9 \
 	bus_generic_print_child.9 \
 	bus_generic_read_ivar.9 \
 	bus_generic_shutdown.9 \
 	BUS_GET_CPUS.9 \
 	bus_get_resource.9 \
 	bus_map_resource.9 \
 	BUS_NEW_PASS.9 \
 	BUS_PRINT_CHILD.9 \
 	BUS_READ_IVAR.9 \
 	BUS_RESCAN.9 \
 	bus_release_resource.9 \
 	bus_set_pass.9 \
 	bus_set_resource.9 \
 	BUS_SETUP_INTR.9 \
 	bus_space.9 \
 	byteorder.9 \
 	callout.9 \
 	casuword.9 \
 	cd.9 \
 	cnv.9 \
 	condvar.9 \
 	config_intrhook.9 \
 	contigmalloc.9 \
 	copy.9 \
 	counter.9 \
 	cpuset.9 \
 	cr_cansee.9 \
 	critical_enter.9 \
 	cr_seeothergids.9 \
 	cr_seeotheruids.9 \
 	crypto.9 \
+	crypto_asym.9 \
+	crypto_driver.9 \
+	crypto_request.9 \
+	crypto_session.9 \
 	CTASSERT.9 \
 	DB_COMMAND.9 \
 	DECLARE_GEOM_CLASS.9 \
 	DECLARE_MODULE.9 \
 	DEFINE_IFUNC.9 \
 	DELAY.9 \
 	devclass.9 \
 	devclass_find.9 \
 	devclass_get_device.9 \
 	devclass_get_devices.9 \
 	devclass_get_drivers.9 \
 	devclass_get_maxunit.9 \
 	devclass_get_name.9 \
 	devclass_get_softc.9 \
 	dev_clone.9 \
 	devfs_set_cdevpriv.9 \
 	device.9 \
 	device_add_child.9 \
 	DEVICE_ATTACH.9 \
 	device_delete_child.9 \
 	device_delete_children.9 \
 	DEVICE_DETACH.9 \
 	device_enable.9 \
 	device_find_child.9 \
 	device_get_children.9 \
 	device_get_devclass.9 \
 	device_get_driver.9 \
 	device_get_ivars.9 \
 	device_get_name.9 \
 	device_get_parent.9 \
 	device_get_softc.9 \
 	device_get_state.9 \
 	device_get_sysctl.9 \
 	device_get_unit.9 \
 	DEVICE_IDENTIFY.9 \
 	device_printf.9 \
 	DEVICE_PROBE.9 \
 	device_probe_and_attach.9 \
 	device_quiet.9 \
 	device_set_desc.9 \
 	device_set_driver.9 \
 	device_set_flags.9 \
 	DEVICE_SHUTDOWN.9 \
 	DEV_MODULE.9 \
 	dev_refthread.9 \
 	devstat.9 \
 	devtoname.9 \
 	disk.9 \
 	dnv.9 \
 	domain.9 \
 	domainset.9 \
 	dpcpu.9 \
 	drbr.9 \
 	driver.9 \
 	DRIVER_MODULE.9 \
 	efirt.9 \
 	epoch.9 \
 	EVENTHANDLER.9 \
 	eventtimers.9 \
 	extattr.9 \
 	fail.9 \
 	fdt_pinctrl.9 \
 	fetch.9 \
 	firmware.9 \
 	fpu_kern.9 \
 	g_access.9 \
 	g_attach.9 \
 	g_bio.9 \
 	g_consumer.9 \
 	g_data.9 \
 	get_cyclecount.9 \
 	getenv.9 \
 	getnewvnode.9 \
 	g_event.9 \
 	g_geom.9 \
 	g_provider.9 \
 	g_provider_by_name.9 \
 	groupmember.9 \
 	g_wither_geom.9 \
 	hash.9 \
 	hashinit.9 \
 	hexdump.9 \
 	hhook.9 \
 	ieee80211.9 \
 	ieee80211_amrr.9 \
 	ieee80211_beacon.9 \
 	ieee80211_bmiss.9 \
 	ieee80211_crypto.9 \
 	ieee80211_ddb.9 \
 	ieee80211_input.9 \
 	ieee80211_node.9 \
 	ieee80211_output.9 \
 	ieee80211_proto.9 \
 	ieee80211_radiotap.9 \
 	ieee80211_regdomain.9 \
 	ieee80211_scan.9 \
 	ieee80211_vap.9 \
 	iflib.9 \
 	iflibdd.9 \
 	iflibdi.9 \
 	iflibtxrx.9 \
 	ifnet.9 \
 	inittodr.9 \
 	insmntque.9 \
 	intro.9 \
 	ithread.9 \
 	KASSERT.9 \
 	kern_testfrwk.9 \
 	kernacc.9 \
 	kernel_mount.9 \
 	khelp.9 \
 	kobj.9 \
 	kproc.9 \
 	kqueue.9 \
 	kthread.9 \
 	ktr.9 \
 	lock.9 \
 	locking.9 \
 	LOCK_PROFILING.9 \
 	mac.9 \
 	make_dev.9 \
 	malloc.9 \
 	mbchain.9 \
 	mbuf.9 \
 	mbuf_tags.9 \
 	MD5.9 \
 	mdchain.9 \
 	memcchr.9 \
 	memguard.9 \
 	microseq.9 \
 	microtime.9 \
 	microuptime.9 \
 	mi_switch.9 \
 	mod_cc.9 \
 	module.9 \
 	MODULE_DEPEND.9 \
 	MODULE_PNP_INFO.9 \
 	MODULE_VERSION.9 \
 	mtx_pool.9 \
 	mutex.9 \
 	namei.9 \
 	netisr.9 \
 	nv.9 \
 	OF_child.9 \
 	OF_device_from_xref.9 \
 	OF_finddevice.9 \
 	OF_getprop.9 \
 	OF_node_from_xref.9 \
 	OF_package_to_path.9 \
 	ofw_bus_is_compatible.9 \
 	ofw_bus_status_okay.9 \
 	osd.9 \
 	owll.9 \
 	own.9 \
 	panic.9 \
 	PCBGROUP.9 \
 	p_candebug.9 \
 	p_cansee.9 \
 	pci.9 \
 	PCI_IOV_ADD_VF.9 \
 	PCI_IOV_INIT.9 \
 	pci_iov_schema.9 \
 	PCI_IOV_UNINIT.9 \
 	pfil.9 \
 	pfind.9 \
 	pget.9 \
 	pgfind.9 \
 	PHOLD.9 \
 	physio.9 \
 	pmap.9 \
 	pmap_activate.9 \
 	pmap_clear_modify.9 \
 	pmap_copy.9 \
 	pmap_enter.9 \
 	pmap_extract.9 \
 	pmap_growkernel.9 \
 	pmap_init.9 \
 	pmap_is_modified.9 \
 	pmap_is_prefaultable.9 \
 	pmap_map.9 \
 	pmap_mincore.9 \
 	pmap_object_init_pt.9 \
 	pmap_page_exists_quick.9 \
 	pmap_page_init.9 \
 	pmap_pinit.9 \
 	pmap_protect.9 \
 	pmap_qenter.9 \
 	pmap_quick_enter_page.9 \
 	pmap_release.9 \
 	pmap_remove.9 \
 	pmap_resident_count.9 \
 	pmap_unwire.9 \
 	pmap_zero_page.9 \
 	printf.9 \
 	prison_check.9 \
 	priv.9 \
 	proc_rwmem.9 \
 	pseudofs.9 \
 	psignal.9 \
 	pwmbus.9 \
 	random.9 \
 	random_harvest.9 \
 	ratecheck.9 \
 	redzone.9 \
 	refcount.9 \
 	resettodr.9 \
 	resource_int_value.9 \
 	rijndael.9 \
 	rman.9 \
 	rmlock.9 \
 	rtalloc.9 \
 	rtentry.9 \
 	runqueue.9 \
 	rwlock.9 \
 	sbuf.9 \
 	scheduler.9 \
 	SDT.9 \
 	securelevel_gt.9 \
 	selrecord.9 \
 	sema.9 \
 	seqc.9 \
 	sf_buf.9 \
 	sglist.9 \
 	shm_map.9 \
 	signal.9 \
 	sleep.9 \
 	sleepqueue.9 \
 	socket.9 \
 	stack.9 \
 	store.9 \
 	style.9 \
 	style.lua.9 \
 	${_superio.9} \
 	swi.9 \
 	sx.9 \
 	syscall_helper_register.9 \
 	SYSCALL_MODULE.9 \
 	sysctl.9 \
 	sysctl_add_oid.9 \
 	sysctl_ctx_init.9 \
 	SYSINIT.9 \
 	taskqueue.9 \
 	tcp_functions.9 \
 	thread_exit.9 \
 	time.9 \
 	tvtohz.9 \
 	ucred.9 \
 	uidinfo.9 \
 	uio.9 \
 	unr.9 \
 	vaccess.9 \
 	vaccess_acl_nfs4.9 \
 	vaccess_acl_posix1e.9 \
 	vcount.9 \
 	vflush.9 \
 	VFS.9 \
 	vfs_busy.9 \
 	VFS_CHECKEXP.9 \
 	vfsconf.9 \
 	VFS_FHTOVP.9 \
 	vfs_getnewfsid.9 \
 	vfs_getopt.9 \
 	vfs_getvfs.9 \
 	VFS_MOUNT.9 \
 	vfs_mountedfrom.9 \
 	VFS_QUOTACTL.9 \
 	VFS_ROOT.9 \
 	vfs_rootmountalloc.9 \
 	VFS_SET.9 \
 	VFS_STATFS.9 \
 	vfs_suser.9 \
 	VFS_SYNC.9 \
 	vfs_timestamp.9 \
 	vfs_unbusy.9 \
 	VFS_UNMOUNT.9 \
 	vfs_unmountall.9 \
 	VFS_VGET.9 \
 	vget.9 \
 	vgone.9 \
 	vhold.9 \
 	vinvalbuf.9 \
 	vm_fault_prefault.9 \
 	vm_map.9 \
 	vm_map_check_protection.9 \
 	vm_map_create.9 \
 	vm_map_delete.9 \
 	vm_map_entry_resize_free.9 \
 	vm_map_find.9 \
 	vm_map_findspace.9 \
 	vm_map_inherit.9 \
 	vm_map_init.9 \
 	vm_map_insert.9 \
 	vm_map_lock.9 \
 	vm_map_lookup.9 \
 	vm_map_madvise.9 \
 	vm_map_max.9 \
 	vm_map_protect.9 \
 	vm_map_remove.9 \
 	vm_map_simplify_entry.9 \
 	vm_map_stack.9 \
 	vm_map_submap.9 \
 	vm_map_sync.9 \
 	vm_map_wire.9 \
 	vm_page_alloc.9 \
 	vm_page_bits.9 \
 	vm_page_busy.9 \
 	vm_page_deactivate.9 \
 	vm_page_dontneed.9 \
 	vm_page_aflag.9 \
 	vm_page_free.9 \
 	vm_page_grab.9 \
 	vm_page_insert.9 \
 	vm_page_lookup.9 \
 	vm_page_rename.9 \
 	vm_page_wire.9 \
 	vm_set_page_size.9 \
 	vmem.9 \
 	vn_fullpath.9 \
 	vn_isdisk.9 \
 	vnet.9 \
 	vnode.9 \
 	VOP_ACCESS.9 \
 	VOP_ACLCHECK.9 \
 	VOP_ADVISE.9 \
 	VOP_ADVLOCK.9 \
 	VOP_ALLOCATE.9 \
 	VOP_ATTRIB.9 \
 	VOP_BMAP.9 \
 	VOP_BWRITE.9 \
 	VOP_COPY_FILE_RANGE.9 \
 	VOP_CREATE.9 \
 	VOP_FSYNC.9 \
 	VOP_GETACL.9 \
 	VOP_GETEXTATTR.9 \
 	VOP_GETPAGES.9 \
 	VOP_INACTIVE.9 \
 	VOP_IOCTL.9 \
 	VOP_LINK.9 \
 	VOP_LISTEXTATTR.9 \
 	VOP_LOCK.9 \
 	VOP_LOOKUP.9 \
 	VOP_OPENCLOSE.9 \
 	VOP_PATHCONF.9 \
 	VOP_PRINT.9 \
 	VOP_RDWR.9 \
 	VOP_READDIR.9 \
 	VOP_READLINK.9 \
 	VOP_REALLOCBLKS.9 \
 	VOP_REMOVE.9 \
 	VOP_RENAME.9 \
 	VOP_REVOKE.9 \
 	VOP_SETACL.9 \
 	VOP_SETEXTATTR.9 \
 	VOP_STRATEGY.9 \
 	VOP_VPTOCNP.9 \
 	VOP_VPTOFH.9 \
 	vref.9 \
 	vrefcnt.9 \
 	vrele.9 \
 	vslock.9 \
 	watchdog.9 \
 	zone.9
 
 MLINKS=	unr.9 alloc_unr.9 \
 	unr.9 alloc_unrl.9 \
 	unr.9 alloc_unr_specific.9 \
 	unr.9 clear_unrhdr.9 \
 	unr.9 delete_unrhdr.9 \
 	unr.9 free_unr.9 \
 	unr.9 new_unrhdr.9
 MLINKS+=accept_filter.9 accept_filt_add.9 \
 	accept_filter.9 accept_filt_del.9 \
 	accept_filter.9 accept_filt_generic_mod_event.9 \
 	accept_filter.9 accept_filt_get.9
 MLINKS+=alq.9 ALQ.9 \
 	alq.9 alq_close.9 \
 	alq.9 alq_flush.9 \
 	alq.9 alq_get.9 \
 	alq.9 alq_getn.9 \
 	alq.9 alq_open.9 \
 	alq.9 alq_open_flags.9 \
 	alq.9 alq_post.9 \
 	alq.9 alq_post_flags.9 \
 	alq.9 alq_write.9 \
 	alq.9 alq_writen.9
 MLINKS+=altq.9 ALTQ.9
 MLINKS+=atomic.9 atomic_add.9 \
 	atomic.9 atomic_clear.9 \
 	atomic.9 atomic_cmpset.9 \
 	atomic.9 atomic_fcmpset.9 \
 	atomic.9 atomic_fetchadd.9 \
 	atomic.9 atomic_load.9 \
 	atomic.9 atomic_readandclear.9 \
 	atomic.9 atomic_set.9 \
 	atomic.9 atomic_store.9 \
 	atomic.9 atomic_subtract.9 \
 	atomic.9 atomic_swap.9 \
 	atomic.9 atomic_testandclear.9 \
 	atomic.9 atomic_testandset.9 \
 	atomic.9 atomic_thread_fence.9
 MLINKS+=bhnd.9 BHND_MATCH_BOARD_TYPE.9 \
 	bhnd.9 BHND_MATCH_BOARD_VENDOR.9 \
 	bhnd.9 BHND_MATCH_CHIP_ID.9 \
 	bhnd.9 BHND_MATCH_CHIP_PKG.9 \
 	bhnd.9 BHND_MATCH_CHIP_REV.9 \
 	bhnd.9 BHND_MATCH_CORE_ID.9 \
 	bhnd.9 BHND_MATCH_CORE_VENDOR.9 \
 	bhnd.9 bhnd_activate_resource.9 \
 	bhnd.9 bhnd_alloc_pmu.9 \
 	bhnd.9 bhnd_alloc_resource.9 \
 	bhnd.9 bhnd_alloc_resource_any.9 \
 	bhnd.9 bhnd_alloc_resources.9 \
 	bhnd.9 bhnd_board_matches.9 \
 	bhnd.9 bhnd_bus_match_child.9 \
 	bhnd.9 bhnd_bus_read_1.9 \
 	bhnd.9 bhnd_bus_read_2.9 \
 	bhnd.9 bhnd_bus_read_4.9 \
 	bhnd.9 bhnd_bus_read_stream_1.9 \
 	bhnd.9 bhnd_bus_read_stream_2.9 \
 	bhnd.9 bhnd_bus_read_stream_4.9 \
 	bhnd.9 bhnd_bus_write_1.9 \
 	bhnd.9 bhnd_bus_write_2.9 \
 	bhnd.9 bhnd_bus_write_4.9 \
 	bhnd.9 bhnd_bus_write_stream_1.9 \
 	bhnd.9 bhnd_bus_write_stream_2.9 \
 	bhnd.9 bhnd_bus_write_stream_4.9 \
 	bhnd.9 bhnd_chip_matches.9 \
 	bhnd.9 bhnd_core_class.9 \
 	bhnd.9 bhnd_core_get_match_desc.9 \
 	bhnd.9 bhnd_core_matches.9 \
 	bhnd.9 bhnd_core_name.9 \
 	bhnd.9 bhnd_cores_equal.9 \
 	bhnd.9 bhnd_deactivate_resource.9 \
 	bhnd.9 bhnd_decode_port_rid.9 \
 	bhnd.9 bhnd_deregister_provider.9 \
 	bhnd.9 bhnd_device_lookup.9 \
 	bhnd.9 bhnd_device_matches.9 \
 	bhnd.9 bhnd_device_quirks.9 \
 	bhnd.9 bhnd_driver_get_erom_class.9 \
 	bhnd.9 bhnd_enable_clocks.9 \
 	bhnd.9 bhnd_find_core_class.9 \
 	bhnd.9 bhnd_find_core_name.9 \
 	bhnd.9 bhnd_format_chip_id.9 \
 	bhnd.9 bhnd_get_attach_type.9 \
 	bhnd.9 bhnd_get_chipid.9 \
 	bhnd.9 bhnd_get_class.9 \
 	bhnd.9 bhnd_get_clock_freq.9 \
 	bhnd.9 bhnd_get_clock_latency.9 \
 	bhnd.9 bhnd_get_core_index.9 \
 	bhnd.9 bhnd_get_core_info.9 \
 	bhnd.9 bhnd_get_core_unit.9 \
 	bhnd.9 bhnd_get_device.9 \
 	bhnd.9 bhnd_get_device_name.9 \
 	bhnd.9 bhnd_get_dma_translation.9 \
 	bhnd.9 bhnd_get_hwrev.9 \
 	bhnd.9 bhnd_get_intr_count.9 \
 	bhnd.9 bhnd_get_intr_ivec.9 \
 	bhnd.9 bhnd_get_port_count.9 \
 	bhnd.9 bhnd_get_port_rid.9 \
 	bhnd.9 bhnd_get_region_addr.9 \
 	bhnd.9 bhnd_get_region_count.9 \
 	bhnd.9 bhnd_get_vendor.9 \
 	bhnd.9 bhnd_get_vendor_name.9 \
 	bhnd.9 bhnd_hwrev_matches.9 \
 	bhnd.9 bhnd_is_hw_suspended.9 \
 	bhnd.9 bhnd_is_region_valid.9 \
 	bhnd.9 bhnd_map_intr.9 \
 	bhnd.9 bhnd_match_core.9 \
 	bhnd.9 bhnd_nvram_getvar.9 \
 	bhnd.9 bhnd_nvram_getvar_array.9 \
 	bhnd.9 bhnd_nvram_getvar_int.9 \
 	bhnd.9 bhnd_nvram_getvar_int16.9 \
 	bhnd.9 bhnd_nvram_getvar_int32.9 \
 	bhnd.9 bhnd_nvram_getvar_int8.9 \
 	bhnd.9 bhnd_nvram_getvar_str.9 \
 	bhnd.9 bhnd_nvram_getvar_uint.9 \
 	bhnd.9 bhnd_nvram_getvar_uint16.9 \
 	bhnd.9 bhnd_nvram_getvar_uint32.9 \
 	bhnd.9 bhnd_nvram_getvar_uint8.9 \
 	bhnd.9 bhnd_nvram_string_array_next.9 \
 	bhnd.9 bhnd_read_board_info.9 \
 	bhnd.9 bhnd_read_config.9 \
 	bhnd.9 bhnd_read_ioctl.9 \
 	bhnd.9 bhnd_read_iost.9 \
 	bhnd.9 bhnd_register_provider.9 \
 	bhnd.9 bhnd_release_ext_rsrc.9 \
 	bhnd.9 bhnd_release_pmu.9 \
 	bhnd.9 bhnd_release_provider.9 \
 	bhnd.9 bhnd_release_resource.9 \
 	bhnd.9 bhnd_release_resources.9 \
 	bhnd.9 bhnd_request_clock.9 \
 	bhnd.9 bhnd_request_ext_rsrc.9 \
 	bhnd.9 bhnd_reset_hw.9 \
 	bhnd.9 bhnd_retain_provider.9 \
 	bhnd.9 bhnd_set_custom_core_desc.9 \
 	bhnd.9 bhnd_set_default_core_desc.9 \
 	bhnd.9 bhnd_suspend_hw.9 \
 	bhnd.9 bhnd_unmap_intr.9 \
 	bhnd.9 bhnd_vendor_name.9 \
 	bhnd.9 bhnd_write_config.9 \
 	bhnd.9 bhnd_write_ioctl.9
 MLINKS+=bhnd_erom.9 bhnd_erom_alloc.9 \
 	bhnd_erom.9 bhnd_erom_dump.9 \
 	bhnd_erom.9 bhnd_erom_fini_static.9 \
 	bhnd_erom.9 bhnd_erom_free.9 \
 	bhnd_erom.9 bhnd_erom_free_core_table.9 \
 	bhnd_erom.9 bhnd_erom_get_core_table.9 \
 	bhnd_erom.9 bhnd_erom_init_static.9 \
 	bhnd_erom.9 bhnd_erom_io.9 \
 	bhnd_erom.9 bhnd_erom_io_fini.9 \
 	bhnd_erom.9 bhnd_erom_io_map.9 \
 	bhnd_erom.9 bhnd_erom_io_read.9 \
 	bhnd_erom.9 bhnd_erom_iobus_init.9 \
 	bhnd_erom.9 bhnd_erom_iores_new.9 \
 	bhnd_erom.9 bhnd_erom_lookup_core.9 \
 	bhnd_erom.9 bhnd_erom_lookup_core_addr.9 \
 	bhnd_erom.9 bhnd_erom_probe.9 \
 	bhnd_erom.9 bhnd_erom_probe_driver_classes.9
 MLINKS+=bitset.9 BITSET_DEFINE.9 \
 	bitset.9 BITSET_T_INITIALIZER.9 \
 	bitset.9 BITSET_FSET.9 \
 	bitset.9 BIT_CLR.9 \
 	bitset.9 BIT_COPY.9 \
 	bitset.9 BIT_ISSET.9 \
 	bitset.9 BIT_SET.9 \
 	bitset.9 BIT_ZERO.9 \
 	bitset.9 BIT_FILL.9 \
 	bitset.9 BIT_SETOF.9 \
 	bitset.9 BIT_EMPTY.9 \
 	bitset.9 BIT_ISFULLSET.9 \
 	bitset.9 BIT_FFS.9 \
 	bitset.9 BIT_COUNT.9 \
 	bitset.9 BIT_SUBSET.9 \
 	bitset.9 BIT_OVERLAP.9 \
 	bitset.9 BIT_CMP.9 \
 	bitset.9 BIT_OR.9 \
 	bitset.9 BIT_AND.9 \
 	bitset.9 BIT_ANDNOT.9 \
 	bitset.9 BIT_CLR_ATOMIC.9 \
 	bitset.9 BIT_SET_ATOMIC.9 \
 	bitset.9 BIT_SET_ATOMIC_ACQ.9 \
 	bitset.9 BIT_AND_ATOMIC.9 \
 	bitset.9 BIT_OR_ATOMIC.9 \
 	bitset.9 BIT_COPY_STORE_REL.9
 MLINKS+=bpf.9 bpfattach.9 \
 	bpf.9 bpfattach2.9 \
 	bpf.9 bpfdetach.9 \
 	bpf.9 bpf_filter.9 \
 	bpf.9 bpf_mtap.9 \
 	bpf.9 bpf_mtap2.9 \
 	bpf.9 bpf_tap.9 \
 	bpf.9 bpf_validate.9
 MLINKS+=buf.9 bp.9
 MLINKS+=buf_ring.9 buf_ring_alloc.9 \
 	buf_ring.9 buf_ring_free.9 \
 	buf_ring.9 buf_ring_enqueue.9 \
 	buf_ring.9 buf_ring_enqueue_bytes.9 \
 	buf_ring.9 buf_ring_dequeue_mc.9 \
 	buf_ring.9 buf_ring_dequeue_sc.9 \
 	buf_ring.9 buf_ring_count.9 \
 	buf_ring.9 buf_ring_empty.9 \
 	buf_ring.9 buf_ring_full.9 \
 	buf_ring.9 buf_ring_peek.9
 MLINKS+=bus_activate_resource.9 bus_deactivate_resource.9
 MLINKS+=bus_alloc_resource.9 bus_alloc_resource_any.9
 MLINKS+=BUS_BIND_INTR.9 bus_bind_intr.9
 MLINKS+=BUS_DESCRIBE_INTR.9 bus_describe_intr.9
 MLINKS+=bus_dma.9 busdma.9 \
 	bus_dma.9 bus_dmamap_create.9 \
 	bus_dma.9 bus_dmamap_destroy.9 \
 	bus_dma.9 bus_dmamap_load.9 \
 	bus_dma.9 bus_dmamap_load_bio.9 \
 	bus_dma.9 bus_dmamap_load_ccb.9 \
 	bus_dma.9 bus_dmamap_load_mbuf.9 \
 	bus_dma.9 bus_dmamap_load_mbuf_sg.9 \
 	bus_dma.9 bus_dmamap_load_uio.9 \
 	bus_dma.9 bus_dmamap_sync.9 \
 	bus_dma.9 bus_dmamap_unload.9 \
 	bus_dma.9 bus_dmamem_alloc.9 \
 	bus_dma.9 bus_dmamem_free.9 \
 	bus_dma.9 bus_dma_tag_create.9 \
 	bus_dma.9 bus_dma_tag_destroy.9
 MLINKS+=bus_generic_read_ivar.9 bus_generic_write_ivar.9
 MLINKS+=BUS_GET_CPUS.9 bus_get_cpus.9
 MLINKS+=bus_map_resource.9 bus_unmap_resource.9 \
 	bus_map_resource.9 resource_init_map_request.9
 MLINKS+=BUS_READ_IVAR.9 BUS_WRITE_IVAR.9
 MLINKS+=BUS_SETUP_INTR.9 bus_setup_intr.9 \
 	BUS_SETUP_INTR.9 BUS_TEARDOWN_INTR.9 \
 	BUS_SETUP_INTR.9 bus_teardown_intr.9
 MLINKS+=bus_space.9 bus_space_alloc.9 \
 	bus_space.9 bus_space_barrier.9 \
 	bus_space.9 bus_space_copy_region_1.9 \
 	bus_space.9 bus_space_copy_region_2.9 \
 	bus_space.9 bus_space_copy_region_4.9 \
 	bus_space.9 bus_space_copy_region_8.9 \
 	bus_space.9 bus_space_copy_region_stream_1.9 \
 	bus_space.9 bus_space_copy_region_stream_2.9 \
 	bus_space.9 bus_space_copy_region_stream_4.9 \
 	bus_space.9 bus_space_copy_region_stream_8.9 \
 	bus_space.9 bus_space_free.9 \
 	bus_space.9 bus_space_map.9 \
 	bus_space.9 bus_space_read_1.9 \
 	bus_space.9 bus_space_read_2.9 \
 	bus_space.9 bus_space_read_4.9 \
 	bus_space.9 bus_space_read_8.9 \
 	bus_space.9 bus_space_read_multi_1.9 \
 	bus_space.9 bus_space_read_multi_2.9 \
 	bus_space.9 bus_space_read_multi_4.9 \
 	bus_space.9 bus_space_read_multi_8.9 \
 	bus_space.9 bus_space_read_multi_stream_1.9 \
 	bus_space.9 bus_space_read_multi_stream_2.9 \
 	bus_space.9 bus_space_read_multi_stream_4.9 \
 	bus_space.9 bus_space_read_multi_stream_8.9 \
 	bus_space.9 bus_space_read_region_1.9 \
 	bus_space.9 bus_space_read_region_2.9 \
 	bus_space.9 bus_space_read_region_4.9 \
 	bus_space.9 bus_space_read_region_8.9 \
 	bus_space.9 bus_space_read_region_stream_1.9 \
 	bus_space.9 bus_space_read_region_stream_2.9 \
 	bus_space.9 bus_space_read_region_stream_4.9 \
 	bus_space.9 bus_space_read_region_stream_8.9 \
 	bus_space.9 bus_space_read_stream_1.9 \
 	bus_space.9 bus_space_read_stream_2.9 \
 	bus_space.9 bus_space_read_stream_4.9 \
 	bus_space.9 bus_space_read_stream_8.9 \
 	bus_space.9 bus_space_set_multi_1.9 \
 	bus_space.9 bus_space_set_multi_2.9 \
 	bus_space.9 bus_space_set_multi_4.9 \
 	bus_space.9 bus_space_set_multi_8.9 \
 	bus_space.9 bus_space_set_multi_stream_1.9 \
 	bus_space.9 bus_space_set_multi_stream_2.9 \
 	bus_space.9 bus_space_set_multi_stream_4.9 \
 	bus_space.9 bus_space_set_multi_stream_8.9 \
 	bus_space.9 bus_space_set_region_1.9 \
 	bus_space.9 bus_space_set_region_2.9 \
 	bus_space.9 bus_space_set_region_4.9 \
 	bus_space.9 bus_space_set_region_8.9 \
 	bus_space.9 bus_space_set_region_stream_1.9 \
 	bus_space.9 bus_space_set_region_stream_2.9 \
 	bus_space.9 bus_space_set_region_stream_4.9 \
 	bus_space.9 bus_space_set_region_stream_8.9 \
 	bus_space.9 bus_space_subregion.9 \
 	bus_space.9 bus_space_unmap.9 \
 	bus_space.9 bus_space_write_1.9 \
 	bus_space.9 bus_space_write_2.9 \
 	bus_space.9 bus_space_write_4.9 \
 	bus_space.9 bus_space_write_8.9 \
 	bus_space.9 bus_space_write_multi_1.9 \
 	bus_space.9 bus_space_write_multi_2.9 \
 	bus_space.9 bus_space_write_multi_4.9 \
 	bus_space.9 bus_space_write_multi_8.9 \
 	bus_space.9 bus_space_write_multi_stream_1.9 \
 	bus_space.9 bus_space_write_multi_stream_2.9 \
 	bus_space.9 bus_space_write_multi_stream_4.9 \
 	bus_space.9 bus_space_write_multi_stream_8.9 \
 	bus_space.9 bus_space_write_region_1.9 \
 	bus_space.9 bus_space_write_region_2.9 \
 	bus_space.9 bus_space_write_region_4.9 \
 	bus_space.9 bus_space_write_region_8.9 \
 	bus_space.9 bus_space_write_region_stream_1.9 \
 	bus_space.9 bus_space_write_region_stream_2.9 \
 	bus_space.9 bus_space_write_region_stream_4.9 \
 	bus_space.9 bus_space_write_region_stream_8.9 \
 	bus_space.9 bus_space_write_stream_1.9 \
 	bus_space.9 bus_space_write_stream_2.9 \
 	bus_space.9 bus_space_write_stream_4.9 \
 	bus_space.9 bus_space_write_stream_8.9
 MLINKS+=byteorder.9 be16dec.9 \
 	byteorder.9 be16enc.9 \
 	byteorder.9 be16toh.9 \
 	byteorder.9 be32dec.9 \
 	byteorder.9 be32enc.9 \
 	byteorder.9 be32toh.9 \
 	byteorder.9 be64dec.9 \
 	byteorder.9 be64enc.9 \
 	byteorder.9 be64toh.9 \
 	byteorder.9 bswap16.9 \
 	byteorder.9 bswap32.9 \
 	byteorder.9 bswap64.9 \
 	byteorder.9 htobe16.9 \
 	byteorder.9 htobe32.9 \
 	byteorder.9 htobe64.9 \
 	byteorder.9 htole16.9 \
 	byteorder.9 htole32.9 \
 	byteorder.9 htole64.9 \
 	byteorder.9 le16dec.9 \
 	byteorder.9 le16enc.9 \
 	byteorder.9 le16toh.9 \
 	byteorder.9 le32dec.9 \
 	byteorder.9 le32enc.9 \
 	byteorder.9 le32toh.9 \
 	byteorder.9 le64dec.9 \
 	byteorder.9 le64enc.9 \
 	byteorder.9 le64toh.9
 MLINKS+=callout.9 callout_active.9 \
 	callout.9 callout_async_drain.9 \
 	callout.9 callout_deactivate.9 \
 	callout.9 callout_drain.9 \
 	callout.9 callout_init.9 \
 	callout.9 callout_init_mtx.9 \
 	callout.9 callout_init_rm.9 \
 	callout.9 callout_init_rw.9 \
 	callout.9 callout_pending.9 \
 	callout.9 callout_reset.9 \
 	callout.9 callout_reset_curcpu.9 \
 	callout.9 callout_reset_on.9 \
 	callout.9 callout_reset_sbt.9 \
 	callout.9 callout_reset_sbt_curcpu.9 \
 	callout.9 callout_reset_sbt_on.9 \
 	callout.9 callout_schedule.9 \
 	callout.9 callout_schedule_curcpu.9 \
 	callout.9 callout_schedule_on.9 \
 	callout.9 callout_schedule_sbt.9 \
 	callout.9 callout_schedule_sbt_curcpu.9 \
 	callout.9 callout_schedule_sbt_on.9 \
 	callout.9 callout_stop.9 \
 	callout.9 callout_when.9
 MLINKS+=cnv.9 cnvlist.9 \
 	cnv.9 cnvlist_free_binary.9 \
 	cnv.9 cnvlist_free_bool.9 \
 	cnv.9 cnvlist_free_bool_array.9 \
 	cnv.9 cnvlist_free_descriptor.9 \
 	cnv.9 cnvlist_free_descriptor_array.9 \
 	cnv.9 cnvlist_free_null.9 \
 	cnv.9 cnvlist_free_number.9 \
 	cnv.9 cnvlist_free_number_array.9 \
 	cnv.9 cnvlist_free_nvlist.9 \
 	cnv.9 cnvlist_free_nvlist_array.9 \
 	cnv.9 cnvlist_free_string.9 \
 	cnv.9 cnvlist_free_string_array.9 \
 	cnv.9 cnvlist_get_binary.9 \
 	cnv.9 cnvlist_get_bool.9 \
 	cnv.9 cnvlist_get_bool_array.9 \
 	cnv.9 cnvlist_get_descriptor.9 \
 	cnv.9 cnvlist_get_descriptor_array.9 \
 	cnv.9 cnvlist_get_number.9 \
 	cnv.9 cnvlist_get_number_array.9 \
 	cnv.9 cnvlist_get_nvlist.9 \
 	cnv.9 cnvlist_get_nvlist_array.9 \
 	cnv.9 cnvlist_get_string.9 \
 	cnv.9 cnvlist_get_string_array.9 \
 	cnv.9 cnvlist_take_binary.9 \
 	cnv.9 cnvlist_take_bool.9 \
 	cnv.9 cnvlist_take_bool_array.9 \
 	cnv.9 cnvlist_take_descriptor.9 \
 	cnv.9 cnvlist_take_descriptor_array.9 \
 	cnv.9 cnvlist_take_number.9 \
 	cnv.9 cnvlist_take_number_array.9 \
 	cnv.9 cnvlist_take_nvlist.9 \
 	cnv.9 cnvlist_take_nvlist_array.9 \
 	cnv.9 cnvlist_take_string.9 \
 	cnv.9 cnvlist_take_string_array.9
 MLINKS+=condvar.9 cv_broadcast.9 \
 	condvar.9 cv_broadcastpri.9 \
 	condvar.9 cv_destroy.9 \
 	condvar.9 cv_init.9 \
 	condvar.9 cv_signal.9 \
 	condvar.9 cv_timedwait.9 \
 	condvar.9 cv_timedwait_sig.9 \
 	condvar.9 cv_timedwait_sig_sbt.9 \
 	condvar.9 cv_wait.9 \
 	condvar.9 cv_wait_sig.9 \
 	condvar.9 cv_wait_unlock.9 \
 	condvar.9 cv_wmesg.9
 MLINKS+=config_intrhook.9 config_intrhook_disestablish.9 \
 	config_intrhook.9 config_intrhook_establish.9 \
 	config_intrhook.9 config_intrhook_oneshot.9
 MLINKS+=contigmalloc.9 contigmalloc_domainset.9 \
 	contigmalloc.9 contigfree.9
 MLINKS+=casuword.9 casueword.9 \
 	casuword.9 casueword32.9 \
 	casuword.9 casuword32.9
 MLINKS+=copy.9 copyin.9 \
 	copy.9 copyin_nofault.9 \
 	copy.9 copyinstr.9 \
 	copy.9 copyout.9 \
 	copy.9 copyout_nofault.9 \
 	copy.9 copystr.9
 MLINKS+=counter.9 counter_u64_alloc.9 \
 	counter.9 counter_u64_free.9 \
 	counter.9 counter_u64_add.9 \
 	counter.9 counter_enter.9 \
 	counter.9 counter_exit.9 \
 	counter.9 counter_u64_add_protected.9 \
 	counter.9 counter_u64_fetch.9 \
 	counter.9 counter_u64_zero.9 \
 	counter.9 SYSCTL_COUNTER_U64.9 \
 	counter.9 SYSCTL_ADD_COUNTER_U64.9 \
 	counter.9 SYSCTL_COUNTER_U64_ARRAY.9 \
 	counter.9 SYSCTL_ADD_COUNTER_U64_ARRAY.9
 MLINKS+=cpuset.9 CPUSET_T_INITIALIZER.9 \
 	cpuset.9 CPUSET_FSET.9 \
 	cpuset.9 CPU_CLR.9 \
 	cpuset.9 CPU_COPY.9 \
 	cpuset.9 CPU_ISSET.9 \
 	cpuset.9 CPU_SET.9 \
 	cpuset.9 CPU_ZERO.9 \
 	cpuset.9 CPU_FILL.9 \
 	cpuset.9 CPU_SETOF.9 \
 	cpuset.9 CPU_EMPTY.9 \
 	cpuset.9 CPU_ISFULLSET.9 \
 	cpuset.9 CPU_FFS.9 \
 	cpuset.9 CPU_COUNT.9 \
 	cpuset.9 CPU_SUBSET.9 \
 	cpuset.9 CPU_OVERLAP.9 \
 	cpuset.9 CPU_CMP.9 \
 	cpuset.9 CPU_OR.9 \
 	cpuset.9 CPU_AND.9 \
 	cpuset.9 CPU_ANDNOT.9 \
 	cpuset.9 CPU_CLR_ATOMIC.9 \
 	cpuset.9 CPU_SET_ATOMIC.9 \
 	cpuset.9 CPU_SET_ATOMIC_ACQ.9 \
 	cpuset.9 CPU_AND_ATOMIC.9 \
 	cpuset.9 CPU_OR_ATOMIC.9 \
 	cpuset.9 CPU_COPY_STORE_REL.9
 MLINKS+=critical_enter.9 critical.9 \
 	critical_enter.9 critical_exit.9
-MLINKS+=crypto.9 crypto_dispatch.9 \
-	crypto.9 crypto_done.9 \
-	crypto.9 crypto_freereq.9 \
-	crypto.9 crypto_freesession.9 \
-	crypto.9 crypto_get_driverid.9 \
-	crypto.9 crypto_getreq.9 \
-	crypto.9 crypto_kdispatch.9 \
-	crypto.9 crypto_kdone.9 \
-	crypto.9 crypto_kregister.9 \
-	crypto.9 crypto_newsession.9 \
-	crypto.9 crypto_register.9 \
-	crypto.9 crypto_unblock.9 \
-	crypto.9 crypto_unregister.9 \
-	crypto.9 crypto_unregister_all.9
+MLINKS+=crypto_asym.9 crypto_kdispatch.9 \
+	crypto_asym.9 crypto_kdone.9 \
+	crypto_asym.9 crypto_kregister.9 \
+	crypto_asym.9 CRYPTODEV_KPROCESS.9
+MLINKS+=crypto_driver.9 crypto_apply.9 \
+	crypto_driver.9 crypto_contiguous_segment.9 \
+	crypto_driver.9 crypto_copyback.9 \
+	crypto_driver.9 crypto_copydata.9 \
+	crypto_driver.9 crypto_done.9 \
+	crypto_driver.9 crypto_get_driverid.9 \
+	crypto_driver.9 crypto_get_driver_session.9 \
+	crypto_driver.9 crypto_unblock.9 \
+	crypto_driver.9 crypto_unregister_all.9 \
+	crypto_driver.9 CRYPTODEV_FREESESSION.9 \
+	crypto_driver.9 CRYPTODEV_NEWSESSION.9 \
+	crypto_driver.9 CRYPTODEV_PROBESESSION.9 \
+	crypto_driver.9 CRYPTODEV_PROCESS.9 \
+	crypto_driver.9 hmac_init_ipad.9 \
+	crypto_driver.9 hmac_init_opad.9
+MLINKS+=crypto_request.9 crypto_dispatch.9 \
+	crypto_request.9 crypto_freereq.9 \
+	crypto_request.9 crypto_getreq.9
+MLINKS+=crypto_session.9 crypto_auth_hash.9 \
+	crypto_session.9 crypto_cipher.9 \
+	crypto_session.9 crypto_get_params.9 \
+	crypto_session.9 crypto_newsession.9 \
+	crypto_session.9 crypto_freesession.9
 MLINKS+=DB_COMMAND.9 DB_SHOW_ALL_COMMAND.9 \
 	DB_COMMAND.9 DB_SHOW_COMMAND.9
 MLINKS+=DECLARE_MODULE.9 DECLARE_MODULE_TIED.9
 MLINKS+=dev_clone.9 drain_dev_clone_events.9
 MLINKS+=dev_refthread.9 devvn_refthread.9 \
 	dev_refthread.9 dev_relthread.9
 MLINKS+=devfs_set_cdevpriv.9 devfs_clear_cdevpriv.9 \
 	devfs_set_cdevpriv.9 devfs_get_cdevpriv.9
 MLINKS+=device_add_child.9 device_add_child_ordered.9
 MLINKS+=device_enable.9 device_disable.9 \
 	device_enable.9 device_is_enabled.9
 MLINKS+=device_get_ivars.9 device_set_ivars.9
 MLINKS+=device_get_name.9 device_get_nameunit.9
 MLINKS+=device_get_state.9 device_busy.9 \
 	device_get_state.9 device_is_alive.9 \
 	device_get_state.9 device_is_attached.9 \
 	device_get_state.9 device_unbusy.9
 MLINKS+=device_get_sysctl.9 device_get_sysctl_ctx.9 \
 	device_get_sysctl.9 device_get_sysctl_tree.9
 MLINKS+=device_quiet.9 device_is_quiet.9 \
 	device_quiet.9 device_verbose.9
 MLINKS+=device_set_desc.9 device_get_desc.9 \
 	device_set_desc.9 device_set_desc_copy.9
 MLINKS+=device_set_flags.9 device_get_flags.9
 MLINKS+=devstat.9 devicestat.9 \
 	devstat.9 devstat_add_entry.9 \
 	devstat.9 devstat_end_transaction.9 \
 	devstat.9 devstat_remove_entry.9 \
 	devstat.9 devstat_start_transaction.9
 MLINKS+=disk.9 disk_add_alias.9 \
 	disk.9 disk_alloc.9 \
 	disk.9 disk_create.9 \
 	disk.9 disk_destroy.9 \
 	disk.9 disk_gone.9 \
 	disk.9 disk_resize.9
 MLINKS+=dnv.9 dnvlist.9 \
 	dnv.9 dnvlist_get_binary.9 \
 	dnv.9 dnvlist_get_bool.9 \
 	dnv.9 dnvlist_get_descriptor.9 \
 	dnv.9 dnvlist_get_number.9 \
 	dnv.9 dnvlist_get_nvlist.9 \
 	dnv.9 dnvlist_get_string.9 \
 	dnv.9 dnvlist_take_binary.9 \
 	dnv.9 dnvlist_take_bool.9 \
 	dnv.9 dnvlist_take_descriptor.9 \
 	dnv.9 dnvlist_take_number.9 \
 	dnv.9 dnvlist_take_nvlist.9 \
 	dnv.9 dnvlist_take_string.9
 MLINKS+=domain.9 DOMAIN_SET.9 \
 	domain.9 domain_add.9 \
 	domain.9 pfctlinput.9 \
 	domain.9 pfctlinput2.9 \
 	domain.9 pffinddomain.9 \
 	domain.9 pffindproto.9 \
 	domain.9 pffindtype.9
 MLINKS+=drbr.9 drbr_free.9 \
 	drbr.9 drbr_enqueue.9 \
 	drbr.9 drbr_dequeue.9 \
 	drbr.9 drbr_dequeue_cond.9 \
 	drbr.9 drbr_flush.9 \
 	drbr.9 drbr_empty.9 \
 	drbr.9 drbr_inuse.9 \
 	drbr.9 drbr_stats_update.9
 MLINKS+=DRIVER_MODULE.9 DRIVER_MODULE_ORDERED.9 \
 	DRIVER_MODULE.9 EARLY_DRIVER_MODULE.9 \
 	DRIVER_MODULE.9 EARLY_DRIVER_MODULE_ORDERED.9
 MLINKS+=epoch.9 epoch_context.9 \
 	epoch.9 epoch_alloc.9 \
 	epoch.9 epoch_free.9 \
 	epoch.9 epoch_enter.9 \
 	epoch.9 epoch_exit.9 \
 	epoch.9 epoch_wait.9 \
 	epoch.9 epoch_call.9 \
 	epoch.9 epoch_drain_callbacks.9 \
 	epoch.9 in_epoch.9
 MLINKS+=EVENTHANDLER.9 EVENTHANDLER_DECLARE.9 \
 	EVENTHANDLER.9 EVENTHANDLER_DEFINE.9 \
 	EVENTHANDLER.9 EVENTHANDLER_DEREGISTER.9 \
 	EVENTHANDLER.9 eventhandler_deregister.9 \
 	EVENTHANDLER.9 eventhandler_find_list.9 \
 	EVENTHANDLER.9 EVENTHANDLER_INVOKE.9 \
 	EVENTHANDLER.9 eventhandler_prune_list.9 \
 	EVENTHANDLER.9 EVENTHANDLER_REGISTER.9 \
 	EVENTHANDLER.9 eventhandler_register.9
 MLINKS+=eventtimers.9 et_register.9 \
 	eventtimers.9 et_deregister.9 \
 	eventtimers.9 et_ban.9 \
 	eventtimers.9 et_find.9 \
 	eventtimers.9 et_free.9 \
 	eventtimers.9 et_init.9 \
 	eventtimers.9 ET_LOCK.9 \
 	eventtimers.9 ET_UNLOCK.9 \
 	eventtimers.9 et_start.9 \
 	eventtimers.9 et_stop.9
 MLINKS+=fail.9 KFAIL_POINT_CODE.9 \
 	fail.9 KFAIL_POINT_ERROR.9 \
 	fail.9 KFAIL_POINT_GOTO.9 \
 	fail.9 KFAIL_POINT_RETURN.9 \
 	fail.9 KFAIL_POINT_RETURN_VOID.9
 MLINKS+=fdt_pinctrl.9 fdt_pinctrl_configure.9 \
 	fdt_pinctrl.9 fdt_pinctrl_configure_by_name.9 \
 	fdt_pinctrl.9 fdt_pinctrl_configure_tree.9 \
 	fdt_pinctrl.9 fdt_pinctrl_register.9
 MLINKS+=fetch.9 fubyte.9 \
 	fetch.9 fuword.9 \
 	fetch.9 fuword16.9 \
 	fetch.9 fuword32.9 \
 	fetch.9 fuword64.9 \
 	fetch.9 fueword.9 \
 	fetch.9 fueword32.9 \
 	fetch.9 fueword64.9
 MLINKS+=firmware.9 firmware_get.9 \
 	firmware.9 firmware_put.9 \
 	firmware.9 firmware_register.9 \
 	firmware.9 firmware_unregister.9
 MLINKS+=fpu_kern.9 fpu_kern_alloc_ctx.9 \
 	fpu_kern.9 fpu_kern_free_ctx.9 \
 	fpu_kern.9 fpu_kern_enter.9 \
 	fpu_kern.9 fpu_kern_leave.9 \
 	fpu_kern.9 fpu_kern_thread.9 \
 	fpu_kern.9 is_fpu_kern_thread.9
 MLINKS+=g_attach.9 g_detach.9
 MLINKS+=g_bio.9 g_alloc_bio.9 \
 	g_bio.9 g_clone_bio.9 \
 	g_bio.9 g_destroy_bio.9 \
 	g_bio.9 g_duplicate_bio.9 \
 	g_bio.9 g_format_bio.9 \
 	g_bio.9 g_new_bio.9 \
 	g_bio.9 g_print_bio.9 \
 	g_bio.9 g_reset_bio.9
 MLINKS+=g_consumer.9 g_destroy_consumer.9 \
 	g_consumer.9 g_new_consumer.9
 MLINKS+=g_data.9 g_read_data.9 \
 	g_data.9 g_write_data.9
 MLINKS+=getenv.9 freeenv.9 \
 	getenv.9 getenv_int.9 \
 	getenv.9 getenv_long.9 \
 	getenv.9 getenv_string.9 \
 	getenv.9 getenv_quad.9 \
 	getenv.9 getenv_uint.9 \
 	getenv.9 getenv_ulong.9 \
 	getenv.9 kern_getenv.9 \
 	getenv.9 kern_setenv.9 \
 	getenv.9 kern_unsetenv.9 \
 	getenv.9 setenv.9 \
 	getenv.9 testenv.9 \
 	getenv.9 unsetenv.9
 MLINKS+=g_event.9 g_cancel_event.9 \
 	g_event.9 g_post_event.9 \
 	g_event.9 g_waitfor_event.9
 MLINKS+=g_geom.9 g_destroy_geom.9 \
 	g_geom.9 g_new_geomf.9
 MLINKS+=g_provider.9 g_destroy_provider.9 \
 	g_provider.9 g_error_provider.9 \
 	g_provider.9 g_new_providerf.9
 MLINKS+=hash.9 hash32.9 \
 	hash.9 hash32_buf.9 \
 	hash.9 hash32_str.9 \
 	hash.9 hash32_stre.9 \
 	hash.9 hash32_strn.9 \
 	hash.9 hash32_strne.9 \
 	hash.9 jenkins_hash.9 \
 	hash.9 jenkins_hash32.9
 MLINKS+=hashinit.9 hashdestroy.9 \
 	hashinit.9 hashinit_flags.9 \
 	hashinit.9 phashinit.9
 MLINKS+=hhook.9 hhook_head_register.9 \
 	hhook.9 hhook_head_deregister.9 \
 	hhook.9 hhook_head_deregister_lookup.9 \
 	hhook.9 hhook_run_hooks.9 \
 	hhook.9 HHOOKS_RUN_IF.9 \
 	hhook.9 HHOOKS_RUN_LOOKUP_IF.9
 MLINKS+=ieee80211.9 ieee80211_ifattach.9 \
 	ieee80211.9 ieee80211_ifdetach.9
 MLINKS+=ieee80211_amrr.9 ieee80211_amrr_choose.9 \
 	ieee80211_amrr.9 ieee80211_amrr_cleanup.9 \
 	ieee80211_amrr.9 ieee80211_amrr_init.9 \
 	ieee80211_amrr.9 ieee80211_amrr_node_init.9 \
 	ieee80211_amrr.9 ieee80211_amrr_setinterval.9 \
 	ieee80211_amrr.9 ieee80211_amrr_tx_complete.9 \
 	ieee80211_amrr.9 ieee80211_amrr_tx_update.9
 MLINKS+=ieee80211_beacon.9 ieee80211_beacon_alloc.9 \
 	ieee80211_beacon.9 ieee80211_beacon_notify.9 \
 	ieee80211_beacon.9 ieee80211_beacon_update.9
 MLINKS+=ieee80211_bmiss.9 ieee80211_beacon_miss.9
 MLINKS+=ieee80211_crypto.9 ieee80211_crypto_available.9 \
 	ieee80211_crypto.9 ieee80211_crypto_decap.9 \
 	ieee80211_crypto.9 ieee80211_crypto_delglobalkeys.9 \
 	ieee80211_crypto.9 ieee80211_crypto_delkey.9 \
 	ieee80211_crypto.9 ieee80211_crypto_demic.9 \
 	ieee80211_crypto.9 ieee80211_crypto_encap.9 \
 	ieee80211_crypto.9 ieee80211_crypto_enmic.9 \
 	ieee80211_crypto.9 ieee80211_crypto_newkey.9 \
 	ieee80211_crypto.9 ieee80211_crypto_register.9 \
 	ieee80211_crypto.9 ieee80211_crypto_reload_keys.9 \
 	ieee80211_crypto.9 ieee80211_crypto_setkey.9 \
 	ieee80211_crypto.9 ieee80211_crypto_unregister.9 \
 	ieee80211_crypto.9 ieee80211_key_update_begin.9 \
 	ieee80211_crypto.9 ieee80211_key_update_end.9 \
 	ieee80211_crypto.9 ieee80211_notify_michael_failure.9 \
 	ieee80211_crypto.9 ieee80211_notify_replay_failure.9
 MLINKS+=ieee80211_input.9 ieee80211_input_all.9
 MLINKS+=ieee80211_node.9 ieee80211_dump_node.9 \
 	ieee80211_node.9 ieee80211_dump_nodes.9 \
 	ieee80211_node.9 ieee80211_find_rxnode.9 \
 	ieee80211_node.9 ieee80211_find_rxnode_withkey.9 \
 	ieee80211_node.9 ieee80211_free_node.9 \
 	ieee80211_node.9 ieee80211_iterate_nodes.9 \
 	ieee80211_node.9 ieee80211_ref_node.9 \
 	ieee80211_node.9 ieee80211_unref_node.9
 MLINKS+=ieee80211_output.9 ieee80211_process_callback.9 \
 	ieee80211_output.9 M_SEQNO_GET.9 \
 	ieee80211_output.9 M_WME_GETAC.9
 MLINKS+=ieee80211_proto.9 ieee80211_new_state.9 \
 	ieee80211_proto.9 ieee80211_resume_all.9 \
 	ieee80211_proto.9 ieee80211_start_all.9 \
 	ieee80211_proto.9 ieee80211_stop_all.9 \
 	ieee80211_proto.9 ieee80211_suspend_all.9 \
 	ieee80211_proto.9 ieee80211_waitfor_parent.9
 MLINKS+=ieee80211_radiotap.9 ieee80211_radiotap_active.9 \
 	ieee80211_radiotap.9 ieee80211_radiotap_active_vap.9 \
 	ieee80211_radiotap.9 ieee80211_radiotap_attach.9 \
 	ieee80211_radiotap.9 ieee80211_radiotap_tx.9 \
 	ieee80211_radiotap.9 radiotap.9
 MLINKS+=ieee80211_regdomain.9 ieee80211_alloc_countryie.9 \
 	ieee80211_regdomain.9 ieee80211_init_channels.9 \
 	ieee80211_regdomain.9 ieee80211_sort_channels.9
 MLINKS+=ieee80211_scan.9 ieee80211_add_scan.9 \
 	ieee80211_scan.9 ieee80211_bg_scan.9 \
 	ieee80211_scan.9 ieee80211_cancel_scan.9 \
 	ieee80211_scan.9 ieee80211_cancel_scan_any.9 \
 	ieee80211_scan.9 ieee80211_check_scan.9 \
 	ieee80211_scan.9 ieee80211_check_scan_current.9 \
 	ieee80211_scan.9 ieee80211_flush.9 \
 	ieee80211_scan.9 ieee80211_probe_curchan.9 \
 	ieee80211_scan.9 ieee80211_scan_assoc_fail.9 \
 	ieee80211_scan.9 ieee80211_scan_done.9 \
 	ieee80211_scan.9 ieee80211_scan_dump_channels.9 \
 	ieee80211_scan.9 ieee80211_scan_flush.9 \
 	ieee80211_scan.9 ieee80211_scan_iterate.9 \
 	ieee80211_scan.9 ieee80211_scan_next.9 \
 	ieee80211_scan.9 ieee80211_scan_timeout.9 \
 	ieee80211_scan.9 ieee80211_scanner_get.9 \
 	ieee80211_scan.9 ieee80211_scanner_register.9 \
 	ieee80211_scan.9 ieee80211_scanner_unregister.9 \
 	ieee80211_scan.9 ieee80211_scanner_unregister_all.9 \
 	ieee80211_scan.9 ieee80211_start_scan.9
 MLINKS+=ieee80211_vap.9 ieee80211_vap_attach.9 \
 	ieee80211_vap.9 ieee80211_vap_detach.9 \
 	ieee80211_vap.9 ieee80211_vap_setup.9
 MLINKS+=iflibdd.9 ifdi_attach_pre.9 \
 	iflibdd.9 ifdi_attach_post.9 \
 	iflibdd.9 ifdi_detach.9 \
 	iflibdd.9 ifdi_get_counter.9 \
 	iflibdd.9 ifdi_i2c_req.9 \
 	iflibdd.9 ifdi_init.9 \
 	iflibdd.9 ifdi_intr_enable.9 \
 	iflibdd.9 ifdi_intr_disable.9 \
 	iflibdd.9 ifdi_led_func.9 \
 	iflibdd.9 ifdi_link_intr_enable.9 \
 	iflibdd.9 ifdi_media_set.9 \
 	iflibdd.9 ifdi_media_status.9 \
 	iflibdd.9 ifdi_media_change.9 \
 	iflibdd.9 ifdi_mtu_set.9 \
 	iflibdd.9 ifdi_multi_set.9 \
 	iflibdd.9 ifdi_promisc_set.9 \
 	iflibdd.9 ifdi_queues_alloc.9 \
 	iflibdd.9 ifdi_queues_free.9 \
 	iflibdd.9 ifdi_queue_intr_enable.9 \
 	iflibdd.9 ifdi_resume.9 \
 	iflibdd.9 ifdi_rxq_setup.9 \
 	iflibdd.9 ifdi_stop.9 \
 	iflibdd.9 ifdi_suspend.9 \
 	iflibdd.9 ifdi_sysctl_int_delay.9 \
 	iflibdd.9 ifdi_timer.9 \
 	iflibdd.9 ifdi_txq_setup.9 \
 	iflibdd.9 ifdi_update_admin_status.9 \
 	iflibdd.9 ifdi_vf_add.9 \
 	iflibdd.9 ifdi_vflr_handle.9 \
 	iflibdd.9 ifdi_vlan_register.9 \
 	iflibdd.9 ifdi_vlan_unregister.9 \
 	iflibdd.9 ifdi_watchdog_reset.9 \
 	iflibdd.9 iov_init.9 \
 	iflibdd.9 iov_uinit.9
 MLINKS+=iflibdi.9 iflib_add_int_delay_sysctl.9 \
 	iflibdi.9 iflib_device_attach.9 \
 	iflibdi.9 iflib_device_deregister.9 \
 	iflibdi.9 iflib_device_detach.9 \
 	iflibdi.9 iflib_device_suspend.9 \
 	iflibdi.9 iflib_device_register.9 \
 	iflibdi.9 iflib_device_resume.9 \
 	iflibdi.9 iflib_led_create.9 \
 	iflibdi.9 iflib_irq_alloc.9 \
 	iflibdi.9 iflib_irq_alloc_generic.9 \
 	iflibdi.9 iflib_link_intr_deferred.9 \
 	iflibdi.9 iflib_link_state_change.9 \
 	iflibdi.9 iflib_rx_intr_deferred.9 \
 	iflibdi.9 iflib_tx_intr_deferred.9
 MLINKS+=iflibtxrx.9 isc_rxd_available.9 \
 	iflibtxrx.9 isc_rxd_refill.9 \
 	iflibtxrx.9 isc_rxd_flush.9 \
 	iflibtxrx.9 isc_rxd_pkt_get.9 \
 	iflibtxrx.9 isc_txd_credits_update.9 \
 	iflibtxrx.9 isc_txd_encap.9 \
 	iflibtxrx.9 isc_txd_flush.9
 MLINKS+=ifnet.9 if_addmulti.9 \
 	ifnet.9 if_alloc.9 \
 	ifnet.9 if_alloc_dev.9 \
 	ifnet.9 if_alloc_domain.9 \
 	ifnet.9 if_allmulti.9 \
 	ifnet.9 if_attach.9 \
 	ifnet.9 if_data.9 \
 	ifnet.9 IF_DEQUEUE.9 \
 	ifnet.9 if_delmulti.9 \
 	ifnet.9 if_detach.9 \
 	ifnet.9 if_down.9 \
 	ifnet.9 if_findmulti.9 \
 	ifnet.9 if_free.9 \
 	ifnet.9 if_free_type.9 \
 	ifnet.9 if_up.9 \
 	ifnet.9 ifa_free.9 \
 	ifnet.9 ifa_ifwithaddr.9 \
 	ifnet.9 ifa_ifwithdstaddr.9 \
 	ifnet.9 ifa_ifwithnet.9 \
 	ifnet.9 ifa_ref.9 \
 	ifnet.9 ifaddr.9 \
 	ifnet.9 ifaddr_byindex.9 \
 	ifnet.9 ifaof_ifpforaddr.9 \
 	ifnet.9 ifioctl.9 \
 	ifnet.9 ifpromisc.9 \
 	ifnet.9 ifqueue.9 \
 	ifnet.9 ifunit.9 \
 	ifnet.9 ifunit_ref.9
 MLINKS+=insmntque.9 insmntque1.9
 MLINKS+=ithread.9 ithread_add_handler.9 \
 	ithread.9 ithread_create.9 \
 	ithread.9 ithread_destroy.9 \
 	ithread.9 ithread_priority.9 \
 	ithread.9 ithread_remove_handler.9 \
 	ithread.9 ithread_schedule.9
 MLINKS+=kernacc.9 useracc.9
 MLINKS+=kernel_mount.9 free_mntarg.9 \
 	kernel_mount.9 kernel_vmount.9 \
 	kernel_mount.9 mount_arg.9 \
 	kernel_mount.9 mount_argb.9 \
 	kernel_mount.9 mount_argf.9 \
 	kernel_mount.9 mount_argsu.9
 MLINKS+=khelp.9 khelp_add_hhook.9 \
 	khelp.9 KHELP_DECLARE_MOD.9 \
 	khelp.9 KHELP_DECLARE_MOD_UMA.9 \
 	khelp.9 khelp_destroy_osd.9 \
 	khelp.9 khelp_get_id.9 \
 	khelp.9 khelp_get_osd.9 \
 	khelp.9 khelp_init_osd.9 \
 	khelp.9 khelp_remove_hhook.9
 MLINKS+=kobj.9 DEFINE_CLASS.9 \
 	kobj.9 kobj_class_compile.9 \
 	kobj.9 kobj_class_compile_static.9 \
 	kobj.9 kobj_class_free.9 \
 	kobj.9 kobj_create.9 \
 	kobj.9 kobj_delete.9 \
 	kobj.9 kobj_init.9 \
 	kobj.9 kobj_init_static.9
 MLINKS+=kproc.9 kproc_create.9 \
 	kproc.9 kproc_exit.9 \
 	kproc.9 kproc_kthread_add.9 \
 	kproc.9 kproc_resume.9 \
 	kproc.9 kproc_shutdown.9 \
 	kproc.9 kproc_start.9 \
 	kproc.9 kproc_suspend.9 \
 	kproc.9 kproc_suspend_check.9 \
 	kproc.9 kthread_create.9
 MLINKS+=kqueue.9 knlist_add.9 \
 	kqueue.9 knlist_clear.9 \
 	kqueue.9 knlist_delete.9 \
 	kqueue.9 knlist_destroy.9 \
 	kqueue.9 knlist_empty.9 \
 	kqueue.9 knlist_init.9 \
 	kqueue.9 knlist_init_mtx.9 \
 	kqueue.9 knlist_init_rw_reader.9 \
 	kqueue.9 knlist_remove.9 \
 	kqueue.9 knlist_remove_inevent.9 \
 	kqueue.9 knote_fdclose.9 \
 	kqueue.9 KNOTE_LOCKED.9 \
 	kqueue.9 KNOTE_UNLOCKED.9 \
 	kqueue.9 kqfd_register.9 \
 	kqueue.9 kqueue_add_filteropts.9 \
 	kqueue.9 kqueue_del_filteropts.9
 MLINKS+=kthread.9 kthread_add.9 \
 	kthread.9 kthread_exit.9 \
 	kthread.9 kthread_resume.9 \
 	kthread.9 kthread_shutdown.9 \
 	kthread.9 kthread_start.9 \
 	kthread.9 kthread_suspend.9 \
 	kthread.9 kthread_suspend_check.9
 MLINKS+=ktr.9 CTR0.9 \
 	ktr.9 CTR1.9 \
 	ktr.9 CTR2.9 \
 	ktr.9 CTR3.9 \
 	ktr.9 CTR4.9 \
 	ktr.9 CTR5.9 \
 	ktr.9 CTR6.9
 MLINKS+=lock.9 lockdestroy.9 \
 	lock.9 lockinit.9 \
 	lock.9 lockmgr.9 \
 	lock.9 lockmgr_args.9 \
 	lock.9 lockmgr_args_rw.9 \
 	lock.9 lockmgr_assert.9 \
 	lock.9 lockmgr_disown.9 \
 	lock.9 lockmgr_printinfo.9 \
 	lock.9 lockmgr_recursed.9 \
 	lock.9 lockmgr_rw.9 \
 	lock.9 lockstatus.9
 MLINKS+=LOCK_PROFILING.9 MUTEX_PROFILING.9
 MLINKS+=make_dev.9 destroy_dev.9 \
 	make_dev.9 destroy_dev_drain.9 \
 	make_dev.9 destroy_dev_sched.9 \
 	make_dev.9 destroy_dev_sched_cb.9 \
 	make_dev.9 dev_depends.9 \
 	make_dev.9 make_dev_alias.9 \
 	make_dev.9 make_dev_alias_p.9 \
 	make_dev.9 make_dev_cred.9 \
 	make_dev.9 make_dev_credf.9 \
 	make_dev.9 make_dev_p.9 \
 	make_dev.9 make_dev_s.9
 MLINKS+=malloc.9 free.9 \
 	malloc.9 malloc_domainset.9 \
 	malloc.9 free_domain.9 \
 	malloc.9 mallocarray.9 \
 	malloc.9 MALLOC_DECLARE.9 \
 	malloc.9 MALLOC_DEFINE.9 \
 	malloc.9 realloc.9 \
 	malloc.9 reallocf.9
 MLINKS+=mbchain.9 mb_detach.9 \
 	mbchain.9 mb_done.9 \
 	mbchain.9 mb_fixhdr.9 \
 	mbchain.9 mb_init.9 \
 	mbchain.9 mb_initm.9 \
 	mbchain.9 mb_put_int64be.9 \
 	mbchain.9 mb_put_int64le.9 \
 	mbchain.9 mb_put_mbuf.9 \
 	mbchain.9 mb_put_mem.9 \
 	mbchain.9 mb_put_uint16be.9 \
 	mbchain.9 mb_put_uint16le.9 \
 	mbchain.9 mb_put_uint32be.9 \
 	mbchain.9 mb_put_uint32le.9 \
 	mbchain.9 mb_put_uint8.9 \
 	mbchain.9 mb_put_uio.9 \
 	mbchain.9 mb_reserve.9
 MLINKS+=\
 	mbuf.9 m_adj.9 \
 	mbuf.9 m_align.9 \
 	mbuf.9 M_ALIGN.9 \
 	mbuf.9 m_append.9 \
 	mbuf.9 m_apply.9 \
 	mbuf.9 m_cat.9 \
 	mbuf.9 m_catpkt.9 \
 	mbuf.9 MCHTYPE.9 \
 	mbuf.9 MCLGET.9 \
 	mbuf.9 m_collapse.9 \
 	mbuf.9 m_copyback.9 \
 	mbuf.9 m_copydata.9 \
 	mbuf.9 m_copym.9 \
 	mbuf.9 m_copypacket.9 \
 	mbuf.9 m_copyup.9 \
 	mbuf.9 m_defrag.9 \
 	mbuf.9 m_devget.9 \
 	mbuf.9 m_dup.9 \
 	mbuf.9 m_dup_pkthdr.9 \
 	mbuf.9 MEXTADD.9 \
 	mbuf.9 m_fixhdr.9 \
 	mbuf.9 m_free.9 \
 	mbuf.9 m_freem.9 \
 	mbuf.9 MGET.9 \
 	mbuf.9 m_get.9 \
 	mbuf.9 m_get2.9 \
 	mbuf.9 m_getjcl.9 \
 	mbuf.9 m_getcl.9 \
 	mbuf.9 MGETHDR.9 \
 	mbuf.9 m_gethdr.9 \
 	mbuf.9 m_getm.9 \
 	mbuf.9 m_getptr.9 \
 	mbuf.9 MH_ALIGN.9 \
 	mbuf.9 M_LEADINGSPACE.9 \
 	mbuf.9 m_length.9 \
 	mbuf.9 M_MOVE_PKTHDR.9 \
 	mbuf.9 m_move_pkthdr.9 \
 	mbuf.9 M_PREPEND.9 \
 	mbuf.9 m_prepend.9 \
 	mbuf.9 m_pulldown.9 \
 	mbuf.9 m_pullup.9 \
 	mbuf.9 m_split.9 \
 	mbuf.9 mtod.9 \
 	mbuf.9 M_TRAILINGSPACE.9 \
 	mbuf.9 m_unshare.9 \
 	mbuf.9 M_WRITABLE.9
 MLINKS+=\
 	mbuf_tags.9 m_tag_alloc.9 \
 	mbuf_tags.9 m_tag_copy.9 \
 	mbuf_tags.9 m_tag_copy_chain.9 \
 	mbuf_tags.9 m_tag_delete.9 \
 	mbuf_tags.9 m_tag_delete_chain.9 \
 	mbuf_tags.9 m_tag_delete_nonpersistent.9 \
 	mbuf_tags.9 m_tag_find.9 \
 	mbuf_tags.9 m_tag_first.9 \
 	mbuf_tags.9 m_tag_free.9 \
 	mbuf_tags.9 m_tag_get.9 \
 	mbuf_tags.9 m_tag_init.9 \
 	mbuf_tags.9 m_tag_locate.9 \
 	mbuf_tags.9 m_tag_next.9 \
 	mbuf_tags.9 m_tag_prepend.9 \
 	mbuf_tags.9 m_tag_unlink.9
 MLINKS+=MD5.9 MD5Init.9 \
 	MD5.9 MD5Transform.9
 MLINKS+=mdchain.9 md_append_record.9 \
 	mdchain.9 md_done.9 \
 	mdchain.9 md_get_int64.9 \
 	mdchain.9 md_get_int64be.9 \
 	mdchain.9 md_get_int64le.9 \
 	mdchain.9 md_get_mbuf.9 \
 	mdchain.9 md_get_mem.9 \
 	mdchain.9 md_get_uint16.9 \
 	mdchain.9 md_get_uint16be.9 \
 	mdchain.9 md_get_uint16le.9 \
 	mdchain.9 md_get_uint32.9 \
 	mdchain.9 md_get_uint32be.9 \
 	mdchain.9 md_get_uint32le.9 \
 	mdchain.9 md_get_uint8.9 \
 	mdchain.9 md_get_uio.9 \
 	mdchain.9 md_initm.9 \
 	mdchain.9 md_next_record.9
 MLINKS+=microtime.9 bintime.9 \
 	microtime.9 getbintime.9 \
 	microtime.9 getmicrotime.9 \
 	microtime.9 getnanotime.9 \
 	microtime.9 nanotime.9
 MLINKS+=microuptime.9 binuptime.9 \
 	microuptime.9 getbinuptime.9 \
 	microuptime.9 getmicrouptime.9 \
 	microuptime.9 getnanouptime.9 \
 	microuptime.9 getsbinuptime.9 \
 	microuptime.9 nanouptime.9 \
 	microuptime.9 sbinuptime.9
 MLINKS+=mi_switch.9 cpu_switch.9 \
 	mi_switch.9 cpu_throw.9
 MLINKS+=mod_cc.9 CCV.9 \
 	mod_cc.9 DECLARE_CC_MODULE.9
 MLINKS+=mtx_pool.9 mtx_pool_alloc.9 \
 	mtx_pool.9 mtx_pool_create.9 \
 	mtx_pool.9 mtx_pool_destroy.9 \
 	mtx_pool.9 mtx_pool_find.9 \
 	mtx_pool.9 mtx_pool_lock.9 \
 	mtx_pool.9 mtx_pool_lock_spin.9 \
 	mtx_pool.9 mtx_pool_unlock.9 \
 	mtx_pool.9 mtx_pool_unlock_spin.9
 MLINKS+=mutex.9 mtx_assert.9 \
 	mutex.9 mtx_destroy.9 \
 	mutex.9 mtx_init.9 \
 	mutex.9 mtx_initialized.9 \
 	mutex.9 mtx_lock.9 \
 	mutex.9 mtx_lock_flags.9 \
 	mutex.9 mtx_lock_spin.9 \
 	mutex.9 mtx_lock_spin_flags.9 \
 	mutex.9 mtx_owned.9 \
 	mutex.9 mtx_recursed.9 \
 	mutex.9 mtx_sleep.9 \
 	mutex.9 MTX_SYSINIT.9 \
 	mutex.9 mtx_trylock.9 \
 	mutex.9 mtx_trylock_flags.9 \
 	mutex.9 mtx_trylock_spin.9 \
 	mutex.9 mtx_trylock_spin_flags.9 \
 	mutex.9 mtx_unlock.9 \
 	mutex.9 mtx_unlock_flags.9 \
 	mutex.9 mtx_unlock_spin.9 \
 	mutex.9 mtx_unlock_spin_flags.9
 MLINKS+=namei.9 NDFREE.9 \
 	namei.9 NDINIT.9
 MLINKS+=netisr.9 netisr_clearqdrops.9 \
 	netisr.9 netisr_default_flow2cpu.9 \
 	netisr.9 netisr_dispatch.9 \
 	netisr.9 netisr_dispatch_src.9 \
 	netisr.9 netisr_get_cpucount.9 \
 	netisr.9 netisr_get_cpuid.9 \
 	netisr.9 netisr_getqdrops.9 \
 	netisr.9 netisr_getqlimit.9 \
 	netisr.9 netisr_queue.9 \
 	netisr.9 netisr_queue_src.9 \
 	netisr.9 netisr_register.9 \
 	netisr.9 netisr_setqlimit.9 \
 	netisr.9 netisr_unregister.9
 MLINKS+=nv.9 libnv.9 \
 	nv.9 nvlist.9 \
 	nv.9 nvlist_add_binary.9 \
 	nv.9 nvlist_add_bool.9 \
 	nv.9 nvlist_add_bool_array.9 \
 	nv.9 nvlist_add_descriptor.9 \
 	nv.9 nvlist_add_descriptor_array.9 \
 	nv.9 nvlist_add_null.9 \
 	nv.9 nvlist_add_number.9 \
 	nv.9 nvlist_add_number_array.9 \
 	nv.9 nvlist_add_nvlist.9 \
 	nv.9 nvlist_add_nvlist_array.9 \
 	nv.9 nvlist_add_string.9 \
 	nv.9 nvlist_add_stringf.9 \
 	nv.9 nvlist_add_stringv.9 \
 	nv.9 nvlist_add_string_array.9 \
 	nv.9 nvlist_clone.9 \
 	nv.9 nvlist_create.9 \
 	nv.9 nvlist_destroy.9 \
 	nv.9 nvlist_dump.9 \
 	nv.9 nvlist_empty.9 \
 	nv.9 nvlist_error.9 \
 	nv.9 nvlist_exists.9 \
 	nv.9 nvlist_exists_binary.9 \
 	nv.9 nvlist_exists_bool.9 \
 	nv.9 nvlist_exists_bool_array.9 \
 	nv.9 nvlist_exists_descriptor.9 \
 	nv.9 nvlist_exists_descriptor_array.9 \
 	nv.9 nvlist_exists_null.9 \
 	nv.9 nvlist_exists_number.9 \
 	nv.9 nvlist_exists_number_array.9 \
 	nv.9 nvlist_exists_nvlist.9 \
 	nv.9 nvlist_exists_nvlist_array.9 \
 	nv.9 nvlist_exists_string.9 \
 	nv.9 nvlist_exists_type.9 \
 	nv.9 nvlist_fdump.9 \
 	nv.9 nvlist_flags.9 \
 	nv.9 nvlist_free.9 \
 	nv.9 nvlist_free_binary.9 \
 	nv.9 nvlist_free_bool.9 \
 	nv.9 nvlist_free_bool_array.9 \
 	nv.9 nvlist_free_descriptor.9 \
 	nv.9 nvlist_free_descriptor_array.9 \
 	nv.9 nvlist_free_null.9 \
 	nv.9 nvlist_free_number.9 \
 	nv.9 nvlist_free_number_array.9 \
 	nv.9 nvlist_free_nvlist.9 \
 	nv.9 nvlist_free_nvlist_array.9 \
 	nv.9 nvlist_free_string.9 \
 	nv.9 nvlist_free_string_array.9 \
 	nv.9 nvlist_free_type.9 \
 	nv.9 nvlist_get_binary.9 \
 	nv.9 nvlist_get_bool.9 \
 	nv.9 nvlist_get_bool_array.9 \
 	nv.9 nvlist_get_descriptor.9 \
 	nv.9 nvlist_get_descriptor_array.9 \
 	nv.9 nvlist_get_number.9 \
 	nv.9 nvlist_get_number_array.9 \
 	nv.9 nvlist_get_nvlist.9 \
 	nv.9 nvlist_get_nvlist_array.9 \
 	nv.9 nvlist_get_parent.9 \
 	nv.9 nvlist_get_string.9 \
 	nv.9 nvlist_get_string_array.9 \
 	nv.9 nvlist_move_binary.9 \
 	nv.9 nvlist_move_descriptor.9 \
 	nv.9 nvlist_move_descriptor_array.9 \
 	nv.9 nvlist_move_nvlist.9 \
 	nv.9 nvlist_move_nvlist_array.9 \
 	nv.9 nvlist_move_string.9 \
 	nv.9 nvlist_move_string_array.9 \
 	nv.9 nvlist_next.9 \
 	nv.9 nvlist_pack.9 \
 	nv.9 nvlist_recv.9 \
 	nv.9 nvlist_send.9 \
 	nv.9 nvlist_set_error.9 \
 	nv.9 nvlist_size.9 \
 	nv.9 nvlist_take_binary.9 \
 	nv.9 nvlist_take_bool.9 \
 	nv.9 nvlist_take_bool_array.9 \
 	nv.9 nvlist_take_descriptor.9 \
 	nv.9 nvlist_take_descriptor_array.9 \
 	nv.9 nvlist_take_number.9 \
 	nv.9 nvlist_take_number_array.9 \
 	nv.9 nvlist_take_nvlist.9 \
 	nv.9 nvlist_take_nvlist_array.9 \
 	nv.9 nvlist_take_string.9 \
 	nv.9 nvlist_take_string_array.9 \
 	nv.9 nvlist_unpack.9 \
 	nv.9 nvlist_xfer.9
 MLINKS+=OF_child.9 OF_parent.9 \
 	OF_child.9 OF_peer.9
 MLINKS+=OF_device_from_xref.9 OF_device_register_xref.9 \
 	OF_device_from_xref.9 OF_xref_from_device.9
 MLINKS+=OF_getprop.9 OF_getencprop.9 \
 	OF_getprop.9 OF_getencprop_alloc.9 \
 	OF_getprop.9 OF_getencprop_alloc_multi.9 \
 	OF_getprop.9 OF_getprop_alloc.9 \
 	OF_getprop.9 OF_getprop_alloc_multi.9 \
 	OF_getprop.9 OF_getproplen.9 \
 	OF_getprop.9 OF_hasprop.9 \
 	OF_getprop.9 OF_nextprop.9 \
 	OF_getprop.9 OF_prop_free.9 \
 	OF_getprop.9 OF_searchencprop.9 \
 	OF_getprop.9 OF_searchprop.9 \
 	OF_getprop.9 OF_setprop.9
 MLINKS+=OF_node_from_xref.9 OF_xref_from_node.9
 MLINKS+=ofw_bus_is_compatible.9 ofw_bus_is_compatible_strict.9 \
 	ofw_bus_is_compatible.9 ofw_bus_node_is_compatible.9 \
 	ofw_bus_is_compatible.9 ofw_bus_search_compatible.9
 MLINKS+= ofw_bus_status_okay.9 ofw_bus_get_status.9 \
 	ofw_bus_status_okay.9 ofw_bus_node_status_okay.9
 MLINKS+=osd.9 osd_call.9 \
 	osd.9 osd_del.9 \
 	osd.9 osd_deregister.9 \
 	osd.9 osd_exit.9 \
 	osd.9 osd_get.9 \
 	osd.9 osd_register.9 \
 	osd.9 osd_set.9
 MLINKS+=panic.9 vpanic.9
 MLINKS+=PCBGROUP.9 in_pcbgroup_byhash.9 \
 	PCBGROUP.9 in_pcbgroup_byinpcb.9 \
 	PCBGROUP.9 in_pcbgroup_destroy.9 \
 	PCBGROUP.9 in_pcbgroup_enabled.9 \
 	PCBGROUP.9 in_pcbgroup_init.9 \
 	PCBGROUP.9 in_pcbgroup_remove.9 \
 	PCBGROUP.9 in_pcbgroup_update.9 \
 	PCBGROUP.9 in_pcbgroup_update_mbuf.9 \
 	PCBGROUP.9 in6_pcbgroup_byhash.9
 MLINKS+=pci.9 pci_alloc_msi.9 \
 	pci.9 pci_alloc_msix.9 \
 	pci.9 pci_disable_busmaster.9 \
 	pci.9 pci_disable_io.9 \
 	pci.9 pci_enable_busmaster.9 \
 	pci.9 pci_enable_io.9 \
 	pci.9 pci_find_bsf.9 \
 	pci.9 pci_find_cap.9 \
 	pci.9 pci_find_dbsf.9 \
 	pci.9 pci_find_device.9 \
 	pci.9 pci_find_extcap.9 \
 	pci.9 pci_find_htcap.9 \
 	pci.9 pci_find_pcie_root_port.9 \
 	pci.9 pci_get_id.9 \
 	pci.9 pci_get_max_read_req.9 \
 	pci.9 pci_get_powerstate.9 \
 	pci.9 pci_get_vpd_ident.9 \
 	pci.9 pci_get_vpd_readonly.9 \
 	pci.9 pci_iov_attach.9 \
 	pci.9 pci_iov_attach_name.9 \
 	pci.9 pci_iov_detach.9 \
 	pci.9 pci_msi_count.9 \
 	pci.9 pci_msix_count.9 \
 	pci.9 pci_msix_pba_bar.9 \
 	pci.9 pci_msix_table_bar.9 \
 	pci.9 pci_pending_msix.9 \
 	pci.9 pci_read_config.9 \
 	pci.9 pci_release_msi.9 \
 	pci.9 pci_remap_msix.9 \
 	pci.9 pci_restore_state.9 \
 	pci.9 pci_save_state.9 \
 	pci.9 pci_set_powerstate.9 \
 	pci.9 pci_set_max_read_req.9 \
 	pci.9 pci_write_config.9 \
 	pci.9 pcie_adjust_config.9 \
 	pci.9 pcie_flr.9 \
 	pci.9 pcie_max_completion_timeout.9 \
 	pci.9 pcie_read_config.9 \
 	pci.9 pcie_wait_for_pending_transactions.9 \
 	pci.9 pcie_write_config.9
 MLINKS+=pci_iov_schema.9 pci_iov_schema_alloc_node.9 \
 	pci_iov_schema.9 pci_iov_schema_add_bool.9 \
 	pci_iov_schema.9 pci_iov_schema_add_string.9 \
 	pci_iov_schema.9 pci_iov_schema_add_uint8.9 \
 	pci_iov_schema.9 pci_iov_schema_add_uint16.9 \
 	pci_iov_schema.9 pci_iov_schema_add_uint32.9 \
 	pci_iov_schema.9 pci_iov_schema_add_uint64.9 \
 	pci_iov_schema.9 pci_iov_schema_add_unicast_mac.9
 MLINKS+=pfil.9 pfil_add_hook.9 \
 	pfil.9 pfil_head_register.9 \
 	pfil.9 pfil_head_unregister.9 \
 	pfil.9 pfil_remove_hook.9 \
 	pfil.9 pfil_run_hooks.9 \
 	pfil.9 pfil_link.9
 MLINKS+=pfind.9 zpfind.9
 MLINKS+=PHOLD.9 PRELE.9 \
 	PHOLD.9 _PHOLD.9 \
 	PHOLD.9 _PRELE.9 \
 	PHOLD.9 PROC_ASSERT_HELD.9 \
 	PHOLD.9 PROC_ASSERT_NOT_HELD.9
 MLINKS+=pmap_copy.9 pmap_copy_page.9
 MLINKS+=pmap_extract.9 pmap_extract_and_hold.9
 MLINKS+=pmap_init.9 pmap_init2.9
 MLINKS+=pmap_is_modified.9 pmap_ts_referenced.9
 MLINKS+=pmap_pinit.9 pmap_pinit0.9 \
 	pmap_pinit.9 pmap_pinit2.9
 MLINKS+=pmap_qenter.9 pmap_qremove.9
 MLINKS+=pmap_quick_enter_page.9 pmap_quick_remove_page.9
 MLINKS+=pmap_remove.9 pmap_remove_all.9 \
 	pmap_remove.9 pmap_remove_pages.9
 MLINKS+=pmap_resident_count.9 pmap_wired_count.9
 MLINKS+=pmap_zero_page.9 pmap_zero_area.9
 MLINKS+=printf.9 log.9 \
 	printf.9 tprintf.9 \
 	printf.9 uprintf.9
 MLINKS+=priv.9 priv_check.9 \
 	priv.9 priv_check_cred.9
 MLINKS+=proc_rwmem.9 proc_readmem.9 \
 	proc_rwmem.9 proc_writemem.9
 MLINKS+=psignal.9 gsignal.9 \
 	psignal.9 pgsignal.9 \
 	psignal.9 tdsignal.9
 MLINKS+=pwmbus.9 pwm.9
 MLINKS+=random.9 arc4rand.9 \
 	random.9 arc4random.9 \
 	random.9 is_random_seeded.9 \
 	random.9 read_random.9 \
 	random.9 read_random_uio.9 \
 	random.9 srandom.9
 MLINKS+=random_harvest.9 random_harvest_direct.9 \
 	random_harvest.9 random_harvest_fast.9 \
 	random_harvest.9 random_harvest_queue.9
 MLINKS+=ratecheck.9 ppsratecheck.9
 MLINKS+=refcount.9 refcount_acquire.9 \
 	refcount.9 refcount_init.9 \
 	refcount.9 refcount_release.9
 MLINKS+=resource_int_value.9 resource_long_value.9 \
 	resource_int_value.9 resource_string_value.9
 MLINKS+=rman.9 rman_activate_resource.9 \
 	rman.9 rman_adjust_resource.9 \
 	rman.9 rman_deactivate_resource.9 \
 	rman.9 rman_fini.9 \
 	rman.9 rman_first_free_region.9 \
 	rman.9 rman_get_bushandle.9 \
 	rman.9 rman_get_bustag.9 \
 	rman.9 rman_get_device.9 \
 	rman.9 rman_get_end.9 \
 	rman.9 rman_get_flags.9 \
 	rman.9 rman_get_mapping.9 \
 	rman.9 rman_get_rid.9 \
 	rman.9 rman_get_size.9 \
 	rman.9 rman_get_start.9 \
 	rman.9 rman_get_virtual.9 \
 	rman.9 rman_init.9 \
 	rman.9 rman_init_from_resource.9 \
 	rman.9 rman_is_region_manager.9 \
 	rman.9 rman_last_free_region.9 \
 	rman.9 rman_make_alignment_flags.9 \
 	rman.9 rman_manage_region.9 \
 	rman.9 rman_release_resource.9 \
 	rman.9 rman_reserve_resource.9 \
 	rman.9 rman_reserve_resource_bound.9 \
 	rman.9 rman_set_bushandle.9 \
 	rman.9 rman_set_bustag.9 \
 	rman.9 rman_set_mapping.9 \
 	rman.9 rman_set_rid.9 \
 	rman.9 rman_set_virtual.9
 MLINKS+=rmlock.9 rm_assert.9 \
 	rmlock.9 rm_destroy.9 \
 	rmlock.9 rm_init.9 \
 	rmlock.9 rm_init_flags.9 \
 	rmlock.9 rm_rlock.9 \
 	rmlock.9 rm_runlock.9 \
 	rmlock.9 rm_sleep.9 \
 	rmlock.9 RM_SYSINIT.9 \
 	rmlock.9 RM_SYSINIT_FLAGS.9 \
 	rmlock.9 rm_try_rlock.9 \
 	rmlock.9 rm_wlock.9 \
 	rmlock.9 rm_wowned.9 \
 	rmlock.9 rm_wunlock.9
 MLINKS+=rtalloc.9 rtalloc1.9 \
 	rtalloc.9 rtalloc_ign.9 \
 	rtalloc.9 RT_ADDREF.9 \
 	rtalloc.9 RT_LOCK.9 \
 	rtalloc.9 RT_REMREF.9 \
 	rtalloc.9 RT_RTFREE.9 \
 	rtalloc.9 RT_UNLOCK.9 \
 	rtalloc.9 RTFREE_LOCKED.9 \
 	rtalloc.9 RTFREE.9 \
 	rtalloc.9 rtfree.9 \
 	rtalloc.9 rtalloc1_fib.9 \
 	rtalloc.9 rtalloc_ign_fib.9 \
 	rtalloc.9 rtalloc_fib.9
 MLINKS+=runqueue.9 choosethread.9 \
 	runqueue.9 procrunnable.9 \
 	runqueue.9 remrunqueue.9 \
 	runqueue.9 setrunqueue.9
 MLINKS+=rwlock.9 rw_assert.9 \
 	rwlock.9 rw_destroy.9 \
 	rwlock.9 rw_downgrade.9 \
 	rwlock.9 rw_init.9 \
 	rwlock.9 rw_init_flags.9 \
 	rwlock.9 rw_initialized.9 \
 	rwlock.9 rw_rlock.9 \
 	rwlock.9 rw_runlock.9 \
 	rwlock.9 rw_unlock.9 \
 	rwlock.9 rw_sleep.9 \
 	rwlock.9 RW_SYSINIT.9 \
 	rwlock.9 RW_SYSINIT_FLAGS.9 \
 	rwlock.9 rw_try_rlock.9 \
 	rwlock.9 rw_try_upgrade.9 \
 	rwlock.9 rw_try_wlock.9 \
 	rwlock.9 rw_wlock.9 \
 	rwlock.9 rw_wowned.9 \
 	rwlock.9 rw_wunlock.9
 MLINKS+=sbuf.9 sbuf_bcat.9 \
 	sbuf.9 sbuf_bcopyin.9 \
 	sbuf.9 sbuf_bcpy.9 \
 	sbuf.9 sbuf_cat.9 \
 	sbuf.9 sbuf_clear.9 \
 	sbuf.9 sbuf_clear_flags.9 \
 	sbuf.9 sbuf_copyin.9 \
 	sbuf.9 sbuf_cpy.9 \
 	sbuf.9 sbuf_data.9 \
 	sbuf.9 sbuf_delete.9 \
 	sbuf.9 sbuf_done.9 \
 	sbuf.9 sbuf_error.9 \
 	sbuf.9 sbuf_finish.9 \
 	sbuf.9 sbuf_get_flags.9 \
 	sbuf.9 sbuf_hexdump.9 \
 	sbuf.9 sbuf_len.9 \
 	sbuf.9 sbuf_new.9 \
 	sbuf.9 sbuf_new_auto.9 \
 	sbuf.9 sbuf_new_for_sysctl.9 \
 	sbuf.9 sbuf_nl_terminate.9 \
 	sbuf.9 sbuf_printf.9 \
 	sbuf.9 sbuf_printf_drain.9 \
 	sbuf.9 sbuf_putbuf.9 \
 	sbuf.9 sbuf_putc.9 \
 	sbuf.9 sbuf_set_drain.9 \
 	sbuf.9 sbuf_set_flags.9 \
 	sbuf.9 sbuf_setpos.9 \
 	sbuf.9 sbuf_start_section.9 \
 	sbuf.9 sbuf_end_section.9  \
 	sbuf.9 sbuf_trim.9 \
 	sbuf.9 sbuf_vprintf.9
 MLINKS+=scheduler.9 curpriority_cmp.9 \
 	scheduler.9 maybe_resched.9 \
 	scheduler.9 propagate_priority.9 \
 	scheduler.9 resetpriority.9 \
 	scheduler.9 roundrobin.9 \
 	scheduler.9 roundrobin_interval.9 \
 	scheduler.9 schedclock.9 \
 	scheduler.9 schedcpu.9 \
 	scheduler.9 sched_setup.9 \
 	scheduler.9 setrunnable.9 \
 	scheduler.9 updatepri.9
 MLINKS+=SDT.9 SDT_PROVIDER_DECLARE.9 \
 	SDT.9 SDT_PROVIDER_DEFINE.9 \
 	SDT.9 SDT_PROBE_DECLARE.9 \
 	SDT.9 SDT_PROBE_DEFINE.9 \
 	SDT.9 SDT_PROBE.9
 MLINKS+=securelevel_gt.9 securelevel_ge.9
 MLINKS+=selrecord.9 seldrain.9 \
 	selrecord.9 selwakeup.9
 MLINKS+=sema.9 sema_destroy.9 \
 	sema.9 sema_init.9 \
 	sema.9 sema_post.9 \
 	sema.9 sema_timedwait.9 \
 	sema.9 sema_trywait.9 \
 	sema.9 sema_value.9 \
 	sema.9 sema_wait.9
 MLINKS+=seqc.9 seqc_consistent.9 \
 	seqc.9 seqc_read.9 \
 	seqc.9 seqc_write_begin.9 \
 	seqc.9 seqc_write_end.9
 MLINKS+=sf_buf.9 sf_buf_alloc.9 \
 	sf_buf.9 sf_buf_free.9 \
 	sf_buf.9 sf_buf_kva.9 \
 	sf_buf.9 sf_buf_page.9
 MLINKS+=sglist.9 sglist_alloc.9 \
 	sglist.9 sglist_append.9 \
 	sglist.9 sglist_append_bio.9 \
 	sglist.9 sglist_append_ext_pgs.9 \
 	sglist.9 sglist_append_mb_ext_pgs.9 \
 	sglist.9 sglist_append_mbuf.9 \
 	sglist.9 sglist_append_phys.9 \
 	sglist.9 sglist_append_sglist.9 \
 	sglist.9 sglist_append_uio.9 \
 	sglist.9 sglist_append_user.9 \
 	sglist.9 sglist_append_vmpages.9 \
 	sglist.9 sglist_build.9 \
 	sglist.9 sglist_clone.9 \
 	sglist.9 sglist_consume_uio.9 \
 	sglist.9 sglist_count.9 \
 	sglist.9 sglist_count_ext_pgs.9 \
 	sglist.9 sglist_count_mb_ext_pgs.9 \
 	sglist.9 sglist_count_vmpages.9 \
 	sglist.9 sglist_free.9 \
 	sglist.9 sglist_hold.9 \
 	sglist.9 sglist_init.9 \
 	sglist.9 sglist_join.9 \
 	sglist.9 sglist_length.9 \
 	sglist.9 sglist_reset.9 \
 	sglist.9 sglist_slice.9 \
 	sglist.9 sglist_split.9
 MLINKS+=shm_map.9 shm_unmap.9
 MLINKS+=signal.9 cursig.9 \
 	signal.9 execsigs.9 \
 	signal.9 issignal.9 \
 	signal.9 killproc.9 \
 	signal.9 pgsigio.9 \
 	signal.9 postsig.9 \
 	signal.9 SETSETNEQ.9 \
 	signal.9 SETSETOR.9 \
 	signal.9 SIGADDSET.9 \
 	signal.9 SIG_CONTSIGMASK.9 \
 	signal.9 SIGDELSET.9 \
 	signal.9 SIGEMPTYSET.9 \
 	signal.9 sigexit.9 \
 	signal.9 SIGFILLSET.9 \
 	signal.9 siginit.9 \
 	signal.9 SIGISEMPTY.9 \
 	signal.9 SIGISMEMBER.9 \
 	signal.9 SIGNOTEMPTY.9 \
 	signal.9 signotify.9 \
 	signal.9 SIGPENDING.9 \
 	signal.9 SIGSETAND.9 \
 	signal.9 SIGSETCANTMASK.9 \
 	signal.9 SIGSETEQ.9 \
 	signal.9 SIGSETNAND.9 \
 	signal.9 SIG_STOPSIGMASK.9 \
 	signal.9 trapsignal.9
 MLINKS+=sleep.9 msleep.9 \
 	sleep.9 msleep_sbt.9 \
 	sleep.9 msleep_spin.9 \
 	sleep.9 msleep_spin_sbt.9 \
 	sleep.9 pause.9 \
 	sleep.9 pause_sig.9 \
 	sleep.9 pause_sbt.9 \
 	sleep.9 tsleep.9 \
 	sleep.9 tsleep_sbt.9 \
 	sleep.9 wakeup.9 \
 	sleep.9 wakeup_one.9 \
 	sleep.9 wakeup_any.9
 MLINKS+=sleepqueue.9 init_sleepqueues.9 \
 	sleepqueue.9 sleepq_abort.9 \
 	sleepqueue.9 sleepq_add.9 \
 	sleepqueue.9 sleepq_alloc.9 \
 	sleepqueue.9 sleepq_broadcast.9 \
 	sleepqueue.9 sleepq_free.9 \
 	sleepqueue.9 sleepq_lookup.9 \
 	sleepqueue.9 sleepq_lock.9 \
 	sleepqueue.9 sleepq_release.9 \
 	sleepqueue.9 sleepq_remove.9 \
 	sleepqueue.9 sleepq_set_timeout.9 \
 	sleepqueue.9 sleepq_set_timeout_sbt.9 \
 	sleepqueue.9 sleepq_signal.9 \
 	sleepqueue.9 sleepq_sleepcnt.9 \
 	sleepqueue.9 sleepq_timedwait.9 \
 	sleepqueue.9 sleepq_timedwait_sig.9 \
 	sleepqueue.9 sleepq_type.9 \
 	sleepqueue.9 sleepq_wait.9 \
 	sleepqueue.9 sleepq_wait_sig.9
 MLINKS+=socket.9 soabort.9 \
 	socket.9 soaccept.9 \
 	socket.9 sobind.9 \
 	socket.9 socheckuid.9 \
 	socket.9 soclose.9 \
 	socket.9 soconnect.9 \
 	socket.9 socreate.9 \
 	socket.9 sodisconnect.9 \
 	socket.9 sodtor_set.9 \
 	socket.9 sodupsockaddr.9 \
 	socket.9 sofree.9 \
 	socket.9 sogetopt.9 \
 	socket.9 sohasoutofband.9 \
 	socket.9 solisten.9 \
 	socket.9 solisten_proto.9 \
 	socket.9 solisten_proto_check.9 \
 	socket.9 sonewconn.9 \
 	socket.9 sooptcopyin.9 \
 	socket.9 sooptcopyout.9 \
 	socket.9 sopoll.9 \
 	socket.9 sopoll_generic.9 \
 	socket.9 soreceive.9 \
 	socket.9 soreceive_dgram.9 \
 	socket.9 soreceive_generic.9 \
 	socket.9 soreceive_stream.9 \
 	socket.9 soreserve.9 \
 	socket.9 sorflush.9 \
 	socket.9 sosend.9 \
 	socket.9 sosend_dgram.9 \
 	socket.9 sosend_generic.9 \
 	socket.9 sosetopt.9 \
 	socket.9 soshutdown.9 \
 	socket.9 sotoxsocket.9 \
 	socket.9 soupcall_clear.9 \
 	socket.9 soupcall_set.9 \
 	socket.9 sowakeup.9
 MLINKS+=stack.9 stack_copy.9 \
 	stack.9 stack_create.9 \
 	stack.9 stack_destroy.9 \
 	stack.9 stack_print.9 \
 	stack.9 stack_print_ddb.9 \
 	stack.9 stack_print_short.9 \
 	stack.9 stack_print_short_ddb.9 \
 	stack.9 stack_put.9 \
 	stack.9 stack_save.9 \
 	stack.9 stack_sbuf_print.9 \
 	stack.9 stack_sbuf_print_ddb.9 \
 	stack.9 stack_zero.9
 MLINKS+=store.9 subyte.9 \
 	store.9 suword.9 \
 	store.9 suword16.9 \
 	store.9 suword32.9 \
 	store.9 suword64.9
 MLINKS+=swi.9 swi_add.9 \
 	swi.9 swi_remove.9 \
 	swi.9 swi_sched.9
 MLINKS+=sx.9 sx_assert.9 \
 	sx.9 sx_destroy.9 \
 	sx.9 sx_downgrade.9 \
 	sx.9 sx_init.9 \
 	sx.9 sx_init_flags.9 \
 	sx.9 sx_sleep.9 \
 	sx.9 sx_slock.9 \
 	sx.9 sx_slock_sig.9 \
 	sx.9 sx_sunlock.9 \
 	sx.9 SX_SYSINIT.9 \
 	sx.9 SX_SYSINIT_FLAGS.9 \
 	sx.9 sx_try_slock.9 \
 	sx.9 sx_try_upgrade.9 \
 	sx.9 sx_try_xlock.9 \
 	sx.9 sx_unlock.9 \
 	sx.9 sx_xholder.9 \
 	sx.9 sx_xlock.9 \
 	sx.9 sx_xlock_sig.9 \
 	sx.9 sx_xlocked.9 \
 	sx.9 sx_xunlock.9
 MLINKS+=syscall_helper_register.9 syscall_helper_unregister.9 \
 	syscall_helper_register.9 SYSCALL_INIT_HELPER.9 \
 	syscall_helper_register.9 SYSCALL_INIT_HELPER_COMPAT.9 \
 	syscall_helper_register.9 SYSCALL_INIT_HELPER_COMPAT_F.9 \
 	syscall_helper_register.9 SYSCALL_INIT_HELPER_F.9
 MLINKS+=sysctl.9 SYSCTL_DECL.9 \
 	sysctl.9 SYSCTL_ADD_INT.9 \
 	sysctl.9 SYSCTL_ADD_LONG.9 \
 	sysctl.9 SYSCTL_ADD_NODE.9 \
 	sysctl.9 SYSCTL_ADD_NODE_WITH_LABEL.9 \
 	sysctl.9 SYSCTL_ADD_OPAQUE.9 \
 	sysctl.9 SYSCTL_ADD_PROC.9 \
 	sysctl.9 SYSCTL_ADD_QUAD.9 \
 	sysctl.9 SYSCTL_ADD_ROOT_NODE.9 \
 	sysctl.9 SYSCTL_ADD_S8.9 \
 	sysctl.9 SYSCTL_ADD_S16.9 \
 	sysctl.9 SYSCTL_ADD_S32.9 \
 	sysctl.9 SYSCTL_ADD_S64.9 \
 	sysctl.9 SYSCTL_ADD_STRING.9 \
 	sysctl.9 SYSCTL_ADD_STRUCT.9 \
 	sysctl.9 SYSCTL_ADD_TIMEVAL_SEC.9 \
 	sysctl.9 SYSCTL_ADD_U8.9 \
 	sysctl.9 SYSCTL_ADD_U16.9 \
 	sysctl.9 SYSCTL_ADD_U32.9 \
 	sysctl.9 SYSCTL_ADD_U64.9 \
 	sysctl.9 SYSCTL_ADD_UAUTO.9 \
 	sysctl.9 SYSCTL_ADD_UINT.9 \
 	sysctl.9 SYSCTL_ADD_ULONG.9 \
 	sysctl.9 SYSCTL_ADD_UMA_CUR.9 \
 	sysctl.9 SYSCTL_ADD_UMA_MAX.9 \
 	sysctl.9 SYSCTL_ADD_UQUAD.9 \
 	sysctl.9 SYSCTL_CHILDREN.9 \
 	sysctl.9 SYSCTL_STATIC_CHILDREN.9 \
 	sysctl.9 SYSCTL_NODE_CHILDREN.9 \
 	sysctl.9 SYSCTL_PARENT.9 \
 	sysctl.9 SYSCTL_INT.9 \
 	sysctl.9 SYSCTL_INT_WITH_LABEL.9 \
 	sysctl.9 SYSCTL_LONG.9 \
 	sysctl.9 sysctl_msec_to_ticks.9 \
 	sysctl.9 SYSCTL_NODE.9 \
 	sysctl.9 SYSCTL_NODE_WITH_LABEL.9 \
 	sysctl.9 SYSCTL_OPAQUE.9 \
 	sysctl.9 SYSCTL_PROC.9 \
 	sysctl.9 SYSCTL_QUAD.9 \
 	sysctl.9 SYSCTL_ROOT_NODE.9 \
 	sysctl.9 SYSCTL_S8.9 \
 	sysctl.9 SYSCTL_S16.9 \
 	sysctl.9 SYSCTL_S32.9 \
 	sysctl.9 SYSCTL_S64.9 \
 	sysctl.9 SYSCTL_STRING.9 \
 	sysctl.9 SYSCTL_STRUCT.9 \
 	sysctl.9 SYSCTL_TIMEVAL_SEC.9 \
 	sysctl.9 SYSCTL_U8.9 \
 	sysctl.9 SYSCTL_U16.9 \
 	sysctl.9 SYSCTL_U32.9 \
 	sysctl.9 SYSCTL_U64.9 \
 	sysctl.9 SYSCTL_UINT.9 \
 	sysctl.9 SYSCTL_ULONG.9 \
 	sysctl.9 SYSCTL_UMA_CUR.9 \
 	sysctl.9 SYSCTL_UMA_MAX.9 \
 	sysctl.9 SYSCTL_UQUAD.9
 MLINKS+=sysctl_add_oid.9 sysctl_move_oid.9 \
 	sysctl_add_oid.9 sysctl_remove_oid.9 \
 	sysctl_add_oid.9 sysctl_remove_name.9
 MLINKS+=sysctl_ctx_init.9 sysctl_ctx_entry_add.9 \
 	sysctl_ctx_init.9 sysctl_ctx_entry_del.9 \
 	sysctl_ctx_init.9 sysctl_ctx_entry_find.9 \
 	sysctl_ctx_init.9 sysctl_ctx_free.9
 MLINKS+=SYSINIT.9 SYSUNINIT.9
 MLINKS+=taskqueue.9 TASK_INIT.9 \
 	taskqueue.9 TASK_INITIALIZER.9 \
 	taskqueue.9 taskqueue_block.9 \
 	taskqueue.9 taskqueue_cancel.9 \
 	taskqueue.9 taskqueue_cancel_timeout.9 \
 	taskqueue.9 taskqueue_create.9 \
 	taskqueue.9 taskqueue_create_fast.9 \
 	taskqueue.9 TASKQUEUE_DECLARE.9 \
 	taskqueue.9 TASKQUEUE_DEFINE.9 \
 	taskqueue.9 TASKQUEUE_DEFINE_THREAD.9 \
 	taskqueue.9 taskqueue_drain.9 \
 	taskqueue.9 taskqueue_drain_all.9 \
 	taskqueue.9 taskqueue_drain_timeout.9 \
 	taskqueue.9 taskqueue_enqueue.9 \
 	taskqueue.9 taskqueue_enqueue_timeout.9 \
 	taskqueue.9 TASKQUEUE_FAST_DEFINE.9 \
 	taskqueue.9 TASKQUEUE_FAST_DEFINE_THREAD.9 \
 	taskqueue.9 taskqueue_free.9 \
 	taskqueue.9 taskqueue_member.9 \
 	taskqueue.9 taskqueue_quiesce.9 \
 	taskqueue.9 taskqueue_run.9 \
 	taskqueue.9 taskqueue_set_callback.9 \
 	taskqueue.9 taskqueue_start_threads.9 \
 	taskqueue.9 taskqueue_start_threads_cpuset.9 \
 	taskqueue.9 taskqueue_start_threads_in_proc.9 \
 	taskqueue.9 taskqueue_unblock.9 \
 	taskqueue.9 TIMEOUT_TASK_INIT.9
 MLINKS+=tcp_functions.9 register_tcp_functions.9 \
 	tcp_functions.9 register_tcp_functions_as_name.9 \
 	tcp_functions.9 register_tcp_functions_as_names.9 \
 	tcp_functions.9 deregister_tcp_functions.9
 MLINKS+=time.9 boottime.9 \
 	time.9 time_second.9 \
 	time.9 time_uptime.9
 MLINKS+=ucred.9 crcopy.9 \
 	ucred.9 crcopysafe.9 \
 	ucred.9 crdup.9 \
 	ucred.9 crfree.9 \
 	ucred.9 crget.9 \
 	ucred.9 crhold.9 \
 	ucred.9 crsetgroups.9 \
 	ucred.9 cru2x.9
 MLINKS+=uidinfo.9 uifind.9 \
 	uidinfo.9 uifree.9 \
 	uidinfo.9 uihashinit.9 \
 	uidinfo.9 uihold.9
 MLINKS+=uio.9 uiomove.9 \
 	uio.9 uiomove_frombuf.9 \
 	uio.9 uiomove_nofault.9
 
 .if ${MK_USB} != "no"
 MAN+=	usbdi.9
 MLINKS+=usbdi.9 usbd_do_request.9 \
 	usbdi.9 usbd_do_request_flags.9 \
 	usbdi.9 usbd_errstr.9 \
 	usbdi.9 usbd_lookup_id_by_info.9 \
 	usbdi.9 usbd_lookup_id_by_uaa.9 \
 	usbdi.9 usbd_transfer_clear_stall.9 \
 	usbdi.9 usbd_transfer_drain.9 \
 	usbdi.9 usbd_transfer_pending.9 \
 	usbdi.9 usbd_transfer_poll.9 \
 	usbdi.9 usbd_transfer_setup.9 \
 	usbdi.9 usbd_transfer_start.9 \
 	usbdi.9 usbd_transfer_stop.9 \
 	usbdi.9 usbd_transfer_submit.9 \
 	usbdi.9 usbd_transfer_unsetup.9 \
 	usbdi.9 usbd_xfer_clr_flag.9 \
 	usbdi.9 usbd_xfer_frame_data.9 \
 	usbdi.9 usbd_xfer_frame_len.9 \
 	usbdi.9 usbd_xfer_get_frame.9 \
 	usbdi.9 usbd_xfer_get_priv.9 \
 	usbdi.9 usbd_xfer_is_stalled.9 \
 	usbdi.9 usbd_xfer_max_framelen.9 \
 	usbdi.9 usbd_xfer_max_frames.9 \
 	usbdi.9 usbd_xfer_max_len.9 \
 	usbdi.9 usbd_xfer_set_flag.9 \
 	usbdi.9 usbd_xfer_set_frame_data.9 \
 	usbdi.9 usbd_xfer_set_frame_len.9 \
 	usbdi.9 usbd_xfer_set_frame_offset.9 \
 	usbdi.9 usbd_xfer_set_frames.9 \
 	usbdi.9 usbd_xfer_set_interval.9 \
 	usbdi.9 usbd_xfer_set_priv.9 \
 	usbdi.9 usbd_xfer_set_stall.9 \
 	usbdi.9 usbd_xfer_set_timeout.9 \
 	usbdi.9 usbd_xfer_softc.9 \
 	usbdi.9 usbd_xfer_state.9 \
 	usbdi.9 usbd_xfer_status.9 \
 	usbdi.9 usb_fifo_alloc_buffer.9 \
 	usbdi.9 usb_fifo_attach.9 \
 	usbdi.9 usb_fifo_detach.9 \
 	usbdi.9 usb_fifo_free_buffer.9 \
 	usbdi.9 usb_fifo_get_data.9 \
 	usbdi.9 usb_fifo_get_data_buffer.9 \
 	usbdi.9 usb_fifo_get_data_error.9 \
 	usbdi.9 usb_fifo_get_data_linear.9 \
 	usbdi.9 usb_fifo_put_bytes_max.9 \
 	usbdi.9 usb_fifo_put_data.9 \
 	usbdi.9 usb_fifo_put_data_buffer.9 \
 	usbdi.9 usb_fifo_put_data_error.9 \
 	usbdi.9 usb_fifo_put_data_linear.9 \
 	usbdi.9 usb_fifo_reset.9 \
 	usbdi.9 usb_fifo_softc.9 \
 	usbdi.9 usb_fifo_wakeup.9
 .endif
 MLINKS+=vcount.9 count_dev.9
 MLINKS+=vfsconf.9 vfs_modevent.9 \
 	vfsconf.9 vfs_register.9 \
 	vfsconf.9 vfs_unregister.9
 MLINKS+=vfs_getopt.9 vfs_copyopt.9 \
 	vfs_getopt.9 vfs_filteropt.9 \
 	vfs_getopt.9 vfs_flagopt.9 \
 	vfs_getopt.9 vfs_getopts.9 \
 	vfs_getopt.9 vfs_scanopt.9 \
 	vfs_getopt.9 vfs_setopt.9 \
 	vfs_getopt.9 vfs_setopt_part.9 \
 	vfs_getopt.9 vfs_setopts.9
 MLINKS+=vhold.9 vdrop.9 \
 	vhold.9 vdropl.9 \
 	vhold.9 vholdl.9
 MLINKS+=vmem.9 vmem_add.9 \
 	vmem.9 vmem_alloc.9 \
 	vmem.9 vmem_create.9 \
 	vmem.9 vmem_destroy.9 \
 	vmem.9 vmem_free.9 \
 	vmem.9 vmem_xalloc.9 \
 	vmem.9 vmem_xfree.9  
 MLINKS+=vm_map_lock.9 vm_map_lock_downgrade.9 \
 	vm_map_lock.9 vm_map_lock_read.9 \
 	vm_map_lock.9 vm_map_lock_upgrade.9 \
 	vm_map_lock.9 vm_map_trylock.9 \
 	vm_map_lock.9 vm_map_trylock_read.9 \
 	vm_map_lock.9 vm_map_unlock.9 \
 	vm_map_lock.9 vm_map_unlock_read.9
 MLINKS+=vm_map_lookup.9 vm_map_lookup_done.9
 MLINKS+=vm_map_max.9 vm_map_min.9 \
 	vm_map_max.9 vm_map_pmap.9
 MLINKS+=vm_map_stack.9 vm_map_growstack.9
 MLINKS+=vm_map_wire.9 vm_map_wire_mapped.9 \
 	vm_page_wire.9 vm_page_unwire.9 \
 	vm_page_wire.9 vm_page_unwire_noq.9
 MLINKS+=vm_page_bits.9 vm_page_clear_dirty.9 \
 	vm_page_bits.9 vm_page_dirty.9 \
 	vm_page_bits.9 vm_page_is_valid.9 \
 	vm_page_bits.9 vm_page_set_invalid.9 \
 	vm_page_bits.9 vm_page_set_validclean.9 \
 	vm_page_bits.9 vm_page_test_dirty.9 \
 	vm_page_bits.9 vm_page_undirty.9 \
 	vm_page_bits.9 vm_page_zero_invalid.9
 MLINKS+=vm_page_busy.9 vm_page_busied.9 \
 	vm_page_busy.9 vm_page_busy_downgrade.9 \
 	vm_page_busy.9 vm_page_busy_sleep.9 \
 	vm_page_busy.9 vm_page_sbusied.9 \
 	vm_page_busy.9 vm_page_sbusy.9 \
 	vm_page_busy.9 vm_page_sleep_if_busy.9 \
 	vm_page_busy.9 vm_page_sunbusy.9 \
 	vm_page_busy.9 vm_page_trysbusy.9 \
 	vm_page_busy.9 vm_page_tryxbusy.9 \
 	vm_page_busy.9 vm_page_xbusied.9 \
 	vm_page_busy.9 vm_page_xbusy.9 \
 	vm_page_busy.9 vm_page_xunbusy.9 \
 	vm_page_busy.9 vm_page_assert_sbusied.9 \
 	vm_page_busy.9 vm_page_assert_unbusied.9 \
 	vm_page_busy.9 vm_page_assert_xbusied.9
 MLINKS+=vm_page_aflag.9 vm_page_aflag_clear.9 \
 	vm_page_aflag.9 vm_page_aflag_set.9 \
 	vm_page_aflag.9 vm_page_reference.9
 MLINKS+=vm_page_free.9 vm_page_free_toq.9 \
 	vm_page_free.9 vm_page_free_zero.9 \
 	vm_page_free.9 vm_page_try_to_free.9
 MLINKS+=vm_page_insert.9 vm_page_remove.9
 MLINKS+=vm_page_wire.9 vm_page_unwire.9
 MLINKS+=VOP_ACCESS.9 VOP_ACCESSX.9
 MLINKS+=VOP_ATTRIB.9 VOP_GETATTR.9 \
 	VOP_ATTRIB.9 VOP_SETATTR.9
 MLINKS+=VOP_CREATE.9 VOP_MKDIR.9 \
 	VOP_CREATE.9 VOP_MKNOD.9 \
 	VOP_CREATE.9 VOP_SYMLINK.9
 MLINKS+=VOP_FSYNC.9 VOP_FDATASYNC.9
 MLINKS+=VOP_GETPAGES.9 VOP_PUTPAGES.9
 MLINKS+=VOP_INACTIVE.9 VOP_RECLAIM.9
 MLINKS+=VOP_LOCK.9 vn_lock.9 \
 	VOP_LOCK.9 VOP_ISLOCKED.9 \
 	VOP_LOCK.9 VOP_UNLOCK.9
 MLINKS+=VOP_OPENCLOSE.9 VOP_CLOSE.9 \
 	VOP_OPENCLOSE.9 VOP_OPEN.9
 MLINKS+=VOP_RDWR.9 VOP_READ.9 \
 	VOP_RDWR.9 VOP_WRITE.9
 MLINKS+=VOP_REMOVE.9 VOP_RMDIR.9
 MLINKS+=vnet.9 vimage.9
 MLINKS+=vref.9 VREF.9 \
 	vref.9 vrefl.9
 MLINKS+=vrele.9 vput.9 \
 	vrele.9 vunref.9
 MLINKS+=vslock.9 vsunlock.9
 MLINKS+=zone.9 uma.9 \
 	zone.9 uma_prealloc.9 \
 	zone.9 uma_reclaim.9 \
 	zone.9 uma_zalloc.9 \
 	zone.9 uma_zalloc_arg.9 \
 	zone.9 uma_zalloc_domain.9 \
 	zone.9 uma_zalloc_pcpu.9 \
 	zone.9 uma_zalloc_pcpu_arg.9 \
 	zone.9 uma_zcache_create.9 \
 	zone.9 uma_zcreate.9 \
 	zone.9 uma_zdestroy.9 \
 	zone.9 uma_zfree.9 \
 	zone.9 uma_zfree_arg.9 \
 	zone.9 uma_zfree_domain.9 \
 	zone.9 uma_zfree_pcpu.9 \
 	zone.9 uma_zfree_pcpu_arg.9 \
 	zone.9 uma_zone_get_cur.9 \
 	zone.9 uma_zone_get_max.9 \
 	zone.9 uma_zone_reclaim.9 \
 	zone.9 uma_zone_reserve.9 \
 	zone.9 uma_zone_reserve_kva.9 \
 	zone.9 uma_zone_set_allocf.9 \
 	zone.9 uma_zone_set_freef.9 \
 	zone.9 uma_zone_set_max.9 \
 	zone.9 uma_zone_set_maxaction.9 \
 	zone.9 uma_zone_set_maxcache.9 \
 	zone.9 uma_zone_set_warning.9 \
 	zone.9 uma_zsecond_create.9
 
 .if ${MACHINE_CPUARCH} == "amd64" || ${MACHINE_CPUARCH} == "i386"
 _superio.9=	superio.9
 MLINKS+=superio.9 superio_devid.9 \
 	superio.9 superio_dev_disable.9 \
 	superio.9 superio_dev_enable.9 \
 	superio.9 superio_dev_enabled.9 \
 	superio.9 superio_find_dev.9 \
 	superio.9 superio_find_dev.9 \
 	superio.9 superio_get_dma.9 \
 	superio.9 superio_get_iobase.9 \
 	superio.9 superio_get_irq.9 \
 	superio.9 superio_get_ldn.9 \
 	superio.9 superio_get_type.9 \
 	superio.9 superio_read.9 \
 	superio.9 superio_revid.9 \
 	superio.9 superio_vendor.9 \
 	superio.9 superio_write.9
 .endif
 
 .include <bsd.prog.mk>
Index: head/share/man/man9/bus_dma.9
===================================================================
--- head/share/man/man9/bus_dma.9	(revision 359373)
+++ head/share/man/man9/bus_dma.9	(revision 359374)
@@ -1,1265 +1,1280 @@
 .\" Copyright (c) 2002, 2003 Hiten M. Pandya.
 .\" All rights reserved.
 .\"
 .\" Redistribution and use in source and binary forms, with or without
 .\" modification, are permitted provided that the following conditions
 .\" are met:
 .\" 1. Redistributions of source code must retain the above copyright
 .\"    notice, this list of conditions, and the following disclaimer,
 .\"    without modification, immediately at the beginning of the file.
 .\" 2. The name of the author may not be used to endorse or promote products
 .\"    derived from this software without specific prior written permission.
 .\"
 .\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 .\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 .\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 .\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR, CONTRIBUTORS OR THE
 .\" VOICES IN HITEN PANDYA'S HEAD BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 .\" SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
 .\" TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 .\" PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 .\" LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 .\" NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 .\" SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 .\"
 .\" Copyright (c) 1996, 1997, 1998, 2001 The NetBSD Foundation, Inc.
 .\" All rights reserved.
 .\"
 .\" This code is derived from software contributed to The NetBSD Foundation
 .\" by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
 .\" NASA Ames Research Center.
 .\"
 .\" Redistribution and use in source and binary forms, with or without
 .\" modification, are permitted provided that the following conditions
 .\" are met:
 .\" 1. Redistributions of source code must retain the above copyright
 .\"    notice, this list of conditions and the following disclaimer.
 .\" 2. Redistributions in binary form must reproduce the above copyright
 .\"    notice, this list of conditions and the following disclaimer in the
 .\"    documentation and/or other materials provided with the distribution.
 .\"
 .\" THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 .\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 .\" TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 .\" PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 .\" BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 .\" CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 .\" SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 .\" INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 .\" CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 .\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 .\" POSSIBILITY OF SUCH DAMAGE.
 .\"
 .\" $FreeBSD$
 .\" $NetBSD: bus_dma.9,v 1.25 2002/10/14 13:43:16 wiz Exp $
 .\"
-.Dd August 11, 2018
+.Dd March 27, 2020
 .Dt BUS_DMA 9
 .Os
 .Sh NAME
 .Nm bus_dma ,
 .Nm bus_dma_tag_create ,
 .Nm bus_dma_tag_destroy ,
 .Nm bus_dma_template_init ,
 .Nm bus_dma_template_tag ,
 .Nm bus_dma_template_clone ,
 .Nm bus_dmamap_create ,
 .Nm bus_dmamap_destroy ,
 .Nm bus_dmamap_load ,
 .Nm bus_dmamap_load_bio ,
 .Nm bus_dmamap_load_ccb ,
+.Nm bus_dmamap_load_crp ,
 .Nm bus_dmamap_load_mbuf ,
 .Nm bus_dmamap_load_mbuf_sg ,
 .Nm bus_dmamap_load_uio ,
 .Nm bus_dmamap_unload ,
 .Nm bus_dmamap_sync ,
 .Nm bus_dmamem_alloc ,
 .Nm bus_dmamem_free
 .Nd Bus and Machine Independent DMA Mapping Interface
 .Sh SYNOPSIS
 .In machine/bus.h
 .Ft int
 .Fn bus_dma_tag_create "bus_dma_tag_t parent" "bus_size_t alignment" \
 "bus_addr_t boundary" "bus_addr_t lowaddr" "bus_addr_t highaddr" \
 "bus_dma_filter_t *filtfunc" "void *filtfuncarg" "bus_size_t maxsize" \
 "int nsegments" "bus_size_t maxsegsz" "int flags" "bus_dma_lock_t *lockfunc" \
 "void *lockfuncarg" "bus_dma_tag_t *dmat"
 .Ft int
 .Fn bus_dma_tag_destroy "bus_dma_tag_t dmat"
 .Ft void
 .Fo bus_dma_template_init
 .Fa "bus_dma_template_t template"
 .Fa "bus_dma_tag_t parent"
 .Fc
 .Ft int
 .Fo bus_dma_template_tag
 .Fa "bus_dma_template_t template"
 .Fa "bus_dma_tag_t *dmat"
 .Fc
 .Ft void
 .Fo bus_dma_template_clone
 .Fa "bus_dma_template_t template"
 .Fa "bus_dma_tag_t dmat"
 .Fc
 .Ft int
 .Fn bus_dmamap_create "bus_dma_tag_t dmat" "int flags" "bus_dmamap_t *mapp"
 .Ft int
 .Fn bus_dmamap_destroy "bus_dma_tag_t dmat" "bus_dmamap_t map"
 .Ft int
 .Fn bus_dmamap_load "bus_dma_tag_t dmat" "bus_dmamap_t map" "void *buf" \
 "bus_size_t buflen" "bus_dmamap_callback_t *callback" "void *callback_arg" \
 "int flags"
 .Ft int
 .Fn bus_dmamap_load_bio "bus_dma_tag_t dmat" "bus_dmamap_t map" \
 "struct bio *bio" "bus_dmamap_callback_t *callback" "void *callback_arg" \
 "int flags"
 .Ft int
 .Fn bus_dmamap_load_ccb "bus_dma_tag_t dmat" "bus_dmamap_t map" \
 "union ccb *ccb" "bus_dmamap_callback_t *callback" "void *callback_arg" \
 "int flags"
 .Ft int
+.Fn bus_dmamap_load_crp "bus_dma_tag_t dmat" "bus_dmamap_t map" \
+"struct crypto *crp" "bus_dmamap_callback_t *callback" "void *callback_arg" \
+"int flags"
+.Ft int
 .Fn bus_dmamap_load_mbuf "bus_dma_tag_t dmat" "bus_dmamap_t map" \
 "struct mbuf *mbuf" "bus_dmamap_callback2_t *callback" "void *callback_arg" \
 "int flags"
 .Ft int
 .Fn bus_dmamap_load_mbuf_sg "bus_dma_tag_t dmat" "bus_dmamap_t map" \
 "struct mbuf *mbuf" "bus_dma_segment_t *segs" "int *nsegs" "int flags"
 .Ft int
 .Fn bus_dmamap_load_uio "bus_dma_tag_t dmat" "bus_dmamap_t map" \
 "struct uio *uio" "bus_dmamap_callback2_t *callback" "void *callback_arg" \
 "int flags"
 .Ft void
 .Fn bus_dmamap_unload "bus_dma_tag_t dmat" "bus_dmamap_t map"
 .Ft void
 .Fn bus_dmamap_sync "bus_dma_tag_t dmat" "bus_dmamap_t map" \
 "op"
 .Ft int
 .Fn bus_dmamem_alloc "bus_dma_tag_t dmat" "void **vaddr" \
 "int flags" "bus_dmamap_t *mapp"
 .Ft void
 .Fn bus_dmamem_free "bus_dma_tag_t dmat" "void *vaddr" \
 "bus_dmamap_t map"
 .Sh DESCRIPTION
 Direct Memory Access (DMA) is a method of transferring data
 without involving the CPU, thus providing higher performance.
 A DMA transaction can be achieved between device to memory,
 device to device, or memory to memory.
 .Pp
 The
 .Nm
 API is a bus, device, and machine-independent (MI) interface to
 DMA mechanisms.
 It provides the client with flexibility and simplicity by
 abstracting machine dependent issues like setting up
 DMA mappings, handling cache issues, bus specific features
 and limitations.
 .Sh OVERVIEW
 A tag structure
 .Vt ( bus_dma_tag_t )
 is used to describe the properties of a group of related DMA
 transactions.
 One way to view this is that a tag describes the limitations of a DMA engine.
 For example, if a DMA engine in a device is limited to 32-bit addresses,
 that limitation is specified by a parameter when creating the tag
 for that device.
 Similarly, a tag can be marked as requiring buffers whose addresses are
 aligned to a specific boundary.
 .Pp
 Some devices may require multiple tags to describe DMA
 transactions with differing properties.
 For example, a device might require 16-byte alignment of its descriptor ring
 while permitting arbitrary alignment of I/O buffers.
 In this case,
 the driver must create one tag for the descriptor ring and a separate tag for
 I/O buffers.
 If a device has restrictions that are common to all DMA transactions
 in addition to restrictions that differ between unrelated groups of
 transactions,
 the driver can first create a
 .Dq parent
 tag that decribes the common restrictions.
 The per-group tags can then inherit these restrictions from this
 .Dq parent
 tag rather than having to list them explicitly when creating the per-group tags.
 .Pp
 A mapping structure
 .Vt ( bus_dmamap_t )
 represents a mapping of a memory region for DMA.
 On systems with I/O MMUs,
 the mapping structure tracks any I/O MMU entries used by a request.
 For DMA requests that require bounce pages,
 the mapping tracks the bounce pages used.
 .Pp
 To prepare for one or more DMA transactions,
 a mapping must be bound to a memory region by calling one of the
 .Fn bus_dmamap_load
 functions.
 These functions configure the mapping which can include programming entries
 in an I/O MMU and/or allocating bounce pages.
 An output of these functions
 (either directly or indirectly by invoking a callback routine)
 is the list of scatter/gather address ranges a consumer can pass to a DMA
 engine to access the memory region.
 When a mapping is no longer needed,
 the mapping must be unloaded via
 .Fn bus_dmamap_unload .
 .Pp
 Before and after each DMA transaction,
 .Fn bus_dmamap_sync
 must be used to ensure that the correct data is used by the DMA engine and
 the CPU.
 If a mapping uses bounce pages,
 the sync operations copy data between the bounce pages and the memory region
 bound to the mapping.
 Sync operations also handle architecture-specific details such as CPU cache
 flushing and CPU memory operation ordering.
 .Sh STATIC VS DYNAMIC
 .Nm
 handles two types of DMA transactions: static and dynamic.
 Static transactions are used with a long-lived memory region that is reused
 for many transactions such as a descriptor ring.
 Dynamic transactions are used for transfers to or from transient buffers
 such as I/O buffers holding a network packet or disk block.
 Each transaction type uses a different subset of the
 .Nm
 API.
 .Ss Static Transactions
 Static transactions use memory regions allocated by
 .Nm .
 Each static memory region is allocated by calling
 .Fn bus_dmamem_alloc .
 This function requires a valid tag describing the properties of the
 DMA transactions to this region such as alignment or address restrictions.
 Multiple regions can share a single tag if they share the same restrictions.
 .Pp
 .Fn bus_dmamem_alloc
 allocates a memory region along with a mapping object.
 The associated tag, memory region, and mapping object must then be passed to
 .Fn bus_dmamap_load
 to bind the mapping to the allocated region and obtain the
 scatter/gather list.
 .Pp
 It is expected that
 .Fn bus_dmamem_alloc
 will attempt to allocate memory requiring less expensive sync operations
 (for example, implementations should not allocate regions requiring bounce
 pages),
 but sync operations should still be used.
 For example, a driver should use
 .Fn bus_dmamap_sync
 in an interrupt handler before reading descriptor ring entries written by the
 device prior to the interrupt.
 .Pp
 When a consumer is finished with a memory region,
 it should unload the mapping via
 .Fn bus_dmamap_unload
 and then release the memory region and mapping object via
 .Fn bus_dmamem_free .
 .Ss Dynamic Transactions
 Dynamic transactions map memory regions provided by other parts of the system.
 A tag must be created via
 .Fn bus_dma_tag_create
 to describe the DMA transactions to and from these memory regions,
 and a pool of mapping objects must be allocated via
 .Fn bus_dmamap_create
 to track the mappings of any in-flight transactions.
 .Pp
 When a consumer wishes to schedule a transaction for a memory region,
 the consumer must first obtain an unused mapping object from its pool
 of mapping objects.
 The memory region must be bound to the mapping object via one of the
 .Fn bus_dmamap_load
 functions.
 Before scheduling the transaction,
 the consumer should sync the memory region via
 .Fn bus_dmamap_sync
 with one or more of the
 .Dq PRE
 flags.
 After the transaction has completed,
 the consumer should sync the memory region via
 .Fn bus_dmamap_sync
 with one or more of the
 .Dq POST
 flags.
 The mapping can then be unloaded via
 .Fn bus_dmamap_unload ,
 and the mapping object can be returned to the pool of unused mapping objects.
 .Pp
 When a consumer is no longer scheduling DMA transactions,
 the mapping objects should be freed via
 .Fn bus_dmamap_destroy ,
 and the tag should be freed via
 .Fn bus_dma_tag_destroy .
 .Sh STRUCTURES AND TYPES
 .Bl -tag -width indent
 .It Vt bus_dma_tag_t
 A machine-dependent (MD) opaque type that describes the
 characteristics of a group of DMA transactions.
 DMA tags are organized into a hierarchy, with each child
 tag inheriting the restrictions of its parent.
 This allows all devices along the path of DMA transactions
 to contribute to the constraints of those transactions.
 .It Vt bus_dma_template_t
 A template structure for creating a
 .Fa bus_dma_tag_t
 from a set of defaults.
 Once initialized with
 .Fn bus_dma_template_init ,
 a driver can over-ride individual fields to suit its needs.
 The following fields have the indicated values:
 .Bd -literal
 	alignment	1
 	boundary	0
 	lowaddr		BUS_SPACE_MAXADDR
 	highaddr	BUS_SPACE_MAXADDR
 	maxsize		BUS_SPACE_MAXSIZE
 	nsegments	BUS_SPACE_UNRESTRICTED
 	maxsegsize	BUS_SPACE_MAXSIZE
 	flags		0
 	lockfunc	NULL
 	lockfuncarg	NULL
 .Ed
 .Pp
 Descriptions of each field are documented with
 .Fn bus_dma_tag_create .
 Note that the
 .Fa filtfunc
 and
 .Fa filtfuncarg
 attributes of the DMA tag are not supported with templates.
 .It Vt bus_dma_filter_t
 Client specified address filter having the format:
 .Bl -tag -width indent
 .It Ft int
 .Fn "client_filter" "void *filtarg" "bus_addr_t testaddr"
 .El
 .Pp
 Address filters can be specified during tag creation to allow
 for devices whose DMA address restrictions cannot be specified
 by a single window.
 The
 .Fa filtarg
 argument is specified by the client during tag creation to be passed to all
 invocations of the callback.
 The
 .Fa testaddr
 argument contains a potential starting address of a DMA mapping.
 The filter function operates on the set of addresses from
 .Fa testaddr
 to
 .Ql trunc_page(testaddr) + PAGE_SIZE - 1 ,
 inclusive.
 The filter function should return zero if any mapping in this range
 can be accommodated by the device and non-zero otherwise.
 .It Vt bus_dma_segment_t
 A machine-dependent type that describes individual
 DMA segments.
 It contains the following fields:
 .Bd -literal
 	bus_addr_t	ds_addr;
 	bus_size_t	ds_len;
 .Ed
 .Pp
 The
 .Fa ds_addr
 field contains the device visible address of the DMA segment, and
 .Fa ds_len
 contains the length of the DMA segment.
 Although the DMA segments returned by a mapping call will adhere to
 all restrictions necessary for a successful DMA operation, some conversion
 (e.g.\& a conversion from host byte order to the device's byte order) is
 almost always required when presenting segment information to the device.
 .It Vt bus_dmamap_t
 A machine-dependent opaque type describing an individual mapping.
 One map is used for each memory allocation that will be loaded.
 Maps can be reused once they have been unloaded.
 Multiple maps can be associated with one DMA tag.
 While the value of the map may evaluate to
 .Dv NULL
 on some platforms under certain conditions,
 it should never be assumed that it will be
 .Dv NULL
 in all cases.
 .It Vt bus_dmamap_callback_t
 Client specified callback for receiving mapping information resulting from
 the load of a
 .Vt bus_dmamap_t
 via
 .Fn bus_dmamap_load ,
-.Fn bus_dmamap_load_bio
+.Fn bus_dmamap_load_bio ,
+.Fn bus_dmamap_load_ccb ,
 or
-.Fn bus_dmamap_load_ccb .
+.Fn bus_dmamap_load_crp .
 Callbacks are of the format:
 .Bl -tag -width indent
 .It Ft void
 .Fn "client_callback" "void *callback_arg" "bus_dma_segment_t *segs" \
 "int nseg" "int error"
 .El
 .Pp
 The
 .Fa callback_arg
 is the callback argument passed to dmamap load functions.
 The
 .Fa segs
 and
 .Fa nseg
 arguments describe an array of
 .Vt bus_dma_segment_t
 structures that represent the mapping.
 This array is only valid within the scope of the callback function.
 The success or failure of the mapping is indicated by the
 .Fa error
 argument.
 More information on the use of callbacks can be found in the
 description of the individual dmamap load functions.
 .It Vt bus_dmamap_callback2_t
 Client specified callback for receiving mapping information resulting from
 the load of a
 .Vt bus_dmamap_t
 via
 .Fn bus_dmamap_load_uio
 or
 .Fn bus_dmamap_load_mbuf .
 .Pp
 Callback2s are of the format:
 .Bl -tag -width indent
 .It Ft void
 .Fn "client_callback2" "void *callback_arg" "bus_dma_segment_t *segs" \
 "int nseg" "bus_size_t mapsize" "int error"
 .El
 .Pp
 Callback2's behavior is the same as
 .Vt bus_dmamap_callback_t
 with the addition that the length of the data mapped is provided via
 .Fa mapsize .
 .It Vt bus_dmasync_op_t
 Memory synchronization operation specifier.
 Bus DMA requires explicit synchronization of memory with its device
 visible mapping in order to guarantee memory coherency.
 The
 .Vt bus_dmasync_op_t
 allows the type of DMA operation that will be or has been performed
 to be communicated to the system so that the correct coherency measures
 are taken.
 The operations are represented as bitfield flags that can be combined together,
 though it only makes sense to combine PRE flags or POST flags, not both.
 See the
 .Fn bus_dmamap_sync
 description below for more details on how to use these operations.
 .Pp
 All operations specified below are performed from the host memory point of view,
 where a read implies data coming from the device to the host memory, and a write
 implies data going from the host memory to the device.
 Alternatively, the operations can be thought of in terms of driver operations,
 where reading a network packet or storage sector corresponds to a read operation
 in
 .Nm .
 .Bl -tag -width ".Dv BUS_DMASYNC_POSTWRITE"
 .It Dv BUS_DMASYNC_PREREAD
 Perform any synchronization required prior to an update of host memory by the
 device.
 .It Dv BUS_DMASYNC_PREWRITE
 Perform any synchronization required after an update of host memory by the CPU
 and prior to device access to host memory.
 .It Dv BUS_DMASYNC_POSTREAD
 Perform any synchronization required after an update of host memory by the
 device and prior to CPU access to host memory.
 .It Dv BUS_DMASYNC_POSTWRITE
 Perform any synchronization required after device access to host memory.
 .El
 .It Vt bus_dma_lock_t
 Client specified lock/mutex manipulation method.
 This will be called from
 within busdma whenever a client lock needs to be manipulated.
 In its current form, the function will be called immediately before
 the callback for a DMA load operation that has been deferred with
 .Dv BUS_DMA_LOCK
 and immediately after with
 .Dv BUS_DMA_UNLOCK .
 If the load operation does not need to be deferred, then it
 will not be called since the function loading the map should
 be holding the appropriate locks.
 This method is of the format:
 .Bl -tag -width indent
 .It Ft void
 .Fn "lockfunc" "void *lockfunc_arg" "bus_dma_lock_op_t op"
 .El
 .Pp
 The
 .Fa lockfuncarg
 argument is specified by the client during tag creation to be passed to all
 invocations of the callback.
 The
 .Fa op
 argument specifies the lock operation to perform.
 .Pp
 Two
 .Vt lockfunc
 implementations are provided for convenience.
 .Fn busdma_lock_mutex
 performs standard mutex operations on the sleep mutex provided via
 .Fa lockfuncarg .
 .Fn dflt_lock
 will generate a system panic if it is called.
 It is substituted into the tag when
 .Fa lockfunc
 is passed as
 .Dv NULL
 to
 .Fn bus_dma_tag_create
 and is useful for tags that should not be used with deferred load operations.
 .It Vt bus_dma_lock_op_t
 Operations to be performed by the client-specified
 .Fn lockfunc .
 .Bl -tag -width ".Dv BUS_DMA_UNLOCK"
 .It Dv BUS_DMA_LOCK
 Acquires and/or locks the client locking primitive.
 .It Dv BUS_DMA_UNLOCK
 Releases and/or unlocks the client locking primitive.
 .El
 .El
 .Sh FUNCTIONS
 .Bl -tag -width indent
 .It Fn bus_dma_tag_create "parent" "alignment" "boundary" "lowaddr" \
 "highaddr" "*filtfunc" "*filtfuncarg" "maxsize" "nsegments" "maxsegsz" \
 "flags" "lockfunc" "lockfuncarg" "*dmat"
 Allocates a DMA tag, and initializes it according to
 the arguments provided:
 .Bl -tag -width ".Fa filtfuncarg"
 .It Fa parent
 A parent tag from which to inherit restrictions.
 The restrictions passed in other arguments can only further tighten the
 restrictions inherited from the parent tag.
 .Pp
 All tags created by a device driver must inherit from the tag returned by
 .Fn bus_get_dma_tag
 to honor restrictions between the parent bridge, CPU memory, and the
 device.
 .It Fa alignment
 Alignment constraint, in bytes, of any mappings created using this tag.
 The alignment must be a power of 2.
 Hardware that can DMA starting at any address would specify
 .Em 1
 for byte alignment.
 Hardware requiring DMA transfers to start on a multiple of 4K
 would specify
 .Em 4096 .
 .It Fa boundary
 Boundary constraint, in bytes, of the target DMA memory region.
 The boundary indicates the set of addresses, all multiples of the
 boundary argument, that cannot be crossed by a single
 .Vt bus_dma_segment_t .
 The boundary must be a power of 2 and must be no smaller than the
 maximum segment size.
 .Ql 0
 indicates that there are no boundary restrictions.
 .It Fa lowaddr , highaddr
 Bounds of the window of bus address space that
 .Em cannot
 be directly accessed by the device.
 The window contains all addresses greater than
 .Fa lowaddr
 and less than or equal to
 .Fa highaddr .
 For example, a device incapable of DMA above 4GB, would specify a
 .Fa highaddr
 of
 .Dv BUS_SPACE_MAXADDR
 and a
 .Fa lowaddr
 of
 .Dv BUS_SPACE_MAXADDR_32BIT .
 Similarly a device that can only perform DMA to addresses below
 16MB would specify a
 .Fa highaddr
 of
 .Dv BUS_SPACE_MAXADDR
 and a
 .Fa lowaddr
 of
 .Dv BUS_SPACE_MAXADDR_24BIT .
 Some implementations require that some region of device visible
 address space, overlapping available host memory, be outside the
 window.
 This area of
 .Ql safe memory
 is used to bounce requests that would otherwise conflict with
 the exclusion window.
 .It Fa filtfunc
 Optional filter function (may be
 .Dv NULL )
 to be called for any attempt to
 map memory into the window described by
 .Fa lowaddr
 and
 .Fa highaddr .
 A filter function is only required when the single window described
 by
 .Fa lowaddr
 and
 .Fa highaddr
 cannot adequately describe the constraints of the device.
 The filter function will be called for every machine page
 that overlaps the exclusion window.
 .It Fa filtfuncarg
 Argument passed to all calls to the filter function for this tag.
 May be
 .Dv NULL .
 .It Fa maxsize
 Maximum size, in bytes, of the sum of all segment lengths in a given
 DMA mapping associated with this tag.
 .It Fa nsegments
 Number of discontinuities (scatter/gather segments) allowed
 in a DMA mapped region.
 If there is no restriction,
 .Dv BUS_SPACE_UNRESTRICTED
 may be specified.
 .It Fa maxsegsz
 Maximum size, in bytes, of a segment in any DMA mapped region associated
 with
 .Fa dmat .
 .It Fa flags
 Are as follows:
 .Bl -tag -width ".Dv BUS_DMA_ALLOCNOW"
 .It Dv BUS_DMA_ALLOCNOW
 Pre-allocate enough resources to handle at least one map load operation on
 this tag.
 If sufficient resources are not available,
 .Er ENOMEM
 is returned.
 This should not be used for tags that only describe buffers that will be
 allocated with
 .Fn bus_dmamem_alloc .
 Also, due to resource sharing with other tags, this flag does not guarantee
 that resources will be allocated or reserved exclusively for this tag.
 It should be treated only as a minor optimization.
 .It Dv BUS_DMA_COHERENT
 Indicate that the DMA engine and CPU are cache-coherent.
 Cached memory may be used to back allocations created by
 .Fn bus_dmamem_alloc .
 For
 .Fn bus_dma_tag_create ,
 the
 .Dv BUS_DMA_COHERENT
 flag is currently implemented on arm64.
 .El
 .It Fa lockfunc
 Optional lock manipulation function (may be
 .Dv NULL )
 to be called when busdma
 needs to manipulate a lock on behalf of the client.
 If
 .Dv NULL
 is specified,
 .Fn dflt_lock
 is used.
 .It Fa lockfuncarg
 Optional argument to be passed to the function specified by
 .Fa lockfunc .
 .It Fa dmat
 Pointer to a bus_dma_tag_t where the resulting DMA tag will
 be stored.
 .El
 .Pp
 Returns
 .Er ENOMEM
 if sufficient memory is not available for tag creation
 or allocating mapping resources.
 .It Fn bus_dma_tag_destroy "dmat"
 Deallocate the DMA tag
 .Fa dmat
 that was created by
 .Fn bus_dma_tag_create .
 .Pp
 Returns
 .Er EBUSY
 if any DMA maps remain associated with
 .Fa dmat
 or
 .Ql 0
 on success.
 .It Fn bus_dma_template_init "*template" "parent"
 Initializes a
 .Fa bus_dma_template_t
 structure and associates it with an optional 
 .Fa parent .
 The
 .Fa parent
 argument may be NULL.
 .It Fn bus_dma_template_tag "*template" "*dmat"
 Unpacks a template into a tag, and returns the tag via the
 .Fa dmat .
 All return values are identical to
 .Fn bus_dma_tag_create .
 .It Fn bus_dma_template_clone "*template" "dmat"
 Clones the fields from a tag to a template.
 This is useful for cloning tags when paired with
 .Fn bus_dma_template_tag .
 A template that is filled in as a clone does not need to be initialized
 first.
 .It Fn bus_dmamap_create "dmat" "flags" "*mapp"
 Allocates and initializes a DMA map.
 Arguments are as follows:
 .Bl -tag -width ".Fa nsegments"
 .It Fa dmat
 DMA tag.
 .It Fa flags
 Are as follows:
 .Bl -tag -width ".Dv BUS_DMA_COHERENT"
 .It Dv BUS_DMA_COHERENT
 Attempt to map the memory loaded with this map such that cache sync
 operations are as cheap as possible.
 This flag is typically set on maps when the memory loaded with these will
 be accessed by both a CPU and a DMA engine, frequently such as control data
 and as opposed to streamable data such as receive and transmit buffers.
 Use of this flag does not remove the requirement of using
 .Fn bus_dmamap_sync ,
 but it may reduce the cost of performing these operations.
 .El
 .It Fa mapp
 Pointer to a
 .Vt bus_dmamap_t
 where the resulting DMA map will be stored.
 .El
 .Pp
 Returns
 .Er ENOMEM
 if sufficient memory is not available for creating the
 map or allocating mapping resources.
 .It Fn bus_dmamap_destroy "dmat" "map"
 Frees all resources associated with a given DMA map.
 Arguments are as follows:
 .Bl -tag -width ".Fa dmat"
 .It Fa dmat
 DMA tag used to allocate
 .Fa map .
 .It Fa map
 The DMA map to destroy.
 .El
 .Pp
 Returns
 .Er EBUSY
 if a mapping is still active for
 .Fa map .
 .It Fn bus_dmamap_load "dmat" "map" "buf" "buflen" "*callback" \
 "callback_arg" "flags"
 Creates a mapping in device visible address space of
 .Fa buflen
 bytes of
 .Fa buf ,
 associated with the DMA map
 .Fa map .
 This call will always return immediately and will not block for any reason.
 Arguments are as follows:
 .Bl -tag -width ".Fa buflen"
 .It Fa dmat
 DMA tag used to allocate
 .Fa map .
 .It Fa map
 A DMA map without a currently active mapping.
 .It Fa buf
 A kernel virtual address pointer to a contiguous (in KVA) buffer, to be
 mapped into device visible address space.
 .It Fa buflen
 The size of the buffer.
 .It Fa callback Fa callback_arg
 The callback function, and its argument.
 This function is called once sufficient mapping resources are available for
 the DMA operation.
 If resources are temporarily unavailable, this function will be deferred until
 later, but the load operation will still return immediately to the caller.
 Thus, callers should not assume that the callback will be called before the
 load returns, and code should be structured appropriately to handle this.
 See below for specific flags and error codes that control this behavior.
 .It Fa flags
 Are as follows:
 .Bl -tag -width ".Dv BUS_DMA_NOWAIT"
 .It Dv BUS_DMA_NOWAIT
 The load should not be deferred in case of insufficient mapping resources,
 and instead should return immediately with an appropriate error.
 .It Dv BUS_DMA_NOCACHE
 The generated transactions to and from the virtual page are non-cacheable.
 .El
 .El
 .Pp
 Return values to the caller are as follows:
 .Bl -tag -width ".Er EINPROGRESS"
 .It 0
 The callback has been called and completed.
 The status of the mapping has been delivered to the callback.
 .It Er EINPROGRESS
 The mapping has been deferred for lack of resources.
 The callback will be called as soon as resources are available.
 Callbacks are serviced in FIFO order.
 .Pp
 Note that subsequent load operations for the same tag that do not require
 extra resources will still succeed.
 This may result in out-of-order processing of requests.
 If the caller requires the order of requests to be preserved,
 then the caller is required to stall subsequent requests until a pending
 request's callback is invoked.
 .It Er ENOMEM
 The load request has failed due to insufficient resources, and the caller
 specifically used the
 .Dv BUS_DMA_NOWAIT
 flag.
 .It Er EINVAL
 The load request was invalid.
 The callback has been called and has been provided the same error.
 This error value may indicate that
 .Fa dmat ,
 .Fa map ,
 .Fa buf ,
 or
 .Fa callback
 were invalid, or
 .Fa buflen
 was larger than the
 .Fa maxsize
 argument used to create the dma tag
 .Fa dmat .
 .El
 .Pp
 When the callback is called, it is presented with an error value
 indicating the disposition of the mapping.
 Error may be one of the following:
 .Bl -tag -width ".Er EINPROGRESS"
 .It 0
 The mapping was successful and the
 .Fa dm_segs
 callback argument contains an array of
 .Vt bus_dma_segment_t
 elements describing the mapping.
 This array is only valid during the scope of the callback function.
 .It Er EFBIG
 A mapping could not be achieved within the segment constraints provided
 in the tag even though the requested allocation size was less than maxsize.
 .El
 .It Fn bus_dmamap_load_bio "dmat" "map" "bio" "callback" "callback_arg" "flags"
 This is a variation of
 .Fn bus_dmamap_load
 which maps buffers pointed to by
 .Fa bio
 for DMA transfers.
 .Fa bio
 may point to either a mapped or unmapped buffer.
 .It Fn bus_dmamap_load_ccb "dmat" "map" "ccb" "callback" "callback_arg" "flags"
 This is a variation of
 .Fn bus_dmamap_load
 which maps data pointed to by
 .Fa ccb
 for DMA transfers.
 The data for
 .Fa ccb
 may be any of the following types:
 .Bl -tag -width ".Er CAM_DATA_SG_PADDR"
 .It CAM_DATA_VADDR
 The data is a single KVA buffer.
 .It CAM_DATA_PADDR
 The data is a single bus address range.
 .It CAM_DATA_SG
 The data is a scatter/gather list of KVA buffers.
 .It CAM_DATA_SG_PADDR
 The data is a scatter/gather list of bus address ranges.
 .It CAM_DATA_BIO
 The data is contained in a
 .Vt struct bio
 attached to the CCB.
 .El
 .Pp
 .Fn bus_dmamap_load_ccb
 supports the following CCB XPT function codes:
 .Pp
 .Bl -item -offset indent -compact
 .It
 XPT_ATA_IO
 .It
 XPT_CONT_TARGET_IO
 .It
 XPT_SCSI_IO
 .El
+.It Fn bus_dmamap_load_crp "dmat" "map" "crp" "callback" "callback_arg" "flags"
+This is a variation of
+.Fn bus_dmamap_load
+which maps buffers pointed to by
+.Fa crp
+for DMA transfers.
+The
+.Dv BUS_DMA_NOWAIT
+flag is implied, thus no callback deferral will happen.
 .It Fn bus_dmamap_load_mbuf "dmat" "map" "mbuf" "callback2" "callback_arg" \
 "flags"
 This is a variation of
 .Fn bus_dmamap_load
 which maps mbuf chains
 for DMA transfers.
 A
 .Vt bus_size_t
 argument is also passed to the callback routine, which
 contains the mbuf chain's packet header length.
 The
 .Dv BUS_DMA_NOWAIT
 flag is implied, thus no callback deferral will happen.
 .Pp
 Mbuf chains are assumed to be in kernel virtual address space.
 .Pp
 Beside the error values listed for
 .Fn bus_dmamap_load ,
 .Er EINVAL
 will be returned if the size of the mbuf chain exceeds the maximum limit of the
 DMA tag.
 .It Fn bus_dmamap_load_mbuf_sg "dmat" "map" "mbuf" "segs" "nsegs" "flags"
 This is just like
 .Fn bus_dmamap_load_mbuf
 except that it returns immediately without calling a callback function.
 It is provided for efficiency.
 The scatter/gather segment array
 .Va segs
 is provided by the caller and filled in directly by the function.
 The
 .Va nsegs
 argument is returned with the number of segments filled in.
 Returns the same errors as
 .Fn bus_dmamap_load_mbuf .
 .It Fn bus_dmamap_load_uio "dmat" "map" "uio" "callback2" "callback_arg" "flags"
 This is a variation of
 .Fn bus_dmamap_load
 which maps buffers pointed to by
 .Fa uio
 for DMA transfers.
 A
 .Vt bus_size_t
 argument is also passed to the callback routine, which contains the size of
 .Fa uio ,
 i.e.
 .Fa uio->uio_resid .
 The
 .Dv BUS_DMA_NOWAIT
 flag is implied, thus no callback deferral will happen.
 Returns the same errors as
 .Fn bus_dmamap_load .
 .Pp
 If
 .Fa uio->uio_segflg
 is
 .Dv UIO_USERSPACE ,
 then it is assumed that the buffer,
 .Fa uio
 is in
 .Fa "uio->uio_td->td_proc" Ns 's
 address space.
 User space memory must be in-core and wired prior to attempting a map
 load operation.
 Pages may be locked using
 .Xr vslock 9 .
 .It Fn bus_dmamap_unload "dmat" "map"
 Unloads a DMA map.
 Arguments are as follows:
 .Bl -tag -width ".Fa dmam"
 .It Fa dmat
 DMA tag used to allocate
 .Fa map .
 .It Fa map
 The DMA map that is to be unloaded.
 .El
 .Pp
 .Fn bus_dmamap_unload
 will not perform any implicit synchronization of DMA buffers.
 This must be done explicitly by a call to
 .Fn bus_dmamap_sync
 prior to unloading the map.
 .It Fn bus_dmamap_sync "dmat" "map" "op"
 Performs synchronization of a device visible mapping with the CPU visible
 memory referenced by that mapping.
 Arguments are as follows:
 .Bl -tag -width ".Fa dmat"
 .It Fa dmat
 DMA tag used to allocate
 .Fa map .
 .It Fa map
 The DMA mapping to be synchronized.
 .It Fa op
 Type of synchronization operation to perform.
 See the definition of
 .Vt bus_dmasync_op_t
 for a description of the acceptable values for
 .Fa op .
 .El
 .Pp
 The
 .Fn bus_dmamap_sync
 function
 is the method used to ensure that CPU's and device's direct
 memory access (DMA) to shared
 memory is coherent.
 For example, the CPU might be used to set up the contents of a buffer
 that is to be made available to a device.
 To ensure that the data are visible via the device's mapping of that
 memory, the buffer must be loaded and a DMA sync operation of
 .Dv BUS_DMASYNC_PREWRITE
 must be performed after the CPU has updated the buffer and before the device
 access is initiated.
 If the CPU modifies this buffer again later, another
 .Dv BUS_DMASYNC_PREWRITE
 sync operation must be performed before an additional device
 access.
 Conversely, suppose a device updates memory that is to be read by a CPU.
 In this case, the buffer must be loaded, and a DMA sync operation of
 .Dv BUS_DMASYNC_PREREAD
 must be performed before the device access is initiated.
 The CPU will only be able to see the results of this memory update
 once the DMA operation has completed and a
 .Dv BUS_DMASYNC_POSTREAD
 sync operation has been performed.
 .Pp
 If read and write operations are not preceded and followed by the
 appropriate synchronization operations, behavior is undefined.
 .It Fn bus_dmamem_alloc "dmat" "**vaddr" "flags" "*mapp"
 Allocates memory that is mapped into KVA at the address returned
 in
 .Fa vaddr
 and that is permanently loaded into the newly created
 .Vt bus_dmamap_t
 returned via
 .Fa mapp .
 Arguments are as follows:
 .Bl -tag -width ".Fa alignment"
 .It Fa dmat
 DMA tag describing the constraints of the DMA mapping.
 .It Fa vaddr
 Pointer to a pointer that will hold the returned KVA mapping of
 the allocated region.
 .It Fa flags
 Flags are defined as follows:
 .Bl -tag -width ".Dv BUS_DMA_NOWAIT"
 .It Dv BUS_DMA_WAITOK
 The routine can safely wait (sleep) for resources.
 .It Dv BUS_DMA_NOWAIT
 The routine is not allowed to wait for resources.
 If resources are not available,
 .Dv ENOMEM
 is returned.
 .It Dv BUS_DMA_COHERENT
 Attempt to map this memory in a coherent fashion.
 See
 .Fn bus_dmamap_create
 above for a description of this flag.
 For
 .Fn bus_dmamem_alloc ,
 the
 .Dv BUS_DMA_COHERENT
 flag is currently implemented on arm and arm64.
 .It Dv BUS_DMA_ZERO
 Causes the allocated memory to be set to all zeros.
 .It Dv BUS_DMA_NOCACHE
 The allocated memory will not be cached in the processor caches.
 All memory accesses appear on the bus and are executed
 without reordering.
 For
 .Fn bus_dmamem_alloc ,
 the
 .Dv BUS_DMA_NOCACHE
 flag is currently implemented on amd64 and i386 where it results in the
 Strong Uncacheable PAT to be set for the allocated virtual address range.
 .El
 .It Fa mapp
 Pointer to a
 .Vt bus_dmamap_t
 where the resulting DMA map will be stored.
 .El
 .Pp
 The size of memory to be allocated is
 .Fa maxsize
 as specified in the call to
 .Fn bus_dma_tag_create
 for
 .Fa dmat .
 .Pp
 The current implementation of
 .Fn bus_dmamem_alloc
 will allocate all requests as a single segment.
 .Pp
 An initial load operation is required to obtain the bus address of the allocated
 memory, and an unload operation is required before freeing the memory, as
 described below in
 .Fn bus_dmamem_free .
 Maps are automatically handled by this function and should not be explicitly
 allocated or destroyed.
 .Pp
 Although an explicit load is not required for each access to the memory
 referenced by the returned map, the synchronization requirements
 as described in the
 .Fn bus_dmamap_sync
 section still apply and should be used to achieve portability on architectures
 without coherent buses.
 .Pp
 Returns
 .Er ENOMEM
 if sufficient memory is not available for completing
 the operation.
 .It Fn bus_dmamem_free "dmat" "*vaddr" "map"
 Frees memory previously allocated by
 .Fn bus_dmamem_alloc .
 Any mappings
 will be invalidated.
 Arguments are as follows:
 .Bl -tag -width ".Fa vaddr"
 .It Fa dmat
 DMA tag.
 .It Fa vaddr
 Kernel virtual address of the memory.
 .It Fa map
 DMA map to be invalidated.
 .El
 .El
 .Sh RETURN VALUES
 Behavior is undefined if invalid arguments are passed to
 any of the above functions.
 If sufficient resources cannot be allocated for a given
 transaction,
 .Er ENOMEM
 is returned.
 All
 routines that are not of type
 .Vt void
 will return 0 on success or an error
 code on failure as discussed above.
 .Pp
 All
 .Vt void
 routines will succeed if provided with valid arguments.
 .Sh LOCKING
 Two locking protocols are used by
 .Nm .
 The first is a private global lock that is used to synchronize access to the
 bounce buffer pool on the architectures that make use of them.
 This lock is strictly a leaf lock that is only used internally to
 .Nm
 and is not exposed to clients of the API.
 .Pp
 The second protocol involves protecting various resources stored in the tag.
 Since almost all
 .Nm
 operations are done through requests from the driver that created the tag,
 the most efficient way to protect the tag resources is through the lock that
 the driver uses.
 In cases where
 .Nm
 acts on its own without being called by the driver, the lock primitive
 specified in the tag is acquired and released automatically.
 An example of this is when the
 .Fn bus_dmamap_load
 callback function is called from a deferred context instead of the driver
 context.
 This means that certain
 .Nm
 functions must always be called with the same lock held that is specified in the
 tag.
 These functions include:
 .Pp
 .Bl -item -offset indent -compact
 .It
 .Fn bus_dmamap_load
 .It
 .Fn bus_dmamap_load_bio
 .It
 .Fn bus_dmamap_load_ccb
 .It
 .Fn bus_dmamap_load_mbuf
 .It
 .Fn bus_dmamap_load_mbuf_sg
 .It
 .Fn bus_dmamap_load_uio
 .It
 .Fn bus_dmamap_unload
 .It
 .Fn bus_dmamap_sync
 .El
 .Pp
 There is one exception to this rule.
 It is common practice to call some of these functions during driver start-up
 without any locks held.
 So long as there is a guarantee of no possible concurrent use of the tag by
 different threads during this operation, it is safe to not hold a lock for
 these functions.
 .Pp
 Certain
 .Nm
 operations should not be called with the driver lock held, either because
 they are already protected by an internal lock, or because they might sleep
 due to memory or resource allocation.
 The following functions must not be
 called with any non-sleepable locks held:
 .Pp
 .Bl -item -offset indent -compact
 .It
 .Fn bus_dma_tag_create
 .It
 .Fn bus_dmamap_create
 .It
 .Fn bus_dmamem_alloc
 .El
 .Pp
 All other functions do not have a locking protocol and can thus be
 called with or without any system or driver locks held.
 .Sh SEE ALSO
 .Xr devclass 9 ,
 .Xr device 9 ,
 .Xr driver 9 ,
 .Xr rman 9 ,
 .Xr vslock 9
 .Pp
 .Rs
 .%A "Jason R. Thorpe"
 .%T "A Machine-Independent DMA Framework for NetBSD"
 .%J "Proceedings of the Summer 1998 USENIX Technical Conference"
 .%Q "USENIX Association"
 .%D "June 1998"
 .Re
 .Sh HISTORY
 The
 .Nm
 interface first appeared in
 .Nx 1.3 .
 .Pp
 The
 .Nm
 API was adopted from
 .Nx
 for use in the CAM SCSI subsystem.
 The alterations to the original API were aimed to remove the need for
 a
 .Vt bus_dma_segment_t
 array stored in each
 .Vt bus_dmamap_t
 while allowing callers to queue up on scarce resources.
 .Sh AUTHORS
 The
 .Nm
 interface was designed and implemented by
 .An Jason R. Thorpe
 of the Numerical Aerospace Simulation Facility, NASA Ames Research Center.
 Additional input on the
 .Nm
 design was provided by
 .An -nosplit
 .An Chris Demetriou ,
 .An Charles Hannum ,
 .An Ross Harvey ,
 .An Matthew Jacob ,
 .An Jonathan Stone ,
 and
 .An Matt Thomas .
 .Pp
 The
 .Nm
 interface in
 .Fx
 benefits from the contributions of
 .An Justin T. Gibbs ,
 .An Peter Wemm ,
 .An Doug Rabson ,
 .An Matthew N. Dodd ,
 .An Sam Leffler ,
 .An Maxime Henrion ,
 .An Jake Burkholder ,
 .An Takahashi Yoshihiro ,
 .An Scott Long
 and many others.
 .Pp
 This manual page was written by
 .An Hiten M. Pandya
 and
 .An Justin T. Gibbs .
Index: head/share/man/man9/crypto.9
===================================================================
--- head/share/man/man9/crypto.9	(revision 359373)
+++ head/share/man/man9/crypto.9	(revision 359374)
@@ -1,756 +1,181 @@
 .\"	$OpenBSD: crypto.9,v 1.19 2002/07/16 06:31:57 angelos Exp $
 .\"
 .\" The author of this manual page is Angelos D. Keromytis (angelos@cis.upenn.edu)
 .\"
 .\" Copyright (c) 2000, 2001 Angelos D. Keromytis
 .\"
 .\" Permission to use, copy, and modify this software with or without fee
 .\" is hereby granted, provided that this entire notice is included in
 .\" all source code copies of any software which is or includes a copy or
 .\" modification of this software.
 .\"
 .\" THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR
 .\" IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY
 .\" REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE
 .\" MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR
 .\" PURPOSE.
 .\"
 .\" $FreeBSD$
 .\"
-.Dd December 17, 2019
+.Dd March 27, 2020
 .Dt CRYPTO 9
 .Os
 .Sh NAME
 .Nm crypto
 .Nd API for cryptographic services in the kernel
 .Sh SYNOPSIS
 .In opencrypto/cryptodev.h
-.Ft int32_t
-.Fn crypto_get_driverid "device_t dev" "size_t session_size" "int flags"
-.Ft int
-.Fn crypto_register "uint32_t driverid" "int alg" "uint16_t maxoplen" "uint32_t flags"
-.Ft int
-.Fn crypto_kregister "uint32_t driverid" "int kalg" "uint32_t flags"
-.Ft int
-.Fn crypto_unregister "uint32_t driverid" "int alg"
-.Ft int
-.Fn crypto_unregister_all "uint32_t driverid"
-.Ft void
-.Fn crypto_done "struct cryptop *crp"
-.Ft void
-.Fn crypto_kdone "struct cryptkop *krp"
-.Ft int
-.Fn crypto_find_driver "const char *match"
-.Ft int
-.Fn crypto_newsession "crypto_session_t *cses" "struct cryptoini *cri" "int crid"
-.Ft int
-.Fn crypto_freesession "crypto_session_t cses"
-.Ft int
-.Fn crypto_dispatch "struct cryptop *crp"
-.Ft int
-.Fn crypto_kdispatch "struct cryptkop *krp"
-.Ft int
-.Fn crypto_unblock "uint32_t driverid" "int what"
-.Ft "struct cryptop *"
-.Fn crypto_getreq "int num"
-.Ft void
-.Fn crypto_freereq "struct cryptop *crp"
-.Bd -literal
-#define	CRYPTO_SYMQ	0x1
-#define	CRYPTO_ASYMQ	0x2
-
-#define EALG_MAX_BLOCK_LEN      16
-
-struct cryptoini {
-	int                cri_alg;
-	int                cri_klen;
-	int                cri_mlen;
-	caddr_t            cri_key;
-	uint8_t            cri_iv[EALG_MAX_BLOCK_LEN];
-	struct cryptoini  *cri_next;
-};
-
-struct cryptodesc {
-	int                crd_skip;
-	int                crd_len;
-	int                crd_inject;
-	int                crd_flags;
-	struct cryptoini   CRD_INI;
-#define crd_iv          CRD_INI.cri_iv
-#define crd_key         CRD_INI.cri_key
-#define crd_alg         CRD_INI.cri_alg
-#define crd_klen        CRD_INI.cri_klen
-	struct cryptodesc *crd_next;
-};
-
-struct cryptop {
-	TAILQ_ENTRY(cryptop) crp_next;
-	crypto_session_t   crp_session;
-	int                crp_ilen;
-	int                crp_olen;
-	int                crp_etype;
-	int                crp_flags;
-	caddr_t            crp_buf;
-	caddr_t            crp_opaque;
-	struct cryptodesc *crp_desc;
-	int              (*crp_callback) (struct cryptop *);
-	caddr_t            crp_mac;
-};
-
-struct crparam {
-        caddr_t         crp_p;
-        u_int           crp_nbits;
-};
-
-#define CRK_MAXPARAM    8
-
-struct cryptkop {
-        TAILQ_ENTRY(cryptkop) krp_next;
-        u_int              krp_op;         /* ie. CRK_MOD_EXP or other */
-        u_int              krp_status;     /* return status */
-        u_short            krp_iparams;    /* # of input parameters */
-        u_short            krp_oparams;    /* # of output parameters */
-        uint32_t           krp_hid;
-        struct crparam     krp_param[CRK_MAXPARAM];
-        int               (*krp_callback)(struct cryptkop *);
-};
-.Ed
 .Sh DESCRIPTION
 .Nm
-is a framework for drivers of cryptographic hardware to register with
-the kernel so
-.Dq consumers
-(other kernel subsystems, and
-users through the
+is a framework for in-kernel cryptography.
+It permits in-kernel consumers to encrypt and decrypt data
+and also enables userland applications to use cryptographic hardware
+through the
 .Pa /dev/crypto
-device) are able to make use of it.
-Drivers register with the framework the algorithms they support,
-and provide entry points (functions) the framework may call to
-establish, use, and tear down sessions.
-Sessions are used to cache cryptographic information in a particular driver
-(or associated hardware), so initialization is not needed with every request.
-Consumers of cryptographic services pass a set of
-descriptors that instruct the framework (and the drivers registered
-with it) of the operations that should be applied on the data (more
-than one cryptographic operation can be requested).
+device.
 .Pp
-Keying operations are supported as well.
-Unlike the symmetric operators described above,
-these sessionless commands perform mathematical operations using
-input and output parameters.
+.Nm
+supports two modes of operation:
+one mode for symmetric-keyed cryptographic requests and digest,
+and a second mode for asymmetric-key requests and modular arithmetic.
+.Ss Symmetric-Key Mode
+Symmetric-key operations include encryption and decryption operations
+using block and stream ciphers as well as computation and verification
+of message authentication codes (MACs).
+In this mode,
+consumers allocate sessions to describe a transform as discussed in
+.Xr crypto_session 9 .
+Consumers then allocate request objects to describe each transformation
+such as encrypting a network packet or decrypting a disk sector.
+Requests are described in
+.Xr crypto_request 9 .
 .Pp
+Device drivers are responsible for processing requests submitted by
+consumers.
+.Xr crypto_driver 9
+describes the interfaces drivers use to register with the framework,
+helper routines the framework provides to faciliate request processing,
+and the interfaces drivers are required to provide.
+.Ss Asymmetric-Key Mode
+Assymteric-key operations do not use sessions.
+Instead,
+these operations perform individual mathematical operations using a set
+of input and output parameters.
+These operations are described in
+.Xr crypto_asym 9 .
+Drivers that support asymmetric operations use additional interfaces
+described in
+.Xr crypto_asym 9
+in addition to the base interfaces described in
+.Xr crypto_driver 9 .
+.Ss Callbacks
 Since the consumers may not be associated with a process, drivers may
 not
 .Xr sleep 9 .
 The same holds for the framework.
 Thus, a callback mechanism is used
 to notify a consumer that a request has been completed (the
 callback is specified by the consumer on a per-request basis).
 The callback is invoked by the framework whether the request was
 successfully completed or not.
-An error indication is provided in the latter case.
-A specific error code,
+Errors are reported to the callback function.
+.Pp
+Session initialization does not use callbacks and returns errors
+synchronously.
+.Ss Session Migration
+For symmetric-key operations,
+a specific error code,
 .Er EAGAIN ,
 is used to indicate that a session handle has changed and that the
 request may be re-submitted immediately with the new session.
-Errors are only returned to the invoking function if not
-enough information to call the callback is available (meaning, there
-was a fatal error in verifying the arguments).
-For session initialization and teardown no callback mechanism is used.
+The consumer should update its saved copy of the session handle
+to the value of
+.Fa crp_session
+so that future requests use the new session.
+.Ss Supported Algorithms
+More details on some algorithms may be found in
+.Xr crypto 7 .
+These algorithms are used for symmetric-mode operations.
+Asymmetric-mode operations support operations described in
+.Xr crypto_asym 9 .
 .Pp
-The
-.Fn crypto_find_driver
-returns the driver id of the device whose name matches
-.Fa match .
-.Fa match
-can either be the exact name of a device including the unit
-or the driver name without a unit.
-In the latter case,
-the id of the first device with the matching driver name is returned.
-If no matching device is found,
-the value -1 is returned.
+The following authentication algorithms are supported:
 .Pp
-The
-.Fn crypto_newsession
-routine is called by consumers of cryptographic services (such as the
-.Xr ipsec 4
-stack) that wish to establish a new session with the framework.
-The
-.Fa cri
-argument points to a
-.Vt cryptoini
-structure containing all the necessary information for
-the driver to establish the session.
-The
-.Fa crid
-argument is either a specific driver id or a bitmask of flags.
-The flags are
-.Dv CRYPTOCAP_F_HARDWARE ,
-to select hardware devices,
-or
-.Dv CRYPTOCAP_F_SOFTWARE ,
-to select software devices.
-If both are specified, hardware devices are preferred over software
-devices.
-On success, the opaque session handle of the new session will be stored in
-.Fa *cses .
-The
-.Vt cryptoini
-structure pointed to by
-.Fa cri
-contains these fields:
-.Bl -tag -width ".Va cri_next"
-.It Va cri_alg
-An algorithm identifier.
-Currently supported algorithms are:
-.Pp
-.Bl -tag -width ".Dv CRYPTO_RIPEMD160_HMAC" -compact
-.It Dv CRYPTO_AES_128_NIST_GMAC
-.It Dv CRYPTO_AES_192_NIST_GMAC
-.It Dv CRYPTO_AES_256_NIST_GMAC
-.It Dv CRYPTO_AES_CBC
-.It Dv CRYPTO_AES_CCM_16
+.Bl -tag -offset indent -width CRYPTO_AES_CCM_CBC_MAC -compact
 .It Dv CRYPTO_AES_CCM_CBC_MAC
-.It Dv CRYPTO_AES_ICM
-.It Dv CRYPTO_AES_NIST_GCM_16
 .It Dv CRYPTO_AES_NIST_GMAC
-.It Dv CRYPTO_AES_XTS
-.It Dv CRYPTO_ARC4
 .It Dv CRYPTO_BLAKE2B
 .It Dv CRYPTO_BLAKE2S
-.It Dv CRYPTO_BLF_CBC
-.It Dv CRYPTO_CAMELLIA_CBC
-.It Dv CRYPTO_CAST_CBC
-.It Dv CRYPTO_CHACHA20
-.It Dv CRYPTO_DEFLATE_COMP
-.It Dv CRYPTO_DES_CBC
-.It Dv CRYPTO_3DES_CBC
 .It Dv CRYPTO_MD5
 .It Dv CRYPTO_MD5_HMAC
 .It Dv CRYPTO_MD5_KPDK
 .It Dv CRYPTO_NULL_HMAC
-.It Dv CRYPTO_NULL_CBC
 .It Dv CRYPTO_POLY1305
 .It Dv CRYPTO_RIPEMD160
 .It Dv CRYPTO_RIPEMD160_HMAC
 .It Dv CRYPTO_SHA1
 .It Dv CRYPTO_SHA1_HMAC
 .It Dv CRYPTO_SHA1_KPDK
 .It Dv CRYPTO_SHA2_224
 .It Dv CRYPTO_SHA2_224_HMAC
 .It Dv CRYPTO_SHA2_256
 .It Dv CRYPTO_SHA2_256_HMAC
 .It Dv CRYPTO_SHA2_384
 .It Dv CRYPTO_SHA2_384_HMAC
 .It Dv CRYPTO_SHA2_512
 .It Dv CRYPTO_SHA2_512_HMAC
-.It Dv CRYPTO_SKIPJACK_CBC
 .El
-.It Va cri_klen
-For variable-size key algorithms, the length of the key in bits.
-.It Va cri_mlen
-If non-zero, truncate the calculated hash to this many bytes.
-.It Va cri_key
-The key to be used.
-.It Va cri_iv
-An explicit initialization vector if it does not prefix
-the data.
-This field is ignored during initialization
-.Pq Nm crypto_newsession .
-If no IV is explicitly passed (see below on details), a random IV is used
-by the device driver processing the request.
-.It Va cri_next
-Pointer to another
-.Vt cryptoini
-structure.
-This is used to establish dual-algorithm sessions, such as combining a
-cipher with a MAC.
-.El
 .Pp
-The
-.Vt cryptoini
-structure and its contents will not be modified or referenced by the
-framework or any cryptographic drivers.
-The memory associated with
-.Fa cri
-can be released once
-.Fn crypto_newsession
-returns.
+The following encryption algorithms are supported:
 .Pp
-.Fn crypto_freesession
-is called with the session handle returned by
-.Fn crypto_newsession
-to free the session.
-.Pp
-.Fn crypto_dispatch
-is called to process a request.
-The various fields in the
-.Vt cryptop
-structure are:
-.Bl -tag -width ".Va crp_callback"
-.It Va crp_session
-The session handle.
-.It Va crp_ilen
-The total length in bytes of the buffer to be processed.
-.It Va crp_olen
-On return, contains the total length of the result.
-For symmetric crypto operations, this will be the same as the input length.
-This will be used if the framework needs to allocate a new
-buffer for the result (or for re-formatting the input).
-.It Va crp_callback
-Callback routine invoked when a request is completed via
-.Fn crypto_done .
-The callback routine should inspect the
-.Va crp_etype
-to determine if the request was successfully completed.
-.It Va crp_etype
-The error type, if any errors were encountered, or zero if
-the request was successfully processed.
-If the
-.Er EAGAIN
-error code is returned, the session handle has changed (and has been recorded
-in the
-.Va crp_session
-field).
-The consumer should record the new session handle and use it in all subsequent
-requests.
-In this case, the request may be re-submitted immediately.
-This mechanism is used by the framework to perform
-session migration (move a session from one driver to another, because
-of availability, performance, or other considerations).
-.Pp
-This field is only valid in the context of the callback routine specified by
-.Va crp_callback .
-Errors are returned to the invoker of
-.Fn crypto_process
-only when enough information is not present to call the callback
-routine (i.e., if the pointer passed is
-.Dv NULL
-or if no callback routine was specified).
-.It Va crp_flags
-A bitmask of flags associated with this request.
-Currently defined flags are:
-.Bl -tag -width ".Dv CRYPTO_F_CBIFSYNC"
-.It Dv CRYPTO_F_IMBUF
-The buffer is an mbuf chain pointed to by
-.Va crp_mbuf .
-.It Dv CRYPTO_F_IOV
-The buffer is a
-.Vt uio
-structure pointed to by
-.Va crp_uio .
-.It Dv CRYPTO_F_BATCH
-Batch operation if possible.
-.It Dv CRYPTO_F_CBIMM
-Do callback immediately instead of doing it from a dedicated kernel thread.
-.It Dv CRYPTO_F_DONE
-Operation completed.
-.It Dv CRYPTO_F_CBIFSYNC
-Do callback immediately if operation is synchronous (that the driver
-specified the
-.Dv CRYPTOCAP_F_SYNC
-flag).
-.It Dv CRYPTO_F_ASYNC
-Try to do the crypto operation in a pool of workers
-if the operation is synchronous (that is, if the driver specified the
-.Dv CRYPTOCAP_F_SYNC
-flag).
-It aims to speed up processing by dispatching crypto operations
-on different processors.
-.It Dv CRYPTO_F_ASYNC_KEEPORDER
-Dispatch callbacks in the same order they are posted.
-Only relevant if the
-.Dv CRYPTO_F_ASYNC
-flag is set and if the operation is synchronous.
+.Bl -tag -offset indent -width CRYPTO_CAMELLIA_CBC -compact
+.It Dv CRYPTO_AES_CBC
+.It Dv CRYPTO_AES_ICM
+.It Dv CRYPTO_AES_XTS
+.It Dv CRYPTO_ARC4
+.It Dv CRYPTO_BLF_CBC
+.It Dv CRYPTO_CAMELLIA_CBC
+.It Dv CRYPTO_CAST_CBC
+.It Dv CRYPTO_CHACHA20
+.It Dv CRYPTO_DES_CBC
+.It Dv CRYPTO_3DES_CBC
+.It Dv CRYPTO_NULL_CBC
+.It Dv CRYPTO_SKIPJACK_CBC
 .El
-.It Va crp_buf
-Data buffer unless
-.Dv CRYPTO_F_IMBUF
-or
-.Dv CRYPTO_F_IOV
-is set in
-.Va crp_flags .
-The length in bytes is set in
-.Va crp_ilen .
-.It Va crp_mbuf
-Data buffer mbuf chain when
-.Dv CRYPTO_F_IMBUF
-is set in
-.Va crp_flags .
-.It Va crp_uio
-.Vt struct uio
-data buffer when
-.Dv CRYPTO_F_IOV
-is set in
-.Va crp_flags .
-.It Va crp_opaque
-Cookie passed through the crypto framework untouched.
-It is
-intended for the invoking application's use.
-.It Va crp_desc
-A linked list of descriptors.
-Each descriptor provides
-information about what type of cryptographic operation should be done
-on the input buffer.
-The various fields are:
-.Bl -tag -width ".Va crd_inject"
-.It Va crd_iv
-When the flag
-.Dv CRD_F_IV_EXPLICIT
-is set, this field contains the IV.
-.It Va crd_key
-When the
-.Dv CRD_F_KEY_EXPLICIT
-flag is set, the
-.Va crd_key
-points to a buffer with encryption or authentication key.
-.It Va crd_alg
-An algorithm to use.
-Must be the same as the one given at newsession time.
-.It Va crd_klen
-The
-.Va crd_key
-key length.
-.It Va crd_skip
-The offset in the input buffer where processing should start.
-.It Va crd_len
-How many bytes, after
-.Va crd_skip ,
-should be processed.
-.It Va crd_inject
-The
-.Va crd_inject
-field specifies an offset in bytes from the beginning of the buffer.
-For encryption algorithms, this may be where the IV will be inserted
-when encrypting or where the IV may be found for
-decryption (subject to
-.Va crd_flags ) .
-For MAC algorithms, this is where the result of the keyed hash will be
-inserted.
-.It Va crd_flags
-The following flags are defined:
-.Bl -tag -width 3n
-.It Dv CRD_F_ENCRYPT
-For encryption algorithms, this bit is set when encryption is required
-(when not set, decryption is performed).
-.It Dv CRD_F_IV_PRESENT
-.\" This flag name has nothing to do w/ it's behavior, fix the name.
-For encryption, if this bit is not set the IV used to encrypt the packet
-will be written at the location pointed to by
-.Va crd_inject .
-The IV length is assumed to be equal to the blocksize of the
-encryption algorithm.
-For encryption, if this bit is set, nothing is done.
-For decryption, this flag has no meaning.
-Applications that do special
-.Dq "IV cooking" ,
-such as the half-IV mode in
-.Xr ipsec 4 ,
-can use this flag to indicate that the IV should not be written on the packet.
-This flag is typically used in conjunction with the
-.Dv CRD_F_IV_EXPLICIT
-flag.
-.It Dv CRD_F_IV_EXPLICIT
-This bit is set when the IV is explicitly
-provided by the consumer in the
-.Va crd_iv
-field.
-Otherwise, for encryption operations the IV is provided for by
-the driver used to perform the operation, whereas for decryption
-operations the offset of the IV is provided by the
-.Va crd_inject
-field.
-This flag is typically used when the IV is calculated
-.Dq "on the fly"
-by the consumer, and does not precede the data.
-.It Dv CRD_F_KEY_EXPLICIT
-For encryption and authentication (MAC) algorithms, this bit is set when the key
-is explicitly provided by the consumer in the
-.Va crd_key
-field for the given operation.
-Otherwise, the key is taken at newsession time from the
-.Va cri_key
-field.
-As calculating the key schedule may take a while, it is recommended that often
-used keys are given their own session.
-.It Dv CRD_F_COMP
-For compression algorithms, this bit is set when compression is required (when
-not set, decompression is performed).
-.El
-.It Va CRD_INI
-This
-.Vt cryptoini
-structure will not be modified by the framework or the device drivers.
-Since this information accompanies every cryptographic
-operation request, drivers may re-initialize state on-demand
-(typically an expensive operation).
-Furthermore, the cryptographic
-framework may re-route requests as a result of full queues or hardware
-failure, as described above.
-.It Va crd_next
-Point to the next descriptor.
-Linked operations are useful in protocols such as
-.Xr ipsec 4 ,
-where multiple cryptographic transforms may be applied on the same
-block of data.
-.El
-.El
 .Pp
-.Fn crypto_getreq
-allocates a
-.Vt cryptop
-structure with a linked list of
-.Fa num
-.Vt cryptodesc
-structures.
+The following authenticated encryption with additional data (AEAD)
+algorithms are supported:
 .Pp
-.Fn crypto_freereq
-deallocates a structure
-.Vt cryptop
-and any
-.Vt cryptodesc
-structures linked to it.
-Note that it is the responsibility of the
-callback routine to do the necessary cleanups associated with the
-opaque field in the
-.Vt cryptop
-structure.
-.Pp
-.Fn crypto_kdispatch
-is called to perform a keying operation.
-The various fields in the
-.Vt cryptkop
-structure are:
-.Bl -tag -width ".Va krp_callback"
-.It Va krp_op
-Operation code, such as
-.Dv CRK_MOD_EXP .
-.It Va krp_status
-Return code.
-This
-.Va errno Ns -style
-variable indicates whether lower level reasons
-for operation failure.
-.It Va krp_iparams
-Number of input parameters to the specified operation.
-Note that each operation has a (typically hardwired) number of such parameters.
-.It Va krp_oparams
-Number of output parameters from the specified operation.
-Note that each operation has a (typically hardwired) number of such parameters.
-.It Va krp_kvp
-An array of kernel memory blocks containing the parameters.
-.It Va krp_hid
-Identifier specifying which low-level driver is being used.
-.It Va krp_callback
-Callback called on completion of a keying operation.
+.Bl -tag -offset indent -width CRYPTO_AES_NIST_GCM_16 -compact
+.It Dv CRYPTO_AES_CCM_16
+.It Dv CRYPTO_AES_NIST_GCM_16
 .El
-.Sh DRIVER-SIDE API
-The
-.Fn crypto_get_driverid ,
-.Fn crypto_get_driver_session ,
-.Fn crypto_register ,
-.Fn crypto_kregister ,
-.Fn crypto_unregister ,
-.Fn crypto_unblock ,
-and
-.Fn crypto_done
-routines are used by drivers that provide support for cryptographic
-primitives to register and unregister with the kernel crypto services
-framework.
 .Pp
-Drivers must first use the
-.Fn crypto_get_driverid
-function to acquire a driver identifier, specifying the
-.Fa flags
-as an argument.
-One of
-.Dv CRYPTOCAP_F_SOFTWARE
-or
-.Dv CRYPTOCAP_F_HARDWARE
-must be specified.
-The
-.Dv CRYPTOCAP_F_SYNC
-may also be specified, and should be specified if the driver does all of
-it's operations synchronously.
-Drivers must pass the size of their session structure as the second argument.
-An appropriately sized memory will be allocated by the framework, zeroed, and
-passed to the driver's
-.Fn newsession
-method.
+The following compression algorithms are supported:
 .Pp
-For each algorithm the driver supports, it must then call
-.Fn crypto_register .
-The first two arguments are the driver and algorithm identifiers.
-The next two arguments specify the largest possible operator length (in bits,
-important for public key operations) and flags for this algorithm.
-.Pp
-.Fn crypto_unregister
-is called by drivers that wish to withdraw support for an algorithm.
-The two arguments are the driver and algorithm identifiers, respectively.
-Typically, drivers for
-PCMCIA
-crypto cards that are being ejected will invoke this routine for all
-algorithms supported by the card.
-.Fn crypto_unregister_all
-will unregister all algorithms registered by a driver
-and the driver will be disabled (no new sessions will be allocated on
-that driver, and any existing sessions will be migrated to other
-drivers).
-The same will be done if all algorithms associated with a driver are
-unregistered one by one.
-After a call to
-.Fn crypto_unregister_all
-there will be no threads in either the newsession or freesession function
-of the driver.
-.Pp
-The calling convention for the driver-supplied routines are:
-.Pp
-.Bl -item -compact
-.It
-.Ft int
-.Fn \*[lp]*newsession\*[rp] "device_t" "crypto_session_t" "struct cryptoini *" ;
-.It
-.Ft void
-.Fn \*[lp]*freesession\*[rp] "device_t" "crypto_session_t" ;
-.It
-.Ft int
-.Fn \*[lp]*process\*[rp] "device_t" "struct cryptop *" "int" ;
-.It
-.Ft int
-.Fn \*[lp]*kprocess\*[rp] "device_t" "struct cryptkop *" "int" ;
+.Bl -tag -offset indent -width CRYPTO_DEFLATE_COMP -compact
+.It Dv CRYPTO_DEFLATE_COMP
 .El
-.Pp
-On invocation, the first argument to
-all routines is the
-.Fa device_t
-that was provided to
-.Fn crypto_get_driverid .
-The second argument to
-.Fn newsession
-is the opaque session handle for the new session.
-The third argument is identical to that of
-.Fn crypto_newsession .
-.Pp
-Drivers obtain a pointer to their session memory by invoking
-.Fn crypto_get_driver_session
-on the opaque
-.Vt crypto_session_t
-handle.
-.Pp
-The
-.Fn freesession
-routine takes as arguments the opaque data value and the session handle.
-It should clear any context associated with the session (clear hardware
-registers, memory, etc.).
-If no resources need to be released other than the contents of session memory,
-the method is optional.
-The
-.Nm
-framework will zero and release the allocated session memory (after running the
-.Fn freesession
-method, if one exists).
-.Pp
-The
-.Fn process
-routine is invoked with a request to perform crypto processing.
-This routine must not block or sleep, but should queue the request and return
-immediately or process the request to completion.
-In case of an unrecoverable error, the error indication must be placed in the
-.Va crp_etype
-field of the
-.Vt cryptop
-structure.
-When the request is completed, or an error is detected, the
-.Fn process
-routine must invoke
-.Fn crypto_done .
-Session migration may be performed, as mentioned previously.
-.Pp
-In case of a temporary resource exhaustion, the
-.Fn process
-routine may return
-.Er ERESTART
-in which case the crypto services will requeue the request, mark the driver
-as
-.Dq blocked ,
-and stop submitting requests for processing.
-The driver is then responsible for notifying the crypto services
-when it is again able to process requests through the
-.Fn crypto_unblock
-routine.
-This simple flow control mechanism should only be used for short-lived
-resource exhaustion as it causes operations to be queued in the crypto
-layer.
-Doing so is preferable to returning an error in such cases as
-it can cause network protocols to degrade performance by treating the
-failure much like a lost packet.
-.Pp
-The
-.Fn kprocess
-routine is invoked with a request to perform crypto key processing.
-This routine must not block, but should queue the request and return
-immediately.
-Upon processing the request, the callback routine should be invoked.
-In case of an unrecoverable error, the error indication must be placed in the
-.Va krp_status
-field of the
-.Vt cryptkop
-structure.
-When the request is completed, or an error is detected, the
-.Fn kprocess
-routine should invoked
-.Fn crypto_kdone .
-.Sh RETURN VALUES
-.Fn crypto_register ,
-.Fn crypto_kregister ,
-.Fn crypto_unregister ,
-.Fn crypto_newsession ,
-.Fn crypto_freesession ,
-and
-.Fn crypto_unblock
-return 0 on success, or an error code on failure.
-.Fn crypto_get_driverid
-returns a non-negative value on error, and \-1 on failure.
-.Fn crypto_getreq
-returns a pointer to a
-.Vt cryptop
-structure and
-.Dv NULL
-on failure.
-.Fn crypto_dispatch
-returns
-.Er EINVAL
-if its argument or the callback function was
-.Dv NULL ,
-and 0 otherwise.
-The callback is provided with an error code in case of failure, in the
-.Va crp_etype
-field.
 .Sh FILES
 .Bl -tag -width ".Pa sys/opencrypto/crypto.c"
 .It Pa sys/opencrypto/crypto.c
 most of the framework code
 .El
 .Sh SEE ALSO
 .Xr crypto 4 ,
 .Xr ipsec 4 ,
 .Xr crypto 7 ,
-.Xr malloc 9 ,
+.Xr crypto_asym 9 ,
+.Xr crypto_driver 9 ,
+.Xr crypto_request 9 ,
+.Xr crypto_session 9 ,
 .Xr sleep 9
 .Sh HISTORY
 The cryptographic framework first appeared in
 .Ox 2.7
 and was written by
 .An Angelos D. Keromytis Aq Mt angelos@openbsd.org .
 .Sh BUGS
-The framework currently assumes that all the algorithms in a
-.Fn crypto_newsession
-operation must be available by the same driver.
-If that is not the case, session initialization will fail.
-.Pp
-The framework also needs a mechanism for determining which driver is
+The framework needs a mechanism for determining which driver is
 best for a specific set of algorithms associated with a session.
 Some type of benchmarking is in order here.
-.Pp
-Multiple instances of the same algorithm in the same session are not
-supported.
Index: head/share/man/man9/crypto_asym.9
===================================================================
--- head/share/man/man9/crypto_asym.9	(nonexistent)
+++ head/share/man/man9/crypto_asym.9	(revision 359374)
@@ -0,0 +1,178 @@
+.\" Copyright (c) 2020, Chelsio Inc
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions are met:
+.\"
+.\" 1. Redistributions of source code must retain the above copyright notice,
+.\"    this list of conditions and the following disclaimer.
+.\"
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" 3. Neither the name of the Chelsio Inc nor the names of its
+.\"    contributors may be used to endorse or promote products derived from
+.\"    this software without specific prior written permission.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+.\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+.\" LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+.\" CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+.\" SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+.\" INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+.\" CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+.\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+.\" POSSIBILITY OF SUCH DAMAGE.
+.\"
+.\" * Other names and brands may be claimed as the property of others.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd March 27, 2020
+.Dt CRYPTO_ASYM 9
+.Os
+.Sh NAME
+.Nm crypto_asym
+.Nd asymmetric cryptographic operations
+.Sh SYNOPSIS
+.In opencrypto/cryptodev.h
+.Ft int
+.Fn crypto_kdispatch "struct cryptkop *krp"
+.Ft void
+.Fn crypto_kdone "struct cryptkop *krp"
+.Ft int
+.Fn crypto_kregister "uint32_t driverid" "int kalg" "uint32_t flags"
+.Ft int
+.Fn CRYPTODEV_KPROCESS "device_t dev" "struct cryptop *krp" "int flags"
+.Sh DESCRIPTION
+The in-kernel cryptographic kernel framework supports asymmetric
+requests (keying requests) in addition to symmetric operations.
+There are currently no in-kernel users of these requests,
+but applications can make requests of hardware drivers via the
+.Pa /dev/crypto
+device .
+.Pp
+Some APIs are shared with the framework's symmetric request support.
+This manual describes the APIs and data structures unique to
+asymmetric requests.
+.Pp
+.Ss Request Objects
+A request is described by a
+.Vt struct cryptkop
+containing the following fields:
+.Bl -tag -width "krp_callback"
+.It Fa krp_op
+Operation to perform.
+Available operations include
+.Dv CRK_MOD_EXP ,
+.Dv CRK_MOD_EXP_CRT ,
+.Dv CRK_DSA_SIGN ,
+.Dv CRK_DSA_VERIFY ,
+and
+.Dv CRK_DH_COMPUTE_KEY .
+.It Fa krp_status
+Error status.
+Either zero on success,
+or an error if an operation fails.
+Set by drivers prior to completing a request via
+.Fn crypto_kdone .
+.It Fa krp_iparams
+Count of input parameters.
+.It Fa krp_oparams
+Count of output parameters.
+.It Fa krp_crid
+Requested device.
+.It Fa krp_hid
+Device used to complete the request.
+.It Fa krp_param
+Array of parameters.
+The array contains the input parameters first followed by the output
+parameters.
+Each parameter is stored as a bignum.
+Each bignum is described by a
+.Vt struct crparam
+containing the following fields:
+.Bl -tag -width "crp_nbits"
+.It Fa crp_p
+Pointer to array of packed bytes.
+.It Fa crp_nbits
+Size of bignum in bits.
+.El
+.It Fa krp_callback
+Callback function.
+This must point to a callback function of type
+.Vt void (*)(struct cryptkop *) .
+The callback function should inspect
+.Fa krp_status
+to determine the status of the completed operation.
+.El
+.Pp
+New requests should be initialized to zero before setting fields to
+appropriate values.
+Once the request has been populated,
+it should be passed to
+.Fn crypto_kdispatch .
+.Pp
+.Fn crypto_kdispatch
+will choose a device driver to perform the operation described by
+.Fa krp
+and invoke that driver's
+.Fn CRYPTO_KPROCESS
+method.
+.Ss Driver API
+Drivers register support for asymmetric operations by calling
+.Fn crypto_kregister
+for each supported algorithm.
+.Fa driverid
+should be the value returned by an earlier call to
+.Fn crypto_get_driverid .
+.Fa kalg
+should list one of the operations that can be set in
+.Fa krp_op .
+.Fa flags
+is a bitmask of zero or more of the following values:
+.Bl -tag -width "CRYPTO_ALG_FLAG_RNG_ENABLE"
+.It Dv CRYPTO_ALG_FLAG_RNG_ENABLE
+Device has a hardware RNG for DH/DSA.
+.It Dv CRYPTO_ALG_FLAG_DSA_SHA
+Device can compute a SHA digest of a message.
+.El
+.Pp
+Drivers unregister with the framework via
+.Fn crypto_unregister_all .
+.Pp
+Similar to
+.Fn CRYPTO_PROCESS ,
+.Fn CRYPTO_KPROCESS
+should complete the request or schedule it for asynchronous
+completion.
+If this method is not able to complete a request due to insufficient
+resources,
+it can defer the request (and future asymmetric requests) by returning
+.Dv ERESTART .
+Once resources are available,
+the driver should invoke
+.Fn crypto_unblock
+with
+.Dv CRYPTO_ASYMQ
+to resume processing of asymmetric requests.
+.Pp
+Once a request is completed,
+the driver should set
+.Fa krp_status
+and then call
+.Fn crypto_kdone .
+.Sh RETURN VALUES
+.Fn crypto_kdispatch ,
+.Fn crypto_kregister ,
+and
+.Fn CRYPTODEV_KPROCESS
+return zero on success or an error on failure.
+.Sh SEE ALSO
+.Xr crypto 7 ,
+.Xr crypto 9 ,
+.Xr crypto_driver 9 ,
+.Xr crypto_request 9 ,
+.Xr crypto_session 9

Property changes on: head/share/man/man9/crypto_asym.9
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Added: svn:keywords
## -0,0 +1 ##
+FreeBSD=%H
\ No newline at end of property
Added: svn:mime-type
## -0,0 +1 ##
+text/plain
\ No newline at end of property
Index: head/share/man/man9/crypto_driver.9
===================================================================
--- head/share/man/man9/crypto_driver.9	(nonexistent)
+++ head/share/man/man9/crypto_driver.9	(revision 359374)
@@ -0,0 +1,392 @@
+.\" Copyright (c) 2020, Chelsio Inc
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions are met:
+.\"
+.\" 1. Redistributions of source code must retain the above copyright notice,
+.\"    this list of conditions and the following disclaimer.
+.\"
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" 3. Neither the name of the Chelsio Inc nor the names of its
+.\"    contributors may be used to endorse or promote products derived from
+.\"    this software without specific prior written permission.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+.\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+.\" LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+.\" CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+.\" SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+.\" INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+.\" CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+.\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+.\" POSSIBILITY OF SUCH DAMAGE.
+.\"
+.\" * Other names and brands may be claimed as the property of others.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd March 27, 2020
+.Dt CRYPTO_DRIVER 9
+.Os
+.Sh NAME
+.Nm crypto_driver
+.Nd interface for symmetric cryptographic drivers
+.Sh SYNOPSIS
+.In opencrypto/cryptodev.h
+.Ft int
+.Fo crypto_apply
+.Fa "struct cryptop *crp"
+.Fa "int off"
+.Fa "int len"
+.Fa "int (*f)(void *, void *, u_int)"
+.Fa "void *arg"
+.Fc
+.Ft void *
+.Fo crypto_contiguous_subsegment
+.Fa "struct cryptop *crp"
+.Fa "size_t skip"
+.Fa "size_t len"
+.Fc
+.Ft void
+.Fn crypto_copyback "struct cryptop *crp" "int off" "int size" "const void *src"
+.Ft void
+.Fn crypto_copydata "struct cryptop *crp" "int off" "int size" "void *dst"
+.Ft void
+.Fn crypto_done "struct cryptop *crp"
+.Ft int32_t
+.Fn crypto_get_driverid "device_t dev" "size_t session_size" "int flags"
+.Ft void *
+.Fn crypto_get_driver_session "crypto_session_t crypto_session"
+.Ft int
+.Fn crypto_unblock "uint32_t driverid" "int what"
+.Ft int
+.Fn crypto_unregister_all "uint32_t driverid"
+.Ft int
+.Fn CRYPTODEV_FREESESSION "device_t dev" "crypto_session_t crypto_session"
+.Ft int
+.Fo CRYPTODEV_NEWSESSION
+.Fa "device_t dev"
+.Fa "crypto_session_t crypto_session"
+.Fa "const struct crypto_session_params *csp"
+.Fc
+.Ft int
+.Fo CRYPTODEV_PROBESESSION
+.Fa "device_t dev"
+.Fa "const struct crypto_session_params *csp"
+.Fc
+.Ft int
+.Fn CRYPTODEV_PROCESS "device_t dev" "struct cryptop *crp" "int flags"
+.Ft void
+.Fo hmac_init_ipad
+.Fa "struct auth_hash *axf"
+.Fa "const char *key"
+.Fa "int klen"
+.Fa "void *auth_ctx"
+.Fc
+.Ft void
+.Fo hmac_init_opad
+.Fa "struct auth_hash *axf"
+.Fa "const char *key"
+.Fa "int klen"
+.Fa "void *auth_ctx"
+.Fc
+.Sh DESCRIPTION
+Symmetric cryptographic drivers process cryptographic requests
+submitted to sessions associated with the driver.
+.Pp
+Cryptographic drivers call
+.Fn crypto_get_driverid
+to register with the cryptographic framework.
+.Fa dev
+is the device used to service requests.
+The
+.Fn CRYPTODEV
+methods are defined in the method table for the device driver attached to
+.Fa dev .
+.Fa session_size
+specifies the size of a driver-specific per-session structure allocated by
+the cryptographic framework.
+.Fa flags
+is a bitmask of properties about the driver.
+Exactly one of
+.Dv CRYPTOCAP_F_SOFTWARE
+or
+.Dv CRYPTOCAP_F_HARDWARE
+must be specified.
+.Dv CRYPTOCAP_F_SOFTWARE
+should be used for drivers which process requests using host CPUs.
+.Dv CRYPTOCAP_F_HARDWARE
+should be used for drivers which process requests on separate co-processors.
+.Dv CRYPTOCAP_F_SYNC
+should be set for drivers which process requests synchronously in
+.Fn CRYPTODEV_PROCESS .
+.Fn crypto_get_driverid
+returns an opaque driver id.
+.Pp
+.Fn crypto_unregister_all
+unregisters a driver from the cryptographic framework.
+If there are any pending operations or open sessions,
+this function will sleep.
+.Fa driverid
+is the value returned by an earlier call to
+.Fn crypto_get_driverid .
+.Pp
+When a new session is created by
+.Fn crypto_newsession ,
+.Fn CRYPTODEV_PROBESESSION
+is invoked by the cryptographic framework on each active driver to
+determine the best driver to use for the session.
+This method should inspect the session parameters in
+.Fa csp .
+If a driver does not support requests described by
+.Fa csp ,
+this method should return an error value.
+If the driver does support requests described by
+.Fa csp ,
+it should return a negative value.
+The framework prefers drivers with the largest negative value,
+similar to
+.Xr DEVICE_PROBE 9 .
+The following values are defined for non-error return values from this
+method:
+.Bl -tag -width "CRYPTODEV_PROBE_ACCEL_SOFTWARE"
+.It Dv CRYPTODEV_PROBE_HARDWARE
+The driver processes requests via a co-processor.
+.It Dv CRYPTODEV_PROBE_ACCEL_SOFTWARE
+The driver processes requests on the host CPU using optimized instructions
+such as AES-NI.
+.It Dv CRYPTODEV_PROBE_SOFTWARE
+The driver processes requests on the host CPU.
+.El
+.Pp
+This method should not sleep.
+.Pp
+Once the framework has chosen a driver for a session,
+the framework invokes the
+.Fn CRYPTODEV_NEWSESSION
+method to initialize driver-specific session state.
+Prior to calling this method,
+the framework allocates a per-session driver-specific data structure.
+This structure is initialized with zeroes,
+and its size is set by the
+.Fa session_size
+passed to
+.Fn crypto_get_driverid .
+This method can retrieve a pointer to this data structure by passing
+.Fa crypto_session
+to
+.Fn crypto_get_driver_session .
+Session parameters are described in
+.Fa csp .
+.Pp
+This method should not sleep.
+.Pp
+.Fn CRYPTODEV_FREESESSION
+is invoked to release any driver-specific state when a session is
+destroyed.
+The per-session driver-specific data structure is explicitly zeroed
+and freed by the framework after this method returns.
+If a driver requires no additional tear-down steps, it can leave
+this method undefined.
+.Pp
+This method should not sleep.
+.Pp
+.Fn CRYPTODEV_PROCESS
+is invoked for each request submitted to an active session.
+This method can either complete a request synchronously or
+schedule it to be completed asynchronously,
+but it must not sleep.
+.Pp
+If this method is not able to complete a request due to insufficient
+resources such as a full command queue,
+it can defer the request by returning
+.Dv ERESTART .
+The request will be queued by the framework and retried once the
+driver releases pending requests via
+.Fn crypto_unblock .
+Any requests submitted to sessions belonging to the driver will also
+be queued until
+.Fn crypto_unblock
+is called.
+.Pp
+If a driver encounters errors while processing a request,
+it should report them via the
+.Fa crp_etype
+field of
+.Fa crp
+rather than returning an error directly.
+.Pp
+.Fa flags
+may be set to
+.Dv CRYPTO_HINT_MORE
+if there are additional requests queued for this driver.
+The driver can use this as a hint to batch completion interrupts.
+Note that these additional requests may be from different sessions.
+.Pp
+.Fn crypto_get_driver_session
+returns a pointer to the driver-specific per-session data structure
+for the session
+.Fa crypto_session .
+This function can be used in the
+.Fn CRYPTODEV_NEWSESSION ,
+.Fn CRYPTODEV_PROCESS ,
+and
+.Fn CRYPTODEV_FREESESSION
+callbacks.
+.Pp
+.Fn crypto_copydata
+copies
+.Fa size
+bytes out of the data buffer for
+.Fa crp
+into a local buffer pointed to by
+.Fa dst .
+The bytes are read starting at an offset of
+.Fa off
+bytes in the request's data buffer.
+.Pp
+.Fn crypto_copyback
+copies
+.Fa size
+bytes from the local buffer pointed to by
+.Fa src
+into the data buffer for
+.Fa crp .
+The bytes are written starting at an offset of
+.Fa off
+bytes in the request's data buffer.
+.Pp
+A driver calls
+.Fn crypto_done
+to mark the request
+.Fa crp
+as completed.
+Any errors should be set in
+.Fa crp_etype
+prior to calling this function.
+.Pp
+If a driver defers a request by returning
+.Dv ERESTART
+from
+.Dv CRYPTO_PROCESS ,
+the framework will queue all requests for the driver until the driver calls
+.Fn crypto_unblock
+to indicate that the temporary resource shortage has been relieved.
+For example,
+if a driver returns
+.Dv ERESTART
+due to a full command ring,
+it would invoke
+.Fn crypto_unblock
+from a command completion interrupt that makes a command ring entry available.
+.Fa driverid
+is the value returned by
+.Fn crypto_get_driverid .
+.Fa what
+indicates which types of requests the driver is able to handle again:
+.Bl -tag -width "CRYPTO_ASYMQ"
+.It Dv CRYPTO_SYMQ
+indicates that the driver is able to handle symmetric requests passed to
+.Fn CRYPTODEV_PROCESS .
+.It Dv CRYPTO_ASYMQ
+indicates that the driver is able to handle asymmetric requests passed to
+.Fn CRYPTODEV_KPROCESS .
+.El
+.Pp
+.Fn crypto_apply
+is a helper routine that can be used to invoke a caller-supplied function
+to a region of the data buffer for
+.Fa crp .
+The function
+.Fa f
+is called one or more times.
+For each invocation,
+the first argument to
+.Fa f
+is the value of
+.Fa arg passed to
+.Fn crypto_apply .
+The second and third arguments to
+.Fa f
+are a pointer and length to a segment of the buffer mapped into the kernel.
+The function is called enough times to cover the
+.Fa len
+bytes of the data buffer which starts at an offset
+.Fa off .
+If any invocation of
+.Fa f
+returns a non-zero value,
+.Fn crypto_apply
+immediately returns that value without invoking
+.Fa f
+on any remaining segments of the region,
+otherwise
+.Fn crypto_apply
+returns the value from the final call to
+.Fa f .
+.Pp
+.Fn crypto_contiguous_subsegment
+attempts to locate a single, virtually-contiguous segment of the data buffer
+for
+.Fa crp .
+The segment must be
+.Fa len
+bytes long and start at an offset of
+.Fa skip
+bytes.
+If a segment is found,
+a pointer to the start of the segment is returned.
+Otherwise,
+.Dv NULL
+is returned.
+.Pp
+.Fn hmac_init_ipad
+prepares an authentication context to generate the inner hash of an HMAC.
+.Fa axf
+is a software implementation of an authentication algorithm such as the
+value returned by
+.Fn crypto_auth_hash .
+.Fa key
+is a pointer to a HMAC key of
+.Fa klen
+bytes.
+.Fa auth_ctx
+points to a valid authentication context for the desired algorithm.
+The function initializes the context with the supplied key.
+.Pp
+.Fn hmac_init_opad
+is similar to
+.Fn hmac_init_ipad
+except that it prepares an authentication context to generate the
+outer hash of an HMAC.
+.Sh RETURN VALUES
+.Fn crypto_apply
+returns the return value from the caller-supplied callback function.
+.Pp
+.Fn crypto_contiguous_subsegment
+returns a pointer to a contiguous segment or
+.Dv NULL .
+.Pp
+.Fn crypto_get_driverid
+returns a driver identifier on success or -1 on error.
+.Pp
+.Fn crypto_unblock ,
+.Fn crypto_unregister_all ,
+.Fn CRYPTODEV_FREESESSION ,
+.Fn CRYPTODEV_NEWSESSION ,
+and
+.Fn CRYPTODEV_PROCESS
+return zero on success or an error on failure.
+.Pp
+.Fn CRYPTODEV_PROBESESSION
+returns a negative value on success or an error on failure.
+.Sh SEE ALSO
+.Xr crypto 7 ,
+.Xr crypto 9 ,
+.Xr crypto_request 9 ,
+.Xr crypto_session 9

Property changes on: head/share/man/man9/crypto_driver.9
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Added: svn:keywords
## -0,0 +1 ##
+FreeBSD=%H
\ No newline at end of property
Added: svn:mime-type
## -0,0 +1 ##
+text/plain
\ No newline at end of property
Index: head/share/man/man9/crypto_request.9
===================================================================
--- head/share/man/man9/crypto_request.9	(nonexistent)
+++ head/share/man/man9/crypto_request.9	(revision 359374)
@@ -0,0 +1,419 @@
+.\" Copyright (c) 2020, Chelsio Inc
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions are met:
+.\"
+.\" 1. Redistributions of source code must retain the above copyright notice,
+.\"    this list of conditions and the following disclaimer.
+.\"
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" 3. Neither the name of the Chelsio Inc nor the names of its
+.\"    contributors may be used to endorse or promote products derived from
+.\"    this software without specific prior written permission.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+.\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+.\" LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+.\" CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+.\" SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+.\" INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+.\" CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+.\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+.\" POSSIBILITY OF SUCH DAMAGE.
+.\"
+.\" * Other names and brands may be claimed as the property of others.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd March 27, 2020
+.Dt CRYPTO_REQUEST 9
+.Os
+.Sh NAME
+.Nm crypto_request
+.Nd symmetric cryptographic operations
+.Sh SYNOPSIS
+.In opencrypto/cryptodev.h
+.Ft int
+.Fn crypto_dispatch "struct cryptop *crp"
+.Ft void
+.Fn crypto_freereq "struct cryptop *crp"
+.Ft "struct cryptop *"
+.Fn crypto_getreq "crypto_session_t cses" "int how"
+.Sh DESCRIPTION
+Each symmetric cryptographic operation in the kernel is described by
+an instance of
+.Vt struct cryptop
+and is associated with an active session.
+.Pp
+New requests are allocated by
+.Fn crypto_getreq .
+.Fa cses
+is a reference to an active session.
+.Fa how
+is passed to
+.Xr malloc 9
+and should be set to either
+.Dv M_NOWAIT
+or
+.Dv M_WAITOK .
+The caller should then set fields in the returned structure to describe
+request-specific parameters.
+Unused fields should be left as-is.
+.Pp
+.Fn crypto_dispatch
+passes a crypto request to the driver attached to the request's session.
+If there are errors in the request's fields, this function may return
+an error to the caller.
+If errors are encountered while servicing the request, they will instead
+be reported to the request's callback function
+.Pq Fa crp_callback
+via
+.Fa crp_etype .
+.Pp
+Note that a request's callback function may be invoked before
+.Fn crypto_dispatch
+returns.
+.Pp
+Once a request has signaled completion by invoking its callback function,
+it should be feed via
+.Fn crypto_freereq .
+.Pp
+Cryptographic operations include several fields to describe the request.
+.Ss Buffer Types
+Requests are associated with a single data buffer that is modified in place.
+The type of the data buffer and the buffer itself are described by the
+following fields:
+.Bl -tag -width crp_buf_type
+.It Fa crp_buf_type
+The type of the data buffer.
+The following types are supported:
+.Bl -tag -width CRYPTO_BUF_CONTIG
+.It Dv CRYPTO_BUF_CONTIG
+An array of bytes mapped into the kernel's address space.
+.It Dv CRYPTO_BUF_UIO
+A scatter/gather list of kernel buffers as described in
+.Xr uio 9 .
+.It Dv CRYPTO_BUF_MBUF
+A network memory buffer as described in
+.Xr mbuf 9 .
+.El
+.It Fa crp_buf
+A pointer to the start of a
+.Dv CRYPTO_BUF_CONTIG
+data buffer.
+.It Fa crp_ilen
+The length of a
+.Dv CRYPTO_BUF_CONTIG
+data buffer
+.It Fa crp_mbuf
+A pointer to a
+.Vt struct mbuf
+for
+.Dv CRYPTO_BUF_MBUF .
+.It Fa crp_uio
+A pointer to a
+.Vt struct uio
+for
+.Dv CRYPTO_BUF_UIO .
+.It Fa crp_olen
+Used with compression and decompression requests to describe the updated
+length of the payload region in the data buffer.
+.Pp
+If a compression request increases the size of the payload,
+then the data buffer is unmodified, the request completes successfully,
+and
+.Fa crp_olen
+is set to the size the compressed data would have used.
+Callers can compare this to the payload region length to determine if
+the compressed data was discarded.
+.El
+.Ss Request Regions
+Each request describes one or more regions in the data buffer using.
+Each region is described by an offset relative to the start of the
+data buffer and a length.
+The length of some regions is the same for all requests belonging to
+a session.
+Those lengths are set in the session parameters of the associated
+session.
+All requests must define a payload region.
+Other regions are only required for specific session modes.
+The following regions are defined:
+.Bl -column "Payload" "crp_payload_start" "crp_payload_length"
+.It Sy Region Ta Sy Start Ta Sy Length Ta Sy Description
+.It AAD Ta Fa crp_aad_start Ta Fa crp_aad_length Ta
+Additional Authenticated Data
+.It IV Ta Fa crp_iv_start Ta Fa csp_ivlen Ta
+Embedded IV or nonce
+.It Payload Ta Fa crp_payload_start Ta Fa crp_payload_length Ta
+Data to encrypt, decrypt, compress, or decompress
+.It Digest Ta Fa crp_digest_start Ta Fa csp_auth_mlen Ta
+Authentication digest, hash, or tag
+.El
+.Pp
+Requests are permitted to operate on only a subset of the data buffer.
+For example,
+requests from IPsec operate on network packets that include headers not
+used as either additional authentication data (AAD) or payload data.
+.Ss Request Operations
+All requests must specify the type of operation to perform in
+.Fa crp_op .
+Available operations depend on the session's mode.
+.Pp
+Compression requests support the following operations:
+.Bl -tag -width CRYPTO_OP_DECOMPRESS
+.It Dv CRYPTO_OP_COMPRESS
+Compress the data in the payload region of the data buffer.
+.It Dv CRYPTO_OP_DECOMPRESS
+Decompress the data in the payload region of the data buffer.
+.El
+.Pp
+Cipher requests support the following operations:
+.Bl -tag -width CRYPTO_OP_DECRYPT
+.It Dv CRYPTO_OP_ENCRYPT
+Encrypt the data in the payload region of the data buffer.
+.It Dv CRYPTO_OP_DECRYPT
+Decrypt the data in the payload region of the data buffer.
+.El
+.Pp
+Digest requests support the following operations:
+.Bl -tag -width CRYPTO_OP_COMPUTE_DIGEST
+.It Dv CRYPTO_OP_COMPUTE_DIGEST
+Calculate a digest over the payload region of the data buffer
+and store the result in the digest region.
+.It Dv CRYPTO_OP_VERIFY_DIGEST
+Calculate a digest over the payload region of the data buffer.
+Compare the calculated digest to the existing digest from the digest region.
+If the digests match,
+complete the request successfully.
+If the digests do not match,
+fail the request with
+.Er EBADMSG .
+.El
+.Pp
+AEAD and Encrypt-then-Authenticate requests support the following
+operations:
+.Bl -tag -width CRYPTO_OP
+.It Dv CRYPTO_OP_ENCRYPT | Dv CRYPTO_OP_COMPUTE_DIGEST
+Encrypt the data in the payload region of the data buffer.
+Calculate a digest over the AAD and payload regions and store the
+result in the data buffer.
+.It Dv CRYPTO_OP_DECRYPT | Dv CRYPTO_OP_VERIFY_DIGEST
+Calculate a digest over the AAD and payload regions of the data buffer.
+Compare the calculated digest to the existing digest from the digest region.
+If the digests match,
+decrypt the payload region.
+If the digests do not match,
+fail the request with
+.Er EBADMSG .
+.El
+.Ss Request IV and/or Nonce
+Some cryptographic operations require an IV or nonce as an input.
+An IV may be stored either in the IV region of the data buffer or in
+.Fa crp_iv .
+By default,
+the IV is assumed to be stored in the IV region.
+If the IV is stored in
+.Fa crp_iv ,
+.Dv CRYPTO_F_IV_SEPARATE
+should be set in
+.Fa crp_flags
+and
+.Fa crp_digest_start
+should be left as zero.
+.Pp
+An encryption request using an IV stored in the IV region may set
+.Dv CRYPTO_F_IV_GENERATE
+in
+.Fa crp_flags
+to request that the driver generate a random IV.
+Note that
+.Dv CRYPTO_F_IV_GENERATE
+cannot be used with decryption operations or in combination with
+.Dv CRYPTO_F_IV_SEPARATE .
+.Pp
+Requests that store part, but not all, of the IV in the data buffer should
+store the partial IV in the data buffer and pass the full IV separately in
+.Fa crp_iv .
+.Ss Request and Callback Scheduling
+The crypto framework provides multiple methods of scheduling the dispatch
+of requests to drivers along with the processing of driver callbacks.
+Requests use flags in
+.Fa crp_flags
+to select the desired scheduling methods.
+.Pp
+.Fn crypto_dispatch
+can pass the request to the session's driver via three different methods:
+.Bl -enum
+.It
+The request is queued to a taskqueue backed by a pool of worker threads.
+By default the pool is sized to provide one thread for each CPU.
+Worker threads dequeue requests and pass them to the driver
+asynchronously.
+.It
+The request is passed to the driver synchronously in the context of the
+thread invoking
+.Fn crypto_dispatch .
+.It
+The request is queued to a queue of pending requests.
+A single worker thread dequeues requests and passes them to the driver
+asynchronously.
+.El
+.Pp
+To select the first method (taskqueue backed by multiple threads),
+requests should set
+.Dv CRYPTO_F_ASYNC .
+To always use the third method (queue to single worker thread),
+requests should set
+.Dv CRYPTO_F_BATCH .
+If both flags are set,
+.Dv CRYPTO_F_ASYNC
+takes precedence.
+If neither flag is set,
+.Fn crypto_dispatch
+will first attempt the second method (invoke driver synchronously).
+If the driver is blocked,
+the request will be queued using the third method.
+One caveat is that the first method is only used for requests using software
+drivers which use host CPUs to process requests.
+Requests whose session is associated with a hardware driver will ignore
+.Dv CRYPTO_F_ASYNC
+and only use
+.Dv CRYPTO_F_BATCH
+to determine how requests should be scheduled.
+.Pp
+In addition to bypassing synchronous dispatch in
+.Fn crypto_dispatch ,
+.Dv CRYPTO_F_BATCH
+requests additional changes aimed at optimizing batches of requests to
+the same driver.
+When the worker thread processes a request with
+.Dv CRYPTO_F_BATCH ,
+it will search the pending request queue for any other requests for the same
+driver,
+including requests from different sessions.
+If any other requests are present,
+.Dv CRYPTO_HINT_MORE
+is passed to the driver's process method.
+Drivers may use this to batch completion interrupts.
+.Pp
+Callback function scheduling is simpler than request scheduling.
+Callbacks can either be invoked synchronously from
+.Fn crypto_done ,
+or they can be queued to a pool of worker threads.
+This pool of worker threads is also sized to provide one worker thread
+for each CPU by default.
+Note that a callback function invoked synchronously from
+.Fn crypto_done
+must follow the same restrictions placed on threaded interrupt handlers.
+.Pp
+By default,
+callbacks are invoked asynchronously by a worker thread.
+If
+.Dv CRYPTO_F_CBIMM
+is set,
+the callback is always invoked synchronously from
+.Fn crypto_done .
+If
+.Dv CRYPTO_F_CBIFSYNC
+is set,
+the callback is invoked synchronously if the request was processed by a
+software driver or asynchronously if the request was processed by a
+hardware driver.
+.Pp
+If a request was scheduled to the taskqueue via
+.Dv CRYPTO_F_ASYNC ,
+callbacks are always invoked asynchronously ignoring
+.Dv CRYPTO_F_CBIMM
+and
+.Dv CRYPTO_F_CBIFSYNC .
+In this case,
+.Dv CRYPTO_F_ASYNC_KEEPORDER
+may be set to ensure that callbacks for requests on a given session are
+invoked in the same order that requests were queued to the session via
+.Fn crypto_dispatch .
+This flag is used by IPsec to ensure that decrypted network packets are
+passed up the network stack in roughly the same order they were received.
+.Pp
+.Ss Other Request Fields
+In addition to the fields and flags enumerated above,
+.Vt struct cryptop
+includes the following:
+.Bl -tag -width crp_payload_length
+.It Fa crp_session
+A reference to the active session.
+This is set when the request is created by
+.Fn crypto_getreq
+and should not be modified.
+Drivers can use this to fetch driver-specific session state or
+session parameters.
+.It Fa crp_etype
+Error status.
+Either zero on success, or an error if a request fails.
+Set by drivers prior to completing a request via
+.Fn crypto_done .
+.It Fa crp_flags
+A bitmask of flags.
+The following flags are available in addition to flags discussed previously:
+.Bl -tag -width CRYPTO_F_DONE
+.It Dv CRYPTO_F_DONE
+Set by
+.Fa crypto_done
+before calling
+.Fa crp_callback .
+This flag is not very useful and will likely be removed in the future.
+It can only be safely checked from the callback routine at which point
+it is always set.
+.El
+.It Fa crp_cipher_key
+Pointer to a request-specific encryption key.
+If this value is not set,
+the request uses the session encryption key.
+.It Fa crp_auth_key
+Pointer to a request-specific authentication key.
+If this value is not set,
+the request uses the session authentication key.
+.It Fa crp_opaque
+An opaque pointer.
+This pointer permits users of the cryptographic framework to store
+information about a request to be used in the callback.
+.It Fa crp_callback
+Callback function.
+This must point to a callback function of type
+.Vt void (*)(struct cryptop *) .
+The callback function should inspect
+.Fa crp_etype
+to determine the status of the completed operation.
+It should also arrange for the request to be freed via
+.Fn crypto_freereq .
+.El
+.Sh RETURN VALUES
+.Fn crypto_dispatch
+returns an error if the request contained invalid fields,
+or zero if the request was valid.
+.Fn crypto_getreq
+returns a pointer to a new request structure on success,
+or
+.Dv NULL
+on failure.
+.Dv NULL
+can only be returned if
+.Dv M_NOWAIT
+was passed in
+.Fa how .
+.Sh SEE ALSO
+.Xr ipsec 4 ,
+.Xr crypto 7 ,
+.Xr crypto 9 ,
+.Xr crypto_session 9 ,
+.Xr mbuf 9
+.Xr uio 9
+.Sh BUGS
+Not all drivers properly handle mixing session and per-request keys
+within a single session.
+Consumers should either use a single key for a session specified in
+the session parameters or always use per-request keys.

Property changes on: head/share/man/man9/crypto_request.9
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Added: svn:keywords
## -0,0 +1 ##
+FreeBSD=%H
\ No newline at end of property
Added: svn:mime-type
## -0,0 +1 ##
+text/plain
\ No newline at end of property
Index: head/share/man/man9/crypto_session.9
===================================================================
--- head/share/man/man9/crypto_session.9	(nonexistent)
+++ head/share/man/man9/crypto_session.9	(revision 359374)
@@ -0,0 +1,245 @@
+.\" Copyright (c) 2020, Chelsio Inc
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions are met:
+.\"
+.\" 1. Redistributions of source code must retain the above copyright notice,
+.\"    this list of conditions and the following disclaimer.
+.\"
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" 3. Neither the name of the Chelsio Inc nor the names of its
+.\"    contributors may be used to endorse or promote products derived from
+.\"    this software without specific prior written permission.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+.\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+.\" LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+.\" CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+.\" SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+.\" INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+.\" CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+.\" ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+.\" POSSIBILITY OF SUCH DAMAGE.
+.\"
+.\" * Other names and brands may be claimed as the property of others.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd March 27, 2020
+.Dt CRYPTO_SESSION 9
+.Os
+.Sh NAME
+.Nm crypto_session
+.Nd state used for symmetric cryptographic services
+.Sh SYNOPSIS
+.In opencrypto/cryptodev.h
+.Ft struct auth_hash *
+.Fn crypto_auth_hash "const struct crypto_session_params *csp"
+.Ft struct enc_xform *
+.Fn crypto_cipher "const struct crypto_session_params *csp"
+.Ft const struct crypto_session_params *
+.Fn crypto_get_params "crypto_session_t cses"
+.Ft int
+.Fo crypto_newsession
+.Fa "crypto_session_t *cses"
+.Fa "const struct crypto_session_params *csp"
+.Fa "int crid"
+.Fc
+.Ft int
+.Fn crypto_freesession "crypto_session_t cses"
+.Sh DESCRIPTION
+Symmetric cryptographic operations in the kernel are associated with
+cryptographic sessions.
+Sessions hold state shared across multiple requests.
+Active sessions are associated with a single cryptographic driver.
+.Pp
+The
+.Vt crypto_session_t
+type represents an opaque reference to an active session.
+Session objects are allocated and managed by the cryptographic
+framework.
+.Pp
+New sessions are created by
+.Fn crypto_newsession .
+.Fa csp
+describes various parameters associated with the new session such as
+the algorithms to use and any session-wide keys.
+.Fa crid
+can be used to request either a specific cryptographic driver or
+classes of drivers.
+For the latter case,
+.Fa crid
+should be set to a mask of the following values:
+.Bl -tag -width "CRYPTOCAP_F_HARDWARE"
+.It Dv CRYPTOCAP_F_HARDWARE
+Request hardware drivers.
+Hardware drivers do not use the host CPU to perform operations.
+Typically, a separate co-processor performs the operations asynchronously.
+.It Dv CRYPTOCAP_F_SOFTWARE
+Request software drivers.
+Software drivers use the host CPU to perform operations.
+The kernel includes a simple, yet portable implementation of each supported
+algorithm in the
+.Xr cryptosoft 4
+driver.
+Additional software drivers may also be available on architectures which
+provide instructions designed to accelerate cryptographic operations.
+.El
+.Pp
+If both hardware and software drivers are requested,
+hardware drivers are preferred over software drivers.
+Accelerated software drivers are preferred over the baseline software driver.
+If multiple hardware drivers are available,
+the framework will distribute sessions across these drivers in a round-robin
+fashion.
+.Pp
+On success,
+.Fn crypto_newsession
+saves a reference to the newly created session in
+.Fa cses .
+.Pp
+.Fn crypto_freesession
+is used to free the resources associated with the session
+.Fa cses .
+.Pp
+.Fn crypto_auth_hash
+returns a structure describing the baseline software implementation of an
+authentication algorithm requested by
+.Fa csp .
+If
+.Fa csp
+does not specify an authentication algorithm,
+or requests an invalid algorithm,
+.Dv NULL
+is returned.
+.Pp
+.Fn crypto_cipher
+returns a structure describing the baseline software implementation of an
+encryption algorithm requested by
+.Fa csp .
+If
+.Fa csp
+does not specify an encryption algorithm,
+or requests an invalid algorithm,
+.Dv NULL
+is returned.
+.Pp
+.Fn crypto_get_params
+returns a pointer to the session parameters used by
+.Fa cses .
+.Ss Session Parameters
+Session parameters are used to describe the cryptographic operations
+performed by cryptographic requests.
+Parameters are stored in an instance of
+.Vt struct crypto_session_params .
+When initializing parameters to pass to
+.Fn crypto_newsession ,
+the entire structure should first be zeroed.
+Needed fields should then be set leaving unused fields as zero.
+This structure contains the following fields:
+.Bl -tag -width csp_cipher_klen
+.It Fa csp_mode
+Type of operation to perform.
+This field must be set to one of the following:
+.Bl -tag -width CSP_MODE_COMPRESS
+.It Dv CSP_MODE_COMPRESS
+Compress or decompress request payload.
+.Pp
+The compression algorithm is specified in
+.Fa csp_cipher_alg .
+.It Dv CSP_MODE_CIPHER
+Encrypt or decrypt request payload.
+.Pp
+The encryption algorithm is specified in
+.Fa csp_cipher_alg .
+.It Dv CSP_MODE_DIGEST
+Compute or verify a digest, or hash, of request payload.
+.Pp
+The authentication algorithm is specified in
+.Fa csp_auth_alg .
+.It Dv CSP_MODE_AEAD
+Authenticated encryption with additional data.
+Decryption operations require the digest, or tag,
+and fail if it does not match.
+.Pp
+The AEAD algorithm is specified in
+.Fa csp_cipher_alg .
+.It Dv CSP_MODE_ETA
+Encrypt-then-Authenticate.
+In this mode, encryption operations encrypt the payload and then
+compute an authentication digest over the request additional authentication
+data followed by the encrypted payload.
+Decryption operations fail without decrypting the data if the provided digest
+does not match.
+.Pp
+The encryption algorithm is specified in
+.Fa csp_cipher_alg
+and the authentication algorithm is specified in
+.Fa csp_auth_alg .
+.El
+.It Fa csp_flags
+Currently, no additional flags are defined and this field should be set to
+zero.
+.It Fa csp_ivlen
+If either the cipher or authentication algorithms require an explicit
+initialization vector (IV) or nonce,
+this specifies the length in bytes.
+All requests for a session use the same IV length.
+.It Fa csp_cipher_alg
+Encryption or compression algorithm.
+.It Fa csp_cipher_klen
+Length of encryption or decryption key in bytes.
+All requests for a session use the same key length.
+.It Fa csp_cipher_key
+Pointer to encryption or decryption key.
+If all requests for a session use request-specific keys,
+this field should be left as
+.Dv NULL .
+This pointer and associated key must remain valid for the duration of the
+crypto session.
+.It Fa csp_auth_alg
+Authentication algorithm.
+.It Fa csp_auth_klen
+Length of authentication key in bytes.
+If the authentication algorithm does not use a key,
+this field should be left as zero.
+.It Fa csp_auth_key
+Pointer to the authentication key.
+If all requests for a session use request-specific keys,
+this field should be left as
+.Dv NULL .
+This pointer and associated key must remain valid for the duration of the
+crypto session.
+.It Fa csp_auth_mlen
+The length in bytes of the digest.
+If zero, the full length of the digest is used.
+If non-zero, the first
+.Fa csp_auth_mlen
+bytes of the digest are used.
+.El
+.Sh RETURN VALUES
+.Fn crypto_newsession
+returns a non-zero value if an error occurs or zero on success.
+.Pp
+.Fn crypto_auth_hash
+and
+.Fn crypto_cipher
+return
+.Dv NULL
+if the request is valid or a pointer to a structure on success.
+.Sh SEE ALSO
+.Xr crypto 7 ,
+.Xr crypto 9 ,
+.Xr crypto_request 9
+.Sh BUGS
+The current implementation of
+.Nm crypto_freesession
+does not provide a way for the caller to know that there are no other
+references to the keys stored in the session's associated parameters.
+This function should probably sleep until any in-flight cryptographic
+operations associated with the session are completed.

Property changes on: head/share/man/man9/crypto_session.9
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Added: svn:keywords
## -0,0 +1 ##
+FreeBSD=%H
\ No newline at end of property
Added: svn:mime-type
## -0,0 +1 ##
+text/plain
\ No newline at end of property
Index: head/sys/crypto/aesni/aesni.c
===================================================================
--- head/sys/crypto/aesni/aesni.c	(revision 359373)
+++ head/sys/crypto/aesni/aesni.c	(revision 359374)
@@ -1,973 +1,859 @@
 /*-
  * Copyright (c) 2005-2008 Pawel Jakub Dawidek <pjd@FreeBSD.org>
  * Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org>
  * Copyright (c) 2014 The FreeBSD Foundation
  * Copyright (c) 2017 Conrad Meyer <cem@FreeBSD.org>
  * All rights reserved.
  *
  * Portions of this software were developed by John-Mark Gurney
  * under sponsorship of the FreeBSD Foundation and
  * Rubicon Communications, LLC (Netgate).
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/bus.h>
 #include <sys/kernel.h>
 #include <sys/kobj.h>
 #include <sys/libkern.h>
 #include <sys/lock.h>
 #include <sys/malloc.h>
 #include <sys/mbuf.h>
 #include <sys/module.h>
 #include <sys/mutex.h>
 #include <sys/smp.h>
 #include <sys/systm.h>
 #include <sys/uio.h>
 
 #include <crypto/aesni/aesni.h>
 #include <crypto/aesni/sha_sse.h>
 #include <crypto/sha1.h>
 #include <crypto/sha2/sha224.h>
 #include <crypto/sha2/sha256.h>
 
 #include <opencrypto/cryptodev.h>
 #include <opencrypto/gmac.h>
 #include <cryptodev_if.h>
 
 #include <machine/md_var.h>
 #include <machine/specialreg.h>
 #if defined(__i386__)
 #include <machine/npx.h>
 #elif defined(__amd64__)
 #include <machine/fpu.h>
 #endif
 
 static struct mtx_padalign *ctx_mtx;
 static struct fpu_kern_ctx **ctx_fpu;
 
 struct aesni_softc {
 	int32_t cid;
 	bool	has_aes;
 	bool	has_sha;
 };
 
 #define ACQUIRE_CTX(i, ctx)					\
 	do {							\
 		(i) = PCPU_GET(cpuid);				\
 		mtx_lock(&ctx_mtx[(i)]);			\
 		(ctx) = ctx_fpu[(i)];				\
 	} while (0)
 #define RELEASE_CTX(i, ctx)					\
 	do {							\
 		mtx_unlock(&ctx_mtx[(i)]);			\
 		(i) = -1;					\
 		(ctx) = NULL;					\
 	} while (0)
 
-static int aesni_newsession(device_t, crypto_session_t cses,
-    struct cryptoini *cri);
 static int aesni_cipher_setup(struct aesni_session *ses,
-    struct cryptoini *encini, struct cryptoini *authini);
-static int aesni_cipher_process(struct aesni_session *ses,
-    struct cryptodesc *enccrd, struct cryptodesc *authcrd, struct cryptop *crp);
-static int aesni_cipher_crypt(struct aesni_session *ses,
-    struct cryptodesc *enccrd, struct cryptodesc *authcrd, struct cryptop *crp);
-static int aesni_cipher_mac(struct aesni_session *ses, struct cryptodesc *crd,
-    struct cryptop *crp);
+    const struct crypto_session_params *csp);
+static int aesni_cipher_process(struct aesni_session *ses, struct cryptop *crp);
+static int aesni_cipher_crypt(struct aesni_session *ses, struct cryptop *crp,
+    const struct crypto_session_params *csp);
+static int aesni_cipher_mac(struct aesni_session *ses, struct cryptop *crp,
+    const struct crypto_session_params *csp);
 
 MALLOC_DEFINE(M_AESNI, "aesni_data", "AESNI Data");
 
 static void
 aesni_identify(driver_t *drv, device_t parent)
 {
 
 	/* NB: order 10 is so we get attached after h/w devices */
 	if (device_find_child(parent, "aesni", -1) == NULL &&
 	    BUS_ADD_CHILD(parent, 10, "aesni", -1) == 0)
 		panic("aesni: could not attach");
 }
 
 static void
 detect_cpu_features(bool *has_aes, bool *has_sha)
 {
 
 	*has_aes = ((cpu_feature2 & CPUID2_AESNI) != 0 &&
 	    (cpu_feature2 & CPUID2_SSE41) != 0);
 	*has_sha = ((cpu_stdext_feature & CPUID_STDEXT_SHA) != 0 &&
 	    (cpu_feature2 & CPUID2_SSSE3) != 0);
 }
 
 static int
 aesni_probe(device_t dev)
 {
 	bool has_aes, has_sha;
 
 	detect_cpu_features(&has_aes, &has_sha);
 	if (!has_aes && !has_sha) {
 		device_printf(dev, "No AES or SHA support.\n");
 		return (EINVAL);
 	} else if (has_aes && has_sha)
 		device_set_desc(dev,
 		    "AES-CBC,AES-CCM,AES-GCM,AES-ICM,AES-XTS,SHA1,SHA256");
 	else if (has_aes)
 		device_set_desc(dev,
 		    "AES-CBC,AES-CCM,AES-GCM,AES-ICM,AES-XTS");
 	else
 		device_set_desc(dev, "SHA1,SHA256");
 
 	return (0);
 }
 
 static void
 aesni_cleanctx(void)
 {
 	int i;
 
 	/* XXX - no way to return driverid */
 	CPU_FOREACH(i) {
 		if (ctx_fpu[i] != NULL) {
 			mtx_destroy(&ctx_mtx[i]);
 			fpu_kern_free_ctx(ctx_fpu[i]);
 		}
 		ctx_fpu[i] = NULL;
 	}
 	free(ctx_mtx, M_AESNI);
 	ctx_mtx = NULL;
 	free(ctx_fpu, M_AESNI);
 	ctx_fpu = NULL;
 }
 
 static int
 aesni_attach(device_t dev)
 {
 	struct aesni_softc *sc;
 	int i;
 
 	sc = device_get_softc(dev);
 
 	sc->cid = crypto_get_driverid(dev, sizeof(struct aesni_session),
-	    CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SYNC);
+	    CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC);
 	if (sc->cid < 0) {
 		device_printf(dev, "Could not get crypto driver id.\n");
 		return (ENOMEM);
 	}
 
 	ctx_mtx = malloc(sizeof *ctx_mtx * (mp_maxid + 1), M_AESNI,
 	    M_WAITOK|M_ZERO);
 	ctx_fpu = malloc(sizeof *ctx_fpu * (mp_maxid + 1), M_AESNI,
 	    M_WAITOK|M_ZERO);
 
 	CPU_FOREACH(i) {
 		ctx_fpu[i] = fpu_kern_alloc_ctx(0);
 		mtx_init(&ctx_mtx[i], "anifpumtx", NULL, MTX_DEF|MTX_NEW);
 	}
 
 	detect_cpu_features(&sc->has_aes, &sc->has_sha);
-	if (sc->has_aes) {
-		crypto_register(sc->cid, CRYPTO_AES_CBC, 0, 0);
-		crypto_register(sc->cid, CRYPTO_AES_ICM, 0, 0);
-		crypto_register(sc->cid, CRYPTO_AES_NIST_GCM_16, 0, 0);
-		crypto_register(sc->cid, CRYPTO_AES_128_NIST_GMAC, 0, 0);
-		crypto_register(sc->cid, CRYPTO_AES_192_NIST_GMAC, 0, 0);
-		crypto_register(sc->cid, CRYPTO_AES_256_NIST_GMAC, 0, 0);
-		crypto_register(sc->cid, CRYPTO_AES_XTS, 0, 0);
-		crypto_register(sc->cid, CRYPTO_AES_CCM_16, 0, 0);
-		crypto_register(sc->cid, CRYPTO_AES_CCM_CBC_MAC, 0, 0);
-	}
-	if (sc->has_sha) {
-		crypto_register(sc->cid, CRYPTO_SHA1, 0, 0);
-		crypto_register(sc->cid, CRYPTO_SHA1_HMAC, 0, 0);
-		crypto_register(sc->cid, CRYPTO_SHA2_224, 0, 0);
-		crypto_register(sc->cid, CRYPTO_SHA2_224_HMAC, 0, 0);
-		crypto_register(sc->cid, CRYPTO_SHA2_256, 0, 0);
-		crypto_register(sc->cid, CRYPTO_SHA2_256_HMAC, 0, 0);
-	}
 	return (0);
 }
 
 static int
 aesni_detach(device_t dev)
 {
 	struct aesni_softc *sc;
 
 	sc = device_get_softc(dev);
 
 	crypto_unregister_all(sc->cid);
 
 	aesni_cleanctx();
 
 	return (0);
 }
 
-static int
-aesni_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
+static bool
+aesni_auth_supported(struct aesni_softc *sc,
+    const struct crypto_session_params *csp)
 {
-	struct aesni_softc *sc;
-	struct aesni_session *ses;
-	struct cryptoini *encini, *authini;
-	bool gcm_hash, gcm;
-	bool cbc_hash, ccm;
-	int error;
 
-	KASSERT(cses != NULL, ("EDOOFUS"));
-	if (cri == NULL) {
-		CRYPTDEB("no cri");
-		return (EINVAL);
+	if (!sc->has_sha)
+		return (false);
+
+	switch (csp->csp_auth_alg) {
+	case CRYPTO_SHA1:
+	case CRYPTO_SHA2_224:
+	case CRYPTO_SHA2_256:
+	case CRYPTO_SHA1_HMAC:
+	case CRYPTO_SHA2_224_HMAC:
+	case CRYPTO_SHA2_256_HMAC:
+		break;
+	default:
+		return (false);
 	}
 
-	sc = device_get_softc(dev);
+	return (true);
+}
 
-	ses = crypto_get_driver_session(cses);
+static bool
+aesni_cipher_supported(struct aesni_softc *sc,
+    const struct crypto_session_params *csp)
+{
 
-	authini = NULL;
-	encini = NULL;
-	gcm = false;
-	gcm_hash = false;
-	ccm = cbc_hash = false;
+	if (!sc->has_aes)
+		return (false);
 
-	for (; cri != NULL; cri = cri->cri_next) {
-		switch (cri->cri_alg) {
+	switch (csp->csp_cipher_alg) {
+	case CRYPTO_AES_CBC:
+	case CRYPTO_AES_ICM:
+		if (csp->csp_ivlen != AES_BLOCK_LEN)
+			return (false);
+		return (sc->has_aes);
+	case CRYPTO_AES_XTS:
+		if (csp->csp_ivlen != AES_XTS_IV_LEN)
+			return (false);
+		return (sc->has_aes);
+	default:
+		return (false);
+	}
+}
+
+static int
+aesni_probesession(device_t dev, const struct crypto_session_params *csp)
+{
+	struct aesni_softc *sc;
+
+	sc = device_get_softc(dev);
+	if (csp->csp_flags != 0)
+		return (EINVAL);
+	switch (csp->csp_mode) {
+	case CSP_MODE_DIGEST:
+		if (!aesni_auth_supported(sc, csp))
+			return (EINVAL);
+		break;
+	case CSP_MODE_CIPHER:
+		if (!aesni_cipher_supported(sc, csp))
+			return (EINVAL);
+		break;
+	case CSP_MODE_AEAD:
+		switch (csp->csp_cipher_alg) {
 		case CRYPTO_AES_NIST_GCM_16:
-		case CRYPTO_AES_CCM_16:
-			if (cri->cri_alg == CRYPTO_AES_NIST_GCM_16) {
-				gcm = true;
-			} else if (cri->cri_alg == CRYPTO_AES_CCM_16) {
-				ccm = true;
-			}
-			/* FALLTHROUGH */
-		case CRYPTO_AES_CBC:
-		case CRYPTO_AES_ICM:
-		case CRYPTO_AES_XTS:
-			if (!sc->has_aes)
-				goto unhandled;
-			if (encini != NULL) {
-				CRYPTDEB("encini already set");
+			if (csp->csp_auth_mlen != 0 &&
+			    csp->csp_auth_mlen != GMAC_DIGEST_LEN)
 				return (EINVAL);
-			}
-			encini = cri;
-			break;
-		case CRYPTO_AES_CCM_CBC_MAC:
-			cbc_hash = true;
-			authini = cri;
-			break;
-		case CRYPTO_AES_128_NIST_GMAC:
-		case CRYPTO_AES_192_NIST_GMAC:
-		case CRYPTO_AES_256_NIST_GMAC:
-			/*
-			 * nothing to do here, maybe in the future cache some
-			 * values for GHASH
-			 */
-			if (authini != NULL) {
-				CRYPTDEB("authini already set");
+			if (csp->csp_ivlen != AES_GCM_IV_LEN ||
+			    !sc->has_aes)
 				return (EINVAL);
-			}
-			gcm_hash = true;
-			authini = cri;
 			break;
-		case CRYPTO_SHA1:
-		case CRYPTO_SHA1_HMAC:
-		case CRYPTO_SHA2_224:
-		case CRYPTO_SHA2_224_HMAC:
-		case CRYPTO_SHA2_256:
-		case CRYPTO_SHA2_256_HMAC:
-			if (!sc->has_sha)
-				goto unhandled;
-			if (authini != NULL) {
-				CRYPTDEB("authini already set");
+		case CRYPTO_AES_CCM_16:
+			if (csp->csp_auth_mlen != 0 &&
+			    csp->csp_auth_mlen != AES_CBC_MAC_HASH_LEN)
 				return (EINVAL);
-			}
-			authini = cri;
+			if (csp->csp_ivlen != AES_CCM_IV_LEN ||
+			    !sc->has_aes)
+				return (EINVAL);
 			break;
 		default:
-unhandled:
-			CRYPTDEB("unhandled algorithm");
 			return (EINVAL);
 		}
-	}
-	if (encini == NULL && authini == NULL) {
-		CRYPTDEB("no cipher");
+		break;
+	case CSP_MODE_ETA:
+		if (!aesni_auth_supported(sc, csp) ||
+		    !aesni_cipher_supported(sc, csp))
+			return (EINVAL);
+		break;
+	default:
 		return (EINVAL);
 	}
-	/*
-	 * GMAC algorithms are only supported with simultaneous GCM.  Likewise
-	 * GCM is not supported without GMAC.
-	 */
-	if (gcm_hash != gcm) {
-		CRYPTDEB("gcm_hash != gcm");
-		return (EINVAL);
-	}
 
-	if (cbc_hash != ccm) {
-		CRYPTDEB("cbc_hash != ccm");
-		return (EINVAL);
-	}
+	return (CRYPTODEV_PROBE_ACCEL_SOFTWARE);
+}
 
-	if (encini != NULL)
-		ses->algo = encini->cri_alg;
-	if (authini != NULL)
-		ses->auth_algo = authini->cri_alg;
+static int
+aesni_newsession(device_t dev, crypto_session_t cses,
+    const struct crypto_session_params *csp)
+{
+	struct aesni_softc *sc;
+	struct aesni_session *ses;
+	int error;
 
-	error = aesni_cipher_setup(ses, encini, authini);
+	sc = device_get_softc(dev);
+
+	ses = crypto_get_driver_session(cses);
+
+	switch (csp->csp_mode) {
+	case CSP_MODE_DIGEST:
+	case CSP_MODE_CIPHER:
+	case CSP_MODE_AEAD:
+	case CSP_MODE_ETA:
+		break;
+	default:
+		return (EINVAL);
+	}
+	error = aesni_cipher_setup(ses, csp);
 	if (error != 0) {
 		CRYPTDEB("setup failed");
 		return (error);
 	}
 
 	return (0);
 }
 
 static int
 aesni_process(device_t dev, struct cryptop *crp, int hint __unused)
 {
 	struct aesni_session *ses;
-	struct cryptodesc *crd, *enccrd, *authcrd;
-	int error, needauth;
+	int error;
 
-	ses = NULL;
-	error = 0;
-	enccrd = NULL;
-	authcrd = NULL;
-	needauth = 0;
-
-	/* Sanity check. */
-	if (crp == NULL)
-		return (EINVAL);
-
-	if (crp->crp_callback == NULL || crp->crp_desc == NULL ||
-	    crp->crp_session == NULL) {
-		error = EINVAL;
-		goto out;
-	}
-
-	for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) {
-		switch (crd->crd_alg) {
-		case CRYPTO_AES_NIST_GCM_16:
-		case CRYPTO_AES_CCM_16:
-			needauth = 1;
-			/* FALLTHROUGH */
-		case CRYPTO_AES_CBC:
-		case CRYPTO_AES_ICM:
-		case CRYPTO_AES_XTS:
-			if (enccrd != NULL) {
-				error = EINVAL;
-				goto out;
-			}
-			enccrd = crd;
-			break;
-
-		case CRYPTO_AES_128_NIST_GMAC:
-		case CRYPTO_AES_192_NIST_GMAC:
-		case CRYPTO_AES_256_NIST_GMAC:
-		case CRYPTO_AES_CCM_CBC_MAC:
-		case CRYPTO_SHA1:
-		case CRYPTO_SHA1_HMAC:
-		case CRYPTO_SHA2_224:
-		case CRYPTO_SHA2_224_HMAC:
-		case CRYPTO_SHA2_256:
-		case CRYPTO_SHA2_256_HMAC:
-			if (authcrd != NULL) {
-				error = EINVAL;
-				goto out;
-			}
-			authcrd = crd;
-			break;
-
-		default:
-			error = EINVAL;
-			goto out;
-		}
-	}
-
-	if ((enccrd == NULL && authcrd == NULL) ||
-	    (needauth && authcrd == NULL)) {
-		error = EINVAL;
-		goto out;
-	}
-
-	/* CBC & XTS can only handle full blocks for now */
-	if (enccrd != NULL && (enccrd->crd_alg == CRYPTO_AES_CBC ||
-	    enccrd->crd_alg == CRYPTO_AES_XTS) &&
-	    (enccrd->crd_len % AES_BLOCK_LEN) != 0) {
-		error = EINVAL;
-		goto out;
-	}
-
 	ses = crypto_get_driver_session(crp->crp_session);
-	KASSERT(ses != NULL, ("EDOOFUS"));
 
-	error = aesni_cipher_process(ses, enccrd, authcrd, crp);
-	if (error != 0)
-		goto out;
+	error = aesni_cipher_process(ses, crp);
 
-out:
 	crp->crp_etype = error;
 	crypto_done(crp);
-	return (error);
+	return (0);
 }
 
 static uint8_t *
-aesni_cipher_alloc(struct cryptodesc *enccrd, struct cryptop *crp,
-    bool *allocated)
+aesni_cipher_alloc(struct cryptop *crp, int start, int length, bool *allocated)
 {
 	uint8_t *addr;
 
-	addr = crypto_contiguous_subsegment(crp->crp_flags,
-	    crp->crp_buf, enccrd->crd_skip, enccrd->crd_len);
+	addr = crypto_contiguous_subsegment(crp, start, length);
 	if (addr != NULL) {
 		*allocated = false;
 		return (addr);
 	}
-	addr = malloc(enccrd->crd_len, M_AESNI, M_NOWAIT);
+	addr = malloc(length, M_AESNI, M_NOWAIT);
 	if (addr != NULL) {
 		*allocated = true;
-		crypto_copydata(crp->crp_flags, crp->crp_buf, enccrd->crd_skip,
-		    enccrd->crd_len, addr);
+		crypto_copydata(crp, start, length, addr);
 	} else
 		*allocated = false;
 	return (addr);
 }
 
 static device_method_t aesni_methods[] = {
 	DEVMETHOD(device_identify, aesni_identify),
 	DEVMETHOD(device_probe, aesni_probe),
 	DEVMETHOD(device_attach, aesni_attach),
 	DEVMETHOD(device_detach, aesni_detach),
 
+	DEVMETHOD(cryptodev_probesession, aesni_probesession),
 	DEVMETHOD(cryptodev_newsession, aesni_newsession),
 	DEVMETHOD(cryptodev_process, aesni_process),
 
 	DEVMETHOD_END
 };
 
 static driver_t aesni_driver = {
 	"aesni",
 	aesni_methods,
 	sizeof(struct aesni_softc),
 };
 static devclass_t aesni_devclass;
 
 DRIVER_MODULE(aesni, nexus, aesni_driver, aesni_devclass, 0, 0);
 MODULE_VERSION(aesni, 1);
 MODULE_DEPEND(aesni, crypto, 1, 1, 1);
 
-static int
-aesni_authprepare(struct aesni_session *ses, int klen, const void *cri_key)
-{
-	int keylen;
-
-	if (klen % 8 != 0)
-		return (EINVAL);
-	keylen = klen / 8;
-	if (keylen > sizeof(ses->hmac_key))
-		return (EINVAL);
-	if (ses->auth_algo == CRYPTO_SHA1 && keylen > 0)
-		return (EINVAL);
-	memcpy(ses->hmac_key, cri_key, keylen);
-	return (0);
-}
-
-static int
-aesni_cipher_setup(struct aesni_session *ses, struct cryptoini *encini,
-    struct cryptoini *authini)
-{
-	struct fpu_kern_ctx *ctx;
-	int kt, ctxidx, error;
-
-	switch (ses->auth_algo) {
-	case CRYPTO_SHA1:
-	case CRYPTO_SHA1_HMAC:
-	case CRYPTO_SHA2_224:
-	case CRYPTO_SHA2_224_HMAC:
-	case CRYPTO_SHA2_256:
-	case CRYPTO_SHA2_256_HMAC:
-		error = aesni_authprepare(ses, authini->cri_klen,
-		    authini->cri_key);
-		if (error != 0)
-			return (error);
-		ses->mlen = authini->cri_mlen;
-	}
-
-	kt = is_fpu_kern_thread(0) || (encini == NULL);
-	if (!kt) {
-		ACQUIRE_CTX(ctxidx, ctx);
-		fpu_kern_enter(curthread, ctx,
-		    FPU_KERN_NORMAL | FPU_KERN_KTHR);
-	}
-
-	error = 0;
-	if (encini != NULL)
-		error = aesni_cipher_setup_common(ses, encini->cri_key,
-		    encini->cri_klen);
-
-	if (!kt) {
-		fpu_kern_leave(curthread, ctx);
-		RELEASE_CTX(ctxidx, ctx);
-	}
-	return (error);
-}
-
-static int
+static void
 intel_sha1_update(void *vctx, const void *vdata, u_int datalen)
 {
 	struct sha1_ctxt *ctx = vctx;
 	const char *data = vdata;
 	size_t gaplen;
 	size_t gapstart;
 	size_t off;
 	size_t copysiz;
 	u_int blocks;
 
 	off = 0;
 	/* Do any aligned blocks without redundant copying. */
 	if (datalen >= 64 && ctx->count % 64 == 0) {
 		blocks = datalen / 64;
 		ctx->c.b64[0] += blocks * 64 * 8;
 		intel_sha1_step(ctx->h.b32, data + off, blocks);
 		off += blocks * 64;
 	}
 
 	while (off < datalen) {
 		gapstart = ctx->count % 64;
 		gaplen = 64 - gapstart;
 
 		copysiz = (gaplen < datalen - off) ? gaplen : datalen - off;
 		bcopy(&data[off], &ctx->m.b8[gapstart], copysiz);
 		ctx->count += copysiz;
 		ctx->count %= 64;
 		ctx->c.b64[0] += copysiz * 8;
 		if (ctx->count % 64 == 0)
 			intel_sha1_step(ctx->h.b32, (void *)ctx->m.b8, 1);
 		off += copysiz;
 	}
-	return (0);
 }
 
 static void
 SHA1_Init_fn(void *ctx)
 {
 	sha1_init(ctx);
 }
 
 static void
 SHA1_Finalize_fn(void *digest, void *ctx)
 {
 	sha1_result(ctx, digest);
 }
 
-static int
+static void
 intel_sha256_update(void *vctx, const void *vdata, u_int len)
 {
 	SHA256_CTX *ctx = vctx;
 	uint64_t bitlen;
 	uint32_t r;
 	u_int blocks;
 	const unsigned char *src = vdata;
 
 	/* Number of bytes left in the buffer from previous updates */
 	r = (ctx->count >> 3) & 0x3f;
 
 	/* Convert the length into a number of bits */
 	bitlen = len << 3;
 
 	/* Update number of bits */
 	ctx->count += bitlen;
 
 	/* Handle the case where we don't need to perform any transforms */
 	if (len < 64 - r) {
 		memcpy(&ctx->buf[r], src, len);
-		return (0);
+		return;
 	}
 
 	/* Finish the current block */
 	memcpy(&ctx->buf[r], src, 64 - r);
 	intel_sha256_step(ctx->state, ctx->buf, 1);
 	src += 64 - r;
 	len -= 64 - r;
 
 	/* Perform complete blocks */
 	if (len >= 64) {
 		blocks = len / 64;
 		intel_sha256_step(ctx->state, src, blocks);
 		src += blocks * 64;
 		len -= blocks * 64;
 	}
 
 	/* Copy left over data into buffer */
 	memcpy(ctx->buf, src, len);
-	return (0);
 }
 
 static void
 SHA224_Init_fn(void *ctx)
 {
 	SHA224_Init(ctx);
 }
 
 static void
 SHA224_Finalize_fn(void *digest, void *ctx)
 {
 	SHA224_Final(digest, ctx);
 }
 
 static void
 SHA256_Init_fn(void *ctx)
 {
 	SHA256_Init(ctx);
 }
 
 static void
 SHA256_Finalize_fn(void *digest, void *ctx)
 {
 	SHA256_Final(digest, ctx);
 }
 
-/*
- * Compute the HASH( (key ^ xorbyte) || buf )
- */
-static void
-hmac_internal(void *ctx, uint32_t *res,
-	int (*update)(void *, const void *, u_int),
-	void (*finalize)(void *, void *), uint8_t *key, uint8_t xorbyte,
-	const void *buf, size_t off, size_t buflen, int crpflags)
+static int
+aesni_authprepare(struct aesni_session *ses, int klen)
 {
-	size_t i;
 
-	for (i = 0; i < 64; i++)
-		key[i] ^= xorbyte;
-	update(ctx, key, 64);
-	for (i = 0; i < 64; i++)
-		key[i] ^= xorbyte;
+	if (klen > SHA1_BLOCK_LEN)
+		return (EINVAL);
+	if ((ses->hmac && klen == 0) || (!ses->hmac && klen != 0))
+		return (EINVAL);
+	return (0);
+}
 
-	crypto_apply(crpflags, __DECONST(void *, buf), off, buflen,
-	    __DECONST(int (*)(void *, void *, u_int), update), ctx);
-	finalize(res, ctx);
+static int
+aesni_cipherprepare(const struct crypto_session_params *csp)
+{
+
+	switch (csp->csp_cipher_alg) {
+	case CRYPTO_AES_ICM:
+	case CRYPTO_AES_NIST_GCM_16:
+	case CRYPTO_AES_CCM_16:
+	case CRYPTO_AES_CBC:
+		switch (csp->csp_cipher_klen * 8) {
+		case 128:
+		case 192:
+		case 256:
+			break;
+		default:
+			CRYPTDEB("invalid CBC/ICM/GCM key length");
+			return (EINVAL);
+		}
+		break;
+	case CRYPTO_AES_XTS:
+		switch (csp->csp_cipher_klen * 8) {
+		case 256:
+		case 512:
+			break;
+		default:
+			CRYPTDEB("invalid XTS key length");
+			return (EINVAL);
+		}
+		break;
+	default:
+		return (EINVAL);
+	}
+	return (0);
 }
 
 static int
-aesni_cipher_process(struct aesni_session *ses, struct cryptodesc *enccrd,
-    struct cryptodesc *authcrd, struct cryptop *crp)
+aesni_cipher_setup(struct aesni_session *ses,
+    const struct crypto_session_params *csp)
 {
 	struct fpu_kern_ctx *ctx;
+	int kt, ctxidx, error;
+
+	switch (csp->csp_auth_alg) {
+	case CRYPTO_SHA1_HMAC:
+		ses->hmac = true;
+		/* FALLTHROUGH */
+	case CRYPTO_SHA1:
+		ses->hash_len = SHA1_HASH_LEN;
+		ses->hash_init = SHA1_Init_fn;
+		ses->hash_update = intel_sha1_update;
+		ses->hash_finalize = SHA1_Finalize_fn;
+		break;
+	case CRYPTO_SHA2_224_HMAC:
+		ses->hmac = true;
+		/* FALLTHROUGH */
+	case CRYPTO_SHA2_224:
+		ses->hash_len = SHA2_224_HASH_LEN;
+		ses->hash_init = SHA224_Init_fn;
+		ses->hash_update = intel_sha256_update;
+		ses->hash_finalize = SHA224_Finalize_fn;
+		break;
+	case CRYPTO_SHA2_256_HMAC:
+		ses->hmac = true;
+		/* FALLTHROUGH */
+	case CRYPTO_SHA2_256:
+		ses->hash_len = SHA2_256_HASH_LEN;
+		ses->hash_init = SHA256_Init_fn;
+		ses->hash_update = intel_sha256_update;
+		ses->hash_finalize = SHA256_Finalize_fn;
+		break;
+	}
+
+	if (ses->hash_len != 0) {
+		if (csp->csp_auth_mlen == 0)
+			ses->mlen = ses->hash_len;
+		else
+			ses->mlen = csp->csp_auth_mlen;
+
+		error = aesni_authprepare(ses, csp->csp_auth_klen);
+		if (error != 0)
+			return (error);
+	}
+
+	error = aesni_cipherprepare(csp);
+	if (error != 0)
+		return (error);
+
+	kt = is_fpu_kern_thread(0) || (csp->csp_cipher_alg == 0);
+	if (!kt) {
+		ACQUIRE_CTX(ctxidx, ctx);
+		fpu_kern_enter(curthread, ctx,
+		    FPU_KERN_NORMAL | FPU_KERN_KTHR);
+	}
+
+	error = 0;
+	if (csp->csp_cipher_key != NULL)
+		aesni_cipher_setup_common(ses, csp, csp->csp_cipher_key,
+		    csp->csp_cipher_klen);
+
+	if (!kt) {
+		fpu_kern_leave(curthread, ctx);
+		RELEASE_CTX(ctxidx, ctx);
+	}
+	return (error);
+}
+
+static int
+aesni_cipher_process(struct aesni_session *ses, struct cryptop *crp)
+{
+	const struct crypto_session_params *csp;
+	struct fpu_kern_ctx *ctx;
 	int error, ctxidx;
 	bool kt;
 
-	if (enccrd != NULL) {
-		if ((enccrd->crd_alg == CRYPTO_AES_ICM ||
-		    enccrd->crd_alg == CRYPTO_AES_CCM_16 ||
-		    enccrd->crd_alg == CRYPTO_AES_NIST_GCM_16) &&
-		    (enccrd->crd_flags & CRD_F_IV_EXPLICIT) == 0)
+	csp = crypto_get_params(crp->crp_session);
+	switch (csp->csp_cipher_alg) {
+	case CRYPTO_AES_ICM:
+	case CRYPTO_AES_NIST_GCM_16:
+	case CRYPTO_AES_CCM_16:
+		if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0)
 			return (EINVAL);
+		break;
+	case CRYPTO_AES_CBC:
+	case CRYPTO_AES_XTS:
+		/* CBC & XTS can only handle full blocks for now */
+		if ((crp->crp_payload_length % AES_BLOCK_LEN) != 0)
+			return (EINVAL);
+		break;
 	}
 
 	ctx = NULL;
 	ctxidx = 0;
 	error = 0;
 	kt = is_fpu_kern_thread(0);
 	if (!kt) {
 		ACQUIRE_CTX(ctxidx, ctx);
 		fpu_kern_enter(curthread, ctx,
 		    FPU_KERN_NORMAL | FPU_KERN_KTHR);
 	}
 
 	/* Do work */
-	if (enccrd != NULL && authcrd != NULL) {
-		/* Perform the first operation */
-		if (crp->crp_desc == enccrd)
-			error = aesni_cipher_crypt(ses, enccrd, authcrd, crp);
-		else
-			error = aesni_cipher_mac(ses, authcrd, crp);
-		if (error != 0)
-			goto out;
-		/* Perform the second operation */
-		if (crp->crp_desc == enccrd)
-			error = aesni_cipher_mac(ses, authcrd, crp);
-		else
-			error = aesni_cipher_crypt(ses, enccrd, authcrd, crp);
-	} else if (enccrd != NULL)
-		error = aesni_cipher_crypt(ses, enccrd, authcrd, crp);
+	if (csp->csp_mode == CSP_MODE_ETA) {
+		if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
+			error = aesni_cipher_crypt(ses, crp, csp);
+			if (error == 0)
+				error = aesni_cipher_mac(ses, crp, csp);
+		} else {
+			error = aesni_cipher_mac(ses, crp, csp);
+			if (error == 0)
+				error = aesni_cipher_crypt(ses, crp, csp);
+		}
+	} else if (csp->csp_mode == CSP_MODE_DIGEST)
+		error = aesni_cipher_mac(ses, crp, csp);
 	else
-		error = aesni_cipher_mac(ses, authcrd, crp);
+		error = aesni_cipher_crypt(ses, crp, csp);
 
-	if (error != 0)
-		goto out;
-
-out:
 	if (!kt) {
 		fpu_kern_leave(curthread, ctx);
 		RELEASE_CTX(ctxidx, ctx);
 	}
 	return (error);
 }
 
 static int
-aesni_cipher_crypt(struct aesni_session *ses, struct cryptodesc *enccrd,
-	struct cryptodesc *authcrd, struct cryptop *crp)
+aesni_cipher_crypt(struct aesni_session *ses, struct cryptop *crp,
+    const struct crypto_session_params *csp)
 {
 	uint8_t iv[AES_BLOCK_LEN], tag[GMAC_DIGEST_LEN], *buf, *authbuf;
-	int error, ivlen;
+	int error;
 	bool encflag, allocated, authallocated;
 
-	KASSERT((ses->algo != CRYPTO_AES_NIST_GCM_16 &&
-		ses->algo != CRYPTO_AES_CCM_16) || authcrd != NULL,
-	    ("AES_NIST_GCM_16/AES_CCM_16  must include MAC descriptor"));
-
-	ivlen = 0;
-	authbuf = NULL;
-
-	buf = aesni_cipher_alloc(enccrd, crp, &allocated);
+	buf = aesni_cipher_alloc(crp, crp->crp_payload_start,
+	    crp->crp_payload_length, &allocated);
 	if (buf == NULL)
 		return (ENOMEM);
 
 	authallocated = false;
-	if (ses->algo == CRYPTO_AES_NIST_GCM_16 ||
-	    ses->algo == CRYPTO_AES_CCM_16) {
-		authbuf = aesni_cipher_alloc(authcrd, crp, &authallocated);
+	authbuf = NULL;
+	if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16 ||
+	    csp->csp_cipher_alg == CRYPTO_AES_CCM_16) {
+		authbuf = aesni_cipher_alloc(crp, crp->crp_aad_start,
+		    crp->crp_aad_length, &authallocated);
 		if (authbuf == NULL) {
 			error = ENOMEM;
 			goto out;
 		}
 	}
 
 	error = 0;
-	encflag = (enccrd->crd_flags & CRD_F_ENCRYPT) == CRD_F_ENCRYPT;
-	if ((enccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) {
-		error = aesni_cipher_setup_common(ses, enccrd->crd_key,
-		    enccrd->crd_klen);
-		if (error != 0)
-			goto out;
-	}
+	encflag = CRYPTO_OP_IS_ENCRYPT(crp->crp_op);
+	if (crp->crp_cipher_key != NULL)
+		aesni_cipher_setup_common(ses, csp, crp->crp_cipher_key,
+		    csp->csp_cipher_klen);
 
-	switch (enccrd->crd_alg) {
-	case CRYPTO_AES_CBC:
-	case CRYPTO_AES_ICM:
-		ivlen = AES_BLOCK_LEN;
-		break;
-	case CRYPTO_AES_XTS:
-		ivlen = 8;
-		break;
-	case CRYPTO_AES_NIST_GCM_16:
-	case CRYPTO_AES_CCM_16:
-		ivlen = 12;	/* should support arbitarily larger */
-		break;
-	}
-
 	/* Setup iv */
-	if (encflag) {
-		if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0)
-			bcopy(enccrd->crd_iv, iv, ivlen);
-		else
-			arc4rand(iv, ivlen, 0);
-		
-		if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0)
-			crypto_copyback(crp->crp_flags, crp->crp_buf,
-			    enccrd->crd_inject, ivlen, iv);
-	} else {
-		if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0)
-			bcopy(enccrd->crd_iv, iv, ivlen);
-		else
-			crypto_copydata(crp->crp_flags, crp->crp_buf,
-			    enccrd->crd_inject, ivlen, iv);
-	}
+	if (crp->crp_flags & CRYPTO_F_IV_GENERATE) {
+		arc4rand(iv, csp->csp_ivlen, 0);
+		crypto_copyback(crp, crp->crp_iv_start, csp->csp_ivlen, iv);
+	} else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+		memcpy(iv, crp->crp_iv, csp->csp_ivlen);
+	else
+		crypto_copydata(crp, crp->crp_iv_start, csp->csp_ivlen, iv);
 
-	switch (ses->algo) {
+	switch (csp->csp_cipher_alg) {
 	case CRYPTO_AES_CBC:
 		if (encflag)
 			aesni_encrypt_cbc(ses->rounds, ses->enc_schedule,
-			    enccrd->crd_len, buf, buf, iv);
+			    crp->crp_payload_length, buf, buf, iv);
 		else
 			aesni_decrypt_cbc(ses->rounds, ses->dec_schedule,
-			    enccrd->crd_len, buf, iv);
+			    crp->crp_payload_length, buf, iv);
 		break;
 	case CRYPTO_AES_ICM:
 		/* encryption & decryption are the same */
 		aesni_encrypt_icm(ses->rounds, ses->enc_schedule,
-		    enccrd->crd_len, buf, buf, iv);
+		    crp->crp_payload_length, buf, buf, iv);
 		break;
 	case CRYPTO_AES_XTS:
 		if (encflag)
 			aesni_encrypt_xts(ses->rounds, ses->enc_schedule,
-			    ses->xts_schedule, enccrd->crd_len, buf, buf,
-			    iv);
+			    ses->xts_schedule, crp->crp_payload_length, buf,
+			    buf, iv);
 		else
 			aesni_decrypt_xts(ses->rounds, ses->dec_schedule,
-			    ses->xts_schedule, enccrd->crd_len, buf, buf,
-			    iv);
+			    ses->xts_schedule, crp->crp_payload_length, buf,
+			    buf, iv);
 		break;
 	case CRYPTO_AES_NIST_GCM_16:
-		if (!encflag)
-			crypto_copydata(crp->crp_flags, crp->crp_buf,
-			    authcrd->crd_inject, sizeof(tag), tag);
-		else
-			bzero(tag, sizeof tag);
-
 		if (encflag) {
+			memset(tag, 0, sizeof(tag));
 			AES_GCM_encrypt(buf, buf, authbuf, iv, tag,
-			    enccrd->crd_len, authcrd->crd_len, ivlen,
-			    ses->enc_schedule, ses->rounds);
-
-			if (authcrd != NULL)
-				crypto_copyback(crp->crp_flags, crp->crp_buf,
-				    authcrd->crd_inject, sizeof(tag), tag);
+			    crp->crp_payload_length, crp->crp_aad_length,
+			    csp->csp_ivlen, ses->enc_schedule, ses->rounds);
+			crypto_copyback(crp, crp->crp_digest_start, sizeof(tag),
+			    tag);
 		} else {
+			crypto_copydata(crp, crp->crp_digest_start, sizeof(tag),
+			    tag);
 			if (!AES_GCM_decrypt(buf, buf, authbuf, iv, tag,
-			    enccrd->crd_len, authcrd->crd_len, ivlen,
-			    ses->enc_schedule, ses->rounds))
+			    crp->crp_payload_length, crp->crp_aad_length,
+			    csp->csp_ivlen, ses->enc_schedule, ses->rounds))
 				error = EBADMSG;
 		}
 		break;
 	case CRYPTO_AES_CCM_16:
-		if (!encflag)
-			crypto_copydata(crp->crp_flags, crp->crp_buf,
-			    authcrd->crd_inject, sizeof(tag), tag);
-		else
-			bzero(tag, sizeof tag);
 		if (encflag) {
+			memset(tag, 0, sizeof(tag));			
 			AES_CCM_encrypt(buf, buf, authbuf, iv, tag,
-			    enccrd->crd_len, authcrd->crd_len, ivlen,
-			    ses->enc_schedule, ses->rounds);
-			if (authcrd != NULL)
-				crypto_copyback(crp->crp_flags, crp->crp_buf,
-				    authcrd->crd_inject, sizeof(tag), tag);
+			    crp->crp_payload_length, crp->crp_aad_length,
+			    csp->csp_ivlen, ses->enc_schedule, ses->rounds);
+			crypto_copyback(crp, crp->crp_digest_start, sizeof(tag),
+			    tag);
 		} else {
+			crypto_copydata(crp, crp->crp_digest_start, sizeof(tag),
+			    tag);
 			if (!AES_CCM_decrypt(buf, buf, authbuf, iv, tag,
-			    enccrd->crd_len, authcrd->crd_len, ivlen,
-			    ses->enc_schedule, ses->rounds))
+			    crp->crp_payload_length, crp->crp_aad_length,
+			    csp->csp_ivlen, ses->enc_schedule, ses->rounds))
 				error = EBADMSG;
 		}
 		break;
 	}
 	if (allocated && error == 0)
-		crypto_copyback(crp->crp_flags, crp->crp_buf, enccrd->crd_skip,
-		    enccrd->crd_len, buf);
+		crypto_copyback(crp, crp->crp_payload_start,
+		    crp->crp_payload_length, buf);
 
 out:
 	if (allocated) {
-		explicit_bzero(buf, enccrd->crd_len);
+		explicit_bzero(buf, crp->crp_payload_length);
 		free(buf, M_AESNI);
 	}
 	if (authallocated) {
-		explicit_bzero(authbuf, authcrd->crd_len);
+		explicit_bzero(authbuf, crp->crp_aad_length);
 		free(authbuf, M_AESNI);
 	}
 	return (error);
 }
 
 static int
-aesni_cipher_mac(struct aesni_session *ses, struct cryptodesc *crd,
-    struct cryptop *crp)
+aesni_cipher_mac(struct aesni_session *ses, struct cryptop *crp,
+    const struct crypto_session_params *csp)
 {
 	union {
 		struct SHA256Context sha2 __aligned(16);
 		struct sha1_ctxt sha1 __aligned(16);
 	} sctx;
+	uint8_t hmac_key[SHA1_BLOCK_LEN] __aligned(16);
 	uint32_t res[SHA2_256_HASH_LEN / sizeof(uint32_t)];
-	int hashlen, error;
-	void *ctx;
-	void (*InitFn)(void *);
-	int (*UpdateFn)(void *, const void *, unsigned);
-	void (*FinalizeFn)(void *, void *);
+	uint32_t res2[SHA2_256_HASH_LEN / sizeof(uint32_t)];
+	const uint8_t *key;
+	int i, keylen;
 
-	bool hmac;
+	if (crp->crp_auth_key != NULL)
+		key = crp->crp_auth_key;
+	else
+		key = csp->csp_auth_key;
+	keylen = csp->csp_auth_klen;
 
-	if ((crd->crd_flags & ~CRD_F_KEY_EXPLICIT) != 0) {
-		CRYPTDEB("%s: Unsupported MAC flags: 0x%x", __func__,
-		    (crd->crd_flags & ~CRD_F_KEY_EXPLICIT));
-		return (EINVAL);
-	}
-	if ((crd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) {
-		error = aesni_authprepare(ses, crd->crd_klen, crd->crd_key);
-		if (error != 0)
-			return (error);
-	}
+	if (ses->hmac) {
+		/* Inner hash: (K ^ IPAD) || data */
+		ses->hash_init(&sctx);
+		for (i = 0; i < keylen; i++)
+			hmac_key[i] = key[i] ^ HMAC_IPAD_VAL;
+		for (i = keylen; i < sizeof(hmac_key); i++)
+			hmac_key[i] = 0 ^ HMAC_IPAD_VAL;
+		ses->hash_update(&sctx, hmac_key, sizeof(hmac_key));
 
-	hmac = false;
-	switch (ses->auth_algo) {
-	case CRYPTO_SHA1_HMAC:
-		hmac = true;
-		/* FALLTHROUGH */
-	case CRYPTO_SHA1:
-		hashlen = SHA1_HASH_LEN;
-		InitFn = SHA1_Init_fn;
-		UpdateFn = intel_sha1_update;
-		FinalizeFn = SHA1_Finalize_fn;
-		ctx = &sctx.sha1;
-		break;
+		crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length,
+		    __DECONST(int (*)(void *, void *, u_int), ses->hash_update),
+		    &sctx);
+		crypto_apply(crp, crp->crp_payload_start,
+		    crp->crp_payload_length,
+		    __DECONST(int (*)(void *, void *, u_int), ses->hash_update),
+		    &sctx);
+		ses->hash_finalize(res, &sctx);
 
-	case CRYPTO_SHA2_256_HMAC:
-		hmac = true;
-		/* FALLTHROUGH */
-	case CRYPTO_SHA2_256:
-		hashlen = SHA2_256_HASH_LEN;
-		InitFn = SHA256_Init_fn;
-		UpdateFn = intel_sha256_update;
-		FinalizeFn = SHA256_Finalize_fn;
-		ctx = &sctx.sha2;
-		break;
-
-	case CRYPTO_SHA2_224_HMAC:
-		hmac = true;
-		/* FALLTHROUGH */
-	case CRYPTO_SHA2_224:
-		hashlen = SHA2_224_HASH_LEN;
-		InitFn = SHA224_Init_fn;
-		UpdateFn = intel_sha256_update;
-		FinalizeFn = SHA224_Finalize_fn;
-		ctx = &sctx.sha2;
-		break;
-	default:
-		/*
-		 * AES-GMAC authentication is verified while processing the
-		 * enccrd
-		 */
-		return (0);
-	}
-
-	if (hmac) {
-		/* Inner hash: (K ^ IPAD) || data */
-		InitFn(ctx);
-		hmac_internal(ctx, res, UpdateFn, FinalizeFn, ses->hmac_key,
-		    0x36, crp->crp_buf, crd->crd_skip, crd->crd_len,
-		    crp->crp_flags);
 		/* Outer hash: (K ^ OPAD) || inner hash */
-		InitFn(ctx);
-		hmac_internal(ctx, res, UpdateFn, FinalizeFn, ses->hmac_key,
-		    0x5C, res, 0, hashlen, 0);
+		ses->hash_init(&sctx);
+		for (i = 0; i < keylen; i++)
+			hmac_key[i] = key[i] ^ HMAC_OPAD_VAL;
+		for (i = keylen; i < sizeof(hmac_key); i++)
+			hmac_key[i] = 0 ^ HMAC_OPAD_VAL;
+		ses->hash_update(&sctx, hmac_key, sizeof(hmac_key));
+		ses->hash_update(&sctx, res, ses->hash_len);
+		ses->hash_finalize(res, &sctx);
 	} else {
-		InitFn(ctx);
-		crypto_apply(crp->crp_flags, crp->crp_buf, crd->crd_skip,
-		    crd->crd_len, __DECONST(int (*)(void *, void *, u_int),
-		    UpdateFn), ctx);
-		FinalizeFn(res, ctx);
-	}
+		ses->hash_init(&sctx);
 
-	if (ses->mlen != 0 && ses->mlen < hashlen)
-		hashlen = ses->mlen;
+		crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length,
+		    __DECONST(int (*)(void *, void *, u_int), ses->hash_update),
+		    &sctx);
+		crypto_apply(crp, crp->crp_payload_start,
+		    crp->crp_payload_length,
+		    __DECONST(int (*)(void *, void *, u_int), ses->hash_update),
+		    &sctx);
 
-	crypto_copyback(crp->crp_flags, crp->crp_buf, crd->crd_inject, hashlen,
-	    (void *)res);
+		ses->hash_finalize(res, &sctx);
+	}
+
+	if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
+		crypto_copydata(crp, crp->crp_digest_start, ses->mlen, res2);
+		if (timingsafe_bcmp(res, res2, ses->mlen) != 0)
+			return (EBADMSG);
+	} else
+		crypto_copyback(crp, crp->crp_digest_start, ses->mlen, res);
 	return (0);
 }
Index: head/sys/crypto/aesni/aesni.h
===================================================================
--- head/sys/crypto/aesni/aesni.h	(revision 359373)
+++ head/sys/crypto/aesni/aesni.h	(revision 359374)
@@ -1,126 +1,126 @@
 /*-
  * Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
  * $FreeBSD$
  */
 
 #ifndef _AESNI_H_
 #define _AESNI_H_
 
 #include <sys/types.h>
 #include <sys/malloc.h>
 #include <sys/queue.h>
 
 #include <opencrypto/cryptodev.h>
 
 #if defined(__amd64__) || defined(__i386__)
 #include <machine/cpufunc.h>
 #include <machine/cputypes.h>
 #include <machine/md_var.h>
 #include <machine/specialreg.h>
 #endif
 #if defined(__i386__)
 #include <machine/npx.h>
 #elif defined(__amd64__)
 #include <machine/fpu.h>
 #endif
 
 #define	AES128_ROUNDS	10
 #define	AES192_ROUNDS	12
 #define	AES256_ROUNDS	14
 #define	AES_SCHED_LEN	((AES256_ROUNDS + 1) * AES_BLOCK_LEN)
 
 struct aesni_session {
 	uint8_t enc_schedule[AES_SCHED_LEN] __aligned(16);
 	uint8_t dec_schedule[AES_SCHED_LEN] __aligned(16);
 	uint8_t xts_schedule[AES_SCHED_LEN] __aligned(16);
-	/* Same as the SHA256 Blocksize. */
-	uint8_t hmac_key[SHA1_BLOCK_LEN] __aligned(16);
-	int algo;
 	int rounds;
 	/* uint8_t *ses_ictx; */
 	/* uint8_t *ses_octx; */
-	/* int ses_mlen; */
 	int used;
-	int auth_algo;
 	int mlen;
+	int hash_len;
+	void (*hash_init)(void *);
+	void (*hash_update)(void *, const void *, unsigned);
+	void (*hash_finalize)(void *, void *);
+	bool hmac;
 };
 
 /*
  * Internal functions, implemented in assembler.
  */
 void aesni_set_enckey(const uint8_t *userkey,
     uint8_t *encrypt_schedule /*__aligned(16)*/, int number_of_rounds);
 void aesni_set_deckey(const uint8_t *encrypt_schedule /*__aligned(16)*/,
     uint8_t *decrypt_schedule /*__aligned(16)*/, int number_of_rounds);
 
 /*
  * Slightly more public interfaces.
  */
 void aesni_encrypt_cbc(int rounds, const void *key_schedule /*__aligned(16)*/,
     size_t len, const uint8_t *from, uint8_t *to,
     const uint8_t iv[__min_size(AES_BLOCK_LEN)]);
 void aesni_decrypt_cbc(int rounds, const void *key_schedule /*__aligned(16)*/,
     size_t len, uint8_t *buf, const uint8_t iv[__min_size(AES_BLOCK_LEN)]);
 void aesni_encrypt_ecb(int rounds, const void *key_schedule /*__aligned(16)*/,
     size_t len, const uint8_t *from, uint8_t *to);
 void aesni_decrypt_ecb(int rounds, const void *key_schedule /*__aligned(16)*/,
     size_t len, const uint8_t *from, uint8_t *to);
 void aesni_encrypt_icm(int rounds, const void *key_schedule /*__aligned(16)*/,
     size_t len, const uint8_t *from, uint8_t *to,
     const uint8_t iv[__min_size(AES_BLOCK_LEN)]);
 
 void aesni_encrypt_xts(int rounds, const void *data_schedule /*__aligned(16)*/,
     const void *tweak_schedule /*__aligned(16)*/, size_t len,
     const uint8_t *from, uint8_t *to,
     const uint8_t iv[__min_size(AES_BLOCK_LEN)]);
 void aesni_decrypt_xts(int rounds, const void *data_schedule /*__aligned(16)*/,
     const void *tweak_schedule /*__aligned(16)*/, size_t len,
     const uint8_t *from, uint8_t *to,
     const uint8_t iv[__min_size(AES_BLOCK_LEN)]);
 
 /* GCM & GHASH functions */
 void AES_GCM_encrypt(const unsigned char *in, unsigned char *out,
     const unsigned char *addt, const unsigned char *ivec,
     unsigned char *tag, uint32_t nbytes, uint32_t abytes, int ibytes,
     const unsigned char *key, int nr);
 int AES_GCM_decrypt(const unsigned char *in, unsigned char *out,
     const unsigned char *addt, const unsigned char *ivec,
     const unsigned char *tag, uint32_t nbytes, uint32_t abytes, int ibytes,
     const unsigned char *key, int nr);
 
 /* CCM + CBC-MAC functions */
 void AES_CCM_encrypt(const unsigned char *in, unsigned char *out,
     const unsigned char *addt, const unsigned char *ivec,
     unsigned char *tag, uint32_t nbytes, uint32_t abytes, int ibytes,
     const unsigned char *key, int nr);
 int AES_CCM_decrypt(const unsigned char *in, unsigned char *out,
     const unsigned char *addt, const unsigned char *ivec,
     const unsigned char *tag, uint32_t nbytes, uint32_t abytes, int ibytes,
     const unsigned char *key, int nr);
-int aesni_cipher_setup_common(struct aesni_session *ses, const uint8_t *key,
-    int keylen);
+void aesni_cipher_setup_common(struct aesni_session *ses,
+    const struct crypto_session_params *csp, const uint8_t *key, int keylen);
 
 #endif /* _AESNI_H_ */
Index: head/sys/crypto/aesni/aesni_wrap.c
===================================================================
--- head/sys/crypto/aesni/aesni_wrap.c	(revision 359373)
+++ head/sys/crypto/aesni/aesni_wrap.c	(revision 359374)
@@ -1,495 +1,479 @@
 /*-
  * Copyright (C) 2008 Damien Miller <djm@mindrot.org>
  * Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org>
  * Copyright (c) 2010-2011 Pawel Jakub Dawidek <pawel@dawidek.net>
  * Copyright 2012-2013 John-Mark Gurney <jmg@FreeBSD.org>
  * Copyright (c) 2014 The FreeBSD Foundation
  * All rights reserved.
  *
  * Portions of this software were developed by John-Mark Gurney
  * under sponsorship of the FreeBSD Foundation and
  * Rubicon Communications, LLC (Netgate).
  * 
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/libkern.h>
 #include <sys/malloc.h>
 #include <sys/proc.h>
 #include <sys/systm.h>
 #include <crypto/aesni/aesni.h>
 
 #include <opencrypto/gmac.h>
 
 #include "aesencdec.h"
 #include <smmintrin.h>
 
 MALLOC_DECLARE(M_AESNI);
 
 struct blocks8 {
 	__m128i	blk[8];
 } __packed;
 
 void
 aesni_encrypt_cbc(int rounds, const void *key_schedule, size_t len,
     const uint8_t *from, uint8_t *to, const uint8_t iv[static AES_BLOCK_LEN])
 {
 	__m128i tot, ivreg;
 	size_t i;
 
 	len /= AES_BLOCK_LEN;
 	ivreg = _mm_loadu_si128((const __m128i *)iv);
 	for (i = 0; i < len; i++) {
 		tot = aesni_enc(rounds - 1, key_schedule,
 		    _mm_loadu_si128((const __m128i *)from) ^ ivreg);
 		ivreg = tot;
 		_mm_storeu_si128((__m128i *)to, tot);
 		from += AES_BLOCK_LEN;
 		to += AES_BLOCK_LEN;
 	}
 }
 
 void
 aesni_decrypt_cbc(int rounds, const void *key_schedule, size_t len,
     uint8_t *buf, const uint8_t iv[static AES_BLOCK_LEN])
 {
 	__m128i blocks[8];
 	struct blocks8 *blks;
 	__m128i ivreg, nextiv;
 	size_t i, j, cnt;
 
 	ivreg = _mm_loadu_si128((const __m128i *)iv);
 	cnt = len / AES_BLOCK_LEN / 8;
 	for (i = 0; i < cnt; i++) {
 		blks = (struct blocks8 *)buf;
 		aesni_dec8(rounds - 1, key_schedule, blks->blk[0], blks->blk[1],
 		    blks->blk[2], blks->blk[3], blks->blk[4], blks->blk[5],
 		    blks->blk[6], blks->blk[7], &blocks[0]);
 		for (j = 0; j < 8; j++) {
 			nextiv = blks->blk[j];
 			blks->blk[j] = blocks[j] ^ ivreg;
 			ivreg = nextiv;
 		}
 		buf += AES_BLOCK_LEN * 8;
 	}
 	i *= 8;
 	cnt = len / AES_BLOCK_LEN;
 	for (; i < cnt; i++) {
 		nextiv = _mm_loadu_si128((void *)buf);
 		_mm_storeu_si128((void *)buf,
 		    aesni_dec(rounds - 1, key_schedule, nextiv) ^ ivreg);
 		ivreg = nextiv;
 		buf += AES_BLOCK_LEN;
 	}
 }
 
 void
 aesni_encrypt_ecb(int rounds, const void *key_schedule, size_t len,
     const uint8_t *from, uint8_t *to)
 {
 	__m128i tot;
 	__m128i tout[8];
 	struct blocks8 *top;
 	const struct blocks8 *blks;
 	size_t i, cnt;
 
 	cnt = len / AES_BLOCK_LEN / 8;
 	for (i = 0; i < cnt; i++) {
 		blks = (const struct blocks8 *)from;
 		top = (struct blocks8 *)to;
 		aesni_enc8(rounds - 1, key_schedule, blks->blk[0], blks->blk[1],
 		    blks->blk[2], blks->blk[3], blks->blk[4], blks->blk[5],
 		    blks->blk[6], blks->blk[7], tout);
 		top->blk[0] = tout[0];
 		top->blk[1] = tout[1];
 		top->blk[2] = tout[2];
 		top->blk[3] = tout[3];
 		top->blk[4] = tout[4];
 		top->blk[5] = tout[5];
 		top->blk[6] = tout[6];
 		top->blk[7] = tout[7];
 		from += AES_BLOCK_LEN * 8;
 		to += AES_BLOCK_LEN * 8;
 	}
 	i *= 8;
 	cnt = len / AES_BLOCK_LEN;
 	for (; i < cnt; i++) {
 		tot = aesni_enc(rounds - 1, key_schedule,
 		    _mm_loadu_si128((const __m128i *)from));
 		_mm_storeu_si128((__m128i *)to, tot);
 		from += AES_BLOCK_LEN;
 		to += AES_BLOCK_LEN;
 	}
 }
 
 void
 aesni_decrypt_ecb(int rounds, const void *key_schedule, size_t len,
     const uint8_t from[AES_BLOCK_LEN], uint8_t to[AES_BLOCK_LEN])
 {
 	__m128i tot;
 	__m128i tout[8];
 	const struct blocks8 *blks;
 	struct blocks8 *top;
 	size_t i, cnt;
 
 	cnt = len / AES_BLOCK_LEN / 8;
 	for (i = 0; i < cnt; i++) {
 		blks = (const struct blocks8 *)from;
 		top = (struct blocks8 *)to;
 		aesni_dec8(rounds - 1, key_schedule, blks->blk[0], blks->blk[1],
 		    blks->blk[2], blks->blk[3], blks->blk[4], blks->blk[5],
 		    blks->blk[6], blks->blk[7], tout);
 		top->blk[0] = tout[0];
 		top->blk[1] = tout[1];
 		top->blk[2] = tout[2];
 		top->blk[3] = tout[3];
 		top->blk[4] = tout[4];
 		top->blk[5] = tout[5];
 		top->blk[6] = tout[6];
 		top->blk[7] = tout[7];
 		from += AES_BLOCK_LEN * 8;
 		to += AES_BLOCK_LEN * 8;
 	}
 	i *= 8;
 	cnt = len / AES_BLOCK_LEN;
 	for (; i < cnt; i++) {
 		tot = aesni_dec(rounds - 1, key_schedule,
 		    _mm_loadu_si128((const __m128i *)from));
 		_mm_storeu_si128((__m128i *)to, tot);
 		from += AES_BLOCK_LEN;
 		to += AES_BLOCK_LEN;
 	}
 }
 
 /*
  * mixed endian increment, low 64bits stored in hi word to be compatible
  * with _icm's BSWAP.
  */
 static inline __m128i
 nextc(__m128i x)
 {
 	const __m128i ONE = _mm_setr_epi32(0, 0, 1, 0);
 	const __m128i ZERO = _mm_setzero_si128();
 
 	x = _mm_add_epi64(x, ONE);
 	__m128i t = _mm_cmpeq_epi64(x, ZERO);
 	t = _mm_unpackhi_epi64(t, ZERO);
 	x = _mm_sub_epi64(x, t);
 
 	return x;
 }
 
 void
 aesni_encrypt_icm(int rounds, const void *key_schedule, size_t len,
     const uint8_t *from, uint8_t *to, const uint8_t iv[static AES_BLOCK_LEN])
 {
 	__m128i tot;
 	__m128i tmp1, tmp2, tmp3, tmp4;
 	__m128i tmp5, tmp6, tmp7, tmp8;
 	__m128i ctr1, ctr2, ctr3, ctr4;
 	__m128i ctr5, ctr6, ctr7, ctr8;
 	__m128i BSWAP_EPI64;
 	__m128i tout[8];
 	struct blocks8 *top;
 	const struct blocks8 *blks;
 	size_t i, cnt;
 
 	BSWAP_EPI64 = _mm_set_epi8(8,9,10,11,12,13,14,15,0,1,2,3,4,5,6,7);
 
 	ctr1 = _mm_loadu_si128((const __m128i *)iv);
 	ctr1 = _mm_shuffle_epi8(ctr1, BSWAP_EPI64);
 
 	cnt = len / AES_BLOCK_LEN / 8;
 	for (i = 0; i < cnt; i++) {
 		tmp1 = _mm_shuffle_epi8(ctr1, BSWAP_EPI64);
 		ctr2 = nextc(ctr1);
 		tmp2 = _mm_shuffle_epi8(ctr2, BSWAP_EPI64);
 		ctr3 = nextc(ctr2);
 		tmp3 = _mm_shuffle_epi8(ctr3, BSWAP_EPI64);
 		ctr4 = nextc(ctr3);
 		tmp4 = _mm_shuffle_epi8(ctr4, BSWAP_EPI64);
 		ctr5 = nextc(ctr4);
 		tmp5 = _mm_shuffle_epi8(ctr5, BSWAP_EPI64);
 		ctr6 = nextc(ctr5);
 		tmp6 = _mm_shuffle_epi8(ctr6, BSWAP_EPI64);
 		ctr7 = nextc(ctr6);
 		tmp7 = _mm_shuffle_epi8(ctr7, BSWAP_EPI64);
 		ctr8 = nextc(ctr7);
 		tmp8 = _mm_shuffle_epi8(ctr8, BSWAP_EPI64);
 		ctr1 = nextc(ctr8);
 
 		blks = (const struct blocks8 *)from;
 		top = (struct blocks8 *)to;
 		aesni_enc8(rounds - 1, key_schedule, tmp1, tmp2, tmp3, tmp4,
 		    tmp5, tmp6, tmp7, tmp8, tout);
 
 		top->blk[0] = blks->blk[0] ^ tout[0];
 		top->blk[1] = blks->blk[1] ^ tout[1];
 		top->blk[2] = blks->blk[2] ^ tout[2];
 		top->blk[3] = blks->blk[3] ^ tout[3];
 		top->blk[4] = blks->blk[4] ^ tout[4];
 		top->blk[5] = blks->blk[5] ^ tout[5];
 		top->blk[6] = blks->blk[6] ^ tout[6];
 		top->blk[7] = blks->blk[7] ^ tout[7];
 
 		from += AES_BLOCK_LEN * 8;
 		to += AES_BLOCK_LEN * 8;
 	}
 	i *= 8;
 	cnt = len / AES_BLOCK_LEN;
 	for (; i < cnt; i++) {
 		tmp1 = _mm_shuffle_epi8(ctr1, BSWAP_EPI64);
 		ctr1 = nextc(ctr1);
 
 		tot = aesni_enc(rounds - 1, key_schedule, tmp1);
 
 		tot = tot ^ _mm_loadu_si128((const __m128i *)from);
 		_mm_storeu_si128((__m128i *)to, tot);
 
 		from += AES_BLOCK_LEN;
 		to += AES_BLOCK_LEN;
 	}
 
 	/* handle remaining partial round */
 	if (len % AES_BLOCK_LEN != 0) {
 		tmp1 = _mm_shuffle_epi8(ctr1, BSWAP_EPI64);
 		tot = aesni_enc(rounds - 1, key_schedule, tmp1);
 		tot = tot ^ _mm_loadu_si128((const __m128i *)from);
 		memcpy(to, &tot, len % AES_BLOCK_LEN);
 	}
 }
 
 #define	AES_XTS_BLOCKSIZE	16
 #define	AES_XTS_IVSIZE		8
 #define	AES_XTS_ALPHA		0x87	/* GF(2^128) generator polynomial */
 
 static inline __m128i
 xts_crank_lfsr(__m128i inp)
 {
 	const __m128i alphamask = _mm_set_epi32(1, 1, 1, AES_XTS_ALPHA);
 	__m128i xtweak, ret;
 
 	/* set up xor mask */
 	xtweak = _mm_shuffle_epi32(inp, 0x93);
 	xtweak = _mm_srai_epi32(xtweak, 31);
 	xtweak &= alphamask;
 
 	/* next term */
 	ret = _mm_slli_epi32(inp, 1);
 	ret ^= xtweak;
 
 	return ret;
 }
 
 static void
 aesni_crypt_xts_block(int rounds, const __m128i *key_schedule, __m128i *tweak,
     const uint8_t *from, uint8_t *to, int do_encrypt)
 {
 	__m128i block;
 
 	block = _mm_loadu_si128((const __m128i *)from) ^ *tweak;
 
 	if (do_encrypt)
 		block = aesni_enc(rounds - 1, key_schedule, block);
 	else
 		block = aesni_dec(rounds - 1, key_schedule, block);
 
 	_mm_storeu_si128((__m128i *)to, block ^ *tweak);
 
 	*tweak = xts_crank_lfsr(*tweak);
 }
 
 static void
 aesni_crypt_xts_block8(int rounds, const __m128i *key_schedule, __m128i *tweak,
     const uint8_t *from, uint8_t *to, int do_encrypt)
 {
 	__m128i tmptweak;
 	__m128i a, b, c, d, e, f, g, h;
 	__m128i tweaks[8];
 	__m128i tmp[8];
 	__m128i *top;
 	const __m128i *fromp;
 
 	tmptweak = *tweak;
 
 	/*
 	 * unroll the loop.  This lets gcc put values directly in the
 	 * register and saves memory accesses.
 	 */
 	fromp = (const __m128i *)from;
 #define PREPINP(v, pos) 					\
 		do {						\
 			tweaks[(pos)] = tmptweak;		\
 			(v) = _mm_loadu_si128(&fromp[pos]) ^	\
 			    tmptweak;				\
 			tmptweak = xts_crank_lfsr(tmptweak);	\
 		} while (0)
 	PREPINP(a, 0);
 	PREPINP(b, 1);
 	PREPINP(c, 2);
 	PREPINP(d, 3);
 	PREPINP(e, 4);
 	PREPINP(f, 5);
 	PREPINP(g, 6);
 	PREPINP(h, 7);
 	*tweak = tmptweak;
 
 	if (do_encrypt)
 		aesni_enc8(rounds - 1, key_schedule, a, b, c, d, e, f, g, h,
 		    tmp);
 	else
 		aesni_dec8(rounds - 1, key_schedule, a, b, c, d, e, f, g, h,
 		    tmp);
 
 	top = (__m128i *)to;
 	_mm_storeu_si128(&top[0], tmp[0] ^ tweaks[0]);
 	_mm_storeu_si128(&top[1], tmp[1] ^ tweaks[1]);
 	_mm_storeu_si128(&top[2], tmp[2] ^ tweaks[2]);
 	_mm_storeu_si128(&top[3], tmp[3] ^ tweaks[3]);
 	_mm_storeu_si128(&top[4], tmp[4] ^ tweaks[4]);
 	_mm_storeu_si128(&top[5], tmp[5] ^ tweaks[5]);
 	_mm_storeu_si128(&top[6], tmp[6] ^ tweaks[6]);
 	_mm_storeu_si128(&top[7], tmp[7] ^ tweaks[7]);
 }
 
 static void
 aesni_crypt_xts(int rounds, const __m128i *data_schedule,
     const __m128i *tweak_schedule, size_t len, const uint8_t *from,
     uint8_t *to, const uint8_t iv[static AES_BLOCK_LEN], int do_encrypt)
 {
 	__m128i tweakreg;
 	uint8_t tweak[AES_XTS_BLOCKSIZE] __aligned(16);
 	size_t i, cnt;
 
 	/*
 	 * Prepare tweak as E_k2(IV). IV is specified as LE representation
 	 * of a 64-bit block number which we allow to be passed in directly.
 	 */
 #if BYTE_ORDER == LITTLE_ENDIAN
 	bcopy(iv, tweak, AES_XTS_IVSIZE);
 	/* Last 64 bits of IV are always zero. */
 	bzero(tweak + AES_XTS_IVSIZE, AES_XTS_IVSIZE);
 #else
 #error Only LITTLE_ENDIAN architectures are supported.
 #endif
 	tweakreg = _mm_loadu_si128((__m128i *)&tweak[0]);
 	tweakreg = aesni_enc(rounds - 1, tweak_schedule, tweakreg);
 
 	cnt = len / AES_XTS_BLOCKSIZE / 8;
 	for (i = 0; i < cnt; i++) {
 		aesni_crypt_xts_block8(rounds, data_schedule, &tweakreg,
 		    from, to, do_encrypt);
 		from += AES_XTS_BLOCKSIZE * 8;
 		to += AES_XTS_BLOCKSIZE * 8;
 	}
 	i *= 8;
 	cnt = len / AES_XTS_BLOCKSIZE;
 	for (; i < cnt; i++) {
 		aesni_crypt_xts_block(rounds, data_schedule, &tweakreg,
 		    from, to, do_encrypt);
 		from += AES_XTS_BLOCKSIZE;
 		to += AES_XTS_BLOCKSIZE;
 	}
 }
 
 void
 aesni_encrypt_xts(int rounds, const void *data_schedule,
     const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to,
     const uint8_t iv[static AES_BLOCK_LEN])
 {
 
 	aesni_crypt_xts(rounds, data_schedule, tweak_schedule, len, from, to,
 	    iv, 1);
 }
 
 void
 aesni_decrypt_xts(int rounds, const void *data_schedule,
     const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to,
     const uint8_t iv[static AES_BLOCK_LEN])
 {
 
 	aesni_crypt_xts(rounds, data_schedule, tweak_schedule, len, from, to,
 	    iv, 0);
 }
 
-int
-aesni_cipher_setup_common(struct aesni_session *ses, const uint8_t *key,
-    int keylen)
+void
+aesni_cipher_setup_common(struct aesni_session *ses,
+    const struct crypto_session_params *csp, const uint8_t *key, int keylen)
 {
 	int decsched;
 
 	decsched = 1;
 
-	switch (ses->algo) {
+	switch (csp->csp_cipher_alg) {
 	case CRYPTO_AES_ICM:
 	case CRYPTO_AES_NIST_GCM_16:
 	case CRYPTO_AES_CCM_16:
 		decsched = 0;
-		/* FALLTHROUGH */
-	case CRYPTO_AES_CBC:
-		switch (keylen) {
-		case 128:
-			ses->rounds = AES128_ROUNDS;
-			break;
-		case 192:
-			ses->rounds = AES192_ROUNDS;
-			break;
-		case 256:
-			ses->rounds = AES256_ROUNDS;
-			break;
-		default:
-			CRYPTDEB("invalid CBC/ICM/GCM key length");
-			return (EINVAL);
-		}
 		break;
-	case CRYPTO_AES_XTS:
-		switch (keylen) {
-		case 256:
-			ses->rounds = AES128_ROUNDS;
-			break;
-		case 512:
-			ses->rounds = AES256_ROUNDS;
-			break;
-		default:
-			CRYPTDEB("invalid XTS key length");
-			return (EINVAL);
-		}
+	}
+
+	if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
+		keylen /= 2;
+
+	switch (keylen * 8) {
+	case 128:
+		ses->rounds = AES128_ROUNDS;
 		break;
+	case 192:
+		ses->rounds = AES192_ROUNDS;
+		break;
+	case 256:
+		ses->rounds = AES256_ROUNDS;
+		break;
 	default:
-		return (EINVAL);
+		panic("shouldn't happen");
 	}
 
 	aesni_set_enckey(key, ses->enc_schedule, ses->rounds);
 	if (decsched)
 		aesni_set_deckey(ses->enc_schedule, ses->dec_schedule,
 		    ses->rounds);
 
-	if (ses->algo == CRYPTO_AES_XTS)
-		aesni_set_enckey(key + keylen / 16, ses->xts_schedule,
+	if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
+		aesni_set_enckey(key + keylen, ses->xts_schedule,
 		    ses->rounds);
-
-	return (0);
 }
Index: head/sys/crypto/armv8/armv8_crypto.c
===================================================================
--- head/sys/crypto/armv8/armv8_crypto.c	(revision 359373)
+++ head/sys/crypto/armv8/armv8_crypto.c	(revision 359374)
@@ -1,472 +1,392 @@
 /*-
  * Copyright (c) 2005-2008 Pawel Jakub Dawidek <pjd@FreeBSD.org>
  * Copyright (c) 2010 Konstantin Belousov <kib@FreeBSD.org>
  * Copyright (c) 2014,2016 The FreeBSD Foundation
  * All rights reserved.
  *
  * Portions of this software were developed by John-Mark Gurney
  * under sponsorship of the FreeBSD Foundation and
  * Rubicon Communications, LLC (Netgate).
  *
  * This software was developed by Andrew Turner under
  * sponsorship from the FreeBSD Foundation.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 /*
  * This is based on the aesni code.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/bus.h>
 #include <sys/endian.h>
 #include <sys/lock.h>
 #include <sys/malloc.h>
 #include <sys/mbuf.h>
 #include <sys/module.h>
 #include <sys/mutex.h>
 #include <sys/queue.h>
 #include <sys/rwlock.h>
 #include <sys/smp.h>
 #include <sys/uio.h>
 
 #include <machine/vfp.h>
 
 #include <opencrypto/cryptodev.h>
 #include <cryptodev_if.h>
 #include <crypto/armv8/armv8_crypto.h>
 #include <crypto/rijndael/rijndael.h>
 
 struct armv8_crypto_softc {
 	int		dieing;
 	int32_t		cid;
 	struct rwlock	lock;
 };
 
 static struct mtx *ctx_mtx;
 static struct fpu_kern_ctx **ctx_vfp;
 
 #define AQUIRE_CTX(i, ctx)					\
 	do {							\
 		(i) = PCPU_GET(cpuid);				\
 		mtx_lock(&ctx_mtx[(i)]);			\
 		(ctx) = ctx_vfp[(i)];				\
 	} while (0)
 #define RELEASE_CTX(i, ctx)					\
 	do {							\
 		mtx_unlock(&ctx_mtx[(i)]);			\
 		(i) = -1;					\
 		(ctx) = NULL;					\
 	} while (0)
 
 static int armv8_crypto_cipher_process(struct armv8_crypto_session *,
-    struct cryptodesc *, struct cryptop *);
+    struct cryptop *);
 
 MALLOC_DEFINE(M_ARMV8_CRYPTO, "armv8_crypto", "ARMv8 Crypto Data");
 
 static void
 armv8_crypto_identify(driver_t *drv, device_t parent)
 {
 
 	/* NB: order 10 is so we get attached after h/w devices */
 	if (device_find_child(parent, "armv8crypto", -1) == NULL &&
 	    BUS_ADD_CHILD(parent, 10, "armv8crypto", -1) == 0)
 		panic("ARMv8 crypto: could not attach");
 }
 
 static int
 armv8_crypto_probe(device_t dev)
 {
 	uint64_t reg;
 	int ret = ENXIO;
 
 	reg = READ_SPECIALREG(id_aa64isar0_el1);
 
 	switch (ID_AA64ISAR0_AES_VAL(reg)) {
 	case ID_AA64ISAR0_AES_BASE:
 	case ID_AA64ISAR0_AES_PMULL:
 		ret = 0;
 		break;
 	}
 
 	device_set_desc_copy(dev, "AES-CBC");
 
 	/* TODO: Check more fields as we support more features */
 
 	return (ret);
 }
 
 static int
 armv8_crypto_attach(device_t dev)
 {
 	struct armv8_crypto_softc *sc;
 	int i;
 
 	sc = device_get_softc(dev);
 	sc->dieing = 0;
 
 	sc->cid = crypto_get_driverid(dev, sizeof(struct armv8_crypto_session),
-	    CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SYNC);
+	    CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC);
 	if (sc->cid < 0) {
 		device_printf(dev, "Could not get crypto driver id.\n");
 		return (ENOMEM);
 	}
 
 	rw_init(&sc->lock, "armv8crypto");
 
 	ctx_mtx = malloc(sizeof(*ctx_mtx) * (mp_maxid + 1), M_ARMV8_CRYPTO,
 	    M_WAITOK|M_ZERO);
 	ctx_vfp = malloc(sizeof(*ctx_vfp) * (mp_maxid + 1), M_ARMV8_CRYPTO,
 	    M_WAITOK|M_ZERO);
 
 	CPU_FOREACH(i) {
 		ctx_vfp[i] = fpu_kern_alloc_ctx(0);
 		mtx_init(&ctx_mtx[i], "armv8cryptoctx", NULL, MTX_DEF|MTX_NEW);
 	}
 
-	crypto_register(sc->cid, CRYPTO_AES_CBC, 0, 0);
-
 	return (0);
 }
 
 static int
 armv8_crypto_detach(device_t dev)
 {
 	struct armv8_crypto_softc *sc;
 	int i;
 
 	sc = device_get_softc(dev);
 
 	rw_wlock(&sc->lock);
 	sc->dieing = 1;
 	rw_wunlock(&sc->lock);
 	crypto_unregister_all(sc->cid);
 
 	rw_destroy(&sc->lock);
 
 	CPU_FOREACH(i) {
 		if (ctx_vfp[i] != NULL) {
 			mtx_destroy(&ctx_mtx[i]);
 			fpu_kern_free_ctx(ctx_vfp[i]);
 		}
 		ctx_vfp[i] = NULL;
 	}
 	free(ctx_mtx, M_ARMV8_CRYPTO);
 	ctx_mtx = NULL;
 	free(ctx_vfp, M_ARMV8_CRYPTO);
 	ctx_vfp = NULL;
 
 	return (0);
 }
 
 static int
-armv8_crypto_cipher_setup(struct armv8_crypto_session *ses,
-    struct cryptoini *encini)
+armv8_crypto_probesession(device_t dev,
+    const struct crypto_session_params *csp)
 {
-	int i;
 
-	switch (ses->algo) {
-	case CRYPTO_AES_CBC:
-		switch (encini->cri_klen) {
-		case 128:
-			ses->rounds = AES128_ROUNDS;
+	if (csp->csp_flags != 0)
+		return (EINVAL);
+	switch (csp->csp_mode) {
+	case CSP_MODE_CIPHER:
+		switch (csp->csp_cipher_alg) {
+		case CRYPTO_AES_CBC:
+			if (csp->csp_ivlen != AES_BLOCK_LEN)
+				return (EINVAL);
+			switch (csp->csp_cipher_klen * 8) {
+			case 128:
+			case 192:
+			case 256:
+				break;
+			default:
+				return (EINVAL);
+			}
 			break;
-		case 192:
-			ses->rounds = AES192_ROUNDS;
-			break;
-		case 256:
-			ses->rounds = AES256_ROUNDS;
-			break;
 		default:
-			CRYPTDEB("invalid CBC/ICM/GCM key length");
 			return (EINVAL);
 		}
-		break;
 	default:
 		return (EINVAL);
 	}
+	return (CRYPTODEV_PROBE_ACCEL_SOFTWARE);		
+}
 
-	rijndaelKeySetupEnc(ses->enc_schedule, encini->cri_key,
-	    encini->cri_klen);
-	rijndaelKeySetupDec(ses->dec_schedule, encini->cri_key,
-	    encini->cri_klen);
+static void
+armv8_crypto_cipher_setup(struct armv8_crypto_session *ses,
+    const struct crypto_session_params *csp)
+{
+	int i;
+
+	switch (csp->csp_cipher_klen * 8) {
+	case 128:
+		ses->rounds = AES128_ROUNDS;
+		break;
+	case 192:
+		ses->rounds = AES192_ROUNDS;
+		break;
+	case 256:
+		ses->rounds = AES256_ROUNDS;
+		break;
+	default:
+		panic("invalid CBC key length");
+	}
+
+	rijndaelKeySetupEnc(ses->enc_schedule, csp->csp_cipher_key,
+	    csp->csp_cipher_klen * 8);
+	rijndaelKeySetupDec(ses->dec_schedule, csp->csp_cipher_key,
+	    csp->csp_cipher_klen * 8);
 	for (i = 0; i < nitems(ses->enc_schedule); i++) {
 		ses->enc_schedule[i] = bswap32(ses->enc_schedule[i]);
 		ses->dec_schedule[i] = bswap32(ses->dec_schedule[i]);
 	}
-
-	return (0);
 }
 
 static int
 armv8_crypto_newsession(device_t dev, crypto_session_t cses,
-    struct cryptoini *cri)
+    const struct crypto_session_params *csp)
 {
 	struct armv8_crypto_softc *sc;
 	struct armv8_crypto_session *ses;
-	struct cryptoini *encini;
-	int error;
 
-	if (cri == NULL) {
-		CRYPTDEB("no cri");
-		return (EINVAL);
-	}
-
 	sc = device_get_softc(dev);
-	if (sc->dieing)
-		return (EINVAL);
-
-	ses = NULL;
-	encini = NULL;
-	for (; cri != NULL; cri = cri->cri_next) {
-		switch (cri->cri_alg) {
-		case CRYPTO_AES_CBC:
-			if (encini != NULL) {
-				CRYPTDEB("encini already set");
-				return (EINVAL);
-			}
-			encini = cri;
-			break;
-		default:
-			CRYPTDEB("unhandled algorithm");
-			return (EINVAL);
-		}
-	}
-	if (encini == NULL) {
-		CRYPTDEB("no cipher");
-		return (EINVAL);
-	}
-
 	rw_wlock(&sc->lock);
 	if (sc->dieing) {
 		rw_wunlock(&sc->lock);
 		return (EINVAL);
 	}
 
 	ses = crypto_get_driver_session(cses);
-	ses->algo = encini->cri_alg;
-
-	error = armv8_crypto_cipher_setup(ses, encini);
-	if (error != 0) {
-		CRYPTDEB("setup failed");
-		rw_wunlock(&sc->lock);
-		return (error);
-	}
-
+	armv8_crypto_cipher_setup(ses, csp);
 	rw_wunlock(&sc->lock);
 	return (0);
 }
 
 static int
 armv8_crypto_process(device_t dev, struct cryptop *crp, int hint __unused)
 {
-	struct cryptodesc *crd, *enccrd;
 	struct armv8_crypto_session *ses;
 	int error;
 
-	error = 0;
-	enccrd = NULL;
-
-	/* Sanity check. */
-	if (crp == NULL)
-		return (EINVAL);
-
-	if (crp->crp_callback == NULL || crp->crp_desc == NULL) {
-		error = EINVAL;
-		goto out;
-	}
-
-	for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) {
-		switch (crd->crd_alg) {
-		case CRYPTO_AES_CBC:
-			if (enccrd != NULL) {
-				error = EINVAL;
-				goto out;
-			}
-			enccrd = crd;
-			break;
-		default:
-			error = EINVAL;
-			goto out;
-		}
-	}
-
-	if (enccrd == NULL) {
-		error = EINVAL;
-		goto out;
-	}
-
 	/* We can only handle full blocks for now */
-	if ((enccrd->crd_len % AES_BLOCK_LEN) != 0) {
+	if ((crp->crp_payload_length % AES_BLOCK_LEN) != 0) {
 		error = EINVAL;
 		goto out;
 	}
 
 	ses = crypto_get_driver_session(crp->crp_session);
-	error = armv8_crypto_cipher_process(ses, enccrd, crp);
+	error = armv8_crypto_cipher_process(ses, crp);
 
 out:
 	crp->crp_etype = error;
 	crypto_done(crp);
 	return (error);
 }
 
 static uint8_t *
-armv8_crypto_cipher_alloc(struct cryptodesc *enccrd, struct cryptop *crp,
-    int *allocated)
+armv8_crypto_cipher_alloc(struct cryptop *crp, int *allocated)
 {
-	struct mbuf *m;
-	struct uio *uio;
-	struct iovec *iov;
 	uint8_t *addr;
 
-	if (crp->crp_flags & CRYPTO_F_IMBUF) {
-		m = (struct mbuf *)crp->crp_buf;
-		if (m->m_next != NULL)
-			goto alloc;
-		addr = mtod(m, uint8_t *);
-	} else if (crp->crp_flags & CRYPTO_F_IOV) {
-		uio = (struct uio *)crp->crp_buf;
-		if (uio->uio_iovcnt != 1)
-			goto alloc;
-		iov = uio->uio_iov;
-		addr = (uint8_t *)iov->iov_base;
-	} else
-		addr = (uint8_t *)crp->crp_buf;
-	*allocated = 0;
-	addr += enccrd->crd_skip;
-	return (addr);
-
-alloc:
-	addr = malloc(enccrd->crd_len, M_ARMV8_CRYPTO, M_NOWAIT);
+	addr = crypto_contiguous_subsegment(crp, crp->crp_payload_start,
+	    crp->crp_payload_length);
 	if (addr != NULL) {
+		*allocated = 0;
+		return (addr);
+	}
+	addr = malloc(crp->crp_payload_length, M_ARMV8_CRYPTO, M_NOWAIT);
+	if (addr != NULL) {
 		*allocated = 1;
-		crypto_copydata(crp->crp_flags, crp->crp_buf, enccrd->crd_skip,
-		    enccrd->crd_len, addr);
+		crypto_copydata(crp, crp->crp_payload_start,
+		    crp->crp_payload_length, addr);
 	} else
 		*allocated = 0;
 	return (addr);
 }
 
 static int
 armv8_crypto_cipher_process(struct armv8_crypto_session *ses,
-    struct cryptodesc *enccrd, struct cryptop *crp)
+    struct cryptop *crp)
 {
+	const struct crypto_session_params *csp;
 	struct fpu_kern_ctx *ctx;
 	uint8_t *buf;
 	uint8_t iv[AES_BLOCK_LEN];
 	int allocated, i;
-	int encflag, ivlen;
+	int encflag;
 	int kt;
 
-	encflag = (enccrd->crd_flags & CRD_F_ENCRYPT) == CRD_F_ENCRYPT;
+	csp = crypto_get_params(crp->crp_session);
+	encflag = CRYPTO_OP_IS_ENCRYPT(crp->crp_op);
 
-	buf = armv8_crypto_cipher_alloc(enccrd, crp, &allocated);
+	buf = armv8_crypto_cipher_alloc(crp, &allocated);
 	if (buf == NULL)
 		return (ENOMEM);
 
 	kt = is_fpu_kern_thread(0);
 	if (!kt) {
 		AQUIRE_CTX(i, ctx);
 		fpu_kern_enter(curthread, ctx,
 		    FPU_KERN_NORMAL | FPU_KERN_KTHR);
 	}
 
-	if ((enccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) {
-		panic("CRD_F_KEY_EXPLICIT");
+	if (crp->crp_cipher_key != NULL) {
+		panic("armv8: new cipher key");
 	}
 
-	switch (enccrd->crd_alg) {
-	case CRYPTO_AES_CBC:
-		ivlen = AES_BLOCK_LEN;
-		break;
-	}
-
 	/* Setup iv */
-	if (encflag) {
-		if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0)
-			bcopy(enccrd->crd_iv, iv, ivlen);
-		else
-			arc4rand(iv, ivlen, 0);
+	if (crp->crp_flags & CRYPTO_F_IV_GENERATE) {
+		arc4rand(iv, csp->csp_ivlen, 0);
+		crypto_copyback(crp, crp->crp_iv_start, csp->csp_ivlen, iv);
+	} else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+		memcpy(iv, crp->crp_iv, csp->csp_ivlen);
+	else
+		crypto_copydata(crp, crp->crp_iv_start, csp->csp_ivlen, iv);
 
-		if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0)
-			crypto_copyback(crp->crp_flags, crp->crp_buf,
-			    enccrd->crd_inject, ivlen, iv);
-	} else {
-		if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0)
-			bcopy(enccrd->crd_iv, iv, ivlen);
-		else
-			crypto_copydata(crp->crp_flags, crp->crp_buf,
-			    enccrd->crd_inject, ivlen, iv);
-	}
-
 	/* Do work */
-	switch (ses->algo) {
+	switch (csp->csp_cipher_alg) {
 	case CRYPTO_AES_CBC:
 		if (encflag)
 			armv8_aes_encrypt_cbc(ses->rounds, ses->enc_schedule,
-			    enccrd->crd_len, buf, buf, iv);
+			    crp->crp_payload_length, buf, buf, iv);
 		else
 			armv8_aes_decrypt_cbc(ses->rounds, ses->dec_schedule,
-			    enccrd->crd_len, buf, iv);
+			    crp->crp_payload_length, buf, iv);
 		break;
 	}
 
 	if (allocated)
-		crypto_copyback(crp->crp_flags, crp->crp_buf, enccrd->crd_skip,
-		    enccrd->crd_len, buf);
+		crypto_copyback(crp, crp->crp_payload_start,
+		    crp->crp_payload_length, buf);
 
 	if (!kt) {
 		fpu_kern_leave(curthread, ctx);
 		RELEASE_CTX(i, ctx);
 	}
 	if (allocated) {
-		bzero(buf, enccrd->crd_len);
+		bzero(buf, crp->crp_payload_length);
 		free(buf, M_ARMV8_CRYPTO);
 	}
 	return (0);
 }
 
 static device_method_t armv8_crypto_methods[] = {
 	DEVMETHOD(device_identify,	armv8_crypto_identify),
 	DEVMETHOD(device_probe,		armv8_crypto_probe),
 	DEVMETHOD(device_attach,	armv8_crypto_attach),
 	DEVMETHOD(device_detach,	armv8_crypto_detach),
 
+	DEVMETHOD(cryptodev_probesession, armv8_crypto_probesession),
 	DEVMETHOD(cryptodev_newsession,	armv8_crypto_newsession),
 	DEVMETHOD(cryptodev_process,	armv8_crypto_process),
 
 	DEVMETHOD_END,
 };
 
 static DEFINE_CLASS_0(armv8crypto, armv8_crypto_driver, armv8_crypto_methods,
     sizeof(struct armv8_crypto_softc));
 static devclass_t armv8_crypto_devclass;
 
 DRIVER_MODULE(armv8crypto, nexus, armv8_crypto_driver, armv8_crypto_devclass,
     0, 0);
Index: head/sys/crypto/blake2/blake2_cryptodev.c
===================================================================
--- head/sys/crypto/blake2/blake2_cryptodev.c	(revision 359373)
+++ head/sys/crypto/blake2/blake2_cryptodev.c	(revision 359374)
@@ -1,447 +1,412 @@
 /*-
  * Copyright (c) 2018 Conrad Meyer <cem@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/bus.h>
 #include <sys/kernel.h>
 #include <sys/kobj.h>
 #include <sys/lock.h>
 #include <sys/malloc.h>
 #include <sys/module.h>
 #include <sys/mutex.h>
 #include <sys/rwlock.h>
 #include <sys/smp.h>
 
 #include <blake2.h>
 
 #include <opencrypto/cryptodev.h>
 #include <cryptodev_if.h>
 
 #if defined(__amd64__)
 #include <machine/fpu.h>
 #elif defined(__i386__)
 #include <machine/npx.h>
 #endif
 
 struct blake2_session {
-	int algo;
-	size_t klen;
 	size_t mlen;
-	uint8_t key[BLAKE2B_KEYBYTES];
 };
 CTASSERT((size_t)BLAKE2B_KEYBYTES > (size_t)BLAKE2S_KEYBYTES);
 
 struct blake2_softc {
 	bool	dying;
 	int32_t cid;
 	struct rwlock lock;
 };
 
 static struct mtx_padalign *ctx_mtx;
 static struct fpu_kern_ctx **ctx_fpu;
 
 #define ACQUIRE_CTX(i, ctx)					\
 	do {							\
 		(i) = PCPU_GET(cpuid);				\
 		mtx_lock(&ctx_mtx[(i)]);			\
 		(ctx) = ctx_fpu[(i)];				\
 	} while (0)
 #define RELEASE_CTX(i, ctx)					\
 	do {							\
 		mtx_unlock(&ctx_mtx[(i)]);			\
 		(i) = -1;					\
 		(ctx) = NULL;					\
 	} while (0)
 
-static int blake2_newsession(device_t, crypto_session_t cses,
-    struct cryptoini *cri);
 static int blake2_cipher_setup(struct blake2_session *ses,
-    struct cryptoini *authini);
+    const struct crypto_session_params *csp);
 static int blake2_cipher_process(struct blake2_session *ses,
     struct cryptop *crp);
 
 MALLOC_DEFINE(M_BLAKE2, "blake2_data", "Blake2 Data");
 
 static void
 blake2_identify(driver_t *drv, device_t parent)
 {
 
 	/* NB: order 10 is so we get attached after h/w devices */
 	if (device_find_child(parent, "blaketwo", -1) == NULL &&
 	    BUS_ADD_CHILD(parent, 10, "blaketwo", -1) == 0)
 		panic("blaketwo: could not attach");
 }
 
 static int
 blake2_probe(device_t dev)
 {
 	device_set_desc(dev, "Blake2");
 	return (0);
 }
 
 static void
 blake2_cleanctx(void)
 {
 	int i;
 
 	/* XXX - no way to return driverid */
 	CPU_FOREACH(i) {
 		if (ctx_fpu[i] != NULL) {
 			mtx_destroy(&ctx_mtx[i]);
 			fpu_kern_free_ctx(ctx_fpu[i]);
 		}
 		ctx_fpu[i] = NULL;
 	}
 	free(ctx_mtx, M_BLAKE2);
 	ctx_mtx = NULL;
 	free(ctx_fpu, M_BLAKE2);
 	ctx_fpu = NULL;
 }
 
 static int
 blake2_attach(device_t dev)
 {
 	struct blake2_softc *sc;
 	int i;
 
 	sc = device_get_softc(dev);
 	sc->dying = false;
 
 	sc->cid = crypto_get_driverid(dev, sizeof(struct blake2_session),
-	    CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SYNC);
+	    CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC);
 	if (sc->cid < 0) {
 		device_printf(dev, "Could not get crypto driver id.\n");
 		return (ENOMEM);
 	}
 
 	ctx_mtx = malloc(sizeof(*ctx_mtx) * (mp_maxid + 1), M_BLAKE2,
 	    M_WAITOK | M_ZERO);
 	ctx_fpu = malloc(sizeof(*ctx_fpu) * (mp_maxid + 1), M_BLAKE2,
 	    M_WAITOK | M_ZERO);
 
 	CPU_FOREACH(i) {
 		ctx_fpu[i] = fpu_kern_alloc_ctx(0);
 		mtx_init(&ctx_mtx[i], "bl2fpumtx", NULL, MTX_DEF | MTX_NEW);
 	}
 
 	rw_init(&sc->lock, "blake2_lock");
 
-	crypto_register(sc->cid, CRYPTO_BLAKE2B, 0, 0);
-	crypto_register(sc->cid, CRYPTO_BLAKE2S, 0, 0);
 	return (0);
 }
 
 static int
 blake2_detach(device_t dev)
 {
 	struct blake2_softc *sc;
 
 	sc = device_get_softc(dev);
 
 	rw_wlock(&sc->lock);
 	sc->dying = true;
 	rw_wunlock(&sc->lock);
 	crypto_unregister_all(sc->cid);
 
 	rw_destroy(&sc->lock);
 
 	blake2_cleanctx();
 
 	return (0);
 }
 
 static int
-blake2_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
+blake2_probesession(device_t dev, const struct crypto_session_params *csp)
 {
-	struct blake2_softc *sc;
-	struct blake2_session *ses;
-	struct cryptoini *authini;
-	int error;
 
-	if (cri == NULL) {
-		CRYPTDEB("no cri");
+	if (csp->csp_flags != 0)
 		return (EINVAL);
-	}
-
-	sc = device_get_softc(dev);
-
-	authini = NULL;
-	for (; cri != NULL; cri = cri->cri_next) {
-		switch (cri->cri_alg) {
+	switch (csp->csp_mode) {
+	case CSP_MODE_DIGEST:
+		switch (csp->csp_auth_alg) {
 		case CRYPTO_BLAKE2B:
 		case CRYPTO_BLAKE2S:
-			if (authini != NULL) {
-				CRYPTDEB("authini already set");
-				return (EINVAL);
-			}
-			authini = cri;
 			break;
 		default:
-			CRYPTDEB("unhandled algorithm");
 			return (EINVAL);
 		}
-	}
-	if (authini == NULL) {
-		CRYPTDEB("no cipher");
+		break;
+	default:
 		return (EINVAL);
 	}
+	return (CRYPTODEV_PROBE_ACCEL_SOFTWARE);
+}
 
-	rw_wlock(&sc->lock);
+static int
+blake2_newsession(device_t dev, crypto_session_t cses,
+    const struct crypto_session_params *csp)
+{
+	struct blake2_softc *sc;
+	struct blake2_session *ses;
+	int error;
+
+	sc = device_get_softc(dev);
+
+	ses = crypto_get_driver_session(cses);
+
+	rw_rlock(&sc->lock);
 	if (sc->dying) {
-		rw_wunlock(&sc->lock);
+		rw_runlock(&sc->lock);
 		return (EINVAL);
 	}
-	rw_wunlock(&sc->lock);
+	rw_runlock(&sc->lock);
 
-	ses = crypto_get_driver_session(cses);
-
-	ses->algo = authini->cri_alg;
-	error = blake2_cipher_setup(ses, authini);
+	error = blake2_cipher_setup(ses, csp);
 	if (error != 0) {
 		CRYPTDEB("setup failed");
 		return (error);
 	}
 
 	return (0);
 }
 
 static int
 blake2_process(device_t dev, struct cryptop *crp, int hint __unused)
 {
 	struct blake2_session *ses;
-	struct cryptodesc *crd, *authcrd;
 	int error;
 
-	ses = NULL;
-	error = 0;
-	authcrd = NULL;
-
-	/* Sanity check. */
-	if (crp == NULL)
-		return (EINVAL);
-
-	if (crp->crp_callback == NULL || crp->crp_desc == NULL) {
-		error = EINVAL;
-		goto out;
-	}
-
-	for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) {
-		switch (crd->crd_alg) {
-		case CRYPTO_BLAKE2B:
-		case CRYPTO_BLAKE2S:
-			if (authcrd != NULL) {
-				error = EINVAL;
-				goto out;
-			}
-			authcrd = crd;
-			break;
-
-		default:
-			error = EINVAL;
-			goto out;
-		}
-	}
-
 	ses = crypto_get_driver_session(crp->crp_session);
 	error = blake2_cipher_process(ses, crp);
-	if (error != 0)
-		goto out;
 
-out:
 	crp->crp_etype = error;
 	crypto_done(crp);
-	return (error);
+	return (0);
 }
 
 static device_method_t blake2_methods[] = {
 	DEVMETHOD(device_identify, blake2_identify),
 	DEVMETHOD(device_probe, blake2_probe),
 	DEVMETHOD(device_attach, blake2_attach),
 	DEVMETHOD(device_detach, blake2_detach),
 
+	DEVMETHOD(cryptodev_probesession, blake2_probesession),
 	DEVMETHOD(cryptodev_newsession, blake2_newsession),
 	DEVMETHOD(cryptodev_process, blake2_process),
 
 	DEVMETHOD_END
 };
 
 static driver_t blake2_driver = {
 	"blaketwo",
 	blake2_methods,
 	sizeof(struct blake2_softc),
 };
 static devclass_t blake2_devclass;
 
 DRIVER_MODULE(blake2, nexus, blake2_driver, blake2_devclass, 0, 0);
 MODULE_VERSION(blake2, 1);
 MODULE_DEPEND(blake2, crypto, 1, 1, 1);
 
+static bool
+blake2_check_klen(const struct crypto_session_params *csp, unsigned klen)
+{
+
+	if (csp->csp_auth_alg == CRYPTO_BLAKE2S)
+		return (klen <= BLAKE2S_KEYBYTES);
+	else
+		return (klen <= BLAKE2B_KEYBYTES);
+}
+
 static int
-blake2_cipher_setup(struct blake2_session *ses, struct cryptoini *authini)
+blake2_cipher_setup(struct blake2_session *ses,
+    const struct crypto_session_params *csp)
 {
-	int keylen;
+	int hashlen;
 
 	CTASSERT((size_t)BLAKE2S_OUTBYTES <= (size_t)BLAKE2B_OUTBYTES);
 
-	if (authini->cri_mlen < 0)
+	if (!blake2_check_klen(csp, csp->csp_auth_klen))
 		return (EINVAL);
 
-	switch (ses->algo) {
+	if (csp->csp_auth_mlen < 0)
+		return (EINVAL);
+
+	switch (csp->csp_auth_alg) {
 	case CRYPTO_BLAKE2S:
-		if (authini->cri_mlen != 0 &&
-		    authini->cri_mlen > BLAKE2S_OUTBYTES)
-			return (EINVAL);
-		/* FALLTHROUGH */
+		hashlen = BLAKE2S_OUTBYTES;
+		break;
 	case CRYPTO_BLAKE2B:
-		if (authini->cri_mlen != 0 &&
-		    authini->cri_mlen > BLAKE2B_OUTBYTES)
-			return (EINVAL);
-
-		if (authini->cri_klen % 8 != 0)
-			return (EINVAL);
-		keylen = authini->cri_klen / 8;
-		if (keylen > sizeof(ses->key) ||
-		    (ses->algo == CRYPTO_BLAKE2S && keylen > BLAKE2S_KEYBYTES))
-			return (EINVAL);
-		ses->klen = keylen;
-		memcpy(ses->key, authini->cri_key, keylen);
-		ses->mlen = authini->cri_mlen;
+		hashlen = BLAKE2B_OUTBYTES;
+		break;
+	default:
+		return (EINVAL);
 	}
+
+	if (csp->csp_auth_mlen > hashlen)
+		return (EINVAL);
+
+	if (csp->csp_auth_mlen == 0)
+		ses->mlen = hashlen;
+	else
+		ses->mlen = csp->csp_auth_mlen;
 	return (0);
 }
 
 static int
 blake2b_applicator(void *state, void *buf, u_int len)
 {
 	int rc;
 
 	rc = blake2b_update(state, buf, len);
 	if (rc != 0)
 		return (EINVAL);
 	return (0);
 }
 
 static int
 blake2s_applicator(void *state, void *buf, u_int len)
 {
 	int rc;
 
 	rc = blake2s_update(state, buf, len);
 	if (rc != 0)
 		return (EINVAL);
 	return (0);
 }
 
 static int
 blake2_cipher_process(struct blake2_session *ses, struct cryptop *crp)
 {
 	union {
 		blake2b_state sb;
 		blake2s_state ss;
 	} bctx;
-	char res[BLAKE2B_OUTBYTES];
+	char res[BLAKE2B_OUTBYTES], res2[BLAKE2B_OUTBYTES];
+	const struct crypto_session_params *csp;
 	struct fpu_kern_ctx *ctx;
+	const void *key;
 	int ctxidx;
 	bool kt;
-	struct cryptodesc *crd;
 	int error, rc;
-	size_t hashlen;
+	unsigned klen;
 
-	crd = crp->crp_desc;
 	ctx = NULL;
 	ctxidx = 0;
 	error = EINVAL;
 
 	kt = is_fpu_kern_thread(0);
 	if (!kt) {
 		ACQUIRE_CTX(ctxidx, ctx);
 		fpu_kern_enter(curthread, ctx,
 		    FPU_KERN_NORMAL | FPU_KERN_KTHR);
 	}
 
-	if (crd->crd_flags != 0)
-		goto out;
-
-	switch (ses->algo) {
+	csp = crypto_get_params(crp->crp_session);
+	if (crp->crp_auth_key != NULL)
+		key = crp->crp_auth_key;
+	else
+		key = csp->csp_auth_key;
+	klen = csp->csp_auth_klen;
+	switch (csp->csp_auth_alg) {
 	case CRYPTO_BLAKE2B:
-		if (ses->mlen != 0)
-			hashlen = ses->mlen;
+		if (klen > 0)
+			rc = blake2b_init_key(&bctx.sb, ses->mlen, key, klen);
 		else
-			hashlen = BLAKE2B_OUTBYTES;
-		if (ses->klen > 0)
-			rc = blake2b_init_key(&bctx.sb, hashlen, ses->key, ses->klen);
-		else
-			rc = blake2b_init(&bctx.sb, hashlen);
+			rc = blake2b_init(&bctx.sb, ses->mlen);
 		if (rc != 0)
 			goto out;
-		error = crypto_apply(crp->crp_flags, crp->crp_buf, crd->crd_skip,
-		    crd->crd_len, blake2b_applicator, &bctx.sb);
+		error = crypto_apply(crp, crp->crp_payload_start,
+		    crp->crp_payload_length, blake2b_applicator, &bctx.sb);
 		if (error != 0)
 			goto out;
-		rc = blake2b_final(&bctx.sb, res, hashlen);
+		rc = blake2b_final(&bctx.sb, res, ses->mlen);
 		if (rc != 0) {
 			error = EINVAL;
 			goto out;
 		}
 		break;
 	case CRYPTO_BLAKE2S:
-		if (ses->mlen != 0)
-			hashlen = ses->mlen;
+		if (klen > 0)
+			rc = blake2s_init_key(&bctx.ss, ses->mlen, key, klen);
 		else
-			hashlen = BLAKE2S_OUTBYTES;
-		if (ses->klen > 0)
-			rc = blake2s_init_key(&bctx.ss, hashlen, ses->key, ses->klen);
-		else
-			rc = blake2s_init(&bctx.ss, hashlen);
+			rc = blake2s_init(&bctx.ss, ses->mlen);
 		if (rc != 0)
 			goto out;
-		error = crypto_apply(crp->crp_flags, crp->crp_buf, crd->crd_skip,
-		    crd->crd_len, blake2s_applicator, &bctx.ss);
+		error = crypto_apply(crp, crp->crp_payload_start,
+		    crp->crp_payload_length, blake2s_applicator, &bctx.ss);
 		if (error != 0)
 			goto out;
-		rc = blake2s_final(&bctx.ss, res, hashlen);
+		rc = blake2s_final(&bctx.ss, res, ses->mlen);
 		if (rc != 0) {
 			error = EINVAL;
 			goto out;
 		}
 		break;
 	default:
 		panic("unreachable");
 	}
 
-	crypto_copyback(crp->crp_flags, crp->crp_buf, crd->crd_inject, hashlen,
-	    (void *)res);
+	if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
+		crypto_copydata(crp, crp->crp_digest_start, ses->mlen, res2);
+		if (timingsafe_bcmp(res, res2, ses->mlen) != 0)
+			return (EBADMSG);
+	} else
+		crypto_copyback(crp, crp->crp_digest_start, ses->mlen, res);
 
 out:
 	if (!kt) {
 		fpu_kern_leave(curthread, ctx);
 		RELEASE_CTX(ctxidx, ctx);
 	}
 	return (error);
 }
Index: head/sys/crypto/ccp/ccp.c
===================================================================
--- head/sys/crypto/ccp/ccp.c	(revision 359373)
+++ head/sys/crypto/ccp/ccp.c	(revision 359374)
@@ -1,885 +1,855 @@
 /*-
  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
  *
  * Copyright (c) 2017 Chelsio Communications, Inc.
  * Copyright (c) 2017 Conrad Meyer <cem@FreeBSD.org>
  * All rights reserved.
  * Largely borrowed from ccr(4), Written by: John Baldwin <jhb@FreeBSD.org>
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include "opt_ddb.h"
 
 #include <sys/param.h>
 #include <sys/bus.h>
 #include <sys/lock.h>
 #include <sys/kernel.h>
 #include <sys/malloc.h>
 #include <sys/mutex.h>
 #include <sys/module.h>
 #include <sys/random.h>
 #include <sys/sglist.h>
 #include <sys/sysctl.h>
 
 #ifdef DDB
 #include <ddb/ddb.h>
 #endif
 
 #include <dev/pci/pcivar.h>
 
 #include <dev/random/randomdev.h>
 
 #include <opencrypto/cryptodev.h>
 #include <opencrypto/xform.h>
 
 #include "cryptodev_if.h"
 
 #include "ccp.h"
 #include "ccp_hardware.h"
 
 MALLOC_DEFINE(M_CCP, "ccp", "AMD CCP crypto");
 
 /*
  * Need a global softc available for garbage random_source API, which lacks any
  * context pointer.  It's also handy for debugging.
  */
 struct ccp_softc *g_ccp_softc;
 
 bool g_debug_print = false;
 SYSCTL_BOOL(_hw_ccp, OID_AUTO, debug, CTLFLAG_RWTUN, &g_debug_print, 0,
     "Set to enable debugging log messages");
 
 static struct pciid {
 	uint32_t devid;
 	const char *desc;
 } ccp_ids[] = {
 	{ 0x14561022, "AMD CCP-5a" },
 	{ 0x14681022, "AMD CCP-5b" },
 };
 
 static struct random_source random_ccp = {
 	.rs_ident = "AMD CCP TRNG",
 	.rs_source = RANDOM_PURE_CCP,
 	.rs_read = random_ccp_read,
 };
 
 /*
  * ccp_populate_sglist() generates a scatter/gather list that covers the entire
  * crypto operation buffer.
  */
 static int
 ccp_populate_sglist(struct sglist *sg, struct cryptop *crp)
 {
 	int error;
 
 	sglist_reset(sg);
-	if (crp->crp_flags & CRYPTO_F_IMBUF)
+	switch (crp->crp_buf_type) {
+	case CRYPTO_BUF_MBUF:
 		error = sglist_append_mbuf(sg, crp->crp_mbuf);
-	else if (crp->crp_flags & CRYPTO_F_IOV)
+		break;
+	case CRYPTO_BUF_UIO:
 		error = sglist_append_uio(sg, crp->crp_uio);
-	else
+		break;
+	case CRYPTO_BUF_CONTIG:
 		error = sglist_append(sg, crp->crp_buf, crp->crp_ilen);
+		break;
+	default:
+		error = EINVAL;
+	}
 	return (error);
 }
 
 /*
  * Handle a GCM request with an empty payload by performing the
- * operation in software.  Derived from swcr_authenc().
+ * operation in software.
  */
 static void
-ccp_gcm_soft(struct ccp_session *s, struct cryptop *crp,
-    struct cryptodesc *crda, struct cryptodesc *crde)
+ccp_gcm_soft(struct ccp_session *s, struct cryptop *crp)
 {
 	struct aes_gmac_ctx gmac_ctx;
 	char block[GMAC_BLOCK_LEN];
 	char digest[GMAC_DIGEST_LEN];
 	char iv[AES_BLOCK_LEN];
 	int i, len;
 
 	/*
 	 * This assumes a 12-byte IV from the crp.  See longer comment
 	 * above in ccp_gcm() for more details.
 	 */
-	if (crde->crd_flags & CRD_F_ENCRYPT) {
-		if (crde->crd_flags & CRD_F_IV_EXPLICIT)
-			memcpy(iv, crde->crd_iv, 12);
-		else
-			arc4rand(iv, 12, 0);
-		if ((crde->crd_flags & CRD_F_IV_PRESENT) == 0)
-			crypto_copyback(crp->crp_flags, crp->crp_buf,
-			    crde->crd_inject, 12, iv);
-	} else {
-		if (crde->crd_flags & CRD_F_IV_EXPLICIT)
-			memcpy(iv, crde->crd_iv, 12);
-		else
-			crypto_copydata(crp->crp_flags, crp->crp_buf,
-			    crde->crd_inject, 12, iv);
+	if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) {
+		crp->crp_etype = EINVAL;
+		goto out;
 	}
+	memcpy(iv, crp->crp_iv, 12);
 	*(uint32_t *)&iv[12] = htobe32(1);
 
 	/* Initialize the MAC. */
 	AES_GMAC_Init(&gmac_ctx);
 	AES_GMAC_Setkey(&gmac_ctx, s->blkcipher.enckey, s->blkcipher.key_len);
 	AES_GMAC_Reinit(&gmac_ctx, iv, sizeof(iv));
 
 	/* MAC the AAD. */
-	for (i = 0; i < crda->crd_len; i += sizeof(block)) {
-		len = imin(crda->crd_len - i, sizeof(block));
-		crypto_copydata(crp->crp_flags, crp->crp_buf, crda->crd_skip +
-		    i, len, block);
+	for (i = 0; i < crp->crp_aad_length; i += sizeof(block)) {
+		len = imin(crp->crp_aad_length - i, sizeof(block));
+		crypto_copydata(crp, crp->crp_aad_start + i, len, block);
 		bzero(block + len, sizeof(block) - len);
 		AES_GMAC_Update(&gmac_ctx, block, sizeof(block));
 	}
 
 	/* Length block. */
 	bzero(block, sizeof(block));
-	((uint32_t *)block)[1] = htobe32(crda->crd_len * 8);
+	((uint32_t *)block)[1] = htobe32(crp->crp_aad_length * 8);
 	AES_GMAC_Update(&gmac_ctx, block, sizeof(block));
 	AES_GMAC_Final(digest, &gmac_ctx);
 
-	if (crde->crd_flags & CRD_F_ENCRYPT) {
-		crypto_copyback(crp->crp_flags, crp->crp_buf, crda->crd_inject,
-		    sizeof(digest), digest);
+	if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
+		crypto_copyback(crp, crp->crp_digest_start, sizeof(digest),
+		    digest);
 		crp->crp_etype = 0;
 	} else {
 		char digest2[GMAC_DIGEST_LEN];
 
-		crypto_copydata(crp->crp_flags, crp->crp_buf, crda->crd_inject,
-		    sizeof(digest2), digest2);
+		crypto_copydata(crp, crp->crp_digest_start, sizeof(digest2),
+		    digest2);
 		if (timingsafe_bcmp(digest, digest2, sizeof(digest)) == 0)
 			crp->crp_etype = 0;
 		else
 			crp->crp_etype = EBADMSG;
 	}
+out:
 	crypto_done(crp);
 }
 
 static int
 ccp_probe(device_t dev)
 {
 	struct pciid *ip;
 	uint32_t id;
 
 	id = pci_get_devid(dev);
 	for (ip = ccp_ids; ip < &ccp_ids[nitems(ccp_ids)]; ip++) {
 		if (id == ip->devid) {
 			device_set_desc(dev, ip->desc);
 			return (0);
 		}
 	}
 	return (ENXIO);
 }
 
 static void
 ccp_initialize_queues(struct ccp_softc *sc)
 {
 	struct ccp_queue *qp;
 	size_t i;
 
 	for (i = 0; i < nitems(sc->queues); i++) {
 		qp = &sc->queues[i];
 
 		qp->cq_softc = sc;
 		qp->cq_qindex = i;
 		mtx_init(&qp->cq_lock, "ccp queue", NULL, MTX_DEF);
 		/* XXX - arbitrarily chosen sizes */
 		qp->cq_sg_crp = sglist_alloc(32, M_WAITOK);
 		/* Two more SGEs than sg_crp to accommodate ipad. */
 		qp->cq_sg_ulptx = sglist_alloc(34, M_WAITOK);
 		qp->cq_sg_dst = sglist_alloc(2, M_WAITOK);
 	}
 }
 
 static void
 ccp_free_queues(struct ccp_softc *sc)
 {
 	struct ccp_queue *qp;
 	size_t i;
 
 	for (i = 0; i < nitems(sc->queues); i++) {
 		qp = &sc->queues[i];
 
 		mtx_destroy(&qp->cq_lock);
 		sglist_free(qp->cq_sg_crp);
 		sglist_free(qp->cq_sg_ulptx);
 		sglist_free(qp->cq_sg_dst);
 	}
 }
 
 static int
 ccp_attach(device_t dev)
 {
 	struct ccp_softc *sc;
 	int error;
 
 	sc = device_get_softc(dev);
 	sc->dev = dev;
 
 	sc->cid = crypto_get_driverid(dev, sizeof(struct ccp_session),
 	    CRYPTOCAP_F_HARDWARE);
 	if (sc->cid < 0) {
 		device_printf(dev, "could not get crypto driver id\n");
 		return (ENXIO);
 	}
 
 	error = ccp_hw_attach(dev);
 	if (error != 0)
 		return (error);
 
 	mtx_init(&sc->lock, "ccp", NULL, MTX_DEF);
 
 	ccp_initialize_queues(sc);
 
 	if (g_ccp_softc == NULL) {
 		g_ccp_softc = sc;
 		if ((sc->hw_features & VERSION_CAP_TRNG) != 0)
 			random_source_register(&random_ccp);
 	}
 
-	if ((sc->hw_features & VERSION_CAP_AES) != 0) {
-		crypto_register(sc->cid, CRYPTO_AES_CBC, 0, 0);
-		crypto_register(sc->cid, CRYPTO_AES_ICM, 0, 0);
-		crypto_register(sc->cid, CRYPTO_AES_NIST_GCM_16, 0, 0);
-		crypto_register(sc->cid, CRYPTO_AES_128_NIST_GMAC, 0, 0);
-		crypto_register(sc->cid, CRYPTO_AES_192_NIST_GMAC, 0, 0);
-		crypto_register(sc->cid, CRYPTO_AES_256_NIST_GMAC, 0, 0);
-		crypto_register(sc->cid, CRYPTO_AES_XTS, 0, 0);
-	}
-	if ((sc->hw_features & VERSION_CAP_SHA) != 0) {
-		crypto_register(sc->cid, CRYPTO_SHA1_HMAC, 0, 0);
-		crypto_register(sc->cid, CRYPTO_SHA2_256_HMAC, 0, 0);
-		crypto_register(sc->cid, CRYPTO_SHA2_384_HMAC, 0, 0);
-		crypto_register(sc->cid, CRYPTO_SHA2_512_HMAC, 0, 0);
-	}
-
 	return (0);
 }
 
 static int
 ccp_detach(device_t dev)
 {
 	struct ccp_softc *sc;
 
 	sc = device_get_softc(dev);
 
 	mtx_lock(&sc->lock);
 	sc->detaching = true;
 	mtx_unlock(&sc->lock);
 
 	crypto_unregister_all(sc->cid);
 	if (g_ccp_softc == sc && (sc->hw_features & VERSION_CAP_TRNG) != 0)
 		random_source_deregister(&random_ccp);
 
 	ccp_hw_detach(dev);
 	ccp_free_queues(sc);
 
 	if (g_ccp_softc == sc)
 		g_ccp_softc = NULL;
 
 	mtx_destroy(&sc->lock);
 	return (0);
 }
 
 static void
-ccp_init_hmac_digest(struct ccp_session *s, int cri_alg, char *key,
-    int klen)
+ccp_init_hmac_digest(struct ccp_session *s, const char *key, int klen)
 {
 	union authctx auth_ctx;
 	struct auth_hash *axf;
 	u_int i;
 
 	/*
 	 * If the key is larger than the block size, use the digest of
 	 * the key as the key instead.
 	 */
 	axf = s->hmac.auth_hash;
-	klen /= 8;
 	if (klen > axf->blocksize) {
 		axf->Init(&auth_ctx);
 		axf->Update(&auth_ctx, key, klen);
 		axf->Final(s->hmac.ipad, &auth_ctx);
 		explicit_bzero(&auth_ctx, sizeof(auth_ctx));
 		klen = axf->hashsize;
 	} else
 		memcpy(s->hmac.ipad, key, klen);
 
 	memset(s->hmac.ipad + klen, 0, axf->blocksize - klen);
 	memcpy(s->hmac.opad, s->hmac.ipad, axf->blocksize);
 
 	for (i = 0; i < axf->blocksize; i++) {
 		s->hmac.ipad[i] ^= HMAC_IPAD_VAL;
 		s->hmac.opad[i] ^= HMAC_OPAD_VAL;
 	}
 }
 
-static int
+static bool
 ccp_aes_check_keylen(int alg, int klen)
 {
 
-	switch (klen) {
+	switch (klen * 8) {
 	case 128:
 	case 192:
 		if (alg == CRYPTO_AES_XTS)
-			return (EINVAL);
+			return (false);
 		break;
 	case 256:
 		break;
 	case 512:
 		if (alg != CRYPTO_AES_XTS)
-			return (EINVAL);
+			return (false);
 		break;
 	default:
-		return (EINVAL);
+		return (false);
 	}
-	return (0);
+	return (true);
 }
 
 static void
 ccp_aes_setkey(struct ccp_session *s, int alg, const void *key, int klen)
 {
 	unsigned kbits;
 
 	if (alg == CRYPTO_AES_XTS)
-		kbits = klen / 2;
+		kbits = (klen / 2) * 8;
 	else
-		kbits = klen;
+		kbits = klen * 8;
 
 	switch (kbits) {
 	case 128:
 		s->blkcipher.cipher_type = CCP_AES_TYPE_128;
 		break;
 	case 192:
 		s->blkcipher.cipher_type = CCP_AES_TYPE_192;
 		break;
 	case 256:
 		s->blkcipher.cipher_type = CCP_AES_TYPE_256;
 		break;
 	default:
 		panic("should not get here");
 	}
 
-	s->blkcipher.key_len = klen / 8;
+	s->blkcipher.key_len = klen;
 	memcpy(s->blkcipher.enckey, key, s->blkcipher.key_len);
 }
 
+static bool
+ccp_auth_supported(struct ccp_softc *sc,
+    const struct crypto_session_params *csp)
+{
+	
+	if ((sc->hw_features & VERSION_CAP_SHA) == 0)
+		return (false);
+	switch (csp->csp_auth_alg) {
+	case CRYPTO_SHA1_HMAC:
+	case CRYPTO_SHA2_256_HMAC:
+	case CRYPTO_SHA2_384_HMAC:
+	case CRYPTO_SHA2_512_HMAC:
+		if (csp->csp_auth_key == NULL)
+			return (false);
+		break;
+	default:
+		return (false);
+	}
+	return (true);
+}
+
+static bool
+ccp_cipher_supported(struct ccp_softc *sc,
+    const struct crypto_session_params *csp)
+{
+
+	if ((sc->hw_features & VERSION_CAP_AES) == 0)
+		return (false);
+	switch (csp->csp_cipher_alg) {
+	case CRYPTO_AES_CBC:
+		if (csp->csp_ivlen != AES_BLOCK_LEN)
+			return (false);
+		break;
+	case CRYPTO_AES_ICM:
+		if (csp->csp_ivlen != AES_BLOCK_LEN)
+			return (false);
+		break;
+	case CRYPTO_AES_XTS:
+		if (csp->csp_ivlen != AES_XTS_IV_LEN)
+			return (false);
+		break;
+	default:
+		return (false);
+	}
+	return (ccp_aes_check_keylen(csp->csp_cipher_alg,
+	    csp->csp_cipher_klen));
+}
+
 static int
-ccp_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
+ccp_probesession(device_t dev, const struct crypto_session_params *csp)
 {
 	struct ccp_softc *sc;
-	struct ccp_session *s;
-	struct auth_hash *auth_hash;
-	struct cryptoini *c, *hash, *cipher;
-	enum ccp_aes_mode cipher_mode;
-	unsigned auth_mode, iv_len;
-	unsigned partial_digest_len;
-	unsigned q;
-	int error;
-	bool gcm_hash;
 
-	if (cri == NULL)
+	if (csp->csp_flags != 0)
 		return (EINVAL);
-
-	s = crypto_get_driver_session(cses);
-
-	gcm_hash = false;
-	cipher = NULL;
-	hash = NULL;
-	auth_hash = NULL;
-	/* XXX reconcile auth_mode with use by ccp_sha */
-	auth_mode = 0;
-	cipher_mode = CCP_AES_MODE_ECB;
-	iv_len = 0;
-	partial_digest_len = 0;
-	for (c = cri; c != NULL; c = c->cri_next) {
-		switch (c->cri_alg) {
-		case CRYPTO_SHA1_HMAC:
-		case CRYPTO_SHA2_256_HMAC:
-		case CRYPTO_SHA2_384_HMAC:
-		case CRYPTO_SHA2_512_HMAC:
-		case CRYPTO_AES_128_NIST_GMAC:
-		case CRYPTO_AES_192_NIST_GMAC:
-		case CRYPTO_AES_256_NIST_GMAC:
-			if (hash)
-				return (EINVAL);
-			hash = c;
-			switch (c->cri_alg) {
-			case CRYPTO_SHA1_HMAC:
-				auth_hash = &auth_hash_hmac_sha1;
-				auth_mode = SHA1;
-				partial_digest_len = SHA1_HASH_LEN;
-				break;
-			case CRYPTO_SHA2_256_HMAC:
-				auth_hash = &auth_hash_hmac_sha2_256;
-				auth_mode = SHA2_256;
-				partial_digest_len = SHA2_256_HASH_LEN;
-				break;
-			case CRYPTO_SHA2_384_HMAC:
-				auth_hash = &auth_hash_hmac_sha2_384;
-				auth_mode = SHA2_384;
-				partial_digest_len = SHA2_512_HASH_LEN;
-				break;
-			case CRYPTO_SHA2_512_HMAC:
-				auth_hash = &auth_hash_hmac_sha2_512;
-				auth_mode = SHA2_512;
-				partial_digest_len = SHA2_512_HASH_LEN;
-				break;
-			case CRYPTO_AES_128_NIST_GMAC:
-			case CRYPTO_AES_192_NIST_GMAC:
-			case CRYPTO_AES_256_NIST_GMAC:
-				gcm_hash = true;
-#if 0
-				auth_mode = CHCR_SCMD_AUTH_MODE_GHASH;
-#endif
-				break;
-			}
-			break;
-		case CRYPTO_AES_CBC:
-		case CRYPTO_AES_ICM:
+	sc = device_get_softc(dev);
+	switch (csp->csp_mode) {
+	case CSP_MODE_DIGEST:
+		if (!ccp_auth_supported(sc, csp))
+			return (EINVAL);
+		break;
+	case CSP_MODE_CIPHER:
+		if (!ccp_cipher_supported(sc, csp))
+			return (EINVAL);
+		break;
+	case CSP_MODE_AEAD:
+		switch (csp->csp_cipher_alg) {
 		case CRYPTO_AES_NIST_GCM_16:
-		case CRYPTO_AES_XTS:
-			if (cipher)
+			if (csp->csp_ivlen != AES_GCM_IV_LEN)
 				return (EINVAL);
-			cipher = c;
-			switch (c->cri_alg) {
-			case CRYPTO_AES_CBC:
-				cipher_mode = CCP_AES_MODE_CBC;
-				iv_len = AES_BLOCK_LEN;
-				break;
-			case CRYPTO_AES_ICM:
-				cipher_mode = CCP_AES_MODE_CTR;
-				iv_len = AES_BLOCK_LEN;
-				break;
-			case CRYPTO_AES_NIST_GCM_16:
-				cipher_mode = CCP_AES_MODE_GCTR;
-				iv_len = AES_GCM_IV_LEN;
-				break;
-			case CRYPTO_AES_XTS:
-				cipher_mode = CCP_AES_MODE_XTS;
-				iv_len = AES_BLOCK_LEN;
-				break;
-			}
-			if (c->cri_key != NULL) {
-				error = ccp_aes_check_keylen(c->cri_alg,
-				    c->cri_klen);
-				if (error != 0)
-					return (error);
-			}
+			if (csp->csp_auth_mlen < 0 ||
+			    csp->csp_auth_mlen > AES_GMAC_HASH_LEN)
+				return (EINVAL);
+			if ((sc->hw_features & VERSION_CAP_AES) == 0)
+				return (EINVAL);
 			break;
 		default:
 			return (EINVAL);
 		}
-	}
-	if (gcm_hash != (cipher_mode == CCP_AES_MODE_GCTR))
+		break;
+	case CSP_MODE_ETA:
+		if (!ccp_auth_supported(sc, csp) ||
+		    !ccp_cipher_supported(sc, csp))
+			return (EINVAL);
+		break;
+	default:
 		return (EINVAL);
-	if (hash == NULL && cipher == NULL)
-		return (EINVAL);
-	if (hash != NULL && hash->cri_key == NULL)
-		return (EINVAL);
+	}
 
+	return (CRYPTODEV_PROBE_HARDWARE);
+}
+
+static int
+ccp_newsession(device_t dev, crypto_session_t cses,
+    const struct crypto_session_params *csp)
+{
+	struct ccp_softc *sc;
+	struct ccp_session *s;
+	struct auth_hash *auth_hash;
+	enum ccp_aes_mode cipher_mode;
+	unsigned auth_mode;
+	unsigned q;
+
+	/* XXX reconcile auth_mode with use by ccp_sha */
+	switch (csp->csp_auth_alg) {
+	case CRYPTO_SHA1_HMAC:
+		auth_hash = &auth_hash_hmac_sha1;
+		auth_mode = SHA1;
+		break;
+	case CRYPTO_SHA2_256_HMAC:
+		auth_hash = &auth_hash_hmac_sha2_256;
+		auth_mode = SHA2_256;
+		break;
+	case CRYPTO_SHA2_384_HMAC:
+		auth_hash = &auth_hash_hmac_sha2_384;
+		auth_mode = SHA2_384;
+		break;
+	case CRYPTO_SHA2_512_HMAC:
+		auth_hash = &auth_hash_hmac_sha2_512;
+		auth_mode = SHA2_512;
+		break;
+	default:
+		auth_hash = NULL;
+		auth_mode = 0;
+		break;
+	}
+
+	switch (csp->csp_cipher_alg) {
+	case CRYPTO_AES_CBC:
+		cipher_mode = CCP_AES_MODE_CBC;
+		break;
+	case CRYPTO_AES_ICM:
+		cipher_mode = CCP_AES_MODE_CTR;
+		break;
+	case CRYPTO_AES_NIST_GCM_16:
+		cipher_mode = CCP_AES_MODE_GCTR;
+		break;
+	case CRYPTO_AES_XTS:
+		cipher_mode = CCP_AES_MODE_XTS;
+		break;
+	default:
+		cipher_mode = CCP_AES_MODE_ECB;
+		break;
+	}
+
 	sc = device_get_softc(dev);
 	mtx_lock(&sc->lock);
 	if (sc->detaching) {
 		mtx_unlock(&sc->lock);
 		return (ENXIO);
 	}
 
+	s = crypto_get_driver_session(cses);
+
 	/* Just grab the first usable queue for now. */
 	for (q = 0; q < nitems(sc->queues); q++)
 		if ((sc->valid_queues & (1 << q)) != 0)
 			break;
 	if (q == nitems(sc->queues)) {
 		mtx_unlock(&sc->lock);
 		return (ENXIO);
 	}
 	s->queue = q;
 
-	if (gcm_hash)
+	switch (csp->csp_mode) {
+	case CSP_MODE_AEAD:
 		s->mode = GCM;
-	else if (hash != NULL && cipher != NULL)
+		break;
+	case CSP_MODE_ETA:
 		s->mode = AUTHENC;
-	else if (hash != NULL)
+		break;
+	case CSP_MODE_DIGEST:
 		s->mode = HMAC;
-	else {
-		MPASS(cipher != NULL);
+		break;
+	case CSP_MODE_CIPHER:
 		s->mode = BLKCIPHER;
+		break;
 	}
-	if (gcm_hash) {
-		if (hash->cri_mlen == 0)
+
+	if (s->mode == GCM) {
+		if (csp->csp_auth_mlen == 0)
 			s->gmac.hash_len = AES_GMAC_HASH_LEN;
 		else
-			s->gmac.hash_len = hash->cri_mlen;
-	} else if (hash != NULL) {
+			s->gmac.hash_len = csp->csp_auth_mlen;
+	} else if (auth_hash != NULL) {
 		s->hmac.auth_hash = auth_hash;
 		s->hmac.auth_mode = auth_mode;
-		s->hmac.partial_digest_len = partial_digest_len;
-		if (hash->cri_mlen == 0)
+		if (csp->csp_auth_mlen == 0)
 			s->hmac.hash_len = auth_hash->hashsize;
 		else
-			s->hmac.hash_len = hash->cri_mlen;
-		ccp_init_hmac_digest(s, hash->cri_alg, hash->cri_key,
-		    hash->cri_klen);
+			s->hmac.hash_len = csp->csp_auth_mlen;
+		ccp_init_hmac_digest(s, csp->csp_auth_key, csp->csp_auth_klen);
 	}
-	if (cipher != NULL) {
+	if (cipher_mode != CCP_AES_MODE_ECB) {
 		s->blkcipher.cipher_mode = cipher_mode;
-		s->blkcipher.iv_len = iv_len;
-		if (cipher->cri_key != NULL)
-			ccp_aes_setkey(s, cipher->cri_alg, cipher->cri_key,
-			    cipher->cri_klen);
+		if (csp->csp_cipher_key != NULL)
+			ccp_aes_setkey(s, csp->csp_cipher_alg,
+			    csp->csp_cipher_key, csp->csp_cipher_klen);
 	}
 
 	s->active = true;
 	mtx_unlock(&sc->lock);
 
 	return (0);
 }
 
 static void
 ccp_freesession(device_t dev, crypto_session_t cses)
 {
 	struct ccp_session *s;
 
 	s = crypto_get_driver_session(cses);
 
 	if (s->pending != 0)
 		device_printf(dev,
 		    "session %p freed with %d pending requests\n", s,
 		    s->pending);
 	s->active = false;
 }
 
 static int
 ccp_process(device_t dev, struct cryptop *crp, int hint)
 {
+	const struct crypto_session_params *csp;
 	struct ccp_softc *sc;
 	struct ccp_queue *qp;
 	struct ccp_session *s;
-	struct cryptodesc *crd, *crda, *crde;
 	int error;
 	bool qpheld;
 
 	qpheld = false;
 	qp = NULL;
-	if (crp == NULL)
-		return (EINVAL);
 
-	crd = crp->crp_desc;
+	csp = crypto_get_params(crp->crp_session);
 	s = crypto_get_driver_session(crp->crp_session);
 	sc = device_get_softc(dev);
 	mtx_lock(&sc->lock);
 	qp = &sc->queues[s->queue];
 	mtx_unlock(&sc->lock);
 	error = ccp_queue_acquire_reserve(qp, 1 /* placeholder */, M_NOWAIT);
 	if (error != 0)
 		goto out;
 	qpheld = true;
 
 	error = ccp_populate_sglist(qp->cq_sg_crp, crp);
 	if (error != 0)
 		goto out;
 
+	if (crp->crp_auth_key != NULL) {
+		KASSERT(s->hmac.auth_hash != NULL, ("auth key without HMAC"));
+		ccp_init_hmac_digest(s, crp->crp_auth_key, csp->csp_auth_klen);
+	}
+	if (crp->crp_cipher_key != NULL)
+		ccp_aes_setkey(s, csp->csp_cipher_alg, crp->crp_cipher_key,
+		    csp->csp_cipher_klen);
+
 	switch (s->mode) {
 	case HMAC:
-		if (crd->crd_flags & CRD_F_KEY_EXPLICIT)
-			ccp_init_hmac_digest(s, crd->crd_alg, crd->crd_key,
-			    crd->crd_klen);
+		if (s->pending != 0) {
+			error = EAGAIN;
+			break;
+		}
 		error = ccp_hmac(qp, s, crp);
 		break;
 	case BLKCIPHER:
-		if (crd->crd_flags & CRD_F_KEY_EXPLICIT) {
-			error = ccp_aes_check_keylen(crd->crd_alg,
-			    crd->crd_klen);
-			if (error != 0)
-				break;
-			ccp_aes_setkey(s, crd->crd_alg, crd->crd_key,
-			    crd->crd_klen);
+		if (s->pending != 0) {
+			error = EAGAIN;
+			break;
 		}
 		error = ccp_blkcipher(qp, s, crp);
 		break;
 	case AUTHENC:
-		error = 0;
-		switch (crd->crd_alg) {
-		case CRYPTO_AES_CBC:
-		case CRYPTO_AES_ICM:
-		case CRYPTO_AES_XTS:
-			/* Only encrypt-then-authenticate supported. */
-			crde = crd;
-			crda = crd->crd_next;
-			if (!(crde->crd_flags & CRD_F_ENCRYPT)) {
-				error = EINVAL;
-				break;
-			}
-			s->cipher_first = true;
+		if (s->pending != 0) {
+			error = EAGAIN;
 			break;
-		default:
-			crda = crd;
-			crde = crd->crd_next;
-			if (crde->crd_flags & CRD_F_ENCRYPT) {
-				error = EINVAL;
-				break;
-			}
-			s->cipher_first = false;
-			break;
 		}
-		if (error != 0)
-			break;
-		if (crda->crd_flags & CRD_F_KEY_EXPLICIT)
-			ccp_init_hmac_digest(s, crda->crd_alg, crda->crd_key,
-			    crda->crd_klen);
-		if (crde->crd_flags & CRD_F_KEY_EXPLICIT) {
-			error = ccp_aes_check_keylen(crde->crd_alg,
-			    crde->crd_klen);
-			if (error != 0)
-				break;
-			ccp_aes_setkey(s, crde->crd_alg, crde->crd_key,
-			    crde->crd_klen);
-		}
-		error = ccp_authenc(qp, s, crp, crda, crde);
+		error = ccp_authenc(qp, s, crp);
 		break;
 	case GCM:
-		error = 0;
-		if (crd->crd_alg == CRYPTO_AES_NIST_GCM_16) {
-			crde = crd;
-			crda = crd->crd_next;
-			s->cipher_first = true;
-		} else {
-			crda = crd;
-			crde = crd->crd_next;
-			s->cipher_first = false;
-		}
-		if (crde->crd_flags & CRD_F_KEY_EXPLICIT) {
-			error = ccp_aes_check_keylen(crde->crd_alg,
-			    crde->crd_klen);
-			if (error != 0)
-				break;
-			ccp_aes_setkey(s, crde->crd_alg, crde->crd_key,
-			    crde->crd_klen);
-		}
-		if (crde->crd_len == 0) {
+		if (crp->crp_payload_length == 0) {
 			mtx_unlock(&qp->cq_lock);
-			ccp_gcm_soft(s, crp, crda, crde);
+			ccp_gcm_soft(s, crp);
 			return (0);
 		}
-		error = ccp_gcm(qp, s, crp, crda, crde);
+		if (s->pending != 0) {
+			error = EAGAIN;
+			break;
+		}
+		error = ccp_gcm(qp, s, crp);
 		break;
 	}
 
 	if (error == 0)
 		s->pending++;
 
 out:
 	if (qpheld) {
 		if (error != 0) {
 			/*
 			 * Squash EAGAIN so callers don't uselessly and
 			 * expensively retry if the ring was full.
 			 */
 			if (error == EAGAIN)
 				error = ENOMEM;
 			ccp_queue_abort(qp);
 		} else
 			ccp_queue_release(qp);
 	}
 
 	if (error != 0) {
 		DPRINTF(dev, "%s: early error:%d\n", __func__, error);
 		crp->crp_etype = error;
 		crypto_done(crp);
 	}
 	return (0);
 }
 
 static device_method_t ccp_methods[] = {
 	DEVMETHOD(device_probe,		ccp_probe),
 	DEVMETHOD(device_attach,	ccp_attach),
 	DEVMETHOD(device_detach,	ccp_detach),
 
+	DEVMETHOD(cryptodev_probesession, ccp_probesession),
 	DEVMETHOD(cryptodev_newsession,	ccp_newsession),
 	DEVMETHOD(cryptodev_freesession, ccp_freesession),
 	DEVMETHOD(cryptodev_process,	ccp_process),
 
 	DEVMETHOD_END
 };
 
 static driver_t ccp_driver = {
 	"ccp",
 	ccp_methods,
 	sizeof(struct ccp_softc)
 };
 
 static devclass_t ccp_devclass;
 DRIVER_MODULE(ccp, pci, ccp_driver, ccp_devclass, NULL, NULL);
 MODULE_VERSION(ccp, 1);
 MODULE_DEPEND(ccp, crypto, 1, 1, 1);
 MODULE_DEPEND(ccp, random_device, 1, 1, 1);
 #if 0	/* There are enough known issues that we shouldn't load automatically */
 MODULE_PNP_INFO("W32:vendor/device", pci, ccp, ccp_ids,
     nitems(ccp_ids));
 #endif
 
 static int
 ccp_queue_reserve_space(struct ccp_queue *qp, unsigned n, int mflags)
 {
 	struct ccp_softc *sc;
 
 	mtx_assert(&qp->cq_lock, MA_OWNED);
 	sc = qp->cq_softc;
 
 	if (n < 1 || n >= (1 << sc->ring_size_order))
 		return (EINVAL);
 
 	while (true) {
 		if (ccp_queue_get_ring_space(qp) >= n)
 			return (0);
 		if ((mflags & M_WAITOK) == 0)
 			return (EAGAIN);
 		qp->cq_waiting = true;
 		msleep(&qp->cq_tail, &qp->cq_lock, 0, "ccpqfull", 0);
 	}
 }
 
 int
 ccp_queue_acquire_reserve(struct ccp_queue *qp, unsigned n, int mflags)
 {
 	int error;
 
 	mtx_lock(&qp->cq_lock);
 	qp->cq_acq_tail = qp->cq_tail;
 	error = ccp_queue_reserve_space(qp, n, mflags);
 	if (error != 0)
 		mtx_unlock(&qp->cq_lock);
 	return (error);
 }
 
 void
 ccp_queue_release(struct ccp_queue *qp)
 {
 
 	mtx_assert(&qp->cq_lock, MA_OWNED);
 	if (qp->cq_tail != qp->cq_acq_tail) {
 		wmb();
 		ccp_queue_write_tail(qp);
 	}
 	mtx_unlock(&qp->cq_lock);
 }
 
 void
 ccp_queue_abort(struct ccp_queue *qp)
 {
 	unsigned i;
 
 	mtx_assert(&qp->cq_lock, MA_OWNED);
 
 	/* Wipe out any descriptors associated with this aborted txn. */
 	for (i = qp->cq_acq_tail; i != qp->cq_tail;
 	    i = (i + 1) % (1 << qp->cq_softc->ring_size_order)) {
 		memset(&qp->desc_ring[i], 0, sizeof(qp->desc_ring[i]));
 	}
 	qp->cq_tail = qp->cq_acq_tail;
 
 	mtx_unlock(&qp->cq_lock);
 }
 
 #ifdef DDB
 #define	_db_show_lock(lo)	LOCK_CLASS(lo)->lc_ddb_show(lo)
 #define	db_show_lock(lk)	_db_show_lock(&(lk)->lock_object)
 static void
 db_show_ccp_sc(struct ccp_softc *sc)
 {
 
 	db_printf("ccp softc at %p\n", sc);
 	db_printf(" cid: %d\n", (int)sc->cid);
 
 	db_printf(" lock: ");
 	db_show_lock(&sc->lock);
 
 	db_printf(" detaching: %d\n", (int)sc->detaching);
 	db_printf(" ring_size_order: %u\n", sc->ring_size_order);
 
 	db_printf(" hw_version: %d\n", (int)sc->hw_version);
 	db_printf(" hw_features: %b\n", (int)sc->hw_features,
 	    "\20\24ELFC\23TRNG\22Zip_Compress\16Zip_Decompress\13ECC\12RSA"
 	    "\11SHA\0103DES\07AES");
 
 	db_printf(" hw status:\n");
 	db_ccp_show_hw(sc);
 }
 
 static void
 db_show_ccp_qp(struct ccp_queue *qp)
 {
 
 	db_printf(" lock: ");
 	db_show_lock(&qp->cq_lock);
 
 	db_printf(" cq_qindex: %u\n", qp->cq_qindex);
 	db_printf(" cq_softc: %p\n", qp->cq_softc);
 
 	db_printf(" head: %u\n", qp->cq_head);
 	db_printf(" tail: %u\n", qp->cq_tail);
 	db_printf(" acq_tail: %u\n", qp->cq_acq_tail);
 	db_printf(" desc_ring: %p\n", qp->desc_ring);
 	db_printf(" completions_ring: %p\n", qp->completions_ring);
 	db_printf(" descriptors (phys): 0x%jx\n",
 	    (uintmax_t)qp->desc_ring_bus_addr);
 
 	db_printf(" hw status:\n");
 	db_ccp_show_queue_hw(qp);
 }
 
 DB_SHOW_COMMAND(ccp, db_show_ccp)
 {
 	struct ccp_softc *sc;
 	unsigned unit, qindex;
 
 	if (!have_addr)
 		goto usage;
 
 	unit = (unsigned)addr;
 
 	sc = devclass_get_softc(ccp_devclass, unit);
 	if (sc == NULL) {
 		db_printf("No such device ccp%u\n", unit);
 		goto usage;
 	}
 
 	if (count == -1) {
 		db_show_ccp_sc(sc);
 		return;
 	}
 
 	qindex = (unsigned)count;
 	if (qindex >= nitems(sc->queues)) {
 		db_printf("No such queue %u\n", qindex);
 		goto usage;
 	}
 	db_show_ccp_qp(&sc->queues[qindex]);
 	return;
 
 usage:
 	db_printf("usage: show ccp <unit>[,<qindex>]\n");
 	return;
 }
 #endif /* DDB */
Index: head/sys/crypto/ccp/ccp.h
===================================================================
--- head/sys/crypto/ccp/ccp.h	(revision 359373)
+++ head/sys/crypto/ccp/ccp.h	(revision 359374)
@@ -1,259 +1,260 @@
 /*-
  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
  *
  * Copyright (c) 2017 Conrad Meyer <cem@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
  * $FreeBSD$
  */
 
 #pragma once
 
 /*
  * Keccak SHAKE128 (if supported by the device?) uses a 1344 bit block.
  * SHA3-224 is the next largest block size, at 1152 bits.  However, crypto(4)
  * doesn't support any SHA3 hash, so SHA2 is the constraint:
  */
 #define CCP_HASH_MAX_BLOCK_SIZE	(SHA2_512_BLOCK_LEN)
 
 #define CCP_AES_MAX_KEY_LEN	(AES_XTS_MAX_KEY)
 #define CCP_MAX_CRYPTO_IV_LEN	32	/* GCM IV + GHASH context */
 
 #define MAX_HW_QUEUES		5
 #define MAX_LSB_REGIONS		8
 
 #ifndef __must_check
 #define __must_check __attribute__((__warn_unused_result__))
 #endif
 
 /*
  * Internal data structures.
  */
 enum sha_version {
 	SHA1,
 #if 0
 	SHA2_224,
 #endif
 	SHA2_256, SHA2_384, SHA2_512
 };
 
+/*
+ * XXX: The hmac.res, gmac.final_block, and blkcipher.iv fields are
+ * used by individual requests meaning that sessions cannot have more
+ * than a single request in flight at a time.
+ */
 struct ccp_session_hmac {
 	struct auth_hash *auth_hash;
 	int hash_len;
-	unsigned int partial_digest_len;
 	unsigned int auth_mode;
-	unsigned int mk_size;
 	char ipad[CCP_HASH_MAX_BLOCK_SIZE];
 	char opad[CCP_HASH_MAX_BLOCK_SIZE];
+	char res[CCP_HASH_MAX_BLOCK_SIZE];
 };
 
 struct ccp_session_gmac {
 	int hash_len;
 	char final_block[GMAC_BLOCK_LEN];
 };
 
 struct ccp_session_blkcipher {
 	unsigned cipher_mode;
 	unsigned cipher_type;
 	unsigned key_len;
-	unsigned iv_len;
 	char enckey[CCP_AES_MAX_KEY_LEN];
 	char iv[CCP_MAX_CRYPTO_IV_LEN];
 };
 
 struct ccp_session {
-	bool active : 1;
-	bool cipher_first : 1;
+	bool active;
 	int pending;
 	enum { HMAC, BLKCIPHER, AUTHENC, GCM } mode;
 	unsigned queue;
 	union {
 		struct ccp_session_hmac hmac;
 		struct ccp_session_gmac gmac;
 	};
 	struct ccp_session_blkcipher blkcipher;
 };
 
 struct ccp_softc;
 struct ccp_queue {
 	struct mtx		cq_lock;
 	unsigned		cq_qindex;
 	struct ccp_softc	*cq_softc;
 
 	/* Host memory and tracking structures for descriptor ring. */
 	bus_dma_tag_t		ring_desc_tag;
 	bus_dmamap_t		ring_desc_map;
 	struct ccp_desc		*desc_ring;
 	bus_addr_t		desc_ring_bus_addr;
 	/* Callbacks and arguments ring; indices correspond to above ring. */
 	struct ccp_completion_ctx *completions_ring;
 
 	uint32_t		qcontrol;	/* Cached register value */
 	unsigned		lsb_mask;	/* LSBs available to queue */
 	int			private_lsb;	/* Reserved LSB #, or -1 */
 
 	unsigned		cq_head;
 	unsigned		cq_tail;
 	unsigned		cq_acq_tail;
 
 	bool			cq_waiting;	/* Thread waiting for space */
 
 	struct sglist		*cq_sg_crp;
 	struct sglist		*cq_sg_ulptx;
 	struct sglist		*cq_sg_dst;
 };
 
 struct ccp_completion_ctx {
 	void (*callback_fn)(struct ccp_queue *qp, struct ccp_session *s,
 	    void *arg, int error);
 	void *callback_arg;
 	struct ccp_session *session;
 };
 
 struct ccp_softc {
 	device_t dev;
 	int32_t cid;
 	struct mtx lock;
 	bool detaching;
 
 	unsigned ring_size_order;
 
 	/*
 	 * Each command queue is either public or private.  "Private"
 	 * (PSP-only) by default.  PSP grants access to some queues to host via
 	 * QMR (Queue Mask Register).  Set bits are host accessible.
 	 */
 	uint8_t valid_queues;
 
 	uint8_t hw_version;
 	uint8_t num_queues;
 	uint16_t hw_features;
 	uint16_t num_lsb_entries;
 
 	/* Primary BAR (RID 2) used for register access */
 	bus_space_tag_t pci_bus_tag;
 	bus_space_handle_t pci_bus_handle;
 	int pci_resource_id;
 	struct resource *pci_resource;
 
 	/* Secondary BAR (RID 5) apparently used for MSI-X */
 	int pci_resource_id_msix;
 	struct resource *pci_resource_msix;
 
 	/* Interrupt resources */
 	void *intr_tag[2];
 	struct resource *intr_res[2];
 	unsigned intr_count;
 
 	struct ccp_queue queues[MAX_HW_QUEUES];
 };
 
 /* Internal globals */
 SYSCTL_DECL(_hw_ccp);
 MALLOC_DECLARE(M_CCP);
 extern bool g_debug_print;
 extern struct ccp_softc *g_ccp_softc;
 
 /*
  * Debug macros.
  */
 #define DPRINTF(dev, ...)	do {				\
 	if (!g_debug_print)					\
 		break;						\
 	if ((dev) != NULL)					\
 		device_printf((dev), "XXX " __VA_ARGS__);	\
 	else							\
 		printf("ccpXXX: " __VA_ARGS__);			\
 } while (0)
 
 #if 0
 #define INSECURE_DEBUG(dev, ...)	do {			\
 	if (!g_debug_print)					\
 		break;						\
 	if ((dev) != NULL)					\
 		device_printf((dev), "XXX " __VA_ARGS__);	\
 	else							\
 		printf("ccpXXX: " __VA_ARGS__);			\
 } while (0)
 #else
 #define INSECURE_DEBUG(dev, ...)
 #endif
 
 /*
  * Internal hardware manipulation routines.
  */
 int ccp_hw_attach(device_t dev);
 void ccp_hw_detach(device_t dev);
 
 void ccp_queue_write_tail(struct ccp_queue *qp);
 
 #ifdef DDB
 void db_ccp_show_hw(struct ccp_softc *sc);
 void db_ccp_show_queue_hw(struct ccp_queue *qp);
 #endif
 
 /*
  * Internal hardware crypt-op submission routines.
  */
 int ccp_authenc(struct ccp_queue *sc, struct ccp_session *s,
-    struct cryptop *crp, struct cryptodesc *crda, struct cryptodesc *crde)
-    __must_check;
+    struct cryptop *crp) __must_check;
 int ccp_blkcipher(struct ccp_queue *sc, struct ccp_session *s,
     struct cryptop *crp) __must_check;
-int ccp_gcm(struct ccp_queue *sc, struct ccp_session *s, struct cryptop *crp,
-    struct cryptodesc *crda, struct cryptodesc *crde) __must_check;
+int ccp_gcm(struct ccp_queue *sc, struct ccp_session *s, struct cryptop *crp)
+    __must_check;
 int ccp_hmac(struct ccp_queue *sc, struct ccp_session *s, struct cryptop *crp)
     __must_check;
 
 /*
  * Internal hardware TRNG read routine.
  */
 u_int random_ccp_read(void *v, u_int c);
 
 /* XXX */
 int ccp_queue_acquire_reserve(struct ccp_queue *qp, unsigned n, int mflags)
     __must_check;
 void ccp_queue_abort(struct ccp_queue *qp);
 void ccp_queue_release(struct ccp_queue *qp);
 
 /*
  * Internal inline routines.
  */
 static inline unsigned
 ccp_queue_get_active(struct ccp_queue *qp)
 {
 	struct ccp_softc *sc;
 
 	sc = qp->cq_softc;
 	return ((qp->cq_tail - qp->cq_head) & ((1 << sc->ring_size_order) - 1));
 }
 
 static inline unsigned
 ccp_queue_get_ring_space(struct ccp_queue *qp)
 {
 	struct ccp_softc *sc;
 
 	sc = qp->cq_softc;
 	return ((1 << sc->ring_size_order) - ccp_queue_get_active(qp) - 1);
 }
Index: head/sys/crypto/ccp/ccp_hardware.c
===================================================================
--- head/sys/crypto/ccp/ccp_hardware.c	(revision 359373)
+++ head/sys/crypto/ccp/ccp_hardware.c	(revision 359374)
@@ -1,2143 +1,2113 @@
 /*-
  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
  *
  * Copyright (c) 2017 Chelsio Communications, Inc.
  * Copyright (c) 2017 Conrad Meyer <cem@FreeBSD.org>
  * All rights reserved.
  * Largely borrowed from ccr(4), Written by: John Baldwin <jhb@FreeBSD.org>
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include "opt_ddb.h"
 
 #include <sys/param.h>
 #include <sys/bus.h>
 #include <sys/lock.h>
 #include <sys/kernel.h>
 #include <sys/malloc.h>
 #include <sys/mutex.h>
 #include <sys/module.h>
 #include <sys/rman.h>
 #include <sys/sglist.h>
 #include <sys/sysctl.h>
 
 #ifdef DDB
 #include <ddb/ddb.h>
 #endif
 
 #include <dev/pci/pcireg.h>
 #include <dev/pci/pcivar.h>
 
 #include <machine/bus.h>
 #include <machine/resource.h>
 #include <machine/vmparam.h>
 
 #include <opencrypto/cryptodev.h>
 #include <opencrypto/xform.h>
 
 #include <vm/vm.h>
 #include <vm/pmap.h>
 
 #include "cryptodev_if.h"
 
 #include "ccp.h"
 #include "ccp_hardware.h"
 #include "ccp_lsb.h"
 
 CTASSERT(sizeof(struct ccp_desc) == 32);
 
 static struct ccp_xts_unitsize_map_entry {
 	enum ccp_xts_unitsize cxu_id;
 	unsigned cxu_size;
 } ccp_xts_unitsize_map[] = {
 	{ CCP_XTS_AES_UNIT_SIZE_16, 16 },
 	{ CCP_XTS_AES_UNIT_SIZE_512, 512 },
 	{ CCP_XTS_AES_UNIT_SIZE_1024, 1024 },
 	{ CCP_XTS_AES_UNIT_SIZE_2048, 2048 },
 	{ CCP_XTS_AES_UNIT_SIZE_4096, 4096 },
 };
 
 SYSCTL_NODE(_hw, OID_AUTO, ccp, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
     "ccp node");
 
 unsigned g_ccp_ring_order = 11;
 SYSCTL_UINT(_hw_ccp, OID_AUTO, ring_order, CTLFLAG_RDTUN, &g_ccp_ring_order,
     0, "Set CCP ring order.  (1 << this) == ring size.  Min: 6, Max: 16");
 
 /*
  * Zero buffer, sufficient for padding LSB entries, that does not span a page
  * boundary
  */
 static const char g_zeroes[32] __aligned(32);
 
 static inline uint32_t
 ccp_read_4(struct ccp_softc *sc, uint32_t offset)
 {
 	return (bus_space_read_4(sc->pci_bus_tag, sc->pci_bus_handle, offset));
 }
 
 static inline void
 ccp_write_4(struct ccp_softc *sc, uint32_t offset, uint32_t value)
 {
 	bus_space_write_4(sc->pci_bus_tag, sc->pci_bus_handle, offset, value);
 }
 
 static inline uint32_t
 ccp_read_queue_4(struct ccp_softc *sc, unsigned queue, uint32_t offset)
 {
 	/*
 	 * Each queue gets its own 4kB register space.  Queue 0 is at 0x1000.
 	 */
 	return (ccp_read_4(sc, (CMD_Q_STATUS_INCR * (1 + queue)) + offset));
 }
 
 static inline void
 ccp_write_queue_4(struct ccp_softc *sc, unsigned queue, uint32_t offset,
     uint32_t value)
 {
 	ccp_write_4(sc, (CMD_Q_STATUS_INCR * (1 + queue)) + offset, value);
 }
 
 void
 ccp_queue_write_tail(struct ccp_queue *qp)
 {
 	ccp_write_queue_4(qp->cq_softc, qp->cq_qindex, CMD_Q_TAIL_LO_BASE,
 	    ((uint32_t)qp->desc_ring_bus_addr) + (Q_DESC_SIZE * qp->cq_tail));
 }
 
 /*
  * Given a queue and a reserved LSB entry index, compute the LSB *entry id* of
  * that entry for the queue's private LSB region.
  */
 static inline uint8_t
 ccp_queue_lsb_entry(struct ccp_queue *qp, unsigned lsb_entry)
 {
 	return ((qp->private_lsb * LSB_REGION_LENGTH + lsb_entry));
 }
 
 /*
  * Given a queue and a reserved LSB entry index, compute the LSB *address* of
  * that entry for the queue's private LSB region.
  */
 static inline uint32_t
 ccp_queue_lsb_address(struct ccp_queue *qp, unsigned lsb_entry)
 {
 	return (ccp_queue_lsb_entry(qp, lsb_entry) * LSB_ENTRY_SIZE);
 }
 
 /*
  * Some terminology:
  *
  * LSB - Local Storage Block
  * =========================
  *
  * 8 segments/regions, each containing 16 entries.
  *
  * Each entry contains 256 bits (32 bytes).
  *
  * Segments are virtually addressed in commands, but accesses cannot cross
  * segment boundaries.  Virtual map uses an identity mapping by default
  * (virtual segment N corresponds to physical segment N).
  *
  * Access to a physical region can be restricted to any subset of all five
  * queues.
  *
  * "Pass-through" mode
  * ===================
  *
  * Pass-through is a generic DMA engine, much like ioat(4).  Some nice
  * features:
  *
  * - Supports byte-swapping for endian conversion (32- or 256-bit words)
  * - AND, OR, XOR with fixed 256-bit mask
  * - CRC32 of data (may be used in tandem with bswap, but not bit operations)
  * - Read/write of LSB
  * - Memset
  *
  * If bit manipulation mode is enabled, input must be a multiple of 256 bits
  * (32 bytes).
  *
  * If byte-swapping is enabled, input must be a multiple of the word size.
  *
  * Zlib mode -- only usable from one queue at a time, single job at a time.
  * ========================================================================
  *
  * Only usable from private host, aka PSP?  Not host processor?
  *
  * RNG.
  * ====
  *
  * Raw bits are conditioned with AES and fed through CTR_DRBG.  Output goes in
  * a ring buffer readable by software.
  *
  * NIST SP 800-90B Repetition Count and Adaptive Proportion health checks are
  * implemented on the raw input stream and may be enabled to verify min-entropy
  * of 0.5 bits per bit.
  */
 
 static void
 ccp_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 {
 	bus_addr_t *baddr;
 
 	KASSERT(error == 0, ("%s: error:%d", __func__, error));
 	baddr = arg;
 	*baddr = segs->ds_addr;
 }
 
 static int
 ccp_hw_attach_queue(device_t dev, uint64_t lsbmask, unsigned queue)
 {
 	struct ccp_softc *sc;
 	struct ccp_queue *qp;
 	void *desc;
 	size_t ringsz, num_descriptors;
 	int error;
 
 	desc = NULL;
 	sc = device_get_softc(dev);
 	qp = &sc->queues[queue];
 
 	/*
 	 * Don't bother allocating a ring for queues the host isn't allowed to
 	 * drive.
 	 */
 	if ((sc->valid_queues & (1 << queue)) == 0)
 		return (0);
 
 	ccp_queue_decode_lsb_regions(sc, lsbmask, queue);
 
 	/* Ignore queues that do not have any LSB access. */
 	if (qp->lsb_mask == 0) {
 		device_printf(dev, "Ignoring queue %u with no LSB access\n",
 		    queue);
 		sc->valid_queues &= ~(1 << queue);
 		return (0);
 	}
 
 	num_descriptors = 1 << sc->ring_size_order;
 	ringsz = sizeof(struct ccp_desc) * num_descriptors;
 
 	/*
 	 * "Queue_Size" is order - 1.
 	 *
 	 * Queue must be aligned to 5+Queue_Size+1 == 5 + order bits.
 	 */
 	error = bus_dma_tag_create(bus_get_dma_tag(dev),
 	    1 << (5 + sc->ring_size_order),
 #if defined(__i386__) && !defined(PAE)
 	    0, BUS_SPACE_MAXADDR,
 #else
 	    (bus_addr_t)1 << 32, BUS_SPACE_MAXADDR_48BIT,
 #endif
 	    BUS_SPACE_MAXADDR, NULL, NULL, ringsz, 1,
 	    ringsz, 0, NULL, NULL, &qp->ring_desc_tag);
 	if (error != 0)
 		goto out;
 
 	error = bus_dmamem_alloc(qp->ring_desc_tag, &desc,
 	    BUS_DMA_ZERO | BUS_DMA_WAITOK, &qp->ring_desc_map);
 	if (error != 0)
 		goto out;
 
 	error = bus_dmamap_load(qp->ring_desc_tag, qp->ring_desc_map, desc,
 	    ringsz, ccp_dmamap_cb, &qp->desc_ring_bus_addr, BUS_DMA_WAITOK);
 	if (error != 0)
 		goto out;
 
 	qp->desc_ring = desc;
 	qp->completions_ring = malloc(num_descriptors *
 	    sizeof(*qp->completions_ring), M_CCP, M_ZERO | M_WAITOK);
 
 	/* Zero control register; among other things, clears the RUN flag. */
 	qp->qcontrol = 0;
 	ccp_write_queue_4(sc, queue, CMD_Q_CONTROL_BASE, qp->qcontrol);
 	ccp_write_queue_4(sc, queue, CMD_Q_INT_ENABLE_BASE, 0);
 
 	/* Clear any leftover interrupt status flags */
 	ccp_write_queue_4(sc, queue, CMD_Q_INTERRUPT_STATUS_BASE,
 	    ALL_INTERRUPTS);
 
 	qp->qcontrol |= (sc->ring_size_order - 1) << CMD_Q_SIZE_SHIFT;
 
 	ccp_write_queue_4(sc, queue, CMD_Q_TAIL_LO_BASE,
 	    (uint32_t)qp->desc_ring_bus_addr);
 	ccp_write_queue_4(sc, queue, CMD_Q_HEAD_LO_BASE,
 	    (uint32_t)qp->desc_ring_bus_addr);
 
 	/*
 	 * Enable completion interrupts, as well as error or administrative
 	 * halt interrupts.  We don't use administrative halts, but they
 	 * shouldn't trip unless we do, so it ought to be harmless.
 	 */
 	ccp_write_queue_4(sc, queue, CMD_Q_INT_ENABLE_BASE,
 	    INT_COMPLETION | INT_ERROR | INT_QUEUE_STOPPED);
 
 	qp->qcontrol |= (qp->desc_ring_bus_addr >> 32) << CMD_Q_PTR_HI_SHIFT;
 	qp->qcontrol |= CMD_Q_RUN;
 	ccp_write_queue_4(sc, queue, CMD_Q_CONTROL_BASE, qp->qcontrol);
 
 out:
 	if (error != 0) {
 		if (qp->desc_ring != NULL)
 			bus_dmamap_unload(qp->ring_desc_tag,
 			    qp->ring_desc_map);
 		if (desc != NULL)
 			bus_dmamem_free(qp->ring_desc_tag, desc,
 			    qp->ring_desc_map);
 		if (qp->ring_desc_tag != NULL)
 			bus_dma_tag_destroy(qp->ring_desc_tag);
 	}
 	return (error);
 }
 
 static void
 ccp_hw_detach_queue(device_t dev, unsigned queue)
 {
 	struct ccp_softc *sc;
 	struct ccp_queue *qp;
 
 	sc = device_get_softc(dev);
 	qp = &sc->queues[queue];
 
 	/*
 	 * Don't bother allocating a ring for queues the host isn't allowed to
 	 * drive.
 	 */
 	if ((sc->valid_queues & (1 << queue)) == 0)
 		return;
 
 	free(qp->completions_ring, M_CCP);
 	bus_dmamap_unload(qp->ring_desc_tag, qp->ring_desc_map);
 	bus_dmamem_free(qp->ring_desc_tag, qp->desc_ring, qp->ring_desc_map);
 	bus_dma_tag_destroy(qp->ring_desc_tag);
 }
 
 static int
 ccp_map_pci_bar(device_t dev)
 {
 	struct ccp_softc *sc;
 
 	sc = device_get_softc(dev);
 
 	sc->pci_resource_id = PCIR_BAR(2);
 	sc->pci_resource = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
 	    &sc->pci_resource_id, RF_ACTIVE);
 	if (sc->pci_resource == NULL) {
 		device_printf(dev, "unable to allocate pci resource\n");
 		return (ENODEV);
 	}
 
 	sc->pci_resource_id_msix = PCIR_BAR(5);
 	sc->pci_resource_msix = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
 	    &sc->pci_resource_id_msix, RF_ACTIVE);
 	if (sc->pci_resource_msix == NULL) {
 		device_printf(dev, "unable to allocate pci resource msix\n");
 		bus_release_resource(dev, SYS_RES_MEMORY, sc->pci_resource_id,
 		    sc->pci_resource);
 		return (ENODEV);
 	}
 
 	sc->pci_bus_tag = rman_get_bustag(sc->pci_resource);
 	sc->pci_bus_handle = rman_get_bushandle(sc->pci_resource);
 	return (0);
 }
 
 static void
 ccp_unmap_pci_bar(device_t dev)
 {
 	struct ccp_softc *sc;
 
 	sc = device_get_softc(dev);
 
 	bus_release_resource(dev, SYS_RES_MEMORY, sc->pci_resource_id_msix,
 	    sc->pci_resource_msix);
 	bus_release_resource(dev, SYS_RES_MEMORY, sc->pci_resource_id,
 	    sc->pci_resource);
 }
 
 const static struct ccp_error_code {
 	uint8_t		ce_code;
 	const char	*ce_name;
 	int		ce_errno;
 	const char	*ce_desc;
 } ccp_error_codes[] = {
 	{ 0x01, "ILLEGAL_ENGINE", EIO, "Requested engine was invalid" },
 	{ 0x03, "ILLEGAL_FUNCTION_TYPE", EIO,
 	    "A non-supported function type was specified" },
 	{ 0x04, "ILLEGAL_FUNCTION_MODE", EIO,
 	    "A non-supported function mode was specified" },
 	{ 0x05, "ILLEGAL_FUNCTION_ENCRYPT", EIO,
 	    "A CMAC type was specified when ENCRYPT was not specified" },
 	{ 0x06, "ILLEGAL_FUNCTION_SIZE", EIO,
 	    "A non-supported function size was specified.\n"
 	    "AES-CFB: Size was not 127 or 7;\n"
 	    "3DES-CFB: Size was not 7;\n"
 	    "RSA: See supported size table (7.4.2);\n"
 	    "ECC: Size was greater than 576 bits." },
 	{ 0x07, "Zlib_MISSING_INIT_EOM", EIO,
 	    "Zlib command does not have INIT and EOM set" },
 	{ 0x08, "ILLEGAL_FUNCTION_RSVD", EIO,
 	    "Reserved bits in a function specification were not 0" },
 	{ 0x09, "ILLEGAL_BUFFER_LENGTH", EIO,
 	    "The buffer length specified was not correct for the selected engine"
 	},
 	{ 0x0A, "VLSB_FAULT", EIO, "Illegal VLSB segment mapping:\n"
 	    "Undefined VLSB segment mapping or\n"
 	    "mapping to unsupported LSB segment id" },
 	{ 0x0B, "ILLEGAL_MEM_ADDR", EFAULT,
 	    "The specified source/destination buffer access was illegal:\n"
 	    "Data buffer located in a LSB location disallowed by the LSB protection masks; or\n"
 	    "Data buffer not completely contained within a single segment; or\n"
 	    "Pointer with Fixed=1 is not 32-bit aligned; or\n"
 	    "Pointer with Fixed=1 attempted to reference non-AXI1 (local) memory."
 	},
 	{ 0x0C, "ILLEGAL_MEM_SEL", EIO,
 	    "A src_mem, dst_mem, or key_mem field was illegal:\n"
 	    "A field was set to a reserved value; or\n"
 	    "A public command attempted to reference AXI1 (local) or GART memory; or\n"
 	    "A Zlib command attmpted to use the LSB." },
 	{ 0x0D, "ILLEGAL_CONTEXT_ADDR", EIO,
 	    "The specified context location was illegal:\n"
 	    "Context located in a LSB location disallowed by the LSB protection masks; or\n"
 	    "Context not completely contained within a single segment." },
 	{ 0x0E, "ILLEGAL_KEY_ADDR", EIO,
 	    "The specified key location was illegal:\n"
 	    "Key located in a LSB location disallowed by the LSB protection masks; or\n"
 	    "Key not completely contained within a single segment." },
 	{ 0x12, "CMD_TIMEOUT", EIO, "A command timeout violation occurred" },
 	/* XXX Could fill out these descriptions too */
 	{ 0x13, "IDMA0_AXI_SLVERR", EIO, "" },
 	{ 0x14, "IDMA0_AXI_DECERR", EIO, "" },
 	{ 0x16, "IDMA1_AXI_SLVERR", EIO, "" },
 	{ 0x17, "IDMA1_AXI_DECERR", EIO, "" },
 	{ 0x19, "ZLIBVHB_AXI_SLVERR", EIO, "" },
 	{ 0x1A, "ZLIBVHB_AXI_DECERR", EIO, "" },
 	{ 0x1C, "ZLIB_UNEXPECTED_EOM", EIO, "" },
 	{ 0x1D, "ZLIB_EXTRA_DATA", EIO, "" },
 	{ 0x1E, "ZLIB_BTYPE", EIO, "" },
 	{ 0x20, "ZLIB_UNDEFINED_DISTANCE_SYMBOL", EIO, "" },
 	{ 0x21, "ZLIB_CODE_LENGTH_SYMBOL", EIO, "" },
 	{ 0x22, "ZLIB_VHB_ILLEGAL_FETCH", EIO, "" },
 	{ 0x23, "ZLIB_UNCOMPRESSED_LEN", EIO, "" },
 	{ 0x24, "ZLIB_LIMIT_REACHED", EIO, "" },
 	{ 0x25, "ZLIB_CHECKSUM_MISMATCH", EIO, "" },
 	{ 0x26, "ODMA0_AXI_SLVERR", EIO, "" },
 	{ 0x27, "ODMA0_AXI_DECERR", EIO, "" },
 	{ 0x29, "ODMA1_AXI_SLVERR", EIO, "" },
 	{ 0x2A, "ODMA1_AXI_DECERR", EIO, "" },
 	{ 0x2B, "LSB_PARITY_ERR", EIO,
 	    "A read from the LSB encountered a parity error" },
 };
 
 static void
 ccp_intr_handle_error(struct ccp_queue *qp, const struct ccp_desc *desc)
 {
 	struct ccp_completion_ctx *cctx;
 	const struct ccp_error_code *ec;
 	struct ccp_softc *sc;
 	uint32_t status, error, esource, faultblock;
 	unsigned q, idx;
 	int errno;
 
 	sc = qp->cq_softc;
 	q = qp->cq_qindex;
 
 	status = ccp_read_queue_4(sc, q, CMD_Q_STATUS_BASE);
 
 	error = status & STATUS_ERROR_MASK;
 
 	/* Decode error status */
 	ec = NULL;
 	for (idx = 0; idx < nitems(ccp_error_codes); idx++)
 		if (ccp_error_codes[idx].ce_code == error) {
 			ec = &ccp_error_codes[idx];
 			break;
 		}
 
 	esource = (status >> STATUS_ERRORSOURCE_SHIFT) &
 	    STATUS_ERRORSOURCE_MASK;
 	faultblock = (status >> STATUS_VLSB_FAULTBLOCK_SHIFT) &
 	    STATUS_VLSB_FAULTBLOCK_MASK;
 	device_printf(sc->dev, "Error: %s (%u) Source: %u Faulting LSB block: %u\n",
 	    (ec != NULL) ? ec->ce_name : "(reserved)", error, esource,
 	    faultblock);
 	if (ec != NULL)
 		device_printf(sc->dev, "Error description: %s\n", ec->ce_desc);
 
 	/* TODO Could format the desc nicely here */
 	idx = desc - qp->desc_ring;
 	DPRINTF(sc->dev, "Bad descriptor index: %u contents: %32D\n", idx,
 	    (const void *)desc, " ");
 
 	/*
 	 * TODO Per § 14.4 "Error Handling," DMA_Status, DMA_Read/Write_Status,
 	 * Zlib Decompress status may be interesting.
 	 */
 
 	while (true) {
 		/* Keep unused descriptors zero for next use. */
 		memset(&qp->desc_ring[idx], 0, sizeof(qp->desc_ring[idx]));
 
 		cctx = &qp->completions_ring[idx];
 
 		/*
 		 * Restart procedure described in § 14.2.5.  Could be used by HoC if we
 		 * used that.
 		 *
 		 * Advance HEAD_LO past bad descriptor + any remaining in
 		 * transaction manually, then restart queue.
 		 */
 		idx = (idx + 1) % (1 << sc->ring_size_order);
 
 		/* Callback function signals end of transaction */
 		if (cctx->callback_fn != NULL) {
 			if (ec == NULL)
 				errno = EIO;
 			else
 				errno = ec->ce_errno;
 			/* TODO More specific error code */
 			cctx->callback_fn(qp, cctx->session, cctx->callback_arg, errno);
 			cctx->callback_fn = NULL;
 			break;
 		}
 	}
 
 	qp->cq_head = idx;
 	qp->cq_waiting = false;
 	wakeup(&qp->cq_tail);
 	DPRINTF(sc->dev, "%s: wrote sw head:%u\n", __func__, qp->cq_head);
 	ccp_write_queue_4(sc, q, CMD_Q_HEAD_LO_BASE,
 	    (uint32_t)qp->desc_ring_bus_addr + (idx * Q_DESC_SIZE));
 	ccp_write_queue_4(sc, q, CMD_Q_CONTROL_BASE, qp->qcontrol);
 	DPRINTF(sc->dev, "%s: Restarted queue\n", __func__);
 }
 
 static void
 ccp_intr_run_completions(struct ccp_queue *qp, uint32_t ints)
 {
 	struct ccp_completion_ctx *cctx;
 	struct ccp_softc *sc;
 	const struct ccp_desc *desc;
 	uint32_t headlo, idx;
 	unsigned q, completed;
 
 	sc = qp->cq_softc;
 	q = qp->cq_qindex;
 
 	mtx_lock(&qp->cq_lock);
 
 	/*
 	 * Hardware HEAD_LO points to the first incomplete descriptor.  Process
 	 * any submitted and completed descriptors, up to but not including
 	 * HEAD_LO.
 	 */
 	headlo = ccp_read_queue_4(sc, q, CMD_Q_HEAD_LO_BASE);
 	idx = (headlo - (uint32_t)qp->desc_ring_bus_addr) / Q_DESC_SIZE;
 
 	DPRINTF(sc->dev, "%s: hw head:%u sw head:%u\n", __func__, idx,
 	    qp->cq_head);
 	completed = 0;
 	while (qp->cq_head != idx) {
 		DPRINTF(sc->dev, "%s: completing:%u\n", __func__, qp->cq_head);
 
 		cctx = &qp->completions_ring[qp->cq_head];
 		if (cctx->callback_fn != NULL) {
 			cctx->callback_fn(qp, cctx->session,
 			    cctx->callback_arg, 0);
 			cctx->callback_fn = NULL;
 		}
 
 		/* Keep unused descriptors zero for next use. */
 		memset(&qp->desc_ring[qp->cq_head], 0,
 		    sizeof(qp->desc_ring[qp->cq_head]));
 
 		qp->cq_head = (qp->cq_head + 1) % (1 << sc->ring_size_order);
 		completed++;
 	}
 	if (completed > 0) {
 		qp->cq_waiting = false;
 		wakeup(&qp->cq_tail);
 	}
 
 	DPRINTF(sc->dev, "%s: wrote sw head:%u\n", __func__, qp->cq_head);
 
 	/*
 	 * Desc points to the first incomplete descriptor, at the time we read
 	 * HEAD_LO.  If there was an error flagged in interrupt status, the HW
 	 * will not proceed past the erroneous descriptor by itself.
 	 */
 	desc = &qp->desc_ring[idx];
 	if ((ints & INT_ERROR) != 0)
 		ccp_intr_handle_error(qp, desc);
 
 	mtx_unlock(&qp->cq_lock);
 }
 
 static void
 ccp_intr_handler(void *arg)
 {
 	struct ccp_softc *sc = arg;
 	size_t i;
 	uint32_t ints;
 
 	DPRINTF(sc->dev, "%s: interrupt\n", __func__);
 
 	/*
 	 * We get one global interrupt per PCI device, shared over all of
 	 * its queues.  Scan each valid queue on interrupt for flags indicating
 	 * activity.
 	 */
 	for (i = 0; i < nitems(sc->queues); i++) {
 		if ((sc->valid_queues & (1 << i)) == 0)
 			continue;
 
 		ints = ccp_read_queue_4(sc, i, CMD_Q_INTERRUPT_STATUS_BASE);
 		if (ints == 0)
 			continue;
 
 #if 0
 		DPRINTF(sc->dev, "%s: %x interrupts on queue %zu\n", __func__,
 		    (unsigned)ints, i);
 #endif
 		/* Write back 1s to clear interrupt status bits. */
 		ccp_write_queue_4(sc, i, CMD_Q_INTERRUPT_STATUS_BASE, ints);
 
 		/*
 		 * If there was an error, we still need to run completions on
 		 * any descriptors prior to the error.  The completions handler
 		 * invoked below will also handle the error descriptor.
 		 */
 		if ((ints & (INT_COMPLETION | INT_ERROR)) != 0)
 			ccp_intr_run_completions(&sc->queues[i], ints);
 
 		if ((ints & INT_QUEUE_STOPPED) != 0)
 			device_printf(sc->dev, "%s: queue %zu stopped\n",
 			    __func__, i);
 	}
 
 	/* Re-enable interrupts after processing */
 	for (i = 0; i < nitems(sc->queues); i++) {
 		if ((sc->valid_queues & (1 << i)) == 0)
 			continue;
 		ccp_write_queue_4(sc, i, CMD_Q_INT_ENABLE_BASE,
 		    INT_COMPLETION | INT_ERROR | INT_QUEUE_STOPPED);
 	}
 }
 
 static int
 ccp_intr_filter(void *arg)
 {
 	struct ccp_softc *sc = arg;
 	size_t i;
 
 	/* TODO: Split individual queues into separate taskqueues? */
 	for (i = 0; i < nitems(sc->queues); i++) {
 		if ((sc->valid_queues & (1 << i)) == 0)
 			continue;
 
 		/* Mask interrupt until task completes */
 		ccp_write_queue_4(sc, i, CMD_Q_INT_ENABLE_BASE, 0);
 	}
 
 	return (FILTER_SCHEDULE_THREAD);
 }
 
 static int
 ccp_setup_interrupts(struct ccp_softc *sc)
 {
 	uint32_t nvec;
 	int rid, error, n, ridcopy;
 
 	n = pci_msix_count(sc->dev);
 	if (n < 1) {
 		device_printf(sc->dev, "%s: msix_count: %d\n", __func__, n);
 		return (ENXIO);
 	}
 
 	nvec = n;
 	error = pci_alloc_msix(sc->dev, &nvec);
 	if (error != 0) {
 		device_printf(sc->dev, "%s: alloc_msix error: %d\n", __func__,
 		    error);
 		return (error);
 	}
 	if (nvec < 1) {
 		device_printf(sc->dev, "%s: alloc_msix: 0 vectors\n",
 		    __func__);
 		return (ENXIO);
 	}
 	if (nvec > nitems(sc->intr_res)) {
 		device_printf(sc->dev, "%s: too many vectors: %u\n", __func__,
 		    nvec);
 		nvec = nitems(sc->intr_res);
 	}
 
 	for (rid = 1; rid < 1 + nvec; rid++) {
 		ridcopy = rid;
 		sc->intr_res[rid - 1] = bus_alloc_resource_any(sc->dev,
 		    SYS_RES_IRQ, &ridcopy, RF_ACTIVE);
 		if (sc->intr_res[rid - 1] == NULL) {
 			device_printf(sc->dev, "%s: Failed to alloc IRQ resource\n",
 			    __func__);
 			return (ENXIO);
 		}
 
 		sc->intr_tag[rid - 1] = NULL;
 		error = bus_setup_intr(sc->dev, sc->intr_res[rid - 1],
 		    INTR_MPSAFE | INTR_TYPE_MISC, ccp_intr_filter,
 		    ccp_intr_handler, sc, &sc->intr_tag[rid - 1]);
 		if (error != 0)
 			device_printf(sc->dev, "%s: setup_intr: %d\n",
 			    __func__, error);
 	}
 	sc->intr_count = nvec;
 
 	return (error);
 }
 
 static void
 ccp_release_interrupts(struct ccp_softc *sc)
 {
 	unsigned i;
 
 	for (i = 0; i < sc->intr_count; i++) {
 		if (sc->intr_tag[i] != NULL)
 			bus_teardown_intr(sc->dev, sc->intr_res[i],
 			    sc->intr_tag[i]);
 		if (sc->intr_res[i] != NULL)
 			bus_release_resource(sc->dev, SYS_RES_IRQ,
 			    rman_get_rid(sc->intr_res[i]), sc->intr_res[i]);
 	}
 
 	pci_release_msi(sc->dev);
 }
 
 int
 ccp_hw_attach(device_t dev)
 {
 	struct ccp_softc *sc;
 	uint64_t lsbmask;
 	uint32_t version, lsbmasklo, lsbmaskhi;
 	unsigned queue_idx, j;
 	int error;
 	bool bars_mapped, interrupts_setup;
 
 	queue_idx = 0;
 	bars_mapped = interrupts_setup = false;
 	sc = device_get_softc(dev);
 
 	error = ccp_map_pci_bar(dev);
 	if (error != 0) {
 		device_printf(dev, "%s: couldn't map BAR(s)\n", __func__);
 		goto out;
 	}
 	bars_mapped = true;
 
 	error = pci_enable_busmaster(dev);
 	if (error != 0) {
 		device_printf(dev, "%s: couldn't enable busmaster\n",
 		    __func__);
 		goto out;
 	}
 
 	sc->ring_size_order = g_ccp_ring_order;
 	if (sc->ring_size_order < 6 || sc->ring_size_order > 16) {
 		device_printf(dev, "bogus hw.ccp.ring_order\n");
 		error = EINVAL;
 		goto out;
 	}
 	sc->valid_queues = ccp_read_4(sc, CMD_QUEUE_MASK_OFFSET);
 
 	version = ccp_read_4(sc, VERSION_REG);
 	if ((version & VERSION_NUM_MASK) < 5) {
 		device_printf(dev,
 		    "driver supports version 5 and later hardware\n");
 		error = ENXIO;
 		goto out;
 	}
 
 	error = ccp_setup_interrupts(sc);
 	if (error != 0)
 		goto out;
 	interrupts_setup = true;
 
 	sc->hw_version = version & VERSION_NUM_MASK;
 	sc->num_queues = (version >> VERSION_NUMVQM_SHIFT) &
 	    VERSION_NUMVQM_MASK;
 	sc->num_lsb_entries = (version >> VERSION_LSBSIZE_SHIFT) &
 	    VERSION_LSBSIZE_MASK;
 	sc->hw_features = version & VERSION_CAP_MASK;
 
 	/*
 	 * Copy private LSB mask to public registers to enable access to LSB
 	 * from all queues allowed by BIOS.
 	 */
 	lsbmasklo = ccp_read_4(sc, LSB_PRIVATE_MASK_LO_OFFSET);
 	lsbmaskhi = ccp_read_4(sc, LSB_PRIVATE_MASK_HI_OFFSET);
 	ccp_write_4(sc, LSB_PUBLIC_MASK_LO_OFFSET, lsbmasklo);
 	ccp_write_4(sc, LSB_PUBLIC_MASK_HI_OFFSET, lsbmaskhi);
 
 	lsbmask = ((uint64_t)lsbmaskhi << 30) | lsbmasklo;
 
 	for (; queue_idx < nitems(sc->queues); queue_idx++) {
 		error = ccp_hw_attach_queue(dev, lsbmask, queue_idx);
 		if (error != 0) {
 			device_printf(dev, "%s: couldn't attach queue %u\n",
 			    __func__, queue_idx);
 			goto out;
 		}
 	}
 	ccp_assign_lsb_regions(sc, lsbmask);
 
 out:
 	if (error != 0) {
 		if (interrupts_setup)
 			ccp_release_interrupts(sc);
 		for (j = 0; j < queue_idx; j++)
 			ccp_hw_detach_queue(dev, j);
 		if (sc->ring_size_order != 0)
 			pci_disable_busmaster(dev);
 		if (bars_mapped)
 			ccp_unmap_pci_bar(dev);
 	}
 	return (error);
 }
 
 void
 ccp_hw_detach(device_t dev)
 {
 	struct ccp_softc *sc;
 	unsigned i;
 
 	sc = device_get_softc(dev);
 
 	for (i = 0; i < nitems(sc->queues); i++)
 		ccp_hw_detach_queue(dev, i);
 
 	ccp_release_interrupts(sc);
 	pci_disable_busmaster(dev);
 	ccp_unmap_pci_bar(dev);
 }
 
 static int __must_check
 ccp_passthrough(struct ccp_queue *qp, bus_addr_t dst,
     enum ccp_memtype dst_type, bus_addr_t src, enum ccp_memtype src_type,
     bus_size_t len, enum ccp_passthru_byteswap swapmode,
     enum ccp_passthru_bitwise bitmode, bool interrupt,
     const struct ccp_completion_ctx *cctx)
 {
 	struct ccp_desc *desc;
 
 	if (ccp_queue_get_ring_space(qp) == 0)
 		return (EAGAIN);
 
 	desc = &qp->desc_ring[qp->cq_tail];
 
 	memset(desc, 0, sizeof(*desc));
 	desc->engine = CCP_ENGINE_PASSTHRU;
 
 	desc->pt.ioc = interrupt;
 	desc->pt.byteswap = swapmode;
 	desc->pt.bitwise = bitmode;
 	desc->length = len;
 
 	desc->src_lo = (uint32_t)src;
 	desc->src_hi = src >> 32;
 	desc->src_mem = src_type;
 
 	desc->dst_lo = (uint32_t)dst;
 	desc->dst_hi = dst >> 32;
 	desc->dst_mem = dst_type;
 
 	if (bitmode != CCP_PASSTHRU_BITWISE_NOOP)
 		desc->lsb_ctx_id = ccp_queue_lsb_entry(qp, LSB_ENTRY_KEY);
 
 	if (cctx != NULL)
 		memcpy(&qp->completions_ring[qp->cq_tail], cctx, sizeof(*cctx));
 
 	qp->cq_tail = (qp->cq_tail + 1) % (1 << qp->cq_softc->ring_size_order);
 	return (0);
 }
 
 static int __must_check
 ccp_passthrough_sgl(struct ccp_queue *qp, bus_addr_t lsb_addr, bool tolsb,
     struct sglist *sgl, bus_size_t len, bool interrupt,
     const struct ccp_completion_ctx *cctx)
 {
 	struct sglist_seg *seg;
 	size_t i, remain, nb;
 	int error;
 
 	remain = len;
 	for (i = 0; i < sgl->sg_nseg && remain != 0; i++) {
 		seg = &sgl->sg_segs[i];
-		/* crd_len is int, so 32-bit min() is ok. */
+		/* crp lengths are int, so 32-bit min() is ok. */
 		nb = min(remain, seg->ss_len);
 
 		if (tolsb)
 			error = ccp_passthrough(qp, lsb_addr, CCP_MEMTYPE_SB,
 			    seg->ss_paddr, CCP_MEMTYPE_SYSTEM, nb,
 			    CCP_PASSTHRU_BYTESWAP_NOOP,
 			    CCP_PASSTHRU_BITWISE_NOOP,
 			    (nb == remain) && interrupt, cctx);
 		else
 			error = ccp_passthrough(qp, seg->ss_paddr,
 			    CCP_MEMTYPE_SYSTEM, lsb_addr, CCP_MEMTYPE_SB, nb,
 			    CCP_PASSTHRU_BYTESWAP_NOOP,
 			    CCP_PASSTHRU_BITWISE_NOOP,
 			    (nb == remain) && interrupt, cctx);
 		if (error != 0)
 			return (error);
 
 		remain -= nb;
 	}
 	return (0);
 }
 
 /*
  * Note that these vectors are in reverse of the usual order.
  */
 const struct SHA_vectors {
 	uint32_t SHA1[8];
 	uint32_t SHA224[8];
 	uint32_t SHA256[8];
 	uint64_t SHA384[8];
 	uint64_t SHA512[8];
 } SHA_H __aligned(PAGE_SIZE) = {
 	.SHA1 = {
 		0xc3d2e1f0ul,
 		0x10325476ul,
 		0x98badcfeul,
 		0xefcdab89ul,
 		0x67452301ul,
 		0,
 		0,
 		0,
 	},
 	.SHA224 = {
 		0xbefa4fa4ul,
 		0x64f98fa7ul,
 		0x68581511ul,
 		0xffc00b31ul,
 		0xf70e5939ul,
 		0x3070dd17ul,
 		0x367cd507ul,
 		0xc1059ed8ul,
 	},
 	.SHA256 = {
 		0x5be0cd19ul,
 		0x1f83d9abul,
 		0x9b05688cul,
 		0x510e527ful,
 		0xa54ff53aul,
 		0x3c6ef372ul,
 		0xbb67ae85ul,
 		0x6a09e667ul,
 	},
 	.SHA384 = {
 		0x47b5481dbefa4fa4ull,
 		0xdb0c2e0d64f98fa7ull,
 		0x8eb44a8768581511ull,
 		0x67332667ffc00b31ull,
 		0x152fecd8f70e5939ull,
 		0x9159015a3070dd17ull,
 		0x629a292a367cd507ull,
 		0xcbbb9d5dc1059ed8ull,
 	},
 	.SHA512 = {
 		0x5be0cd19137e2179ull,
 		0x1f83d9abfb41bd6bull,
 		0x9b05688c2b3e6c1full,
 		0x510e527fade682d1ull,
 		0xa54ff53a5f1d36f1ull,
 		0x3c6ef372fe94f82bull,
 		0xbb67ae8584caa73bull,
 		0x6a09e667f3bcc908ull,
 	},
 };
 /*
  * Ensure vectors do not cross a page boundary.
  *
  * Disabled due to a new Clang error:  "expression is not an integral constant
  * expression."  GCC (cross toolchain) seems to handle this assertion with
  * _Static_assert just fine.
  */
 #if 0
 CTASSERT(PAGE_SIZE - ((uintptr_t)&SHA_H % PAGE_SIZE) >= sizeof(SHA_H));
 #endif
 
 const struct SHA_Defn {
 	enum sha_version version;
 	const void *H_vectors;
 	size_t H_size;
 	struct auth_hash *axf;
 	enum ccp_sha_type engine_type;
 } SHA_definitions[] = {
 	{
 		.version = SHA1,
 		.H_vectors = SHA_H.SHA1,
 		.H_size = sizeof(SHA_H.SHA1),
 		.axf = &auth_hash_hmac_sha1,
 		.engine_type = CCP_SHA_TYPE_1,
 	},
 #if 0
 	{
 		.version = SHA2_224,
 		.H_vectors = SHA_H.SHA224,
 		.H_size = sizeof(SHA_H.SHA224),
 		.axf = &auth_hash_hmac_sha2_224,
 		.engine_type = CCP_SHA_TYPE_224,
 	},
 #endif
 	{
 		.version = SHA2_256,
 		.H_vectors = SHA_H.SHA256,
 		.H_size = sizeof(SHA_H.SHA256),
 		.axf = &auth_hash_hmac_sha2_256,
 		.engine_type = CCP_SHA_TYPE_256,
 	},
 	{
 		.version = SHA2_384,
 		.H_vectors = SHA_H.SHA384,
 		.H_size = sizeof(SHA_H.SHA384),
 		.axf = &auth_hash_hmac_sha2_384,
 		.engine_type = CCP_SHA_TYPE_384,
 	},
 	{
 		.version = SHA2_512,
 		.H_vectors = SHA_H.SHA512,
 		.H_size = sizeof(SHA_H.SHA512),
 		.axf = &auth_hash_hmac_sha2_512,
 		.engine_type = CCP_SHA_TYPE_512,
 	},
 };
 
 static int __must_check
 ccp_sha_single_desc(struct ccp_queue *qp, const struct SHA_Defn *defn,
     vm_paddr_t addr, size_t len, bool start, bool end, uint64_t msgbits)
 {
 	struct ccp_desc *desc;
 
 	if (ccp_queue_get_ring_space(qp) == 0)
 		return (EAGAIN);
 
 	desc = &qp->desc_ring[qp->cq_tail];
 
 	memset(desc, 0, sizeof(*desc));
 	desc->engine = CCP_ENGINE_SHA;
 	desc->som = start;
 	desc->eom = end;
 
 	desc->sha.type = defn->engine_type;
 	desc->length = len;
 
 	if (end) {
 		desc->sha_len_lo = (uint32_t)msgbits;
 		desc->sha_len_hi = msgbits >> 32;
 	}
 
 	desc->src_lo = (uint32_t)addr;
 	desc->src_hi = addr >> 32;
 	desc->src_mem = CCP_MEMTYPE_SYSTEM;
 
 	desc->lsb_ctx_id = ccp_queue_lsb_entry(qp, LSB_ENTRY_SHA);
 
 	qp->cq_tail = (qp->cq_tail + 1) % (1 << qp->cq_softc->ring_size_order);
 	return (0);
 }
 
 static int __must_check
 ccp_sha(struct ccp_queue *qp, enum sha_version version, struct sglist *sgl_src,
     struct sglist *sgl_dst, const struct ccp_completion_ctx *cctx)
 {
 	const struct SHA_Defn *defn;
 	struct sglist_seg *seg;
 	size_t i, msgsize, remaining, nb;
 	uint32_t lsbaddr;
 	int error;
 
 	for (i = 0; i < nitems(SHA_definitions); i++)
 		if (SHA_definitions[i].version == version)
 			break;
 	if (i == nitems(SHA_definitions))
 		return (EINVAL);
 	defn = &SHA_definitions[i];
 
 	/* XXX validate input ??? */
 
 	/* Load initial SHA state into LSB */
 	/* XXX ensure H_vectors don't span page boundaries */
 	error = ccp_passthrough(qp, ccp_queue_lsb_address(qp, LSB_ENTRY_SHA),
 	    CCP_MEMTYPE_SB, pmap_kextract((vm_offset_t)defn->H_vectors),
 	    CCP_MEMTYPE_SYSTEM, roundup2(defn->H_size, LSB_ENTRY_SIZE),
 	    CCP_PASSTHRU_BYTESWAP_NOOP, CCP_PASSTHRU_BITWISE_NOOP, false,
 	    NULL);
 	if (error != 0)
 		return (error);
 
 	/* Execute series of SHA updates on correctly sized buffers */
 	msgsize = 0;
 	for (i = 0; i < sgl_src->sg_nseg; i++) {
 		seg = &sgl_src->sg_segs[i];
 		msgsize += seg->ss_len;
 		error = ccp_sha_single_desc(qp, defn, seg->ss_paddr,
 		    seg->ss_len, i == 0, i == sgl_src->sg_nseg - 1,
 		    msgsize << 3);
 		if (error != 0)
 			return (error);
 	}
 
 	/* Copy result out to sgl_dst */
 	remaining = roundup2(defn->H_size, LSB_ENTRY_SIZE);
 	lsbaddr = ccp_queue_lsb_address(qp, LSB_ENTRY_SHA);
 	for (i = 0; i < sgl_dst->sg_nseg; i++) {
 		seg = &sgl_dst->sg_segs[i];
-		/* crd_len is int, so 32-bit min() is ok. */
+		/* crp lengths are int, so 32-bit min() is ok. */
 		nb = min(remaining, seg->ss_len);
 
 		error = ccp_passthrough(qp, seg->ss_paddr, CCP_MEMTYPE_SYSTEM,
 		    lsbaddr, CCP_MEMTYPE_SB, nb, CCP_PASSTHRU_BYTESWAP_NOOP,
 		    CCP_PASSTHRU_BITWISE_NOOP,
 		    (cctx != NULL) ? (nb == remaining) : false,
 		    (nb == remaining) ? cctx : NULL);
 		if (error != 0)
 			return (error);
 
 		remaining -= nb;
 		lsbaddr += nb;
 		if (remaining == 0)
 			break;
 	}
 
 	return (0);
 }
 
 static void
 byteswap256(uint64_t *buffer)
 {
 	uint64_t t;
 
 	t = bswap64(buffer[3]);
 	buffer[3] = bswap64(buffer[0]);
 	buffer[0] = t;
 
 	t = bswap64(buffer[2]);
 	buffer[2] = bswap64(buffer[1]);
 	buffer[1] = t;
 }
 
 /*
  * Translate CCP internal LSB hash format into a standard hash ouput.
  *
  * Manipulates input buffer with byteswap256 operation.
  */
 static void
 ccp_sha_copy_result(char *output, char *buffer, enum sha_version version)
 {
 	const struct SHA_Defn *defn;
 	size_t i;
 
 	for (i = 0; i < nitems(SHA_definitions); i++)
 		if (SHA_definitions[i].version == version)
 			break;
 	if (i == nitems(SHA_definitions))
 		panic("bogus sha version auth_mode %u\n", (unsigned)version);
 
 	defn = &SHA_definitions[i];
 
 	/* Swap 256bit manually -- DMA engine can, but with limitations */
 	byteswap256((void *)buffer);
 	if (defn->axf->hashsize > LSB_ENTRY_SIZE)
 		byteswap256((void *)(buffer + LSB_ENTRY_SIZE));
 
 	switch (defn->version) {
 	case SHA1:
 		memcpy(output, buffer + 12, defn->axf->hashsize);
 		break;
 #if 0
 	case SHA2_224:
 		memcpy(output, buffer + XXX, defn->axf->hashsize);
 		break;
 #endif
 	case SHA2_256:
 		memcpy(output, buffer, defn->axf->hashsize);
 		break;
 	case SHA2_384:
 		memcpy(output,
 		    buffer + LSB_ENTRY_SIZE * 3 - defn->axf->hashsize,
 		    defn->axf->hashsize - LSB_ENTRY_SIZE);
 		memcpy(output + defn->axf->hashsize - LSB_ENTRY_SIZE, buffer,
 		    LSB_ENTRY_SIZE);
 		break;
 	case SHA2_512:
 		memcpy(output, buffer + LSB_ENTRY_SIZE, LSB_ENTRY_SIZE);
 		memcpy(output + LSB_ENTRY_SIZE, buffer, LSB_ENTRY_SIZE);
 		break;
 	}
 }
 
 static void
 ccp_do_hmac_done(struct ccp_queue *qp, struct ccp_session *s,
-    struct cryptop *crp, struct cryptodesc *crd, int error)
+    struct cryptop *crp, int error)
 {
 	char ihash[SHA2_512_HASH_LEN /* max hash len */];
 	union authctx auth_ctx;
 	struct auth_hash *axf;
 
 	axf = s->hmac.auth_hash;
 
 	s->pending--;
 
 	if (error != 0) {
 		crp->crp_etype = error;
 		goto out;
 	}
 
 	/* Do remaining outer hash over small inner hash in software */
 	axf->Init(&auth_ctx);
 	axf->Update(&auth_ctx, s->hmac.opad, axf->blocksize);
-	ccp_sha_copy_result(ihash, s->hmac.ipad, s->hmac.auth_mode);
+	ccp_sha_copy_result(ihash, s->hmac.res, s->hmac.auth_mode);
 #if 0
 	INSECURE_DEBUG(dev, "%s sha intermediate=%64D\n", __func__,
 	    (u_char *)ihash, " ");
 #endif
 	axf->Update(&auth_ctx, ihash, axf->hashsize);
-	axf->Final(s->hmac.ipad, &auth_ctx);
+	axf->Final(s->hmac.res, &auth_ctx);
 
-	crypto_copyback(crp->crp_flags, crp->crp_buf, crd->crd_inject,
-	    s->hmac.hash_len, s->hmac.ipad);
+	if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
+		crypto_copydata(crp, crp->crp_digest_start, s->hmac.hash_len,
+		    ihash);
+		if (timingsafe_bcmp(s->hmac.res, ihash, s->hmac.hash_len) != 0)
+			crp->crp_etype = EBADMSG;
+	} else
+		crypto_copyback(crp, crp->crp_digest_start, s->hmac.hash_len,
+		    s->hmac.res);
 
 	/* Avoid leaking key material */
 	explicit_bzero(&auth_ctx, sizeof(auth_ctx));
-	explicit_bzero(s->hmac.ipad, sizeof(s->hmac.ipad));
-	explicit_bzero(s->hmac.opad, sizeof(s->hmac.opad));
+	explicit_bzero(s->hmac.res, sizeof(s->hmac.res));
 
 out:
 	crypto_done(crp);
 }
 
 static void
 ccp_hmac_done(struct ccp_queue *qp, struct ccp_session *s, void *vcrp,
     int error)
 {
-	struct cryptodesc *crd;
 	struct cryptop *crp;
 
 	crp = vcrp;
-	crd = crp->crp_desc;
-	ccp_do_hmac_done(qp, s, crp, crd, error);
+	ccp_do_hmac_done(qp, s, crp, error);
 }
 
 static int __must_check
 ccp_do_hmac(struct ccp_queue *qp, struct ccp_session *s, struct cryptop *crp,
-    struct cryptodesc *crd, const struct ccp_completion_ctx *cctx)
+    const struct ccp_completion_ctx *cctx)
 {
 	device_t dev;
 	struct auth_hash *axf;
 	int error;
 
 	dev = qp->cq_softc->dev;
 	axf = s->hmac.auth_hash;
 
 	/*
 	 * Populate the SGL describing inside hash contents.  We want to hash
 	 * the ipad (key XOR fixed bit pattern) concatenated with the user
 	 * data.
 	 */
 	sglist_reset(qp->cq_sg_ulptx);
 	error = sglist_append(qp->cq_sg_ulptx, s->hmac.ipad, axf->blocksize);
 	if (error != 0)
 		return (error);
+	if (crp->crp_aad_length != 0) {
+		error = sglist_append_sglist(qp->cq_sg_ulptx, qp->cq_sg_crp,
+		    crp->crp_aad_start, crp->crp_aad_length);
+		if (error != 0)
+			return (error);
+	}
 	error = sglist_append_sglist(qp->cq_sg_ulptx, qp->cq_sg_crp,
-	    crd->crd_skip, crd->crd_len);
+	    crp->crp_payload_start, crp->crp_payload_length);
 	if (error != 0) {
 		DPRINTF(dev, "%s: sglist too short\n", __func__);
 		return (error);
 	}
-	/* Populate SGL for output -- just reuse hmac.ipad buffer. */
+	/* Populate SGL for output -- use hmac.res buffer. */
 	sglist_reset(qp->cq_sg_dst);
-	error = sglist_append(qp->cq_sg_dst, s->hmac.ipad,
+	error = sglist_append(qp->cq_sg_dst, s->hmac.res,
 	    roundup2(axf->hashsize, LSB_ENTRY_SIZE));
 	if (error != 0)
 		return (error);
 
 	error = ccp_sha(qp, s->hmac.auth_mode, qp->cq_sg_ulptx, qp->cq_sg_dst,
 	    cctx);
 	if (error != 0) {
 		DPRINTF(dev, "%s: ccp_sha error\n", __func__);
 		return (error);
 	}
 	return (0);
 }
 
 int __must_check
 ccp_hmac(struct ccp_queue *qp, struct ccp_session *s, struct cryptop *crp)
 {
 	struct ccp_completion_ctx ctx;
-	struct cryptodesc *crd;
 
-	crd = crp->crp_desc;
-
 	ctx.callback_fn = ccp_hmac_done;
 	ctx.callback_arg = crp;
 	ctx.session = s;
 
-	return (ccp_do_hmac(qp, s, crp, crd, &ctx));
+	return (ccp_do_hmac(qp, s, crp, &ctx));
 }
 
 static void
 ccp_byteswap(char *data, size_t len)
 {
 	size_t i;
 	char t;
 
 	len--;
 	for (i = 0; i < len; i++, len--) {
 		t = data[i];
 		data[i] = data[len];
 		data[len] = t;
 	}
 }
 
 static void
 ccp_blkcipher_done(struct ccp_queue *qp, struct ccp_session *s, void *vcrp,
     int error)
 {
 	struct cryptop *crp;
 
-	explicit_bzero(&s->blkcipher, sizeof(s->blkcipher));
+	explicit_bzero(&s->blkcipher.iv, sizeof(s->blkcipher.iv));
 
 	crp = vcrp;
 
 	s->pending--;
 
 	if (error != 0)
 		crp->crp_etype = error;
 
 	DPRINTF(qp->cq_softc->dev, "%s: qp=%p crp=%p\n", __func__, qp, crp);
 	crypto_done(crp);
 }
 
 static void
-ccp_collect_iv(struct ccp_session *s, struct cryptop *crp,
-    struct cryptodesc *crd)
-{
+ccp_collect_iv(struct cryptop *crp, const struct crypto_session_params *csp,
+    char *iv)
+{	
 
-	if (crd->crd_flags & CRD_F_ENCRYPT) {
-		if (crd->crd_flags & CRD_F_IV_EXPLICIT)
-			memcpy(s->blkcipher.iv, crd->crd_iv,
-			    s->blkcipher.iv_len);
-		else
-			arc4rand(s->blkcipher.iv, s->blkcipher.iv_len, 0);
-		if ((crd->crd_flags & CRD_F_IV_PRESENT) == 0)
-			crypto_copyback(crp->crp_flags, crp->crp_buf,
-			    crd->crd_inject, s->blkcipher.iv_len,
-			    s->blkcipher.iv);
-	} else {
-		if (crd->crd_flags & CRD_F_IV_EXPLICIT)
-			memcpy(s->blkcipher.iv, crd->crd_iv,
-			    s->blkcipher.iv_len);
-		else
-			crypto_copydata(crp->crp_flags, crp->crp_buf,
-			    crd->crd_inject, s->blkcipher.iv_len,
-			    s->blkcipher.iv);
-	}
+	if (crp->crp_flags & CRYPTO_F_IV_GENERATE) {
+		arc4rand(iv, csp->csp_ivlen, 0);
+		crypto_copyback(crp, crp->crp_iv_start, csp->csp_ivlen, iv);
+	} else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+		memcpy(iv, crp->crp_iv, csp->csp_ivlen);
+	else
+		crypto_copydata(crp, crp->crp_iv_start, csp->csp_ivlen, iv);
 
 	/*
 	 * If the input IV is 12 bytes, append an explicit counter of 1.
 	 */
-	if (crd->crd_alg == CRYPTO_AES_NIST_GCM_16 &&
-	    s->blkcipher.iv_len == 12) {
-		*(uint32_t *)&s->blkcipher.iv[12] = htobe32(1);
-		s->blkcipher.iv_len = AES_BLOCK_LEN;
-	}
+	if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16 &&
+	    csp->csp_ivlen == 12)
+		*(uint32_t *)&iv[12] = htobe32(1);
 
-	if (crd->crd_alg == CRYPTO_AES_XTS && s->blkcipher.iv_len != AES_BLOCK_LEN) {
-		DPRINTF(NULL, "got ivlen != 16: %u\n", s->blkcipher.iv_len);
-		if (s->blkcipher.iv_len < AES_BLOCK_LEN)
-			memset(&s->blkcipher.iv[s->blkcipher.iv_len], 0,
-			    AES_BLOCK_LEN - s->blkcipher.iv_len);
-		s->blkcipher.iv_len = AES_BLOCK_LEN;
-	}
+	if (csp->csp_cipher_alg == CRYPTO_AES_XTS &&
+	    csp->csp_ivlen < AES_BLOCK_LEN)
+		memset(&iv[csp->csp_ivlen], 0, AES_BLOCK_LEN - csp->csp_ivlen);
 
 	/* Reverse order of IV material for HW */
-	INSECURE_DEBUG(NULL, "%s: IV: %16D len: %u\n", __func__,
-	    s->blkcipher.iv, " ", s->blkcipher.iv_len);
+	INSECURE_DEBUG(NULL, "%s: IV: %16D len: %u\n", __func__, iv, " ",
+	    csp->csp_ivlen);
 
 	/*
 	 * For unknown reasons, XTS mode expects the IV in the reverse byte
 	 * order to every other AES mode.
 	 */
-	if (crd->crd_alg != CRYPTO_AES_XTS)
-		ccp_byteswap(s->blkcipher.iv, s->blkcipher.iv_len);
+	if (csp->csp_cipher_alg != CRYPTO_AES_XTS)
+		ccp_byteswap(iv, AES_BLOCK_LEN);
 }
 
 static int __must_check
 ccp_do_pst_to_lsb(struct ccp_queue *qp, uint32_t lsbaddr, const void *src,
     size_t len)
 {
 	int error;
 
 	sglist_reset(qp->cq_sg_ulptx);
 	error = sglist_append(qp->cq_sg_ulptx, __DECONST(void *, src), len);
 	if (error != 0)
 		return (error);
 
 	error = ccp_passthrough_sgl(qp, lsbaddr, true, qp->cq_sg_ulptx, len,
 	    false, NULL);
 	return (error);
 }
 
 static int __must_check
 ccp_do_xts(struct ccp_queue *qp, struct ccp_session *s, struct cryptop *crp,
-    struct cryptodesc *crd, enum ccp_cipher_dir dir,
-    const struct ccp_completion_ctx *cctx)
+    enum ccp_cipher_dir dir, const struct ccp_completion_ctx *cctx)
 {
 	struct ccp_desc *desc;
 	device_t dev;
 	unsigned i;
 	enum ccp_xts_unitsize usize;
 
 	/* IV and Key data are already loaded */
 
 	dev = qp->cq_softc->dev;
 
 	for (i = 0; i < nitems(ccp_xts_unitsize_map); i++)
-		if (ccp_xts_unitsize_map[i].cxu_size == crd->crd_len) {
+		if (ccp_xts_unitsize_map[i].cxu_size ==
+		    crp->crp_payload_length) {
 			usize = ccp_xts_unitsize_map[i].cxu_id;
 			break;
 		}
 	if (i >= nitems(ccp_xts_unitsize_map))
 		return (EINVAL);
 
 	for (i = 0; i < qp->cq_sg_ulptx->sg_nseg; i++) {
 		struct sglist_seg *seg;
 
 		seg = &qp->cq_sg_ulptx->sg_segs[i];
 
 		desc = &qp->desc_ring[qp->cq_tail];
 		desc->engine = CCP_ENGINE_XTS_AES;
 		desc->som = (i == 0);
 		desc->eom = (i == qp->cq_sg_ulptx->sg_nseg - 1);
 		desc->ioc = (desc->eom && cctx != NULL);
 		DPRINTF(dev, "%s: XTS %u: som:%d eom:%d ioc:%d dir:%d\n",
 		    __func__, qp->cq_tail, (int)desc->som, (int)desc->eom,
 		    (int)desc->ioc, (int)dir);
 
 		if (desc->ioc)
 			memcpy(&qp->completions_ring[qp->cq_tail], cctx,
 			    sizeof(*cctx));
 
 		desc->aes_xts.encrypt = dir;
 		desc->aes_xts.type = s->blkcipher.cipher_type;
 		desc->aes_xts.size = usize;
 
 		DPRINTF(dev, "XXX %s: XTS %u: type:%u size:%u\n", __func__,
 		    qp->cq_tail, (unsigned)desc->aes_xts.type,
 		    (unsigned)desc->aes_xts.size);
 
 		desc->length = seg->ss_len;
 		desc->src_lo = (uint32_t)seg->ss_paddr;
 		desc->src_hi = (seg->ss_paddr >> 32);
 		desc->src_mem = CCP_MEMTYPE_SYSTEM;
 
 		/* Crypt in-place */
 		desc->dst_lo = desc->src_lo;
 		desc->dst_hi = desc->src_hi;
 		desc->dst_mem = desc->src_mem;
 
 		desc->key_lo = ccp_queue_lsb_address(qp, LSB_ENTRY_KEY);
 		desc->key_hi = 0;
 		desc->key_mem = CCP_MEMTYPE_SB;
 
 		desc->lsb_ctx_id = ccp_queue_lsb_entry(qp, LSB_ENTRY_IV);
 
 		qp->cq_tail = (qp->cq_tail + 1) %
 		    (1 << qp->cq_softc->ring_size_order);
 	}
 	return (0);
 }
 
 static int __must_check
 ccp_do_blkcipher(struct ccp_queue *qp, struct ccp_session *s,
-    struct cryptop *crp, struct cryptodesc *crd,
-    const struct ccp_completion_ctx *cctx)
+    struct cryptop *crp, const struct ccp_completion_ctx *cctx)
 {
+	const struct crypto_session_params *csp;
 	struct ccp_desc *desc;
 	char *keydata;
 	device_t dev;
 	enum ccp_cipher_dir dir;
-	int error;
+	int error, iv_len;
 	size_t keydata_len;
 	unsigned i, j;
 
 	dev = qp->cq_softc->dev;
 
-	if (s->blkcipher.key_len == 0 || crd->crd_len == 0) {
+	if (s->blkcipher.key_len == 0 || crp->crp_payload_length == 0) {
 		DPRINTF(dev, "%s: empty\n", __func__);
 		return (EINVAL);
 	}
-	if ((crd->crd_len % AES_BLOCK_LEN) != 0) {
-		DPRINTF(dev, "%s: len modulo: %d\n", __func__, crd->crd_len);
+	if ((crp->crp_payload_length % AES_BLOCK_LEN) != 0) {
+		DPRINTF(dev, "%s: len modulo: %d\n", __func__,
+		    crp->crp_payload_length);
 		return (EINVAL);
 	}
 
 	/*
 	 * Individual segments must be multiples of AES block size for the HW
 	 * to process it.  Non-compliant inputs aren't bogus, just not doable
 	 * on this hardware.
 	 */
 	for (i = 0; i < qp->cq_sg_crp->sg_nseg; i++)
 		if ((qp->cq_sg_crp->sg_segs[i].ss_len % AES_BLOCK_LEN) != 0) {
 			DPRINTF(dev, "%s: seg modulo: %zu\n", __func__,
 			    qp->cq_sg_crp->sg_segs[i].ss_len);
 			return (EINVAL);
 		}
 
 	/* Gather IV/nonce data */
-	ccp_collect_iv(s, crp, crd);
+	csp = crypto_get_params(crp->crp_session);
+	ccp_collect_iv(crp, csp, s->blkcipher.iv);
+	iv_len = csp->csp_ivlen;
+	if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
+		iv_len = AES_BLOCK_LEN;
 
-	if ((crd->crd_flags & CRD_F_ENCRYPT) != 0)
+	if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
 		dir = CCP_CIPHER_DIR_ENCRYPT;
 	else
 		dir = CCP_CIPHER_DIR_DECRYPT;
 
 	/* Set up passthrough op(s) to copy IV into LSB */
 	error = ccp_do_pst_to_lsb(qp, ccp_queue_lsb_address(qp, LSB_ENTRY_IV),
-	    s->blkcipher.iv, s->blkcipher.iv_len);
+	    s->blkcipher.iv, iv_len);
 	if (error != 0)
 		return (error);
 
 	/*
 	 * Initialize keydata and keydata_len for GCC.  The default case of the
 	 * following switch is impossible to reach, but GCC doesn't know that.
 	 */
 	keydata_len = 0;
 	keydata = NULL;
 
-	switch (crd->crd_alg) {
+	switch (csp->csp_cipher_alg) {
 	case CRYPTO_AES_XTS:
 		for (j = 0; j < nitems(ccp_xts_unitsize_map); j++)
-			if (ccp_xts_unitsize_map[j].cxu_size == crd->crd_len)
+			if (ccp_xts_unitsize_map[j].cxu_size ==
+			    crp->crp_payload_length)
 				break;
 		/* Input buffer must be a supported UnitSize */
 		if (j >= nitems(ccp_xts_unitsize_map)) {
 			device_printf(dev, "%s: rejected block size: %u\n",
-			    __func__, crd->crd_len);
+			    __func__, crp->crp_payload_length);
 			return (EOPNOTSUPP);
 		}
 		/* FALLTHROUGH */
 	case CRYPTO_AES_CBC:
 	case CRYPTO_AES_ICM:
 		keydata = s->blkcipher.enckey;
 		keydata_len = s->blkcipher.key_len;
 		break;
 	}
 
 	INSECURE_DEBUG(dev, "%s: KEY(%zu): %16D\n", __func__, keydata_len,
 	    keydata, " ");
-	if (crd->crd_alg == CRYPTO_AES_XTS)
+	if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
 		INSECURE_DEBUG(dev, "%s: KEY(XTS): %64D\n", __func__, keydata, " ");
 
 	/* Reverse order of key material for HW */
 	ccp_byteswap(keydata, keydata_len);
 
 	/* Store key material into LSB to avoid page boundaries */
-	if (crd->crd_alg == CRYPTO_AES_XTS) {
+	if (csp->csp_cipher_alg == CRYPTO_AES_XTS) {
 		/*
 		 * XTS mode uses 2 256-bit vectors for the primary key and the
 		 * tweak key.  For 128-bit keys, the vectors are zero-padded.
 		 *
 		 * After byteswapping the combined OCF-provided K1:K2 vector
 		 * above, we need to reverse the order again so the hardware
 		 * gets the swapped keys in the order K1':K2'.
 		 */
 		error = ccp_do_pst_to_lsb(qp,
 		    ccp_queue_lsb_address(qp, LSB_ENTRY_KEY + 1), keydata,
 		    keydata_len / 2);
 		if (error != 0)
 			return (error);
 		error = ccp_do_pst_to_lsb(qp,
 		    ccp_queue_lsb_address(qp, LSB_ENTRY_KEY),
 		    keydata + (keydata_len / 2), keydata_len / 2);
 
 		/* Zero-pad 128 bit keys */
 		if (keydata_len == 32) {
 			if (error != 0)
 				return (error);
 			error = ccp_do_pst_to_lsb(qp,
 			    ccp_queue_lsb_address(qp, LSB_ENTRY_KEY) +
 			    keydata_len / 2, g_zeroes, keydata_len / 2);
 			if (error != 0)
 				return (error);
 			error = ccp_do_pst_to_lsb(qp,
 			    ccp_queue_lsb_address(qp, LSB_ENTRY_KEY + 1) +
 			    keydata_len / 2, g_zeroes, keydata_len / 2);
 		}
 	} else
 		error = ccp_do_pst_to_lsb(qp,
 		    ccp_queue_lsb_address(qp, LSB_ENTRY_KEY), keydata,
 		    keydata_len);
 	if (error != 0)
 		return (error);
 
 	/*
 	 * Point SGLs at the subset of cryptop buffer contents representing the
 	 * data.
 	 */
 	sglist_reset(qp->cq_sg_ulptx);
 	error = sglist_append_sglist(qp->cq_sg_ulptx, qp->cq_sg_crp,
-	    crd->crd_skip, crd->crd_len);
+	    crp->crp_payload_start, crp->crp_payload_length);
 	if (error != 0)
 		return (error);
 
 	INSECURE_DEBUG(dev, "%s: Contents: %16D\n", __func__,
 	    (void *)PHYS_TO_DMAP(qp->cq_sg_ulptx->sg_segs[0].ss_paddr), " ");
 
 	DPRINTF(dev, "%s: starting AES ops @ %u\n", __func__, qp->cq_tail);
 
 	if (ccp_queue_get_ring_space(qp) < qp->cq_sg_ulptx->sg_nseg)
 		return (EAGAIN);
 
-	if (crd->crd_alg == CRYPTO_AES_XTS)
-		return (ccp_do_xts(qp, s, crp, crd, dir, cctx));
+	if (csp->csp_cipher_alg == CRYPTO_AES_XTS)
+		return (ccp_do_xts(qp, s, crp, dir, cctx));
 
 	for (i = 0; i < qp->cq_sg_ulptx->sg_nseg; i++) {
 		struct sglist_seg *seg;
 
 		seg = &qp->cq_sg_ulptx->sg_segs[i];
 
 		desc = &qp->desc_ring[qp->cq_tail];
 		desc->engine = CCP_ENGINE_AES;
 		desc->som = (i == 0);
 		desc->eom = (i == qp->cq_sg_ulptx->sg_nseg - 1);
 		desc->ioc = (desc->eom && cctx != NULL);
 		DPRINTF(dev, "%s: AES %u: som:%d eom:%d ioc:%d dir:%d\n",
 		    __func__, qp->cq_tail, (int)desc->som, (int)desc->eom,
 		    (int)desc->ioc, (int)dir);
 
 		if (desc->ioc)
 			memcpy(&qp->completions_ring[qp->cq_tail], cctx,
 			    sizeof(*cctx));
 
 		desc->aes.encrypt = dir;
 		desc->aes.mode = s->blkcipher.cipher_mode;
 		desc->aes.type = s->blkcipher.cipher_type;
-		if (crd->crd_alg == CRYPTO_AES_ICM)
+		if (csp->csp_cipher_alg == CRYPTO_AES_ICM)
 			/*
 			 * Size of CTR value in bits, - 1.  ICM mode uses all
 			 * 128 bits as counter.
 			 */
 			desc->aes.size = 127;
 
 		DPRINTF(dev, "%s: AES %u: mode:%u type:%u size:%u\n", __func__,
 		    qp->cq_tail, (unsigned)desc->aes.mode,
 		    (unsigned)desc->aes.type, (unsigned)desc->aes.size);
 
 		desc->length = seg->ss_len;
 		desc->src_lo = (uint32_t)seg->ss_paddr;
 		desc->src_hi = (seg->ss_paddr >> 32);
 		desc->src_mem = CCP_MEMTYPE_SYSTEM;
 
 		/* Crypt in-place */
 		desc->dst_lo = desc->src_lo;
 		desc->dst_hi = desc->src_hi;
 		desc->dst_mem = desc->src_mem;
 
 		desc->key_lo = ccp_queue_lsb_address(qp, LSB_ENTRY_KEY);
 		desc->key_hi = 0;
 		desc->key_mem = CCP_MEMTYPE_SB;
 
 		desc->lsb_ctx_id = ccp_queue_lsb_entry(qp, LSB_ENTRY_IV);
 
 		qp->cq_tail = (qp->cq_tail + 1) %
 		    (1 << qp->cq_softc->ring_size_order);
 	}
 	return (0);
 }
 
 int __must_check
 ccp_blkcipher(struct ccp_queue *qp, struct ccp_session *s, struct cryptop *crp)
 {
 	struct ccp_completion_ctx ctx;
-	struct cryptodesc *crd;
 
-	crd = crp->crp_desc;
-
 	ctx.callback_fn = ccp_blkcipher_done;
 	ctx.session = s;
 	ctx.callback_arg = crp;
 
-	return (ccp_do_blkcipher(qp, s, crp, crd, &ctx));
+	return (ccp_do_blkcipher(qp, s, crp, &ctx));
 }
 
 static void
 ccp_authenc_done(struct ccp_queue *qp, struct ccp_session *s, void *vcrp,
     int error)
 {
-	struct cryptodesc *crda;
 	struct cryptop *crp;
 
-	explicit_bzero(&s->blkcipher, sizeof(s->blkcipher));
+	explicit_bzero(&s->blkcipher.iv, sizeof(s->blkcipher.iv));
 
 	crp = vcrp;
-	if (s->cipher_first)
-		crda = crp->crp_desc->crd_next;
-	else
-		crda = crp->crp_desc;
 
-	ccp_do_hmac_done(qp, s, crp, crda, error);
+	ccp_do_hmac_done(qp, s, crp, error);
 }
 
 int __must_check
-ccp_authenc(struct ccp_queue *qp, struct ccp_session *s, struct cryptop *crp,
-    struct cryptodesc *crda, struct cryptodesc *crde)
+ccp_authenc(struct ccp_queue *qp, struct ccp_session *s, struct cryptop *crp)
 {
 	struct ccp_completion_ctx ctx;
 	int error;
 
 	ctx.callback_fn = ccp_authenc_done;
 	ctx.session = s;
 	ctx.callback_arg = crp;
 
 	/* Perform first operation */
-	if (s->cipher_first)
-		error = ccp_do_blkcipher(qp, s, crp, crde, NULL);
+	if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
+		error = ccp_do_blkcipher(qp, s, crp, NULL);
 	else
-		error = ccp_do_hmac(qp, s, crp, crda, NULL);
+		error = ccp_do_hmac(qp, s, crp, NULL);
 	if (error != 0)
 		return (error);
 
 	/* Perform second operation */
-	if (s->cipher_first)
-		error = ccp_do_hmac(qp, s, crp, crda, &ctx);
+	if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
+		error = ccp_do_hmac(qp, s, crp, &ctx);
 	else
-		error = ccp_do_blkcipher(qp, s, crp, crde, &ctx);
+		error = ccp_do_blkcipher(qp, s, crp, &ctx);
 	return (error);
 }
 
 static int __must_check
 ccp_do_ghash_aad(struct ccp_queue *qp, struct ccp_session *s)
 {
 	struct ccp_desc *desc;
 	struct sglist_seg *seg;
 	unsigned i;
 
 	if (ccp_queue_get_ring_space(qp) < qp->cq_sg_ulptx->sg_nseg)
 		return (EAGAIN);
 
 	for (i = 0; i < qp->cq_sg_ulptx->sg_nseg; i++) {
 		seg = &qp->cq_sg_ulptx->sg_segs[i];
 
 		desc = &qp->desc_ring[qp->cq_tail];
 
 		desc->engine = CCP_ENGINE_AES;
 		desc->aes.mode = CCP_AES_MODE_GHASH;
 		desc->aes.type = s->blkcipher.cipher_type;
 		desc->aes.encrypt = CCP_AES_MODE_GHASH_AAD;
 
 		desc->som = (i == 0);
 		desc->length = seg->ss_len;
 
 		desc->src_lo = (uint32_t)seg->ss_paddr;
 		desc->src_hi = (seg->ss_paddr >> 32);
 		desc->src_mem = CCP_MEMTYPE_SYSTEM;
 
 		desc->lsb_ctx_id = ccp_queue_lsb_entry(qp, LSB_ENTRY_IV);
 
 		desc->key_lo = ccp_queue_lsb_address(qp, LSB_ENTRY_KEY);
 		desc->key_mem = CCP_MEMTYPE_SB;
 
 		qp->cq_tail = (qp->cq_tail + 1) %
 		    (1 << qp->cq_softc->ring_size_order);
 	}
 	return (0);
 }
 
 static int __must_check
 ccp_do_gctr(struct ccp_queue *qp, struct ccp_session *s,
     enum ccp_cipher_dir dir, struct sglist_seg *seg, bool som, bool eom)
 {
 	struct ccp_desc *desc;
 
 	if (ccp_queue_get_ring_space(qp) == 0)
 		return (EAGAIN);
 
 	desc = &qp->desc_ring[qp->cq_tail];
 
 	desc->engine = CCP_ENGINE_AES;
 	desc->aes.mode = CCP_AES_MODE_GCTR;
 	desc->aes.type = s->blkcipher.cipher_type;
 	desc->aes.encrypt = dir;
 	desc->aes.size = 8 * (seg->ss_len % GMAC_BLOCK_LEN) - 1;
 
 	desc->som = som;
 	desc->eom = eom;
 
 	/* Trailing bytes will be masked off by aes.size above. */
 	desc->length = roundup2(seg->ss_len, GMAC_BLOCK_LEN);
 
 	desc->dst_lo = desc->src_lo = (uint32_t)seg->ss_paddr;
 	desc->dst_hi = desc->src_hi = seg->ss_paddr >> 32;
 	desc->dst_mem = desc->src_mem = CCP_MEMTYPE_SYSTEM;
 
 	desc->lsb_ctx_id = ccp_queue_lsb_entry(qp, LSB_ENTRY_IV);
 
 	desc->key_lo = ccp_queue_lsb_address(qp, LSB_ENTRY_KEY);
 	desc->key_mem = CCP_MEMTYPE_SB;
 
 	qp->cq_tail = (qp->cq_tail + 1) %
 	    (1 << qp->cq_softc->ring_size_order);
 	return (0);
 }
 
 static int __must_check
 ccp_do_ghash_final(struct ccp_queue *qp, struct ccp_session *s)
 {
 	struct ccp_desc *desc;
 
 	if (ccp_queue_get_ring_space(qp) == 0)
 		return (EAGAIN);
 
 	desc = &qp->desc_ring[qp->cq_tail];
 
 	desc->engine = CCP_ENGINE_AES;
 	desc->aes.mode = CCP_AES_MODE_GHASH;
 	desc->aes.type = s->blkcipher.cipher_type;
 	desc->aes.encrypt = CCP_AES_MODE_GHASH_FINAL;
 
 	desc->length = GMAC_BLOCK_LEN;
 
 	desc->src_lo = ccp_queue_lsb_address(qp, LSB_ENTRY_GHASH_IN);
 	desc->src_mem = CCP_MEMTYPE_SB;
 
 	desc->lsb_ctx_id = ccp_queue_lsb_entry(qp, LSB_ENTRY_IV);
 
 	desc->key_lo = ccp_queue_lsb_address(qp, LSB_ENTRY_KEY);
 	desc->key_mem = CCP_MEMTYPE_SB;
 
 	desc->dst_lo = ccp_queue_lsb_address(qp, LSB_ENTRY_GHASH);
 	desc->dst_mem = CCP_MEMTYPE_SB;
 
 	qp->cq_tail = (qp->cq_tail + 1) %
 	    (1 << qp->cq_softc->ring_size_order);
 	return (0);
 }
 
 static void
 ccp_gcm_done(struct ccp_queue *qp, struct ccp_session *s, void *vcrp,
     int error)
 {
 	char tag[GMAC_DIGEST_LEN];
-	struct cryptodesc *crde, *crda;
 	struct cryptop *crp;
 
 	crp = vcrp;
-	if (s->cipher_first) {
-		crde = crp->crp_desc;
-		crda = crp->crp_desc->crd_next;
-	} else {
-		crde = crp->crp_desc->crd_next;
-		crda = crp->crp_desc;
-	}
 
 	s->pending--;
 
 	if (error != 0) {
 		crp->crp_etype = error;
 		goto out;
 	}
 
 	/* Encrypt is done.  Decrypt needs to verify tag. */
-	if ((crde->crd_flags & CRD_F_ENCRYPT) != 0)
+	if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
 		goto out;
 
 	/* Copy in message tag. */
-	crypto_copydata(crp->crp_flags, crp->crp_buf, crda->crd_inject,
-	    sizeof(tag), tag);
+	crypto_copydata(crp, crp->crp_digest_start, s->gmac.hash_len, tag);
 
 	/* Verify tag against computed GMAC */
 	if (timingsafe_bcmp(tag, s->gmac.final_block, s->gmac.hash_len) != 0)
 		crp->crp_etype = EBADMSG;
 
 out:
-	explicit_bzero(&s->blkcipher, sizeof(s->blkcipher));
-	explicit_bzero(&s->gmac, sizeof(s->gmac));
+	explicit_bzero(&s->blkcipher.iv, sizeof(s->blkcipher.iv));
+	explicit_bzero(&s->gmac.final_block, sizeof(s->gmac.final_block));
 	crypto_done(crp);
 }
 
 int __must_check
-ccp_gcm(struct ccp_queue *qp, struct ccp_session *s, struct cryptop *crp,
-    struct cryptodesc *crda, struct cryptodesc *crde)
+ccp_gcm(struct ccp_queue *qp, struct ccp_session *s, struct cryptop *crp)
 {
+	const struct crypto_session_params *csp;
 	struct ccp_completion_ctx ctx;
 	enum ccp_cipher_dir dir;
 	device_t dev;
 	unsigned i;
 	int error;
 
 	if (s->blkcipher.key_len == 0)
 		return (EINVAL);
 
-	/*
-	 * AAD is only permitted before the cipher/plain text, not
-	 * after.
-	 */
-	if (crda->crd_len + crda->crd_skip > crde->crd_len + crde->crd_skip)
-		return (EINVAL);
-
 	dev = qp->cq_softc->dev;
 
-	if ((crde->crd_flags & CRD_F_ENCRYPT) != 0)
+	if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
 		dir = CCP_CIPHER_DIR_ENCRYPT;
 	else
 		dir = CCP_CIPHER_DIR_DECRYPT;
 
 	/* Zero initial GHASH portion of context */
 	memset(s->blkcipher.iv, 0, sizeof(s->blkcipher.iv));
 
 	/* Gather IV data */
-	ccp_collect_iv(s, crp, crde);
+	csp = crypto_get_params(crp->crp_session);
+	ccp_collect_iv(crp, csp, s->blkcipher.iv);
 
 	/* Reverse order of key material for HW */
 	ccp_byteswap(s->blkcipher.enckey, s->blkcipher.key_len);
 
 	/* Prepare input buffer of concatenated lengths for final GHASH */
-	be64enc(s->gmac.final_block, (uint64_t)crda->crd_len * 8);
-	be64enc(&s->gmac.final_block[8], (uint64_t)crde->crd_len * 8);
+	be64enc(s->gmac.final_block, (uint64_t)crp->crp_aad_length * 8);
+	be64enc(&s->gmac.final_block[8], (uint64_t)crp->crp_payload_length * 8);
 
 	/* Send IV + initial zero GHASH, key data, and lengths buffer to LSB */
 	error = ccp_do_pst_to_lsb(qp, ccp_queue_lsb_address(qp, LSB_ENTRY_IV),
 	    s->blkcipher.iv, 32);
 	if (error != 0)
 		return (error);
 	error = ccp_do_pst_to_lsb(qp, ccp_queue_lsb_address(qp, LSB_ENTRY_KEY),
 	    s->blkcipher.enckey, s->blkcipher.key_len);
 	if (error != 0)
 		return (error);
 	error = ccp_do_pst_to_lsb(qp,
 	    ccp_queue_lsb_address(qp, LSB_ENTRY_GHASH_IN), s->gmac.final_block,
 	    GMAC_BLOCK_LEN);
 	if (error != 0)
 		return (error);
 
 	/* First step - compute GHASH over AAD */
-	if (crda->crd_len != 0) {
+	if (crp->crp_aad_length != 0) {
 		sglist_reset(qp->cq_sg_ulptx);
 		error = sglist_append_sglist(qp->cq_sg_ulptx, qp->cq_sg_crp,
-		    crda->crd_skip, crda->crd_len);
+		    crp->crp_aad_start, crp->crp_aad_length);
 		if (error != 0)
 			return (error);
 
 		/* This engine cannot process non-block multiple AAD data. */
 		for (i = 0; i < qp->cq_sg_ulptx->sg_nseg; i++)
 			if ((qp->cq_sg_ulptx->sg_segs[i].ss_len %
 			    GMAC_BLOCK_LEN) != 0) {
 				DPRINTF(dev, "%s: AD seg modulo: %zu\n",
 				    __func__,
 				    qp->cq_sg_ulptx->sg_segs[i].ss_len);
 				return (EINVAL);
 			}
 
 		error = ccp_do_ghash_aad(qp, s);
 		if (error != 0)
 			return (error);
 	}
 
 	/* Feed data piece by piece into GCTR */
 	sglist_reset(qp->cq_sg_ulptx);
 	error = sglist_append_sglist(qp->cq_sg_ulptx, qp->cq_sg_crp,
-	    crde->crd_skip, crde->crd_len);
+	    crp->crp_payload_start, crp->crp_payload_length);
 	if (error != 0)
 		return (error);
 
 	/*
 	 * All segments except the last must be even multiples of AES block
 	 * size for the HW to process it.  Non-compliant inputs aren't bogus,
 	 * just not doable on this hardware.
 	 *
 	 * XXX: Well, the hardware will produce a valid tag for shorter final
 	 * segment inputs, but it will still write out a block-sized plaintext
 	 * or ciphertext chunk.  For a typical CRP this tramples trailing data,
 	 * including the provided message tag.  So, reject such inputs for now.
 	 */
 	for (i = 0; i < qp->cq_sg_ulptx->sg_nseg; i++)
 		if ((qp->cq_sg_ulptx->sg_segs[i].ss_len % AES_BLOCK_LEN) != 0) {
 			DPRINTF(dev, "%s: seg modulo: %zu\n", __func__,
 			    qp->cq_sg_ulptx->sg_segs[i].ss_len);
 			return (EINVAL);
 		}
 
 	for (i = 0; i < qp->cq_sg_ulptx->sg_nseg; i++) {
 		struct sglist_seg *seg;
 
 		seg = &qp->cq_sg_ulptx->sg_segs[i];
 		error = ccp_do_gctr(qp, s, dir, seg,
-		    (i == 0 && crda->crd_len == 0),
+		    (i == 0 && crp->crp_aad_length == 0),
 		    i == (qp->cq_sg_ulptx->sg_nseg - 1));
 		if (error != 0)
 			return (error);
 	}
 
 	/* Send just initial IV (not GHASH!) to LSB again */
 	error = ccp_do_pst_to_lsb(qp, ccp_queue_lsb_address(qp, LSB_ENTRY_IV),
-	    s->blkcipher.iv, s->blkcipher.iv_len);
+	    s->blkcipher.iv, AES_BLOCK_LEN);
 	if (error != 0)
 		return (error);
 
 	ctx.callback_fn = ccp_gcm_done;
 	ctx.session = s;
 	ctx.callback_arg = crp;
 
 	/* Compute final hash and copy result back */
 	error = ccp_do_ghash_final(qp, s);
 	if (error != 0)
 		return (error);
 
 	/* When encrypting, copy computed tag out to caller buffer. */
 	sglist_reset(qp->cq_sg_ulptx);
 	if (dir == CCP_CIPHER_DIR_ENCRYPT)
 		error = sglist_append_sglist(qp->cq_sg_ulptx, qp->cq_sg_crp,
-		    crda->crd_inject, s->gmac.hash_len);
+		    crp->crp_digest_start, s->gmac.hash_len);
 	else
 		/*
 		 * For decrypting, copy the computed tag out to our session
 		 * buffer to verify in our callback.
 		 */
 		error = sglist_append(qp->cq_sg_ulptx, s->gmac.final_block,
 		    s->gmac.hash_len);
 	if (error != 0)
 		return (error);
 	error = ccp_passthrough_sgl(qp,
 	    ccp_queue_lsb_address(qp, LSB_ENTRY_GHASH), false, qp->cq_sg_ulptx,
 	    s->gmac.hash_len, true, &ctx);
 	return (error);
 }
 
 #define MAX_TRNG_RETRIES	10
 u_int
 random_ccp_read(void *v, u_int c)
 {
 	uint32_t *buf;
 	u_int i, j;
 
 	KASSERT(c % sizeof(*buf) == 0, ("%u not multiple of u_long", c));
 
 	buf = v;
 	for (i = c; i > 0; i -= sizeof(*buf)) {
 		for (j = 0; j < MAX_TRNG_RETRIES; j++) {
 			*buf = ccp_read_4(g_ccp_softc, TRNG_OUT_OFFSET);
 			if (*buf != 0)
 				break;
 		}
 		if (j == MAX_TRNG_RETRIES)
 			return (0);
 		buf++;
 	}
 	return (c);
 
 }
 
 #ifdef DDB
 void
 db_ccp_show_hw(struct ccp_softc *sc)
 {
 
 	db_printf("  queue mask: 0x%x\n",
 	    ccp_read_4(sc, CMD_QUEUE_MASK_OFFSET));
 	db_printf("  queue prio: 0x%x\n",
 	    ccp_read_4(sc, CMD_QUEUE_PRIO_OFFSET));
 	db_printf("  reqid: 0x%x\n", ccp_read_4(sc, CMD_REQID_CONFIG_OFFSET));
 	db_printf("  trng output: 0x%x\n", ccp_read_4(sc, TRNG_OUT_OFFSET));
 	db_printf("  cmd timeout: 0x%x\n",
 	    ccp_read_4(sc, CMD_CMD_TIMEOUT_OFFSET));
 	db_printf("  lsb public mask lo: 0x%x\n",
 	    ccp_read_4(sc, LSB_PUBLIC_MASK_LO_OFFSET));
 	db_printf("  lsb public mask hi: 0x%x\n",
 	    ccp_read_4(sc, LSB_PUBLIC_MASK_HI_OFFSET));
 	db_printf("  lsb private mask lo: 0x%x\n",
 	    ccp_read_4(sc, LSB_PRIVATE_MASK_LO_OFFSET));
 	db_printf("  lsb private mask hi: 0x%x\n",
 	    ccp_read_4(sc, LSB_PRIVATE_MASK_HI_OFFSET));
 	db_printf("  version: 0x%x\n", ccp_read_4(sc, VERSION_REG));
 }
 
 void
 db_ccp_show_queue_hw(struct ccp_queue *qp)
 {
 	const struct ccp_error_code *ec;
 	struct ccp_softc *sc;
 	uint32_t status, error, esource, faultblock, headlo, qcontrol;
 	unsigned q, i;
 
 	sc = qp->cq_softc;
 	q = qp->cq_qindex;
 
 	qcontrol = ccp_read_queue_4(sc, q, CMD_Q_CONTROL_BASE);
 	db_printf("  qcontrol: 0x%x%s%s\n", qcontrol,
 	    (qcontrol & CMD_Q_RUN) ? " RUN" : "",
 	    (qcontrol & CMD_Q_HALTED) ? " HALTED" : "");
 	db_printf("  tail_lo: 0x%x\n",
 	    ccp_read_queue_4(sc, q, CMD_Q_TAIL_LO_BASE));
 	headlo = ccp_read_queue_4(sc, q, CMD_Q_HEAD_LO_BASE);
 	db_printf("  head_lo: 0x%x\n", headlo);
 	db_printf("  int enable: 0x%x\n",
 	    ccp_read_queue_4(sc, q, CMD_Q_INT_ENABLE_BASE));
 	db_printf("  interrupt status: 0x%x\n",
 	    ccp_read_queue_4(sc, q, CMD_Q_INTERRUPT_STATUS_BASE));
 	status = ccp_read_queue_4(sc, q, CMD_Q_STATUS_BASE);
 	db_printf("  status: 0x%x\n", status);
 	db_printf("  int stats: 0x%x\n",
 	    ccp_read_queue_4(sc, q, CMD_Q_INT_STATUS_BASE));
 
 	error = status & STATUS_ERROR_MASK;
 	if (error == 0)
 		return;
 
 	esource = (status >> STATUS_ERRORSOURCE_SHIFT) &
 	    STATUS_ERRORSOURCE_MASK;
 	faultblock = (status >> STATUS_VLSB_FAULTBLOCK_SHIFT) &
 	    STATUS_VLSB_FAULTBLOCK_MASK;
 
 	ec = NULL;
 	for (i = 0; i < nitems(ccp_error_codes); i++)
 		if (ccp_error_codes[i].ce_code == error)
 			break;
 	if (i < nitems(ccp_error_codes))
 		ec = &ccp_error_codes[i];
 
 	db_printf("  Error: %s (%u) Source: %u Faulting LSB block: %u\n",
 	    (ec != NULL) ? ec->ce_name : "(reserved)", error, esource,
 	    faultblock);
 	if (ec != NULL)
 		db_printf("  Error description: %s\n", ec->ce_desc);
 
 	i = (headlo - (uint32_t)qp->desc_ring_bus_addr) / Q_DESC_SIZE;
 	db_printf("  Bad descriptor idx: %u contents:\n  %32D\n", i,
 	    (void *)&qp->desc_ring[i], " ");
 }
 #endif
Index: head/sys/crypto/via/padlock.c
===================================================================
--- head/sys/crypto/via/padlock.c	(revision 359373)
+++ head/sys/crypto/via/padlock.c	(revision 359374)
@@ -1,340 +1,304 @@
 /*-
  * Copyright (c) 2005-2008 Pawel Jakub Dawidek <pjd@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/module.h>
 #include <sys/lock.h>
 #include <sys/rwlock.h>
 #include <sys/malloc.h>
 #include <sys/libkern.h>
 #if defined(__amd64__) || defined(__i386__)
 #include <machine/cpufunc.h>
 #include <machine/cputypes.h>
 #include <machine/md_var.h>
 #include <machine/specialreg.h>
 #endif
 
 #include <opencrypto/cryptodev.h>
 
 #include <crypto/via/padlock.h>
 
 #include <sys/kobj.h>
 #include <sys/bus.h>
 #include "cryptodev_if.h"
 
 /*
  * Technical documentation about the PadLock engine can be found here:
  *
  * http://www.via.com.tw/en/downloads/whitepapers/initiatives/padlock/programming_guide.pdf
  */
 
 struct padlock_softc {
 	int32_t		sc_cid;
 };
 
-static int padlock_newsession(device_t, crypto_session_t cses, struct cryptoini *cri);
+static int padlock_probesession(device_t, const struct crypto_session_params *);
+static int padlock_newsession(device_t, crypto_session_t cses,
+    const struct crypto_session_params *);
 static void padlock_freesession(device_t, crypto_session_t cses);
 static void padlock_freesession_one(struct padlock_softc *sc,
     struct padlock_session *ses);
 static int padlock_process(device_t, struct cryptop *crp, int hint __unused);
 
 MALLOC_DEFINE(M_PADLOCK, "padlock_data", "PadLock Data");
 
 static void
 padlock_identify(driver_t *drv, device_t parent)
 {
 	/* NB: order 10 is so we get attached after h/w devices */
 	if (device_find_child(parent, "padlock", -1) == NULL &&
 	    BUS_ADD_CHILD(parent, 10, "padlock", -1) == 0)
 		panic("padlock: could not attach");
 }
 
 static int
 padlock_probe(device_t dev)
 {
 	char capp[256];
 
 #if defined(__amd64__) || defined(__i386__)
 	/* If there is no AES support, we has nothing to do here. */
 	if (!(via_feature_xcrypt & VIA_HAS_AES)) {
 		device_printf(dev, "No ACE support.\n");
 		return (EINVAL);
 	}
 	strlcpy(capp, "AES-CBC", sizeof(capp));
 #if 0
 	strlcat(capp, ",AES-EBC", sizeof(capp));
 	strlcat(capp, ",AES-CFB", sizeof(capp));
 	strlcat(capp, ",AES-OFB", sizeof(capp));
 #endif
 	if (via_feature_xcrypt & VIA_HAS_SHA) {
 		strlcat(capp, ",SHA1", sizeof(capp));
 		strlcat(capp, ",SHA256", sizeof(capp));
 	}
 #if 0
 	if (via_feature_xcrypt & VIA_HAS_AESCTR)
 		strlcat(capp, ",AES-CTR", sizeof(capp));
 	if (via_feature_xcrypt & VIA_HAS_MM)
 		strlcat(capp, ",RSA", sizeof(capp));
 #endif
 	device_set_desc_copy(dev, capp);
 	return (0);
 #else
 	return (EINVAL);
 #endif
 }
 
 static int
 padlock_attach(device_t dev)
 {
 	struct padlock_softc *sc = device_get_softc(dev);
 
 	sc->sc_cid = crypto_get_driverid(dev, sizeof(struct padlock_session),
 	    CRYPTOCAP_F_HARDWARE);
 	if (sc->sc_cid < 0) {
 		device_printf(dev, "Could not get crypto driver id.\n");
 		return (ENOMEM);
 	}
 
-	crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
-	crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
-	crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
-	crypto_register(sc->sc_cid, CRYPTO_RIPEMD160_HMAC, 0, 0);
-	crypto_register(sc->sc_cid, CRYPTO_SHA2_256_HMAC, 0, 0);
-	crypto_register(sc->sc_cid, CRYPTO_SHA2_384_HMAC, 0, 0);
-	crypto_register(sc->sc_cid, CRYPTO_SHA2_512_HMAC, 0, 0);
 	return (0);
 }
 
 static int
 padlock_detach(device_t dev)
 {
 	struct padlock_softc *sc = device_get_softc(dev);
 
 	crypto_unregister_all(sc->sc_cid);
 	return (0);
 }
 
 static int
-padlock_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
+padlock_probesession(device_t dev, const struct crypto_session_params *csp)
 {
-	struct padlock_softc *sc = device_get_softc(dev);
-	struct padlock_session *ses = NULL;
-	struct cryptoini *encini, *macini;
-	struct thread *td;
-	int error;
 
-	if (cri == NULL)
+	if (csp->csp_flags != 0)
 		return (EINVAL);
 
-	encini = macini = NULL;
-	for (; cri != NULL; cri = cri->cri_next) {
-		switch (cri->cri_alg) {
-		case CRYPTO_NULL_HMAC:
-		case CRYPTO_MD5_HMAC:
-		case CRYPTO_SHA1_HMAC:
-		case CRYPTO_RIPEMD160_HMAC:
-		case CRYPTO_SHA2_256_HMAC:
-		case CRYPTO_SHA2_384_HMAC:
-		case CRYPTO_SHA2_512_HMAC:
-			if (macini != NULL)
-				return (EINVAL);
-			macini = cri;
-			break;
+	/*
+	 * We only support HMAC algorithms to be able to work with
+	 * ipsec(4), so if we are asked only for authentication without
+	 * encryption, don't pretend we can accelerate it.
+	 *
+	 * XXX: For CPUs with SHA instructions we should probably
+	 * permit CSP_MODE_DIGEST so that those can be tested.
+	 */
+	switch (csp->csp_mode) {
+	case CSP_MODE_ETA:
+		if (!padlock_hash_check(csp))
+			return (EINVAL);
+		/* FALLTHROUGH */
+	case CSP_MODE_CIPHER:
+		switch (csp->csp_cipher_alg) {
 		case CRYPTO_AES_CBC:
-			if (encini != NULL)
+			if (csp->csp_ivlen != AES_BLOCK_LEN)
 				return (EINVAL);
-			encini = cri;
 			break;
 		default:
 			return (EINVAL);
 		}
+		break;
+	default:
+		return (EINVAL);
 	}
 
-	/*
-	 * We only support HMAC algorithms to be able to work with
-	 * ipsec(4), so if we are asked only for authentication without
-	 * encryption, don't pretend we can accellerate it.
-	 */
-	if (encini == NULL)
-		return (EINVAL);
+	return (CRYPTODEV_PROBE_ACCEL_SOFTWARE);
+}
 
+static int
+padlock_newsession(device_t dev, crypto_session_t cses,
+    const struct crypto_session_params *csp)
+{
+	struct padlock_softc *sc = device_get_softc(dev);
+	struct padlock_session *ses = NULL;
+	struct thread *td;
+	int error;
+
 	ses = crypto_get_driver_session(cses);
 	ses->ses_fpu_ctx = fpu_kern_alloc_ctx(FPU_KERN_NORMAL);
 
-	error = padlock_cipher_setup(ses, encini);
+	error = padlock_cipher_setup(ses, csp);
 	if (error != 0) {
 		padlock_freesession_one(sc, ses);
 		return (error);
 	}
 
-	if (macini != NULL) {
+	if (csp->csp_mode == CSP_MODE_ETA) {
 		td = curthread;
 		fpu_kern_enter(td, ses->ses_fpu_ctx, FPU_KERN_NORMAL |
 		    FPU_KERN_KTHR);
-		error = padlock_hash_setup(ses, macini);
+		error = padlock_hash_setup(ses, csp);
 		fpu_kern_leave(td, ses->ses_fpu_ctx);
 		if (error != 0) {
 			padlock_freesession_one(sc, ses);
 			return (error);
 		}
 	}
 
 	return (0);
 }
 
 static void
 padlock_freesession_one(struct padlock_softc *sc, struct padlock_session *ses)
 {
 
 	padlock_hash_free(ses);
 	fpu_kern_free_ctx(ses->ses_fpu_ctx);
 }
 
 static void
 padlock_freesession(device_t dev, crypto_session_t cses)
 {
 	struct padlock_softc *sc = device_get_softc(dev);
 	struct padlock_session *ses;
 
 	ses = crypto_get_driver_session(cses);
 	padlock_freesession_one(sc, ses);
 }
 
 static int
 padlock_process(device_t dev, struct cryptop *crp, int hint __unused)
 {
-	struct padlock_session *ses = NULL;
-	struct cryptodesc *crd, *enccrd, *maccrd;
-	int error = 0;
+	const struct crypto_session_params *csp;
+	struct padlock_session *ses;
+	int error;
 
-	enccrd = maccrd = NULL;
-
-	/* Sanity check. */
-	if (crp == NULL)
-		return (EINVAL);
-
-	if (crp->crp_callback == NULL || crp->crp_desc == NULL) {
+	if ((crp->crp_payload_length % AES_BLOCK_LEN) != 0) {
 		error = EINVAL;
 		goto out;
 	}
 
-	for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) {
-		switch (crd->crd_alg) {
-		case CRYPTO_NULL_HMAC:
-		case CRYPTO_MD5_HMAC:
-		case CRYPTO_SHA1_HMAC:
-		case CRYPTO_RIPEMD160_HMAC:
-		case CRYPTO_SHA2_256_HMAC:
-		case CRYPTO_SHA2_384_HMAC:
-		case CRYPTO_SHA2_512_HMAC:
-			if (maccrd != NULL) {
-				error = EINVAL;
-				goto out;
-			}
-			maccrd = crd;
-			break;
-		case CRYPTO_AES_CBC:
-			if (enccrd != NULL) {
-				error = EINVAL;
-				goto out;
-			}
-			enccrd = crd;
-			break;
-		default:
-			return (EINVAL);
-		}
-	}
-	if (enccrd == NULL || (enccrd->crd_len % AES_BLOCK_LEN) != 0) {
-		error = EINVAL;
-		goto out;
-	}
-
 	ses = crypto_get_driver_session(crp->crp_session);
+	csp = crypto_get_params(crp->crp_session);
 
-	/* Perform data authentication if requested before encryption. */
-	if (maccrd != NULL && maccrd->crd_next == enccrd) {
-		error = padlock_hash_process(ses, maccrd, crp);
+	/* Perform data authentication if requested before decryption. */
+	if (csp->csp_mode == CSP_MODE_ETA &&
+	    !CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
+		error = padlock_hash_process(ses, crp, csp);
 		if (error != 0)
 			goto out;
 	}
 
-	error = padlock_cipher_process(ses, enccrd, crp);
+	error = padlock_cipher_process(ses, crp, csp);
 	if (error != 0)
 		goto out;
 
 	/* Perform data authentication if requested after encryption. */
-	if (maccrd != NULL && enccrd->crd_next == maccrd) {
-		error = padlock_hash_process(ses, maccrd, crp);
+	if (csp->csp_mode == CSP_MODE_ETA &&
+	    CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
+		error = padlock_hash_process(ses, crp, csp);
 		if (error != 0)
 			goto out;
 	}
 
 out:
 #if 0
 	/*
 	 * This code is not necessary, because contexts will be freed on next
 	 * padlock_setup_mackey() call or at padlock_freesession() call.
 	 */
 	if (ses != NULL && maccrd != NULL &&
 	    (maccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) {
 		padlock_free_ctx(ses->ses_axf, ses->ses_ictx);
 		padlock_free_ctx(ses->ses_axf, ses->ses_octx);
 	}
 #endif
 	crp->crp_etype = error;
 	crypto_done(crp);
 	return (error);
 }
 
 static device_method_t padlock_methods[] = {
 	DEVMETHOD(device_identify,	padlock_identify),
 	DEVMETHOD(device_probe,		padlock_probe),
 	DEVMETHOD(device_attach,	padlock_attach),
 	DEVMETHOD(device_detach,	padlock_detach),
 
+	DEVMETHOD(cryptodev_probesession, padlock_probesession),
 	DEVMETHOD(cryptodev_newsession,	padlock_newsession),
 	DEVMETHOD(cryptodev_freesession,padlock_freesession),
 	DEVMETHOD(cryptodev_process,	padlock_process),
 
 	{0, 0},
 };
 
 static driver_t padlock_driver = {
 	"padlock",
 	padlock_methods,
 	sizeof(struct padlock_softc),
 };
 static devclass_t padlock_devclass;
 
 /* XXX where to attach */
 DRIVER_MODULE(padlock, nexus, padlock_driver, padlock_devclass, 0, 0);
 MODULE_VERSION(padlock, 1);
 MODULE_DEPEND(padlock, crypto, 1, 1, 1);
Index: head/sys/crypto/via/padlock.h
===================================================================
--- head/sys/crypto/via/padlock.h	(revision 359373)
+++ head/sys/crypto/via/padlock.h	(revision 359374)
@@ -1,91 +1,91 @@
 /*-
  * Copyright (c) 2005-2006 Pawel Jakub Dawidek <pjd@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
  * $FreeBSD$
  */
 
 #ifndef _PADLOCK_H_
 #define _PADLOCK_H_
 
 #include <opencrypto/cryptodev.h>
 #include <crypto/rijndael/rijndael.h>
 
 #if defined(__i386__)
 #include <machine/npx.h>
 #elif defined(__amd64__)
 #include <machine/fpu.h>
 #endif
 
 union padlock_cw {
 	uint64_t raw;
 	struct {
 		u_int round_count : 4;
 		u_int algorithm_type : 3;
 		u_int key_generation : 1;
 		u_int intermediate : 1;
 		u_int direction : 1;
 		u_int key_size : 2;
 		u_int filler0 : 20;
 		u_int filler1 : 32;
 		u_int filler2 : 32;
 		u_int filler3 : 32;
 	} __field;
 };
 #define	cw_round_count		__field.round_count
 #define	cw_algorithm_type	__field.algorithm_type
 #define	cw_key_generation	__field.key_generation
 #define	cw_intermediate		__field.intermediate
 #define	cw_direction		__field.direction
 #define	cw_key_size		__field.key_size
 #define	cw_filler0		__field.filler0
 #define	cw_filler1		__field.filler1
 #define	cw_filler2		__field.filler2
 #define	cw_filler3		__field.filler3
 
 struct padlock_session {
 	union padlock_cw ses_cw __aligned(16);
 	uint32_t	ses_ekey[4 * (RIJNDAEL_MAXNR + 1) + 4] __aligned(16);	/* 128 bit aligned */
 	uint32_t	ses_dkey[4 * (RIJNDAEL_MAXNR + 1) + 4] __aligned(16);	/* 128 bit aligned */
-	uint8_t		ses_iv[16] __aligned(16);			/* 128 bit aligned */
 	struct auth_hash *ses_axf;
 	uint8_t		*ses_ictx;
 	uint8_t		*ses_octx;
 	int		ses_mlen;
 	struct fpu_kern_ctx *ses_fpu_ctx;
 };
 
 #define	PADLOCK_ALIGN(p)	(void *)(roundup2((uintptr_t)(p), 16))
 
 int	padlock_cipher_setup(struct padlock_session *ses,
-	    struct cryptoini *encini);
+	    const struct crypto_session_params *csp);
 int	padlock_cipher_process(struct padlock_session *ses,
-	    struct cryptodesc *enccrd, struct cryptop *crp);
+	    struct cryptop *crp, const struct crypto_session_params *csp);
+bool	padlock_hash_check(const struct crypto_session_params *csp);
 int	padlock_hash_setup(struct padlock_session *ses,
-	    struct cryptoini *macini);
+	    const struct crypto_session_params *csp);
 int	padlock_hash_process(struct padlock_session *ses,
-	    struct cryptodesc *maccrd, struct cryptop *crp);
+	    struct cryptop *crp, const struct crypto_session_params *csp);
 void	padlock_hash_free(struct padlock_session *ses);
 
 #endif	/* !_PADLOCK_H_ */
Index: head/sys/crypto/via/padlock_cipher.c
===================================================================
--- head/sys/crypto/via/padlock_cipher.c	(revision 359373)
+++ head/sys/crypto/via/padlock_cipher.c	(revision 359374)
@@ -1,275 +1,265 @@
 /*-
  * Copyright (c) 2005-2006 Pawel Jakub Dawidek <pjd@FreeBSD.org>
  * Copyright (c) 2004 Mark R V Murray
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 /*	$OpenBSD: via.c,v 1.3 2004/06/15 23:36:55 deraadt Exp $	*/
 /*-
  * Copyright (c) 2003 Jason Wright
  * Copyright (c) 2003, 2004 Theo de Raadt
  * All rights reserved.
  *
  * Permission to use, copy, modify, and distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
  * copyright notice and this permission notice appear in all copies.
  *
  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/module.h>
 #include <sys/malloc.h>
 #include <sys/libkern.h>
 #include <sys/pcpu.h>
 #include <sys/uio.h>
 
 #include <opencrypto/cryptodev.h>
 #include <crypto/rijndael/rijndael.h>
 
 #include <crypto/via/padlock.h>
 
 #define	PADLOCK_ROUND_COUNT_AES128	10
 #define	PADLOCK_ROUND_COUNT_AES192	12
 #define	PADLOCK_ROUND_COUNT_AES256	14
 
 #define	PADLOCK_ALGORITHM_TYPE_AES	0
 
 #define	PADLOCK_KEY_GENERATION_HW	0
 #define	PADLOCK_KEY_GENERATION_SW	1
 
 #define	PADLOCK_DIRECTION_ENCRYPT	0
 #define	PADLOCK_DIRECTION_DECRYPT	1
 
 #define	PADLOCK_KEY_SIZE_128	0
 #define	PADLOCK_KEY_SIZE_192	1
 #define	PADLOCK_KEY_SIZE_256	2
 
 MALLOC_DECLARE(M_PADLOCK);
 
 static __inline void
 padlock_cbc(void *in, void *out, size_t count, void *key, union padlock_cw *cw,
     void *iv)
 {
 #ifdef __GNUCLIKE_ASM
 	/* The .byte line is really VIA C3 "xcrypt-cbc" instruction */
 	__asm __volatile(
 		"pushf				\n\t"
 		"popf				\n\t"
 		"rep				\n\t"
 		".byte	0x0f, 0xa7, 0xd0"
 			: "+a" (iv), "+c" (count), "+D" (out), "+S" (in)
 			: "b" (key), "d" (cw)
 			: "cc", "memory"
 		);
 #endif
 }
 
 static void
-padlock_cipher_key_setup(struct padlock_session *ses, caddr_t key, int klen)
+padlock_cipher_key_setup(struct padlock_session *ses, const void *key, int klen)
 {
 	union padlock_cw *cw;
 	int i;
 
 	cw = &ses->ses_cw;
 	if (cw->cw_key_generation == PADLOCK_KEY_GENERATION_SW) {
 		/* Build expanded keys for both directions */
-		rijndaelKeySetupEnc(ses->ses_ekey, key, klen);
-		rijndaelKeySetupDec(ses->ses_dkey, key, klen);
+		rijndaelKeySetupEnc(ses->ses_ekey, key, klen * 8);
+		rijndaelKeySetupDec(ses->ses_dkey, key, klen * 8);
 		for (i = 0; i < 4 * (RIJNDAEL_MAXNR + 1); i++) {
 			ses->ses_ekey[i] = ntohl(ses->ses_ekey[i]);
 			ses->ses_dkey[i] = ntohl(ses->ses_dkey[i]);
 		}
 	} else {
 		bcopy(key, ses->ses_ekey, klen);
 		bcopy(key, ses->ses_dkey, klen);
 	}
 }
 
 int
-padlock_cipher_setup(struct padlock_session *ses, struct cryptoini *encini)
+padlock_cipher_setup(struct padlock_session *ses,
+    const struct crypto_session_params *csp)
 {
 	union padlock_cw *cw;
 
-	if (encini->cri_klen != 128 && encini->cri_klen != 192 &&
-	    encini->cri_klen != 256) {
+	if (csp->csp_cipher_klen != 16 && csp->csp_cipher_klen != 25 &&
+	    csp->csp_cipher_klen != 32) {
 		return (EINVAL);
 	}
 
 	cw = &ses->ses_cw;
 	bzero(cw, sizeof(*cw));
 	cw->cw_algorithm_type = PADLOCK_ALGORITHM_TYPE_AES;
 	cw->cw_key_generation = PADLOCK_KEY_GENERATION_SW;
 	cw->cw_intermediate = 0;
-	switch (encini->cri_klen) {
+	switch (csp->csp_cipher_klen * 8) {
 	case 128:
 		cw->cw_round_count = PADLOCK_ROUND_COUNT_AES128;
 		cw->cw_key_size = PADLOCK_KEY_SIZE_128;
 #ifdef HW_KEY_GENERATION
 		/* This doesn't buy us much, that's why it is commented out. */
 		cw->cw_key_generation = PADLOCK_KEY_GENERATION_HW;
 #endif
 		break;
 	case 192:
 		cw->cw_round_count = PADLOCK_ROUND_COUNT_AES192;
 		cw->cw_key_size = PADLOCK_KEY_SIZE_192;
 		break;
 	case 256:
 		cw->cw_round_count = PADLOCK_ROUND_COUNT_AES256;
 		cw->cw_key_size = PADLOCK_KEY_SIZE_256;
 		break;
 	}
-	if (encini->cri_key != NULL) {
-		padlock_cipher_key_setup(ses, encini->cri_key,
-		    encini->cri_klen);
+	if (csp->csp_cipher_key != NULL) {
+		padlock_cipher_key_setup(ses, csp->csp_cipher_key,
+		    csp->csp_cipher_klen);
 	}
-
-	arc4rand(ses->ses_iv, sizeof(ses->ses_iv), 0);
 	return (0);
 }
 
 /*
  * Function checks if the given buffer is already 16 bytes aligned.
  * If it is there is no need to allocate new buffer.
  * If it isn't, new buffer is allocated.
  */
 static u_char *
-padlock_cipher_alloc(struct cryptodesc *enccrd, struct cryptop *crp,
-    int *allocated)
+padlock_cipher_alloc(struct cryptop *crp, int *allocated)
 {
 	u_char *addr;
 
-	if (crp->crp_flags & CRYPTO_F_IMBUF)
-		goto alloc;
-	else {
-		if (crp->crp_flags & CRYPTO_F_IOV) {
-			struct uio *uio;
-			struct iovec *iov;
+	switch (crp->crp_buf_type) {
+	case CRYPTO_BUF_MBUF:
+		break;
+	case CRYPTO_BUF_UIO: {
+		struct uio *uio;
+		struct iovec *iov;
 
-			uio = (struct uio *)crp->crp_buf;
-			if (uio->uio_iovcnt != 1)
-				goto alloc;
-			iov = uio->uio_iov;
-			addr = (u_char *)iov->iov_base + enccrd->crd_skip;
-		} else {
-			addr = (u_char *)crp->crp_buf;
-		}
+		uio = crp->crp_uio;
+		if (uio->uio_iovcnt != 1)
+			break;
+		iov = uio->uio_iov;
+		addr = (u_char *)iov->iov_base + crp->crp_payload_start;
 		if (((uintptr_t)addr & 0xf) != 0) /* 16 bytes aligned? */
-			goto alloc;
+			break;
 		*allocated = 0;
 		return (addr);
 	}
-alloc:
+	case CRYPTO_BUF_CONTIG:
+		addr = (u_char *)crp->crp_buf + crp->crp_payload_start;
+		if (((uintptr_t)addr & 0xf) != 0) /* 16 bytes aligned? */
+			break;
+		*allocated = 0;
+		return (addr);
+	}
+
 	*allocated = 1;
-	addr = malloc(enccrd->crd_len + 16, M_PADLOCK, M_NOWAIT);
+	addr = malloc(crp->crp_payload_length + 16, M_PADLOCK, M_NOWAIT);
 	return (addr);
 }
 
 int
-padlock_cipher_process(struct padlock_session *ses, struct cryptodesc *enccrd,
-    struct cryptop *crp)
+padlock_cipher_process(struct padlock_session *ses, struct cryptop *crp,
+    const struct crypto_session_params *csp)
 {
 	union padlock_cw *cw;
 	struct thread *td;
 	u_char *buf, *abuf;
 	uint32_t *key;
+	uint8_t iv[AES_BLOCK_LEN] __aligned(16);
 	int allocated;
 
-	buf = padlock_cipher_alloc(enccrd, crp, &allocated);
+	buf = padlock_cipher_alloc(crp, &allocated);
 	if (buf == NULL)
 		return (ENOMEM);
 	/* Buffer has to be 16 bytes aligned. */
 	abuf = PADLOCK_ALIGN(buf);
 
-	if ((enccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) {
-		padlock_cipher_key_setup(ses, enccrd->crd_key,
-		    enccrd->crd_klen);
+	if (crp->crp_cipher_key != NULL) {
+		padlock_cipher_key_setup(ses, crp->crp_cipher_key,
+		    csp->csp_cipher_klen);
 	}
 
 	cw = &ses->ses_cw;
 	cw->cw_filler0 = 0;
 	cw->cw_filler1 = 0;
 	cw->cw_filler2 = 0;
 	cw->cw_filler3 = 0;
-	if ((enccrd->crd_flags & CRD_F_ENCRYPT) != 0) {
+
+	if (crp->crp_flags & CRYPTO_F_IV_GENERATE) {
+		arc4rand(iv, AES_BLOCK_LEN, 0);
+		crypto_copyback(crp, crp->crp_iv_start, AES_BLOCK_LEN, iv);
+	} else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+		memcpy(iv, crp->crp_iv, AES_BLOCK_LEN);
+	else
+		crypto_copydata(crp, crp->crp_iv_start, AES_BLOCK_LEN, iv);
+
+	if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
 		cw->cw_direction = PADLOCK_DIRECTION_ENCRYPT;
 		key = ses->ses_ekey;
-		if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0)
-			bcopy(enccrd->crd_iv, ses->ses_iv, AES_BLOCK_LEN);
-
-		if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
-			crypto_copyback(crp->crp_flags, crp->crp_buf,
-			    enccrd->crd_inject, AES_BLOCK_LEN, ses->ses_iv);
-		}
 	} else {
 		cw->cw_direction = PADLOCK_DIRECTION_DECRYPT;
 		key = ses->ses_dkey;
-		if ((enccrd->crd_flags & CRD_F_IV_EXPLICIT) != 0)
-			bcopy(enccrd->crd_iv, ses->ses_iv, AES_BLOCK_LEN);
-		else {
-			crypto_copydata(crp->crp_flags, crp->crp_buf,
-			    enccrd->crd_inject, AES_BLOCK_LEN, ses->ses_iv);
-		}
 	}
 
 	if (allocated) {
-		crypto_copydata(crp->crp_flags, crp->crp_buf, enccrd->crd_skip,
-		    enccrd->crd_len, abuf);
+		crypto_copydata(crp, crp->crp_payload_start,
+		    crp->crp_payload_length, abuf);
 	}
 
 	td = curthread;
 	fpu_kern_enter(td, ses->ses_fpu_ctx, FPU_KERN_NORMAL | FPU_KERN_KTHR);
-	padlock_cbc(abuf, abuf, enccrd->crd_len / AES_BLOCK_LEN, key, cw,
-	    ses->ses_iv);
+	padlock_cbc(abuf, abuf, crp->crp_payload_length / AES_BLOCK_LEN, key,
+	    cw, iv);
 	fpu_kern_leave(td, ses->ses_fpu_ctx);
 
 	if (allocated) {
-		crypto_copyback(crp->crp_flags, crp->crp_buf, enccrd->crd_skip,
-		    enccrd->crd_len, abuf);
-	}
+		crypto_copyback(crp, crp->crp_payload_start,
+		    crp->crp_payload_length, abuf);
 
-	/* copy out last block for use as next session IV */
-	if ((enccrd->crd_flags & CRD_F_ENCRYPT) != 0) {
-		crypto_copydata(crp->crp_flags, crp->crp_buf,
-		    enccrd->crd_skip + enccrd->crd_len - AES_BLOCK_LEN,
-		    AES_BLOCK_LEN, ses->ses_iv);
-	}
-
-	if (allocated) {
-		bzero(buf, enccrd->crd_len + 16);
+		explicit_bzero(buf, crp->crp_payload_length + 16);
 		free(buf, M_PADLOCK);
 	}
 	return (0);
 }
Index: head/sys/crypto/via/padlock_hash.c
===================================================================
--- head/sys/crypto/via/padlock_hash.c	(revision 359373)
+++ head/sys/crypto/via/padlock_hash.c	(revision 359374)
@@ -1,405 +1,423 @@
 /*-
  * Copyright (c) 2006 Pawel Jakub Dawidek <pjd@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/module.h>
 #include <sys/malloc.h>
 #include <sys/libkern.h>
 #include <sys/endian.h>
 #include <sys/pcpu.h>
 #if defined(__amd64__) || defined(__i386__)
 #include <machine/cpufunc.h>
 #include <machine/cputypes.h>
 #include <machine/md_var.h>
 #include <machine/specialreg.h>
 #endif
 #include <machine/pcb.h>
 
 #include <opencrypto/cryptodev.h>
-#include <opencrypto/cryptosoft.h> /* for hmac_ipad_buffer and hmac_opad_buffer */
 #include <opencrypto/xform.h>
 
 #include <crypto/via/padlock.h>
 
 /*
  * Implementation notes.
  *
  * Some VIA CPUs provides SHA1 and SHA256 acceleration.
  * We implement all HMAC algorithms provided by crypto(9) framework, but we do
  * the crypto work in software unless this is HMAC/SHA1 or HMAC/SHA256 and
  * our CPU can accelerate it.
  *
  * Additional CPU instructions, which preform SHA1 and SHA256 are one-shot
  * functions - we have only one chance to give the data, CPU itself will add
  * the padding and calculate hash automatically.
  * This means, it is not possible to implement common init(), update(), final()
  * methods.
  * The way I've choosen is to keep adding data to the buffer on update()
  * (reallocating the buffer if necessary) and call XSHA{1,256} instruction on
  * final().
  */
 
 struct padlock_sha_ctx {
 	uint8_t	*psc_buf;
 	int	 psc_offset;
 	int	 psc_size;
 };
 CTASSERT(sizeof(struct padlock_sha_ctx) <= sizeof(union authctx));
 
 static void padlock_sha_init(struct padlock_sha_ctx *ctx);
 static int padlock_sha_update(struct padlock_sha_ctx *ctx, const uint8_t *buf,
     uint16_t bufsize);
 static void padlock_sha1_final(uint8_t *hash, struct padlock_sha_ctx *ctx);
 static void padlock_sha256_final(uint8_t *hash, struct padlock_sha_ctx *ctx);
 
 static struct auth_hash padlock_hmac_sha1 = {
 	.type = CRYPTO_SHA1_HMAC,
 	.name = "HMAC-SHA1",
 	.keysize = SHA1_BLOCK_LEN,
 	.hashsize = SHA1_HASH_LEN,
 	.ctxsize = sizeof(struct padlock_sha_ctx),
 	.blocksize = SHA1_BLOCK_LEN,
         .Init = (void (*)(void *))padlock_sha_init,
 	.Update = (int (*)(void *, const uint8_t *, uint16_t))padlock_sha_update,
 	.Final = (void (*)(uint8_t *, void *))padlock_sha1_final,
 };
 
 static struct auth_hash padlock_hmac_sha256 = {
 	.type = CRYPTO_SHA2_256_HMAC,
 	.name = "HMAC-SHA2-256",
 	.keysize = SHA2_256_BLOCK_LEN,
 	.hashsize = SHA2_256_HASH_LEN,
 	.ctxsize = sizeof(struct padlock_sha_ctx),
 	.blocksize = SHA2_256_BLOCK_LEN,
         .Init = (void (*)(void *))padlock_sha_init,
 	.Update = (int (*)(void *, const uint8_t *, uint16_t))padlock_sha_update,
 	.Final = (void (*)(uint8_t *, void *))padlock_sha256_final,
 };
 
 MALLOC_DECLARE(M_PADLOCK);
 
 static __inline void
 padlock_output_block(uint32_t *src, uint32_t *dst, size_t count)
 {
 
 	while (count-- > 0)
 		*dst++ = bswap32(*src++);
 }
 
 static void
 padlock_do_sha1(const u_char *in, u_char *out, int count)
 {
 	u_char buf[128+16];	/* PadLock needs at least 128 bytes buffer. */
 	u_char *result = PADLOCK_ALIGN(buf);
 
 	((uint32_t *)result)[0] = 0x67452301;
 	((uint32_t *)result)[1] = 0xEFCDAB89;
 	((uint32_t *)result)[2] = 0x98BADCFE;
 	((uint32_t *)result)[3] = 0x10325476;
 	((uint32_t *)result)[4] = 0xC3D2E1F0;
 
 #ifdef __GNUCLIKE_ASM
 	__asm __volatile(
 		".byte  0xf3, 0x0f, 0xa6, 0xc8" /* rep xsha1 */
 			: "+S"(in), "+D"(result)
 			: "c"(count), "a"(0)
 		);
 #endif
 
 	padlock_output_block((uint32_t *)result, (uint32_t *)out,
 	    SHA1_HASH_LEN / sizeof(uint32_t));
 }
 
 static void
 padlock_do_sha256(const char *in, char *out, int count)
 {
 	char buf[128+16];	/* PadLock needs at least 128 bytes buffer. */
 	char *result = PADLOCK_ALIGN(buf);
 
 	((uint32_t *)result)[0] = 0x6A09E667;
 	((uint32_t *)result)[1] = 0xBB67AE85;
 	((uint32_t *)result)[2] = 0x3C6EF372;
 	((uint32_t *)result)[3] = 0xA54FF53A;
 	((uint32_t *)result)[4] = 0x510E527F;
 	((uint32_t *)result)[5] = 0x9B05688C;
 	((uint32_t *)result)[6] = 0x1F83D9AB;
 	((uint32_t *)result)[7] = 0x5BE0CD19;
 
 #ifdef __GNUCLIKE_ASM
 	__asm __volatile(
 		".byte  0xf3, 0x0f, 0xa6, 0xd0" /* rep xsha256 */
 			: "+S"(in), "+D"(result)
 			: "c"(count), "a"(0)
 		);
 #endif
 
 	padlock_output_block((uint32_t *)result, (uint32_t *)out,
 	    SHA2_256_HASH_LEN / sizeof(uint32_t));
 }
 
 static void
 padlock_sha_init(struct padlock_sha_ctx *ctx)
 {
 
 	ctx->psc_buf = NULL;
 	ctx->psc_offset = 0;
 	ctx->psc_size = 0;
 }
 
 static int
 padlock_sha_update(struct padlock_sha_ctx *ctx, const uint8_t *buf, uint16_t bufsize)
 {
 
 	if (ctx->psc_size - ctx->psc_offset < bufsize) {
 		ctx->psc_size = MAX(ctx->psc_size * 2, ctx->psc_size + bufsize);
 		ctx->psc_buf = realloc(ctx->psc_buf, ctx->psc_size, M_PADLOCK,
 		    M_NOWAIT);
 		if(ctx->psc_buf == NULL)
 			return (ENOMEM);
 	}
 	bcopy(buf, ctx->psc_buf + ctx->psc_offset, bufsize);
 	ctx->psc_offset += bufsize;
 	return (0);
 }
 
 static void
 padlock_sha_free(struct padlock_sha_ctx *ctx)
 {
 
 	if (ctx->psc_buf != NULL) {
 		//bzero(ctx->psc_buf, ctx->psc_size);
 		free(ctx->psc_buf, M_PADLOCK);
 		ctx->psc_buf = NULL;
 		ctx->psc_offset = 0;
 		ctx->psc_size = 0;
 	}
 }
 
 static void
 padlock_sha1_final(uint8_t *hash, struct padlock_sha_ctx *ctx)
 {
 
 	padlock_do_sha1(ctx->psc_buf, hash, ctx->psc_offset);
 	padlock_sha_free(ctx);
 }
 
 static void
 padlock_sha256_final(uint8_t *hash, struct padlock_sha_ctx *ctx)
 {
 
 	padlock_do_sha256(ctx->psc_buf, hash, ctx->psc_offset);
 	padlock_sha_free(ctx);
 }
 
 static void
 padlock_copy_ctx(struct auth_hash *axf, void *sctx, void *dctx)
 {
 
 	if ((via_feature_xcrypt & VIA_HAS_SHA) != 0 &&
 	    (axf->type == CRYPTO_SHA1_HMAC ||
 	     axf->type == CRYPTO_SHA2_256_HMAC)) {
 		struct padlock_sha_ctx *spctx = sctx, *dpctx = dctx;
 
 		dpctx->psc_offset = spctx->psc_offset;
 		dpctx->psc_size = spctx->psc_size;
 		dpctx->psc_buf = malloc(dpctx->psc_size, M_PADLOCK, M_WAITOK);
 		bcopy(spctx->psc_buf, dpctx->psc_buf, dpctx->psc_size);
 	} else {
 		bcopy(sctx, dctx, axf->ctxsize);
 	}
 }
 
 static void
 padlock_free_ctx(struct auth_hash *axf, void *ctx)
 {
 
 	if ((via_feature_xcrypt & VIA_HAS_SHA) != 0 &&
 	    (axf->type == CRYPTO_SHA1_HMAC ||
 	     axf->type == CRYPTO_SHA2_256_HMAC)) {
 		padlock_sha_free(ctx);
 	}
 }
 
 static void
-padlock_hash_key_setup(struct padlock_session *ses, caddr_t key, int klen)
+padlock_hash_key_setup(struct padlock_session *ses, const uint8_t *key,
+    int klen)
 {
 	struct auth_hash *axf;
-	int i;
 
-	klen /= 8;
 	axf = ses->ses_axf;
 
 	/*
 	 * Try to free contexts before using them, because
 	 * padlock_hash_key_setup() can be called twice - once from
 	 * padlock_newsession() and again from padlock_process().
 	 */
 	padlock_free_ctx(axf, ses->ses_ictx);
 	padlock_free_ctx(axf, ses->ses_octx);
 
-	for (i = 0; i < klen; i++)
-		key[i] ^= HMAC_IPAD_VAL;
-
-	axf->Init(ses->ses_ictx);
-	axf->Update(ses->ses_ictx, key, klen);
-	axf->Update(ses->ses_ictx, hmac_ipad_buffer, axf->blocksize - klen);
-
-	for (i = 0; i < klen; i++)
-		key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
-
-	axf->Init(ses->ses_octx);
-	axf->Update(ses->ses_octx, key, klen);
-	axf->Update(ses->ses_octx, hmac_opad_buffer, axf->blocksize - klen);
-
-	for (i = 0; i < klen; i++)
-		key[i] ^= HMAC_OPAD_VAL;
+	hmac_init_ipad(axf, key, klen, ses->ses_ictx);
+	hmac_init_opad(axf, key, klen, ses->ses_octx);
 }
 
 /*
  * Compute keyed-hash authenticator.
  */
 static int
-padlock_authcompute(struct padlock_session *ses, struct cryptodesc *crd,
-    caddr_t buf, int flags)
+padlock_authcompute(struct padlock_session *ses, struct cryptop *crp)
 {
-	u_char hash[HASH_MAX_LEN];
+	u_char hash[HASH_MAX_LEN], hash2[HASH_MAX_LEN];
 	struct auth_hash *axf;
 	union authctx ctx;
 	int error;
 
 	axf = ses->ses_axf;
 
 	padlock_copy_ctx(axf, ses->ses_ictx, &ctx);
-	error = crypto_apply(flags, buf, crd->crd_skip, crd->crd_len,
+	error = crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length,
 	    (int (*)(void *, void *, unsigned int))axf->Update, (caddr_t)&ctx);
 	if (error != 0) {
 		padlock_free_ctx(axf, &ctx);
 		return (error);
 	}
+	error = crypto_apply(crp, crp->crp_payload_start,
+	    crp->crp_payload_length,
+	    (int (*)(void *, void *, unsigned int))axf->Update, (caddr_t)&ctx);
+	if (error != 0) {
+		padlock_free_ctx(axf, &ctx);
+		return (error);
+	}
 	axf->Final(hash, &ctx);
 
 	padlock_copy_ctx(axf, ses->ses_octx, &ctx);
 	axf->Update(&ctx, hash, axf->hashsize);
 	axf->Final(hash, &ctx);
 
-	/* Inject the authentication data */
-	crypto_copyback(flags, buf, crd->crd_inject,
-	    ses->ses_mlen == 0 ? axf->hashsize : ses->ses_mlen, hash);
+	if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
+		crypto_copydata(crp, crp->crp_digest_start, ses->ses_mlen,
+		    hash2);
+		if (timingsafe_bcmp(hash, hash2, ses->ses_mlen) != 0)
+			return (EBADMSG);
+	} else
+		crypto_copyback(crp, crp->crp_digest_start, ses->ses_mlen,
+		    hash);
 	return (0);
 }
 
-int
-padlock_hash_setup(struct padlock_session *ses, struct cryptoini *macini)
+/* Find software structure which describes HMAC algorithm. */
+static struct auth_hash *
+padlock_hash_lookup(int alg)
 {
+	struct auth_hash *axf;
 
-	ses->ses_mlen = macini->cri_mlen;
-
-	/* Find software structure which describes HMAC algorithm. */
-	switch (macini->cri_alg) {
+	switch (alg) {
 	case CRYPTO_NULL_HMAC:
-		ses->ses_axf = &auth_hash_null;
+		axf = &auth_hash_null;
 		break;
 	case CRYPTO_MD5_HMAC:
-		ses->ses_axf = &auth_hash_hmac_md5;
+		axf = &auth_hash_hmac_md5;
 		break;
 	case CRYPTO_SHA1_HMAC:
 		if ((via_feature_xcrypt & VIA_HAS_SHA) != 0)
-			ses->ses_axf = &padlock_hmac_sha1;
+			axf = &padlock_hmac_sha1;
 		else
-			ses->ses_axf = &auth_hash_hmac_sha1;
+			axf = &auth_hash_hmac_sha1;
 		break;
 	case CRYPTO_RIPEMD160_HMAC:
-		ses->ses_axf = &auth_hash_hmac_ripemd_160;
+		axf = &auth_hash_hmac_ripemd_160;
 		break;
 	case CRYPTO_SHA2_256_HMAC:
 		if ((via_feature_xcrypt & VIA_HAS_SHA) != 0)
-			ses->ses_axf = &padlock_hmac_sha256;
+			axf = &padlock_hmac_sha256;
 		else
-			ses->ses_axf = &auth_hash_hmac_sha2_256;
+			axf = &auth_hash_hmac_sha2_256;
 		break;
 	case CRYPTO_SHA2_384_HMAC:
-		ses->ses_axf = &auth_hash_hmac_sha2_384;
+		axf = &auth_hash_hmac_sha2_384;
 		break;
 	case CRYPTO_SHA2_512_HMAC:
-		ses->ses_axf = &auth_hash_hmac_sha2_512;
+		axf = &auth_hash_hmac_sha2_512;
 		break;
+	default:
+		axf = NULL;
+		break;
 	}
+	return (axf);
+}
 
+bool
+padlock_hash_check(const struct crypto_session_params *csp)
+{
+
+	return (padlock_hash_lookup(csp->csp_auth_alg) != NULL);
+}
+
+int
+padlock_hash_setup(struct padlock_session *ses,
+    const struct crypto_session_params *csp)
+{
+
+	ses->ses_axf = padlock_hash_lookup(csp->csp_auth_alg);
+	if (csp->csp_auth_mlen == 0)
+		ses->ses_mlen = ses->ses_axf->hashsize;
+	else
+		ses->ses_mlen = csp->csp_auth_mlen;
+
 	/* Allocate memory for HMAC inner and outer contexts. */
 	ses->ses_ictx = malloc(ses->ses_axf->ctxsize, M_PADLOCK,
 	    M_ZERO | M_NOWAIT);
 	ses->ses_octx = malloc(ses->ses_axf->ctxsize, M_PADLOCK,
 	    M_ZERO | M_NOWAIT);
 	if (ses->ses_ictx == NULL || ses->ses_octx == NULL)
 		return (ENOMEM);
 
 	/* Setup key if given. */
-	if (macini->cri_key != NULL) {
-		padlock_hash_key_setup(ses, macini->cri_key,
-		    macini->cri_klen);
+	if (csp->csp_auth_key != NULL) {
+		padlock_hash_key_setup(ses, csp->csp_auth_key,
+		    csp->csp_auth_klen);
 	}
 	return (0);
 }
 
 int
-padlock_hash_process(struct padlock_session *ses, struct cryptodesc *maccrd,
-    struct cryptop *crp)
+padlock_hash_process(struct padlock_session *ses, struct cryptop *crp,
+    const struct crypto_session_params *csp)
 {
 	struct thread *td;
 	int error;
 
 	td = curthread;
 	fpu_kern_enter(td, ses->ses_fpu_ctx, FPU_KERN_NORMAL | FPU_KERN_KTHR);
-	if ((maccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0)
-		padlock_hash_key_setup(ses, maccrd->crd_key, maccrd->crd_klen);
+	if (crp->crp_auth_key != NULL)
+		padlock_hash_key_setup(ses, crp->crp_auth_key,
+		    csp->csp_auth_klen);
 
-	error = padlock_authcompute(ses, maccrd, crp->crp_buf, crp->crp_flags);
+	error = padlock_authcompute(ses, crp);
 	fpu_kern_leave(td, ses->ses_fpu_ctx);
 	return (error);
 }
 
 void
 padlock_hash_free(struct padlock_session *ses)
 {
 
 	if (ses->ses_ictx != NULL) {
 		padlock_free_ctx(ses->ses_axf, ses->ses_ictx);
 		bzero(ses->ses_ictx, ses->ses_axf->ctxsize);
 		free(ses->ses_ictx, M_PADLOCK);
 		ses->ses_ictx = NULL;
 	}
 	if (ses->ses_octx != NULL) {
 		padlock_free_ctx(ses->ses_axf, ses->ses_octx);
 		bzero(ses->ses_octx, ses->ses_axf->ctxsize);
 		free(ses->ses_octx, M_PADLOCK);
 		ses->ses_octx = NULL;
 	}
 }
Index: head/sys/dev/cesa/cesa.c
===================================================================
--- head/sys/dev/cesa/cesa.c	(revision 359373)
+++ head/sys/dev/cesa/cesa.c	(revision 359374)
@@ -1,1829 +1,1842 @@
 /*-
  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
  *
  * Copyright (C) 2009-2011 Semihalf.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 /*
  * CESA SRAM Memory Map:
  *
  * +------------------------+ <= sc->sc_sram_base_va + CESA_SRAM_SIZE
  * |                        |
  * |          DATA          |
  * |                        |
  * +------------------------+ <= sc->sc_sram_base_va + CESA_DATA(0)
  * |  struct cesa_sa_data   |
  * +------------------------+
  * |  struct cesa_sa_hdesc  |
  * +------------------------+ <= sc->sc_sram_base_va
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/bus.h>
 #include <sys/endian.h>
 #include <sys/kernel.h>
 #include <sys/lock.h>
 #include <sys/mbuf.h>
 #include <sys/module.h>
 #include <sys/mutex.h>
 #include <sys/rman.h>
 
 #include <machine/bus.h>
 #include <machine/intr.h>
 #include <machine/resource.h>
 #include <machine/fdt.h>
 
 #include <dev/fdt/simplebus.h>
 #include <dev/fdt/fdt_common.h>
 #include <dev/ofw/ofw_bus.h>
 #include <dev/ofw/ofw_bus_subr.h>
 
 #include <sys/md5.h>
 #include <crypto/sha1.h>
 #include <crypto/sha2/sha256.h>
 #include <crypto/rijndael/rijndael.h>
 #include <opencrypto/cryptodev.h>
+#include <opencrypto/xform.h>
 #include "cryptodev_if.h"
 
 #include <arm/mv/mvreg.h>
 #include <arm/mv/mvvar.h>
 #include "cesa.h"
 
 static int	cesa_probe(device_t);
 static int	cesa_attach(device_t);
 static int	cesa_attach_late(device_t);
 static int	cesa_detach(device_t);
 static void	cesa_intr(void *);
-static int	cesa_newsession(device_t, crypto_session_t, struct cryptoini *);
+static int	cesa_probesession(device_t,
+    const struct crypto_session_params *);
+static int	cesa_newsession(device_t, crypto_session_t,
+    const struct crypto_session_params *);
 static int	cesa_process(device_t, struct cryptop *, int);
 
 static struct resource_spec cesa_res_spec[] = {
 	{ SYS_RES_MEMORY, 0, RF_ACTIVE },
 	{ SYS_RES_MEMORY, 1, RF_ACTIVE },
 	{ SYS_RES_IRQ, 0, RF_ACTIVE | RF_SHAREABLE },
 	{ -1, 0 }
 };
 
 static device_method_t cesa_methods[] = {
 	/* Device interface */
 	DEVMETHOD(device_probe,		cesa_probe),
 	DEVMETHOD(device_attach,	cesa_attach),
 	DEVMETHOD(device_detach,	cesa_detach),
 
 	/* Crypto device methods */
+	DEVMETHOD(cryptodev_probesession, cesa_probesession),
 	DEVMETHOD(cryptodev_newsession,	cesa_newsession),
 	DEVMETHOD(cryptodev_process,	cesa_process),
 
 	DEVMETHOD_END
 };
 
 static driver_t cesa_driver = {
 	"cesa",
 	cesa_methods,
 	sizeof (struct cesa_softc)
 };
 static devclass_t cesa_devclass;
 
 DRIVER_MODULE(cesa, simplebus, cesa_driver, cesa_devclass, 0, 0);
 MODULE_DEPEND(cesa, crypto, 1, 1, 1);
 
 static void
 cesa_dump_cshd(struct cesa_softc *sc, struct cesa_sa_hdesc *cshd)
 {
 #ifdef DEBUG
 	device_t dev;
 
 	dev = sc->sc_dev;
 	device_printf(dev, "CESA SA Hardware Descriptor:\n");
 	device_printf(dev, "\t\tconfig: 0x%08X\n", cshd->cshd_config);
 	device_printf(dev, "\t\te_src:  0x%08X\n", cshd->cshd_enc_src);
 	device_printf(dev, "\t\te_dst:  0x%08X\n", cshd->cshd_enc_dst);
 	device_printf(dev, "\t\te_dlen: 0x%08X\n", cshd->cshd_enc_dlen);
 	device_printf(dev, "\t\te_key:  0x%08X\n", cshd->cshd_enc_key);
 	device_printf(dev, "\t\te_iv_1: 0x%08X\n", cshd->cshd_enc_iv);
 	device_printf(dev, "\t\te_iv_2: 0x%08X\n", cshd->cshd_enc_iv_buf);
 	device_printf(dev, "\t\tm_src:  0x%08X\n", cshd->cshd_mac_src);
 	device_printf(dev, "\t\tm_dst:  0x%08X\n", cshd->cshd_mac_dst);
 	device_printf(dev, "\t\tm_dlen: 0x%08X\n", cshd->cshd_mac_dlen);
 	device_printf(dev, "\t\tm_tlen: 0x%08X\n", cshd->cshd_mac_total_dlen);
 	device_printf(dev, "\t\tm_iv_i: 0x%08X\n", cshd->cshd_mac_iv_in);
 	device_printf(dev, "\t\tm_iv_o: 0x%08X\n", cshd->cshd_mac_iv_out);
 #endif
 }
 
 static void
 cesa_alloc_dma_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 {
 	struct cesa_dma_mem *cdm;
 
 	if (error)
 		return;
 
 	KASSERT(nseg == 1, ("Got wrong number of DMA segments, should be 1."));
 	cdm = arg;
 	cdm->cdm_paddr = segs->ds_addr;
 }
 
 static int
 cesa_alloc_dma_mem(struct cesa_softc *sc, struct cesa_dma_mem *cdm,
     bus_size_t size)
 {
 	int error;
 
 	KASSERT(cdm->cdm_vaddr == NULL,
 	    ("%s(): DMA memory descriptor in use.", __func__));
 
 	error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev),	/* parent */
 	    PAGE_SIZE, 0,			/* alignment, boundary */
 	    BUS_SPACE_MAXADDR_32BIT,		/* lowaddr */
 	    BUS_SPACE_MAXADDR,			/* highaddr */
 	    NULL, NULL,				/* filtfunc, filtfuncarg */
 	    size, 1,				/* maxsize, nsegments */
 	    size, 0,				/* maxsegsz, flags */
 	    NULL, NULL,				/* lockfunc, lockfuncarg */
 	    &cdm->cdm_tag);			/* dmat */
 	if (error) {
 		device_printf(sc->sc_dev, "failed to allocate busdma tag, error"
 		    " %i!\n", error);
 
 		goto err1;
 	}
 
 	error = bus_dmamem_alloc(cdm->cdm_tag, &cdm->cdm_vaddr,
 	    BUS_DMA_NOWAIT | BUS_DMA_ZERO, &cdm->cdm_map);
 	if (error) {
 		device_printf(sc->sc_dev, "failed to allocate DMA safe"
 		    " memory, error %i!\n", error);
 
 		goto err2;
 	}
 
 	error = bus_dmamap_load(cdm->cdm_tag, cdm->cdm_map, cdm->cdm_vaddr,
 	    size, cesa_alloc_dma_mem_cb, cdm, BUS_DMA_NOWAIT);
 	if (error) {
 		device_printf(sc->sc_dev, "cannot get address of the DMA"
 		    " memory, error %i\n", error);
 
 		goto err3;
 	}
 
 	return (0);
 err3:
 	bus_dmamem_free(cdm->cdm_tag, cdm->cdm_vaddr, cdm->cdm_map);
 err2:
 	bus_dma_tag_destroy(cdm->cdm_tag);
 err1:
 	cdm->cdm_vaddr = NULL;
 	return (error);
 }
 
 static void
 cesa_free_dma_mem(struct cesa_dma_mem *cdm)
 {
 
 	bus_dmamap_unload(cdm->cdm_tag, cdm->cdm_map);
 	bus_dmamem_free(cdm->cdm_tag, cdm->cdm_vaddr, cdm->cdm_map);
 	bus_dma_tag_destroy(cdm->cdm_tag);
 	cdm->cdm_vaddr = NULL;
 }
 
 static void
 cesa_sync_dma_mem(struct cesa_dma_mem *cdm, bus_dmasync_op_t op)
 {
 
 	/* Sync only if dma memory is valid */
         if (cdm->cdm_vaddr != NULL)
 		bus_dmamap_sync(cdm->cdm_tag, cdm->cdm_map, op);
 }
 
 static void
 cesa_sync_desc(struct cesa_softc *sc, bus_dmasync_op_t op)
 {
 
 	cesa_sync_dma_mem(&sc->sc_tdesc_cdm, op);
 	cesa_sync_dma_mem(&sc->sc_sdesc_cdm, op);
 	cesa_sync_dma_mem(&sc->sc_requests_cdm, op);
 }
 
 static struct cesa_request *
 cesa_alloc_request(struct cesa_softc *sc)
 {
 	struct cesa_request *cr;
 
 	CESA_GENERIC_ALLOC_LOCKED(sc, cr, requests);
 	if (!cr)
 		return (NULL);
 
 	STAILQ_INIT(&cr->cr_tdesc);
 	STAILQ_INIT(&cr->cr_sdesc);
 
 	return (cr);
 }
 
 static void
 cesa_free_request(struct cesa_softc *sc, struct cesa_request *cr)
 {
 
 	/* Free TDMA descriptors assigned to this request */
 	CESA_LOCK(sc, tdesc);
 	STAILQ_CONCAT(&sc->sc_free_tdesc, &cr->cr_tdesc);
 	CESA_UNLOCK(sc, tdesc);
 
 	/* Free SA descriptors assigned to this request */
 	CESA_LOCK(sc, sdesc);
 	STAILQ_CONCAT(&sc->sc_free_sdesc, &cr->cr_sdesc);
 	CESA_UNLOCK(sc, sdesc);
 
 	/* Unload DMA memory associated with request */
 	if (cr->cr_dmap_loaded) {
 		bus_dmamap_unload(sc->sc_data_dtag, cr->cr_dmap);
 		cr->cr_dmap_loaded = 0;
 	}
 
 	CESA_GENERIC_FREE_LOCKED(sc, cr, requests);
 }
 
 static void
 cesa_enqueue_request(struct cesa_softc *sc, struct cesa_request *cr)
 {
 
 	CESA_LOCK(sc, requests);
 	STAILQ_INSERT_TAIL(&sc->sc_ready_requests, cr, cr_stq);
 	CESA_UNLOCK(sc, requests);
 }
 
 static struct cesa_tdma_desc *
 cesa_alloc_tdesc(struct cesa_softc *sc)
 {
 	struct cesa_tdma_desc *ctd;
 
 	CESA_GENERIC_ALLOC_LOCKED(sc, ctd, tdesc);
 
 	if (!ctd)
 		device_printf(sc->sc_dev, "TDMA descriptors pool exhaused. "
 		    "Consider increasing CESA_TDMA_DESCRIPTORS.\n");
 
 	return (ctd);
 }
 
 static struct cesa_sa_desc *
 cesa_alloc_sdesc(struct cesa_softc *sc, struct cesa_request *cr)
 {
 	struct cesa_sa_desc *csd;
 
 	CESA_GENERIC_ALLOC_LOCKED(sc, csd, sdesc);
 	if (!csd) {
 		device_printf(sc->sc_dev, "SA descriptors pool exhaused. "
 		    "Consider increasing CESA_SA_DESCRIPTORS.\n");
 		return (NULL);
 	}
 
 	STAILQ_INSERT_TAIL(&cr->cr_sdesc, csd, csd_stq);
 
 	/* Fill-in SA descriptor with default values */
 	csd->csd_cshd->cshd_enc_key = CESA_SA_DATA(csd_key);
 	csd->csd_cshd->cshd_enc_iv = CESA_SA_DATA(csd_iv);
 	csd->csd_cshd->cshd_enc_iv_buf = CESA_SA_DATA(csd_iv);
 	csd->csd_cshd->cshd_enc_src = 0;
 	csd->csd_cshd->cshd_enc_dst = 0;
 	csd->csd_cshd->cshd_enc_dlen = 0;
 	csd->csd_cshd->cshd_mac_dst = CESA_SA_DATA(csd_hash);
 	csd->csd_cshd->cshd_mac_iv_in = CESA_SA_DATA(csd_hiv_in);
 	csd->csd_cshd->cshd_mac_iv_out = CESA_SA_DATA(csd_hiv_out);
 	csd->csd_cshd->cshd_mac_src = 0;
 	csd->csd_cshd->cshd_mac_dlen = 0;
 
 	return (csd);
 }
 
 static struct cesa_tdma_desc *
 cesa_tdma_copy(struct cesa_softc *sc, bus_addr_t dst, bus_addr_t src,
     bus_size_t size)
 {
 	struct cesa_tdma_desc *ctd;
 
 	ctd = cesa_alloc_tdesc(sc);
 	if (!ctd)
 		return (NULL);
 
 	ctd->ctd_cthd->cthd_dst = dst;
 	ctd->ctd_cthd->cthd_src = src;
 	ctd->ctd_cthd->cthd_byte_count = size;
 
 	/* Handle special control packet */
 	if (size != 0)
 		ctd->ctd_cthd->cthd_flags = CESA_CTHD_OWNED;
 	else
 		ctd->ctd_cthd->cthd_flags = 0;
 
 	return (ctd);
 }
 
 static struct cesa_tdma_desc *
 cesa_tdma_copyin_sa_data(struct cesa_softc *sc, struct cesa_request *cr)
 {
 
 	return (cesa_tdma_copy(sc, sc->sc_sram_base_pa +
 	    sizeof(struct cesa_sa_hdesc), cr->cr_csd_paddr,
 	    sizeof(struct cesa_sa_data)));
 }
 
 static struct cesa_tdma_desc *
 cesa_tdma_copyout_sa_data(struct cesa_softc *sc, struct cesa_request *cr)
 {
 
 	return (cesa_tdma_copy(sc, cr->cr_csd_paddr, sc->sc_sram_base_pa +
 	    sizeof(struct cesa_sa_hdesc), sizeof(struct cesa_sa_data)));
 }
 
 static struct cesa_tdma_desc *
 cesa_tdma_copy_sdesc(struct cesa_softc *sc, struct cesa_sa_desc *csd)
 {
 
 	return (cesa_tdma_copy(sc, sc->sc_sram_base_pa, csd->csd_cshd_paddr,
 	    sizeof(struct cesa_sa_hdesc)));
 }
 
 static void
 cesa_append_tdesc(struct cesa_request *cr, struct cesa_tdma_desc *ctd)
 {
 	struct cesa_tdma_desc *ctd_prev;
 
 	if (!STAILQ_EMPTY(&cr->cr_tdesc)) {
 		ctd_prev = STAILQ_LAST(&cr->cr_tdesc, cesa_tdma_desc, ctd_stq);
 		ctd_prev->ctd_cthd->cthd_next = ctd->ctd_cthd_paddr;
 	}
 
 	ctd->ctd_cthd->cthd_next = 0;
 	STAILQ_INSERT_TAIL(&cr->cr_tdesc, ctd, ctd_stq);
 }
 
 static int
 cesa_append_packet(struct cesa_softc *sc, struct cesa_request *cr,
     struct cesa_packet *cp, struct cesa_sa_desc *csd)
 {
 	struct cesa_tdma_desc *ctd, *tmp;
 
 	/* Copy SA descriptor for this packet */
 	ctd = cesa_tdma_copy_sdesc(sc, csd);
 	if (!ctd)
 		return (ENOMEM);
 
 	cesa_append_tdesc(cr, ctd);
 
 	/* Copy data to be processed */
 	STAILQ_FOREACH_SAFE(ctd, &cp->cp_copyin, ctd_stq, tmp)
 		cesa_append_tdesc(cr, ctd);
 	STAILQ_INIT(&cp->cp_copyin);
 
 	/* Insert control descriptor */
 	ctd = cesa_tdma_copy(sc, 0, 0, 0);
 	if (!ctd)
 		return (ENOMEM);
 
 	cesa_append_tdesc(cr, ctd);
 
 	/* Copy back results */
 	STAILQ_FOREACH_SAFE(ctd, &cp->cp_copyout, ctd_stq, tmp)
 		cesa_append_tdesc(cr, ctd);
 	STAILQ_INIT(&cp->cp_copyout);
 
 	return (0);
 }
 
-static int
+static void
 cesa_set_mkey(struct cesa_session *cs, int alg, const uint8_t *mkey, int mklen)
 {
-	uint8_t ipad[CESA_MAX_HMAC_BLOCK_LEN];
-	uint8_t opad[CESA_MAX_HMAC_BLOCK_LEN];
-	SHA1_CTX sha1ctx;
-	SHA256_CTX sha256ctx;
-	MD5_CTX md5ctx;
+	union authctx auth_ctx;
 	uint32_t *hout;
 	uint32_t *hin;
 	int i;
 
-	memset(ipad, HMAC_IPAD_VAL, CESA_MAX_HMAC_BLOCK_LEN);
-	memset(opad, HMAC_OPAD_VAL, CESA_MAX_HMAC_BLOCK_LEN);
-	for (i = 0; i < mklen; i++) {
-		ipad[i] ^= mkey[i];
-		opad[i] ^= mkey[i];
-	}
-
 	hin = (uint32_t *)cs->cs_hiv_in;
 	hout = (uint32_t *)cs->cs_hiv_out;
 
 	switch (alg) {
 	case CRYPTO_MD5_HMAC:
-		MD5Init(&md5ctx);
-		MD5Update(&md5ctx, ipad, MD5_BLOCK_LEN);
-		memcpy(hin, md5ctx.state, sizeof(md5ctx.state));
-		MD5Init(&md5ctx);
-		MD5Update(&md5ctx, opad, MD5_BLOCK_LEN);
-		memcpy(hout, md5ctx.state, sizeof(md5ctx.state));
+		hmac_init_ipad(&auth_hash_hmac_md5, mkey, mklen, &auth_ctx);
+		memcpy(hin, auth_ctx.md5ctx.state,
+		    sizeof(auth_ctx.md5ctx.state));
+		hmac_init_opad(&auth_hash_hmac_md5, mkey, mklen, &auth_ctx);
+		memcpy(hout, auth_ctx.md5ctx.state,
+		    sizeof(auth_ctx.md5ctx.state));
 		break;
 	case CRYPTO_SHA1_HMAC:
-		SHA1Init(&sha1ctx);
-		SHA1Update(&sha1ctx, ipad, SHA1_BLOCK_LEN);
-		memcpy(hin, sha1ctx.h.b32, sizeof(sha1ctx.h.b32));
-		SHA1Init(&sha1ctx);
-		SHA1Update(&sha1ctx, opad, SHA1_BLOCK_LEN);
-		memcpy(hout, sha1ctx.h.b32, sizeof(sha1ctx.h.b32));
+		hmac_init_ipad(&auth_hash_hmac_sha1, mkey, mklen, &auth_ctx);
+		memcpy(hin, auth_ctx.sha1ctx.h.b32,
+		    sizeof(auth_ctx.sha1ctx.h.b32));
+		hmac_init_opad(&auth_hash_hmac_sha1, mkey, mklen, &auth_ctx);
+		memcpy(hout, auth_ctx.sha1ctx.h.b32,
+		    sizeof(auth_ctx.sha1ctx.h.b32));
 		break;
 	case CRYPTO_SHA2_256_HMAC:
-		SHA256_Init(&sha256ctx);
-		SHA256_Update(&sha256ctx, ipad, SHA2_256_BLOCK_LEN);
-		memcpy(hin, sha256ctx.state, sizeof(sha256ctx.state));
-		SHA256_Init(&sha256ctx);
-		SHA256_Update(&sha256ctx, opad, SHA2_256_BLOCK_LEN);
-		memcpy(hout, sha256ctx.state, sizeof(sha256ctx.state));
+		hmac_init_ipad(&auth_hash_hmac_sha2_256, mkey, mklen,
+		    &auth_ctx);
+		memcpy(hin, auth_ctx.sha256ctx.state,
+		    sizeof(auth_ctx.sha256ctx.state));
+		hmac_init_opad(&auth_hash_hmac_sha2_256, mkey, mklen,
+		    &auth_ctx);
+		memcpy(hout, auth_ctx.sha256ctx.state,
+		    sizeof(auth_ctx.sha256ctx.state));
 		break;
 	default:
-		return (EINVAL);
+		panic("shouldn't get here");
 	}
 
 	for (i = 0; i < CESA_MAX_HASH_LEN / sizeof(uint32_t); i++) {
 		hin[i] = htobe32(hin[i]);
 		hout[i] = htobe32(hout[i]);
 	}
-
-	return (0);
 }
 
 static int
-cesa_prep_aes_key(struct cesa_session *cs)
+cesa_prep_aes_key(struct cesa_session *cs,
+    const struct crypto_session_params *csp)
 {
 	uint32_t ek[4 * (RIJNDAEL_MAXNR + 1)];
 	uint32_t *dkey;
 	int i;
 
-	rijndaelKeySetupEnc(ek, cs->cs_key, cs->cs_klen * 8);
+	rijndaelKeySetupEnc(ek, cs->cs_key, csp->csp_cipher_klen * 8);
 
 	cs->cs_config &= ~CESA_CSH_AES_KLEN_MASK;
 	dkey = (uint32_t *)cs->cs_aes_dkey;
 
-	switch (cs->cs_klen) {
+	switch (csp->csp_cipher_klen) {
 	case 16:
 		cs->cs_config |= CESA_CSH_AES_KLEN_128;
 		for (i = 0; i < 4; i++)
 			*dkey++ = htobe32(ek[4 * 10 + i]);
 		break;
 	case 24:
 		cs->cs_config |= CESA_CSH_AES_KLEN_192;
 		for (i = 0; i < 4; i++)
 			*dkey++ = htobe32(ek[4 * 12 + i]);
 		for (i = 0; i < 2; i++)
 			*dkey++ = htobe32(ek[4 * 11 + 2 + i]);
 		break;
 	case 32:
 		cs->cs_config |= CESA_CSH_AES_KLEN_256;
 		for (i = 0; i < 4; i++)
 			*dkey++ = htobe32(ek[4 * 14 + i]);
 		for (i = 0; i < 4; i++)
 			*dkey++ = htobe32(ek[4 * 13 + i]);
 		break;
 	default:
 		return (EINVAL);
 	}
 
 	return (0);
 }
 
-static int
-cesa_is_hash(int alg)
-{
-
-	switch (alg) {
-	case CRYPTO_MD5:
-	case CRYPTO_MD5_HMAC:
-	case CRYPTO_SHA1:
-	case CRYPTO_SHA1_HMAC:
-	case CRYPTO_SHA2_256_HMAC:
-		return (1);
-	default:
-		return (0);
-	}
-}
-
 static void
 cesa_start_packet(struct cesa_packet *cp, unsigned int size)
 {
 
 	cp->cp_size = size;
 	cp->cp_offset = 0;
 	STAILQ_INIT(&cp->cp_copyin);
 	STAILQ_INIT(&cp->cp_copyout);
 }
 
 static int
 cesa_fill_packet(struct cesa_softc *sc, struct cesa_packet *cp,
     bus_dma_segment_t *seg)
 {
 	struct cesa_tdma_desc *ctd;
 	unsigned int bsize;
 
 	/* Calculate size of block copy */
 	bsize = MIN(seg->ds_len, cp->cp_size - cp->cp_offset);
 
 	if (bsize > 0) {
 		ctd = cesa_tdma_copy(sc, sc->sc_sram_base_pa +
 		    CESA_DATA(cp->cp_offset), seg->ds_addr, bsize);
 		if (!ctd)
 			return (-ENOMEM);
 
 		STAILQ_INSERT_TAIL(&cp->cp_copyin, ctd, ctd_stq);
 
 		ctd = cesa_tdma_copy(sc, seg->ds_addr, sc->sc_sram_base_pa +
 		    CESA_DATA(cp->cp_offset), bsize);
 		if (!ctd)
 			return (-ENOMEM);
 
 		STAILQ_INSERT_TAIL(&cp->cp_copyout, ctd, ctd_stq);
 
 		seg->ds_len -= bsize;
 		seg->ds_addr += bsize;
 		cp->cp_offset += bsize;
 	}
 
 	return (bsize);
 }
 
 static void
 cesa_create_chain_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 {
 	unsigned int mpsize, fragmented;
 	unsigned int mlen, mskip, tmlen;
 	struct cesa_chain_info *cci;
 	unsigned int elen, eskip;
 	unsigned int skip, len;
 	struct cesa_sa_desc *csd;
 	struct cesa_request *cr;
+	struct cryptop *crp;
 	struct cesa_softc *sc;
 	struct cesa_packet cp;
 	bus_dma_segment_t seg;
 	uint32_t config;
 	int size;
 
 	cci = arg;
 	sc = cci->cci_sc;
 	cr = cci->cci_cr;
+	crp = cr->cr_crp;
 
 	if (error) {
 		cci->cci_error = error;
 		return;
 	}
 
-	elen = cci->cci_enc ? cci->cci_enc->crd_len : 0;
-	eskip = cci->cci_enc ? cci->cci_enc->crd_skip : 0;
-	mlen = cci->cci_mac ? cci->cci_mac->crd_len : 0;
-	mskip = cci->cci_mac ? cci->cci_mac->crd_skip : 0;
-
-	if (elen && mlen &&
-	    ((eskip > mskip && ((eskip - mskip) & (cr->cr_cs->cs_ivlen - 1))) ||
-	    (mskip > eskip && ((mskip - eskip) & (cr->cr_cs->cs_mblen - 1))) ||
-	    (eskip > (mskip + mlen)) || (mskip > (eskip + elen)))) {
+	/*
+	 * Only do a combined op if the AAD is adjacent to the payload
+	 * and the AAD length is a multiple of the IV length.  The
+	 * checks against 'config' are to avoid recursing when the
+	 * logic below invokes separate operations.
+	 */
+	config = cci->cci_config;
+	if (((config & CESA_CSHD_OP_MASK) == CESA_CSHD_MAC_AND_ENC ||
+	    (config & CESA_CSHD_OP_MASK) == CESA_CSHD_ENC_AND_MAC) &&
+	    crp->crp_aad_length != 0 &&
+	    (crp->crp_aad_length & (cr->cr_cs->cs_ivlen - 1)) != 0) {
 		/*
 		 * Data alignment in the request does not meet CESA requiremnts
 		 * for combined encryption/decryption and hashing. We have to
 		 * split the request to separate operations and process them
 		 * one by one.
 		 */
-		config = cci->cci_config;
 		if ((config & CESA_CSHD_OP_MASK) == CESA_CSHD_MAC_AND_ENC) {
 			config &= ~CESA_CSHD_OP_MASK;
 
 			cci->cci_config = config | CESA_CSHD_MAC;
-			cci->cci_enc = NULL;
-			cci->cci_mac = cr->cr_mac;
-			cesa_create_chain_cb(cci, segs, nseg, cci->cci_error);
+			cesa_create_chain_cb(cci, segs, nseg, 0);
 
 			cci->cci_config = config | CESA_CSHD_ENC;
-			cci->cci_enc = cr->cr_enc;
-			cci->cci_mac = NULL;
-			cesa_create_chain_cb(cci, segs, nseg, cci->cci_error);
+			cesa_create_chain_cb(cci, segs, nseg, 0);
 		} else {
 			config &= ~CESA_CSHD_OP_MASK;
 
 			cci->cci_config = config | CESA_CSHD_ENC;
-			cci->cci_enc = cr->cr_enc;
-			cci->cci_mac = NULL;
-			cesa_create_chain_cb(cci, segs, nseg, cci->cci_error);
+			cesa_create_chain_cb(cci, segs, nseg, 0);
 
 			cci->cci_config = config | CESA_CSHD_MAC;
-			cci->cci_enc = NULL;
-			cci->cci_mac = cr->cr_mac;
-			cesa_create_chain_cb(cci, segs, nseg, cci->cci_error);
+			cesa_create_chain_cb(cci, segs, nseg, 0);
 		}
 
 		return;
 	}
 
+	mskip = mlen = eskip = elen = 0;
+
+	if (crp->crp_aad_length == 0) {
+		skip = crp->crp_payload_start;
+		len = crp->crp_payload_length;
+		switch (config & CESA_CSHD_OP_MASK) {
+		case CESA_CSHD_ENC:
+			eskip = skip;
+			elen = len;
+			break;
+		case CESA_CSHD_MAC:
+			mskip = skip;
+			mlen = len;
+			break;
+		default:
+			eskip = skip;
+			elen = len;
+			mskip = skip;
+			mlen = len;
+			break;
+		}
+	} else {
+		/*
+		 * For an encryption-only separate request, only
+		 * process the payload.  For combined requests and
+		 * hash-only requests, process the entire region.
+		 */
+		switch (config & CESA_CSHD_OP_MASK) {
+		case CESA_CSHD_ENC:
+			skip = crp->crp_payload_start;
+			len = crp->crp_payload_length;
+			eskip = skip;
+			elen = len;
+			break;
+		case CESA_CSHD_MAC:
+			skip = crp->crp_aad_start;
+			len = crp->crp_aad_length + crp->crp_payload_length;
+			mskip = skip;
+			mlen = len;
+			break;
+		default:
+			skip = crp->crp_aad_start;
+			len = crp->crp_aad_length + crp->crp_payload_length;
+			mskip = skip;
+			mlen = len;
+			eskip = crp->crp_payload_start;
+			elen = crp->crp_payload_length;
+			break;
+		}
+	}
+
 	tmlen = mlen;
 	fragmented = 0;
 	mpsize = CESA_MAX_PACKET_SIZE;
 	mpsize &= ~((cr->cr_cs->cs_ivlen - 1) | (cr->cr_cs->cs_mblen - 1));
 
-	if (elen && mlen) {
-		skip = MIN(eskip, mskip);
-		len = MAX(elen + eskip, mlen + mskip) - skip;
-	} else if (elen) {
-		skip = eskip;
-		len = elen;
-	} else {
-		skip = mskip;
-		len = mlen;
-	}
-
 	/* Start first packet in chain */
 	cesa_start_packet(&cp, MIN(mpsize, len));
 
 	while (nseg-- && len > 0) {
 		seg = *(segs++);
 
 		/*
 		 * Skip data in buffer on which neither ENC nor MAC operation
 		 * is requested.
 		 */
 		if (skip > 0) {
 			size = MIN(skip, seg.ds_len);
 			skip -= size;
 
 			seg.ds_addr += size;
 			seg.ds_len -= size;
 
 			if (eskip > 0)
 				eskip -= size;
 
 			if (mskip > 0)
 				mskip -= size;
 
 			if (seg.ds_len == 0)
 				continue;
 		}
 
 		while (1) {
 			/*
 			 * Fill in current packet with data. Break if there is
 			 * no more data in current DMA segment or an error
 			 * occurred.
 			 */
 			size = cesa_fill_packet(sc, &cp, &seg);
 			if (size <= 0) {
 				error = -size;
 				break;
 			}
 
 			len -= size;
 
 			/* If packet is full, append it to the chain */
 			if (cp.cp_size == cp.cp_offset) {
 				csd = cesa_alloc_sdesc(sc, cr);
 				if (!csd) {
 					error = ENOMEM;
 					break;
 				}
 
 				/* Create SA descriptor for this packet */
 				csd->csd_cshd->cshd_config = cci->cci_config;
 				csd->csd_cshd->cshd_mac_total_dlen = tmlen;
 
 				/*
 				 * Enable fragmentation if request will not fit
 				 * into one packet.
 				 */
 				if (len > 0) {
 					if (!fragmented) {
 						fragmented = 1;
 						csd->csd_cshd->cshd_config |=
 						    CESA_CSHD_FRAG_FIRST;
 					} else
 						csd->csd_cshd->cshd_config |=
 						    CESA_CSHD_FRAG_MIDDLE;
 				} else if (fragmented)
 					csd->csd_cshd->cshd_config |=
 					    CESA_CSHD_FRAG_LAST;
 
 				if (eskip < cp.cp_size && elen > 0) {
 					csd->csd_cshd->cshd_enc_src =
 					    CESA_DATA(eskip);
 					csd->csd_cshd->cshd_enc_dst =
 					    CESA_DATA(eskip);
 					csd->csd_cshd->cshd_enc_dlen =
 					    MIN(elen, cp.cp_size - eskip);
 				}
 
 				if (mskip < cp.cp_size && mlen > 0) {
 					csd->csd_cshd->cshd_mac_src =
 					    CESA_DATA(mskip);
 					csd->csd_cshd->cshd_mac_dlen =
 					    MIN(mlen, cp.cp_size - mskip);
 				}
 
 				elen -= csd->csd_cshd->cshd_enc_dlen;
 				eskip -= MIN(eskip, cp.cp_size);
 				mlen -= csd->csd_cshd->cshd_mac_dlen;
 				mskip -= MIN(mskip, cp.cp_size);
 
 				cesa_dump_cshd(sc, csd->csd_cshd);
 
 				/* Append packet to the request */
 				error = cesa_append_packet(sc, cr, &cp, csd);
 				if (error)
 					break;
 
 				/* Start a new packet, as current is full */
 				cesa_start_packet(&cp, MIN(mpsize, len));
 			}
 		}
 
 		if (error)
 			break;
 	}
 
 	if (error) {
 		/*
 		 * Move all allocated resources to the request. They will be
 		 * freed later.
 		 */
 		STAILQ_CONCAT(&cr->cr_tdesc, &cp.cp_copyin);
 		STAILQ_CONCAT(&cr->cr_tdesc, &cp.cp_copyout);
 		cci->cci_error = error;
 	}
 }
 
-static void
-cesa_create_chain_cb2(void *arg, bus_dma_segment_t *segs, int nseg,
-    bus_size_t size, int error)
-{
-
-	cesa_create_chain_cb(arg, segs, nseg, error);
-}
-
 static int
-cesa_create_chain(struct cesa_softc *sc, struct cesa_request *cr)
+cesa_create_chain(struct cesa_softc *sc,
+    const struct crypto_session_params *csp, struct cesa_request *cr)
 {
 	struct cesa_chain_info cci;
 	struct cesa_tdma_desc *ctd;
 	uint32_t config;
 	int error;
 
 	error = 0;
 	CESA_LOCK_ASSERT(sc, sessions);
 
 	/* Create request metadata */
-	if (cr->cr_enc) {
-		if (cr->cr_enc->crd_alg == CRYPTO_AES_CBC &&
-		    (cr->cr_enc->crd_flags & CRD_F_ENCRYPT) == 0)
+	if (csp->csp_cipher_klen != 0) {
+		if (csp->csp_cipher_alg == CRYPTO_AES_CBC &&
+		    !CRYPTO_OP_IS_ENCRYPT(cr->cr_crp->crp_op))
 			memcpy(cr->cr_csd->csd_key, cr->cr_cs->cs_aes_dkey,
-			    cr->cr_cs->cs_klen);
+			    csp->csp_cipher_klen);
 		else
 			memcpy(cr->cr_csd->csd_key, cr->cr_cs->cs_key,
-			    cr->cr_cs->cs_klen);
+			    csp->csp_cipher_klen);
 	}
 
-	if (cr->cr_mac) {
+	if (csp->csp_auth_klen != 0) {
 		memcpy(cr->cr_csd->csd_hiv_in, cr->cr_cs->cs_hiv_in,
 		    CESA_MAX_HASH_LEN);
 		memcpy(cr->cr_csd->csd_hiv_out, cr->cr_cs->cs_hiv_out,
 		    CESA_MAX_HASH_LEN);
 	}
 
 	ctd = cesa_tdma_copyin_sa_data(sc, cr);
 	if (!ctd)
 		return (ENOMEM);
 
 	cesa_append_tdesc(cr, ctd);
 
 	/* Prepare SA configuration */
 	config = cr->cr_cs->cs_config;
 
-	if (cr->cr_enc && (cr->cr_enc->crd_flags & CRD_F_ENCRYPT) == 0)
+	if (csp->csp_cipher_alg != 0 &&
+	    !CRYPTO_OP_IS_ENCRYPT(cr->cr_crp->crp_op))
 		config |= CESA_CSHD_DECRYPT;
-	if (cr->cr_enc && !cr->cr_mac)
+	switch (csp->csp_mode) {
+	case CSP_MODE_CIPHER:
 		config |= CESA_CSHD_ENC;
-	if (!cr->cr_enc && cr->cr_mac)
+		break;
+	case CSP_MODE_DIGEST:
 		config |= CESA_CSHD_MAC;
-	if (cr->cr_enc && cr->cr_mac)
+		break;
+	case CSP_MODE_ETA:
 		config |= (config & CESA_CSHD_DECRYPT) ? CESA_CSHD_MAC_AND_ENC :
 		    CESA_CSHD_ENC_AND_MAC;
+		break;
+	}
 
 	/* Create data packets */
 	cci.cci_sc = sc;
 	cci.cci_cr = cr;
-	cci.cci_enc = cr->cr_enc;
-	cci.cci_mac = cr->cr_mac;
 	cci.cci_config = config;
 	cci.cci_error = 0;
 
-	if (cr->cr_crp->crp_flags & CRYPTO_F_IOV)
-		error = bus_dmamap_load_uio(sc->sc_data_dtag,
-		    cr->cr_dmap, (struct uio *)cr->cr_crp->crp_buf,
-		    cesa_create_chain_cb2, &cci, BUS_DMA_NOWAIT);
-	else if (cr->cr_crp->crp_flags & CRYPTO_F_IMBUF)
-		error = bus_dmamap_load_mbuf(sc->sc_data_dtag,
-		    cr->cr_dmap, (struct mbuf *)cr->cr_crp->crp_buf,
-		    cesa_create_chain_cb2, &cci, BUS_DMA_NOWAIT);
-	else
-		error = bus_dmamap_load(sc->sc_data_dtag,
-		    cr->cr_dmap, cr->cr_crp->crp_buf,
-		    cr->cr_crp->crp_ilen, cesa_create_chain_cb, &cci,
-		    BUS_DMA_NOWAIT);
+	error = bus_dmamap_load_crp(sc->sc_data_dtag, cr->cr_dmap, cr->cr_crp,
+	    cesa_create_chain_cb, &cci, BUS_DMA_NOWAIT);
 
 	if (!error)
 		cr->cr_dmap_loaded = 1;
 
 	if (cci.cci_error)
 		error = cci.cci_error;
 
 	if (error)
 		return (error);
 
 	/* Read back request metadata */
 	ctd = cesa_tdma_copyout_sa_data(sc, cr);
 	if (!ctd)
 		return (ENOMEM);
 
 	cesa_append_tdesc(cr, ctd);
 
 	return (0);
 }
 
 static void
 cesa_execute(struct cesa_softc *sc)
 {
 	struct cesa_tdma_desc *prev_ctd, *ctd;
 	struct cesa_request *prev_cr, *cr;
 
 	CESA_LOCK(sc, requests);
 
 	/*
 	 * If ready list is empty, there is nothing to execute. If queued list
 	 * is not empty, the hardware is busy and we cannot start another
 	 * execution.
 	 */
 	if (STAILQ_EMPTY(&sc->sc_ready_requests) ||
 	    !STAILQ_EMPTY(&sc->sc_queued_requests)) {
 		CESA_UNLOCK(sc, requests);
 		return;
 	}
 
 	/* Move all ready requests to queued list */
 	STAILQ_CONCAT(&sc->sc_queued_requests, &sc->sc_ready_requests);
 	STAILQ_INIT(&sc->sc_ready_requests);
 
 	/* Create one execution chain from all requests on the list */
 	if (STAILQ_FIRST(&sc->sc_queued_requests) !=
 	    STAILQ_LAST(&sc->sc_queued_requests, cesa_request, cr_stq)) {
 		prev_cr = NULL;
 		cesa_sync_dma_mem(&sc->sc_tdesc_cdm, BUS_DMASYNC_POSTREAD |
 		    BUS_DMASYNC_POSTWRITE);
 
 		STAILQ_FOREACH(cr, &sc->sc_queued_requests, cr_stq) {
 			if (prev_cr) {
 				ctd = STAILQ_FIRST(&cr->cr_tdesc);
 				prev_ctd = STAILQ_LAST(&prev_cr->cr_tdesc,
 				    cesa_tdma_desc, ctd_stq);
 
 				prev_ctd->ctd_cthd->cthd_next =
 				    ctd->ctd_cthd_paddr;
 			}
 
 			prev_cr = cr;
 		}
 
 		cesa_sync_dma_mem(&sc->sc_tdesc_cdm, BUS_DMASYNC_PREREAD |
 		    BUS_DMASYNC_PREWRITE);
 	}
 
 	/* Start chain execution in hardware */
 	cr = STAILQ_FIRST(&sc->sc_queued_requests);
 	ctd = STAILQ_FIRST(&cr->cr_tdesc);
 
 	CESA_TDMA_WRITE(sc, CESA_TDMA_ND, ctd->ctd_cthd_paddr);
 
 	if (sc->sc_soc_id == MV_DEV_88F6828 ||
 	    sc->sc_soc_id == MV_DEV_88F6820 ||
 	    sc->sc_soc_id == MV_DEV_88F6810)
 		CESA_REG_WRITE(sc, CESA_SA_CMD, CESA_SA_CMD_ACTVATE | CESA_SA_CMD_SHA2);
 	else
 		CESA_REG_WRITE(sc, CESA_SA_CMD, CESA_SA_CMD_ACTVATE);
 
 	CESA_UNLOCK(sc, requests);
 }
 
 static int
 cesa_setup_sram(struct cesa_softc *sc)
 {
 	phandle_t sram_node;
 	ihandle_t sram_ihandle;
 	pcell_t sram_handle, sram_reg[2];
 	void *sram_va;
 	int rv;
 
 	rv = OF_getencprop(ofw_bus_get_node(sc->sc_dev), "sram-handle",
 	    (void *)&sram_handle, sizeof(sram_handle));
 	if (rv <= 0)
 		return (rv);
 
 	sram_ihandle = (ihandle_t)sram_handle;
 	sram_node = OF_instance_to_package(sram_ihandle);
 
 	rv = OF_getencprop(sram_node, "reg", (void *)sram_reg, sizeof(sram_reg));
 	if (rv <= 0)
 		return (rv);
 
 	sc->sc_sram_base_pa = sram_reg[0];
 	/* Store SRAM size to be able to unmap in detach() */
 	sc->sc_sram_size = sram_reg[1];
 
 	if (sc->sc_soc_id != MV_DEV_88F6828 &&
 	    sc->sc_soc_id != MV_DEV_88F6820 &&
 	    sc->sc_soc_id != MV_DEV_88F6810)
 		return (0);
 
 	/* SRAM memory was not mapped in platform_sram_devmap(), map it now */
 	sram_va = pmap_mapdev(sc->sc_sram_base_pa, sc->sc_sram_size);
 	if (sram_va == NULL)
 		return (ENOMEM);
 	sc->sc_sram_base_va = (vm_offset_t)sram_va;
 
 	return (0);
 }
 
 /*
  * Function: device_from_node
  * This function returns appropriate device_t to phandle_t
  * Parameters:
  * root - device where you want to start search
  *     if you provide NULL here, function will take
  *     "root0" device as root.
  * node - we are checking every device_t to be
  *     appropriate with this.
  */
 static device_t
 device_from_node(device_t root, phandle_t node)
 {
 	device_t *children, retval;
 	int nkid, i;
 
 	/* Nothing matches no node */
 	if (node == -1)
 		return (NULL);
 
 	if (root == NULL)
 		/* Get root of device tree */
 		if ((root = device_lookup_by_name("root0")) == NULL)
 			return (NULL);
 
 	if (device_get_children(root, &children, &nkid) != 0)
 		return (NULL);
 
 	retval = NULL;
 	for (i = 0; i < nkid; i++) {
 		/* Check if device and node matches */
 		if (OFW_BUS_GET_NODE(root, children[i]) == node) {
 			retval = children[i];
 			break;
 		}
 		/* or go deeper */
 		if ((retval = device_from_node(children[i], node)) != NULL)
 			break;
 	}
 	free(children, M_TEMP);
 
 	return (retval);
 }
 
 static int
 cesa_setup_sram_armada(struct cesa_softc *sc)
 {
 	phandle_t sram_node;
 	ihandle_t sram_ihandle;
 	pcell_t sram_handle[2];
 	void *sram_va;
 	int rv, j;
 	struct resource_list rl;
 	struct resource_list_entry *rle;
 	struct simplebus_softc *ssc;
 	device_t sdev;
 
 	/* Get refs to SRAMS from CESA node */
 	rv = OF_getencprop(ofw_bus_get_node(sc->sc_dev), "marvell,crypto-srams",
 	    (void *)sram_handle, sizeof(sram_handle));
 	if (rv <= 0)
 		return (rv);
 
 	if (sc->sc_cesa_engine_id >= 2)
 		return (ENXIO);
 
 	/* Get SRAM node on the basis of sc_cesa_engine_id */
 	sram_ihandle = (ihandle_t)sram_handle[sc->sc_cesa_engine_id];
 	sram_node = OF_instance_to_package(sram_ihandle);
 
 	/* Get device_t of simplebus (sram_node parent) */
 	sdev = device_from_node(NULL, OF_parent(sram_node));
 	if (!sdev)
 		return (ENXIO);
 
 	ssc = device_get_softc(sdev);
 
 	resource_list_init(&rl);
 	/* Parse reg property to resource list */
 	ofw_bus_reg_to_rl(sdev, sram_node, ssc->acells,
 	    ssc->scells, &rl);
 
 	/* We expect only one resource */
 	rle = resource_list_find(&rl, SYS_RES_MEMORY, 0);
 	if (rle == NULL)
 		return (ENXIO);
 
 	/* Remap through ranges property */
 	for (j = 0; j < ssc->nranges; j++) {
 		if (rle->start >= ssc->ranges[j].bus &&
 		    rle->end < ssc->ranges[j].bus + ssc->ranges[j].size) {
 			rle->start -= ssc->ranges[j].bus;
 			rle->start += ssc->ranges[j].host;
 			rle->end -= ssc->ranges[j].bus;
 			rle->end += ssc->ranges[j].host;
 		}
 	}
 
 	sc->sc_sram_base_pa = rle->start;
 	sc->sc_sram_size = rle->count;
 
 	/* SRAM memory was not mapped in platform_sram_devmap(), map it now */
 	sram_va = pmap_mapdev(sc->sc_sram_base_pa, sc->sc_sram_size);
 	if (sram_va == NULL)
 		return (ENOMEM);
 	sc->sc_sram_base_va = (vm_offset_t)sram_va;
 
 	return (0);
 }
 
 struct ofw_compat_data cesa_devices[] = {
 	{ "mrvl,cesa", (uintptr_t)true },
 	{ "marvell,armada-38x-crypto", (uintptr_t)true },
 	{ NULL, 0 }
 };
 
 static int
 cesa_probe(device_t dev)
 {
 
 	if (!ofw_bus_status_okay(dev))
 		return (ENXIO);
 
 	if (!ofw_bus_search_compatible(dev, cesa_devices)->ocd_data)
 		return (ENXIO);
 
 	device_set_desc(dev, "Marvell Cryptographic Engine and Security "
 	    "Accelerator");
 
 	return (BUS_PROBE_DEFAULT);
 }
 
 static int
 cesa_attach(device_t dev)
 {
 	static int engine_idx = 0;
 	struct simplebus_devinfo *ndi;
 	struct resource_list *rl;
 	struct cesa_softc *sc;
 
 	if (!ofw_bus_is_compatible(dev, "marvell,armada-38x-crypto"))
 		return (cesa_attach_late(dev));
 
 	/*
 	 * Get simplebus_devinfo which contains
 	 * resource list filled with adresses and
 	 * interrupts read form FDT.
 	 * Let's correct it by splitting resources
 	 * for each engine.
 	 */
 	if ((ndi = device_get_ivars(dev)) == NULL)
 		return (ENXIO);
 
 	rl = &ndi->rl;
 
 	switch (engine_idx) {
 		case 0:
 			/* Update regs values */
 			resource_list_add(rl, SYS_RES_MEMORY, 0, CESA0_TDMA_ADDR,
 			    CESA0_TDMA_ADDR + CESA_TDMA_SIZE - 1, CESA_TDMA_SIZE);
 			resource_list_add(rl, SYS_RES_MEMORY, 1, CESA0_CESA_ADDR,
 			    CESA0_CESA_ADDR + CESA_CESA_SIZE - 1, CESA_CESA_SIZE);
 
 			/* Remove unused interrupt */
 			resource_list_delete(rl, SYS_RES_IRQ, 1);
 			break;
 
 		case 1:
 			/* Update regs values */
 			resource_list_add(rl, SYS_RES_MEMORY, 0, CESA1_TDMA_ADDR,
 			    CESA1_TDMA_ADDR + CESA_TDMA_SIZE - 1, CESA_TDMA_SIZE);
 			resource_list_add(rl, SYS_RES_MEMORY, 1, CESA1_CESA_ADDR,
 			    CESA1_CESA_ADDR + CESA_CESA_SIZE - 1, CESA_CESA_SIZE);
 
 			/* Remove unused interrupt */
 			resource_list_delete(rl, SYS_RES_IRQ, 0);
 			resource_list_find(rl, SYS_RES_IRQ, 1)->rid = 0;
 			break;
 
 		default:
 			device_printf(dev, "Bad cesa engine_idx\n");
 			return (ENXIO);
 	}
 
 	sc = device_get_softc(dev);
 	sc->sc_cesa_engine_id = engine_idx;
 
 	/*
 	 * Call simplebus_add_device only once.
 	 * It will create second cesa driver instance
 	 * with the same FDT node as first instance.
 	 * When second driver reach this function,
 	 * it will be configured to use second cesa engine
 	 */
 	if (engine_idx == 0)
 		simplebus_add_device(device_get_parent(dev), ofw_bus_get_node(dev),
 		    0, "cesa", 1, NULL);
 
 	engine_idx++;
 
 	return (cesa_attach_late(dev));
 }
 
 static int
 cesa_attach_late(device_t dev)
 {
 	struct cesa_softc *sc;
 	uint32_t d, r, val;
 	int error;
 	int i;
 
 	sc = device_get_softc(dev);
 	sc->sc_blocked = 0;
 	sc->sc_error = 0;
 	sc->sc_dev = dev;
 
 	soc_id(&d, &r);
 
 	switch (d) {
 	case MV_DEV_88F6281:
 	case MV_DEV_88F6282:
 		/* Check if CESA peripheral device has power turned on */
 		if (soc_power_ctrl_get(CPU_PM_CTRL_CRYPTO) ==
 		    CPU_PM_CTRL_CRYPTO) {
 			device_printf(dev, "not powered on\n");
 			return (ENXIO);
 		}
 		sc->sc_tperr = 0;
 		break;
 	case MV_DEV_88F6828:
 	case MV_DEV_88F6820:
 	case MV_DEV_88F6810:
 		sc->sc_tperr = 0;
 		break;
 	case MV_DEV_MV78100:
 	case MV_DEV_MV78100_Z0:
 		/* Check if CESA peripheral device has power turned on */
 		if (soc_power_ctrl_get(CPU_PM_CTRL_CRYPTO) !=
 		    CPU_PM_CTRL_CRYPTO) {
 			device_printf(dev, "not powered on\n");
 			return (ENXIO);
 		}
 		sc->sc_tperr = CESA_ICR_TPERR;
 		break;
 	default:
 		return (ENXIO);
 	}
 
 	sc->sc_soc_id = d;
 
 	/* Initialize mutexes */
 	mtx_init(&sc->sc_sc_lock, device_get_nameunit(dev),
 	    "CESA Shared Data", MTX_DEF);
 	mtx_init(&sc->sc_tdesc_lock, device_get_nameunit(dev),
 	    "CESA TDMA Descriptors Pool", MTX_DEF);
 	mtx_init(&sc->sc_sdesc_lock, device_get_nameunit(dev),
 	    "CESA SA Descriptors Pool", MTX_DEF);
 	mtx_init(&sc->sc_requests_lock, device_get_nameunit(dev),
 	    "CESA Requests Pool", MTX_DEF);
 	mtx_init(&sc->sc_sessions_lock, device_get_nameunit(dev),
 	    "CESA Sessions Pool", MTX_DEF);
 
 	/* Allocate I/O and IRQ resources */
 	error = bus_alloc_resources(dev, cesa_res_spec, sc->sc_res);
 	if (error) {
 		device_printf(dev, "could not allocate resources\n");
 		goto err0;
 	}
 
 	/* Acquire SRAM base address */
 	if (!ofw_bus_is_compatible(dev, "marvell,armada-38x-crypto"))
 		error = cesa_setup_sram(sc);
 	else
 		error = cesa_setup_sram_armada(sc);
 
 	if (error) {
 		device_printf(dev, "could not setup SRAM\n");
 		goto err1;
 	}
 
 	/* Setup interrupt handler */
 	error = bus_setup_intr(dev, sc->sc_res[RES_CESA_IRQ], INTR_TYPE_NET |
 	    INTR_MPSAFE, NULL, cesa_intr, sc, &(sc->sc_icookie));
 	if (error) {
 		device_printf(dev, "could not setup engine completion irq\n");
 		goto err2;
 	}
 
 	/* Create DMA tag for processed data */
 	error = bus_dma_tag_create(bus_get_dma_tag(dev),	/* parent */
 	    1, 0,				/* alignment, boundary */
 	    BUS_SPACE_MAXADDR_32BIT,		/* lowaddr */
 	    BUS_SPACE_MAXADDR,			/* highaddr */
 	    NULL, NULL,				/* filtfunc, filtfuncarg */
 	    CESA_MAX_REQUEST_SIZE,		/* maxsize */
 	    CESA_MAX_FRAGMENTS,			/* nsegments */
 	    CESA_MAX_REQUEST_SIZE, 0,		/* maxsegsz, flags */
 	    NULL, NULL,				/* lockfunc, lockfuncarg */
 	    &sc->sc_data_dtag);			/* dmat */
 	if (error)
 		goto err3;
 
 	/* Initialize data structures: TDMA Descriptors Pool */
 	error = cesa_alloc_dma_mem(sc, &sc->sc_tdesc_cdm,
 	    CESA_TDMA_DESCRIPTORS * sizeof(struct cesa_tdma_hdesc));
 	if (error)
 		goto err4;
 
 	STAILQ_INIT(&sc->sc_free_tdesc);
 	for (i = 0; i < CESA_TDMA_DESCRIPTORS; i++) {
 		sc->sc_tdesc[i].ctd_cthd =
 		    (struct cesa_tdma_hdesc *)(sc->sc_tdesc_cdm.cdm_vaddr) + i;
 		sc->sc_tdesc[i].ctd_cthd_paddr = sc->sc_tdesc_cdm.cdm_paddr +
 		    (i * sizeof(struct cesa_tdma_hdesc));
 		STAILQ_INSERT_TAIL(&sc->sc_free_tdesc, &sc->sc_tdesc[i],
 		    ctd_stq);
 	}
 
 	/* Initialize data structures: SA Descriptors Pool */
 	error = cesa_alloc_dma_mem(sc, &sc->sc_sdesc_cdm,
 	    CESA_SA_DESCRIPTORS * sizeof(struct cesa_sa_hdesc));
 	if (error)
 		goto err5;
 
 	STAILQ_INIT(&sc->sc_free_sdesc);
 	for (i = 0; i < CESA_SA_DESCRIPTORS; i++) {
 		sc->sc_sdesc[i].csd_cshd =
 		    (struct cesa_sa_hdesc *)(sc->sc_sdesc_cdm.cdm_vaddr) + i;
 		sc->sc_sdesc[i].csd_cshd_paddr = sc->sc_sdesc_cdm.cdm_paddr +
 		    (i * sizeof(struct cesa_sa_hdesc));
 		STAILQ_INSERT_TAIL(&sc->sc_free_sdesc, &sc->sc_sdesc[i],
 		    csd_stq);
 	}
 
 	/* Initialize data structures: Requests Pool */
 	error = cesa_alloc_dma_mem(sc, &sc->sc_requests_cdm,
 	    CESA_REQUESTS * sizeof(struct cesa_sa_data));
 	if (error)
 		goto err6;
 
 	STAILQ_INIT(&sc->sc_free_requests);
 	STAILQ_INIT(&sc->sc_ready_requests);
 	STAILQ_INIT(&sc->sc_queued_requests);
 	for (i = 0; i < CESA_REQUESTS; i++) {
 		sc->sc_requests[i].cr_csd =
 		    (struct cesa_sa_data *)(sc->sc_requests_cdm.cdm_vaddr) + i;
 		sc->sc_requests[i].cr_csd_paddr =
 		    sc->sc_requests_cdm.cdm_paddr +
 		    (i * sizeof(struct cesa_sa_data));
 
 		/* Preallocate DMA maps */
 		error = bus_dmamap_create(sc->sc_data_dtag, 0,
 		    &sc->sc_requests[i].cr_dmap);
 		if (error && i > 0) {
 			i--;
 			do {
 				bus_dmamap_destroy(sc->sc_data_dtag,
 				    sc->sc_requests[i].cr_dmap);
 			} while (i--);
 
 			goto err7;
 		}
 
 		STAILQ_INSERT_TAIL(&sc->sc_free_requests, &sc->sc_requests[i],
 		    cr_stq);
 	}
 
 	/*
 	 * Initialize TDMA:
 	 * - Burst limit: 128 bytes,
 	 * - Outstanding reads enabled,
 	 * - No byte-swap.
 	 */
 	val = CESA_TDMA_CR_DBL128 | CESA_TDMA_CR_SBL128 |
 	    CESA_TDMA_CR_ORDEN | CESA_TDMA_CR_NBS | CESA_TDMA_CR_ENABLE;
 
 	if (sc->sc_soc_id == MV_DEV_88F6828 ||
 	    sc->sc_soc_id == MV_DEV_88F6820 ||
 	    sc->sc_soc_id == MV_DEV_88F6810)
 		val |= CESA_TDMA_NUM_OUTSTAND;
 
 	CESA_TDMA_WRITE(sc, CESA_TDMA_CR, val);
 
 	/*
 	 * Initialize SA:
 	 * - SA descriptor is present at beginning of CESA SRAM,
 	 * - Multi-packet chain mode,
 	 * - Cooperation with TDMA enabled.
 	 */
 	CESA_REG_WRITE(sc, CESA_SA_DPR, 0);
 	CESA_REG_WRITE(sc, CESA_SA_CR, CESA_SA_CR_ACTIVATE_TDMA |
 	    CESA_SA_CR_WAIT_FOR_TDMA | CESA_SA_CR_MULTI_MODE);
 
 	/* Unmask interrupts */
 	CESA_REG_WRITE(sc, CESA_ICR, 0);
 	CESA_REG_WRITE(sc, CESA_ICM, CESA_ICM_ACCTDMA | sc->sc_tperr);
 	CESA_TDMA_WRITE(sc, CESA_TDMA_ECR, 0);
 	CESA_TDMA_WRITE(sc, CESA_TDMA_EMR, CESA_TDMA_EMR_MISS |
 	    CESA_TDMA_EMR_DOUBLE_HIT | CESA_TDMA_EMR_BOTH_HIT |
 	    CESA_TDMA_EMR_DATA_ERROR);
 
 	/* Register in OCF */
 	sc->sc_cid = crypto_get_driverid(dev, sizeof(struct cesa_session),
 	    CRYPTOCAP_F_HARDWARE);
 	if (sc->sc_cid < 0) {
 		device_printf(dev, "could not get crypto driver id\n");
 		goto err8;
 	}
 
-	crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
-	crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
-	crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
-	crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0);
-	crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
-	crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0);
-	crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
-	if (sc->sc_soc_id == MV_DEV_88F6828 ||
-	    sc->sc_soc_id == MV_DEV_88F6820 ||
-	    sc->sc_soc_id == MV_DEV_88F6810)
-		crypto_register(sc->sc_cid, CRYPTO_SHA2_256_HMAC, 0, 0);
-
 	return (0);
 err8:
 	for (i = 0; i < CESA_REQUESTS; i++)
 		bus_dmamap_destroy(sc->sc_data_dtag,
 		    sc->sc_requests[i].cr_dmap);
 err7:
 	cesa_free_dma_mem(&sc->sc_requests_cdm);
 err6:
 	cesa_free_dma_mem(&sc->sc_sdesc_cdm);
 err5:
 	cesa_free_dma_mem(&sc->sc_tdesc_cdm);
 err4:
 	bus_dma_tag_destroy(sc->sc_data_dtag);
 err3:
 	bus_teardown_intr(dev, sc->sc_res[RES_CESA_IRQ], sc->sc_icookie);
 err2:
 	if (sc->sc_soc_id == MV_DEV_88F6828 ||
 	    sc->sc_soc_id == MV_DEV_88F6820 ||
 	    sc->sc_soc_id == MV_DEV_88F6810)
 		pmap_unmapdev(sc->sc_sram_base_va, sc->sc_sram_size);
 err1:
 	bus_release_resources(dev, cesa_res_spec, sc->sc_res);
 err0:
 	mtx_destroy(&sc->sc_sessions_lock);
 	mtx_destroy(&sc->sc_requests_lock);
 	mtx_destroy(&sc->sc_sdesc_lock);
 	mtx_destroy(&sc->sc_tdesc_lock);
 	mtx_destroy(&sc->sc_sc_lock);
 	return (ENXIO);
 }
 
 static int
 cesa_detach(device_t dev)
 {
 	struct cesa_softc *sc;
 	int i;
  
 	sc = device_get_softc(dev);
 
 	/* TODO: Wait for queued requests completion before shutdown. */
 
 	/* Mask interrupts */
 	CESA_REG_WRITE(sc, CESA_ICM, 0);
 	CESA_TDMA_WRITE(sc, CESA_TDMA_EMR, 0);
 
 	/* Unregister from OCF */
 	crypto_unregister_all(sc->sc_cid);
 
 	/* Free DMA Maps */
 	for (i = 0; i < CESA_REQUESTS; i++)
 		bus_dmamap_destroy(sc->sc_data_dtag,
 		    sc->sc_requests[i].cr_dmap);
 
 	/* Free DMA Memory */
 	cesa_free_dma_mem(&sc->sc_requests_cdm);
 	cesa_free_dma_mem(&sc->sc_sdesc_cdm);
 	cesa_free_dma_mem(&sc->sc_tdesc_cdm);
 
 	/* Free DMA Tag */
 	bus_dma_tag_destroy(sc->sc_data_dtag);
 
 	/* Stop interrupt */
 	bus_teardown_intr(dev, sc->sc_res[RES_CESA_IRQ], sc->sc_icookie);
 
 	/* Relase I/O and IRQ resources */
 	bus_release_resources(dev, cesa_res_spec, sc->sc_res);
 
 	/* Unmap SRAM memory */
 	if (sc->sc_soc_id == MV_DEV_88F6828 ||
 	    sc->sc_soc_id == MV_DEV_88F6820 ||
 	    sc->sc_soc_id == MV_DEV_88F6810)
 		pmap_unmapdev(sc->sc_sram_base_va, sc->sc_sram_size);
 
 	/* Destroy mutexes */
 	mtx_destroy(&sc->sc_sessions_lock);
 	mtx_destroy(&sc->sc_requests_lock);
 	mtx_destroy(&sc->sc_sdesc_lock);
 	mtx_destroy(&sc->sc_tdesc_lock);
 	mtx_destroy(&sc->sc_sc_lock);
 
 	return (0);
 }
 
 static void
 cesa_intr(void *arg)
 {
 	STAILQ_HEAD(, cesa_request) requests;
 	struct cesa_request *cr, *tmp;
 	struct cesa_softc *sc;
 	uint32_t ecr, icr;
+	uint8_t hash[HASH_MAX_LEN];
 	int blocked;
 
 	sc = arg;
 
 	/* Ack interrupt */
 	ecr = CESA_TDMA_READ(sc, CESA_TDMA_ECR);
 	CESA_TDMA_WRITE(sc, CESA_TDMA_ECR, 0);
 	icr = CESA_REG_READ(sc, CESA_ICR);
 	CESA_REG_WRITE(sc, CESA_ICR, 0);
 
 	/* Check for TDMA errors */
 	if (ecr & CESA_TDMA_ECR_MISS) {
 		device_printf(sc->sc_dev, "TDMA Miss error detected!\n");
 		sc->sc_error = EIO;
 	}
 
 	if (ecr & CESA_TDMA_ECR_DOUBLE_HIT) {
 		device_printf(sc->sc_dev, "TDMA Double Hit error detected!\n");
 		sc->sc_error = EIO;
 	}
 
 	if (ecr & CESA_TDMA_ECR_BOTH_HIT) {
 		device_printf(sc->sc_dev, "TDMA Both Hit error detected!\n");
 		sc->sc_error = EIO;
 	}
 
 	if (ecr & CESA_TDMA_ECR_DATA_ERROR) {
 		device_printf(sc->sc_dev, "TDMA Data error detected!\n");
 		sc->sc_error = EIO;
 	}
 
 	/* Check for CESA errors */
 	if (icr & sc->sc_tperr) {
 		device_printf(sc->sc_dev, "CESA SRAM Parity error detected!\n");
 		sc->sc_error = EIO;
 	}
 
 	/* If there is nothing more to do, return */
 	if ((icr & CESA_ICR_ACCTDMA) == 0)
 		return;
 
 	/* Get all finished requests */
 	CESA_LOCK(sc, requests);
 	STAILQ_INIT(&requests);
 	STAILQ_CONCAT(&requests, &sc->sc_queued_requests);
 	STAILQ_INIT(&sc->sc_queued_requests);
 	CESA_UNLOCK(sc, requests);
 
 	/* Execute all ready requests */
 	cesa_execute(sc);
 
 	/* Process completed requests */
 	cesa_sync_dma_mem(&sc->sc_requests_cdm, BUS_DMASYNC_POSTREAD |
 	    BUS_DMASYNC_POSTWRITE);
 
 	STAILQ_FOREACH_SAFE(cr, &requests, cr_stq, tmp) {
 		bus_dmamap_sync(sc->sc_data_dtag, cr->cr_dmap,
 		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 
 		cr->cr_crp->crp_etype = sc->sc_error;
-		if (cr->cr_mac)
-			crypto_copyback(cr->cr_crp->crp_flags,
-			    cr->cr_crp->crp_buf, cr->cr_mac->crd_inject,
-			    cr->cr_cs->cs_hlen, cr->cr_csd->csd_hash);
-
+		if (cr->cr_cs->cs_hlen != 0 && cr->cr_crp->crp_etype == 0) {
+			if (cr->cr_crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
+				crypto_copydata(cr->cr_crp,
+				    cr->cr_crp->crp_digest_start,
+				    cr->cr_cs->cs_hlen, hash);
+				if (timingsafe_bcmp(hash, cr->cr_csd->csd_hash,
+				    cr->cr_cs->cs_hlen) != 0)
+					cr->cr_crp->crp_etype = EBADMSG;
+			} else
+				crypto_copyback(cr->cr_crp,
+				    cr->cr_crp->crp_digest_start,
+				    cr->cr_cs->cs_hlen, cr->cr_csd->csd_hash);
+		}
 		crypto_done(cr->cr_crp);
 		cesa_free_request(sc, cr);
 	}
 
 	cesa_sync_dma_mem(&sc->sc_requests_cdm, BUS_DMASYNC_PREREAD |
 	    BUS_DMASYNC_PREWRITE);
 
 	sc->sc_error = 0;
 
 	/* Unblock driver if it ran out of resources */
 	CESA_LOCK(sc, sc);
 	blocked = sc->sc_blocked;
 	sc->sc_blocked = 0;
 	CESA_UNLOCK(sc, sc);
 
 	if (blocked)
 		crypto_unblock(sc->sc_cid, blocked);
 }
 
-static int
-cesa_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
+static bool
+cesa_cipher_supported(const struct crypto_session_params *csp)
 {
-	struct cesa_session *cs;
-	struct cesa_softc *sc;
-	struct cryptoini *enc;
-	struct cryptoini *mac;
-	int error;
- 
-	sc = device_get_softc(dev);
-	enc = NULL;
-	mac = NULL;
-	error = 0;
 
-	/* Check and parse input */
-	if (cesa_is_hash(cri->cri_alg))
-		mac = cri;
-	else
-		enc = cri;
+	switch (csp->csp_cipher_alg) {
+	case CRYPTO_AES_CBC:
+		if (csp->csp_ivlen != AES_BLOCK_LEN)
+			return (false);
+		break;
+	case CRYPTO_DES_CBC:
+		if (csp->csp_ivlen != DES_BLOCK_LEN)
+			return (false);
+		break;
+	case CRYPTO_3DES_CBC:
+		if (csp->csp_ivlen != DES3_BLOCK_LEN)
+			return (false);
+		break;
+	default:
+		return (false);
+	}
 
-	cri = cri->cri_next;
+	if (csp->csp_cipher_klen > CESA_MAX_KEY_LEN)
+		return (false);
 
-	if (cri) {
-		if (!enc && !cesa_is_hash(cri->cri_alg))
-			enc = cri;
+	return (true);
+}
 
-		if (!mac && cesa_is_hash(cri->cri_alg))
-			mac = cri;
+static bool
+cesa_auth_supported(struct cesa_softc *sc,
+    const struct crypto_session_params *csp)
+{
 
-		if (cri->cri_next || !(enc && mac))
+	switch (csp->csp_auth_alg) {
+	case CRYPTO_SHA2_256_HMAC:
+		if (!(sc->sc_soc_id == MV_DEV_88F6828 ||
+		    sc->sc_soc_id == MV_DEV_88F6820 ||
+		    sc->sc_soc_id == MV_DEV_88F6810))
+			return (false);
+		/* FALLTHROUGH */
+	case CRYPTO_MD5:
+	case CRYPTO_MD5_HMAC:
+	case CRYPTO_SHA1:
+	case CRYPTO_SHA1_HMAC:
+		break;
+	default:
+		return (false);
+	}
+
+	if (csp->csp_auth_klen > CESA_MAX_MKEY_LEN)
+		return (false);
+
+	return (true);
+}
+
+static int
+cesa_probesession(device_t dev, const struct crypto_session_params *csp)
+{
+	struct cesa_softc *sc;
+
+	sc = device_get_softc(dev);
+	if (csp->csp_flags != 0)
+		return (EINVAL);
+	switch (csp->csp_mode) {
+	case CSP_MODE_DIGEST:
+		if (!cesa_auth_supported(sc, csp))
 			return (EINVAL);
+		break;
+	case CSP_MODE_CIPHER:
+		if (!cesa_cipher_supported(csp))
+			return (EINVAL);
+		break;
+	case CSP_MODE_ETA:
+		if (!cesa_auth_supported(sc, csp) ||
+		    !cesa_cipher_supported(csp))
+			return (EINVAL);
+		break;
+	default:
+		return (EINVAL);
 	}
+	return (CRYPTODEV_PROBE_HARDWARE);
+}
 
-	if ((enc && (enc->cri_klen / 8) > CESA_MAX_KEY_LEN) ||
-	    (mac && (mac->cri_klen / 8) > CESA_MAX_MKEY_LEN))
-		return (E2BIG);
+static int
+cesa_newsession(device_t dev, crypto_session_t cses,
+    const struct crypto_session_params *csp)
+{
+	struct cesa_session *cs;
+	struct cesa_softc *sc;
+	int error;
+ 
+	sc = device_get_softc(dev);
+	error = 0;
 
 	/* Allocate session */
 	cs = crypto_get_driver_session(cses);
 
 	/* Prepare CESA configuration */
 	cs->cs_config = 0;
 	cs->cs_ivlen = 1;
 	cs->cs_mblen = 1;
 
-	if (enc) {
-		switch (enc->cri_alg) {
-		case CRYPTO_AES_CBC:
-			cs->cs_config |= CESA_CSHD_AES | CESA_CSHD_CBC;
-			cs->cs_ivlen = AES_BLOCK_LEN;
-			break;
-		case CRYPTO_DES_CBC:
-			cs->cs_config |= CESA_CSHD_DES | CESA_CSHD_CBC;
-			cs->cs_ivlen = DES_BLOCK_LEN;
-			break;
-		case CRYPTO_3DES_CBC:
-			cs->cs_config |= CESA_CSHD_3DES | CESA_CSHD_3DES_EDE |
-			    CESA_CSHD_CBC;
-			cs->cs_ivlen = DES3_BLOCK_LEN;
-			break;
-		default:
-			error = EINVAL;
-			break;
-		}
+	switch (csp->csp_cipher_alg) {
+	case CRYPTO_AES_CBC:
+		cs->cs_config |= CESA_CSHD_AES | CESA_CSHD_CBC;
+		cs->cs_ivlen = AES_BLOCK_LEN;
+		break;
+	case CRYPTO_DES_CBC:
+		cs->cs_config |= CESA_CSHD_DES | CESA_CSHD_CBC;
+		cs->cs_ivlen = DES_BLOCK_LEN;
+		break;
+	case CRYPTO_3DES_CBC:
+		cs->cs_config |= CESA_CSHD_3DES | CESA_CSHD_3DES_EDE |
+		    CESA_CSHD_CBC;
+		cs->cs_ivlen = DES3_BLOCK_LEN;
+		break;
 	}
 
-	if (!error && mac) {
-		switch (mac->cri_alg) {
-		case CRYPTO_MD5:
-			cs->cs_mblen = 1;
-			cs->cs_hlen = (mac->cri_mlen == 0) ? MD5_HASH_LEN :
-			    mac->cri_mlen;
-			cs->cs_config |= CESA_CSHD_MD5;
-			break;
-		case CRYPTO_MD5_HMAC:
-			cs->cs_mblen = MD5_BLOCK_LEN;
-			cs->cs_hlen = (mac->cri_mlen == 0) ? MD5_HASH_LEN :
-			    mac->cri_mlen;
-			cs->cs_config |= CESA_CSHD_MD5_HMAC;
-			if (cs->cs_hlen == CESA_HMAC_TRUNC_LEN)
-				cs->cs_config |= CESA_CSHD_96_BIT_HMAC;
-			break;
-		case CRYPTO_SHA1:
-			cs->cs_mblen = 1;
-			cs->cs_hlen = (mac->cri_mlen == 0) ? SHA1_HASH_LEN :
-			    mac->cri_mlen;
-			cs->cs_config |= CESA_CSHD_SHA1;
-			break;
-		case CRYPTO_SHA1_HMAC:
-			cs->cs_mblen = SHA1_BLOCK_LEN;
-			cs->cs_hlen = (mac->cri_mlen == 0) ? SHA1_HASH_LEN :
-			    mac->cri_mlen;
-			cs->cs_config |= CESA_CSHD_SHA1_HMAC;
-			if (cs->cs_hlen == CESA_HMAC_TRUNC_LEN)
-				cs->cs_config |= CESA_CSHD_96_BIT_HMAC;
-			break;
-		case CRYPTO_SHA2_256_HMAC:
-			cs->cs_mblen = SHA2_256_BLOCK_LEN;
-			cs->cs_hlen = (mac->cri_mlen == 0) ? SHA2_256_HASH_LEN :
-			    mac->cri_mlen;
-			cs->cs_config |= CESA_CSHD_SHA2_256_HMAC;
-			break;
-		default:
-			error = EINVAL;
-			break;
-		}
+	switch (csp->csp_auth_alg) {
+	case CRYPTO_MD5:
+		cs->cs_mblen = 1;
+		cs->cs_hlen = (csp->csp_auth_mlen == 0) ? MD5_HASH_LEN :
+		    csp->csp_auth_mlen;
+		cs->cs_config |= CESA_CSHD_MD5;
+		break;
+	case CRYPTO_MD5_HMAC:
+		cs->cs_mblen = MD5_BLOCK_LEN;
+		cs->cs_hlen = (csp->csp_auth_mlen == 0) ? MD5_HASH_LEN :
+		    csp->csp_auth_mlen;
+		cs->cs_config |= CESA_CSHD_MD5_HMAC;
+		if (cs->cs_hlen == CESA_HMAC_TRUNC_LEN)
+			cs->cs_config |= CESA_CSHD_96_BIT_HMAC;
+		break;
+	case CRYPTO_SHA1:
+		cs->cs_mblen = 1;
+		cs->cs_hlen = (csp->csp_auth_mlen == 0) ? SHA1_HASH_LEN :
+		    csp->csp_auth_mlen;
+		cs->cs_config |= CESA_CSHD_SHA1;
+		break;
+	case CRYPTO_SHA1_HMAC:
+		cs->cs_mblen = SHA1_BLOCK_LEN;
+		cs->cs_hlen = (csp->csp_auth_mlen == 0) ? SHA1_HASH_LEN :
+		    csp->csp_auth_mlen;
+		cs->cs_config |= CESA_CSHD_SHA1_HMAC;
+		if (cs->cs_hlen == CESA_HMAC_TRUNC_LEN)
+			cs->cs_config |= CESA_CSHD_96_BIT_HMAC;
+		break;
+	case CRYPTO_SHA2_256_HMAC:
+		cs->cs_mblen = SHA2_256_BLOCK_LEN;
+		cs->cs_hlen = (csp->csp_auth_mlen == 0) ? SHA2_256_HASH_LEN :
+		    csp->csp_auth_mlen;
+		cs->cs_config |= CESA_CSHD_SHA2_256_HMAC;
+		break;
 	}
 
 	/* Save cipher key */
-	if (!error && enc && enc->cri_key) {
-		cs->cs_klen = enc->cri_klen / 8;
-		memcpy(cs->cs_key, enc->cri_key, cs->cs_klen);
-		if (enc->cri_alg == CRYPTO_AES_CBC)
-			error = cesa_prep_aes_key(cs);
+	if (csp->csp_cipher_key != NULL) {
+		memcpy(cs->cs_key, csp->csp_cipher_key,
+		    csp->csp_cipher_klen);
+		if (csp->csp_cipher_alg == CRYPTO_AES_CBC)
+			error = cesa_prep_aes_key(cs, csp);
 	}
 
 	/* Save digest key */
-	if (!error && mac && mac->cri_key)
-		error = cesa_set_mkey(cs, mac->cri_alg, mac->cri_key,
-		    mac->cri_klen / 8);
+	if (csp->csp_auth_key != NULL)
+		cesa_set_mkey(cs, csp->csp_auth_alg, csp->csp_auth_key,
+		    csp->csp_auth_klen);
 
-	if (error)
-		return (error);
-
-	return (0);
+	return (error);
 }
 
 static int
 cesa_process(device_t dev, struct cryptop *crp, int hint)
 {
+	const struct crypto_session_params *csp;
 	struct cesa_request *cr;
 	struct cesa_session *cs;
-	struct cryptodesc *crd;
-	struct cryptodesc *enc;
-	struct cryptodesc *mac;
 	struct cesa_softc *sc;
 	int error;
 
 	sc = device_get_softc(dev);
-	crd = crp->crp_desc;
-	enc = NULL;
-	mac = NULL;
 	error = 0;
 
 	cs = crypto_get_driver_session(crp->crp_session);
+	csp = crypto_get_params(crp->crp_session);
 
 	/* Check and parse input */
 	if (crp->crp_ilen > CESA_MAX_REQUEST_SIZE) {
 		crp->crp_etype = E2BIG;
 		crypto_done(crp);
 		return (0);
 	}
 
-	if (cesa_is_hash(crd->crd_alg))
-		mac = crd;
-	else
-		enc = crd;
-
-	crd = crd->crd_next;
-
-	if (crd) {
-		if (!enc && !cesa_is_hash(crd->crd_alg))
-			enc = crd;
-
-		if (!mac && cesa_is_hash(crd->crd_alg))
-			mac = crd;
-
-		if (crd->crd_next || !(enc && mac)) {
-			crp->crp_etype = EINVAL;
-			crypto_done(crp);
-			return (0);
-		}
+	/*
+	 * For requests with AAD, only requests where the AAD is
+	 * immediately adjacent to the payload are supported.
+	 */
+	if (crp->crp_aad_length != 0 &&
+	    (crp->crp_aad_start + crp->crp_aad_length) !=
+	    crp->crp_payload_start) {
+		crp->crp_etype = EINVAL;
+		crypto_done(crp);
+		return (0);
 	}
 
 	/*
 	 * Get request descriptor. Block driver if there is no free
 	 * descriptors in pool.
 	 */
 	cr = cesa_alloc_request(sc);
 	if (!cr) {
 		CESA_LOCK(sc, sc);
 		sc->sc_blocked = CRYPTO_SYMQ;
 		CESA_UNLOCK(sc, sc);
 		return (ERESTART);
 	}
 
 	/* Prepare request */
 	cr->cr_crp = crp;
-	cr->cr_enc = enc;
-	cr->cr_mac = mac;
 	cr->cr_cs = cs;
 
 	CESA_LOCK(sc, sessions);
 	cesa_sync_desc(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 
-	if (enc && enc->crd_flags & CRD_F_ENCRYPT) {
-		if (enc->crd_flags & CRD_F_IV_EXPLICIT)
-			memcpy(cr->cr_csd->csd_iv, enc->crd_iv, cs->cs_ivlen);
+	if (csp->csp_cipher_alg != 0) {
+		if (crp->crp_flags & CRYPTO_F_IV_GENERATE) {
+			arc4rand(cr->cr_csd->csd_iv, csp->csp_ivlen, 0);
+			crypto_copyback(crp, crp->crp_iv_start, csp->csp_ivlen,
+		    cr->cr_csd->csd_iv);
+		} else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+			memcpy(cr->cr_csd->csd_iv, crp->crp_iv, csp->csp_ivlen);
 		else
-			arc4rand(cr->cr_csd->csd_iv, cs->cs_ivlen, 0);
-
-		if ((enc->crd_flags & CRD_F_IV_PRESENT) == 0)
-			crypto_copyback(crp->crp_flags, crp->crp_buf,
-			    enc->crd_inject, cs->cs_ivlen, cr->cr_csd->csd_iv);
-	} else if (enc) {
-		if (enc->crd_flags & CRD_F_IV_EXPLICIT)
-			memcpy(cr->cr_csd->csd_iv, enc->crd_iv, cs->cs_ivlen);
-		else
-			crypto_copydata(crp->crp_flags, crp->crp_buf,
-			    enc->crd_inject, cs->cs_ivlen, cr->cr_csd->csd_iv);
+			crypto_copydata(crp, crp->crp_iv_start, csp->csp_ivlen,
+		    cr->cr_csd->csd_iv);
 	}
 
-	if (enc && enc->crd_flags & CRD_F_KEY_EXPLICIT) {
-		if ((enc->crd_klen / 8) <= CESA_MAX_KEY_LEN) {
-			cs->cs_klen = enc->crd_klen / 8;
-			memcpy(cs->cs_key, enc->crd_key, cs->cs_klen);
-			if (enc->crd_alg == CRYPTO_AES_CBC)
-				error = cesa_prep_aes_key(cs);
-		} else
-			error = E2BIG;
+	if (crp->crp_cipher_key != NULL) {
+		memcpy(cs->cs_key, crp->crp_cipher_key,
+		    csp->csp_cipher_klen);
+		if (csp->csp_cipher_alg == CRYPTO_AES_CBC)
+			error = cesa_prep_aes_key(cs, csp);
 	}
 
-	if (!error && mac && mac->crd_flags & CRD_F_KEY_EXPLICIT) {
-		if ((mac->crd_klen / 8) <= CESA_MAX_MKEY_LEN)
-			error = cesa_set_mkey(cs, mac->crd_alg, mac->crd_key,
-			    mac->crd_klen / 8);
-		else
-			error = E2BIG;
-	}
+	if (!error && crp->crp_auth_key != NULL)
+		cesa_set_mkey(cs, csp->csp_auth_alg, crp->crp_auth_key,
+		    csp->csp_auth_klen);
 
 	/* Convert request to chain of TDMA and SA descriptors */
 	if (!error)
-		error = cesa_create_chain(sc, cr);
+		error = cesa_create_chain(sc, csp, cr);
 
 	cesa_sync_desc(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 	CESA_UNLOCK(sc, sessions);
 
 	if (error) {
 		cesa_free_request(sc, cr);
 		crp->crp_etype = error;
 		crypto_done(crp);
 		return (0);
 	}
 
 	bus_dmamap_sync(sc->sc_data_dtag, cr->cr_dmap, BUS_DMASYNC_PREREAD |
 	    BUS_DMASYNC_PREWRITE);
 
 	/* Enqueue request to execution */
 	cesa_enqueue_request(sc, cr);
 
 	/* Start execution, if we have no more requests in queue */
 	if ((hint & CRYPTO_HINT_MORE) == 0)
 		cesa_execute(sc);
 
 	return (0);
 }
Index: head/sys/dev/cesa/cesa.h
===================================================================
--- head/sys/dev/cesa/cesa.h	(revision 359373)
+++ head/sys/dev/cesa/cesa.h	(revision 359374)
@@ -1,371 +1,366 @@
 /*-
  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
  *
  * Copyright (C) 2009-2011 Semihalf.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
  * $FreeBSD$
  */
 
 #ifndef _DEV_CESA_H_
 #define _DEV_CESA_H_
 
 /* Maximum number of allocated sessions */
 #define CESA_SESSIONS			64
 
 /* Maximum number of queued requests */
 #define CESA_REQUESTS			256
 
 /*
  * CESA is able to process data only in CESA SRAM, which is quite small (2 kB).
  * We have to fit a packet there, which contains SA descriptor, keys, IV
  * and data to be processed. Every request must be converted into chain of
  * packets and each packet can hold about 1.75 kB of data.
  *
  * To process each packet we need at least 1 SA descriptor and at least 4 TDMA
  * descriptors. However there are cases when we use 2 SA and 8 TDMA descriptors
  * per packet. Number of used TDMA descriptors can increase beyond given values
  * if data in the request is fragmented in physical memory.
  *
  * The driver uses preallocated SA and TDMA descriptors pools to get best
  * performace. Size of these pools should match expected request size. Example:
  *
  * Expected average request size:			1.5 kB (Ethernet MTU)
  * Packets per average request:				(1.5 kB / 1.75 kB) = 1
  * SA decriptors per average request (worst case):	1 * 2 = 2
  * TDMA desctiptors per average request (worst case):	1 * 8 = 8
  *
  * More TDMA descriptors should be allocated, if data fragmentation is expected
  * (for example while processing mbufs larger than MCLBYTES). The driver may use
  * 2 additional TDMA descriptors per each discontinuity in the physical data
  * layout.
  */
 
 /* Values below are optimized for requests containing about 1.5 kB of data */
 #define CESA_SA_DESC_PER_REQ		2
 #define CESA_TDMA_DESC_PER_REQ		8
 
 #define CESA_SA_DESCRIPTORS		(CESA_SA_DESC_PER_REQ * CESA_REQUESTS)
 #define CESA_TDMA_DESCRIPTORS		(CESA_TDMA_DESC_PER_REQ * CESA_REQUESTS)
 
 /* Useful constants */
 #define CESA_HMAC_TRUNC_LEN		12
 #define CESA_MAX_FRAGMENTS		64
 #define CESA_SRAM_SIZE			2048
 
 /*
  * CESA_MAX_HASH_LEN is maximum length of hash generated by CESA.
  * As CESA supports MD5, SHA1 and SHA-256 this equals to 32 bytes.
  */
 #define CESA_MAX_HASH_LEN		32
 #define CESA_MAX_KEY_LEN		32
 #define CESA_MAX_IV_LEN			16
 #define CESA_MAX_HMAC_BLOCK_LEN		64
 #define CESA_MAX_MKEY_LEN		CESA_MAX_HMAC_BLOCK_LEN
 #define CESA_MAX_PACKET_SIZE		(CESA_SRAM_SIZE - CESA_DATA(0))
 #define CESA_MAX_REQUEST_SIZE		65535
 
 /* Locking macros */
 #define CESA_LOCK(sc, what)		mtx_lock(&(sc)->sc_ ## what ## _lock)
 #define CESA_UNLOCK(sc, what)		mtx_unlock(&(sc)->sc_ ## what ## _lock)
 #define CESA_LOCK_ASSERT(sc, what)	\
 	mtx_assert(&(sc)->sc_ ## what ## _lock, MA_OWNED)
 
 /* Registers read/write macros */
 #define CESA_REG_READ(sc, reg)		\
 	bus_read_4((sc)->sc_res[RES_CESA_REGS], (reg))
 #define CESA_REG_WRITE(sc, reg, val)	\
 	bus_write_4((sc)->sc_res[RES_CESA_REGS], (reg), (val))
 
 #define CESA_TDMA_READ(sc, reg)		\
 	bus_read_4((sc)->sc_res[RES_TDMA_REGS], (reg))
 #define CESA_TDMA_WRITE(sc, reg, val)	\
 	bus_write_4((sc)->sc_res[RES_TDMA_REGS], (reg), (val))
 
 /* Generic allocator for objects */
 #define CESA_GENERIC_ALLOC_LOCKED(sc, obj, pool) do {		\
 	CESA_LOCK(sc, pool);					\
 								\
 	if (STAILQ_EMPTY(&(sc)->sc_free_ ## pool))		\
 		obj = NULL;					\
 	else {							\
 		obj = STAILQ_FIRST(&(sc)->sc_free_ ## pool);	\
 		STAILQ_REMOVE_HEAD(&(sc)->sc_free_ ## pool,	\
 		    obj ## _stq);				\
 	}							\
 								\
 	CESA_UNLOCK(sc, pool);					\
 } while (0)
 
 #define CESA_GENERIC_FREE_LOCKED(sc, obj, pool) do {		\
 	CESA_LOCK(sc, pool);					\
 	STAILQ_INSERT_TAIL(&(sc)->sc_free_ ## pool, obj,	\
 	    obj ## _stq);					\
 	CESA_UNLOCK(sc, pool);					\
 } while (0)
 
 /* CESA SRAM offset calculation macros */
 #define CESA_SA_DATA(member)					\
 	(sizeof(struct cesa_sa_hdesc) + offsetof(struct cesa_sa_data, member))
 #define CESA_DATA(offset)					\
 	(sizeof(struct cesa_sa_hdesc) + sizeof(struct cesa_sa_data) + offset)
 
 /* CESA memory and IRQ resources */
 enum cesa_res_type {
 	RES_TDMA_REGS,
 	RES_CESA_REGS,
 	RES_CESA_IRQ,
 	RES_CESA_NUM
 };
 
 struct cesa_tdma_hdesc {
 	uint16_t	cthd_byte_count;
 	uint16_t	cthd_flags;
 	uint32_t	cthd_src;
 	uint32_t	cthd_dst;
 	uint32_t	cthd_next;
 };
 
 struct cesa_sa_hdesc {
 	uint32_t	cshd_config;
 	uint16_t	cshd_enc_src;
 	uint16_t	cshd_enc_dst;
 	uint32_t	cshd_enc_dlen;
 	uint32_t	cshd_enc_key;
 	uint16_t	cshd_enc_iv;
 	uint16_t	cshd_enc_iv_buf;
 	uint16_t	cshd_mac_src;
 	uint16_t	cshd_mac_total_dlen;
 	uint16_t	cshd_mac_dst;
 	uint16_t	cshd_mac_dlen;
 	uint16_t	cshd_mac_iv_in;
 	uint16_t	cshd_mac_iv_out;
 };
 
 struct cesa_sa_data {
 	uint8_t		csd_key[CESA_MAX_KEY_LEN];
 	uint8_t		csd_iv[CESA_MAX_IV_LEN];
 	uint8_t		csd_hiv_in[CESA_MAX_HASH_LEN];
 	uint8_t		csd_hiv_out[CESA_MAX_HASH_LEN];
 	uint8_t		csd_hash[CESA_MAX_HASH_LEN];
 };
 
 struct cesa_dma_mem {
 	void		*cdm_vaddr;
 	bus_addr_t	cdm_paddr;
 	bus_dma_tag_t	cdm_tag;
 	bus_dmamap_t	cdm_map;
 };
 
 struct cesa_tdma_desc {
 	struct cesa_tdma_hdesc		*ctd_cthd;
 	bus_addr_t			ctd_cthd_paddr;
 
 	STAILQ_ENTRY(cesa_tdma_desc)	ctd_stq;
 };
 
 struct cesa_sa_desc {
 	struct cesa_sa_hdesc		*csd_cshd;
 	bus_addr_t			csd_cshd_paddr;
 
 	STAILQ_ENTRY(cesa_sa_desc)	csd_stq;
 };
 
 struct cesa_session {
 	uint32_t			cs_config;
-	unsigned int			cs_klen;
 	unsigned int			cs_ivlen;
 	unsigned int			cs_hlen;
 	unsigned int			cs_mblen;
 	uint8_t				cs_key[CESA_MAX_KEY_LEN];
 	uint8_t				cs_aes_dkey[CESA_MAX_KEY_LEN];
 	uint8_t				cs_hiv_in[CESA_MAX_HASH_LEN];
 	uint8_t				cs_hiv_out[CESA_MAX_HASH_LEN];
 };
 
 struct cesa_request {
 	struct cesa_sa_data		*cr_csd;
 	bus_addr_t			cr_csd_paddr;
 	struct cryptop			*cr_crp;
-	struct cryptodesc		*cr_enc;
-	struct cryptodesc		*cr_mac;
 	struct cesa_session		*cr_cs;
 	bus_dmamap_t			cr_dmap;
 	int				cr_dmap_loaded;
 
 	STAILQ_HEAD(, cesa_tdma_desc)	cr_tdesc;
 	STAILQ_HEAD(, cesa_sa_desc)	cr_sdesc;
 
 	STAILQ_ENTRY(cesa_request)	cr_stq;
 };
 
 struct cesa_packet {
 	STAILQ_HEAD(, cesa_tdma_desc)	cp_copyin;
 	STAILQ_HEAD(, cesa_tdma_desc)	cp_copyout;
 	unsigned int			cp_size;
 	unsigned int			cp_offset;
 };
 
 struct cesa_softc {
 	device_t			sc_dev;
 	int32_t				sc_cid;
 	uint32_t			sc_soc_id;
 	struct resource			*sc_res[RES_CESA_NUM];
 	void				*sc_icookie;
 	bus_dma_tag_t			sc_data_dtag;
 	int				sc_error;
 	int				sc_tperr;
 	uint8_t				sc_cesa_engine_id;
 
 	struct mtx			sc_sc_lock;
 	int				sc_blocked;
 
 	/* TDMA descriptors pool */
 	struct mtx			sc_tdesc_lock;
 	struct cesa_tdma_desc		sc_tdesc[CESA_TDMA_DESCRIPTORS];
 	struct cesa_dma_mem		sc_tdesc_cdm;
 	STAILQ_HEAD(, cesa_tdma_desc)	sc_free_tdesc;
 
 	/* SA descriptors pool */
 	struct mtx			sc_sdesc_lock;
 	struct cesa_sa_desc		sc_sdesc[CESA_SA_DESCRIPTORS];
 	struct cesa_dma_mem		sc_sdesc_cdm;
 	STAILQ_HEAD(, cesa_sa_desc)	sc_free_sdesc;
 
 	/* Requests pool */
 	struct mtx			sc_requests_lock;
 	struct cesa_request		sc_requests[CESA_REQUESTS];
 	struct cesa_dma_mem		sc_requests_cdm;
 	STAILQ_HEAD(, cesa_request)	sc_free_requests;
 	STAILQ_HEAD(, cesa_request)	sc_ready_requests;
 	STAILQ_HEAD(, cesa_request)	sc_queued_requests;
 
 	struct mtx			sc_sessions_lock;
 
 	/* CESA SRAM Address */
 	bus_addr_t			sc_sram_base_pa;
 	vm_offset_t			sc_sram_base_va;
 	bus_size_t			sc_sram_size;
 };
 
 struct cesa_chain_info {
 	struct cesa_softc		*cci_sc;
 	struct cesa_request		*cci_cr;
-	struct cryptodesc		*cci_enc;
-	struct cryptodesc		*cci_mac;
 	uint32_t			cci_config;
 	int				cci_error;
 };
 
 /* CESA descriptors flags definitions */
 #define CESA_CTHD_OWNED			(1 << 15)
 
 #define CESA_CSHD_MAC			(0 << 0)
 #define CESA_CSHD_ENC			(1 << 0)
 #define CESA_CSHD_MAC_AND_ENC		(2 << 0)
 #define CESA_CSHD_ENC_AND_MAC		(3 << 0)
 #define CESA_CSHD_OP_MASK		(3 << 0)
 
 #define CESA_CSHD_MD5			(4 << 4)
 #define CESA_CSHD_SHA1			(5 << 4)
 #define CESA_CSHD_SHA2_256		(1 << 4)
 #define CESA_CSHD_MD5_HMAC		(6 << 4)
 #define CESA_CSHD_SHA1_HMAC		(7 << 4)
 #define CESA_CSHD_SHA2_256_HMAC		(3 << 4)
 
 #define CESA_CSHD_96_BIT_HMAC		(1 << 7)
 
 #define CESA_CSHD_DES			(1 << 8)
 #define CESA_CSHD_3DES			(2 << 8)
 #define CESA_CSHD_AES			(3 << 8)
 
 #define CESA_CSHD_DECRYPT		(1 << 12)
 #define CESA_CSHD_CBC			(1 << 16)
 #define CESA_CSHD_3DES_EDE		(1 << 20)
 
 #define CESA_CSH_AES_KLEN_128		(0 << 24)
 #define CESA_CSH_AES_KLEN_192		(1 << 24)
 #define CESA_CSH_AES_KLEN_256		(2 << 24)
 #define CESA_CSH_AES_KLEN_MASK		(3 << 24)
 
 #define CESA_CSHD_FRAG_FIRST		(1 << 30)
 #define CESA_CSHD_FRAG_LAST		(2U << 30)
 #define CESA_CSHD_FRAG_MIDDLE		(3U << 30)
 
 /* CESA registers definitions */
 #define CESA_ICR			0x0E20
 #define CESA_ICR_ACCTDMA		(1 << 7)
 #define CESA_ICR_TPERR			(1 << 12)
 
 #define CESA_ICM			0x0E24
 #define CESA_ICM_ACCTDMA		CESA_ICR_ACCTDMA
 #define CESA_ICM_TPERR			CESA_ICR_TPERR
 
 /* CESA TDMA registers definitions */
 #define CESA_TDMA_ND			0x0830
 
 #define CESA_TDMA_CR			0x0840
 #define CESA_TDMA_CR_DBL128		(4 << 0)
 #define CESA_TDMA_CR_ORDEN		(1 << 4)
 #define CESA_TDMA_CR_SBL128		(4 << 6)
 #define CESA_TDMA_CR_NBS		(1 << 11)
 #define CESA_TDMA_CR_ENABLE		(1 << 12)
 #define CESA_TDMA_CR_FETCHND		(1 << 13)
 #define CESA_TDMA_CR_ACTIVE		(1 << 14)
 #define CESA_TDMA_NUM_OUTSTAND		(2 << 16)
 
 #define CESA_TDMA_ECR			0x08C8
 #define CESA_TDMA_ECR_MISS		(1 << 0)
 #define CESA_TDMA_ECR_DOUBLE_HIT	(1 << 1)
 #define CESA_TDMA_ECR_BOTH_HIT		(1 << 2)
 #define CESA_TDMA_ECR_DATA_ERROR	(1 << 3)
 
 #define CESA_TDMA_EMR			0x08CC
 #define CESA_TDMA_EMR_MISS		CESA_TDMA_ECR_MISS
 #define CESA_TDMA_EMR_DOUBLE_HIT	CESA_TDMA_ECR_DOUBLE_HIT
 #define CESA_TDMA_EMR_BOTH_HIT		CESA_TDMA_ECR_BOTH_HIT
 #define CESA_TDMA_EMR_DATA_ERROR	CESA_TDMA_ECR_DATA_ERROR
 
 /* CESA SA registers definitions */
 #define CESA_SA_CMD			0x0E00
 #define CESA_SA_CMD_ACTVATE		(1 << 0)
 #define CESA_SA_CMD_SHA2		(1 << 31)
 
 #define CESA_SA_DPR			0x0E04
 
 #define CESA_SA_CR			0x0E08
 #define CESA_SA_CR_WAIT_FOR_TDMA	(1 << 7)
 #define CESA_SA_CR_ACTIVATE_TDMA	(1 << 9)
 #define CESA_SA_CR_MULTI_MODE		(1 << 11)
 
 #define CESA_SA_SR			0x0E0C
 #define CESA_SA_SR_ACTIVE		(1 << 0)
 
 #define CESA_TDMA_SIZE			0x1000
 #define CESA_CESA_SIZE			0x1000
 #define CESA0_TDMA_ADDR			0x90000
 #define CESA0_CESA_ADDR			0x9D000
 #define CESA1_TDMA_ADDR			0x92000
 #define CESA1_CESA_ADDR			0x9F000
 #endif
Index: head/sys/dev/cxgbe/adapter.h
===================================================================
--- head/sys/dev/cxgbe/adapter.h	(revision 359373)
+++ head/sys/dev/cxgbe/adapter.h	(revision 359374)
@@ -1,1351 +1,1351 @@
 /*-
  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
  *
  * Copyright (c) 2011 Chelsio Communications, Inc.
  * All rights reserved.
  * Written by: Navdeep Parhar <np@FreeBSD.org>
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
  * $FreeBSD$
  *
  */
 
 #ifndef __T4_ADAPTER_H__
 #define __T4_ADAPTER_H__
 
 #include <sys/kernel.h>
 #include <sys/bus.h>
 #include <sys/counter.h>
 #include <sys/rman.h>
 #include <sys/types.h>
 #include <sys/lock.h>
 #include <sys/malloc.h>
 #include <sys/rwlock.h>
 #include <sys/sx.h>
 #include <sys/vmem.h>
 #include <vm/uma.h>
 
 #include <dev/pci/pcivar.h>
 #include <dev/pci/pcireg.h>
 #include <machine/bus.h>
 #include <sys/socket.h>
 #include <sys/sysctl.h>
 #include <net/ethernet.h>
 #include <net/if.h>
 #include <net/if_var.h>
 #include <net/if_media.h>
 #include <net/pfil.h>
 #include <netinet/in.h>
 #include <netinet/tcp_lro.h>
 
 #include "offload.h"
 #include "t4_ioctl.h"
 #include "common/t4_msg.h"
 #include "firmware/t4fw_interface.h"
 
 #define KTR_CXGBE	KTR_SPARE3
 MALLOC_DECLARE(M_CXGBE);
 #define CXGBE_UNIMPLEMENTED(s) \
     panic("%s (%s, line %d) not implemented yet.", s, __FILE__, __LINE__)
 
 #if defined(__i386__) || defined(__amd64__)
 static __inline void
 prefetch(void *x)
 {
 	__asm volatile("prefetcht0 %0" :: "m" (*(unsigned long *)x));
 }
 #else
 #define prefetch(x) __builtin_prefetch(x)
 #endif
 
 #ifndef SYSCTL_ADD_UQUAD
 #define SYSCTL_ADD_UQUAD SYSCTL_ADD_QUAD
 #define sysctl_handle_64 sysctl_handle_quad
 #define CTLTYPE_U64 CTLTYPE_QUAD
 #endif
 
 SYSCTL_DECL(_hw_cxgbe);
 
 struct adapter;
 typedef struct adapter adapter_t;
 
 enum {
 	/*
 	 * All ingress queues use this entry size.  Note that the firmware event
 	 * queue and any iq expecting CPL_RX_PKT in the descriptor needs this to
 	 * be at least 64.
 	 */
 	IQ_ESIZE = 64,
 
 	/* Default queue sizes for all kinds of ingress queues */
 	FW_IQ_QSIZE = 256,
 	RX_IQ_QSIZE = 1024,
 
 	/* All egress queues use this entry size */
 	EQ_ESIZE = 64,
 
 	/* Default queue sizes for all kinds of egress queues */
 	CTRL_EQ_QSIZE = 1024,
 	TX_EQ_QSIZE = 1024,
 
 #if MJUMPAGESIZE != MCLBYTES
 	SW_ZONE_SIZES = 4,	/* cluster, jumbop, jumbo9k, jumbo16k */
 #else
 	SW_ZONE_SIZES = 3,	/* cluster, jumbo9k, jumbo16k */
 #endif
 	CL_METADATA_SIZE = CACHE_LINE_SIZE,
 
 	SGE_MAX_WR_NDESC = SGE_MAX_WR_LEN / EQ_ESIZE, /* max WR size in desc */
 	TX_SGL_SEGS = 39,
 	TX_SGL_SEGS_TSO = 38,
 	TX_SGL_SEGS_EO_TSO = 30,	/* XXX: lower for IPv6. */
 	TX_WR_FLITS = SGE_MAX_WR_LEN / 8
 };
 
 enum {
 	/* adapter intr_type */
 	INTR_INTX	= (1 << 0),
 	INTR_MSI 	= (1 << 1),
 	INTR_MSIX	= (1 << 2)
 };
 
 enum {
 	XGMAC_MTU	= (1 << 0),
 	XGMAC_PROMISC	= (1 << 1),
 	XGMAC_ALLMULTI	= (1 << 2),
 	XGMAC_VLANEX	= (1 << 3),
 	XGMAC_UCADDR	= (1 << 4),
 	XGMAC_MCADDRS	= (1 << 5),
 
 	XGMAC_ALL	= 0xffff
 };
 
 enum {
 	/* flags understood by begin_synchronized_op */
 	HOLD_LOCK	= (1 << 0),
 	SLEEP_OK	= (1 << 1),
 	INTR_OK		= (1 << 2),
 
 	/* flags understood by end_synchronized_op */
 	LOCK_HELD	= HOLD_LOCK,
 };
 
 enum {
 	/* adapter flags */
 	FULL_INIT_DONE	= (1 << 0),
 	FW_OK		= (1 << 1),
 	CHK_MBOX_ACCESS	= (1 << 2),
 	MASTER_PF	= (1 << 3),
 	ADAP_SYSCTL_CTX	= (1 << 4),
 	ADAP_ERR	= (1 << 5),
 	BUF_PACKING_OK	= (1 << 6),
 	IS_VF		= (1 << 7),
 	KERN_TLS_OK	= (1 << 8),
 
 	CXGBE_BUSY	= (1 << 9),
 
 	/* port flags */
 	HAS_TRACEQ	= (1 << 3),
 	FIXED_IFMEDIA	= (1 << 4),	/* ifmedia list doesn't change. */
 
 	/* VI flags */
 	DOOMED		= (1 << 0),
 	VI_INIT_DONE	= (1 << 1),
 	VI_SYSCTL_CTX	= (1 << 2),
 
 	/* adapter debug_flags */
 	DF_DUMP_MBOX		= (1 << 0),	/* Log all mbox cmd/rpl. */
 	DF_LOAD_FW_ANYTIME	= (1 << 1),	/* Allow LOAD_FW after init */
 	DF_DISABLE_TCB_CACHE	= (1 << 2),	/* Disable TCB cache (T6+) */
 	DF_DISABLE_CFG_RETRY	= (1 << 3),	/* Disable fallback config */
 	DF_VERBOSE_SLOWINTR	= (1 << 4),	/* Chatty slow intr handler */
 };
 
 #define IS_DOOMED(vi)	((vi)->flags & DOOMED)
 #define SET_DOOMED(vi)	do {(vi)->flags |= DOOMED;} while (0)
 #define IS_BUSY(sc)	((sc)->flags & CXGBE_BUSY)
 #define SET_BUSY(sc)	do {(sc)->flags |= CXGBE_BUSY;} while (0)
 #define CLR_BUSY(sc)	do {(sc)->flags &= ~CXGBE_BUSY;} while (0)
 
 struct vi_info {
 	device_t dev;
 	struct port_info *pi;
 
 	struct ifnet *ifp;
 	struct pfil_head *pfil;
 
 	unsigned long flags;
 	int if_flags;
 
 	uint16_t *rss, *nm_rss;
 	uint16_t viid;		/* opaque VI identifier */
 	uint16_t smt_idx;
 	uint16_t vin;
 	uint8_t vfvld;
 	int16_t  xact_addr_filt;/* index of exact MAC address filter */
 	uint16_t rss_size;	/* size of VI's RSS table slice */
 	uint16_t rss_base;	/* start of VI's RSS table slice */
 	int hashen;
 
 	int nintr;
 	int first_intr;
 
 	/* These need to be int as they are used in sysctl */
 	int ntxq;		/* # of tx queues */
 	int first_txq;		/* index of first tx queue */
 	int rsrv_noflowq; 	/* Reserve queue 0 for non-flowid packets */
 	int nrxq;		/* # of rx queues */
 	int first_rxq;		/* index of first rx queue */
 	int nofldtxq;		/* # of offload tx queues */
 	int first_ofld_txq;	/* index of first offload tx queue */
 	int nofldrxq;		/* # of offload rx queues */
 	int first_ofld_rxq;	/* index of first offload rx queue */
 	int nnmtxq;
 	int first_nm_txq;
 	int nnmrxq;
 	int first_nm_rxq;
 	int tmr_idx;
 	int ofld_tmr_idx;
 	int pktc_idx;
 	int ofld_pktc_idx;
 	int qsize_rxq;
 	int qsize_txq;
 
 	struct timeval last_refreshed;
 	struct fw_vi_stats_vf stats;
 
 	struct callout tick;
 	struct sysctl_ctx_list ctx;	/* from ifconfig up to driver detach */
 
 	uint8_t hw_addr[ETHER_ADDR_LEN]; /* factory MAC address, won't change */
 };
 
 struct tx_ch_rl_params {
 	enum fw_sched_params_rate ratemode;	/* %port (REL) or kbps (ABS) */
 	uint32_t maxrate;
 };
 
 enum {
 	CLRL_USER	= (1 << 0),	/* allocated manually. */
 	CLRL_SYNC	= (1 << 1),	/* sync hw update in progress. */
 	CLRL_ASYNC	= (1 << 2),	/* async hw update requested. */
 	CLRL_ERR	= (1 << 3),	/* last hw setup ended in error. */
 };
 
 struct tx_cl_rl_params {
 	int refcount;
 	uint8_t flags;
 	enum fw_sched_params_rate ratemode;	/* %port REL or ABS value */
 	enum fw_sched_params_unit rateunit;	/* kbps or pps (when ABS) */
 	enum fw_sched_params_mode mode;		/* aggr or per-flow */
 	uint32_t maxrate;
 	uint16_t pktsize;
 	uint16_t burstsize;
 };
 
 /* Tx scheduler parameters for a channel/port */
 struct tx_sched_params {
 	/* Channel Rate Limiter */
 	struct tx_ch_rl_params ch_rl;
 
 	/* Class WRR */
 	/* XXX */
 
 	/* Class Rate Limiter (including the default pktsize and burstsize). */
 	int pktsize;
 	int burstsize;
 	struct tx_cl_rl_params cl_rl[];
 };
 
 struct port_info {
 	device_t dev;
 	struct adapter *adapter;
 
 	struct vi_info *vi;
 	int nvi;
 	int up_vis;
 	int uld_vis;
 
 	struct tx_sched_params *sched_params;
 
 	struct mtx pi_lock;
 	char lockname[16];
 	unsigned long flags;
 
 	uint8_t  lport;		/* associated offload logical port */
 	int8_t   mdio_addr;
 	uint8_t  port_type;
 	uint8_t  mod_type;
 	uint8_t  port_id;
 	uint8_t  tx_chan;
 	uint8_t  mps_bg_map;	/* rx MPS buffer group bitmap */
 	uint8_t  rx_e_chan_map;	/* rx TP e-channel bitmap */
 
 	struct link_config link_cfg;
 	struct ifmedia media;
 
 	struct timeval last_refreshed;
  	struct port_stats stats;
 	u_int tnl_cong_drops;
 	u_int tx_parse_error;
 	u_long	tx_toe_tls_records;
 	u_long	tx_toe_tls_octets;
 	u_long	rx_toe_tls_records;
 	u_long	rx_toe_tls_octets;
 
 	struct callout tick;
 };
 
 #define	IS_MAIN_VI(vi)		((vi) == &((vi)->pi->vi[0]))
 
 struct cluster_metadata {
 	uma_zone_t zone;
 	caddr_t cl;
 	u_int refcount;
 };
 
 struct fl_sdesc {
 	caddr_t cl;
 	uint16_t nmbuf;	/* # of driver originated mbufs with ref on cluster */
 	int16_t moff;	/* offset of metadata from cl */
 	uint8_t zidx;
 };
 
 struct tx_desc {
 	__be64 flit[8];
 };
 
 struct tx_sdesc {
 	struct mbuf *m;		/* m_nextpkt linked chain of frames */
 	uint8_t desc_used;	/* # of hardware descriptors used by the WR */
 };
 
 
 #define IQ_PAD (IQ_ESIZE - sizeof(struct rsp_ctrl) - sizeof(struct rss_header))
 struct iq_desc {
 	struct rss_header rss;
 	uint8_t cpl[IQ_PAD];
 	struct rsp_ctrl rsp;
 };
 #undef IQ_PAD
 CTASSERT(sizeof(struct iq_desc) == IQ_ESIZE);
 
 enum {
 	/* iq flags */
 	IQ_ALLOCATED	= (1 << 0),	/* firmware resources allocated */
 	IQ_HAS_FL	= (1 << 1),	/* iq associated with a freelist */
 	IQ_RX_TIMESTAMP	= (1 << 2),	/* provide the SGE rx timestamp */
 	IQ_LRO_ENABLED	= (1 << 3),	/* iq is an eth rxq with LRO enabled */
 	IQ_ADJ_CREDIT	= (1 << 4),	/* hw is off by 1 credit for this iq */
 
 	/* iq state */
 	IQS_DISABLED	= 0,
 	IQS_BUSY	= 1,
 	IQS_IDLE	= 2,
 
 	/* netmap related flags */
 	NM_OFF	= 0,
 	NM_ON	= 1,
 	NM_BUSY	= 2,
 };
 
 enum {
 	CPL_COOKIE_RESERVED = 0,
 	CPL_COOKIE_FILTER,
 	CPL_COOKIE_DDP0,
 	CPL_COOKIE_DDP1,
 	CPL_COOKIE_TOM,
 	CPL_COOKIE_HASHFILTER,
 	CPL_COOKIE_ETHOFLD,
 	CPL_COOKIE_KERN_TLS,
 
 	NUM_CPL_COOKIES = 8	/* Limited by M_COOKIE.  Do not increase. */
 };
 
 struct sge_iq;
 struct rss_header;
 typedef int (*cpl_handler_t)(struct sge_iq *, const struct rss_header *,
     struct mbuf *);
 typedef int (*an_handler_t)(struct sge_iq *, const struct rsp_ctrl *);
 typedef int (*fw_msg_handler_t)(struct adapter *, const __be64 *);
 
 /*
  * Ingress Queue: T4 is producer, driver is consumer.
  */
 struct sge_iq {
 	uint32_t flags;
 	volatile int state;
 	struct adapter *adapter;
 	struct iq_desc  *desc;	/* KVA of descriptor ring */
 	int8_t   intr_pktc_idx;	/* packet count threshold index */
 	uint8_t  gen;		/* generation bit */
 	uint8_t  intr_params;	/* interrupt holdoff parameters */
 	uint8_t  intr_next;	/* XXX: holdoff for next interrupt */
 	uint16_t qsize;		/* size (# of entries) of the queue */
 	uint16_t sidx;		/* index of the entry with the status page */
 	uint16_t cidx;		/* consumer index */
 	uint16_t cntxt_id;	/* SGE context id for the iq */
 	uint16_t abs_id;	/* absolute SGE id for the iq */
 
 	STAILQ_ENTRY(sge_iq) link;
 
 	bus_dma_tag_t desc_tag;
 	bus_dmamap_t desc_map;
 	bus_addr_t ba;		/* bus address of descriptor ring */
 };
 
 enum {
 	EQ_CTRL		= 1,
 	EQ_ETH		= 2,
 	EQ_OFLD		= 3,
 
 	/* eq flags */
 	EQ_TYPEMASK	= 0x3,		/* 2 lsbits hold the type (see above) */
 	EQ_ALLOCATED	= (1 << 2),	/* firmware resources allocated */
 	EQ_ENABLED	= (1 << 3),	/* open for business */
 	EQ_QFLUSH	= (1 << 4),	/* if_qflush in progress */
 };
 
 /* Listed in order of preference.  Update t4_sysctls too if you change these */
 enum {DOORBELL_UDB, DOORBELL_WCWR, DOORBELL_UDBWC, DOORBELL_KDB};
 
 /*
  * Egress Queue: driver is producer, T4 is consumer.
  *
  * Note: A free list is an egress queue (driver produces the buffers and T4
  * consumes them) but it's special enough to have its own struct (see sge_fl).
  */
 struct sge_eq {
 	unsigned int flags;	/* MUST be first */
 	unsigned int cntxt_id;	/* SGE context id for the eq */
 	unsigned int abs_id;	/* absolute SGE id for the eq */
 	struct mtx eq_lock;
 
 	struct tx_desc *desc;	/* KVA of descriptor ring */
 	uint8_t doorbells;
 	volatile uint32_t *udb;	/* KVA of doorbell (lies within BAR2) */
 	u_int udb_qid;		/* relative qid within the doorbell page */
 	uint16_t sidx;		/* index of the entry with the status page */
 	uint16_t cidx;		/* consumer idx (desc idx) */
 	uint16_t pidx;		/* producer idx (desc idx) */
 	uint16_t equeqidx;	/* EQUEQ last requested at this pidx */
 	uint16_t dbidx;		/* pidx of the most recent doorbell */
 	uint16_t iqid;		/* iq that gets egr_update for the eq */
 	uint8_t tx_chan;	/* tx channel used by the eq */
 	volatile u_int equiq;	/* EQUIQ outstanding */
 
 	bus_dma_tag_t desc_tag;
 	bus_dmamap_t desc_map;
 	bus_addr_t ba;		/* bus address of descriptor ring */
 	char lockname[16];
 };
 
 struct rx_buf_info {
 	uma_zone_t zone;	/* zone that this cluster comes from */
 	uint16_t size1;		/* same as size of cluster: 2K/4K/9K/16K.
 				 * hwsize[hwidx1] = size1.  No spare. */
 	uint16_t size2;		/* hwsize[hwidx2] = size2.
 				 * spare in cluster = size1 - size2. */
 	int8_t hwidx1;		/* SGE bufsize idx for size1 */
 	int8_t hwidx2;		/* SGE bufsize idx for size2 */
 	uint8_t type;		/* EXT_xxx type of the cluster */
 };
 
 enum {
 	NUM_MEMWIN = 3,
 
 	MEMWIN0_APERTURE = 2048,
 	MEMWIN0_BASE     = 0x1b800,
 
 	MEMWIN1_APERTURE = 32768,
 	MEMWIN1_BASE     = 0x28000,
 
 	MEMWIN2_APERTURE_T4 = 65536,
 	MEMWIN2_BASE_T4     = 0x30000,
 
 	MEMWIN2_APERTURE_T5 = 128 * 1024,
 	MEMWIN2_BASE_T5     = 0x60000,
 };
 
 struct memwin {
 	struct rwlock mw_lock __aligned(CACHE_LINE_SIZE);
 	uint32_t mw_base;	/* constant after setup_memwin */
 	uint32_t mw_aperture;	/* ditto */
 	uint32_t mw_curpos;	/* protected by mw_lock */
 };
 
 enum {
 	FL_STARVING	= (1 << 0), /* on the adapter's list of starving fl's */
 	FL_DOOMED	= (1 << 1), /* about to be destroyed */
 	FL_BUF_PACKING	= (1 << 2), /* buffer packing enabled */
 	FL_BUF_RESUME	= (1 << 3), /* resume from the middle of the frame */
 };
 
 #define FL_RUNNING_LOW(fl) \
     (IDXDIFF(fl->dbidx * 8, fl->cidx, fl->sidx * 8) <= fl->lowat)
 #define FL_NOT_RUNNING_LOW(fl) \
     (IDXDIFF(fl->dbidx * 8, fl->cidx, fl->sidx * 8) >= 2 * fl->lowat)
 
 struct sge_fl {
 	struct mtx fl_lock;
 	__be64 *desc;		/* KVA of descriptor ring, ptr to addresses */
 	struct fl_sdesc *sdesc;	/* KVA of software descriptor ring */
 	uint16_t zidx;		/* refill zone idx */
 	uint16_t safe_zidx;
 	uint16_t lowat;		/* # of buffers <= this means fl needs help */
 	int flags;
 	uint16_t buf_boundary;
 
 	/* The 16b idx all deal with hw descriptors */
 	uint16_t dbidx;		/* hw pidx after last doorbell */
 	uint16_t sidx;		/* index of status page */
 	volatile uint16_t hw_cidx;
 
 	/* The 32b idx are all buffer idx, not hardware descriptor idx */
 	uint32_t cidx;		/* consumer index */
 	uint32_t pidx;		/* producer index */
 
 	uint32_t dbval;
 	u_int rx_offset;	/* offset in fl buf (when buffer packing) */
 	volatile uint32_t *udb;
 
 	uint64_t cl_allocated;	/* # of clusters allocated */
 	uint64_t cl_recycled;	/* # of clusters recycled */
 	uint64_t cl_fast_recycled; /* # of clusters recycled (fast) */
 
 	/* These 3 are valid when FL_BUF_RESUME is set, stale otherwise. */
 	struct mbuf *m0;
 	struct mbuf **pnext;
 	u_int remaining;
 
 	uint16_t qsize;		/* # of hw descriptors (status page included) */
 	uint16_t cntxt_id;	/* SGE context id for the freelist */
 	TAILQ_ENTRY(sge_fl) link; /* All starving freelists */
 	bus_dma_tag_t desc_tag;
 	bus_dmamap_t desc_map;
 	char lockname[16];
 	bus_addr_t ba;		/* bus address of descriptor ring */
 };
 
 struct mp_ring;
 
 /* txq: SGE egress queue + what's needed for Ethernet NIC */
 struct sge_txq {
 	struct sge_eq eq;	/* MUST be first */
 
 	struct ifnet *ifp;	/* the interface this txq belongs to */
 	struct mp_ring *r;	/* tx software ring */
 	struct tx_sdesc *sdesc;	/* KVA of software descriptor ring */
 	struct sglist *gl;
 	__be32 cpl_ctrl0;	/* for convenience */
 	int tc_idx;		/* traffic class */
 
 	struct task tx_reclaim_task;
 	/* stats for common events first */
 
 	uint64_t txcsum;	/* # of times hardware assisted with checksum */
 	uint64_t tso_wrs;	/* # of TSO work requests */
 	uint64_t vlan_insertion;/* # of times VLAN tag was inserted */
 	uint64_t imm_wrs;	/* # of work requests with immediate data */
 	uint64_t sgl_wrs;	/* # of work requests with direct SGL */
 	uint64_t txpkt_wrs;	/* # of txpkt work requests (not coalesced) */
 	uint64_t txpkts0_wrs;	/* # of type0 coalesced tx work requests */
 	uint64_t txpkts1_wrs;	/* # of type1 coalesced tx work requests */
 	uint64_t txpkts0_pkts;	/* # of frames in type0 coalesced tx WRs */
 	uint64_t txpkts1_pkts;	/* # of frames in type1 coalesced tx WRs */
 	uint64_t raw_wrs;	/* # of raw work requests (alloc_wr_mbuf) */
 
 	uint64_t kern_tls_records;
 	uint64_t kern_tls_short;
 	uint64_t kern_tls_partial;
 	uint64_t kern_tls_full;
 	uint64_t kern_tls_octets;
 	uint64_t kern_tls_waste;
 	uint64_t kern_tls_options;
 	uint64_t kern_tls_header;
 	uint64_t kern_tls_fin;
 	uint64_t kern_tls_fin_short;
 	uint64_t kern_tls_cbc;
 	uint64_t kern_tls_gcm;
 
 	/* stats for not-that-common events */
 
 	/* Optional scratch space for constructing work requests. */
 	uint8_t ss[SGE_MAX_WR_LEN] __aligned(16);
 } __aligned(CACHE_LINE_SIZE);
 
 /* rxq: SGE ingress queue + SGE free list + miscellaneous items */
 struct sge_rxq {
 	struct sge_iq iq;	/* MUST be first */
 	struct sge_fl fl;	/* MUST follow iq */
 
 	struct ifnet *ifp;	/* the interface this rxq belongs to */
 #if defined(INET) || defined(INET6)
 	struct lro_ctrl lro;	/* LRO state */
 #endif
 
 	/* stats for common events first */
 
 	uint64_t rxcsum;	/* # of times hardware assisted with checksum */
 	uint64_t vlan_extraction;/* # of times VLAN tag was extracted */
 
 	/* stats for not-that-common events */
 
 } __aligned(CACHE_LINE_SIZE);
 
 static inline struct sge_rxq *
 iq_to_rxq(struct sge_iq *iq)
 {
 
 	return (__containerof(iq, struct sge_rxq, iq));
 }
 
 
 /* ofld_rxq: SGE ingress queue + SGE free list + miscellaneous items */
 struct sge_ofld_rxq {
 	struct sge_iq iq;	/* MUST be first */
 	struct sge_fl fl;	/* MUST follow iq */
 } __aligned(CACHE_LINE_SIZE);
 
 static inline struct sge_ofld_rxq *
 iq_to_ofld_rxq(struct sge_iq *iq)
 {
 
 	return (__containerof(iq, struct sge_ofld_rxq, iq));
 }
 
 struct wrqe {
 	STAILQ_ENTRY(wrqe) link;
 	struct sge_wrq *wrq;
 	int wr_len;
 	char wr[] __aligned(16);
 };
 
 struct wrq_cookie {
 	TAILQ_ENTRY(wrq_cookie) link;
 	int ndesc;
 	int pidx;
 };
 
 /*
  * wrq: SGE egress queue that is given prebuilt work requests.  Both the control
  * and offload tx queues are of this type.
  */
 struct sge_wrq {
 	struct sge_eq eq;	/* MUST be first */
 
 	struct adapter *adapter;
 	struct task wrq_tx_task;
 
 	/* Tx desc reserved but WR not "committed" yet. */
 	TAILQ_HEAD(wrq_incomplete_wrs , wrq_cookie) incomplete_wrs;
 
 	/* List of WRs ready to go out as soon as descriptors are available. */
 	STAILQ_HEAD(, wrqe) wr_list;
 	u_int nwr_pending;
 	u_int ndesc_needed;
 
 	/* stats for common events first */
 
 	uint64_t tx_wrs_direct;	/* # of WRs written directly to desc ring. */
 	uint64_t tx_wrs_ss;	/* # of WRs copied from scratch space. */
 	uint64_t tx_wrs_copied;	/* # of WRs queued and copied to desc ring. */
 
 	/* stats for not-that-common events */
 
 	/*
 	 * Scratch space for work requests that wrap around after reaching the
 	 * status page, and some information about the last WR that used it.
 	 */
 	uint16_t ss_pidx;
 	uint16_t ss_len;
 	uint8_t ss[SGE_MAX_WR_LEN];
 
 } __aligned(CACHE_LINE_SIZE);
 
 #define INVALID_NM_RXQ_CNTXT_ID ((uint16_t)(-1))
 struct sge_nm_rxq {
 	/* Items used by the driver rx ithread are in this cacheline. */
 	volatile int nm_state __aligned(CACHE_LINE_SIZE);	/* NM_OFF, NM_ON, or NM_BUSY */
 	u_int nid;		/* netmap ring # for this queue */
 	struct vi_info *vi;
 
 	struct iq_desc *iq_desc;
 	uint16_t iq_abs_id;
 	uint16_t iq_cntxt_id;
 	uint16_t iq_cidx;
 	uint16_t iq_sidx;
 	uint8_t iq_gen;
 	uint32_t fl_sidx;
 
 	/* Items used by netmap rxsync are in this cacheline. */
 	__be64  *fl_desc __aligned(CACHE_LINE_SIZE);
 	uint16_t fl_cntxt_id;
 	uint32_t fl_pidx;
 	uint32_t fl_sidx2;	/* copy of fl_sidx */
 	uint32_t fl_db_val;
 	u_int fl_db_saved;
 	u_int fl_hwidx:4;
 
 	/*
 	 * fl_cidx is used by both the ithread and rxsync, the rest are not used
 	 * in the rx fast path.
 	 */
 	uint32_t fl_cidx __aligned(CACHE_LINE_SIZE);
 
 	bus_dma_tag_t iq_desc_tag;
 	bus_dmamap_t iq_desc_map;
 	bus_addr_t iq_ba;
 	int intr_idx;
 
 	bus_dma_tag_t fl_desc_tag;
 	bus_dmamap_t fl_desc_map;
 	bus_addr_t fl_ba;
 };
 
 #define INVALID_NM_TXQ_CNTXT_ID ((u_int)(-1))
 struct sge_nm_txq {
 	struct tx_desc *desc;
 	uint16_t cidx;
 	uint16_t pidx;
 	uint16_t sidx;
 	uint16_t equiqidx;	/* EQUIQ last requested at this pidx */
 	uint16_t equeqidx;	/* EQUEQ last requested at this pidx */
 	uint16_t dbidx;		/* pidx of the most recent doorbell */
 	uint8_t doorbells;
 	volatile uint32_t *udb;
 	u_int udb_qid;
 	u_int cntxt_id;
 	__be32 cpl_ctrl0;	/* for convenience */
 	__be32 op_pkd;		/* ditto */
 	u_int nid;		/* netmap ring # for this queue */
 
 	/* infrequently used items after this */
 
 	bus_dma_tag_t desc_tag;
 	bus_dmamap_t desc_map;
 	bus_addr_t ba;
 	int iqidx;
 } __aligned(CACHE_LINE_SIZE);
 
 struct sge {
 	int nrxq;	/* total # of Ethernet rx queues */
 	int ntxq;	/* total # of Ethernet tx queues */
 	int nofldrxq;	/* total # of TOE rx queues */
 	int nofldtxq;	/* total # of TOE tx queues */
 	int nnmrxq;	/* total # of netmap rx queues */
 	int nnmtxq;	/* total # of netmap tx queues */
 	int niq;	/* total # of ingress queues */
 	int neq;	/* total # of egress queues */
 
 	struct sge_iq fwq;	/* Firmware event queue */
 	struct sge_wrq *ctrlq;	/* Control queues */
 	struct sge_txq *txq;	/* NIC tx queues */
 	struct sge_rxq *rxq;	/* NIC rx queues */
 	struct sge_wrq *ofld_txq;	/* TOE tx queues */
 	struct sge_ofld_rxq *ofld_rxq;	/* TOE rx queues */
 	struct sge_nm_txq *nm_txq;	/* netmap tx queues */
 	struct sge_nm_rxq *nm_rxq;	/* netmap rx queues */
 
 	uint16_t iq_start;	/* first cntxt_id */
 	uint16_t iq_base;	/* first abs_id */
 	int eq_start;		/* first cntxt_id */
 	int eq_base;		/* first abs_id */
 	struct sge_iq **iqmap;	/* iq->cntxt_id to iq mapping */
 	struct sge_eq **eqmap;	/* eq->cntxt_id to eq mapping */
 
 	int8_t safe_zidx;
 	struct rx_buf_info rx_buf_info[SW_ZONE_SIZES];
 };
 
 struct devnames {
 	const char *nexus_name;
 	const char *ifnet_name;
 	const char *vi_ifnet_name;
 	const char *pf03_drv_name;
 	const char *vf_nexus_name;
 	const char *vf_ifnet_name;
 };
 
 struct clip_entry;
 
 struct adapter {
 	SLIST_ENTRY(adapter) link;
 	device_t dev;
 	struct cdev *cdev;
 	const struct devnames *names;
 
 	/* PCIe register resources */
 	int regs_rid;
 	struct resource *regs_res;
 	int msix_rid;
 	struct resource *msix_res;
 	bus_space_handle_t bh;
 	bus_space_tag_t bt;
 	bus_size_t mmio_len;
 	int udbs_rid;
 	struct resource *udbs_res;
 	volatile uint8_t *udbs_base;
 
 	unsigned int pf;
 	unsigned int mbox;
 	unsigned int vpd_busy;
 	unsigned int vpd_flag;
 
 	/* Interrupt information */
 	int intr_type;
 	int intr_count;
 	struct irq {
 		struct resource *res;
 		int rid;
 		void *tag;
 		struct sge_rxq *rxq;
 		struct sge_nm_rxq *nm_rxq;
 	} __aligned(CACHE_LINE_SIZE) *irq;
 	int sge_gts_reg;
 	int sge_kdoorbell_reg;
 
 	bus_dma_tag_t dmat;	/* Parent DMA tag */
 
 	struct sge sge;
 	int lro_timeout;
 	int sc_do_rxcopy;
 
 	struct taskqueue *tq[MAX_NCHAN];	/* General purpose taskqueues */
 	struct port_info *port[MAX_NPORTS];
 	uint8_t chan_map[MAX_NCHAN];		/* channel -> port */
 
 	struct mtx clip_table_lock;
 	TAILQ_HEAD(, clip_entry) clip_table;
 	int clip_gen;
 
 	void *tom_softc;	/* (struct tom_data *) */
 	struct tom_tunables tt;
 	struct t4_offload_policy *policy;
 	struct rwlock policy_lock;
 
 	void *iwarp_softc;	/* (struct c4iw_dev *) */
 	struct iw_tunables iwt;
 	void *iscsi_ulp_softc;	/* (struct cxgbei_data *) */
 	void *ccr_softc;	/* (struct ccr_softc *) */
 	struct l2t_data *l2t;	/* L2 table */
 	struct smt_data *smt;	/* Source MAC Table */
 	struct tid_info tids;
 	vmem_t *key_map;
 	struct tls_tunables tlst;
 
 	uint8_t doorbells;
 	int offload_map;	/* ports with IFCAP_TOE enabled */
 	int active_ulds;	/* ULDs activated on this adapter */
 	int flags;
 	int debug_flags;
 
 	char ifp_lockname[16];
 	struct mtx ifp_lock;
 	struct ifnet *ifp;	/* tracer ifp */
 	struct ifmedia media;
 	int traceq;		/* iq used by all tracers, -1 if none */
 	int tracer_valid;	/* bitmap of valid tracers */
 	int tracer_enabled;	/* bitmap of enabled tracers */
 
 	char fw_version[16];
 	char tp_version[16];
 	char er_version[16];
 	char bs_version[16];
 	char cfg_file[32];
 	u_int cfcsum;
 	struct adapter_params params;
 	const struct chip_params *chip_params;
 	struct t4_virt_res vres;
 
 	uint16_t nbmcaps;
 	uint16_t linkcaps;
 	uint16_t switchcaps;
 	uint16_t niccaps;
 	uint16_t toecaps;
 	uint16_t rdmacaps;
 	uint16_t cryptocaps;
 	uint16_t iscsicaps;
 	uint16_t fcoecaps;
 
 	struct sysctl_ctx_list ctx; /* from adapter_full_init to full_uninit */
 
 	struct mtx sc_lock;
 	char lockname[16];
 
 	/* Starving free lists */
 	struct mtx sfl_lock;	/* same cache-line as sc_lock? but that's ok */
 	TAILQ_HEAD(, sge_fl) sfl;
 	struct callout sfl_callout;
 
 	struct mtx reg_lock;	/* for indirect register access */
 
 	struct memwin memwin[NUM_MEMWIN];	/* memory windows */
 
 	struct mtx tc_lock;
 	struct task tc_task;
 
 	const char *last_op;
 	const void *last_op_thr;
 	int last_op_flags;
 
 	int swintr;
 	int sensor_resets;
 
 	struct callout ktls_tick;
 };
 
 #define ADAPTER_LOCK(sc)		mtx_lock(&(sc)->sc_lock)
 #define ADAPTER_UNLOCK(sc)		mtx_unlock(&(sc)->sc_lock)
 #define ADAPTER_LOCK_ASSERT_OWNED(sc)	mtx_assert(&(sc)->sc_lock, MA_OWNED)
 #define ADAPTER_LOCK_ASSERT_NOTOWNED(sc) mtx_assert(&(sc)->sc_lock, MA_NOTOWNED)
 
 #define ASSERT_SYNCHRONIZED_OP(sc)	\
     KASSERT(IS_BUSY(sc) && \
 	(mtx_owned(&(sc)->sc_lock) || sc->last_op_thr == curthread), \
 	("%s: operation not synchronized.", __func__))
 
 #define PORT_LOCK(pi)			mtx_lock(&(pi)->pi_lock)
 #define PORT_UNLOCK(pi)			mtx_unlock(&(pi)->pi_lock)
 #define PORT_LOCK_ASSERT_OWNED(pi)	mtx_assert(&(pi)->pi_lock, MA_OWNED)
 #define PORT_LOCK_ASSERT_NOTOWNED(pi)	mtx_assert(&(pi)->pi_lock, MA_NOTOWNED)
 
 #define FL_LOCK(fl)			mtx_lock(&(fl)->fl_lock)
 #define FL_TRYLOCK(fl)			mtx_trylock(&(fl)->fl_lock)
 #define FL_UNLOCK(fl)			mtx_unlock(&(fl)->fl_lock)
 #define FL_LOCK_ASSERT_OWNED(fl)	mtx_assert(&(fl)->fl_lock, MA_OWNED)
 #define FL_LOCK_ASSERT_NOTOWNED(fl)	mtx_assert(&(fl)->fl_lock, MA_NOTOWNED)
 
 #define RXQ_FL_LOCK(rxq)		FL_LOCK(&(rxq)->fl)
 #define RXQ_FL_UNLOCK(rxq)		FL_UNLOCK(&(rxq)->fl)
 #define RXQ_FL_LOCK_ASSERT_OWNED(rxq)	FL_LOCK_ASSERT_OWNED(&(rxq)->fl)
 #define RXQ_FL_LOCK_ASSERT_NOTOWNED(rxq) FL_LOCK_ASSERT_NOTOWNED(&(rxq)->fl)
 
 #define EQ_LOCK(eq)			mtx_lock(&(eq)->eq_lock)
 #define EQ_TRYLOCK(eq)			mtx_trylock(&(eq)->eq_lock)
 #define EQ_UNLOCK(eq)			mtx_unlock(&(eq)->eq_lock)
 #define EQ_LOCK_ASSERT_OWNED(eq)	mtx_assert(&(eq)->eq_lock, MA_OWNED)
 #define EQ_LOCK_ASSERT_NOTOWNED(eq)	mtx_assert(&(eq)->eq_lock, MA_NOTOWNED)
 
 #define TXQ_LOCK(txq)			EQ_LOCK(&(txq)->eq)
 #define TXQ_TRYLOCK(txq)		EQ_TRYLOCK(&(txq)->eq)
 #define TXQ_UNLOCK(txq)			EQ_UNLOCK(&(txq)->eq)
 #define TXQ_LOCK_ASSERT_OWNED(txq)	EQ_LOCK_ASSERT_OWNED(&(txq)->eq)
 #define TXQ_LOCK_ASSERT_NOTOWNED(txq)	EQ_LOCK_ASSERT_NOTOWNED(&(txq)->eq)
 
 #define for_each_txq(vi, iter, q) \
 	for (q = &vi->pi->adapter->sge.txq[vi->first_txq], iter = 0; \
 	    iter < vi->ntxq; ++iter, ++q)
 #define for_each_rxq(vi, iter, q) \
 	for (q = &vi->pi->adapter->sge.rxq[vi->first_rxq], iter = 0; \
 	    iter < vi->nrxq; ++iter, ++q)
 #define for_each_ofld_txq(vi, iter, q) \
 	for (q = &vi->pi->adapter->sge.ofld_txq[vi->first_ofld_txq], iter = 0; \
 	    iter < vi->nofldtxq; ++iter, ++q)
 #define for_each_ofld_rxq(vi, iter, q) \
 	for (q = &vi->pi->adapter->sge.ofld_rxq[vi->first_ofld_rxq], iter = 0; \
 	    iter < vi->nofldrxq; ++iter, ++q)
 #define for_each_nm_txq(vi, iter, q) \
 	for (q = &vi->pi->adapter->sge.nm_txq[vi->first_nm_txq], iter = 0; \
 	    iter < vi->nnmtxq; ++iter, ++q)
 #define for_each_nm_rxq(vi, iter, q) \
 	for (q = &vi->pi->adapter->sge.nm_rxq[vi->first_nm_rxq], iter = 0; \
 	    iter < vi->nnmrxq; ++iter, ++q)
 #define for_each_vi(_pi, _iter, _vi) \
 	for ((_vi) = (_pi)->vi, (_iter) = 0; (_iter) < (_pi)->nvi; \
 	     ++(_iter), ++(_vi))
 
 #define IDXINCR(idx, incr, wrap) do { \
 	idx = wrap - idx > incr ? idx + incr : incr - (wrap - idx); \
 } while (0)
 #define IDXDIFF(head, tail, wrap) \
 	((head) >= (tail) ? (head) - (tail) : (wrap) - (tail) + (head))
 
 /* One for errors, one for firmware events */
 #define T4_EXTRA_INTR 2
 
 /* One for firmware events */
 #define T4VF_EXTRA_INTR 1
 
 static inline int
 forwarding_intr_to_fwq(struct adapter *sc)
 {
 
 	return (sc->intr_count == 1);
 }
 
 static inline uint32_t
 t4_read_reg(struct adapter *sc, uint32_t reg)
 {
 
 	return bus_space_read_4(sc->bt, sc->bh, reg);
 }
 
 static inline void
 t4_write_reg(struct adapter *sc, uint32_t reg, uint32_t val)
 {
 
 	bus_space_write_4(sc->bt, sc->bh, reg, val);
 }
 
 static inline uint64_t
 t4_read_reg64(struct adapter *sc, uint32_t reg)
 {
 
 #ifdef __LP64__
 	return bus_space_read_8(sc->bt, sc->bh, reg);
 #else
 	return (uint64_t)bus_space_read_4(sc->bt, sc->bh, reg) +
 	    ((uint64_t)bus_space_read_4(sc->bt, sc->bh, reg + 4) << 32);
 
 #endif
 }
 
 static inline void
 t4_write_reg64(struct adapter *sc, uint32_t reg, uint64_t val)
 {
 
 #ifdef __LP64__
 	bus_space_write_8(sc->bt, sc->bh, reg, val);
 #else
 	bus_space_write_4(sc->bt, sc->bh, reg, val);
 	bus_space_write_4(sc->bt, sc->bh, reg + 4, val>> 32);
 #endif
 }
 
 static inline void
 t4_os_pci_read_cfg1(struct adapter *sc, int reg, uint8_t *val)
 {
 
 	*val = pci_read_config(sc->dev, reg, 1);
 }
 
 static inline void
 t4_os_pci_write_cfg1(struct adapter *sc, int reg, uint8_t val)
 {
 
 	pci_write_config(sc->dev, reg, val, 1);
 }
 
 static inline void
 t4_os_pci_read_cfg2(struct adapter *sc, int reg, uint16_t *val)
 {
 
 	*val = pci_read_config(sc->dev, reg, 2);
 }
 
 static inline void
 t4_os_pci_write_cfg2(struct adapter *sc, int reg, uint16_t val)
 {
 
 	pci_write_config(sc->dev, reg, val, 2);
 }
 
 static inline void
 t4_os_pci_read_cfg4(struct adapter *sc, int reg, uint32_t *val)
 {
 
 	*val = pci_read_config(sc->dev, reg, 4);
 }
 
 static inline void
 t4_os_pci_write_cfg4(struct adapter *sc, int reg, uint32_t val)
 {
 
 	pci_write_config(sc->dev, reg, val, 4);
 }
 
 static inline struct port_info *
 adap2pinfo(struct adapter *sc, int idx)
 {
 
 	return (sc->port[idx]);
 }
 
 static inline void
 t4_os_set_hw_addr(struct port_info *pi, uint8_t hw_addr[])
 {
 
 	bcopy(hw_addr, pi->vi[0].hw_addr, ETHER_ADDR_LEN);
 }
 
 static inline int
 tx_resume_threshold(struct sge_eq *eq)
 {
 
 	/* not quite the same as qsize / 4, but this will do. */
 	return (eq->sidx / 4);
 }
 
 static inline int
 t4_use_ldst(struct adapter *sc)
 {
 
 #ifdef notyet
 	return (sc->flags & FW_OK || !sc->use_bd);
 #else
 	return (0);
 #endif
 }
 
 static inline void
 CH_DUMP_MBOX(struct adapter *sc, int mbox, const int reg,
     const char *msg, const __be64 *const p, const bool err)
 {
 
 	if (!(sc->debug_flags & DF_DUMP_MBOX) && !err)
 		return;
 	if (p != NULL) {
 		log(err ? LOG_ERR : LOG_DEBUG,
 		    "%s: mbox %u %s %016llx %016llx %016llx %016llx "
 		    "%016llx %016llx %016llx %016llx\n",
 		    device_get_nameunit(sc->dev), mbox, msg,
 		    (long long)be64_to_cpu(p[0]), (long long)be64_to_cpu(p[1]),
 		    (long long)be64_to_cpu(p[2]), (long long)be64_to_cpu(p[3]),
 		    (long long)be64_to_cpu(p[4]), (long long)be64_to_cpu(p[5]),
 		    (long long)be64_to_cpu(p[6]), (long long)be64_to_cpu(p[7]));
 	} else {
 		log(err ? LOG_ERR : LOG_DEBUG,
 		    "%s: mbox %u %s %016llx %016llx %016llx %016llx "
 		    "%016llx %016llx %016llx %016llx\n",
 		    device_get_nameunit(sc->dev), mbox, msg,
 		    (long long)t4_read_reg64(sc, reg),
 		    (long long)t4_read_reg64(sc, reg + 8),
 		    (long long)t4_read_reg64(sc, reg + 16),
 		    (long long)t4_read_reg64(sc, reg + 24),
 		    (long long)t4_read_reg64(sc, reg + 32),
 		    (long long)t4_read_reg64(sc, reg + 40),
 		    (long long)t4_read_reg64(sc, reg + 48),
 		    (long long)t4_read_reg64(sc, reg + 56));
 	}
 }
 
 /* t4_main.c */
 extern int t4_ntxq;
 extern int t4_nrxq;
 extern int t4_intr_types;
 extern int t4_tmr_idx;
 extern int t4_pktc_idx;
 extern unsigned int t4_qsize_rxq;
 extern unsigned int t4_qsize_txq;
 extern device_method_t cxgbe_methods[];
 
 int t4_os_find_pci_capability(struct adapter *, int);
 int t4_os_pci_save_state(struct adapter *);
 int t4_os_pci_restore_state(struct adapter *);
 void t4_os_portmod_changed(struct port_info *);
 void t4_os_link_changed(struct port_info *);
 void t4_iterate(void (*)(struct adapter *, void *), void *);
 void t4_init_devnames(struct adapter *);
 void t4_add_adapter(struct adapter *);
 int t4_detach_common(device_t);
 int t4_map_bars_0_and_4(struct adapter *);
 int t4_map_bar_2(struct adapter *);
 int t4_setup_intr_handlers(struct adapter *);
 void t4_sysctls(struct adapter *);
 int begin_synchronized_op(struct adapter *, struct vi_info *, int, char *);
 void doom_vi(struct adapter *, struct vi_info *);
 void end_synchronized_op(struct adapter *, int);
 int update_mac_settings(struct ifnet *, int);
 int adapter_full_init(struct adapter *);
 int adapter_full_uninit(struct adapter *);
 uint64_t cxgbe_get_counter(struct ifnet *, ift_counter);
 void cxgbe_snd_tag_init(struct cxgbe_snd_tag *, struct ifnet *, int);
 int vi_full_init(struct vi_info *);
 int vi_full_uninit(struct vi_info *);
 void vi_sysctls(struct vi_info *);
 void vi_tick(void *);
 int rw_via_memwin(struct adapter *, int, uint32_t, uint32_t *, int, int);
 int alloc_atid(struct adapter *, void *);
 void *lookup_atid(struct adapter *, int);
 void free_atid(struct adapter *, int);
 void release_tid(struct adapter *, int, struct sge_wrq *);
 int cxgbe_media_change(struct ifnet *);
 void cxgbe_media_status(struct ifnet *, struct ifmediareq *);
 bool t4_os_dump_cimla(struct adapter *, int, bool);
 void t4_os_dump_devlog(struct adapter *);
 
 #ifdef KERN_TLS
 /* t4_kern_tls.c */
 int cxgbe_tls_tag_alloc(struct ifnet *, union if_snd_tag_alloc_params *,
     struct m_snd_tag **);
 void cxgbe_tls_tag_free(struct m_snd_tag *);
 void t6_ktls_modload(void);
 void t6_ktls_modunload(void);
 int t6_ktls_try(struct ifnet *, struct socket *, struct ktls_session *);
 int t6_ktls_parse_pkt(struct mbuf *, int *, int *);
 int t6_ktls_write_wr(struct sge_txq *, void *, struct mbuf *, u_int, u_int);
 #endif
 
 /* t4_keyctx.c */
 struct auth_hash;
 union authctx;
 
 void t4_aes_getdeckey(void *, const void *, unsigned int);
 void t4_copy_partial_hash(int, union authctx *, void *);
 void t4_init_gmac_hash(const char *, int, char *);
-void t4_init_hmac_digest(struct auth_hash *, u_int, char *, int, char *);
+void t4_init_hmac_digest(struct auth_hash *, u_int, const char *, int, char *);
 
 #ifdef DEV_NETMAP
 /* t4_netmap.c */
 struct sge_nm_rxq;
 void cxgbe_nm_attach(struct vi_info *);
 void cxgbe_nm_detach(struct vi_info *);
 void service_nm_rxq(struct sge_nm_rxq *);
 #endif
 
 /* t4_sge.c */
 void t4_sge_modload(void);
 void t4_sge_modunload(void);
 uint64_t t4_sge_extfree_refs(void);
 void t4_tweak_chip_settings(struct adapter *);
 int t4_read_chip_settings(struct adapter *);
 int t4_create_dma_tag(struct adapter *);
 void t4_sge_sysctls(struct adapter *, struct sysctl_ctx_list *,
     struct sysctl_oid_list *);
 int t4_destroy_dma_tag(struct adapter *);
 int t4_setup_adapter_queues(struct adapter *);
 int t4_teardown_adapter_queues(struct adapter *);
 int t4_setup_vi_queues(struct vi_info *);
 int t4_teardown_vi_queues(struct vi_info *);
 void t4_intr_all(void *);
 void t4_intr(void *);
 #ifdef DEV_NETMAP
 void t4_nm_intr(void *);
 void t4_vi_intr(void *);
 #endif
 void t4_intr_err(void *);
 void t4_intr_evt(void *);
 void t4_wrq_tx_locked(struct adapter *, struct sge_wrq *, struct wrqe *);
 void t4_update_fl_bufsize(struct ifnet *);
 struct mbuf *alloc_wr_mbuf(int, int);
 int parse_pkt(struct adapter *, struct mbuf **);
 void *start_wrq_wr(struct sge_wrq *, int, struct wrq_cookie *);
 void commit_wrq_wr(struct sge_wrq *, void *, struct wrq_cookie *);
 int tnl_cong(struct port_info *, int);
 void t4_register_an_handler(an_handler_t);
 void t4_register_fw_msg_handler(int, fw_msg_handler_t);
 void t4_register_cpl_handler(int, cpl_handler_t);
 void t4_register_shared_cpl_handler(int, cpl_handler_t, int);
 #ifdef RATELIMIT
 int ethofld_transmit(struct ifnet *, struct mbuf *);
 void send_etid_flush_wr(struct cxgbe_rate_tag *);
 #endif
 
 /* t4_tracer.c */
 struct t4_tracer;
 void t4_tracer_modload(void);
 void t4_tracer_modunload(void);
 void t4_tracer_port_detach(struct adapter *);
 int t4_get_tracer(struct adapter *, struct t4_tracer *);
 int t4_set_tracer(struct adapter *, struct t4_tracer *);
 int t4_trace_pkt(struct sge_iq *, const struct rss_header *, struct mbuf *);
 int t5_trace_pkt(struct sge_iq *, const struct rss_header *, struct mbuf *);
 
 /* t4_sched.c */
 int t4_set_sched_class(struct adapter *, struct t4_sched_params *);
 int t4_set_sched_queue(struct adapter *, struct t4_sched_queue *);
 int t4_init_tx_sched(struct adapter *);
 int t4_free_tx_sched(struct adapter *);
 void t4_update_tx_sched(struct adapter *);
 int t4_reserve_cl_rl_kbps(struct adapter *, int, u_int, int *);
 void t4_release_cl_rl(struct adapter *, int, int);
 int sysctl_tc(SYSCTL_HANDLER_ARGS);
 int sysctl_tc_params(SYSCTL_HANDLER_ARGS);
 #ifdef RATELIMIT
 void t4_init_etid_table(struct adapter *);
 void t4_free_etid_table(struct adapter *);
 struct cxgbe_rate_tag *lookup_etid(struct adapter *, int);
 int cxgbe_rate_tag_alloc(struct ifnet *, union if_snd_tag_alloc_params *,
     struct m_snd_tag **);
 int cxgbe_rate_tag_modify(struct m_snd_tag *, union if_snd_tag_modify_params *);
 int cxgbe_rate_tag_query(struct m_snd_tag *, union if_snd_tag_query_params *);
 void cxgbe_rate_tag_free(struct m_snd_tag *);
 void cxgbe_rate_tag_free_locked(struct cxgbe_rate_tag *);
 void cxgbe_ratelimit_query(struct ifnet *, struct if_ratelimit_query_results *);
 #endif
 
 /* t4_filter.c */
 int get_filter_mode(struct adapter *, uint32_t *);
 int set_filter_mode(struct adapter *, uint32_t);
 int get_filter(struct adapter *, struct t4_filter *);
 int set_filter(struct adapter *, struct t4_filter *);
 int del_filter(struct adapter *, struct t4_filter *);
 int t4_filter_rpl(struct sge_iq *, const struct rss_header *, struct mbuf *);
 int t4_hashfilter_ao_rpl(struct sge_iq *, const struct rss_header *, struct mbuf *);
 int t4_hashfilter_tcb_rpl(struct sge_iq *, const struct rss_header *, struct mbuf *);
 int t4_del_hashfilter_rpl(struct sge_iq *, const struct rss_header *, struct mbuf *);
 void free_hftid_hash(struct tid_info *);
 
 static inline struct wrqe *
 alloc_wrqe(int wr_len, struct sge_wrq *wrq)
 {
 	int len = offsetof(struct wrqe, wr) + wr_len;
 	struct wrqe *wr;
 
 	wr = malloc(len, M_CXGBE, M_NOWAIT);
 	if (__predict_false(wr == NULL))
 		return (NULL);
 	wr->wr_len = wr_len;
 	wr->wrq = wrq;
 	return (wr);
 }
 
 static inline void *
 wrtod(struct wrqe *wr)
 {
 	return (&wr->wr[0]);
 }
 
 static inline void
 free_wrqe(struct wrqe *wr)
 {
 	free(wr, M_CXGBE);
 }
 
 static inline void
 t4_wrq_tx(struct adapter *sc, struct wrqe *wr)
 {
 	struct sge_wrq *wrq = wr->wrq;
 
 	TXQ_LOCK(wrq);
 	t4_wrq_tx_locked(sc, wrq, wr);
 	TXQ_UNLOCK(wrq);
 }
 
 static inline int
 read_via_memwin(struct adapter *sc, int idx, uint32_t addr, uint32_t *val,
     int len)
 {
 
 	return (rw_via_memwin(sc, idx, addr, val, len, 0));
 }
 
 static inline int
 write_via_memwin(struct adapter *sc, int idx, uint32_t addr,
     const uint32_t *val, int len)
 {
 
 	return (rw_via_memwin(sc, idx, addr, (void *)(uintptr_t)val, len, 1));
 }
 #endif
Index: head/sys/dev/cxgbe/crypto/t4_crypto.c
===================================================================
--- head/sys/dev/cxgbe/crypto/t4_crypto.c	(revision 359373)
+++ head/sys/dev/cxgbe/crypto/t4_crypto.c	(revision 359374)
@@ -1,2856 +1,2735 @@
 /*-
  * Copyright (c) 2017 Chelsio Communications, Inc.
  * All rights reserved.
  * Written by: John Baldwin <jhb@FreeBSD.org>
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include <sys/types.h>
 #include <sys/bus.h>
 #include <sys/lock.h>
 #include <sys/malloc.h>
 #include <sys/mutex.h>
 #include <sys/module.h>
 #include <sys/sglist.h>
 
 #include <opencrypto/cryptodev.h>
 #include <opencrypto/xform.h>
 
 #include "cryptodev_if.h"
 
 #include "common/common.h"
 #include "crypto/t4_crypto.h"
 
 /*
  * Requests consist of:
  *
  * +-------------------------------+
  * | struct fw_crypto_lookaside_wr |
  * +-------------------------------+
  * | struct ulp_txpkt              |
  * +-------------------------------+
  * | struct ulptx_idata            |
  * +-------------------------------+
  * | struct cpl_tx_sec_pdu         |
  * +-------------------------------+
  * | struct cpl_tls_tx_scmd_fmt    |
  * +-------------------------------+
  * | key context header            |
  * +-------------------------------+
  * | AES key                       |  ----- For requests with AES
  * +-------------------------------+
  * | Hash state                    |  ----- For hash-only requests
  * +-------------------------------+ -
  * | IPAD (16-byte aligned)        |  \
  * +-------------------------------+  +---- For requests with HMAC
  * | OPAD (16-byte aligned)        |  /
  * +-------------------------------+ -
  * | GMAC H                        |  ----- For AES-GCM
  * +-------------------------------+ -
  * | struct cpl_rx_phys_dsgl       |  \
  * +-------------------------------+  +---- Destination buffer for
  * | PHYS_DSGL entries             |  /     non-hash-only requests
  * +-------------------------------+ -
  * | 16 dummy bytes                |  ----- Only for HMAC/hash-only requests
  * +-------------------------------+
  * | IV                            |  ----- If immediate IV
  * +-------------------------------+
  * | Payload                       |  ----- If immediate Payload
  * +-------------------------------+ -
  * | struct ulptx_sgl              |  \
  * +-------------------------------+  +---- If payload via SGL
  * | SGL entries                   |  /
  * +-------------------------------+ -
  *
  * Note that the key context must be padded to ensure 16-byte alignment.
  * For HMAC requests, the key consists of the partial hash of the IPAD
  * followed by the partial hash of the OPAD.
  *
  * Replies consist of:
  *
  * +-------------------------------+
  * | struct cpl_fw6_pld            |
  * +-------------------------------+
  * | hash digest                   |  ----- For HMAC request with
  * +-------------------------------+        'hash_size' set in work request
  *
  * A 32-bit big-endian error status word is supplied in the last 4
  * bytes of data[0] in the CPL_FW6_PLD message.  bit 0 indicates a
  * "MAC" error and bit 1 indicates a "PAD" error.
  *
  * The 64-bit 'cookie' field from the fw_crypto_lookaside_wr message
  * in the request is returned in data[1] of the CPL_FW6_PLD message.
  *
  * For block cipher replies, the updated IV is supplied in data[2] and
  * data[3] of the CPL_FW6_PLD message.
  *
  * For hash replies where the work request set 'hash_size' to request
  * a copy of the hash in the reply, the hash digest is supplied
  * immediately following the CPL_FW6_PLD message.
  */
 
 /*
  * The crypto engine supports a maximum AAD size of 511 bytes.
  */
 #define	MAX_AAD_LEN		511
 
 /*
  * The documentation for CPL_RX_PHYS_DSGL claims a maximum of 32 SG
  * entries.  While the CPL includes a 16-bit length field, the T6 can
  * sometimes hang if an error occurs while processing a request with a
  * single DSGL entry larger than 2k.
  */
 #define	MAX_RX_PHYS_DSGL_SGE	32
 #define	DSGL_SGE_MAXLEN		2048
 
 /*
  * The adapter only supports requests with a total input or output
  * length of 64k-1 or smaller.  Longer requests either result in hung
  * requests or incorrect results.
  */
 #define	MAX_REQUEST_SIZE	65535
 
 static MALLOC_DEFINE(M_CCR, "ccr", "Chelsio T6 crypto");
 
 struct ccr_session_hmac {
 	struct auth_hash *auth_hash;
 	int hash_len;
 	unsigned int partial_digest_len;
 	unsigned int auth_mode;
 	unsigned int mk_size;
 	char pads[CHCR_HASH_MAX_BLOCK_SIZE_128 * 2];
 };
 
 struct ccr_session_gmac {
 	int hash_len;
 	char ghash_h[GMAC_BLOCK_LEN];
 };
 
 struct ccr_session_ccm_mac {
 	int hash_len;
 };
 
 struct ccr_session_blkcipher {
 	unsigned int cipher_mode;
 	unsigned int key_len;
 	unsigned int iv_len;
 	__be32 key_ctx_hdr;
 	char enckey[CHCR_AES_MAX_KEY_LEN];
 	char deckey[CHCR_AES_MAX_KEY_LEN];
 };
 
 struct ccr_session {
 	bool active;
 	int pending;
-	enum { HASH, HMAC, BLKCIPHER, AUTHENC, GCM, CCM } mode;
+	enum { HASH, HMAC, BLKCIPHER, ETA, GCM, CCM } mode;
 	union {
 		struct ccr_session_hmac hmac;
 		struct ccr_session_gmac gmac;
 		struct ccr_session_ccm_mac ccm_mac;
 	};
 	struct ccr_session_blkcipher blkcipher;
 };
 
 struct ccr_softc {
 	struct adapter *adapter;
 	device_t dev;
 	uint32_t cid;
 	int tx_channel_id;
 	struct mtx lock;
 	bool detaching;
 	struct sge_wrq *txq;
 	struct sge_rxq *rxq;
 
 	/*
 	 * Pre-allocate S/G lists used when preparing a work request.
 	 * 'sg_crp' contains an sglist describing the entire buffer
 	 * for a 'struct cryptop'.  'sg_ulptx' is used to describe
 	 * the data the engine should DMA as input via ULPTX_SGL.
 	 * 'sg_dsgl' is used to describe the destination that cipher
 	 * text and a tag should be written to.
 	 */
 	struct sglist *sg_crp;
 	struct sglist *sg_ulptx;
 	struct sglist *sg_dsgl;
 
 	/*
 	 * Pre-allocate a dummy output buffer for the IV and AAD for
 	 * AEAD requests.
 	 */
 	char *iv_aad_buf;
 	struct sglist *sg_iv_aad;
 
 	/* Statistics. */
 	uint64_t stats_blkcipher_encrypt;
 	uint64_t stats_blkcipher_decrypt;
 	uint64_t stats_hash;
 	uint64_t stats_hmac;
-	uint64_t stats_authenc_encrypt;
-	uint64_t stats_authenc_decrypt;
+	uint64_t stats_eta_encrypt;
+	uint64_t stats_eta_decrypt;
 	uint64_t stats_gcm_encrypt;
 	uint64_t stats_gcm_decrypt;
 	uint64_t stats_ccm_encrypt;
 	uint64_t stats_ccm_decrypt;
 	uint64_t stats_wr_nomem;
 	uint64_t stats_inflight;
 	uint64_t stats_mac_error;
 	uint64_t stats_pad_error;
 	uint64_t stats_bad_session;
 	uint64_t stats_sglist_error;
 	uint64_t stats_process_error;
 	uint64_t stats_sw_fallback;
 };
 
 /*
  * Crypto requests involve two kind of scatter/gather lists.
  *
  * Non-hash-only requests require a PHYS_DSGL that describes the
  * location to store the results of the encryption or decryption
  * operation.  This SGL uses a different format (PHYS_DSGL) and should
- * exclude the crd_skip bytes at the start of the data as well as
- * any AAD or IV.  For authenticated encryption requests it should
- * cover include the destination of the hash or tag.
+ * exclude the skip bytes at the start of the data as well as any AAD
+ * or IV.  For authenticated encryption requests it should include the
+ * destination of the hash or tag.
  *
  * The input payload may either be supplied inline as immediate data,
  * or via a standard ULP_TX SGL.  This SGL should include AAD,
  * ciphertext, and the hash or tag for authenticated decryption
  * requests.
  *
  * These scatter/gather lists can describe different subsets of the
  * buffer described by the crypto operation.  ccr_populate_sglist()
  * generates a scatter/gather list that covers the entire crypto
  * operation buffer that is then used to construct the other
  * scatter/gather lists.
  */
 static int
 ccr_populate_sglist(struct sglist *sg, struct cryptop *crp)
 {
 	int error;
 
 	sglist_reset(sg);
-	if (crp->crp_flags & CRYPTO_F_IMBUF)
-		error = sglist_append_mbuf(sg, (struct mbuf *)crp->crp_buf);
-	else if (crp->crp_flags & CRYPTO_F_IOV)
-		error = sglist_append_uio(sg, (struct uio *)crp->crp_buf);
-	else
+	switch (crp->crp_buf_type) {
+	case CRYPTO_BUF_MBUF:
+		error = sglist_append_mbuf(sg, crp->crp_mbuf);
+		break;
+	case CRYPTO_BUF_UIO:
+		error = sglist_append_uio(sg, crp->crp_uio);
+		break;
+	case CRYPTO_BUF_CONTIG:
 		error = sglist_append(sg, crp->crp_buf, crp->crp_ilen);
+		break;
+	default:
+		error = EINVAL;
+	}
 	return (error);
 }
 
 /*
  * Segments in 'sg' larger than 'maxsegsize' are counted as multiple
  * segments.
  */
 static int
 ccr_count_sgl(struct sglist *sg, int maxsegsize)
 {
 	int i, nsegs;
 
 	nsegs = 0;
 	for (i = 0; i < sg->sg_nseg; i++)
 		nsegs += howmany(sg->sg_segs[i].ss_len, maxsegsize);
 	return (nsegs);
 }
 
 /* These functions deal with PHYS_DSGL for the reply buffer. */
 static inline int
 ccr_phys_dsgl_len(int nsegs)
 {
 	int len;
 
 	len = (nsegs / 8) * sizeof(struct phys_sge_pairs);
 	if ((nsegs % 8) != 0) {
 		len += sizeof(uint16_t) * 8;
 		len += roundup2(nsegs % 8, 2) * sizeof(uint64_t);
 	}
 	return (len);
 }
 
 static void
 ccr_write_phys_dsgl(struct ccr_softc *sc, void *dst, int nsegs)
 {
 	struct sglist *sg;
 	struct cpl_rx_phys_dsgl *cpl;
 	struct phys_sge_pairs *sgl;
 	vm_paddr_t paddr;
 	size_t seglen;
 	u_int i, j;
 
 	sg = sc->sg_dsgl;
 	cpl = dst;
 	cpl->op_to_tid = htobe32(V_CPL_RX_PHYS_DSGL_OPCODE(CPL_RX_PHYS_DSGL) |
 	    V_CPL_RX_PHYS_DSGL_ISRDMA(0));
 	cpl->pcirlxorder_to_noofsgentr = htobe32(
 	    V_CPL_RX_PHYS_DSGL_PCIRLXORDER(0) |
 	    V_CPL_RX_PHYS_DSGL_PCINOSNOOP(0) |
 	    V_CPL_RX_PHYS_DSGL_PCITPHNTENB(0) | V_CPL_RX_PHYS_DSGL_DCAID(0) |
 	    V_CPL_RX_PHYS_DSGL_NOOFSGENTR(nsegs));
 	cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR;
 	cpl->rss_hdr_int.qid = htobe16(sc->rxq->iq.abs_id);
 	cpl->rss_hdr_int.hash_val = 0;
 	sgl = (struct phys_sge_pairs *)(cpl + 1);
 	j = 0;
 	for (i = 0; i < sg->sg_nseg; i++) {
 		seglen = sg->sg_segs[i].ss_len;
 		paddr = sg->sg_segs[i].ss_paddr;
 		do {
 			sgl->addr[j] = htobe64(paddr);
 			if (seglen > DSGL_SGE_MAXLEN) {
 				sgl->len[j] = htobe16(DSGL_SGE_MAXLEN);
 				paddr += DSGL_SGE_MAXLEN;
 				seglen -= DSGL_SGE_MAXLEN;
 			} else {
 				sgl->len[j] = htobe16(seglen);
 				seglen = 0;
 			}
 			j++;
 			if (j == 8) {
 				sgl++;
 				j = 0;
 			}
 		} while (seglen != 0);
 	}
 	MPASS(j + 8 * (sgl - (struct phys_sge_pairs *)(cpl + 1)) == nsegs);
 }
 
 /* These functions deal with the ULPTX_SGL for input payload. */
 static inline int
 ccr_ulptx_sgl_len(int nsegs)
 {
 	u_int n;
 
 	nsegs--; /* first segment is part of ulptx_sgl */
 	n = sizeof(struct ulptx_sgl) + 8 * ((3 * nsegs) / 2 + (nsegs & 1));
 	return (roundup2(n, 16));
 }
 
 static void
 ccr_write_ulptx_sgl(struct ccr_softc *sc, void *dst, int nsegs)
 {
 	struct ulptx_sgl *usgl;
 	struct sglist *sg;
 	struct sglist_seg *ss;
 	int i;
 
 	sg = sc->sg_ulptx;
 	MPASS(nsegs == sg->sg_nseg);
 	ss = &sg->sg_segs[0];
 	usgl = dst;
 	usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) |
 	    V_ULPTX_NSGE(nsegs));
 	usgl->len0 = htobe32(ss->ss_len);
 	usgl->addr0 = htobe64(ss->ss_paddr);
 	ss++;
 	for (i = 0; i < sg->sg_nseg - 1; i++) {
 		usgl->sge[i / 2].len[i & 1] = htobe32(ss->ss_len);
 		usgl->sge[i / 2].addr[i & 1] = htobe64(ss->ss_paddr);
 		ss++;
 	}
 	
 }
 
 static bool
 ccr_use_imm_data(u_int transhdr_len, u_int input_len)
 {
 
 	if (input_len > CRYPTO_MAX_IMM_TX_PKT_LEN)
 		return (false);
 	if (roundup2(transhdr_len, 16) + roundup2(input_len, 16) >
 	    SGE_MAX_WR_LEN)
 		return (false);
 	return (true);
 }
 
 static void
 ccr_populate_wreq(struct ccr_softc *sc, struct chcr_wr *crwr, u_int kctx_len,
     u_int wr_len, u_int imm_len, u_int sgl_len, u_int hash_size,
     struct cryptop *crp)
 {
 	u_int cctx_size, idata_len;
 
 	cctx_size = sizeof(struct _key_ctx) + kctx_len;
 	crwr->wreq.op_to_cctx_size = htobe32(
 	    V_FW_CRYPTO_LOOKASIDE_WR_OPCODE(FW_CRYPTO_LOOKASIDE_WR) |
 	    V_FW_CRYPTO_LOOKASIDE_WR_COMPL(0) |
 	    V_FW_CRYPTO_LOOKASIDE_WR_IMM_LEN(imm_len) |
 	    V_FW_CRYPTO_LOOKASIDE_WR_CCTX_LOC(1) |
 	    V_FW_CRYPTO_LOOKASIDE_WR_CCTX_SIZE(cctx_size >> 4));
 	crwr->wreq.len16_pkd = htobe32(
 	    V_FW_CRYPTO_LOOKASIDE_WR_LEN16(wr_len / 16));
 	crwr->wreq.session_id = 0;
 	crwr->wreq.rx_chid_to_rx_q_id = htobe32(
 	    V_FW_CRYPTO_LOOKASIDE_WR_RX_CHID(sc->tx_channel_id) |
 	    V_FW_CRYPTO_LOOKASIDE_WR_LCB(0) |
 	    V_FW_CRYPTO_LOOKASIDE_WR_PHASH(0) |
 	    V_FW_CRYPTO_LOOKASIDE_WR_IV(IV_NOP) |
 	    V_FW_CRYPTO_LOOKASIDE_WR_FQIDX(0) |
 	    V_FW_CRYPTO_LOOKASIDE_WR_TX_CH(0) |
 	    V_FW_CRYPTO_LOOKASIDE_WR_RX_Q_ID(sc->rxq->iq.abs_id));
 	crwr->wreq.key_addr = 0;
 	crwr->wreq.pld_size_hash_size = htobe32(
 	    V_FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE(sgl_len) |
 	    V_FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE(hash_size));
 	crwr->wreq.cookie = htobe64((uintptr_t)crp);
 
 	crwr->ulptx.cmd_dest = htobe32(V_ULPTX_CMD(ULP_TX_PKT) |
 	    V_ULP_TXPKT_DATAMODIFY(0) |
 	    V_ULP_TXPKT_CHANNELID(sc->tx_channel_id) | V_ULP_TXPKT_DEST(0) |
 	    V_ULP_TXPKT_FID(sc->rxq->iq.abs_id) | V_ULP_TXPKT_RO(1));
 	crwr->ulptx.len = htobe32(
 	    ((wr_len - sizeof(struct fw_crypto_lookaside_wr)) / 16));
 
 	crwr->sc_imm.cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM) |
 	    V_ULP_TX_SC_MORE(sgl_len != 0 ? 1 : 0));
 	idata_len = wr_len - offsetof(struct chcr_wr, sec_cpl) - sgl_len;
 	if (imm_len % 16 != 0)
 		idata_len -= 16 - imm_len % 16;
 	crwr->sc_imm.len = htobe32(idata_len);
 }
 
 static int
 ccr_hash(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
 {
 	struct chcr_wr *crwr;
 	struct wrqe *wr;
 	struct auth_hash *axf;
-	struct cryptodesc *crd;
 	char *dst;
 	u_int hash_size_in_response, kctx_flits, kctx_len, transhdr_len, wr_len;
 	u_int hmac_ctrl, imm_len, iopad_size;
 	int error, sgl_nsegs, sgl_len, use_opad;
 
-	crd = crp->crp_desc;
-
 	/* Reject requests with too large of an input buffer. */
-	if (crd->crd_len > MAX_REQUEST_SIZE)
+	if (crp->crp_payload_length > MAX_REQUEST_SIZE)
 		return (EFBIG);
 
 	axf = s->hmac.auth_hash;
 
 	if (s->mode == HMAC) {
 		use_opad = 1;
 		hmac_ctrl = SCMD_HMAC_CTRL_NO_TRUNC;
 	} else {
 		use_opad = 0;
 		hmac_ctrl = SCMD_HMAC_CTRL_NOP;
 	}
 
 	/* PADs must be 128-bit aligned. */
 	iopad_size = roundup2(s->hmac.partial_digest_len, 16);
 
 	/*
 	 * The 'key' part of the context includes the aligned IPAD and
 	 * OPAD.
 	 */
 	kctx_len = iopad_size;
 	if (use_opad)
 		kctx_len += iopad_size;
 	hash_size_in_response = axf->hashsize;
 	transhdr_len = HASH_TRANSHDR_SIZE(kctx_len);
 
-	if (crd->crd_len == 0) {
+	if (crp->crp_payload_length == 0) {
 		imm_len = axf->blocksize;
 		sgl_nsegs = 0;
 		sgl_len = 0;
-	} else if (ccr_use_imm_data(transhdr_len, crd->crd_len)) {
-		imm_len = crd->crd_len;
+	} else if (ccr_use_imm_data(transhdr_len, crp->crp_payload_length)) {
+		imm_len = crp->crp_payload_length;
 		sgl_nsegs = 0;
 		sgl_len = 0;
 	} else {
 		imm_len = 0;
 		sglist_reset(sc->sg_ulptx);
 		error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
-		    crd->crd_skip, crd->crd_len);
+		    crp->crp_payload_start, crp->crp_payload_length);
 		if (error)
 			return (error);
 		sgl_nsegs = sc->sg_ulptx->sg_nseg;
 		sgl_len = ccr_ulptx_sgl_len(sgl_nsegs);
 	}
 
 	wr_len = roundup2(transhdr_len, 16) + roundup2(imm_len, 16) + sgl_len;
 	if (wr_len > SGE_MAX_WR_LEN)
 		return (EFBIG);
 	wr = alloc_wrqe(wr_len, sc->txq);
 	if (wr == NULL) {
 		sc->stats_wr_nomem++;
 		return (ENOMEM);
 	}
 	crwr = wrtod(wr);
 	memset(crwr, 0, wr_len);
 
 	ccr_populate_wreq(sc, crwr, kctx_len, wr_len, imm_len, sgl_len,
 	    hash_size_in_response, crp);
 
 	/* XXX: Hardcodes SGE loopback channel of 0. */
 	crwr->sec_cpl.op_ivinsrtofst = htobe32(
 	    V_CPL_TX_SEC_PDU_OPCODE(CPL_TX_SEC_PDU) |
 	    V_CPL_TX_SEC_PDU_RXCHID(sc->tx_channel_id) |
 	    V_CPL_TX_SEC_PDU_ACKFOLLOWS(0) | V_CPL_TX_SEC_PDU_ULPTXLPBK(1) |
 	    V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) |
 	    V_CPL_TX_SEC_PDU_IVINSRTOFST(0));
 
-	crwr->sec_cpl.pldlen = htobe32(crd->crd_len == 0 ? axf->blocksize :
-	    crd->crd_len);
+	crwr->sec_cpl.pldlen = htobe32(crp->crp_payload_length == 0 ?
+	    axf->blocksize : crp->crp_payload_length);
 
 	crwr->sec_cpl.cipherstop_lo_authinsert = htobe32(
 	    V_CPL_TX_SEC_PDU_AUTHSTART(1) | V_CPL_TX_SEC_PDU_AUTHSTOP(0));
 
 	/* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
 	crwr->sec_cpl.seqno_numivs = htobe32(
 	    V_SCMD_SEQ_NO_CTRL(0) |
 	    V_SCMD_PROTO_VERSION(SCMD_PROTO_VERSION_GENERIC) |
 	    V_SCMD_CIPH_MODE(SCMD_CIPH_MODE_NOP) |
 	    V_SCMD_AUTH_MODE(s->hmac.auth_mode) |
 	    V_SCMD_HMAC_CTRL(hmac_ctrl));
 	crwr->sec_cpl.ivgen_hdrlen = htobe32(
 	    V_SCMD_LAST_FRAG(0) |
-	    V_SCMD_MORE_FRAGS(crd->crd_len == 0 ? 1 : 0) | V_SCMD_MAC_ONLY(1));
+	    V_SCMD_MORE_FRAGS(crp->crp_payload_length == 0 ? 1 : 0) |
+	    V_SCMD_MAC_ONLY(1));
 
 	memcpy(crwr->key_ctx.key, s->hmac.pads, kctx_len);
 
 	/* XXX: F_KEY_CONTEXT_SALT_PRESENT set, but 'salt' not set. */
 	kctx_flits = (sizeof(struct _key_ctx) + kctx_len) / 16;
 	crwr->key_ctx.ctx_hdr = htobe32(V_KEY_CONTEXT_CTX_LEN(kctx_flits) |
 	    V_KEY_CONTEXT_OPAD_PRESENT(use_opad) |
 	    V_KEY_CONTEXT_SALT_PRESENT(1) |
 	    V_KEY_CONTEXT_CK_SIZE(CHCR_KEYCTX_NO_KEY) |
 	    V_KEY_CONTEXT_MK_SIZE(s->hmac.mk_size) | V_KEY_CONTEXT_VALID(1));
 
 	dst = (char *)(crwr + 1) + kctx_len + DUMMY_BYTES;
-	if (crd->crd_len == 0) {
+	if (crp->crp_payload_length == 0) {
 		dst[0] = 0x80;
 		if (s->mode == HMAC)
 			*(uint64_t *)(dst + axf->blocksize - sizeof(uint64_t)) =
 			    htobe64(axf->blocksize << 3);
 	} else if (imm_len != 0)
-		crypto_copydata(crp->crp_flags, crp->crp_buf, crd->crd_skip,
-		    crd->crd_len, dst);
+		crypto_copydata(crp, crp->crp_payload_start,
+		    crp->crp_payload_length, dst);
 	else
 		ccr_write_ulptx_sgl(sc, dst, sgl_nsegs);
 
 	/* XXX: TODO backpressure */
 	t4_wrq_tx(sc->adapter, wr);
 
 	return (0);
 }
 
 static int
 ccr_hash_done(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
     const struct cpl_fw6_pld *cpl, int error)
 {
-	struct cryptodesc *crd;
+	uint8_t hash[HASH_MAX_LEN];
 
-	crd = crp->crp_desc;
-	if (error == 0) {
-		crypto_copyback(crp->crp_flags, crp->crp_buf, crd->crd_inject,
-		    s->hmac.hash_len, (c_caddr_t)(cpl + 1));
-	}
+	if (error)
+		return (error);
 
-	return (error);
+	if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
+		crypto_copydata(crp, crp->crp_digest_start, s->hmac.hash_len,
+		    hash);
+		if (timingsafe_bcmp((cpl + 1), hash, s->hmac.hash_len) != 0)
+			return (EBADMSG);
+	} else
+		crypto_copyback(crp, crp->crp_digest_start, s->hmac.hash_len,
+		    (cpl + 1));
+	return (0);
 }
 
 static int
 ccr_blkcipher(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
 {
 	char iv[CHCR_MAX_CRYPTO_IV_LEN];
 	struct chcr_wr *crwr;
 	struct wrqe *wr;
-	struct cryptodesc *crd;
 	char *dst;
 	u_int kctx_len, key_half, op_type, transhdr_len, wr_len;
-	u_int imm_len;
+	u_int imm_len, iv_len;
 	int dsgl_nsegs, dsgl_len;
 	int sgl_nsegs, sgl_len;
 	int error;
 
-	crd = crp->crp_desc;
-
-	if (s->blkcipher.key_len == 0 || crd->crd_len == 0)
+	if (s->blkcipher.key_len == 0 || crp->crp_payload_length == 0)
 		return (EINVAL);
-	if (crd->crd_alg == CRYPTO_AES_CBC &&
-	    (crd->crd_len % AES_BLOCK_LEN) != 0)
+	if (s->blkcipher.cipher_mode == SCMD_CIPH_MODE_AES_CBC &&
+	    (crp->crp_payload_length % AES_BLOCK_LEN) != 0)
 		return (EINVAL);
 
 	/* Reject requests with too large of an input buffer. */
-	if (crd->crd_len > MAX_REQUEST_SIZE)
+	if (crp->crp_payload_length > MAX_REQUEST_SIZE)
 		return (EFBIG);
 
-	if (crd->crd_flags & CRD_F_ENCRYPT)
+	if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
 		op_type = CHCR_ENCRYPT_OP;
 	else
 		op_type = CHCR_DECRYPT_OP;
 	
 	sglist_reset(sc->sg_dsgl);
-	error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp, crd->crd_skip,
-	    crd->crd_len);
+	error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp,
+	    crp->crp_payload_start, crp->crp_payload_length);
 	if (error)
 		return (error);
 	dsgl_nsegs = ccr_count_sgl(sc->sg_dsgl, DSGL_SGE_MAXLEN);
 	if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE)
 		return (EFBIG);
 	dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs);
 
 	/* The 'key' must be 128-bit aligned. */
 	kctx_len = roundup2(s->blkcipher.key_len, 16);
 	transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dsgl_len);
 
-	if (ccr_use_imm_data(transhdr_len, crd->crd_len +
-	    s->blkcipher.iv_len)) {
-		imm_len = crd->crd_len;
+	/* For AES-XTS we send a 16-byte IV in the work request. */
+	if (s->blkcipher.cipher_mode == SCMD_CIPH_MODE_AES_XTS)
+		iv_len = AES_BLOCK_LEN;
+	else
+		iv_len = s->blkcipher.iv_len;
+
+	if (ccr_use_imm_data(transhdr_len, crp->crp_payload_length + iv_len)) {
+		imm_len = crp->crp_payload_length;
 		sgl_nsegs = 0;
 		sgl_len = 0;
 	} else {
 		imm_len = 0;
 		sglist_reset(sc->sg_ulptx);
 		error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
-		    crd->crd_skip, crd->crd_len);
+		    crp->crp_payload_start, crp->crp_payload_length);
 		if (error)
 			return (error);
 		sgl_nsegs = sc->sg_ulptx->sg_nseg;
 		sgl_len = ccr_ulptx_sgl_len(sgl_nsegs);
 	}
 
-	wr_len = roundup2(transhdr_len, 16) + s->blkcipher.iv_len +
+	wr_len = roundup2(transhdr_len, 16) + iv_len +
 	    roundup2(imm_len, 16) + sgl_len;
 	if (wr_len > SGE_MAX_WR_LEN)
 		return (EFBIG);
 	wr = alloc_wrqe(wr_len, sc->txq);
 	if (wr == NULL) {
 		sc->stats_wr_nomem++;
 		return (ENOMEM);
 	}
 	crwr = wrtod(wr);
 	memset(crwr, 0, wr_len);
 
 	/*
 	 * Read the existing IV from the request or generate a random
-	 * one if none is provided.  Optionally copy the generated IV
-	 * into the output buffer if requested.
+	 * one if none is provided.
 	 */
-	if (op_type == CHCR_ENCRYPT_OP) {
-		if (crd->crd_flags & CRD_F_IV_EXPLICIT)
-			memcpy(iv, crd->crd_iv, s->blkcipher.iv_len);
-		else
-			arc4rand(iv, s->blkcipher.iv_len, 0);
-		if ((crd->crd_flags & CRD_F_IV_PRESENT) == 0)
-			crypto_copyback(crp->crp_flags, crp->crp_buf,
-			    crd->crd_inject, s->blkcipher.iv_len, iv);
-	} else {
-		if (crd->crd_flags & CRD_F_IV_EXPLICIT)
-			memcpy(iv, crd->crd_iv, s->blkcipher.iv_len);
-		else
-			crypto_copydata(crp->crp_flags, crp->crp_buf,
-			    crd->crd_inject, s->blkcipher.iv_len, iv);
-	}
+	if (crp->crp_flags & CRYPTO_F_IV_GENERATE) {
+		arc4rand(iv, s->blkcipher.iv_len, 0);
+		crypto_copyback(crp, crp->crp_iv_start, s->blkcipher.iv_len,
+		    iv);
+	} else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+		memcpy(iv, crp->crp_iv, s->blkcipher.iv_len);
+	else
+		crypto_copydata(crp, crp->crp_iv_start, s->blkcipher.iv_len,
+		    iv);
 
+	/* Zero the remainder of the IV for AES-XTS. */
+	memset(iv + s->blkcipher.iv_len, 0, iv_len - s->blkcipher.iv_len);
+
 	ccr_populate_wreq(sc, crwr, kctx_len, wr_len, imm_len, sgl_len, 0,
 	    crp);
 
 	/* XXX: Hardcodes SGE loopback channel of 0. */
 	crwr->sec_cpl.op_ivinsrtofst = htobe32(
 	    V_CPL_TX_SEC_PDU_OPCODE(CPL_TX_SEC_PDU) |
 	    V_CPL_TX_SEC_PDU_RXCHID(sc->tx_channel_id) |
 	    V_CPL_TX_SEC_PDU_ACKFOLLOWS(0) | V_CPL_TX_SEC_PDU_ULPTXLPBK(1) |
 	    V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) |
 	    V_CPL_TX_SEC_PDU_IVINSRTOFST(1));
 
-	crwr->sec_cpl.pldlen = htobe32(s->blkcipher.iv_len + crd->crd_len);
+	crwr->sec_cpl.pldlen = htobe32(iv_len + crp->crp_payload_length);
 
 	crwr->sec_cpl.aadstart_cipherstop_hi = htobe32(
-	    V_CPL_TX_SEC_PDU_CIPHERSTART(s->blkcipher.iv_len + 1) |
+	    V_CPL_TX_SEC_PDU_CIPHERSTART(iv_len + 1) |
 	    V_CPL_TX_SEC_PDU_CIPHERSTOP_HI(0));
 	crwr->sec_cpl.cipherstop_lo_authinsert = htobe32(
 	    V_CPL_TX_SEC_PDU_CIPHERSTOP_LO(0));
 
 	/* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
 	crwr->sec_cpl.seqno_numivs = htobe32(
 	    V_SCMD_SEQ_NO_CTRL(0) |
 	    V_SCMD_PROTO_VERSION(SCMD_PROTO_VERSION_GENERIC) |
 	    V_SCMD_ENC_DEC_CTRL(op_type) |
 	    V_SCMD_CIPH_MODE(s->blkcipher.cipher_mode) |
 	    V_SCMD_AUTH_MODE(SCMD_AUTH_MODE_NOP) |
 	    V_SCMD_HMAC_CTRL(SCMD_HMAC_CTRL_NOP) |
-	    V_SCMD_IV_SIZE(s->blkcipher.iv_len / 2) |
+	    V_SCMD_IV_SIZE(iv_len / 2) |
 	    V_SCMD_NUM_IVS(0));
 	crwr->sec_cpl.ivgen_hdrlen = htobe32(
 	    V_SCMD_IV_GEN_CTRL(0) |
 	    V_SCMD_MORE_FRAGS(0) | V_SCMD_LAST_FRAG(0) | V_SCMD_MAC_ONLY(0) |
 	    V_SCMD_AADIVDROP(1) | V_SCMD_HDR_LEN(dsgl_len));
 
 	crwr->key_ctx.ctx_hdr = s->blkcipher.key_ctx_hdr;
-	switch (crd->crd_alg) {
-	case CRYPTO_AES_CBC:
-		if (crd->crd_flags & CRD_F_ENCRYPT)
+	switch (s->blkcipher.cipher_mode) {
+	case SCMD_CIPH_MODE_AES_CBC:
+		if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
 			memcpy(crwr->key_ctx.key, s->blkcipher.enckey,
 			    s->blkcipher.key_len);
 		else
 			memcpy(crwr->key_ctx.key, s->blkcipher.deckey,
 			    s->blkcipher.key_len);
 		break;
-	case CRYPTO_AES_ICM:
+	case SCMD_CIPH_MODE_AES_CTR:
 		memcpy(crwr->key_ctx.key, s->blkcipher.enckey,
 		    s->blkcipher.key_len);
 		break;
-	case CRYPTO_AES_XTS:
+	case SCMD_CIPH_MODE_AES_XTS:
 		key_half = s->blkcipher.key_len / 2;
 		memcpy(crwr->key_ctx.key, s->blkcipher.enckey + key_half,
 		    key_half);
-		if (crd->crd_flags & CRD_F_ENCRYPT)
+		if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
 			memcpy(crwr->key_ctx.key + key_half,
 			    s->blkcipher.enckey, key_half);
 		else
 			memcpy(crwr->key_ctx.key + key_half,
 			    s->blkcipher.deckey, key_half);
 		break;
 	}
 
 	dst = (char *)(crwr + 1) + kctx_len;
 	ccr_write_phys_dsgl(sc, dst, dsgl_nsegs);
 	dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len;
-	memcpy(dst, iv, s->blkcipher.iv_len);
-	dst += s->blkcipher.iv_len;
+	memcpy(dst, iv, iv_len);
+	dst += iv_len;
 	if (imm_len != 0)
-		crypto_copydata(crp->crp_flags, crp->crp_buf, crd->crd_skip,
-		    crd->crd_len, dst);
+		crypto_copydata(crp, crp->crp_payload_start,
+		    crp->crp_payload_length, dst);
 	else
 		ccr_write_ulptx_sgl(sc, dst, sgl_nsegs);
 
 	/* XXX: TODO backpressure */
 	t4_wrq_tx(sc->adapter, wr);
 
 	return (0);
 }
 
 static int
 ccr_blkcipher_done(struct ccr_softc *sc, struct ccr_session *s,
     struct cryptop *crp, const struct cpl_fw6_pld *cpl, int error)
 {
 
 	/*
 	 * The updated IV to permit chained requests is at
 	 * cpl->data[2], but OCF doesn't permit chained requests.
 	 */
 	return (error);
 }
 
 /*
  * 'hashsize' is the length of a full digest.  'authsize' is the
  * requested digest length for this operation which may be less
  * than 'hashsize'.
  */
 static int
 ccr_hmac_ctrl(unsigned int hashsize, unsigned int authsize)
 {
 
 	if (authsize == 10)
 		return (SCMD_HMAC_CTRL_TRUNC_RFC4366);
 	if (authsize == 12)
 		return (SCMD_HMAC_CTRL_IPSEC_96BIT);
 	if (authsize == hashsize / 2)
 		return (SCMD_HMAC_CTRL_DIV2);
 	return (SCMD_HMAC_CTRL_NO_TRUNC);
 }
 
 static int
-ccr_authenc(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
-    struct cryptodesc *crda, struct cryptodesc *crde)
+ccr_eta(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
 {
 	char iv[CHCR_MAX_CRYPTO_IV_LEN];
 	struct chcr_wr *crwr;
 	struct wrqe *wr;
 	struct auth_hash *axf;
 	char *dst;
 	u_int kctx_len, key_half, op_type, transhdr_len, wr_len;
-	u_int hash_size_in_response, imm_len, iopad_size;
-	u_int aad_start, aad_len, aad_stop;
-	u_int auth_start, auth_stop, auth_insert;
+	u_int hash_size_in_response, imm_len, iopad_size, iv_len;
+	u_int aad_start, aad_stop;
+	u_int auth_insert;
 	u_int cipher_start, cipher_stop;
 	u_int hmac_ctrl, input_len;
 	int dsgl_nsegs, dsgl_len;
 	int sgl_nsegs, sgl_len;
 	int error;
 
 	/*
 	 * If there is a need in the future, requests with an empty
 	 * payload could be supported as HMAC-only requests.
 	 */
-	if (s->blkcipher.key_len == 0 || crde->crd_len == 0)
+	if (s->blkcipher.key_len == 0 || crp->crp_payload_length == 0)
 		return (EINVAL);
-	if (crde->crd_alg == CRYPTO_AES_CBC &&
-	    (crde->crd_len % AES_BLOCK_LEN) != 0)
+	if (s->blkcipher.cipher_mode == SCMD_CIPH_MODE_AES_CBC &&
+	    (crp->crp_payload_length % AES_BLOCK_LEN) != 0)
 		return (EINVAL);
 
-	/*
-	 * Compute the length of the AAD (data covered by the
-	 * authentication descriptor but not the encryption
-	 * descriptor).  To simplify the logic, AAD is only permitted
-	 * before the cipher/plain text, not after.  This is true of
-	 * all currently-generated requests.
-	 */
-	if (crda->crd_len + crda->crd_skip > crde->crd_len + crde->crd_skip)
+	/* For AES-XTS we send a 16-byte IV in the work request. */
+	if (s->blkcipher.cipher_mode == SCMD_CIPH_MODE_AES_XTS)
+		iv_len = AES_BLOCK_LEN;
+	else
+		iv_len = s->blkcipher.iv_len;
+
+	if (crp->crp_aad_length + iv_len > MAX_AAD_LEN)
 		return (EINVAL);
-	if (crda->crd_skip < crde->crd_skip) {
-		if (crda->crd_skip + crda->crd_len > crde->crd_skip)
-			aad_len = (crde->crd_skip - crda->crd_skip);
-		else
-			aad_len = crda->crd_len;
-	} else
-		aad_len = 0;
-	if (aad_len + s->blkcipher.iv_len > MAX_AAD_LEN)
-		return (EINVAL);
 
 	axf = s->hmac.auth_hash;
 	hash_size_in_response = s->hmac.hash_len;
-	if (crde->crd_flags & CRD_F_ENCRYPT)
+	if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
 		op_type = CHCR_ENCRYPT_OP;
 	else
 		op_type = CHCR_DECRYPT_OP;
 
 	/*
 	 * The output buffer consists of the cipher text followed by
 	 * the hash when encrypting.  For decryption it only contains
 	 * the plain text.
 	 *
 	 * Due to a firmware bug, the output buffer must include a
 	 * dummy output buffer for the IV and AAD prior to the real
 	 * output buffer.
 	 */
 	if (op_type == CHCR_ENCRYPT_OP) {
-		if (s->blkcipher.iv_len + aad_len + crde->crd_len +
+		if (iv_len + crp->crp_aad_length + crp->crp_payload_length +
 		    hash_size_in_response > MAX_REQUEST_SIZE)
 			return (EFBIG);
 	} else {
-		if (s->blkcipher.iv_len + aad_len + crde->crd_len >
+		if (iv_len + crp->crp_aad_length + crp->crp_payload_length >
 		    MAX_REQUEST_SIZE)
 			return (EFBIG);
 	}
 	sglist_reset(sc->sg_dsgl);
 	error = sglist_append_sglist(sc->sg_dsgl, sc->sg_iv_aad, 0,
-	    s->blkcipher.iv_len + aad_len);
+	    iv_len + crp->crp_aad_length);
 	if (error)
 		return (error);
-	error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp, crde->crd_skip,
-	    crde->crd_len);
+	error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp,
+	    crp->crp_payload_start, crp->crp_payload_length);
 	if (error)
 		return (error);
 	if (op_type == CHCR_ENCRYPT_OP) {
 		error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp,
-		    crda->crd_inject, hash_size_in_response);
+		    crp->crp_digest_start, hash_size_in_response);
 		if (error)
 			return (error);
 	}
 	dsgl_nsegs = ccr_count_sgl(sc->sg_dsgl, DSGL_SGE_MAXLEN);
 	if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE)
 		return (EFBIG);
 	dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs);
 
 	/* PADs must be 128-bit aligned. */
 	iopad_size = roundup2(s->hmac.partial_digest_len, 16);
 
 	/*
 	 * The 'key' part of the key context consists of the key followed
 	 * by the IPAD and OPAD.
 	 */
 	kctx_len = roundup2(s->blkcipher.key_len, 16) + iopad_size * 2;
 	transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dsgl_len);
 
 	/*
 	 * The input buffer consists of the IV, any AAD, and then the
 	 * cipher/plain text.  For decryption requests the hash is
 	 * appended after the cipher text.
 	 *
 	 * The IV is always stored at the start of the input buffer
 	 * even though it may be duplicated in the payload.  The
 	 * crypto engine doesn't work properly if the IV offset points
 	 * inside of the AAD region, so a second copy is always
 	 * required.
 	 */
-	input_len = aad_len + crde->crd_len;
+	input_len = crp->crp_aad_length + crp->crp_payload_length;
 
 	/*
 	 * The firmware hangs if sent a request which is a
 	 * bit smaller than MAX_REQUEST_SIZE.  In particular, the
 	 * firmware appears to require 512 - 16 bytes of spare room
 	 * along with the size of the hash even if the hash isn't
 	 * included in the input buffer.
 	 */
 	if (input_len + roundup2(axf->hashsize, 16) + (512 - 16) >
 	    MAX_REQUEST_SIZE)
 		return (EFBIG);
 	if (op_type == CHCR_DECRYPT_OP)
 		input_len += hash_size_in_response;
-	if (ccr_use_imm_data(transhdr_len, s->blkcipher.iv_len + input_len)) {
+
+	if (ccr_use_imm_data(transhdr_len, iv_len + input_len)) {
 		imm_len = input_len;
 		sgl_nsegs = 0;
 		sgl_len = 0;
 	} else {
 		imm_len = 0;
 		sglist_reset(sc->sg_ulptx);
-		if (aad_len != 0) {
+		if (crp->crp_aad_length != 0) {
 			error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
-			    crda->crd_skip, aad_len);
+			    crp->crp_aad_start, crp->crp_aad_length);
 			if (error)
 				return (error);
 		}
 		error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
-		    crde->crd_skip, crde->crd_len);
+		    crp->crp_payload_start, crp->crp_payload_length);
 		if (error)
 			return (error);
 		if (op_type == CHCR_DECRYPT_OP) {
 			error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
-			    crda->crd_inject, hash_size_in_response);
+			    crp->crp_digest_start, hash_size_in_response);
 			if (error)
 				return (error);
 		}
 		sgl_nsegs = sc->sg_ulptx->sg_nseg;
 		sgl_len = ccr_ulptx_sgl_len(sgl_nsegs);
 	}
 
 	/*
 	 * Any auth-only data before the cipher region is marked as AAD.
 	 * Auth-data that overlaps with the cipher region is placed in
 	 * the auth section.
 	 */
-	if (aad_len != 0) {
-		aad_start = s->blkcipher.iv_len + 1;
-		aad_stop = aad_start + aad_len - 1;
+	if (crp->crp_aad_length != 0) {
+		aad_start = iv_len + 1;
+		aad_stop = aad_start + crp->crp_aad_length - 1;
 	} else {
 		aad_start = 0;
 		aad_stop = 0;
 	}
-	cipher_start = s->blkcipher.iv_len + aad_len + 1;
+	cipher_start = iv_len + crp->crp_aad_length + 1;
 	if (op_type == CHCR_DECRYPT_OP)
 		cipher_stop = hash_size_in_response;
 	else
 		cipher_stop = 0;
-	if (aad_len == crda->crd_len) {
-		auth_start = 0;
-		auth_stop = 0;
-	} else {
-		if (aad_len != 0)
-			auth_start = cipher_start;
-		else
-			auth_start = s->blkcipher.iv_len + crda->crd_skip -
-			    crde->crd_skip + 1;
-		auth_stop = (crde->crd_skip + crde->crd_len) -
-		    (crda->crd_skip + crda->crd_len) + cipher_stop;
-	}
 	if (op_type == CHCR_DECRYPT_OP)
 		auth_insert = hash_size_in_response;
 	else
 		auth_insert = 0;
 
-	wr_len = roundup2(transhdr_len, 16) + s->blkcipher.iv_len +
-	    roundup2(imm_len, 16) + sgl_len;
+	wr_len = roundup2(transhdr_len, 16) + iv_len + roundup2(imm_len, 16) +
+	    sgl_len;
 	if (wr_len > SGE_MAX_WR_LEN)
 		return (EFBIG);
 	wr = alloc_wrqe(wr_len, sc->txq);
 	if (wr == NULL) {
 		sc->stats_wr_nomem++;
 		return (ENOMEM);
 	}
 	crwr = wrtod(wr);
 	memset(crwr, 0, wr_len);
 
 	/*
 	 * Read the existing IV from the request or generate a random
-	 * one if none is provided.  Optionally copy the generated IV
-	 * into the output buffer if requested.
+	 * one if none is provided.
 	 */
-	if (op_type == CHCR_ENCRYPT_OP) {
-		if (crde->crd_flags & CRD_F_IV_EXPLICIT)
-			memcpy(iv, crde->crd_iv, s->blkcipher.iv_len);
-		else
-			arc4rand(iv, s->blkcipher.iv_len, 0);
-		if ((crde->crd_flags & CRD_F_IV_PRESENT) == 0)
-			crypto_copyback(crp->crp_flags, crp->crp_buf,
-			    crde->crd_inject, s->blkcipher.iv_len, iv);
-	} else {
-		if (crde->crd_flags & CRD_F_IV_EXPLICIT)
-			memcpy(iv, crde->crd_iv, s->blkcipher.iv_len);
-		else
-			crypto_copydata(crp->crp_flags, crp->crp_buf,
-			    crde->crd_inject, s->blkcipher.iv_len, iv);
-	}
+	if (crp->crp_flags & CRYPTO_F_IV_GENERATE) {
+		arc4rand(iv, s->blkcipher.iv_len, 0);
+		crypto_copyback(crp, crp->crp_iv_start, s->blkcipher.iv_len,
+		    iv);
+	} else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+		memcpy(iv, crp->crp_iv, s->blkcipher.iv_len);
+	else
+		crypto_copydata(crp, crp->crp_iv_start, s->blkcipher.iv_len,
+		    iv);
 
+	/* Zero the remainder of the IV for AES-XTS. */
+	memset(iv + s->blkcipher.iv_len, 0, iv_len - s->blkcipher.iv_len);
+
 	ccr_populate_wreq(sc, crwr, kctx_len, wr_len, imm_len, sgl_len,
 	    op_type == CHCR_DECRYPT_OP ? hash_size_in_response : 0, crp);
 
 	/* XXX: Hardcodes SGE loopback channel of 0. */
 	crwr->sec_cpl.op_ivinsrtofst = htobe32(
 	    V_CPL_TX_SEC_PDU_OPCODE(CPL_TX_SEC_PDU) |
 	    V_CPL_TX_SEC_PDU_RXCHID(sc->tx_channel_id) |
 	    V_CPL_TX_SEC_PDU_ACKFOLLOWS(0) | V_CPL_TX_SEC_PDU_ULPTXLPBK(1) |
 	    V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) |
 	    V_CPL_TX_SEC_PDU_IVINSRTOFST(1));
 
-	crwr->sec_cpl.pldlen = htobe32(s->blkcipher.iv_len + input_len);
+	crwr->sec_cpl.pldlen = htobe32(iv_len + input_len);
 
 	crwr->sec_cpl.aadstart_cipherstop_hi = htobe32(
 	    V_CPL_TX_SEC_PDU_AADSTART(aad_start) |
 	    V_CPL_TX_SEC_PDU_AADSTOP(aad_stop) |
 	    V_CPL_TX_SEC_PDU_CIPHERSTART(cipher_start) |
 	    V_CPL_TX_SEC_PDU_CIPHERSTOP_HI(cipher_stop >> 4));
 	crwr->sec_cpl.cipherstop_lo_authinsert = htobe32(
 	    V_CPL_TX_SEC_PDU_CIPHERSTOP_LO(cipher_stop & 0xf) |
-	    V_CPL_TX_SEC_PDU_AUTHSTART(auth_start) |
-	    V_CPL_TX_SEC_PDU_AUTHSTOP(auth_stop) |
+	    V_CPL_TX_SEC_PDU_AUTHSTART(cipher_start) |
+	    V_CPL_TX_SEC_PDU_AUTHSTOP(cipher_stop) |
 	    V_CPL_TX_SEC_PDU_AUTHINSERT(auth_insert));
 
 	/* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
 	hmac_ctrl = ccr_hmac_ctrl(axf->hashsize, hash_size_in_response);
 	crwr->sec_cpl.seqno_numivs = htobe32(
 	    V_SCMD_SEQ_NO_CTRL(0) |
 	    V_SCMD_PROTO_VERSION(SCMD_PROTO_VERSION_GENERIC) |
 	    V_SCMD_ENC_DEC_CTRL(op_type) |
 	    V_SCMD_CIPH_AUTH_SEQ_CTRL(op_type == CHCR_ENCRYPT_OP ? 1 : 0) |
 	    V_SCMD_CIPH_MODE(s->blkcipher.cipher_mode) |
 	    V_SCMD_AUTH_MODE(s->hmac.auth_mode) |
 	    V_SCMD_HMAC_CTRL(hmac_ctrl) |
-	    V_SCMD_IV_SIZE(s->blkcipher.iv_len / 2) |
+	    V_SCMD_IV_SIZE(iv_len / 2) |
 	    V_SCMD_NUM_IVS(0));
 	crwr->sec_cpl.ivgen_hdrlen = htobe32(
 	    V_SCMD_IV_GEN_CTRL(0) |
 	    V_SCMD_MORE_FRAGS(0) | V_SCMD_LAST_FRAG(0) | V_SCMD_MAC_ONLY(0) |
 	    V_SCMD_AADIVDROP(0) | V_SCMD_HDR_LEN(dsgl_len));
 
 	crwr->key_ctx.ctx_hdr = s->blkcipher.key_ctx_hdr;
-	switch (crde->crd_alg) {
-	case CRYPTO_AES_CBC:
-		if (crde->crd_flags & CRD_F_ENCRYPT)
+	switch (s->blkcipher.cipher_mode) {
+	case SCMD_CIPH_MODE_AES_CBC:
+		if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
 			memcpy(crwr->key_ctx.key, s->blkcipher.enckey,
 			    s->blkcipher.key_len);
 		else
 			memcpy(crwr->key_ctx.key, s->blkcipher.deckey,
 			    s->blkcipher.key_len);
 		break;
-	case CRYPTO_AES_ICM:
+	case SCMD_CIPH_MODE_AES_CTR:
 		memcpy(crwr->key_ctx.key, s->blkcipher.enckey,
 		    s->blkcipher.key_len);
 		break;
-	case CRYPTO_AES_XTS:
+	case SCMD_CIPH_MODE_AES_XTS:
 		key_half = s->blkcipher.key_len / 2;
 		memcpy(crwr->key_ctx.key, s->blkcipher.enckey + key_half,
 		    key_half);
-		if (crde->crd_flags & CRD_F_ENCRYPT)
+		if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
 			memcpy(crwr->key_ctx.key + key_half,
 			    s->blkcipher.enckey, key_half);
 		else
 			memcpy(crwr->key_ctx.key + key_half,
 			    s->blkcipher.deckey, key_half);
 		break;
 	}
 
 	dst = crwr->key_ctx.key + roundup2(s->blkcipher.key_len, 16);
 	memcpy(dst, s->hmac.pads, iopad_size * 2);
 
 	dst = (char *)(crwr + 1) + kctx_len;
 	ccr_write_phys_dsgl(sc, dst, dsgl_nsegs);
 	dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len;
-	memcpy(dst, iv, s->blkcipher.iv_len);
-	dst += s->blkcipher.iv_len;
+	memcpy(dst, iv, iv_len);
+	dst += iv_len;
 	if (imm_len != 0) {
-		if (aad_len != 0) {
-			crypto_copydata(crp->crp_flags, crp->crp_buf,
-			    crda->crd_skip, aad_len, dst);
-			dst += aad_len;
+		if (crp->crp_aad_length != 0) {
+			crypto_copydata(crp, crp->crp_aad_start,
+			    crp->crp_aad_length, dst);
+			dst += crp->crp_aad_length;
 		}
-		crypto_copydata(crp->crp_flags, crp->crp_buf, crde->crd_skip,
-		    crde->crd_len, dst);
-		dst += crde->crd_len;
+		crypto_copydata(crp, crp->crp_payload_start,
+		    crp->crp_payload_length, dst);
+		dst += crp->crp_payload_length;
 		if (op_type == CHCR_DECRYPT_OP)
-			crypto_copydata(crp->crp_flags, crp->crp_buf,
-			    crda->crd_inject, hash_size_in_response, dst);
+			crypto_copydata(crp, crp->crp_digest_start,
+			    hash_size_in_response, dst);
 	} else
 		ccr_write_ulptx_sgl(sc, dst, sgl_nsegs);
 
 	/* XXX: TODO backpressure */
 	t4_wrq_tx(sc->adapter, wr);
 
 	return (0);
 }
 
 static int
-ccr_authenc_done(struct ccr_softc *sc, struct ccr_session *s,
+ccr_eta_done(struct ccr_softc *sc, struct ccr_session *s,
     struct cryptop *crp, const struct cpl_fw6_pld *cpl, int error)
 {
-	struct cryptodesc *crd;
 
 	/*
 	 * The updated IV to permit chained requests is at
 	 * cpl->data[2], but OCF doesn't permit chained requests.
-	 *
-	 * For a decryption request, the hardware may do a verification
-	 * of the HMAC which will fail if the existing HMAC isn't in the
-	 * buffer.  If that happens, clear the error and copy the HMAC
-	 * from the CPL reply into the buffer.
-	 *
-	 * For encryption requests, crd should be the cipher request
-	 * which will have CRD_F_ENCRYPT set.  For decryption
-	 * requests, crp_desc will be the HMAC request which should
-	 * not have this flag set.
 	 */
-	crd = crp->crp_desc;
-	if (error == EBADMSG && !CHK_PAD_ERR_BIT(be64toh(cpl->data[0])) &&
-	    !(crd->crd_flags & CRD_F_ENCRYPT)) {
-		crypto_copyback(crp->crp_flags, crp->crp_buf, crd->crd_inject,
-		    s->hmac.hash_len, (c_caddr_t)(cpl + 1));
-		error = 0;
-	}
 	return (error);
 }
 
 static int
-ccr_gcm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
-    struct cryptodesc *crda, struct cryptodesc *crde)
+ccr_gcm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
 {
 	char iv[CHCR_MAX_CRYPTO_IV_LEN];
 	struct chcr_wr *crwr;
 	struct wrqe *wr;
 	char *dst;
 	u_int iv_len, kctx_len, op_type, transhdr_len, wr_len;
 	u_int hash_size_in_response, imm_len;
 	u_int aad_start, aad_stop, cipher_start, cipher_stop, auth_insert;
 	u_int hmac_ctrl, input_len;
 	int dsgl_nsegs, dsgl_len;
 	int sgl_nsegs, sgl_len;
 	int error;
 
 	if (s->blkcipher.key_len == 0)
 		return (EINVAL);
 
 	/*
 	 * The crypto engine doesn't handle GCM requests with an empty
 	 * payload, so handle those in software instead.
 	 */
-	if (crde->crd_len == 0)
+	if (crp->crp_payload_length == 0)
 		return (EMSGSIZE);
 
-	/*
-	 * AAD is only permitted before the cipher/plain text, not
-	 * after.
-	 */
-	if (crda->crd_len + crda->crd_skip > crde->crd_len + crde->crd_skip)
+	if (crp->crp_aad_length + AES_BLOCK_LEN > MAX_AAD_LEN)
 		return (EMSGSIZE);
 
-	if (crda->crd_len + AES_BLOCK_LEN > MAX_AAD_LEN)
-		return (EMSGSIZE);
-
 	hash_size_in_response = s->gmac.hash_len;
-	if (crde->crd_flags & CRD_F_ENCRYPT)
+	if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
 		op_type = CHCR_ENCRYPT_OP;
 	else
 		op_type = CHCR_DECRYPT_OP;
 
 	/*
 	 * The IV handling for GCM in OCF is a bit more complicated in
 	 * that IPSec provides a full 16-byte IV (including the
 	 * counter), whereas the /dev/crypto interface sometimes
 	 * provides a full 16-byte IV (if no IV is provided in the
 	 * ioctl) and sometimes a 12-byte IV (if the IV was explicit).
 	 *
 	 * When provided a 12-byte IV, assume the IV is really 16 bytes
 	 * with a counter in the last 4 bytes initialized to 1.
 	 *
 	 * While iv_len is checked below, the value is currently
 	 * always set to 12 when creating a GCM session in this driver
 	 * due to limitations in OCF (there is no way to know what the
 	 * IV length of a given request will be).  This means that the
 	 * driver always assumes as 12-byte IV for now.
 	 */
 	if (s->blkcipher.iv_len == 12)
 		iv_len = AES_BLOCK_LEN;
 	else
 		iv_len = s->blkcipher.iv_len;
 
 	/*
+	 * GCM requests should always provide an explicit IV.
+	 */
+	if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0)
+		return (EINVAL);
+
+	/*
 	 * The output buffer consists of the cipher text followed by
 	 * the tag when encrypting.  For decryption it only contains
 	 * the plain text.
 	 *
 	 * Due to a firmware bug, the output buffer must include a
 	 * dummy output buffer for the IV and AAD prior to the real
 	 * output buffer.
 	 */
 	if (op_type == CHCR_ENCRYPT_OP) {
-		if (iv_len + crda->crd_len + crde->crd_len +
+		if (iv_len + crp->crp_aad_length + crp->crp_payload_length +
 		    hash_size_in_response > MAX_REQUEST_SIZE)
 			return (EFBIG);
 	} else {
-		if (iv_len + crda->crd_len + crde->crd_len > MAX_REQUEST_SIZE)
+		if (iv_len + crp->crp_aad_length + crp->crp_payload_length >
+		    MAX_REQUEST_SIZE)
 			return (EFBIG);
 	}
 	sglist_reset(sc->sg_dsgl);
 	error = sglist_append_sglist(sc->sg_dsgl, sc->sg_iv_aad, 0, iv_len +
-	    crda->crd_len);
+	    crp->crp_aad_length);
 	if (error)
 		return (error);
-	error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp, crde->crd_skip,
-	    crde->crd_len);
+	error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp,
+	    crp->crp_payload_start, crp->crp_payload_length);
 	if (error)
 		return (error);
 	if (op_type == CHCR_ENCRYPT_OP) {
 		error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp,
-		    crda->crd_inject, hash_size_in_response);
+		    crp->crp_digest_start, hash_size_in_response);
 		if (error)
 			return (error);
 	}
 	dsgl_nsegs = ccr_count_sgl(sc->sg_dsgl, DSGL_SGE_MAXLEN);
 	if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE)
 		return (EFBIG);
 	dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs);
 
 	/*
 	 * The 'key' part of the key context consists of the key followed
 	 * by the Galois hash key.
 	 */
 	kctx_len = roundup2(s->blkcipher.key_len, 16) + GMAC_BLOCK_LEN;
 	transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dsgl_len);
 
 	/*
 	 * The input buffer consists of the IV, any AAD, and then the
 	 * cipher/plain text.  For decryption requests the hash is
 	 * appended after the cipher text.
 	 *
 	 * The IV is always stored at the start of the input buffer
 	 * even though it may be duplicated in the payload.  The
 	 * crypto engine doesn't work properly if the IV offset points
 	 * inside of the AAD region, so a second copy is always
 	 * required.
 	 */
-	input_len = crda->crd_len + crde->crd_len;
+	input_len = crp->crp_aad_length + crp->crp_payload_length;
 	if (op_type == CHCR_DECRYPT_OP)
 		input_len += hash_size_in_response;
 	if (input_len > MAX_REQUEST_SIZE)
 		return (EFBIG);
 	if (ccr_use_imm_data(transhdr_len, iv_len + input_len)) {
 		imm_len = input_len;
 		sgl_nsegs = 0;
 		sgl_len = 0;
 	} else {
 		imm_len = 0;
 		sglist_reset(sc->sg_ulptx);
-		if (crda->crd_len != 0) {
+		if (crp->crp_aad_length != 0) {
 			error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
-			    crda->crd_skip, crda->crd_len);
+			    crp->crp_aad_start, crp->crp_aad_length);
 			if (error)
 				return (error);
 		}
 		error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
-		    crde->crd_skip, crde->crd_len);
+		    crp->crp_payload_start, crp->crp_payload_length);
 		if (error)
 			return (error);
 		if (op_type == CHCR_DECRYPT_OP) {
 			error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
-			    crda->crd_inject, hash_size_in_response);
+			    crp->crp_digest_start, hash_size_in_response);
 			if (error)
 				return (error);
 		}
 		sgl_nsegs = sc->sg_ulptx->sg_nseg;
 		sgl_len = ccr_ulptx_sgl_len(sgl_nsegs);
 	}
 
-	if (crda->crd_len != 0) {
+	if (crp->crp_aad_length != 0) {
 		aad_start = iv_len + 1;
-		aad_stop = aad_start + crda->crd_len - 1;
+		aad_stop = aad_start + crp->crp_aad_length - 1;
 	} else {
 		aad_start = 0;
 		aad_stop = 0;
 	}
-	cipher_start = iv_len + crda->crd_len + 1;
+	cipher_start = iv_len + crp->crp_aad_length + 1;
 	if (op_type == CHCR_DECRYPT_OP)
 		cipher_stop = hash_size_in_response;
 	else
 		cipher_stop = 0;
 	if (op_type == CHCR_DECRYPT_OP)
 		auth_insert = hash_size_in_response;
 	else
 		auth_insert = 0;
 
 	wr_len = roundup2(transhdr_len, 16) + iv_len + roundup2(imm_len, 16) +
 	    sgl_len;
 	if (wr_len > SGE_MAX_WR_LEN)
 		return (EFBIG);
 	wr = alloc_wrqe(wr_len, sc->txq);
 	if (wr == NULL) {
 		sc->stats_wr_nomem++;
 		return (ENOMEM);
 	}
 	crwr = wrtod(wr);
 	memset(crwr, 0, wr_len);
 
-	/*
-	 * Read the existing IV from the request or generate a random
-	 * one if none is provided.  Optionally copy the generated IV
-	 * into the output buffer if requested.
-	 *
-	 * If the input IV is 12 bytes, append an explicit 4-byte
-	 * counter of 1.
-	 */
-	if (op_type == CHCR_ENCRYPT_OP) {
-		if (crde->crd_flags & CRD_F_IV_EXPLICIT)
-			memcpy(iv, crde->crd_iv, s->blkcipher.iv_len);
-		else
-			arc4rand(iv, s->blkcipher.iv_len, 0);
-		if ((crde->crd_flags & CRD_F_IV_PRESENT) == 0)
-			crypto_copyback(crp->crp_flags, crp->crp_buf,
-			    crde->crd_inject, s->blkcipher.iv_len, iv);
-	} else {
-		if (crde->crd_flags & CRD_F_IV_EXPLICIT)
-			memcpy(iv, crde->crd_iv, s->blkcipher.iv_len);
-		else
-			crypto_copydata(crp->crp_flags, crp->crp_buf,
-			    crde->crd_inject, s->blkcipher.iv_len, iv);
-	}
+	memcpy(iv, crp->crp_iv, s->blkcipher.iv_len);
 	if (s->blkcipher.iv_len == 12)
 		*(uint32_t *)&iv[12] = htobe32(1);
 
 	ccr_populate_wreq(sc, crwr, kctx_len, wr_len, imm_len, sgl_len, 0,
 	    crp);
 
 	/* XXX: Hardcodes SGE loopback channel of 0. */
 	crwr->sec_cpl.op_ivinsrtofst = htobe32(
 	    V_CPL_TX_SEC_PDU_OPCODE(CPL_TX_SEC_PDU) |
 	    V_CPL_TX_SEC_PDU_RXCHID(sc->tx_channel_id) |
 	    V_CPL_TX_SEC_PDU_ACKFOLLOWS(0) | V_CPL_TX_SEC_PDU_ULPTXLPBK(1) |
 	    V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) |
 	    V_CPL_TX_SEC_PDU_IVINSRTOFST(1));
 
 	crwr->sec_cpl.pldlen = htobe32(iv_len + input_len);
 
 	/*
 	 * NB: cipherstop is explicitly set to 0.  On encrypt it
-	 * should normally be set to 0 anyway (as the encrypt crd ends
-	 * at the end of the input).  However, for decrypt the cipher
-	 * ends before the tag in the AUTHENC case (and authstop is
-	 * set to stop before the tag), but for GCM the cipher still
-	 * runs to the end of the buffer.  Not sure if this is
-	 * intentional or a firmware quirk, but it is required for
-	 * working tag validation with GCM decryption.
+	 * should normally be set to 0 anyway.  However, for decrypt
+	 * the cipher ends before the tag in the ETA case (and
+	 * authstop is set to stop before the tag), but for GCM the
+	 * cipher still runs to the end of the buffer.  Not sure if
+	 * this is intentional or a firmware quirk, but it is required
+	 * for working tag validation with GCM decryption.
 	 */
 	crwr->sec_cpl.aadstart_cipherstop_hi = htobe32(
 	    V_CPL_TX_SEC_PDU_AADSTART(aad_start) |
 	    V_CPL_TX_SEC_PDU_AADSTOP(aad_stop) |
 	    V_CPL_TX_SEC_PDU_CIPHERSTART(cipher_start) |
 	    V_CPL_TX_SEC_PDU_CIPHERSTOP_HI(0));
 	crwr->sec_cpl.cipherstop_lo_authinsert = htobe32(
 	    V_CPL_TX_SEC_PDU_CIPHERSTOP_LO(0) |
 	    V_CPL_TX_SEC_PDU_AUTHSTART(cipher_start) |
 	    V_CPL_TX_SEC_PDU_AUTHSTOP(cipher_stop) |
 	    V_CPL_TX_SEC_PDU_AUTHINSERT(auth_insert));
 
 	/* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
 	hmac_ctrl = ccr_hmac_ctrl(AES_GMAC_HASH_LEN, hash_size_in_response);
 	crwr->sec_cpl.seqno_numivs = htobe32(
 	    V_SCMD_SEQ_NO_CTRL(0) |
 	    V_SCMD_PROTO_VERSION(SCMD_PROTO_VERSION_GENERIC) |
 	    V_SCMD_ENC_DEC_CTRL(op_type) |
 	    V_SCMD_CIPH_AUTH_SEQ_CTRL(op_type == CHCR_ENCRYPT_OP ? 1 : 0) |
 	    V_SCMD_CIPH_MODE(SCMD_CIPH_MODE_AES_GCM) |
 	    V_SCMD_AUTH_MODE(SCMD_AUTH_MODE_GHASH) |
 	    V_SCMD_HMAC_CTRL(hmac_ctrl) |
 	    V_SCMD_IV_SIZE(iv_len / 2) |
 	    V_SCMD_NUM_IVS(0));
 	crwr->sec_cpl.ivgen_hdrlen = htobe32(
 	    V_SCMD_IV_GEN_CTRL(0) |
 	    V_SCMD_MORE_FRAGS(0) | V_SCMD_LAST_FRAG(0) | V_SCMD_MAC_ONLY(0) |
 	    V_SCMD_AADIVDROP(0) | V_SCMD_HDR_LEN(dsgl_len));
 
 	crwr->key_ctx.ctx_hdr = s->blkcipher.key_ctx_hdr;
 	memcpy(crwr->key_ctx.key, s->blkcipher.enckey, s->blkcipher.key_len);
 	dst = crwr->key_ctx.key + roundup2(s->blkcipher.key_len, 16);
 	memcpy(dst, s->gmac.ghash_h, GMAC_BLOCK_LEN);
 
 	dst = (char *)(crwr + 1) + kctx_len;
 	ccr_write_phys_dsgl(sc, dst, dsgl_nsegs);
 	dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len;
 	memcpy(dst, iv, iv_len);
 	dst += iv_len;
 	if (imm_len != 0) {
-		if (crda->crd_len != 0) {
-			crypto_copydata(crp->crp_flags, crp->crp_buf,
-			    crda->crd_skip, crda->crd_len, dst);
-			dst += crda->crd_len;
+		if (crp->crp_aad_length != 0) {
+			crypto_copydata(crp, crp->crp_aad_start,
+			    crp->crp_aad_length, dst);
+			dst += crp->crp_aad_length;
 		}
-		crypto_copydata(crp->crp_flags, crp->crp_buf, crde->crd_skip,
-		    crde->crd_len, dst);
-		dst += crde->crd_len;
+		crypto_copydata(crp, crp->crp_payload_start,
+		    crp->crp_payload_length, dst);
+		dst += crp->crp_payload_length;
 		if (op_type == CHCR_DECRYPT_OP)
-			crypto_copydata(crp->crp_flags, crp->crp_buf,
-			    crda->crd_inject, hash_size_in_response, dst);
+			crypto_copydata(crp, crp->crp_digest_start,
+			    hash_size_in_response, dst);
 	} else
 		ccr_write_ulptx_sgl(sc, dst, sgl_nsegs);
 
 	/* XXX: TODO backpressure */
 	t4_wrq_tx(sc->adapter, wr);
 
 	return (0);
 }
 
 static int
 ccr_gcm_done(struct ccr_softc *sc, struct ccr_session *s,
     struct cryptop *crp, const struct cpl_fw6_pld *cpl, int error)
 {
 
 	/*
 	 * The updated IV to permit chained requests is at
 	 * cpl->data[2], but OCF doesn't permit chained requests.
 	 *
 	 * Note that the hardware should always verify the GMAC hash.
 	 */
 	return (error);
 }
 
 /*
  * Handle a GCM request that is not supported by the crypto engine by
  * performing the operation in software.  Derived from swcr_authenc().
  */
 static void
-ccr_gcm_soft(struct ccr_session *s, struct cryptop *crp,
-    struct cryptodesc *crda, struct cryptodesc *crde)
+ccr_gcm_soft(struct ccr_session *s, struct cryptop *crp)
 {
 	struct auth_hash *axf;
 	struct enc_xform *exf;
 	void *auth_ctx;
 	uint8_t *kschedule;
 	char block[GMAC_BLOCK_LEN];
 	char digest[GMAC_DIGEST_LEN];
 	char iv[AES_BLOCK_LEN];
 	int error, i, len;
 
 	auth_ctx = NULL;
 	kschedule = NULL;
 
 	/* Initialize the MAC. */
 	switch (s->blkcipher.key_len) {
 	case 16:
 		axf = &auth_hash_nist_gmac_aes_128;
 		break;
 	case 24:
 		axf = &auth_hash_nist_gmac_aes_192;
 		break;
 	case 32:
 		axf = &auth_hash_nist_gmac_aes_256;
 		break;
 	default:
 		error = EINVAL;
 		goto out;
 	}
 	auth_ctx = malloc(axf->ctxsize, M_CCR, M_NOWAIT);
 	if (auth_ctx == NULL) {
 		error = ENOMEM;
 		goto out;
 	}
 	axf->Init(auth_ctx);
 	axf->Setkey(auth_ctx, s->blkcipher.enckey, s->blkcipher.key_len);
 
 	/* Initialize the cipher. */
 	exf = &enc_xform_aes_nist_gcm;
 	error = exf->setkey(&kschedule, s->blkcipher.enckey,
 	    s->blkcipher.key_len);
 	if (error)
 		goto out;
 
 	/*
 	 * This assumes a 12-byte IV from the crp.  See longer comment
 	 * above in ccr_gcm() for more details.
 	 */
-	if (crde->crd_flags & CRD_F_ENCRYPT) {
-		if (crde->crd_flags & CRD_F_IV_EXPLICIT)
-			memcpy(iv, crde->crd_iv, 12);
-		else
-			arc4rand(iv, 12, 0);
-		if ((crde->crd_flags & CRD_F_IV_PRESENT) == 0)
-			crypto_copyback(crp->crp_flags, crp->crp_buf,
-			    crde->crd_inject, 12, iv);
-	} else {
-		if (crde->crd_flags & CRD_F_IV_EXPLICIT)
-			memcpy(iv, crde->crd_iv, 12);
-		else
-			crypto_copydata(crp->crp_flags, crp->crp_buf,
-			    crde->crd_inject, 12, iv);
+	if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) {
+		error = EINVAL;
+		goto out;
 	}
+	memcpy(iv, crp->crp_iv, 12);
 	*(uint32_t *)&iv[12] = htobe32(1);
 
 	axf->Reinit(auth_ctx, iv, sizeof(iv));
 
 	/* MAC the AAD. */
-	for (i = 0; i < crda->crd_len; i += sizeof(block)) {
-		len = imin(crda->crd_len - i, sizeof(block));
-		crypto_copydata(crp->crp_flags, crp->crp_buf, crda->crd_skip +
-		    i, len, block);
+	for (i = 0; i < crp->crp_aad_length; i += sizeof(block)) {
+		len = imin(crp->crp_aad_length - i, sizeof(block));
+		crypto_copydata(crp, crp->crp_aad_start + i, len, block);
 		bzero(block + len, sizeof(block) - len);
 		axf->Update(auth_ctx, block, sizeof(block));
 	}
 
 	exf->reinit(kschedule, iv);
 
 	/* Do encryption with MAC */
-	for (i = 0; i < crde->crd_len; i += sizeof(block)) {
-		len = imin(crde->crd_len - i, sizeof(block));
-		crypto_copydata(crp->crp_flags, crp->crp_buf, crde->crd_skip +
-		    i, len, block);
+	for (i = 0; i < crp->crp_payload_length; i += sizeof(block)) {
+		len = imin(crp->crp_payload_length - i, sizeof(block));
+		crypto_copydata(crp, crp->crp_payload_start + i, len, block);
 		bzero(block + len, sizeof(block) - len);
-		if (crde->crd_flags & CRD_F_ENCRYPT) {
+		if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
 			exf->encrypt(kschedule, block);
 			axf->Update(auth_ctx, block, len);
-			crypto_copyback(crp->crp_flags, crp->crp_buf,
-			    crde->crd_skip + i, len, block);
+			crypto_copyback(crp, crp->crp_payload_start + i, len,
+			    block);
 		} else {
 			axf->Update(auth_ctx, block, len);
 		}
 	}
 
 	/* Length block. */
 	bzero(block, sizeof(block));
-	((uint32_t *)block)[1] = htobe32(crda->crd_len * 8);
-	((uint32_t *)block)[3] = htobe32(crde->crd_len * 8);
+	((uint32_t *)block)[1] = htobe32(crp->crp_aad_length * 8);
+	((uint32_t *)block)[3] = htobe32(crp->crp_payload_length * 8);
 	axf->Update(auth_ctx, block, sizeof(block));
 
 	/* Finalize MAC. */
 	axf->Final(digest, auth_ctx);
 
 	/* Inject or validate tag. */
-	if (crde->crd_flags & CRD_F_ENCRYPT) {
-		crypto_copyback(crp->crp_flags, crp->crp_buf, crda->crd_inject,
-		    sizeof(digest), digest);
+	if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
+		crypto_copyback(crp, crp->crp_digest_start, sizeof(digest),
+		    digest);
 		error = 0;
 	} else {
 		char digest2[GMAC_DIGEST_LEN];
 
-		crypto_copydata(crp->crp_flags, crp->crp_buf, crda->crd_inject,
-		    sizeof(digest2), digest2);
+		crypto_copydata(crp, crp->crp_digest_start, sizeof(digest2),
+		    digest2);
 		if (timingsafe_bcmp(digest, digest2, sizeof(digest)) == 0) {
 			error = 0;
 
 			/* Tag matches, decrypt data. */
-			for (i = 0; i < crde->crd_len; i += sizeof(block)) {
-				len = imin(crde->crd_len - i, sizeof(block));
-				crypto_copydata(crp->crp_flags, crp->crp_buf,
-				    crde->crd_skip + i, len, block);
+			for (i = 0; i < crp->crp_payload_length;
+			     i += sizeof(block)) {
+				len = imin(crp->crp_payload_length - i,
+				    sizeof(block));
+				crypto_copydata(crp, crp->crp_payload_start + i,
+				    len, block);
 				bzero(block + len, sizeof(block) - len);
 				exf->decrypt(kschedule, block);
-				crypto_copyback(crp->crp_flags, crp->crp_buf,
-				    crde->crd_skip + i, len, block);
+				crypto_copyback(crp, crp->crp_payload_start + i,
+				    len, block);
 			}
 		} else
 			error = EBADMSG;
 	}
 
 	exf->zerokey(&kschedule);
 out:
 	if (auth_ctx != NULL) {
 		memset(auth_ctx, 0, axf->ctxsize);
 		free(auth_ctx, M_CCR);
 	}
 	crp->crp_etype = error;
 	crypto_done(crp);
 }
 
 static void
-generate_ccm_b0(struct cryptodesc *crda, struct cryptodesc *crde,
-    u_int hash_size_in_response, const char *iv, char *b0)
+generate_ccm_b0(struct cryptop *crp, u_int hash_size_in_response,
+    const char *iv, char *b0)
 {
 	u_int i, payload_len;
 
 	/* NB: L is already set in the first byte of the IV. */
 	memcpy(b0, iv, CCM_B0_SIZE);
 
 	/* Set length of hash in bits 3 - 5. */
 	b0[0] |= (((hash_size_in_response - 2) / 2) << 3);
 
 	/* Store the payload length as a big-endian value. */
-	payload_len = crde->crd_len;
+	payload_len = crp->crp_payload_length;
 	for (i = 0; i < iv[0]; i++) {
 		b0[CCM_CBC_BLOCK_LEN - 1 - i] = payload_len;
 		payload_len >>= 8;
 	}
 
 	/*
 	 * If there is AAD in the request, set bit 6 in the flags
 	 * field and store the AAD length as a big-endian value at the
 	 * start of block 1.  This only assumes a 16-bit AAD length
 	 * since T6 doesn't support large AAD sizes.
 	 */
-	if (crda->crd_len != 0) {
+	if (crp->crp_aad_length != 0) {
 		b0[0] |= (1 << 6);
-		*(uint16_t *)(b0 + CCM_B0_SIZE) = htobe16(crda->crd_len);
+		*(uint16_t *)(b0 + CCM_B0_SIZE) = htobe16(crp->crp_aad_length);
 	}
 }
 
 static int
-ccr_ccm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp,
-    struct cryptodesc *crda, struct cryptodesc *crde)
+ccr_ccm(struct ccr_softc *sc, struct ccr_session *s, struct cryptop *crp)
 {
 	char iv[CHCR_MAX_CRYPTO_IV_LEN];
 	struct ulptx_idata *idata;
 	struct chcr_wr *crwr;
 	struct wrqe *wr;
 	char *dst;
 	u_int iv_len, kctx_len, op_type, transhdr_len, wr_len;
 	u_int aad_len, b0_len, hash_size_in_response, imm_len;
 	u_int aad_start, aad_stop, cipher_start, cipher_stop, auth_insert;
 	u_int hmac_ctrl, input_len;
 	int dsgl_nsegs, dsgl_len;
 	int sgl_nsegs, sgl_len;
 	int error;
 
 	if (s->blkcipher.key_len == 0)
 		return (EINVAL);
 
 	/*
 	 * The crypto engine doesn't handle CCM requests with an empty
 	 * payload, so handle those in software instead.
 	 */
-	if (crde->crd_len == 0)
+	if (crp->crp_payload_length == 0)
 		return (EMSGSIZE);
 
 	/*
-	 * AAD is only permitted before the cipher/plain text, not
-	 * after.
-	 */
-	if (crda->crd_len + crda->crd_skip > crde->crd_len + crde->crd_skip)
-		return (EMSGSIZE);
-
-	/*
 	 * CCM always includes block 0 in the AAD before AAD from the
 	 * request.
 	 */
 	b0_len = CCM_B0_SIZE;
-	if (crda->crd_len != 0)
+	if (crp->crp_aad_length != 0)
 		b0_len += CCM_AAD_FIELD_SIZE;
-	aad_len = b0_len + crda->crd_len;
+	aad_len = b0_len + crp->crp_aad_length;
 
 	/*
-	 * Always assume a 12 byte input IV for now since that is what
-	 * OCF always generates.  The full IV in the work request is
-	 * 16 bytes.
+	 * CCM requests should always provide an explicit IV (really
+	 * the nonce).
 	 */
+	if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0)
+		return (EINVAL);
+
+	/*
+	 * Always assume a 12 byte input nonce for now since that is
+	 * what OCF always generates.  The full IV in the work request
+	 * is 16 bytes.
+	 */
 	iv_len = AES_BLOCK_LEN;
 
 	if (iv_len + aad_len > MAX_AAD_LEN)
 		return (EMSGSIZE);
 
 	hash_size_in_response = s->ccm_mac.hash_len;
-	if (crde->crd_flags & CRD_F_ENCRYPT)
+	if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
 		op_type = CHCR_ENCRYPT_OP;
 	else
 		op_type = CHCR_DECRYPT_OP;
 
 	/*
 	 * The output buffer consists of the cipher text followed by
 	 * the tag when encrypting.  For decryption it only contains
 	 * the plain text.
 	 *
 	 * Due to a firmware bug, the output buffer must include a
 	 * dummy output buffer for the IV and AAD prior to the real
 	 * output buffer.
 	 */
 	if (op_type == CHCR_ENCRYPT_OP) {
-		if (iv_len + aad_len + crde->crd_len + hash_size_in_response >
-		    MAX_REQUEST_SIZE)
+		if (iv_len + aad_len + crp->crp_payload_length +
+		    hash_size_in_response > MAX_REQUEST_SIZE)
 			return (EFBIG);
 	} else {
-		if (iv_len + aad_len + crde->crd_len > MAX_REQUEST_SIZE)
+		if (iv_len + aad_len + crp->crp_payload_length >
+		    MAX_REQUEST_SIZE)
 			return (EFBIG);
 	}
 	sglist_reset(sc->sg_dsgl);
 	error = sglist_append_sglist(sc->sg_dsgl, sc->sg_iv_aad, 0, iv_len +
 	    aad_len);
 	if (error)
 		return (error);
-	error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp, crde->crd_skip,
-	    crde->crd_len);
+	error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp,
+	    crp->crp_payload_start, crp->crp_payload_length);
 	if (error)
 		return (error);
 	if (op_type == CHCR_ENCRYPT_OP) {
 		error = sglist_append_sglist(sc->sg_dsgl, sc->sg_crp,
-		    crda->crd_inject, hash_size_in_response);
+		    crp->crp_digest_start, hash_size_in_response);
 		if (error)
 			return (error);
 	}
 	dsgl_nsegs = ccr_count_sgl(sc->sg_dsgl, DSGL_SGE_MAXLEN);
 	if (dsgl_nsegs > MAX_RX_PHYS_DSGL_SGE)
 		return (EFBIG);
 	dsgl_len = ccr_phys_dsgl_len(dsgl_nsegs);
 
 	/*
 	 * The 'key' part of the key context consists of two copies of
 	 * the AES key.
 	 */
 	kctx_len = roundup2(s->blkcipher.key_len, 16) * 2;
 	transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dsgl_len);
 
 	/*
 	 * The input buffer consists of the IV, AAD (including block
 	 * 0), and then the cipher/plain text.  For decryption
 	 * requests the hash is appended after the cipher text.
 	 *
 	 * The IV is always stored at the start of the input buffer
 	 * even though it may be duplicated in the payload.  The
 	 * crypto engine doesn't work properly if the IV offset points
 	 * inside of the AAD region, so a second copy is always
 	 * required.
 	 */
-	input_len = aad_len + crde->crd_len;
+	input_len = aad_len + crp->crp_payload_length;
 	if (op_type == CHCR_DECRYPT_OP)
 		input_len += hash_size_in_response;
 	if (input_len > MAX_REQUEST_SIZE)
 		return (EFBIG);
 	if (ccr_use_imm_data(transhdr_len, iv_len + input_len)) {
 		imm_len = input_len;
 		sgl_nsegs = 0;
 		sgl_len = 0;
 	} else {
 		/* Block 0 is passed as immediate data. */
 		imm_len = b0_len;
 
 		sglist_reset(sc->sg_ulptx);
-		if (crda->crd_len != 0) {
+		if (crp->crp_aad_length != 0) {
 			error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
-			    crda->crd_skip, crda->crd_len);
+			    crp->crp_aad_start, crp->crp_aad_length);
 			if (error)
 				return (error);
 		}
 		error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
-		    crde->crd_skip, crde->crd_len);
+		    crp->crp_payload_start, crp->crp_payload_length);
 		if (error)
 			return (error);
 		if (op_type == CHCR_DECRYPT_OP) {
 			error = sglist_append_sglist(sc->sg_ulptx, sc->sg_crp,
-			    crda->crd_inject, hash_size_in_response);
+			    crp->crp_digest_start, hash_size_in_response);
 			if (error)
 				return (error);
 		}
 		sgl_nsegs = sc->sg_ulptx->sg_nseg;
 		sgl_len = ccr_ulptx_sgl_len(sgl_nsegs);
 	}
 
 	aad_start = iv_len + 1;
 	aad_stop = aad_start + aad_len - 1;
 	cipher_start = aad_stop + 1;
 	if (op_type == CHCR_DECRYPT_OP)
 		cipher_stop = hash_size_in_response;
 	else
 		cipher_stop = 0;
 	if (op_type == CHCR_DECRYPT_OP)
 		auth_insert = hash_size_in_response;
 	else
 		auth_insert = 0;
 
 	wr_len = roundup2(transhdr_len, 16) + iv_len + roundup2(imm_len, 16) +
 	    sgl_len;
 	if (wr_len > SGE_MAX_WR_LEN)
 		return (EFBIG);
 	wr = alloc_wrqe(wr_len, sc->txq);
 	if (wr == NULL) {
 		sc->stats_wr_nomem++;
 		return (ENOMEM);
 	}
 	crwr = wrtod(wr);
 	memset(crwr, 0, wr_len);
 
 	/*
-	 * Read the nonce from the request or generate a random one if
-	 * none is provided.  Use the nonce to generate the full IV
-	 * with the counter set to 0.
+	 * Read the nonce from the request.  Use the nonce to generate
+	 * the full IV with the counter set to 0.
 	 */
 	memset(iv, 0, iv_len);
 	iv[0] = (15 - AES_CCM_IV_LEN) - 1;
-	if (op_type == CHCR_ENCRYPT_OP) {
-		if (crde->crd_flags & CRD_F_IV_EXPLICIT)
-			memcpy(iv + 1, crde->crd_iv, AES_CCM_IV_LEN);
-		else
-			arc4rand(iv + 1, AES_CCM_IV_LEN, 0);
-		if ((crde->crd_flags & CRD_F_IV_PRESENT) == 0)
-			crypto_copyback(crp->crp_flags, crp->crp_buf,
-			    crde->crd_inject, AES_CCM_IV_LEN, iv + 1);
-	} else {
-		if (crde->crd_flags & CRD_F_IV_EXPLICIT)
-			memcpy(iv + 1, crde->crd_iv, AES_CCM_IV_LEN);
-		else
-			crypto_copydata(crp->crp_flags, crp->crp_buf,
-			    crde->crd_inject, AES_CCM_IV_LEN, iv + 1);
-	}
+	memcpy(iv + 1, crp->crp_iv, AES_CCM_IV_LEN);
 
 	ccr_populate_wreq(sc, crwr, kctx_len, wr_len, imm_len, sgl_len, 0,
 	    crp);
 
 	/* XXX: Hardcodes SGE loopback channel of 0. */
 	crwr->sec_cpl.op_ivinsrtofst = htobe32(
 	    V_CPL_TX_SEC_PDU_OPCODE(CPL_TX_SEC_PDU) |
 	    V_CPL_TX_SEC_PDU_RXCHID(sc->tx_channel_id) |
 	    V_CPL_TX_SEC_PDU_ACKFOLLOWS(0) | V_CPL_TX_SEC_PDU_ULPTXLPBK(1) |
 	    V_CPL_TX_SEC_PDU_CPLLEN(2) | V_CPL_TX_SEC_PDU_PLACEHOLDER(0) |
 	    V_CPL_TX_SEC_PDU_IVINSRTOFST(1));
 
 	crwr->sec_cpl.pldlen = htobe32(iv_len + input_len);
 
 	/*
 	 * NB: cipherstop is explicitly set to 0.  See comments above
 	 * in ccr_gcm().
 	 */
 	crwr->sec_cpl.aadstart_cipherstop_hi = htobe32(
 	    V_CPL_TX_SEC_PDU_AADSTART(aad_start) |
 	    V_CPL_TX_SEC_PDU_AADSTOP(aad_stop) |
 	    V_CPL_TX_SEC_PDU_CIPHERSTART(cipher_start) |
 	    V_CPL_TX_SEC_PDU_CIPHERSTOP_HI(0));
 	crwr->sec_cpl.cipherstop_lo_authinsert = htobe32(
 	    V_CPL_TX_SEC_PDU_CIPHERSTOP_LO(0) |
 	    V_CPL_TX_SEC_PDU_AUTHSTART(cipher_start) |
 	    V_CPL_TX_SEC_PDU_AUTHSTOP(cipher_stop) |
 	    V_CPL_TX_SEC_PDU_AUTHINSERT(auth_insert));
 
 	/* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
 	hmac_ctrl = ccr_hmac_ctrl(AES_CBC_MAC_HASH_LEN, hash_size_in_response);
 	crwr->sec_cpl.seqno_numivs = htobe32(
 	    V_SCMD_SEQ_NO_CTRL(0) |
 	    V_SCMD_PROTO_VERSION(SCMD_PROTO_VERSION_GENERIC) |
 	    V_SCMD_ENC_DEC_CTRL(op_type) |
 	    V_SCMD_CIPH_AUTH_SEQ_CTRL(op_type == CHCR_ENCRYPT_OP ? 0 : 1) |
 	    V_SCMD_CIPH_MODE(SCMD_CIPH_MODE_AES_CCM) |
 	    V_SCMD_AUTH_MODE(SCMD_AUTH_MODE_CBCMAC) |
 	    V_SCMD_HMAC_CTRL(hmac_ctrl) |
 	    V_SCMD_IV_SIZE(iv_len / 2) |
 	    V_SCMD_NUM_IVS(0));
 	crwr->sec_cpl.ivgen_hdrlen = htobe32(
 	    V_SCMD_IV_GEN_CTRL(0) |
 	    V_SCMD_MORE_FRAGS(0) | V_SCMD_LAST_FRAG(0) | V_SCMD_MAC_ONLY(0) |
 	    V_SCMD_AADIVDROP(0) | V_SCMD_HDR_LEN(dsgl_len));
 
 	crwr->key_ctx.ctx_hdr = s->blkcipher.key_ctx_hdr;
 	memcpy(crwr->key_ctx.key, s->blkcipher.enckey, s->blkcipher.key_len);
 	memcpy(crwr->key_ctx.key + roundup(s->blkcipher.key_len, 16),
 	    s->blkcipher.enckey, s->blkcipher.key_len);
 
 	dst = (char *)(crwr + 1) + kctx_len;
 	ccr_write_phys_dsgl(sc, dst, dsgl_nsegs);
 	dst += sizeof(struct cpl_rx_phys_dsgl) + dsgl_len;
 	memcpy(dst, iv, iv_len);
 	dst += iv_len;
-	generate_ccm_b0(crda, crde, hash_size_in_response, iv, dst);
+	generate_ccm_b0(crp, hash_size_in_response, iv, dst);
 	if (sgl_nsegs == 0) {
 		dst += b0_len;
-		if (crda->crd_len != 0) {
-			crypto_copydata(crp->crp_flags, crp->crp_buf,
-			    crda->crd_skip, crda->crd_len, dst);
-			dst += crda->crd_len;
+		if (crp->crp_aad_length != 0) {
+			crypto_copydata(crp, crp->crp_aad_start,
+			    crp->crp_aad_length, dst);
+			dst += crp->crp_aad_length;
 		}
-		crypto_copydata(crp->crp_flags, crp->crp_buf, crde->crd_skip,
-		    crde->crd_len, dst);
-		dst += crde->crd_len;
+		crypto_copydata(crp, crp->crp_payload_start,
+		    crp->crp_payload_length, dst);
+		dst += crp->crp_payload_length;
 		if (op_type == CHCR_DECRYPT_OP)
-			crypto_copydata(crp->crp_flags, crp->crp_buf,
-			    crda->crd_inject, hash_size_in_response, dst);
+			crypto_copydata(crp, crp->crp_digest_start,
+			    hash_size_in_response, dst);
 	} else {
 		dst += CCM_B0_SIZE;
 		if (b0_len > CCM_B0_SIZE) {
 			/*
 			 * If there is AAD, insert padding including a
 			 * ULP_TX_SC_NOOP so that the ULP_TX_SC_DSGL
 			 * is 16-byte aligned.
 			 */
 			KASSERT(b0_len - CCM_B0_SIZE == CCM_AAD_FIELD_SIZE,
 			    ("b0_len mismatch"));
 			memset(dst + CCM_AAD_FIELD_SIZE, 0,
 			    8 - CCM_AAD_FIELD_SIZE);
 			idata = (void *)(dst + 8);
 			idata->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP));
 			idata->len = htobe32(0);
 			dst = (void *)(idata + 1);
 		}
 		ccr_write_ulptx_sgl(sc, dst, sgl_nsegs);
 	}
 
 	/* XXX: TODO backpressure */
 	t4_wrq_tx(sc->adapter, wr);
 
 	return (0);
 }
 
 static int
 ccr_ccm_done(struct ccr_softc *sc, struct ccr_session *s,
     struct cryptop *crp, const struct cpl_fw6_pld *cpl, int error)
 {
 
 	/*
 	 * The updated IV to permit chained requests is at
 	 * cpl->data[2], but OCF doesn't permit chained requests.
 	 *
 	 * Note that the hardware should always verify the CBC MAC
 	 * hash.
 	 */
 	return (error);
 }
 
 /*
  * Handle a CCM request that is not supported by the crypto engine by
  * performing the operation in software.  Derived from swcr_authenc().
  */
 static void
-ccr_ccm_soft(struct ccr_session *s, struct cryptop *crp,
-    struct cryptodesc *crda, struct cryptodesc *crde)
+ccr_ccm_soft(struct ccr_session *s, struct cryptop *crp)
 {
 	struct auth_hash *axf;
 	struct enc_xform *exf;
 	union authctx *auth_ctx;
 	uint8_t *kschedule;
 	char block[CCM_CBC_BLOCK_LEN];
 	char digest[AES_CBC_MAC_HASH_LEN];
 	char iv[AES_CCM_IV_LEN];
 	int error, i, len;
 
 	auth_ctx = NULL;
 	kschedule = NULL;
 
 	/* Initialize the MAC. */
 	switch (s->blkcipher.key_len) {
 	case 16:
 		axf = &auth_hash_ccm_cbc_mac_128;
 		break;
 	case 24:
 		axf = &auth_hash_ccm_cbc_mac_192;
 		break;
 	case 32:
 		axf = &auth_hash_ccm_cbc_mac_256;
 		break;
 	default:
 		error = EINVAL;
 		goto out;
 	}
 	auth_ctx = malloc(axf->ctxsize, M_CCR, M_NOWAIT);
 	if (auth_ctx == NULL) {
 		error = ENOMEM;
 		goto out;
 	}
 	axf->Init(auth_ctx);
 	axf->Setkey(auth_ctx, s->blkcipher.enckey, s->blkcipher.key_len);
 
 	/* Initialize the cipher. */
 	exf = &enc_xform_ccm;
 	error = exf->setkey(&kschedule, s->blkcipher.enckey,
 	    s->blkcipher.key_len);
 	if (error)
 		goto out;
 
-	if (crde->crd_flags & CRD_F_ENCRYPT) {
-		if (crde->crd_flags & CRD_F_IV_EXPLICIT)
-			memcpy(iv, crde->crd_iv, AES_CCM_IV_LEN);
-		else
-			arc4rand(iv, AES_CCM_IV_LEN, 0);
-		if ((crde->crd_flags & CRD_F_IV_PRESENT) == 0)
-			crypto_copyback(crp->crp_flags, crp->crp_buf,
-			    crde->crd_inject, AES_CCM_IV_LEN, iv);
-	} else {
-		if (crde->crd_flags & CRD_F_IV_EXPLICIT)
-			memcpy(iv, crde->crd_iv, AES_CCM_IV_LEN);
-		else
-			crypto_copydata(crp->crp_flags, crp->crp_buf,
-			    crde->crd_inject, AES_CCM_IV_LEN, iv);
+	if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0) {
+		error = EINVAL;
+		goto out;
 	}
+	memcpy(iv, crp->crp_iv, AES_CCM_IV_LEN);
 
-	auth_ctx->aes_cbc_mac_ctx.authDataLength = crda->crd_len;
-	auth_ctx->aes_cbc_mac_ctx.cryptDataLength = crde->crd_len;
+	auth_ctx->aes_cbc_mac_ctx.authDataLength = crp->crp_aad_length;
+	auth_ctx->aes_cbc_mac_ctx.cryptDataLength = crp->crp_payload_length;
 	axf->Reinit(auth_ctx, iv, sizeof(iv));
 
 	/* MAC the AAD. */
-	for (i = 0; i < crda->crd_len; i += sizeof(block)) {
-		len = imin(crda->crd_len - i, sizeof(block));
-		crypto_copydata(crp->crp_flags, crp->crp_buf, crda->crd_skip +
-		    i, len, block);
+	for (i = 0; i < crp->crp_aad_length; i += sizeof(block)) {
+		len = imin(crp->crp_aad_length - i, sizeof(block));
+		crypto_copydata(crp, crp->crp_aad_start + i, len, block);
 		bzero(block + len, sizeof(block) - len);
 		axf->Update(auth_ctx, block, sizeof(block));
 	}
 
 	exf->reinit(kschedule, iv);
 
 	/* Do encryption/decryption with MAC */
-	for (i = 0; i < crde->crd_len; i += sizeof(block)) {
-		len = imin(crde->crd_len - i, sizeof(block));
-		crypto_copydata(crp->crp_flags, crp->crp_buf, crde->crd_skip +
-		    i, len, block);
+	for (i = 0; i < crp->crp_payload_length; i += sizeof(block)) {
+		len = imin(crp->crp_payload_length - i, sizeof(block));
+		crypto_copydata(crp, crp->crp_payload_start + i, len, block);
 		bzero(block + len, sizeof(block) - len);
-		if (crde->crd_flags & CRD_F_ENCRYPT) {
+		if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
 			axf->Update(auth_ctx, block, len);
 			exf->encrypt(kschedule, block);
-			crypto_copyback(crp->crp_flags, crp->crp_buf,
-			    crde->crd_skip + i, len, block);
+			crypto_copyback(crp, crp->crp_payload_start + i, len,
+			    block);
 		} else {
 			exf->decrypt(kschedule, block);
 			axf->Update(auth_ctx, block, len);
 		}
 	}
 
 	/* Finalize MAC. */
 	axf->Final(digest, auth_ctx);
 
 	/* Inject or validate tag. */
-	if (crde->crd_flags & CRD_F_ENCRYPT) {
-		crypto_copyback(crp->crp_flags, crp->crp_buf, crda->crd_inject,
-		    sizeof(digest), digest);
+	if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
+		crypto_copyback(crp, crp->crp_digest_start, sizeof(digest),
+		    digest);
 		error = 0;
 	} else {
-		char digest2[GMAC_DIGEST_LEN];
+		char digest2[AES_CBC_MAC_HASH_LEN];
 
-		crypto_copydata(crp->crp_flags, crp->crp_buf, crda->crd_inject,
-		    sizeof(digest2), digest2);
+		crypto_copydata(crp, crp->crp_digest_start, sizeof(digest2),
+		    digest2);
 		if (timingsafe_bcmp(digest, digest2, sizeof(digest)) == 0) {
 			error = 0;
 
 			/* Tag matches, decrypt data. */
 			exf->reinit(kschedule, iv);
-			for (i = 0; i < crde->crd_len; i += sizeof(block)) {
-				len = imin(crde->crd_len - i, sizeof(block));
-				crypto_copydata(crp->crp_flags, crp->crp_buf,
-				    crde->crd_skip + i, len, block);
+			for (i = 0; i < crp->crp_payload_length;
+			     i += sizeof(block)) {
+				len = imin(crp->crp_payload_length - i,
+				    sizeof(block));
+				crypto_copydata(crp, crp->crp_payload_start + i,
+				    len, block);
 				bzero(block + len, sizeof(block) - len);
 				exf->decrypt(kschedule, block);
-				crypto_copyback(crp->crp_flags, crp->crp_buf,
-				    crde->crd_skip + i, len, block);
+				crypto_copyback(crp, crp->crp_payload_start + i,
+				    len, block);
 			}
 		} else
 			error = EBADMSG;
 	}
 
 	exf->zerokey(&kschedule);
 out:
 	if (auth_ctx != NULL) {
 		memset(auth_ctx, 0, axf->ctxsize);
 		free(auth_ctx, M_CCR);
 	}
 	crp->crp_etype = error;
 	crypto_done(crp);
 }
 
 static void
 ccr_identify(driver_t *driver, device_t parent)
 {
 	struct adapter *sc;
 
 	sc = device_get_softc(parent);
 	if (sc->cryptocaps & FW_CAPS_CONFIG_CRYPTO_LOOKASIDE &&
 	    device_find_child(parent, "ccr", -1) == NULL)
 		device_add_child(parent, "ccr", -1);
 }
 
 static int
 ccr_probe(device_t dev)
 {
 
 	device_set_desc(dev, "Chelsio Crypto Accelerator");
 	return (BUS_PROBE_DEFAULT);
 }
 
 static void
 ccr_sysctls(struct ccr_softc *sc)
 {
 	struct sysctl_ctx_list *ctx;
 	struct sysctl_oid *oid;
 	struct sysctl_oid_list *children;
 
 	ctx = device_get_sysctl_ctx(sc->dev);
 
 	/*
 	 * dev.ccr.X.
 	 */
 	oid = device_get_sysctl_tree(sc->dev);
 	children = SYSCTL_CHILDREN(oid);
 
 	/*
 	 * dev.ccr.X.stats.
 	 */
 	oid = SYSCTL_ADD_NODE(ctx, children, OID_AUTO, "stats",
 	    CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "statistics");
 	children = SYSCTL_CHILDREN(oid);
 
 	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "hash", CTLFLAG_RD,
 	    &sc->stats_hash, 0, "Hash requests submitted");
 	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "hmac", CTLFLAG_RD,
 	    &sc->stats_hmac, 0, "HMAC requests submitted");
 	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "cipher_encrypt", CTLFLAG_RD,
 	    &sc->stats_blkcipher_encrypt, 0,
 	    "Cipher encryption requests submitted");
 	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "cipher_decrypt", CTLFLAG_RD,
 	    &sc->stats_blkcipher_decrypt, 0,
 	    "Cipher decryption requests submitted");
-	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "authenc_encrypt", CTLFLAG_RD,
-	    &sc->stats_authenc_encrypt, 0,
+	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "eta_encrypt", CTLFLAG_RD,
+	    &sc->stats_eta_encrypt, 0,
 	    "Combined AES+HMAC encryption requests submitted");
-	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "authenc_decrypt", CTLFLAG_RD,
-	    &sc->stats_authenc_decrypt, 0,
+	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "eta_decrypt", CTLFLAG_RD,
+	    &sc->stats_eta_decrypt, 0,
 	    "Combined AES+HMAC decryption requests submitted");
 	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "gcm_encrypt", CTLFLAG_RD,
 	    &sc->stats_gcm_encrypt, 0, "AES-GCM encryption requests submitted");
 	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "gcm_decrypt", CTLFLAG_RD,
 	    &sc->stats_gcm_decrypt, 0, "AES-GCM decryption requests submitted");
 	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "ccm_encrypt", CTLFLAG_RD,
 	    &sc->stats_ccm_encrypt, 0, "AES-CCM encryption requests submitted");
 	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "ccm_decrypt", CTLFLAG_RD,
 	    &sc->stats_ccm_decrypt, 0, "AES-CCM decryption requests submitted");
 	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "wr_nomem", CTLFLAG_RD,
 	    &sc->stats_wr_nomem, 0, "Work request memory allocation failures");
 	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "inflight", CTLFLAG_RD,
 	    &sc->stats_inflight, 0, "Requests currently pending");
 	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "mac_error", CTLFLAG_RD,
 	    &sc->stats_mac_error, 0, "MAC errors");
 	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "pad_error", CTLFLAG_RD,
 	    &sc->stats_pad_error, 0, "Padding errors");
 	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "bad_session", CTLFLAG_RD,
 	    &sc->stats_bad_session, 0, "Requests with invalid session ID");
 	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "sglist_error", CTLFLAG_RD,
 	    &sc->stats_sglist_error, 0,
 	    "Requests for which DMA mapping failed");
 	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "process_error", CTLFLAG_RD,
 	    &sc->stats_process_error, 0, "Requests failed during queueing");
 	SYSCTL_ADD_U64(ctx, children, OID_AUTO, "sw_fallback", CTLFLAG_RD,
 	    &sc->stats_sw_fallback, 0,
 	    "Requests processed by falling back to software");
 }
 
 static int
 ccr_attach(device_t dev)
 {
 	struct ccr_softc *sc;
 	int32_t cid;
 
 	sc = device_get_softc(dev);
 	sc->dev = dev;
 	sc->adapter = device_get_softc(device_get_parent(dev));
 	sc->txq = &sc->adapter->sge.ctrlq[0];
 	sc->rxq = &sc->adapter->sge.rxq[0];
 	cid = crypto_get_driverid(dev, sizeof(struct ccr_session),
 	    CRYPTOCAP_F_HARDWARE);
 	if (cid < 0) {
 		device_printf(dev, "could not get crypto driver id\n");
 		return (ENXIO);
 	}
 	sc->cid = cid;
 	sc->adapter->ccr_softc = sc;
 
 	/* XXX: TODO? */
 	sc->tx_channel_id = 0;
 
 	mtx_init(&sc->lock, "ccr", NULL, MTX_DEF);
 	sc->sg_crp = sglist_alloc(TX_SGL_SEGS, M_WAITOK);
 	sc->sg_ulptx = sglist_alloc(TX_SGL_SEGS, M_WAITOK);
 	sc->sg_dsgl = sglist_alloc(MAX_RX_PHYS_DSGL_SGE, M_WAITOK);
 	sc->iv_aad_buf = malloc(MAX_AAD_LEN, M_CCR, M_WAITOK);
 	sc->sg_iv_aad = sglist_build(sc->iv_aad_buf, MAX_AAD_LEN, M_WAITOK);
 	ccr_sysctls(sc);
 
-	crypto_register(cid, CRYPTO_SHA1, 0, 0);
-	crypto_register(cid, CRYPTO_SHA2_224, 0, 0);
-	crypto_register(cid, CRYPTO_SHA2_256, 0, 0);
-	crypto_register(cid, CRYPTO_SHA2_384, 0, 0);
-	crypto_register(cid, CRYPTO_SHA2_512, 0, 0);
-	crypto_register(cid, CRYPTO_SHA1_HMAC, 0, 0);
-	crypto_register(cid, CRYPTO_SHA2_224_HMAC, 0, 0);
-	crypto_register(cid, CRYPTO_SHA2_256_HMAC, 0, 0);
-	crypto_register(cid, CRYPTO_SHA2_384_HMAC, 0, 0);
-	crypto_register(cid, CRYPTO_SHA2_512_HMAC, 0, 0);
-	crypto_register(cid, CRYPTO_AES_CBC, 0, 0);
-	crypto_register(cid, CRYPTO_AES_ICM, 0, 0);
-	crypto_register(cid, CRYPTO_AES_NIST_GCM_16, 0, 0);
-	crypto_register(cid, CRYPTO_AES_128_NIST_GMAC, 0, 0);
-	crypto_register(cid, CRYPTO_AES_192_NIST_GMAC, 0, 0);
-	crypto_register(cid, CRYPTO_AES_256_NIST_GMAC, 0, 0);
-	crypto_register(cid, CRYPTO_AES_XTS, 0, 0);
-	crypto_register(cid, CRYPTO_AES_CCM_16, 0, 0);
-	crypto_register(cid, CRYPTO_AES_CCM_CBC_MAC, 0, 0);
 	return (0);
 }
 
 static int
 ccr_detach(device_t dev)
 {
 	struct ccr_softc *sc;
 
 	sc = device_get_softc(dev);
 
 	mtx_lock(&sc->lock);
 	sc->detaching = true;
 	mtx_unlock(&sc->lock);
 
 	crypto_unregister_all(sc->cid);
 
 	mtx_destroy(&sc->lock);
 	sglist_free(sc->sg_iv_aad);
 	free(sc->iv_aad_buf, M_CCR);
 	sglist_free(sc->sg_dsgl);
 	sglist_free(sc->sg_ulptx);
 	sglist_free(sc->sg_crp);
 	sc->adapter->ccr_softc = NULL;
 	return (0);
 }
 
 static void
-ccr_init_hash_digest(struct ccr_session *s, int cri_alg)
+ccr_init_hash_digest(struct ccr_session *s)
 {
 	union authctx auth_ctx;
 	struct auth_hash *axf;
 
 	axf = s->hmac.auth_hash;
 	axf->Init(&auth_ctx);
-	t4_copy_partial_hash(cri_alg, &auth_ctx, s->hmac.pads);
+	t4_copy_partial_hash(axf->type, &auth_ctx, s->hmac.pads);
 }
 
-static int
+static bool
 ccr_aes_check_keylen(int alg, int klen)
 {
 
-	switch (klen) {
+	switch (klen * 8) {
 	case 128:
 	case 192:
 		if (alg == CRYPTO_AES_XTS)
-			return (EINVAL);
+			return (false);
 		break;
 	case 256:
 		break;
 	case 512:
 		if (alg != CRYPTO_AES_XTS)
-			return (EINVAL);
+			return (false);
 		break;
 	default:
-		return (EINVAL);
+		return (false);
 	}
-	return (0);
+	return (true);
 }
 
 static void
-ccr_aes_setkey(struct ccr_session *s, int alg, const void *key, int klen)
+ccr_aes_setkey(struct ccr_session *s, const void *key, int klen)
 {
 	unsigned int ck_size, iopad_size, kctx_flits, kctx_len, kbits, mk_size;
 	unsigned int opad_present;
 
-	if (alg == CRYPTO_AES_XTS)
-		kbits = klen / 2;
+	if (s->blkcipher.cipher_mode == SCMD_CIPH_MODE_AES_XTS)
+		kbits = (klen / 2) * 8;
 	else
-		kbits = klen;
+		kbits = klen * 8;
 	switch (kbits) {
 	case 128:
 		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
 		break;
 	case 192:
 		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
 		break;
 	case 256:
 		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
 		break;
 	default:
 		panic("should not get here");
 	}
 
-	s->blkcipher.key_len = klen / 8;
+	s->blkcipher.key_len = klen;
 	memcpy(s->blkcipher.enckey, key, s->blkcipher.key_len);
-	switch (alg) {
-	case CRYPTO_AES_CBC:
-	case CRYPTO_AES_XTS:
+	switch (s->blkcipher.cipher_mode) {
+	case SCMD_CIPH_MODE_AES_CBC:
+	case SCMD_CIPH_MODE_AES_XTS:
 		t4_aes_getdeckey(s->blkcipher.deckey, key, kbits);
 		break;
 	}
 
 	kctx_len = roundup2(s->blkcipher.key_len, 16);
 	switch (s->mode) {
-	case AUTHENC:
+	case ETA:
 		mk_size = s->hmac.mk_size;
 		opad_present = 1;
 		iopad_size = roundup2(s->hmac.partial_digest_len, 16);
 		kctx_len += iopad_size * 2;
 		break;
 	case GCM:
 		mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_128;
 		opad_present = 0;
 		kctx_len += GMAC_BLOCK_LEN;
 		break;
 	case CCM:
 		switch (kbits) {
 		case 128:
 			mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_128;
 			break;
 		case 192:
 			mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_192;
 			break;
 		case 256:
 			mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
 			break;
 		default:
 			panic("should not get here");
 		}
 		opad_present = 0;
 		kctx_len *= 2;
 		break;
 	default:
 		mk_size = CHCR_KEYCTX_NO_KEY;
 		opad_present = 0;
 		break;
 	}
 	kctx_flits = (sizeof(struct _key_ctx) + kctx_len) / 16;
 	s->blkcipher.key_ctx_hdr = htobe32(V_KEY_CONTEXT_CTX_LEN(kctx_flits) |
-	    V_KEY_CONTEXT_DUAL_CK(alg == CRYPTO_AES_XTS) |
+	    V_KEY_CONTEXT_DUAL_CK(s->blkcipher.cipher_mode ==
+	    SCMD_CIPH_MODE_AES_XTS) |
 	    V_KEY_CONTEXT_OPAD_PRESENT(opad_present) |
 	    V_KEY_CONTEXT_SALT_PRESENT(1) | V_KEY_CONTEXT_CK_SIZE(ck_size) |
 	    V_KEY_CONTEXT_MK_SIZE(mk_size) | V_KEY_CONTEXT_VALID(1));
 }
 
+static bool
+ccr_auth_supported(const struct crypto_session_params *csp)
+{
+
+	switch (csp->csp_auth_alg) {
+	case CRYPTO_SHA1:
+	case CRYPTO_SHA2_224:
+	case CRYPTO_SHA2_256:
+	case CRYPTO_SHA2_384:
+	case CRYPTO_SHA2_512:
+	case CRYPTO_SHA1_HMAC:
+	case CRYPTO_SHA2_224_HMAC:
+	case CRYPTO_SHA2_256_HMAC:
+	case CRYPTO_SHA2_384_HMAC:
+	case CRYPTO_SHA2_512_HMAC:
+		break;
+	default:
+		return (false);
+	}
+	return (true);
+}
+
+static bool
+ccr_cipher_supported(const struct crypto_session_params *csp)
+{
+
+	switch (csp->csp_cipher_alg) {
+	case CRYPTO_AES_CBC:
+		if (csp->csp_ivlen != AES_BLOCK_LEN)
+			return (false);
+		break;
+	case CRYPTO_AES_ICM:
+		if (csp->csp_ivlen != AES_BLOCK_LEN)
+			return (false);
+		break;
+	case CRYPTO_AES_XTS:
+		if (csp->csp_ivlen != AES_XTS_IV_LEN)
+			return (false);
+		break;
+	default:
+		return (false);
+	}
+	return (ccr_aes_check_keylen(csp->csp_cipher_alg,
+	    csp->csp_cipher_klen));
+}
+
 static int
-ccr_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
+ccr_cipher_mode(const struct crypto_session_params *csp)
 {
-	struct ccr_softc *sc;
-	struct ccr_session *s;
-	struct auth_hash *auth_hash;
-	struct cryptoini *c, *hash, *cipher;
-	unsigned int auth_mode, cipher_mode, iv_len, mk_size;
-	unsigned int partial_digest_len;
-	int error;
-	bool gcm_hash, hmac;
 
-	if (cri == NULL)
-		return (EINVAL);
+	switch (csp->csp_cipher_alg) {
+	case CRYPTO_AES_CBC:
+		return (SCMD_CIPH_MODE_AES_CBC);
+	case CRYPTO_AES_ICM:
+		return (SCMD_CIPH_MODE_AES_CTR);
+	case CRYPTO_AES_NIST_GCM_16:
+		return (SCMD_CIPH_MODE_AES_GCM);
+	case CRYPTO_AES_XTS:
+		return (SCMD_CIPH_MODE_AES_XTS);
+	case CRYPTO_AES_CCM_16:
+		return (SCMD_CIPH_MODE_AES_CCM);
+	default:
+		return (SCMD_CIPH_MODE_NOP);
+	}
+}
 
-	gcm_hash = false;
-	hmac = false;
-	cipher = NULL;
-	hash = NULL;
-	auth_hash = NULL;
-	auth_mode = SCMD_AUTH_MODE_NOP;
-	cipher_mode = SCMD_CIPH_MODE_NOP;
-	iv_len = 0;
-	mk_size = 0;
-	partial_digest_len = 0;
-	for (c = cri; c != NULL; c = c->cri_next) {
-		switch (c->cri_alg) {
-		case CRYPTO_SHA1:
-		case CRYPTO_SHA2_224:
-		case CRYPTO_SHA2_256:
-		case CRYPTO_SHA2_384:
-		case CRYPTO_SHA2_512:
-		case CRYPTO_SHA1_HMAC:
-		case CRYPTO_SHA2_224_HMAC:
-		case CRYPTO_SHA2_256_HMAC:
-		case CRYPTO_SHA2_384_HMAC:
-		case CRYPTO_SHA2_512_HMAC:
-		case CRYPTO_AES_128_NIST_GMAC:
-		case CRYPTO_AES_192_NIST_GMAC:
-		case CRYPTO_AES_256_NIST_GMAC:
-		case CRYPTO_AES_CCM_CBC_MAC:
-			if (hash)
+static int
+ccr_probesession(device_t dev, const struct crypto_session_params *csp)
+{
+	unsigned int cipher_mode;
+
+	if (csp->csp_flags != 0)
+		return (EINVAL);
+	switch (csp->csp_mode) {
+	case CSP_MODE_DIGEST:
+		if (!ccr_auth_supported(csp))
+			return (EINVAL);
+		break;
+	case CSP_MODE_CIPHER:
+		if (!ccr_cipher_supported(csp))
+			return (EINVAL);
+		break;
+	case CSP_MODE_AEAD:
+		switch (csp->csp_cipher_alg) {
+		case CRYPTO_AES_NIST_GCM_16:
+			if (csp->csp_ivlen != AES_GCM_IV_LEN)
 				return (EINVAL);
-			hash = c;
-			switch (c->cri_alg) {
-			case CRYPTO_SHA1:
-			case CRYPTO_SHA1_HMAC:
-				auth_hash = &auth_hash_hmac_sha1;
-				auth_mode = SCMD_AUTH_MODE_SHA1;
-				mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160;
-				partial_digest_len = SHA1_HASH_LEN;
-				break;
-			case CRYPTO_SHA2_224:
-			case CRYPTO_SHA2_224_HMAC:
-				auth_hash = &auth_hash_hmac_sha2_224;
-				auth_mode = SCMD_AUTH_MODE_SHA224;
-				mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
-				partial_digest_len = SHA2_256_HASH_LEN;
-				break;
-			case CRYPTO_SHA2_256:
-			case CRYPTO_SHA2_256_HMAC:
-				auth_hash = &auth_hash_hmac_sha2_256;
-				auth_mode = SCMD_AUTH_MODE_SHA256;
-				mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
-				partial_digest_len = SHA2_256_HASH_LEN;
-				break;
-			case CRYPTO_SHA2_384:
-			case CRYPTO_SHA2_384_HMAC:
-				auth_hash = &auth_hash_hmac_sha2_384;
-				auth_mode = SCMD_AUTH_MODE_SHA512_384;
-				mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
-				partial_digest_len = SHA2_512_HASH_LEN;
-				break;
-			case CRYPTO_SHA2_512:
-			case CRYPTO_SHA2_512_HMAC:
-				auth_hash = &auth_hash_hmac_sha2_512;
-				auth_mode = SCMD_AUTH_MODE_SHA512_512;
-				mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
-				partial_digest_len = SHA2_512_HASH_LEN;
-				break;
-			case CRYPTO_AES_128_NIST_GMAC:
-			case CRYPTO_AES_192_NIST_GMAC:
-			case CRYPTO_AES_256_NIST_GMAC:
-				gcm_hash = true;
-				auth_mode = SCMD_AUTH_MODE_GHASH;
-				mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_128;
-				break;
-			case CRYPTO_AES_CCM_CBC_MAC:
-				auth_mode = SCMD_AUTH_MODE_CBCMAC;
-				break;
-			}
-			switch (c->cri_alg) {
-			case CRYPTO_SHA1_HMAC:
-			case CRYPTO_SHA2_224_HMAC:
-			case CRYPTO_SHA2_256_HMAC:
-			case CRYPTO_SHA2_384_HMAC:
-			case CRYPTO_SHA2_512_HMAC:
-				hmac = true;
-				break;
-			}
+			if (csp->csp_auth_mlen < 0 ||
+			    csp->csp_auth_mlen > AES_GMAC_HASH_LEN)
+				return (EINVAL);
 			break;
-		case CRYPTO_AES_CBC:
-		case CRYPTO_AES_ICM:
-		case CRYPTO_AES_NIST_GCM_16:
-		case CRYPTO_AES_XTS:
 		case CRYPTO_AES_CCM_16:
-			if (cipher)
+			if (csp->csp_ivlen != AES_CCM_IV_LEN)
 				return (EINVAL);
-			cipher = c;
-			switch (c->cri_alg) {
-			case CRYPTO_AES_CBC:
-				cipher_mode = SCMD_CIPH_MODE_AES_CBC;
-				iv_len = AES_BLOCK_LEN;
-				break;
-			case CRYPTO_AES_ICM:
-				cipher_mode = SCMD_CIPH_MODE_AES_CTR;
-				iv_len = AES_BLOCK_LEN;
-				break;
-			case CRYPTO_AES_NIST_GCM_16:
-				cipher_mode = SCMD_CIPH_MODE_AES_GCM;
-				iv_len = AES_GCM_IV_LEN;
-				break;
-			case CRYPTO_AES_XTS:
-				cipher_mode = SCMD_CIPH_MODE_AES_XTS;
-				iv_len = AES_BLOCK_LEN;
-				break;
-			case CRYPTO_AES_CCM_16:
-				cipher_mode = SCMD_CIPH_MODE_AES_CCM;
-				iv_len = AES_CCM_IV_LEN;
-				break;
-			}
-			if (c->cri_key != NULL) {
-				error = ccr_aes_check_keylen(c->cri_alg,
-				    c->cri_klen);
-				if (error)
-					return (error);
-			}
+			if (csp->csp_auth_mlen < 0 ||
+			    csp->csp_auth_mlen > AES_CBC_MAC_HASH_LEN)
+				return (EINVAL);
 			break;
 		default:
 			return (EINVAL);
 		}
-	}
-	if (gcm_hash != (cipher_mode == SCMD_CIPH_MODE_AES_GCM))
+		break;
+	case CSP_MODE_ETA:
+		if (!ccr_auth_supported(csp) || !ccr_cipher_supported(csp))
+			return (EINVAL);
+		break;
+	default:
 		return (EINVAL);
-	if ((auth_mode == SCMD_AUTH_MODE_CBCMAC) !=
-	    (cipher_mode == SCMD_CIPH_MODE_AES_CCM))
-		return (EINVAL);
-	if (hash == NULL && cipher == NULL)
-		return (EINVAL);
-	if (hash != NULL) {
-		if (hmac || gcm_hash || auth_mode == SCMD_AUTH_MODE_CBCMAC) {
-			if (hash->cri_key == NULL)
-				return (EINVAL);
-		} else {
-			if (hash->cri_key != NULL)
-				return (EINVAL);
-		}
 	}
 
+	if (csp->csp_cipher_klen != 0) {
+		cipher_mode = ccr_cipher_mode(csp);
+		if (cipher_mode == SCMD_CIPH_MODE_NOP)
+			return (EINVAL);
+	}
+
+	return (CRYPTODEV_PROBE_HARDWARE);
+}
+
+static int
+ccr_newsession(device_t dev, crypto_session_t cses,
+    const struct crypto_session_params *csp)
+{
+	struct ccr_softc *sc;
+	struct ccr_session *s;
+	struct auth_hash *auth_hash;
+	unsigned int auth_mode, cipher_mode, mk_size;
+	unsigned int partial_digest_len;
+
+	switch (csp->csp_auth_alg) {
+	case CRYPTO_SHA1:
+	case CRYPTO_SHA1_HMAC:
+		auth_hash = &auth_hash_hmac_sha1;
+		auth_mode = SCMD_AUTH_MODE_SHA1;
+		mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160;
+		partial_digest_len = SHA1_HASH_LEN;
+		break;
+	case CRYPTO_SHA2_224:
+	case CRYPTO_SHA2_224_HMAC:
+		auth_hash = &auth_hash_hmac_sha2_224;
+		auth_mode = SCMD_AUTH_MODE_SHA224;
+		mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
+		partial_digest_len = SHA2_256_HASH_LEN;
+		break;
+	case CRYPTO_SHA2_256:
+	case CRYPTO_SHA2_256_HMAC:
+		auth_hash = &auth_hash_hmac_sha2_256;
+		auth_mode = SCMD_AUTH_MODE_SHA256;
+		mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
+		partial_digest_len = SHA2_256_HASH_LEN;
+		break;
+	case CRYPTO_SHA2_384:
+	case CRYPTO_SHA2_384_HMAC:
+		auth_hash = &auth_hash_hmac_sha2_384;
+		auth_mode = SCMD_AUTH_MODE_SHA512_384;
+		mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
+		partial_digest_len = SHA2_512_HASH_LEN;
+		break;
+	case CRYPTO_SHA2_512:
+	case CRYPTO_SHA2_512_HMAC:
+		auth_hash = &auth_hash_hmac_sha2_512;
+		auth_mode = SCMD_AUTH_MODE_SHA512_512;
+		mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
+		partial_digest_len = SHA2_512_HASH_LEN;
+		break;
+	default:
+		auth_hash = NULL;
+		auth_mode = SCMD_AUTH_MODE_NOP;
+		mk_size = 0;
+		partial_digest_len = 0;
+		break;
+	}
+
+	cipher_mode = ccr_cipher_mode(csp);
+
+#ifdef INVARIANTS
+	switch (csp->csp_mode) {
+	case CSP_MODE_CIPHER:
+		if (cipher_mode == SCMD_CIPH_MODE_NOP ||
+		    cipher_mode == SCMD_CIPH_MODE_AES_GCM ||
+		    cipher_mode == SCMD_CIPH_MODE_AES_CCM)
+			panic("invalid cipher algo");
+		break;
+	case CSP_MODE_DIGEST:
+		if (auth_mode == SCMD_AUTH_MODE_NOP)
+			panic("invalid auth algo");
+		break;
+	case CSP_MODE_AEAD:
+		if (cipher_mode != SCMD_CIPH_MODE_AES_GCM &&
+		    cipher_mode != SCMD_CIPH_MODE_AES_CCM)
+			panic("invalid aead cipher algo");
+		if (auth_mode != SCMD_AUTH_MODE_NOP)
+			panic("invalid aead auth aglo");
+		break;
+	case CSP_MODE_ETA:
+		if (cipher_mode == SCMD_CIPH_MODE_NOP ||
+		    cipher_mode == SCMD_CIPH_MODE_AES_GCM ||
+		    cipher_mode == SCMD_CIPH_MODE_AES_CCM)
+			panic("invalid cipher algo");
+		if (auth_mode == SCMD_AUTH_MODE_NOP)
+			panic("invalid auth algo");
+		break;
+	default:
+		panic("invalid csp mode");
+	}
+#endif
+
 	sc = device_get_softc(dev);
 
 	/*
 	 * XXX: Don't create a session if the queues aren't
 	 * initialized.  This is racy as the rxq can be destroyed by
 	 * the associated VI detaching.  Eventually ccr should use
 	 * dedicated queues.
 	 */
 	if (sc->rxq->iq.adapter == NULL || sc->txq->adapter == NULL)
 		return (ENXIO);
 	
 	mtx_lock(&sc->lock);
 	if (sc->detaching) {
 		mtx_unlock(&sc->lock);
 		return (ENXIO);
 	}
 
 	s = crypto_get_driver_session(cses);
 
-	if (gcm_hash)
-		s->mode = GCM;
-	else if (cipher_mode == SCMD_CIPH_MODE_AES_CCM)
-		s->mode = CCM;
-	else if (hash != NULL && cipher != NULL)
-		s->mode = AUTHENC;
-	else if (hash != NULL) {
-		if (hmac)
+	switch (csp->csp_mode) {
+	case CSP_MODE_AEAD:
+		if (cipher_mode == SCMD_CIPH_MODE_AES_CCM)
+			s->mode = CCM;
+		else
+			s->mode = GCM;
+		break;
+	case CSP_MODE_ETA:
+		s->mode = ETA;
+		break;
+	case CSP_MODE_DIGEST:
+		if (csp->csp_auth_klen != 0)
 			s->mode = HMAC;
 		else
 			s->mode = HASH;
-	} else {
-		MPASS(cipher != NULL);
+		break;
+	case CSP_MODE_CIPHER:
 		s->mode = BLKCIPHER;
+		break;
 	}
-	if (gcm_hash) {
-		if (hash->cri_mlen == 0)
+
+	if (s->mode == GCM) {
+		if (csp->csp_auth_mlen == 0)
 			s->gmac.hash_len = AES_GMAC_HASH_LEN;
 		else
-			s->gmac.hash_len = hash->cri_mlen;
-		t4_init_gmac_hash(hash->cri_key, hash->cri_klen,
+			s->gmac.hash_len = csp->csp_auth_mlen;
+		t4_init_gmac_hash(csp->csp_cipher_key, csp->csp_cipher_klen,
 		    s->gmac.ghash_h);
-	} else if (auth_mode == SCMD_AUTH_MODE_CBCMAC) {
-		if (hash->cri_mlen == 0)
+	} else if (s->mode == CCM) {
+		if (csp->csp_auth_mlen == 0)
 			s->ccm_mac.hash_len = AES_CBC_MAC_HASH_LEN;
 		else
-			s->ccm_mac.hash_len = hash->cri_mlen;
-	} else if (hash != NULL) {
+			s->ccm_mac.hash_len = csp->csp_auth_mlen;
+	} else if (auth_mode != SCMD_AUTH_MODE_NOP) {
 		s->hmac.auth_hash = auth_hash;
 		s->hmac.auth_mode = auth_mode;
 		s->hmac.mk_size = mk_size;
 		s->hmac.partial_digest_len = partial_digest_len;
-		if (hash->cri_mlen == 0)
+		if (csp->csp_auth_mlen == 0)
 			s->hmac.hash_len = auth_hash->hashsize;
 		else
-			s->hmac.hash_len = hash->cri_mlen;
-		if (hmac)
+			s->hmac.hash_len = csp->csp_auth_mlen;
+		if (csp->csp_auth_key != NULL)
 			t4_init_hmac_digest(auth_hash, partial_digest_len,
-			    hash->cri_key, hash->cri_klen, s->hmac.pads);
+			    csp->csp_auth_key, csp->csp_auth_klen,
+			    s->hmac.pads);
 		else
-			ccr_init_hash_digest(s, hash->cri_alg);
+			ccr_init_hash_digest(s);
 	}
-	if (cipher != NULL) {
+	if (cipher_mode != SCMD_CIPH_MODE_NOP) {
 		s->blkcipher.cipher_mode = cipher_mode;
-		s->blkcipher.iv_len = iv_len;
-		if (cipher->cri_key != NULL)
-			ccr_aes_setkey(s, cipher->cri_alg, cipher->cri_key,
-			    cipher->cri_klen);
+		s->blkcipher.iv_len = csp->csp_ivlen;
+		if (csp->csp_cipher_key != NULL)
+			ccr_aes_setkey(s, csp->csp_cipher_key,
+			    csp->csp_cipher_klen);
 	}
 
 	s->active = true;
 	mtx_unlock(&sc->lock);
 	return (0);
 }
 
 static void
 ccr_freesession(device_t dev, crypto_session_t cses)
 {
 	struct ccr_softc *sc;
 	struct ccr_session *s;
 
 	sc = device_get_softc(dev);
 	s = crypto_get_driver_session(cses);
 	mtx_lock(&sc->lock);
 	if (s->pending != 0)
 		device_printf(dev,
 		    "session %p freed with %d pending requests\n", s,
 		    s->pending);
 	s->active = false;
 	mtx_unlock(&sc->lock);
 }
 
 static int
 ccr_process(device_t dev, struct cryptop *crp, int hint)
 {
+	const struct crypto_session_params *csp;
 	struct ccr_softc *sc;
 	struct ccr_session *s;
-	struct cryptodesc *crd, *crda, *crde;
 	int error;
 
-	if (crp == NULL)
-		return (EINVAL);
-
-	crd = crp->crp_desc;
+	csp = crypto_get_params(crp->crp_session);
 	s = crypto_get_driver_session(crp->crp_session);
 	sc = device_get_softc(dev);
 
 	mtx_lock(&sc->lock);
 	error = ccr_populate_sglist(sc->sg_crp, crp);
 	if (error) {
 		sc->stats_sglist_error++;
 		goto out;
 	}
 
 	switch (s->mode) {
 	case HASH:
 		error = ccr_hash(sc, s, crp);
 		if (error == 0)
 			sc->stats_hash++;
 		break;
 	case HMAC:
-		if (crd->crd_flags & CRD_F_KEY_EXPLICIT)
+		if (crp->crp_auth_key != NULL)
 			t4_init_hmac_digest(s->hmac.auth_hash,
-			    s->hmac.partial_digest_len, crd->crd_key,
-			    crd->crd_klen, s->hmac.pads);
+			    s->hmac.partial_digest_len, crp->crp_auth_key,
+			    csp->csp_auth_klen, s->hmac.pads);
 		error = ccr_hash(sc, s, crp);
 		if (error == 0)
 			sc->stats_hmac++;
 		break;
 	case BLKCIPHER:
-		if (crd->crd_flags & CRD_F_KEY_EXPLICIT) {
-			error = ccr_aes_check_keylen(crd->crd_alg,
-			    crd->crd_klen);
-			if (error)
-				break;
-			ccr_aes_setkey(s, crd->crd_alg, crd->crd_key,
-			    crd->crd_klen);
-		}
+		if (crp->crp_cipher_key != NULL)
+			ccr_aes_setkey(s, crp->crp_cipher_key,
+			    csp->csp_cipher_klen);
 		error = ccr_blkcipher(sc, s, crp);
 		if (error == 0) {
-			if (crd->crd_flags & CRD_F_ENCRYPT)
+			if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
 				sc->stats_blkcipher_encrypt++;
 			else
 				sc->stats_blkcipher_decrypt++;
 		}
 		break;
-	case AUTHENC:
-		error = 0;
-		switch (crd->crd_alg) {
-		case CRYPTO_AES_CBC:
-		case CRYPTO_AES_ICM:
-		case CRYPTO_AES_XTS:
-			/* Only encrypt-then-authenticate supported. */
-			crde = crd;
-			crda = crd->crd_next;
-			if (!(crde->crd_flags & CRD_F_ENCRYPT)) {
-				error = EINVAL;
-				break;
-			}
-			break;
-		default:
-			crda = crd;
-			crde = crd->crd_next;
-			if (crde->crd_flags & CRD_F_ENCRYPT) {
-				error = EINVAL;
-				break;
-			}
-			break;
-		}
-		if (error)
-			break;
-		if (crda->crd_flags & CRD_F_KEY_EXPLICIT)
+	case ETA:
+		if (crp->crp_auth_key != NULL)
 			t4_init_hmac_digest(s->hmac.auth_hash,
-			    s->hmac.partial_digest_len, crda->crd_key,
-			    crda->crd_klen, s->hmac.pads);
-		if (crde->crd_flags & CRD_F_KEY_EXPLICIT) {
-			error = ccr_aes_check_keylen(crde->crd_alg,
-			    crde->crd_klen);
-			if (error)
-				break;
-			ccr_aes_setkey(s, crde->crd_alg, crde->crd_key,
-			    crde->crd_klen);
-		}
-		error = ccr_authenc(sc, s, crp, crda, crde);
+			    s->hmac.partial_digest_len, crp->crp_auth_key,
+			    csp->csp_auth_klen, s->hmac.pads);
+		if (crp->crp_cipher_key != NULL)
+			ccr_aes_setkey(s, crp->crp_cipher_key,
+			    csp->csp_cipher_klen);
+		error = ccr_eta(sc, s, crp);
 		if (error == 0) {
-			if (crde->crd_flags & CRD_F_ENCRYPT)
-				sc->stats_authenc_encrypt++;
+			if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
+				sc->stats_eta_encrypt++;
 			else
-				sc->stats_authenc_decrypt++;
+				sc->stats_eta_decrypt++;
 		}
 		break;
 	case GCM:
-		error = 0;
-		if (crd->crd_alg == CRYPTO_AES_NIST_GCM_16) {
-			crde = crd;
-			crda = crd->crd_next;
-		} else {
-			crda = crd;
-			crde = crd->crd_next;
+		if (crp->crp_cipher_key != NULL) {
+			t4_init_gmac_hash(crp->crp_cipher_key,
+			    csp->csp_cipher_klen, s->gmac.ghash_h);
+			ccr_aes_setkey(s, crp->crp_cipher_key,
+			    csp->csp_cipher_klen);
 		}
-		if (crda->crd_flags & CRD_F_KEY_EXPLICIT)
-			t4_init_gmac_hash(crda->crd_key, crda->crd_klen,
-			    s->gmac.ghash_h);
-		if (crde->crd_flags & CRD_F_KEY_EXPLICIT) {
-			error = ccr_aes_check_keylen(crde->crd_alg,
-			    crde->crd_klen);
-			if (error)
-				break;
-			ccr_aes_setkey(s, crde->crd_alg, crde->crd_key,
-			    crde->crd_klen);
-		}
-		if (crde->crd_len == 0) {
+		if (crp->crp_payload_length == 0) {
 			mtx_unlock(&sc->lock);
-			ccr_gcm_soft(s, crp, crda, crde);
+			ccr_gcm_soft(s, crp);
 			return (0);
 		}
-		error = ccr_gcm(sc, s, crp, crda, crde);
+		error = ccr_gcm(sc, s, crp);
 		if (error == EMSGSIZE) {
 			sc->stats_sw_fallback++;
 			mtx_unlock(&sc->lock);
-			ccr_gcm_soft(s, crp, crda, crde);
+			ccr_gcm_soft(s, crp);
 			return (0);
 		}
 		if (error == 0) {
-			if (crde->crd_flags & CRD_F_ENCRYPT)
+			if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
 				sc->stats_gcm_encrypt++;
 			else
 				sc->stats_gcm_decrypt++;
 		}
 		break;
 	case CCM:
-		error = 0;
-		if (crd->crd_alg == CRYPTO_AES_CCM_16) {
-			crde = crd;
-			crda = crd->crd_next;
-		} else {
-			crda = crd;
-			crde = crd->crd_next;
+		if (crp->crp_cipher_key != NULL) {
+			ccr_aes_setkey(s, crp->crp_cipher_key,
+			    csp->csp_cipher_klen);
 		}
-		if (crde->crd_flags & CRD_F_KEY_EXPLICIT) {
-			error = ccr_aes_check_keylen(crde->crd_alg,
-			    crde->crd_klen);
-			if (error)
-				break;
-			ccr_aes_setkey(s, crde->crd_alg, crde->crd_key,
-			    crde->crd_klen);
-		}
-		error = ccr_ccm(sc, s, crp, crda, crde);
+		error = ccr_ccm(sc, s, crp);
 		if (error == EMSGSIZE) {
 			sc->stats_sw_fallback++;
 			mtx_unlock(&sc->lock);
-			ccr_ccm_soft(s, crp, crda, crde);
+			ccr_ccm_soft(s, crp);
 			return (0);
 		}
 		if (error == 0) {
-			if (crde->crd_flags & CRD_F_ENCRYPT)
+			if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
 				sc->stats_ccm_encrypt++;
 			else
 				sc->stats_ccm_decrypt++;
 		}
 		break;
 	}
 
 	if (error == 0) {
 		s->pending++;
 		sc->stats_inflight++;
 	} else
 		sc->stats_process_error++;
 
 out:
 	mtx_unlock(&sc->lock);
 
 	if (error) {
 		crp->crp_etype = error;
 		crypto_done(crp);
 	}
 
 	return (0);
 }
 
 static int
 do_cpl6_fw_pld(struct sge_iq *iq, const struct rss_header *rss,
     struct mbuf *m)
 {
 	struct ccr_softc *sc = iq->adapter->ccr_softc;
 	struct ccr_session *s;
 	const struct cpl_fw6_pld *cpl;
 	struct cryptop *crp;
 	uint32_t status;
 	int error;
 
 	if (m != NULL)
 		cpl = mtod(m, const void *);
 	else
 		cpl = (const void *)(rss + 1);
 
 	crp = (struct cryptop *)(uintptr_t)be64toh(cpl->data[1]);
 	s = crypto_get_driver_session(crp->crp_session);
 	status = be64toh(cpl->data[0]);
 	if (CHK_MAC_ERR_BIT(status) || CHK_PAD_ERR_BIT(status))
 		error = EBADMSG;
 	else
 		error = 0;
 
 	mtx_lock(&sc->lock);
 	s->pending--;
 	sc->stats_inflight--;
 
 	switch (s->mode) {
 	case HASH:
 	case HMAC:
 		error = ccr_hash_done(sc, s, crp, cpl, error);
 		break;
 	case BLKCIPHER:
 		error = ccr_blkcipher_done(sc, s, crp, cpl, error);
 		break;
-	case AUTHENC:
-		error = ccr_authenc_done(sc, s, crp, cpl, error);
+	case ETA:
+		error = ccr_eta_done(sc, s, crp, cpl, error);
 		break;
 	case GCM:
 		error = ccr_gcm_done(sc, s, crp, cpl, error);
 		break;
 	case CCM:
 		error = ccr_ccm_done(sc, s, crp, cpl, error);
 		break;
 	}
 
 	if (error == EBADMSG) {
 		if (CHK_MAC_ERR_BIT(status))
 			sc->stats_mac_error++;
 		if (CHK_PAD_ERR_BIT(status))
 			sc->stats_pad_error++;
 	}
 	mtx_unlock(&sc->lock);
 	crp->crp_etype = error;
 	crypto_done(crp);
 	m_freem(m);
 	return (0);
 }
 
 static int
 ccr_modevent(module_t mod, int cmd, void *arg)
 {
 
 	switch (cmd) {
 	case MOD_LOAD:
 		t4_register_cpl_handler(CPL_FW6_PLD, do_cpl6_fw_pld);
 		return (0);
 	case MOD_UNLOAD:
 		t4_register_cpl_handler(CPL_FW6_PLD, NULL);
 		return (0);
 	default:
 		return (EOPNOTSUPP);
 	}
 }
 
 static device_method_t ccr_methods[] = {
 	DEVMETHOD(device_identify,	ccr_identify),
 	DEVMETHOD(device_probe,		ccr_probe),
 	DEVMETHOD(device_attach,	ccr_attach),
 	DEVMETHOD(device_detach,	ccr_detach),
 
+	DEVMETHOD(cryptodev_probesession, ccr_probesession),
 	DEVMETHOD(cryptodev_newsession,	ccr_newsession),
 	DEVMETHOD(cryptodev_freesession, ccr_freesession),
 	DEVMETHOD(cryptodev_process,	ccr_process),
 
 	DEVMETHOD_END
 };
 
 static driver_t ccr_driver = {
 	"ccr",
 	ccr_methods,
 	sizeof(struct ccr_softc)
 };
 
 static devclass_t ccr_devclass;
 
 DRIVER_MODULE(ccr, t6nex, ccr_driver, ccr_devclass, ccr_modevent, NULL);
 MODULE_VERSION(ccr, 1);
 MODULE_DEPEND(ccr, crypto, 1, 1, 1);
 MODULE_DEPEND(ccr, t6nex, 1, 1, 1);
Index: head/sys/dev/cxgbe/crypto/t4_keyctx.c
===================================================================
--- head/sys/dev/cxgbe/crypto/t4_keyctx.c	(revision 359373)
+++ head/sys/dev/cxgbe/crypto/t4_keyctx.c	(revision 359374)
@@ -1,198 +1,172 @@
 /*-
  * Copyright (c) 2017-2019 Chelsio Communications, Inc.
  * All rights reserved.
  * Written by: John Baldwin <jhb@FreeBSD.org>
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include <sys/types.h>
 #include <sys/malloc.h>
 
 #include <opencrypto/cryptodev.h>
 #include <opencrypto/xform.h>
 
 #include "common/common.h"
 #include "crypto/t4_crypto.h"
 
 /*
  * Crypto operations use a key context to store cipher keys and
  * partial hash digests.  They can either be passed inline as part of
  * a work request using crypto or they can be stored in card RAM.  For
  * the latter case, work requests must replace the inline key context
  * with a request to read the context from card RAM.
  *
  * The format of a key context:
  *
  * +-------------------------------+
  * | key context header            |
  * +-------------------------------+
  * | AES key                       |  ----- For requests with AES
  * +-------------------------------+
  * | Hash state                    |  ----- For hash-only requests
  * +-------------------------------+ -
  * | IPAD (16-byte aligned)        |  \
  * +-------------------------------+  +---- For requests with HMAC
  * | OPAD (16-byte aligned)        |  /
  * +-------------------------------+ -
  * | GMAC H                        |  ----- For AES-GCM
  * +-------------------------------+ -
  */
 
 /*
  * Generate the initial GMAC hash state for a AES-GCM key.
  *
  * Borrowed from AES_GMAC_Setkey().
  */
 void
 t4_init_gmac_hash(const char *key, int klen, char *ghash)
 {
 	static char zeroes[GMAC_BLOCK_LEN];
 	uint32_t keysched[4 * (RIJNDAEL_MAXNR + 1)];
 	int rounds;
 
-	rounds = rijndaelKeySetupEnc(keysched, key, klen);
+	rounds = rijndaelKeySetupEnc(keysched, key, klen * 8);
 	rijndaelEncrypt(keysched, rounds, zeroes, ghash);
 }
 
 /* Copy out the partial hash state from a software hash implementation. */
 void
 t4_copy_partial_hash(int alg, union authctx *auth_ctx, void *dst)
 {
 	uint32_t *u32;
 	uint64_t *u64;
 	u_int i;
 
 	u32 = (uint32_t *)dst;
 	u64 = (uint64_t *)dst;
 	switch (alg) {
 	case CRYPTO_SHA1:
 	case CRYPTO_SHA1_HMAC:
 		for (i = 0; i < SHA1_HASH_LEN / 4; i++)
 			u32[i] = htobe32(auth_ctx->sha1ctx.h.b32[i]);
 		break;
 	case CRYPTO_SHA2_224:
 	case CRYPTO_SHA2_224_HMAC:
 		for (i = 0; i < SHA2_256_HASH_LEN / 4; i++)
 			u32[i] = htobe32(auth_ctx->sha224ctx.state[i]);
 		break;
 	case CRYPTO_SHA2_256:
 	case CRYPTO_SHA2_256_HMAC:
 		for (i = 0; i < SHA2_256_HASH_LEN / 4; i++)
 			u32[i] = htobe32(auth_ctx->sha256ctx.state[i]);
 		break;
 	case CRYPTO_SHA2_384:
 	case CRYPTO_SHA2_384_HMAC:
 		for (i = 0; i < SHA2_512_HASH_LEN / 8; i++)
 			u64[i] = htobe64(auth_ctx->sha384ctx.state[i]);
 		break;
 	case CRYPTO_SHA2_512:
 	case CRYPTO_SHA2_512_HMAC:
 		for (i = 0; i < SHA2_512_HASH_LEN / 8; i++)
 			u64[i] = htobe64(auth_ctx->sha512ctx.state[i]);
 		break;
 	}
 }
 
 void
 t4_init_hmac_digest(struct auth_hash *axf, u_int partial_digest_len,
-    char *key, int klen, char *dst)
+    const char *key, int klen, char *dst)
 {
 	union authctx auth_ctx;
-	char ipad[SHA2_512_BLOCK_LEN], opad[SHA2_512_BLOCK_LEN];
-	u_int i;
 
-	/*
-	 * If the key is larger than the block size, use the digest of
-	 * the key as the key instead.
-	 */
-	klen /= 8;
-	if (klen > axf->blocksize) {
-		axf->Init(&auth_ctx);
-		axf->Update(&auth_ctx, key, klen);
-		axf->Final(ipad, &auth_ctx);
-		klen = axf->hashsize;
-	} else
-		memcpy(ipad, key, klen);
-
-	memset(ipad + klen, 0, axf->blocksize - klen);
-	memcpy(opad, ipad, axf->blocksize);
-
-	for (i = 0; i < axf->blocksize; i++) {
-		ipad[i] ^= HMAC_IPAD_VAL;
-		opad[i] ^= HMAC_OPAD_VAL;
-	}
-
-	/*
-	 * Hash the raw ipad and opad and store the partial results in
-	 * the key context.
-	 */
-	axf->Init(&auth_ctx);
-	axf->Update(&auth_ctx, ipad, axf->blocksize);
+	hmac_init_ipad(axf, key, klen, &auth_ctx);
 	t4_copy_partial_hash(axf->type, &auth_ctx, dst);
 
 	dst += roundup2(partial_digest_len, 16);
-	axf->Init(&auth_ctx);
-	axf->Update(&auth_ctx, opad, axf->blocksize);
+
+	hmac_init_opad(axf, key, klen, &auth_ctx);
 	t4_copy_partial_hash(axf->type, &auth_ctx, dst);
+
+	explicit_bzero(&auth_ctx, sizeof(auth_ctx));
 }
 
 /*
  * Borrowed from cesa_prep_aes_key().
  *
  * NB: The crypto engine wants the words in the decryption key in reverse
  * order.
  */
 void
 t4_aes_getdeckey(void *dec_key, const void *enc_key, unsigned int kbits)
 {
 	uint32_t ek[4 * (RIJNDAEL_MAXNR + 1)];
 	uint32_t *dkey;
 	int i;
 
 	rijndaelKeySetupEnc(ek, enc_key, kbits);
 	dkey = dec_key;
 	dkey += (kbits / 8) / 4;
 
 	switch (kbits) {
 	case 128:
 		for (i = 0; i < 4; i++)
 			*--dkey = htobe32(ek[4 * 10 + i]);
 		break;
 	case 192:
 		for (i = 0; i < 2; i++)
 			*--dkey = htobe32(ek[4 * 11 + 2 + i]);
 		for (i = 0; i < 4; i++)
 			*--dkey = htobe32(ek[4 * 12 + i]);
 		break;
 	case 256:
 		for (i = 0; i < 4; i++)
 			*--dkey = htobe32(ek[4 * 13 + i]);
 		for (i = 0; i < 4; i++)
 			*--dkey = htobe32(ek[4 * 14 + i]);
 		break;
 	}
 	MPASS(dkey == dec_key);
 }
Index: head/sys/dev/cxgbe/tom/t4_tls.c
===================================================================
--- head/sys/dev/cxgbe/tom/t4_tls.c	(revision 359373)
+++ head/sys/dev/cxgbe/tom/t4_tls.c	(revision 359374)
@@ -1,2116 +1,2116 @@
 /*-
  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
  *
  * Copyright (c) 2017-2018 Chelsio Communications, Inc.
  * All rights reserved.
  * Written by: John Baldwin <jhb@FreeBSD.org>
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 #include "opt_inet.h"
 #include "opt_kern_tls.h"
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/ktr.h>
 #ifdef KERN_TLS
 #include <sys/ktls.h>
 #endif
 #include <sys/sglist.h>
 #include <sys/socket.h>
 #include <sys/socketvar.h>
 #include <sys/systm.h>
 #include <netinet/in.h>
 #include <netinet/in_pcb.h>
 #include <netinet/tcp_var.h>
 #include <netinet/toecore.h>
 #ifdef KERN_TLS
 #include <opencrypto/cryptodev.h>
 #include <opencrypto/xform.h>
 #endif
 
 #ifdef TCP_OFFLOAD
 #include "common/common.h"
 #include "common/t4_tcb.h"
 #include "crypto/t4_crypto.h"
 #include "tom/t4_tom_l2t.h"
 #include "tom/t4_tom.h"
 
 /*
  * The TCP sequence number of a CPL_TLS_DATA mbuf is saved here while
  * the mbuf is in the ulp_pdu_reclaimq.
  */
 #define	tls_tcp_seq	PH_loc.thirtytwo[0]
 
 /*
  * Handshake lock used for the handshake timer.  Having a global lock
  * is perhaps not ideal, but it avoids having to use callout_drain()
  * in tls_uninit_toep() which can't block.  Also, the timer shouldn't
  * actually fire for most connections.
  */
 static struct mtx tls_handshake_lock;
 
 static void
 t4_set_tls_tcb_field(struct toepcb *toep, uint16_t word, uint64_t mask,
     uint64_t val)
 {
 	struct adapter *sc = td_adapter(toep->td);
 
 	t4_set_tcb_field(sc, toep->ofld_txq, toep, word, mask, val, 0, 0);
 }
 
 /* TLS and DTLS common routines */
 bool
 can_tls_offload(struct adapter *sc)
 {
 
 	return (sc->tt.tls && sc->cryptocaps & FW_CAPS_CONFIG_TLSKEYS);
 }
 
 int
 tls_tx_key(struct toepcb *toep)
 {
 	struct tls_ofld_info *tls_ofld = &toep->tls;
 
 	return (tls_ofld->tx_key_addr >= 0);
 }
 
 int
 tls_rx_key(struct toepcb *toep)
 {
 	struct tls_ofld_info *tls_ofld = &toep->tls;
 
 	return (tls_ofld->rx_key_addr >= 0);
 }
 
 static int
 key_size(struct toepcb *toep)
 {
 	struct tls_ofld_info *tls_ofld = &toep->tls;
 
 	return ((tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_IMMEDIATE) ?
 		tls_ofld->k_ctx.tx_key_info_size : KEY_IN_DDR_SIZE);
 }
 
 /* Set TLS Key-Id in TCB */
 static void
 t4_set_tls_keyid(struct toepcb *toep, unsigned int key_id)
 {
 
 	t4_set_tls_tcb_field(toep, W_TCB_RX_TLS_KEY_TAG,
 			 V_TCB_RX_TLS_KEY_TAG(M_TCB_RX_TLS_BUF_TAG),
 			 V_TCB_RX_TLS_KEY_TAG(key_id));
 }
 
 /* Clear TF_RX_QUIESCE to re-enable receive. */
 static void
 t4_clear_rx_quiesce(struct toepcb *toep)
 {
 
 	t4_set_tls_tcb_field(toep, W_TCB_T_FLAGS, V_TF_RX_QUIESCE(1), 0);
 }
 
 static void
 tls_clr_ofld_mode(struct toepcb *toep)
 {
 
 	tls_stop_handshake_timer(toep);
 
 	/* Operate in PDU extraction mode only. */
 	t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW,
 	    V_TCB_ULP_RAW(M_TCB_ULP_RAW),
 	    V_TCB_ULP_RAW(V_TF_TLS_ENABLE(1)));
 	t4_clear_rx_quiesce(toep);
 }
 
 static void
 tls_clr_quiesce(struct toepcb *toep)
 {
 
 	tls_stop_handshake_timer(toep);
 	t4_clear_rx_quiesce(toep);
 }
 
 /*
  * Calculate the TLS data expansion size
  */
 static int
 tls_expansion_size(struct toepcb *toep, int data_len, int full_pdus_only,
     unsigned short *pdus_per_ulp)
 {
 	struct tls_ofld_info *tls_ofld = &toep->tls;
 	struct tls_scmd *scmd = &tls_ofld->scmd0;
 	int expn_size = 0, frag_count = 0, pad_per_pdu = 0,
 	    pad_last_pdu = 0, last_frag_size = 0, max_frag_size = 0;
 	int exp_per_pdu = 0;
 	int hdr_len = TLS_HEADER_LENGTH;
 
 	do {
 		max_frag_size = tls_ofld->k_ctx.frag_size;
 		if (G_SCMD_CIPH_MODE(scmd->seqno_numivs) ==
 		   SCMD_CIPH_MODE_AES_GCM) {
 			frag_count = (data_len / max_frag_size);
 			exp_per_pdu = GCM_TAG_SIZE + AEAD_EXPLICIT_DATA_SIZE +
 				hdr_len;
 			expn_size =  frag_count * exp_per_pdu;
 			if (full_pdus_only) {
 				*pdus_per_ulp = data_len / (exp_per_pdu +
 					max_frag_size);
 				if (*pdus_per_ulp > 32)
 					*pdus_per_ulp = 32;
 				else if(!*pdus_per_ulp)
 					*pdus_per_ulp = 1;
 				expn_size = (*pdus_per_ulp) * exp_per_pdu;
 				break;
 			}
 			if ((last_frag_size = data_len % max_frag_size) > 0) {
 				frag_count += 1;
 				expn_size += exp_per_pdu;
 			}
 			break;
 		} else if (G_SCMD_CIPH_MODE(scmd->seqno_numivs) !=
 			   SCMD_CIPH_MODE_NOP) {
 			/* Calculate the number of fragments we can make */
 			frag_count  = (data_len / max_frag_size);
 			if (frag_count > 0) {
 				pad_per_pdu = (((howmany((max_frag_size +
 						       tls_ofld->mac_length),
 						      CIPHER_BLOCK_SIZE)) *
 						CIPHER_BLOCK_SIZE) -
 					       (max_frag_size +
 						tls_ofld->mac_length));
 				if (!pad_per_pdu)
 					pad_per_pdu = CIPHER_BLOCK_SIZE;
 				exp_per_pdu = pad_per_pdu +
 				       	tls_ofld->mac_length +
 					hdr_len + CIPHER_BLOCK_SIZE;
 				expn_size = frag_count * exp_per_pdu;
 			}
 			if (full_pdus_only) {
 				*pdus_per_ulp = data_len / (exp_per_pdu +
 					max_frag_size);
 				if (*pdus_per_ulp > 32)
 					*pdus_per_ulp = 32;
 				else if (!*pdus_per_ulp)
 					*pdus_per_ulp = 1;
 				expn_size = (*pdus_per_ulp) * exp_per_pdu;
 				break;
 			}
 			/* Consider the last fragment */
 			if ((last_frag_size = data_len % max_frag_size) > 0) {
 				pad_last_pdu = (((howmany((last_frag_size +
 							tls_ofld->mac_length),
 						       CIPHER_BLOCK_SIZE)) *
 						 CIPHER_BLOCK_SIZE) -
 						(last_frag_size +
 						 tls_ofld->mac_length));
 				if (!pad_last_pdu)
 					pad_last_pdu = CIPHER_BLOCK_SIZE;
 				expn_size += (pad_last_pdu +
 					      tls_ofld->mac_length + hdr_len +
 					      CIPHER_BLOCK_SIZE);
 			}
 		}
 	} while (0);
 
 	return (expn_size);
 }
 
 /* Copy Key to WR */
 static void
 tls_copy_tx_key(struct toepcb *toep, void *dst)
 {
 	struct tls_ofld_info *tls_ofld = &toep->tls;
 	struct ulptx_sc_memrd *sc_memrd;
 	struct ulptx_idata *sc;
 
 	if (tls_ofld->k_ctx.tx_key_info_size <= 0)
 		return;
 
 	if (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_DDR) {
 		sc = dst;
 		sc->cmd_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_NOOP));
 		sc->len = htobe32(0);
 		sc_memrd = (struct ulptx_sc_memrd *)(sc + 1);
 		sc_memrd->cmd_to_len = htobe32(V_ULPTX_CMD(ULP_TX_SC_MEMRD) |
 		    V_ULP_TX_SC_MORE(1) |
 		    V_ULPTX_LEN16(tls_ofld->k_ctx.tx_key_info_size >> 4));
 		sc_memrd->addr = htobe32(tls_ofld->tx_key_addr >> 5);
 	} else if (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_IMMEDIATE) {
 		memcpy(dst, &tls_ofld->k_ctx.tx,
 		    tls_ofld->k_ctx.tx_key_info_size);
 	}
 }
 
 /* TLS/DTLS content type  for CPL SFO */
 static inline unsigned char
 tls_content_type(unsigned char content_type)
 {
 	/*
 	 * XXX: Shouldn't this map CONTENT_TYPE_APP_DATA to DATA and
 	 * default to "CUSTOM" for all other types including
 	 * heartbeat?
 	 */
 	switch (content_type) {
 	case CONTENT_TYPE_CCS:
 		return CPL_TX_TLS_SFO_TYPE_CCS;
 	case CONTENT_TYPE_ALERT:
 		return CPL_TX_TLS_SFO_TYPE_ALERT;
 	case CONTENT_TYPE_HANDSHAKE:
 		return CPL_TX_TLS_SFO_TYPE_HANDSHAKE;
 	case CONTENT_TYPE_HEARTBEAT:
 		return CPL_TX_TLS_SFO_TYPE_HEARTBEAT;
 	}
 	return CPL_TX_TLS_SFO_TYPE_DATA;
 }
 
 static unsigned char
 get_cipher_key_size(unsigned int ck_size)
 {
 	switch (ck_size) {
 	case AES_NOP: /* NOP */
 		return 15;
 	case AES_128: /* AES128 */
 		return CH_CK_SIZE_128;
 	case AES_192: /* AES192 */
 		return CH_CK_SIZE_192;
 	case AES_256: /* AES256 */
 		return CH_CK_SIZE_256;
 	default:
 		return CH_CK_SIZE_256;
 	}
 }
 
 static unsigned char
 get_mac_key_size(unsigned int mk_size)
 {
 	switch (mk_size) {
 	case SHA_NOP: /* NOP */
 		return CH_MK_SIZE_128;
 	case SHA_GHASH: /* GHASH */
 	case SHA_512: /* SHA512 */
 		return CH_MK_SIZE_512;
 	case SHA_224: /* SHA2-224 */
 		return CH_MK_SIZE_192;
 	case SHA_256: /* SHA2-256*/
 		return CH_MK_SIZE_256;
 	case SHA_384: /* SHA384 */
 		return CH_MK_SIZE_512;
 	case SHA1: /* SHA1 */
 	default:
 		return CH_MK_SIZE_160;
 	}
 }
 
 static unsigned int
 get_proto_ver(int proto_ver)
 {
 	switch (proto_ver) {
 	case TLS1_2_VERSION:
 		return TLS_1_2_VERSION;
 	case TLS1_1_VERSION:
 		return TLS_1_1_VERSION;
 	case DTLS1_2_VERSION:
 		return DTLS_1_2_VERSION;
 	default:
 		return TLS_VERSION_MAX;
 	}
 }
 
 static void
 tls_rxkey_flit1(struct tls_keyctx *kwr, struct tls_key_context *kctx)
 {
 
 	if (kctx->state.enc_mode == CH_EVP_CIPH_GCM_MODE) {
 		kwr->u.rxhdr.ivinsert_to_authinsrt =
 		    htobe64(V_TLS_KEYCTX_TX_WR_IVINSERT(6ULL) |
 			V_TLS_KEYCTX_TX_WR_AADSTRTOFST(1ULL) |
 			V_TLS_KEYCTX_TX_WR_AADSTOPOFST(5ULL) |
 			V_TLS_KEYCTX_TX_WR_AUTHSRTOFST(14ULL) |
 			V_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(16ULL) |
 			V_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(14ULL) |
 			V_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(0ULL) |
 			V_TLS_KEYCTX_TX_WR_AUTHINSRT(16ULL));
 		kwr->u.rxhdr.ivpresent_to_rxmk_size &=
 			~(V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(1));
 		kwr->u.rxhdr.authmode_to_rxvalid &=
 			~(V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(1));
 	} else {
 		kwr->u.rxhdr.ivinsert_to_authinsrt =
 		    htobe64(V_TLS_KEYCTX_TX_WR_IVINSERT(6ULL) |
 			V_TLS_KEYCTX_TX_WR_AADSTRTOFST(1ULL) |
 			V_TLS_KEYCTX_TX_WR_AADSTOPOFST(5ULL) |
 			V_TLS_KEYCTX_TX_WR_AUTHSRTOFST(22ULL) |
 			V_TLS_KEYCTX_TX_WR_AUTHSTOPOFST(0ULL) |
 			V_TLS_KEYCTX_TX_WR_CIPHERSRTOFST(22ULL) |
 			V_TLS_KEYCTX_TX_WR_CIPHERSTOPOFST(0ULL) |
 			V_TLS_KEYCTX_TX_WR_AUTHINSRT(0ULL));
 	}
 }
 
 /* Rx key */
 static void
 prepare_rxkey_wr(struct tls_keyctx *kwr, struct tls_key_context *kctx)
 {
 	unsigned int ck_size = kctx->cipher_secret_size;
 	unsigned int mk_size = kctx->mac_secret_size;
 	int proto_ver = kctx->proto_ver;
 
 	kwr->u.rxhdr.flitcnt_hmacctrl =
 		((kctx->tx_key_info_size >> 4) << 3) | kctx->hmac_ctrl;
 
 	kwr->u.rxhdr.protover_ciphmode =
 		V_TLS_KEYCTX_TX_WR_PROTOVER(get_proto_ver(proto_ver)) |
 		V_TLS_KEYCTX_TX_WR_CIPHMODE(kctx->state.enc_mode);
 
 	kwr->u.rxhdr.authmode_to_rxvalid =
 		V_TLS_KEYCTX_TX_WR_AUTHMODE(kctx->state.auth_mode) |
 		V_TLS_KEYCTX_TX_WR_CIPHAUTHSEQCTRL(1) |
 		V_TLS_KEYCTX_TX_WR_SEQNUMCTRL(3) |
 		V_TLS_KEYCTX_TX_WR_RXVALID(1);
 
 	kwr->u.rxhdr.ivpresent_to_rxmk_size =
 		V_TLS_KEYCTX_TX_WR_IVPRESENT(0) |
 		V_TLS_KEYCTX_TX_WR_RXOPAD_PRESENT(1) |
 		V_TLS_KEYCTX_TX_WR_RXCK_SIZE(get_cipher_key_size(ck_size)) |
 		V_TLS_KEYCTX_TX_WR_RXMK_SIZE(get_mac_key_size(mk_size));
 
 	tls_rxkey_flit1(kwr, kctx);
 
 	/* No key reversal for GCM */
 	if (kctx->state.enc_mode != CH_EVP_CIPH_GCM_MODE) {
 		t4_aes_getdeckey(kwr->keys.edkey, kctx->rx.key,
 				 (kctx->cipher_secret_size << 3));
 		memcpy(kwr->keys.edkey + kctx->cipher_secret_size,
 		       kctx->rx.key + kctx->cipher_secret_size,
 		       (IPAD_SIZE + OPAD_SIZE));
 	} else {
 		memcpy(kwr->keys.edkey, kctx->rx.key,
 		       (kctx->tx_key_info_size - SALT_SIZE));
 		memcpy(kwr->u.rxhdr.rxsalt, kctx->rx.salt, SALT_SIZE);
 	}
 }
 
 /* Tx key */
 static void
 prepare_txkey_wr(struct tls_keyctx *kwr, struct tls_key_context *kctx)
 {
 	unsigned int ck_size = kctx->cipher_secret_size;
 	unsigned int mk_size = kctx->mac_secret_size;
 
 	kwr->u.txhdr.ctxlen =
 		(kctx->tx_key_info_size >> 4);
 	kwr->u.txhdr.dualck_to_txvalid =
 		V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1) |
 		V_TLS_KEYCTX_TX_WR_SALT_PRESENT(1) |
 		V_TLS_KEYCTX_TX_WR_TXCK_SIZE(get_cipher_key_size(ck_size)) |
 		V_TLS_KEYCTX_TX_WR_TXMK_SIZE(get_mac_key_size(mk_size)) |
 		V_TLS_KEYCTX_TX_WR_TXVALID(1);
 
 	memcpy(kwr->keys.edkey, kctx->tx.key, HDR_KCTX_SIZE);
 	if (kctx->state.enc_mode == CH_EVP_CIPH_GCM_MODE) {
 		memcpy(kwr->u.txhdr.txsalt, kctx->tx.salt, SALT_SIZE);
 		kwr->u.txhdr.dualck_to_txvalid &=
 			~(V_TLS_KEYCTX_TX_WR_TXOPAD_PRESENT(1));
 	}
 	kwr->u.txhdr.dualck_to_txvalid = htons(kwr->u.txhdr.dualck_to_txvalid);
 }
 
 /* TLS Key memory management */
 static int
 get_new_keyid(struct toepcb *toep)
 {
 	struct adapter *sc = td_adapter(toep->td);
 	vmem_addr_t addr;
 
 	if (vmem_alloc(sc->key_map, TLS_KEY_CONTEXT_SZ, M_NOWAIT | M_FIRSTFIT,
 	    &addr) != 0)
 		return (-1);
 
 	return (addr);
 }
 
 static void
 free_keyid(struct toepcb *toep, int keyid)
 {
 	struct adapter *sc = td_adapter(toep->td);
 
 	vmem_free(sc->key_map, keyid, TLS_KEY_CONTEXT_SZ);
 }
 
 static void
 clear_tls_keyid(struct toepcb *toep)
 {
 	struct tls_ofld_info *tls_ofld = &toep->tls;
 
 	if (tls_ofld->rx_key_addr >= 0) {
 		free_keyid(toep, tls_ofld->rx_key_addr);
 		tls_ofld->rx_key_addr = -1;
 	}
 	if (tls_ofld->tx_key_addr >= 0) {
 		free_keyid(toep, tls_ofld->tx_key_addr);
 		tls_ofld->tx_key_addr = -1;
 	}
 }
 
 static int
 get_keyid(struct tls_ofld_info *tls_ofld, unsigned int ops)
 {
 	return (ops & KEY_WRITE_RX ? tls_ofld->rx_key_addr :
 		((ops & KEY_WRITE_TX) ? tls_ofld->tx_key_addr : -1));
 }
 
 static int
 get_tp_plen_max(struct tls_ofld_info *tls_ofld)
 {
 	int plen = ((min(3*4096, TP_TX_PG_SZ))/1448) * 1448;
 
 	return (tls_ofld->k_ctx.frag_size <= 8192 ? plen : FC_TP_PLEN_MAX);
 }
 
 /* Send request to get the key-id */
 static int
 tls_program_key_id(struct toepcb *toep, struct tls_key_context *k_ctx)
 {
 	struct tls_ofld_info *tls_ofld = &toep->tls;
 	struct adapter *sc = td_adapter(toep->td);
 	struct ofld_tx_sdesc *txsd;
 	int kwrlen, kctxlen, keyid, len;
 	struct wrqe *wr;
 	struct tls_key_req *kwr;
 	struct tls_keyctx *kctx;
 
 	kwrlen = sizeof(*kwr);
 	kctxlen = roundup2(sizeof(*kctx), 32);
 	len = roundup2(kwrlen + kctxlen, 16);
 
 	if (toep->txsd_avail == 0)
 		return (EAGAIN);
 
 	/* Dont initialize key for re-neg */
 	if (!G_KEY_CLR_LOC(k_ctx->l_p_key)) {
 		if ((keyid = get_new_keyid(toep)) < 0) {
 			return (ENOSPC);
 		}
 	} else {
 		keyid = get_keyid(tls_ofld, k_ctx->l_p_key);
 	}
 
 	wr = alloc_wrqe(len, toep->ofld_txq);
 	if (wr == NULL) {
 		free_keyid(toep, keyid);
 		return (ENOMEM);
 	}
 	kwr = wrtod(wr);
 	memset(kwr, 0, kwrlen);
 
 	kwr->wr_hi = htobe32(V_FW_WR_OP(FW_ULPTX_WR) | F_FW_WR_COMPL |
 	    F_FW_WR_ATOMIC);
 	kwr->wr_mid = htobe32(V_FW_WR_LEN16(DIV_ROUND_UP(len, 16)) |
 	    V_FW_WR_FLOWID(toep->tid));
 	kwr->protocol = get_proto_ver(k_ctx->proto_ver);
 	kwr->mfs = htons(k_ctx->frag_size);
 	kwr->reneg_to_write_rx = k_ctx->l_p_key;
 
 	/* master command */
 	kwr->cmd = htobe32(V_ULPTX_CMD(ULP_TX_MEM_WRITE) |
 	    V_T5_ULP_MEMIO_ORDER(1) | V_T5_ULP_MEMIO_IMM(1));
 	kwr->dlen = htobe32(V_ULP_MEMIO_DATA_LEN(kctxlen >> 5));
 	kwr->len16 = htobe32((toep->tid << 8) |
 	    DIV_ROUND_UP(len - sizeof(struct work_request_hdr), 16));
 	kwr->kaddr = htobe32(V_ULP_MEMIO_ADDR(keyid >> 5));
 
 	/* sub command */
 	kwr->sc_more = htobe32(V_ULPTX_CMD(ULP_TX_SC_IMM));
 	kwr->sc_len = htobe32(kctxlen);
 
 	kctx = (struct tls_keyctx *)(kwr + 1);
 	memset(kctx, 0, kctxlen);
 
 	if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_TX) {
 		tls_ofld->tx_key_addr = keyid;
 		prepare_txkey_wr(kctx, k_ctx);
 	} else if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) {
 		tls_ofld->rx_key_addr = keyid;
 		prepare_rxkey_wr(kctx, k_ctx);
 	}
 
 	txsd = &toep->txsd[toep->txsd_pidx];
 	txsd->tx_credits = DIV_ROUND_UP(len, 16);
 	txsd->plen = 0;
 	toep->tx_credits -= txsd->tx_credits;
 	if (__predict_false(++toep->txsd_pidx == toep->txsd_total))
 		toep->txsd_pidx = 0;
 	toep->txsd_avail--;
 
 	t4_wrq_tx(sc, wr);
 
 	return (0);
 }
 
 /* Store a key received from SSL in DDR. */
 static int
 program_key_context(struct tcpcb *tp, struct toepcb *toep,
     struct tls_key_context *uk_ctx)
 {
 	struct adapter *sc = td_adapter(toep->td);
 	struct tls_ofld_info *tls_ofld = &toep->tls;
 	struct tls_key_context *k_ctx;
 	int error, key_offset;
 
 	if (tp->t_state != TCPS_ESTABLISHED) {
 		/*
 		 * XXX: Matches Linux driver, but not sure this is a
 		 * very appropriate error.
 		 */
 		return (ENOENT);
 	}
 
 	/* Stop timer on handshake completion */
 	tls_stop_handshake_timer(toep);
 
 	toep->flags &= ~TPF_FORCE_CREDITS;
 
 	CTR4(KTR_CXGBE, "%s: tid %d %s proto_ver %#x", __func__, toep->tid,
 	    G_KEY_GET_LOC(uk_ctx->l_p_key) == KEY_WRITE_RX ? "KEY_WRITE_RX" :
 	    "KEY_WRITE_TX", uk_ctx->proto_ver);
 
 	if (G_KEY_GET_LOC(uk_ctx->l_p_key) == KEY_WRITE_RX &&
 	    ulp_mode(toep) != ULP_MODE_TLS)
 		return (EOPNOTSUPP);
 
 	/* Don't copy the 'tx' and 'rx' fields. */
 	k_ctx = &tls_ofld->k_ctx;
 	memcpy(&k_ctx->l_p_key, &uk_ctx->l_p_key,
 	    sizeof(*k_ctx) - offsetof(struct tls_key_context, l_p_key));
 
 	/* TLS version != 1.1 and !1.2 OR DTLS != 1.2 */
 	if (get_proto_ver(k_ctx->proto_ver) > DTLS_1_2_VERSION) {
 		if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) {
 			tls_ofld->rx_key_addr = -1;
 			t4_clear_rx_quiesce(toep);
 		} else {
 			tls_ofld->tx_key_addr = -1;
 		}
 		return (0);
 	}
 
 	if (k_ctx->state.enc_mode == CH_EVP_CIPH_GCM_MODE) {
 		k_ctx->iv_size = 4;
 		k_ctx->mac_first = 0;
 		k_ctx->hmac_ctrl = 0;
 	} else {
 		k_ctx->iv_size = 8; /* for CBC, iv is 16B, unit of 2B */
 		k_ctx->mac_first = 1;
 	}
 
 	tls_ofld->scmd0.seqno_numivs =
 		(V_SCMD_SEQ_NO_CTRL(3) |
 		 V_SCMD_PROTO_VERSION(get_proto_ver(k_ctx->proto_ver)) |
 		 V_SCMD_ENC_DEC_CTRL(SCMD_ENCDECCTRL_ENCRYPT) |
 		 V_SCMD_CIPH_AUTH_SEQ_CTRL((k_ctx->mac_first == 0)) |
 		 V_SCMD_CIPH_MODE(k_ctx->state.enc_mode) |
 		 V_SCMD_AUTH_MODE(k_ctx->state.auth_mode) |
 		 V_SCMD_HMAC_CTRL(k_ctx->hmac_ctrl) |
 		 V_SCMD_IV_SIZE(k_ctx->iv_size));
 
 	tls_ofld->scmd0.ivgen_hdrlen =
 		(V_SCMD_IV_GEN_CTRL(k_ctx->iv_ctrl) |
 		 V_SCMD_KEY_CTX_INLINE(0) |
 		 V_SCMD_TLS_FRAG_ENABLE(1));
 
 	tls_ofld->mac_length = k_ctx->mac_secret_size;
 
 	if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) {
 		k_ctx->rx = uk_ctx->rx;
 		/* Dont initialize key for re-neg */
 		if (!G_KEY_CLR_LOC(k_ctx->l_p_key))
 			tls_ofld->rx_key_addr = -1;
 	} else {
 		k_ctx->tx = uk_ctx->tx;
 		/* Dont initialize key for re-neg */
 		if (!G_KEY_CLR_LOC(k_ctx->l_p_key))
 			tls_ofld->tx_key_addr = -1;
 	}
 
 	/* Flush pending data before new Tx key becomes active */
 	if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_TX) {
 		struct sockbuf *sb;
 
 		/* XXX: This might not drain everything. */
 		t4_push_frames(sc, toep, 0);
 		sb = &toep->inp->inp_socket->so_snd;
 		SOCKBUF_LOCK(sb);
 
 		/* XXX: This asserts that everything has been pushed. */
 		MPASS(sb->sb_sndptr == NULL || sb->sb_sndptr->m_next == NULL);
 		sb->sb_sndptr = NULL;
 		tls_ofld->sb_off = sbavail(sb);
 		SOCKBUF_UNLOCK(sb);
 		tls_ofld->tx_seq_no = 0;
 	}
 
 	if ((G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) ||
 	    (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_DDR)) {
 		error = tls_program_key_id(toep, k_ctx);
 		if (error) {
 			/* XXX: Only clear quiesce for KEY_WRITE_RX? */
 			t4_clear_rx_quiesce(toep);
 			return (error);
 		}
 	}
 
 	if (G_KEY_GET_LOC(k_ctx->l_p_key) == KEY_WRITE_RX) {
 		/*
 		 * RX key tags are an index into the key portion of MA
 		 * memory stored as an offset from the base address in
 		 * units of 64 bytes.
 		 */
 		key_offset = tls_ofld->rx_key_addr - sc->vres.key.start;
 		t4_set_tls_keyid(toep, key_offset / 64);
 		t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW,
 				 V_TCB_ULP_RAW(M_TCB_ULP_RAW),
 				 V_TCB_ULP_RAW((V_TF_TLS_KEY_SIZE(3) |
 						V_TF_TLS_CONTROL(1) |
 						V_TF_TLS_ACTIVE(1) |
 						V_TF_TLS_ENABLE(1))));
 		t4_set_tls_tcb_field(toep, W_TCB_TLS_SEQ,
 				 V_TCB_TLS_SEQ(M_TCB_TLS_SEQ),
 				 V_TCB_TLS_SEQ(0));
 		t4_clear_rx_quiesce(toep);
 	} else {
 		unsigned short pdus_per_ulp;
 
 		if (tls_ofld->key_location == TLS_SFO_WR_CONTEXTLOC_IMMEDIATE)
 			tls_ofld->tx_key_addr = 1;
 
 		tls_ofld->fcplenmax = get_tp_plen_max(tls_ofld);
 		tls_ofld->expn_per_ulp = tls_expansion_size(toep,
 				tls_ofld->fcplenmax, 1, &pdus_per_ulp);
 		tls_ofld->pdus_per_ulp = pdus_per_ulp;
 		tls_ofld->adjusted_plen = tls_ofld->pdus_per_ulp *
 			((tls_ofld->expn_per_ulp/tls_ofld->pdus_per_ulp) +
 			 tls_ofld->k_ctx.frag_size);
 	}
 
 	return (0);
 }
 
 /*
  * In some cases a client connection can hang without sending the
  * ServerHelloDone message from the NIC to the host.  Send a dummy
  * RX_DATA_ACK with RX_MODULATE to unstick the connection.
  */
 static void
 tls_send_handshake_ack(void *arg)
 {
 	struct toepcb *toep = arg;
 	struct tls_ofld_info *tls_ofld = &toep->tls;
 	struct adapter *sc = td_adapter(toep->td);
 
 	/*
 	 * XXX: Does not have the t4_get_tcb() checks to refine the
 	 * workaround.
 	 */
 	callout_schedule(&tls_ofld->handshake_timer, TLS_SRV_HELLO_RD_TM * hz);
 
 	CTR2(KTR_CXGBE, "%s: tid %d sending RX_DATA_ACK", __func__, toep->tid);
 	send_rx_modulate(sc, toep);
 }
 
 static void
 tls_start_handshake_timer(struct toepcb *toep)
 {
 	struct tls_ofld_info *tls_ofld = &toep->tls;
 
 	mtx_lock(&tls_handshake_lock);
 	callout_reset(&tls_ofld->handshake_timer, TLS_SRV_HELLO_BKOFF_TM * hz,
 	    tls_send_handshake_ack, toep);
 	mtx_unlock(&tls_handshake_lock);
 }
 
 void
 tls_stop_handshake_timer(struct toepcb *toep)
 {
 	struct tls_ofld_info *tls_ofld = &toep->tls;
 
 	mtx_lock(&tls_handshake_lock);
 	callout_stop(&tls_ofld->handshake_timer);
 	mtx_unlock(&tls_handshake_lock);
 }
 
 int
 t4_ctloutput_tls(struct socket *so, struct sockopt *sopt)
 {
 	struct tls_key_context uk_ctx;
 	struct inpcb *inp;
 	struct tcpcb *tp;
 	struct toepcb *toep;
 	int error, optval;
 
 	error = 0;
 	if (sopt->sopt_dir == SOPT_SET &&
 	    sopt->sopt_name == TCP_TLSOM_SET_TLS_CONTEXT) {
 		error = sooptcopyin(sopt, &uk_ctx, sizeof(uk_ctx),
 		    sizeof(uk_ctx));
 		if (error)
 			return (error);
 	}
 
 	inp = sotoinpcb(so);
 	KASSERT(inp != NULL, ("tcp_ctloutput: inp == NULL"));
 	INP_WLOCK(inp);
 	if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
 		INP_WUNLOCK(inp);
 		return (ECONNRESET);
 	}
 	tp = intotcpcb(inp);
 	toep = tp->t_toe;
 	switch (sopt->sopt_dir) {
 	case SOPT_SET:
 		switch (sopt->sopt_name) {
 		case TCP_TLSOM_SET_TLS_CONTEXT:
 			if (toep->tls.mode == TLS_MODE_KTLS)
 				error = EINVAL;
 			else {
 				error = program_key_context(tp, toep, &uk_ctx);
 				if (error == 0)
 					toep->tls.mode = TLS_MODE_TLSOM;
 			}
 			INP_WUNLOCK(inp);
 			break;
 		case TCP_TLSOM_CLR_TLS_TOM:
 			if (toep->tls.mode == TLS_MODE_KTLS)
 				error = EINVAL;
 			else if (ulp_mode(toep) == ULP_MODE_TLS) {
 				CTR2(KTR_CXGBE, "%s: tid %d CLR_TLS_TOM",
 				    __func__, toep->tid);
 				tls_clr_ofld_mode(toep);
 			} else
 				error = EOPNOTSUPP;
 			INP_WUNLOCK(inp);
 			break;
 		case TCP_TLSOM_CLR_QUIES:
 			if (toep->tls.mode == TLS_MODE_KTLS)
 				error = EINVAL;
 			else if (ulp_mode(toep) == ULP_MODE_TLS) {
 				CTR2(KTR_CXGBE, "%s: tid %d CLR_QUIES",
 				    __func__, toep->tid);
 				tls_clr_quiesce(toep);
 			} else
 				error = EOPNOTSUPP;
 			INP_WUNLOCK(inp);
 			break;
 		default:
 			INP_WUNLOCK(inp);
 			error = EOPNOTSUPP;
 			break;
 		}
 		break;
 	case SOPT_GET:
 		switch (sopt->sopt_name) {
 		case TCP_TLSOM_GET_TLS_TOM:
 			/*
 			 * TLS TX is permitted on any TOE socket, but
 			 * TLS RX requires a TLS ULP mode.
 			 */
 			optval = TLS_TOM_NONE;
 			if (can_tls_offload(td_adapter(toep->td)) &&
 			    toep->tls.mode != TLS_MODE_KTLS) {
 				switch (ulp_mode(toep)) {
 				case ULP_MODE_NONE:
 				case ULP_MODE_TCPDDP:
 					optval = TLS_TOM_TXONLY;
 					break;
 				case ULP_MODE_TLS:
 					optval = TLS_TOM_BOTH;
 					break;
 				}
 			}
 			CTR3(KTR_CXGBE, "%s: tid %d GET_TLS_TOM = %d",
 			    __func__, toep->tid, optval);
 			INP_WUNLOCK(inp);
 			error = sooptcopyout(sopt, &optval, sizeof(optval));
 			break;
 		default:
 			INP_WUNLOCK(inp);
 			error = EOPNOTSUPP;
 			break;
 		}
 		break;
 	}
 	return (error);
 }
 
 #ifdef KERN_TLS
 static void
 init_ktls_key_context(struct ktls_session *tls, struct tls_key_context *k_ctx)
 {
 	struct auth_hash *axf;
 	u_int mac_key_size;
 	char *hash;
 
 	k_ctx->l_p_key = V_KEY_GET_LOC(KEY_WRITE_TX);
 	if (tls->params.tls_vminor == TLS_MINOR_VER_ONE)
 		k_ctx->proto_ver = SCMD_PROTO_VERSION_TLS_1_1;
 	else
 		k_ctx->proto_ver = SCMD_PROTO_VERSION_TLS_1_2;
 	k_ctx->cipher_secret_size = tls->params.cipher_key_len;
 	k_ctx->tx_key_info_size = sizeof(struct tx_keyctx_hdr) +
 	    k_ctx->cipher_secret_size;
 	memcpy(k_ctx->tx.key, tls->params.cipher_key,
 	    tls->params.cipher_key_len);
 	hash = k_ctx->tx.key + tls->params.cipher_key_len;
 	if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16) {
 		k_ctx->state.auth_mode = SCMD_AUTH_MODE_GHASH;
 		k_ctx->state.enc_mode = SCMD_CIPH_MODE_AES_GCM;
 		k_ctx->iv_size = 4;
 		k_ctx->mac_first = 0;
 		k_ctx->hmac_ctrl = SCMD_HMAC_CTRL_NOP;
 		k_ctx->tx_key_info_size += GMAC_BLOCK_LEN;
 		memcpy(k_ctx->tx.salt, tls->params.iv, SALT_SIZE);
 		t4_init_gmac_hash(tls->params.cipher_key,
-		    tls->params.cipher_key_len * 8, hash);
+		    tls->params.cipher_key_len, hash);
 	} else {
 		switch (tls->params.auth_algorithm) {
 		case CRYPTO_SHA1_HMAC:
 			axf = &auth_hash_hmac_sha1;
 			mac_key_size = SHA1_HASH_LEN;
 			k_ctx->state.auth_mode = SCMD_AUTH_MODE_SHA1;
 			break;
 		case CRYPTO_SHA2_256_HMAC:
 			axf = &auth_hash_hmac_sha2_256;
 			mac_key_size = SHA2_256_HASH_LEN;
 			k_ctx->state.auth_mode = SCMD_AUTH_MODE_SHA256;
 			break;
 		case CRYPTO_SHA2_384_HMAC:
 			axf = &auth_hash_hmac_sha2_384;
 			mac_key_size = SHA2_512_HASH_LEN;
 			k_ctx->state.auth_mode = SCMD_AUTH_MODE_SHA512_384;
 			break;
 		default:
 			panic("bad auth mode");
 		}
 		k_ctx->state.enc_mode = SCMD_CIPH_MODE_AES_CBC;
 		k_ctx->iv_size = 8; /* for CBC, iv is 16B, unit of 2B */
 		k_ctx->mac_first = 1;
 		k_ctx->hmac_ctrl = SCMD_HMAC_CTRL_NO_TRUNC;
 		k_ctx->tx_key_info_size += roundup2(mac_key_size, 16) * 2;
 		k_ctx->mac_secret_size = mac_key_size;
 		t4_init_hmac_digest(axf, mac_key_size, tls->params.auth_key,
-		    tls->params.auth_key_len * 8, hash);
+		    tls->params.auth_key_len, hash);
 	}
 
 	k_ctx->frag_size = tls->params.max_frame_len;
 	k_ctx->iv_ctrl = 1;
 }
 
 int
 tls_alloc_ktls(struct toepcb *toep, struct ktls_session *tls)
 {
 	struct tls_key_context *k_ctx;
 	int error;
 
 	if (toep->tls.mode == TLS_MODE_TLSOM)
 		return (EINVAL);
 	if (!can_tls_offload(td_adapter(toep->td)))
 		return (EINVAL);
 	switch (ulp_mode(toep)) {
 	case ULP_MODE_NONE:
 	case ULP_MODE_TCPDDP:
 		break;
 	default:
 		return (EINVAL);
 	}
 
 	switch (tls->params.cipher_algorithm) {
 	case CRYPTO_AES_CBC:
 		/* XXX: Explicitly ignore any provided IV. */
 		switch (tls->params.cipher_key_len) {
 		case 128 / 8:
 		case 192 / 8:
 		case 256 / 8:
 			break;
 		default:
 			return (EINVAL);
 		}
 		switch (tls->params.auth_algorithm) {
 		case CRYPTO_SHA1_HMAC:
 		case CRYPTO_SHA2_256_HMAC:
 		case CRYPTO_SHA2_384_HMAC:
 			break;
 		default:
 			return (EPROTONOSUPPORT);
 		}
 		break;
 	case CRYPTO_AES_NIST_GCM_16:
 		if (tls->params.iv_len != SALT_SIZE)
 			return (EINVAL);
 		switch (tls->params.cipher_key_len) {
 		case 128 / 8:
 		case 192 / 8:
 		case 256 / 8:
 			break;
 		default:
 			return (EINVAL);
 		}
 		break;
 	default:
 		return (EPROTONOSUPPORT);
 	}
 
 	/* Only TLS 1.1 and TLS 1.2 are currently supported. */
 	if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE ||
 	    tls->params.tls_vminor < TLS_MINOR_VER_ONE ||
 	    tls->params.tls_vminor > TLS_MINOR_VER_TWO)
 		return (EPROTONOSUPPORT);
 
 	/*
 	 * XXX: This assumes no key renegotation.  If KTLS ever supports
 	 * that we will want to allocate TLS sessions dynamically rather
 	 * than as a static member of toep.
 	 */
 	k_ctx = &toep->tls.k_ctx;
 	init_ktls_key_context(tls, k_ctx);
 
 	toep->tls.scmd0.seqno_numivs =
 		(V_SCMD_SEQ_NO_CTRL(3) |
 		 V_SCMD_PROTO_VERSION(k_ctx->proto_ver) |
 		 V_SCMD_ENC_DEC_CTRL(SCMD_ENCDECCTRL_ENCRYPT) |
 		 V_SCMD_CIPH_AUTH_SEQ_CTRL((k_ctx->mac_first == 0)) |
 		 V_SCMD_CIPH_MODE(k_ctx->state.enc_mode) |
 		 V_SCMD_AUTH_MODE(k_ctx->state.auth_mode) |
 		 V_SCMD_HMAC_CTRL(k_ctx->hmac_ctrl) |
 		 V_SCMD_IV_SIZE(k_ctx->iv_size));
 
 	toep->tls.scmd0.ivgen_hdrlen =
 		(V_SCMD_IV_GEN_CTRL(k_ctx->iv_ctrl) |
 		 V_SCMD_KEY_CTX_INLINE(0) |
 		 V_SCMD_TLS_FRAG_ENABLE(1));
 
 	if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16)
 		toep->tls.iv_len = 8;
 	else
 		toep->tls.iv_len = AES_BLOCK_LEN;
 
 	toep->tls.mac_length = k_ctx->mac_secret_size;
 
 	toep->tls.tx_key_addr = -1;
 
 	error = tls_program_key_id(toep, k_ctx);
 	if (error)
 		return (error);
 
 	toep->tls.fcplenmax = get_tp_plen_max(&toep->tls);
 	toep->tls.expn_per_ulp = tls->params.tls_hlen + tls->params.tls_tlen;
 	toep->tls.pdus_per_ulp = 1;
 	toep->tls.adjusted_plen = toep->tls.expn_per_ulp +
 	    toep->tls.k_ctx.frag_size;
 
 	toep->tls.mode = TLS_MODE_KTLS;
 
 	return (0);
 }
 #endif
 
 void
 tls_init_toep(struct toepcb *toep)
 {
 	struct tls_ofld_info *tls_ofld = &toep->tls;
 
 	tls_ofld->mode = TLS_MODE_OFF;
 	tls_ofld->key_location = TLS_SFO_WR_CONTEXTLOC_DDR;
 	tls_ofld->rx_key_addr = -1;
 	tls_ofld->tx_key_addr = -1;
 	if (ulp_mode(toep) == ULP_MODE_TLS)
 		callout_init_mtx(&tls_ofld->handshake_timer,
 		    &tls_handshake_lock, 0);
 }
 
 void
 tls_establish(struct toepcb *toep)
 {
 
 	/*
 	 * Enable PDU extraction.
 	 *
 	 * XXX: Supposedly this should be done by the firmware when
 	 * the ULP_MODE FLOWC parameter is set in send_flowc_wr(), but
 	 * in practice this seems to be required.
 	 */
 	CTR2(KTR_CXGBE, "%s: tid %d setting TLS_ENABLE", __func__, toep->tid);
 	t4_set_tls_tcb_field(toep, W_TCB_ULP_RAW, V_TCB_ULP_RAW(M_TCB_ULP_RAW),
 	    V_TCB_ULP_RAW(V_TF_TLS_ENABLE(1)));
 
 	toep->flags |= TPF_FORCE_CREDITS;
 
 	tls_start_handshake_timer(toep);
 }
 
 void
 tls_uninit_toep(struct toepcb *toep)
 {
 
 	if (ulp_mode(toep) == ULP_MODE_TLS)
 		tls_stop_handshake_timer(toep);
 	clear_tls_keyid(toep);
 }
 
 #define MAX_OFLD_TX_CREDITS (SGE_MAX_WR_LEN / 16)
 #define	MIN_OFLD_TLSTX_CREDITS(toep)					\
 	(howmany(sizeof(struct fw_tlstx_data_wr) +			\
 	    sizeof(struct cpl_tx_tls_sfo) + key_size((toep)) +		\
 	    CIPHER_BLOCK_SIZE + 1, 16))
 
 static inline u_int
 max_imm_tls_space(int tx_credits)
 {
 	const int n = 2;	/* Use only up to 2 desc for imm. data WR */
 	int space;
 
 	KASSERT(tx_credits >= 0 &&
 		tx_credits <= MAX_OFLD_TX_CREDITS,
 		("%s: %d credits", __func__, tx_credits));
 
 	if (tx_credits >= (n * EQ_ESIZE) / 16)
 		space = (n * EQ_ESIZE);
 	else
 		space = tx_credits * 16;
 	return (space);
 }
 
 static int
 count_mbuf_segs(struct mbuf *m, int skip, int len, int *max_nsegs_1mbufp)
 {
 	int max_nsegs_1mbuf, n, nsegs;
 
 	while (skip >= m->m_len) {
 		skip -= m->m_len;
 		m = m->m_next;
 	}
 
 	nsegs = 0;
 	max_nsegs_1mbuf = 0;
 	while (len > 0) {
 		n = sglist_count(mtod(m, char *) + skip, m->m_len - skip);
 		if (n > max_nsegs_1mbuf)
 			max_nsegs_1mbuf = n;
 		nsegs += n;
 		len -= m->m_len - skip;
 		skip = 0;
 		m = m->m_next;
 	}
 	*max_nsegs_1mbufp = max_nsegs_1mbuf;
 	return (nsegs);
 }
 
 static void
 write_tlstx_wr(struct fw_tlstx_data_wr *txwr, struct toepcb *toep,
     unsigned int immdlen, unsigned int plen, unsigned int expn,
     unsigned int pdus, uint8_t credits, int shove, int imm_ivs)
 {
 	struct tls_ofld_info *tls_ofld = &toep->tls;
 	unsigned int len = plen + expn;
 
 	txwr->op_to_immdlen = htobe32(V_WR_OP(FW_TLSTX_DATA_WR) |
 	    V_FW_TLSTX_DATA_WR_COMPL(1) |
 	    V_FW_TLSTX_DATA_WR_IMMDLEN(immdlen));
 	txwr->flowid_len16 = htobe32(V_FW_TLSTX_DATA_WR_FLOWID(toep->tid) |
 	    V_FW_TLSTX_DATA_WR_LEN16(credits));
 	txwr->plen = htobe32(len);
 	txwr->lsodisable_to_flags = htobe32(V_TX_ULP_MODE(ULP_MODE_TLS) |
 	    V_TX_URG(0) | /* F_T6_TX_FORCE | */ V_TX_SHOVE(shove));
 	txwr->ctxloc_to_exp = htobe32(V_FW_TLSTX_DATA_WR_NUMIVS(pdus) |
 	    V_FW_TLSTX_DATA_WR_EXP(expn) |
 	    V_FW_TLSTX_DATA_WR_CTXLOC(tls_ofld->key_location) |
 	    V_FW_TLSTX_DATA_WR_IVDSGL(!imm_ivs) |
 	    V_FW_TLSTX_DATA_WR_KEYSIZE(tls_ofld->k_ctx.tx_key_info_size >> 4));
 	txwr->mfs = htobe16(tls_ofld->k_ctx.frag_size);
 	txwr->adjustedplen_pkd = htobe16(
 	    V_FW_TLSTX_DATA_WR_ADJUSTEDPLEN(tls_ofld->adjusted_plen));
 	txwr->expinplenmax_pkd = htobe16(
 	    V_FW_TLSTX_DATA_WR_EXPINPLENMAX(tls_ofld->expn_per_ulp));
 	txwr->pdusinplenmax_pkd = 
 	    V_FW_TLSTX_DATA_WR_PDUSINPLENMAX(tls_ofld->pdus_per_ulp);
 }
 
 static void
 write_tlstx_cpl(struct cpl_tx_tls_sfo *cpl, struct toepcb *toep,
     struct tls_hdr *tls_hdr, unsigned int plen, unsigned int pdus)
 {
 	struct tls_ofld_info *tls_ofld = &toep->tls;
 	int data_type, seglen;
 
 	if (plen < tls_ofld->k_ctx.frag_size)
 		seglen = plen;
 	else
 		seglen = tls_ofld->k_ctx.frag_size;
 	data_type = tls_content_type(tls_hdr->type);
 	cpl->op_to_seg_len = htobe32(V_CPL_TX_TLS_SFO_OPCODE(CPL_TX_TLS_SFO) |
 	    V_CPL_TX_TLS_SFO_DATA_TYPE(data_type) |
 	    V_CPL_TX_TLS_SFO_CPL_LEN(2) | V_CPL_TX_TLS_SFO_SEG_LEN(seglen));
 	cpl->pld_len = htobe32(plen);
 	if (data_type == CPL_TX_TLS_SFO_TYPE_HEARTBEAT)
 		cpl->type_protover = htobe32(
 		    V_CPL_TX_TLS_SFO_TYPE(tls_hdr->type));
 	cpl->seqno_numivs = htobe32(tls_ofld->scmd0.seqno_numivs |
 	    V_SCMD_NUM_IVS(pdus));
 	cpl->ivgen_hdrlen = htobe32(tls_ofld->scmd0.ivgen_hdrlen);
 	cpl->scmd1 = htobe64(tls_ofld->tx_seq_no);
 	tls_ofld->tx_seq_no += pdus;
 }
 
 /*
  * Similar to write_tx_sgl() except that it accepts an optional
  * trailer buffer for IVs.
  */
 static void
 write_tlstx_sgl(void *dst, struct mbuf *start, int skip, int plen,
     void *iv_buffer, int iv_len, int nsegs, int n)
 {
 	struct mbuf *m;
 	struct ulptx_sgl *usgl = dst;
 	int i, j, rc;
 	struct sglist sg;
 	struct sglist_seg segs[n];
 
 	KASSERT(nsegs > 0, ("%s: nsegs 0", __func__));
 
 	sglist_init(&sg, n, segs);
 	usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) |
 	    V_ULPTX_NSGE(nsegs));
 
 	for (m = start; skip >= m->m_len; m = m->m_next)
 		skip -= m->m_len;
 
 	i = -1;
 	for (m = start; plen > 0; m = m->m_next) {
 		rc = sglist_append(&sg, mtod(m, char *) + skip,
 		    m->m_len - skip);
 		if (__predict_false(rc != 0))
 			panic("%s: sglist_append %d", __func__, rc);
 		plen -= m->m_len - skip;
 		skip = 0;
 
 		for (j = 0; j < sg.sg_nseg; i++, j++) {
 			if (i < 0) {
 				usgl->len0 = htobe32(segs[j].ss_len);
 				usgl->addr0 = htobe64(segs[j].ss_paddr);
 			} else {
 				usgl->sge[i / 2].len[i & 1] =
 				    htobe32(segs[j].ss_len);
 				usgl->sge[i / 2].addr[i & 1] =
 				    htobe64(segs[j].ss_paddr);
 			}
 #ifdef INVARIANTS
 			nsegs--;
 #endif
 		}
 		sglist_reset(&sg);
 	}
 	if (iv_buffer != NULL) {
 		rc = sglist_append(&sg, iv_buffer, iv_len);
 		if (__predict_false(rc != 0))
 			panic("%s: sglist_append %d", __func__, rc);
 
 		for (j = 0; j < sg.sg_nseg; i++, j++) {
 			if (i < 0) {
 				usgl->len0 = htobe32(segs[j].ss_len);
 				usgl->addr0 = htobe64(segs[j].ss_paddr);
 			} else {
 				usgl->sge[i / 2].len[i & 1] =
 				    htobe32(segs[j].ss_len);
 				usgl->sge[i / 2].addr[i & 1] =
 				    htobe64(segs[j].ss_paddr);
 			}
 #ifdef INVARIANTS
 			nsegs--;
 #endif
 		}
 	}
 	if (i & 1)
 		usgl->sge[i / 2].len[1] = htobe32(0);
 	KASSERT(nsegs == 0, ("%s: nsegs %d, start %p, iv_buffer %p",
 	    __func__, nsegs, start, iv_buffer));
 }
 
 /*
  * Similar to t4_push_frames() but handles TLS sockets when TLS offload
  * is enabled.  Rather than transmitting bulk data, the socket buffer
  * contains TLS records.  The work request requires a full TLS record,
  * so batch mbufs up until a full TLS record is seen.  This requires
  * reading the TLS header out of the start of each record to determine
  * its length.
  */
 void
 t4_push_tls_records(struct adapter *sc, struct toepcb *toep, int drop)
 {
 	struct tls_hdr thdr;
 	struct mbuf *sndptr;
 	struct fw_tlstx_data_wr *txwr;
 	struct cpl_tx_tls_sfo *cpl;
 	struct wrqe *wr;
 	u_int plen, nsegs, credits, space, max_nsegs_1mbuf, wr_len;
 	u_int expn_size, iv_len, pdus, sndptroff;
 	struct tls_ofld_info *tls_ofld = &toep->tls;
 	struct inpcb *inp = toep->inp;
 	struct tcpcb *tp = intotcpcb(inp);
 	struct socket *so = inp->inp_socket;
 	struct sockbuf *sb = &so->so_snd;
 	int tls_size, tx_credits, shove, /* compl,*/ sowwakeup;
 	struct ofld_tx_sdesc *txsd;
 	bool imm_ivs, imm_payload;
 	void *iv_buffer, *iv_dst, *buf;
 
 	INP_WLOCK_ASSERT(inp);
 	KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
 	    ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid));
 
 	KASSERT(ulp_mode(toep) == ULP_MODE_NONE ||
 	    ulp_mode(toep) == ULP_MODE_TCPDDP || ulp_mode(toep) == ULP_MODE_TLS,
 	    ("%s: ulp_mode %u for toep %p", __func__, ulp_mode(toep), toep));
 	KASSERT(tls_tx_key(toep),
 	    ("%s: TX key not set for toep %p", __func__, toep));
 
 #ifdef VERBOSE_TRACES
 	CTR4(KTR_CXGBE, "%s: tid %d toep flags %#x tp flags %#x drop %d",
 	    __func__, toep->tid, toep->flags, tp->t_flags);
 #endif
 	if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN))
 		return;
 
 #ifdef RATELIMIT
 	if (__predict_false(inp->inp_flags2 & INP_RATE_LIMIT_CHANGED) &&
 	    (update_tx_rate_limit(sc, toep, so->so_max_pacing_rate) == 0)) {
 		inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED;
 	}
 #endif
 
 	/*
 	 * This function doesn't resume by itself.  Someone else must clear the
 	 * flag and call this function.
 	 */
 	if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) {
 		KASSERT(drop == 0,
 		    ("%s: drop (%d) != 0 but tx is suspended", __func__, drop));
 		return;
 	}
 
 	txsd = &toep->txsd[toep->txsd_pidx];
 	for (;;) {
 		tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS);
 		space = max_imm_tls_space(tx_credits);
 		wr_len = sizeof(struct fw_tlstx_data_wr) +
 		    sizeof(struct cpl_tx_tls_sfo) + key_size(toep);
 		if (wr_len + CIPHER_BLOCK_SIZE + 1 > space) {
 #ifdef VERBOSE_TRACES
 			CTR5(KTR_CXGBE,
 			    "%s: tid %d tx_credits %d min_wr %d space %d",
 			    __func__, toep->tid, tx_credits, wr_len +
 			    CIPHER_BLOCK_SIZE + 1, space);
 #endif
 			return;
 		}
 
 		SOCKBUF_LOCK(sb);
 		sowwakeup = drop;
 		if (drop) {
 			sbdrop_locked(sb, drop);
 			MPASS(tls_ofld->sb_off >= drop);
 			tls_ofld->sb_off -= drop;
 			drop = 0;
 		}
 
 		/*
 		 * Send a FIN if requested, but only if there's no
 		 * more data to send.
 		 */
 		if (sbavail(sb) == tls_ofld->sb_off &&
 		    toep->flags & TPF_SEND_FIN) {
 			if (sowwakeup)
 				sowwakeup_locked(so);
 			else
 				SOCKBUF_UNLOCK(sb);
 			SOCKBUF_UNLOCK_ASSERT(sb);
 			t4_close_conn(sc, toep);
 			return;
 		}
 
 		if (sbavail(sb) < tls_ofld->sb_off + TLS_HEADER_LENGTH) {
 			/*
 			 * A full TLS header is not yet queued, stop
 			 * for now until more data is added to the
 			 * socket buffer.  However, if the connection
 			 * has been closed, we will never get the rest
 			 * of the header so just discard the partial
 			 * header and close the connection.
 			 */
 #ifdef VERBOSE_TRACES
 			CTR5(KTR_CXGBE, "%s: tid %d sbavail %d sb_off %d%s",
 			    __func__, toep->tid, sbavail(sb), tls_ofld->sb_off,
 			    toep->flags & TPF_SEND_FIN ? "" : " SEND_FIN");
 #endif
 			if (sowwakeup)
 				sowwakeup_locked(so);
 			else
 				SOCKBUF_UNLOCK(sb);
 			SOCKBUF_UNLOCK_ASSERT(sb);
 			if (toep->flags & TPF_SEND_FIN)
 				t4_close_conn(sc, toep);
 			return;
 		}
 
 		/* Read the header of the next TLS record. */
 		sndptr = sbsndmbuf(sb, tls_ofld->sb_off, &sndptroff);
 		m_copydata(sndptr, sndptroff, sizeof(thdr), (caddr_t)&thdr);
 		tls_size = htons(thdr.length);
 		plen = TLS_HEADER_LENGTH + tls_size;
 		pdus = howmany(tls_size, tls_ofld->k_ctx.frag_size);
 		iv_len = pdus * CIPHER_BLOCK_SIZE;
 
 		if (sbavail(sb) < tls_ofld->sb_off + plen) {
 			/*
 			 * The full TLS record is not yet queued, stop
 			 * for now until more data is added to the
 			 * socket buffer.  However, if the connection
 			 * has been closed, we will never get the rest
 			 * of the record so just discard the partial
 			 * record and close the connection.
 			 */
 #ifdef VERBOSE_TRACES
 			CTR6(KTR_CXGBE,
 			    "%s: tid %d sbavail %d sb_off %d plen %d%s",
 			    __func__, toep->tid, sbavail(sb), tls_ofld->sb_off,
 			    plen, toep->flags & TPF_SEND_FIN ? "" :
 			    " SEND_FIN");
 #endif
 			if (sowwakeup)
 				sowwakeup_locked(so);
 			else
 				SOCKBUF_UNLOCK(sb);
 			SOCKBUF_UNLOCK_ASSERT(sb);
 			if (toep->flags & TPF_SEND_FIN)
 				t4_close_conn(sc, toep);
 			return;
 		}
 
 		/* Shove if there is no additional data pending. */
 		shove = (sbavail(sb) == tls_ofld->sb_off + plen) &&
 		    !(tp->t_flags & TF_MORETOCOME);
 
 		if (sb->sb_flags & SB_AUTOSIZE &&
 		    V_tcp_do_autosndbuf &&
 		    sb->sb_hiwat < V_tcp_autosndbuf_max &&
 		    sbused(sb) >= sb->sb_hiwat * 7 / 8) {
 			int newsize = min(sb->sb_hiwat + V_tcp_autosndbuf_inc,
 			    V_tcp_autosndbuf_max);
 
 			if (!sbreserve_locked(sb, newsize, so, NULL))
 				sb->sb_flags &= ~SB_AUTOSIZE;
 			else
 				sowwakeup = 1;	/* room available */
 		}
 		if (sowwakeup)
 			sowwakeup_locked(so);
 		else
 			SOCKBUF_UNLOCK(sb);
 		SOCKBUF_UNLOCK_ASSERT(sb);
 
 		if (__predict_false(toep->flags & TPF_FIN_SENT))
 			panic("%s: excess tx.", __func__);
 
 		/* Determine whether to use immediate vs SGL. */
 		imm_payload = false;
 		imm_ivs = false;
 		if (wr_len + iv_len <= space) {
 			imm_ivs = true;
 			wr_len += iv_len;
 			if (wr_len + tls_size <= space) {
 				wr_len += tls_size;
 				imm_payload = true;
 			}
 		}
 
 		/* Allocate space for IVs if needed. */
 		if (!imm_ivs) {
 			iv_buffer = malloc(iv_len, M_CXGBE, M_NOWAIT);
 			if (iv_buffer == NULL) {
 				/*
 				 * XXX: How to restart this?
 				 */
 				if (sowwakeup)
 					sowwakeup_locked(so);
 				else
 					SOCKBUF_UNLOCK(sb);
 				SOCKBUF_UNLOCK_ASSERT(sb);
 				CTR3(KTR_CXGBE,
 			    "%s: tid %d failed to alloc IV space len %d",
 				    __func__, toep->tid, iv_len);
 				return;
 			}
 		} else
 			iv_buffer = NULL;
 
 		/* Determine size of SGL. */
 		nsegs = 0;
 		max_nsegs_1mbuf = 0; /* max # of SGL segments in any one mbuf */
 		if (!imm_payload) {
 			nsegs = count_mbuf_segs(sndptr, sndptroff +
 			    TLS_HEADER_LENGTH, tls_size, &max_nsegs_1mbuf);
 			if (!imm_ivs) {
 				int n = sglist_count(iv_buffer, iv_len);
 				nsegs += n;
 				if (n > max_nsegs_1mbuf)
 					max_nsegs_1mbuf = n;
 			}
 
 			/* Account for SGL in work request length. */
 			wr_len += sizeof(struct ulptx_sgl) +
 			    ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8;
 		}
 
 		wr = alloc_wrqe(roundup2(wr_len, 16), toep->ofld_txq);
 		if (wr == NULL) {
 			/* XXX: how will we recover from this? */
 			toep->flags |= TPF_TX_SUSPENDED;
 			return;
 		}
 
 #ifdef VERBOSE_TRACES
 		CTR5(KTR_CXGBE, "%s: tid %d TLS record %d len %#x pdus %d",
 		    __func__, toep->tid, thdr.type, tls_size, pdus);
 #endif
 		txwr = wrtod(wr);
 		cpl = (struct cpl_tx_tls_sfo *)(txwr + 1);
 		memset(txwr, 0, roundup2(wr_len, 16));
 		credits = howmany(wr_len, 16);
 		expn_size = tls_expansion_size(toep, tls_size, 0, NULL);
 		write_tlstx_wr(txwr, toep, imm_payload ? tls_size : 0,
 		    tls_size, expn_size, pdus, credits, shove, imm_ivs ? 1 : 0);
 		write_tlstx_cpl(cpl, toep, &thdr, tls_size, pdus);
 		tls_copy_tx_key(toep, cpl + 1);
 
 		/* Generate random IVs */
 		buf = (char *)(cpl + 1) + key_size(toep);
 		if (imm_ivs) {
 			MPASS(iv_buffer == NULL);
 			iv_dst = buf;
 			buf = (char *)iv_dst + iv_len;
 		} else
 			iv_dst = iv_buffer;
 		arc4rand(iv_dst, iv_len, 0);
 
 		if (imm_payload) {
 			m_copydata(sndptr, sndptroff + TLS_HEADER_LENGTH,
 			    tls_size, buf);
 		} else {
 			write_tlstx_sgl(buf, sndptr,
 			    sndptroff + TLS_HEADER_LENGTH, tls_size, iv_buffer,
 			    iv_len, nsegs, max_nsegs_1mbuf);
 		}
 
 		KASSERT(toep->tx_credits >= credits,
 			("%s: not enough credits", __func__));
 
 		toep->tx_credits -= credits;
 
 		tp->snd_nxt += plen;
 		tp->snd_max += plen;
 
 		SOCKBUF_LOCK(sb);
 		sbsndptr_adv(sb, sb->sb_sndptr, plen);
 		tls_ofld->sb_off += plen;
 		SOCKBUF_UNLOCK(sb);
 
 		toep->flags |= TPF_TX_DATA_SENT;
 		if (toep->tx_credits < MIN_OFLD_TLSTX_CREDITS(toep))
 			toep->flags |= TPF_TX_SUSPENDED;
 
 		KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__));
 		txsd->plen = plen;
 		txsd->tx_credits = credits;
 		txsd->iv_buffer = iv_buffer;
 		txsd++;
 		if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) {
 			toep->txsd_pidx = 0;
 			txsd = &toep->txsd[0];
 		}
 		toep->txsd_avail--;
 
 		atomic_add_long(&toep->vi->pi->tx_toe_tls_records, 1);
 		atomic_add_long(&toep->vi->pi->tx_toe_tls_octets, plen);
 
 		t4_l2t_send(sc, wr, toep->l2te);
 	}
 }
 
 #ifdef KERN_TLS
 static int
 count_ext_pgs_segs(struct mbuf_ext_pgs *ext_pgs)
 {
 	vm_paddr_t nextpa;
 	u_int i, nsegs;
 
 	MPASS(ext_pgs->npgs > 0);
 	nsegs = 1;
 	nextpa = ext_pgs->pa[0] + PAGE_SIZE;
 	for (i = 1; i < ext_pgs->npgs; i++) {
 		if (nextpa != ext_pgs->pa[i])
 			nsegs++;
 		nextpa = ext_pgs->pa[i] + PAGE_SIZE;
 	}
 	return (nsegs);
 }
 
 static void
 write_ktlstx_sgl(void *dst, struct mbuf_ext_pgs *ext_pgs, int nsegs)
 {
 	struct ulptx_sgl *usgl = dst;
 	vm_paddr_t pa;
 	uint32_t len;
 	int i, j;
 
 	KASSERT(nsegs > 0, ("%s: nsegs 0", __func__));
 
 	usgl->cmd_nsge = htobe32(V_ULPTX_CMD(ULP_TX_SC_DSGL) |
 	    V_ULPTX_NSGE(nsegs));
 
 	/* Figure out the first S/G length. */
 	pa = ext_pgs->pa[0] + ext_pgs->first_pg_off;
 	usgl->addr0 = htobe64(pa);
 	len = mbuf_ext_pg_len(ext_pgs, 0, ext_pgs->first_pg_off);
 	pa += len;
 	for (i = 1; i < ext_pgs->npgs; i++) {
 		if (ext_pgs->pa[i] != pa)
 			break;
 		len += mbuf_ext_pg_len(ext_pgs, i, 0);
 		pa += mbuf_ext_pg_len(ext_pgs, i, 0);
 	}
 	usgl->len0 = htobe32(len);
 #ifdef INVARIANTS
 	nsegs--;
 #endif
 
 	j = -1;
 	for (; i < ext_pgs->npgs; i++) {
 		if (j == -1 || ext_pgs->pa[i] != pa) {
 			if (j >= 0)
 				usgl->sge[j / 2].len[j & 1] = htobe32(len);
 			j++;
 #ifdef INVARIANTS
 			nsegs--;
 #endif
 			pa = ext_pgs->pa[i];
 			usgl->sge[j / 2].addr[j & 1] = htobe64(pa);
 			len = mbuf_ext_pg_len(ext_pgs, i, 0);
 			pa += len;
 		} else {
 			len += mbuf_ext_pg_len(ext_pgs, i, 0);
 			pa += mbuf_ext_pg_len(ext_pgs, i, 0);
 		}
 	}
 	if (j >= 0) {
 		usgl->sge[j / 2].len[j & 1] = htobe32(len);
 
 		if ((j & 1) == 0)
 			usgl->sge[j / 2].len[1] = htobe32(0);
 	}
 	KASSERT(nsegs == 0, ("%s: nsegs %d, ext_pgs %p", __func__, nsegs,
 	    ext_pgs));
 }
 
 /*
  * Similar to t4_push_frames() but handles sockets that contain TLS
  * record mbufs.  Unlike TLSOM, each mbuf is a complete TLS record and
  * corresponds to a single work request.
  */
 void
 t4_push_ktls(struct adapter *sc, struct toepcb *toep, int drop)
 {
 	struct tls_hdr *thdr;
 	struct fw_tlstx_data_wr *txwr;
 	struct cpl_tx_tls_sfo *cpl;
 	struct wrqe *wr;
 	struct mbuf *m;
 	u_int nsegs, credits, wr_len;
 	u_int expn_size;
 	struct inpcb *inp = toep->inp;
 	struct tcpcb *tp = intotcpcb(inp);
 	struct socket *so = inp->inp_socket;
 	struct sockbuf *sb = &so->so_snd;
 	int tls_size, tx_credits, shove, sowwakeup;
 	struct ofld_tx_sdesc *txsd;
 	char *buf;
 
 	INP_WLOCK_ASSERT(inp);
 	KASSERT(toep->flags & TPF_FLOWC_WR_SENT,
 	    ("%s: flowc_wr not sent for tid %u.", __func__, toep->tid));
 
 	KASSERT(ulp_mode(toep) == ULP_MODE_NONE ||
 	    ulp_mode(toep) == ULP_MODE_TCPDDP,
 	    ("%s: ulp_mode %u for toep %p", __func__, ulp_mode(toep), toep));
 	KASSERT(tls_tx_key(toep),
 	    ("%s: TX key not set for toep %p", __func__, toep));
 
 #ifdef VERBOSE_TRACES
 	CTR4(KTR_CXGBE, "%s: tid %d toep flags %#x tp flags %#x drop %d",
 	    __func__, toep->tid, toep->flags, tp->t_flags);
 #endif
 	if (__predict_false(toep->flags & TPF_ABORT_SHUTDOWN))
 		return;
 
 #ifdef RATELIMIT
 	if (__predict_false(inp->inp_flags2 & INP_RATE_LIMIT_CHANGED) &&
 	    (update_tx_rate_limit(sc, toep, so->so_max_pacing_rate) == 0)) {
 		inp->inp_flags2 &= ~INP_RATE_LIMIT_CHANGED;
 	}
 #endif
 
 	/*
 	 * This function doesn't resume by itself.  Someone else must clear the
 	 * flag and call this function.
 	 */
 	if (__predict_false(toep->flags & TPF_TX_SUSPENDED)) {
 		KASSERT(drop == 0,
 		    ("%s: drop (%d) != 0 but tx is suspended", __func__, drop));
 		return;
 	}
 
 	txsd = &toep->txsd[toep->txsd_pidx];
 	for (;;) {
 		tx_credits = min(toep->tx_credits, MAX_OFLD_TX_CREDITS);
 
 		SOCKBUF_LOCK(sb);
 		sowwakeup = drop;
 		if (drop) {
 			sbdrop_locked(sb, drop);
 			drop = 0;
 		}
 
 		m = sb->sb_sndptr != NULL ? sb->sb_sndptr->m_next : sb->sb_mb;
 
 		/*
 		 * Send a FIN if requested, but only if there's no
 		 * more data to send.
 		 */
 		if (m == NULL && toep->flags & TPF_SEND_FIN) {
 			if (sowwakeup)
 				sowwakeup_locked(so);
 			else
 				SOCKBUF_UNLOCK(sb);
 			SOCKBUF_UNLOCK_ASSERT(sb);
 			t4_close_conn(sc, toep);
 			return;
 		}
 
 		/*
 		 * If there is no ready data to send, wait until more
 		 * data arrives.
 		 */
 		if (m == NULL || (m->m_flags & M_NOTAVAIL) != 0) {
 			if (sowwakeup)
 				sowwakeup_locked(so);
 			else
 				SOCKBUF_UNLOCK(sb);
 			SOCKBUF_UNLOCK_ASSERT(sb);
 #ifdef VERBOSE_TRACES
 			CTR2(KTR_CXGBE, "%s: tid %d no ready data to send",
 			    __func__, toep->tid);
 #endif
 			return;
 		}
 
 		KASSERT(m->m_flags & M_NOMAP, ("%s: mbuf %p is not NOMAP",
 		    __func__, m));
 		KASSERT(m->m_ext.ext_pgs->tls != NULL,
 		    ("%s: mbuf %p doesn't have TLS session", __func__, m));
 
 		/* Calculate WR length. */
 		wr_len = sizeof(struct fw_tlstx_data_wr) +
 		    sizeof(struct cpl_tx_tls_sfo) + key_size(toep);
 
 		/* Explicit IVs for AES-CBC and AES-GCM are <= 16. */
 		MPASS(toep->tls.iv_len <= AES_BLOCK_LEN);
 		wr_len += AES_BLOCK_LEN;
 
 		/* Account for SGL in work request length. */
 		nsegs = count_ext_pgs_segs(m->m_ext.ext_pgs);
 		wr_len += sizeof(struct ulptx_sgl) +
 		    ((3 * (nsegs - 1)) / 2 + ((nsegs - 1) & 1)) * 8;
 
 		/* Not enough credits for this work request. */
 		if (howmany(wr_len, 16) > tx_credits) {
 			if (sowwakeup)
 				sowwakeup_locked(so);
 			else
 				SOCKBUF_UNLOCK(sb);
 			SOCKBUF_UNLOCK_ASSERT(sb);
 #ifdef VERBOSE_TRACES
 			CTR5(KTR_CXGBE,
 	    "%s: tid %d mbuf %p requires %d credits, but only %d available",
 			    __func__, toep->tid, m, howmany(wr_len, 16),
 			    tx_credits);
 #endif
 			toep->flags |= TPF_TX_SUSPENDED;
 			return;
 		}
 	
 		/* Shove if there is no additional data pending. */
 		shove = ((m->m_next == NULL ||
 		    (m->m_next->m_flags & M_NOTAVAIL) != 0)) &&
 		    (tp->t_flags & TF_MORETOCOME) == 0;
 
 		if (sb->sb_flags & SB_AUTOSIZE &&
 		    V_tcp_do_autosndbuf &&
 		    sb->sb_hiwat < V_tcp_autosndbuf_max &&
 		    sbused(sb) >= sb->sb_hiwat * 7 / 8) {
 			int newsize = min(sb->sb_hiwat + V_tcp_autosndbuf_inc,
 			    V_tcp_autosndbuf_max);
 
 			if (!sbreserve_locked(sb, newsize, so, NULL))
 				sb->sb_flags &= ~SB_AUTOSIZE;
 			else
 				sowwakeup = 1;	/* room available */
 		}
 		if (sowwakeup)
 			sowwakeup_locked(so);
 		else
 			SOCKBUF_UNLOCK(sb);
 		SOCKBUF_UNLOCK_ASSERT(sb);
 
 		if (__predict_false(toep->flags & TPF_FIN_SENT))
 			panic("%s: excess tx.", __func__);
 
 		wr = alloc_wrqe(roundup2(wr_len, 16), toep->ofld_txq);
 		if (wr == NULL) {
 			/* XXX: how will we recover from this? */
 			toep->flags |= TPF_TX_SUSPENDED;
 			return;
 		}
 
 		thdr = (struct tls_hdr *)m->m_ext.ext_pgs->hdr;
 #ifdef VERBOSE_TRACES
 		CTR5(KTR_CXGBE, "%s: tid %d TLS record %ju type %d len %#x",
 		    __func__, toep->tid, m->m_ext.ext_pgs->seqno, thdr->type,
 		    m->m_len);
 #endif
 		txwr = wrtod(wr);
 		cpl = (struct cpl_tx_tls_sfo *)(txwr + 1);
 		memset(txwr, 0, roundup2(wr_len, 16));
 		credits = howmany(wr_len, 16);
 		expn_size = m->m_ext.ext_pgs->hdr_len +
 		    m->m_ext.ext_pgs->trail_len;
 		tls_size = m->m_len - expn_size;
 		write_tlstx_wr(txwr, toep, 0,
 		    tls_size, expn_size, 1, credits, shove, 1);
 		toep->tls.tx_seq_no = m->m_ext.ext_pgs->seqno;
 		write_tlstx_cpl(cpl, toep, thdr, tls_size, 1);
 		tls_copy_tx_key(toep, cpl + 1);
 
 		/* Copy IV. */
 		buf = (char *)(cpl + 1) + key_size(toep);
 		memcpy(buf, thdr + 1, toep->tls.iv_len);
 		buf += AES_BLOCK_LEN;
 
 		write_ktlstx_sgl(buf, m->m_ext.ext_pgs, nsegs);
 
 		KASSERT(toep->tx_credits >= credits,
 			("%s: not enough credits", __func__));
 
 		toep->tx_credits -= credits;
 
 		tp->snd_nxt += m->m_len;
 		tp->snd_max += m->m_len;
 
 		SOCKBUF_LOCK(sb);
 		sb->sb_sndptr = m;
 		SOCKBUF_UNLOCK(sb);
 
 		toep->flags |= TPF_TX_DATA_SENT;
 		if (toep->tx_credits < MIN_OFLD_TLSTX_CREDITS(toep))
 			toep->flags |= TPF_TX_SUSPENDED;
 
 		KASSERT(toep->txsd_avail > 0, ("%s: no txsd", __func__));
 		txsd->plen = m->m_len;
 		txsd->tx_credits = credits;
 		txsd++;
 		if (__predict_false(++toep->txsd_pidx == toep->txsd_total)) {
 			toep->txsd_pidx = 0;
 			txsd = &toep->txsd[0];
 		}
 		toep->txsd_avail--;
 
 		atomic_add_long(&toep->vi->pi->tx_toe_tls_records, 1);
 		atomic_add_long(&toep->vi->pi->tx_toe_tls_octets, m->m_len);
 
 		t4_l2t_send(sc, wr, toep->l2te);
 	}
 }
 #endif
 
 /*
  * For TLS data we place received mbufs received via CPL_TLS_DATA into
  * an mbufq in the TLS offload state.  When CPL_RX_TLS_CMP is
  * received, the completed PDUs are placed into the socket receive
  * buffer.
  *
  * The TLS code reuses the ulp_pdu_reclaimq to hold the pending mbufs.
  */
 static int
 do_tls_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
 {
 	struct adapter *sc = iq->adapter;
 	const struct cpl_tls_data *cpl = mtod(m, const void *);
 	unsigned int tid = GET_TID(cpl);
 	struct toepcb *toep = lookup_tid(sc, tid);
 	struct inpcb *inp = toep->inp;
 	struct tcpcb *tp;
 	int len;
 
 	/* XXX: Should this match do_rx_data instead? */
 	KASSERT(!(toep->flags & TPF_SYNQE),
 	    ("%s: toep %p claims to be a synq entry", __func__, toep));
 
 	KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__));
 
 	/* strip off CPL header */
 	m_adj(m, sizeof(*cpl));
 	len = m->m_pkthdr.len;
 
 	atomic_add_long(&toep->vi->pi->rx_toe_tls_octets, len);
 
 	KASSERT(len == G_CPL_TLS_DATA_LENGTH(be32toh(cpl->length_pkd)),
 	    ("%s: payload length mismatch", __func__));
 
 	INP_WLOCK(inp);
 	if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) {
 		CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x",
 		    __func__, tid, len, inp->inp_flags);
 		INP_WUNLOCK(inp);
 		m_freem(m);
 		return (0);
 	}
 
 	/* Save TCP sequence number. */
 	m->m_pkthdr.tls_tcp_seq = be32toh(cpl->seq);
 
 	if (mbufq_enqueue(&toep->ulp_pdu_reclaimq, m)) {
 #ifdef INVARIANTS
 		panic("Failed to queue TLS data packet");
 #else
 		printf("%s: Failed to queue TLS data packet\n", __func__);
 		INP_WUNLOCK(inp);
 		m_freem(m);
 		return (0);
 #endif
 	}
 
 	tp = intotcpcb(inp);
 	tp->t_rcvtime = ticks;
 
 #ifdef VERBOSE_TRACES
 	CTR4(KTR_CXGBE, "%s: tid %u len %d seq %u", __func__, tid, len,
 	    be32toh(cpl->seq));
 #endif
 
 	INP_WUNLOCK(inp);
 	return (0);
 }
 
 static int
 do_rx_tls_cmp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
 {
 	struct adapter *sc = iq->adapter;
 	const struct cpl_rx_tls_cmp *cpl = mtod(m, const void *);
 	struct tlsrx_hdr_pkt *tls_hdr_pkt;
 	unsigned int tid = GET_TID(cpl);
 	struct toepcb *toep = lookup_tid(sc, tid);
 	struct inpcb *inp = toep->inp;
 	struct tcpcb *tp;
 	struct socket *so;
 	struct sockbuf *sb;
 	struct mbuf *tls_data;
 	int len, pdu_length, rx_credits;
 
 	KASSERT(toep->tid == tid, ("%s: toep tid/atid mismatch", __func__));
 	KASSERT(!(toep->flags & TPF_SYNQE),
 	    ("%s: toep %p claims to be a synq entry", __func__, toep));
 
 	/* strip off CPL header */
 	m_adj(m, sizeof(*cpl));
 	len = m->m_pkthdr.len;
 
 	atomic_add_long(&toep->vi->pi->rx_toe_tls_records, 1);
 
 	KASSERT(len == G_CPL_RX_TLS_CMP_LENGTH(be32toh(cpl->pdulength_length)),
 	    ("%s: payload length mismatch", __func__));
 
 	INP_WLOCK(inp);
 	if (inp->inp_flags & (INP_DROPPED | INP_TIMEWAIT)) {
 		CTR4(KTR_CXGBE, "%s: tid %u, rx (%d bytes), inp_flags 0x%x",
 		    __func__, tid, len, inp->inp_flags);
 		INP_WUNLOCK(inp);
 		m_freem(m);
 		return (0);
 	}
 
 	pdu_length = G_CPL_RX_TLS_CMP_PDULENGTH(be32toh(cpl->pdulength_length));
 
 	tp = intotcpcb(inp);
 
 #ifdef VERBOSE_TRACES
 	CTR6(KTR_CXGBE, "%s: tid %u PDU len %d len %d seq %u, rcv_nxt %u",
 	    __func__, tid, pdu_length, len, be32toh(cpl->seq), tp->rcv_nxt);
 #endif
 
 	tp->rcv_nxt += pdu_length;
 	if (tp->rcv_wnd < pdu_length) {
 		toep->tls.rcv_over += pdu_length - tp->rcv_wnd;
 		tp->rcv_wnd = 0;
 	} else
 		tp->rcv_wnd -= pdu_length;
 
 	/* XXX: Not sure what to do about urgent data. */
 
 	/*
 	 * The payload of this CPL is the TLS header followed by
 	 * additional fields.
 	 */
 	KASSERT(m->m_len >= sizeof(*tls_hdr_pkt),
 	    ("%s: payload too small", __func__));
 	tls_hdr_pkt = mtod(m, void *);
 
 	/*
 	 * Only the TLS header is sent to OpenSSL, so report errors by
 	 * altering the record type.
 	 */
 	if ((tls_hdr_pkt->res_to_mac_error & M_TLSRX_HDR_PKT_ERROR) != 0)
 		tls_hdr_pkt->type = CONTENT_TYPE_ERROR;
 
 	/* Trim this CPL's mbuf to only include the TLS header. */
 	KASSERT(m->m_len == len && m->m_next == NULL,
 	    ("%s: CPL spans multiple mbufs", __func__));
 	m->m_len = TLS_HEADER_LENGTH;
 	m->m_pkthdr.len = TLS_HEADER_LENGTH;
 
 	tls_data = mbufq_dequeue(&toep->ulp_pdu_reclaimq);
 	if (tls_data != NULL) {
 		KASSERT(be32toh(cpl->seq) == tls_data->m_pkthdr.tls_tcp_seq,
 		    ("%s: sequence mismatch", __func__));
 
 		/*
 		 * Update the TLS header length to be the length of
 		 * the payload data.
 		 */
 		tls_hdr_pkt->length = htobe16(tls_data->m_pkthdr.len);
 
 		m->m_next = tls_data;
 		m->m_pkthdr.len += tls_data->m_len;
 	}
 
 	so = inp_inpcbtosocket(inp);
 	sb = &so->so_rcv;
 	SOCKBUF_LOCK(sb);
 
 	if (__predict_false(sb->sb_state & SBS_CANTRCVMORE)) {
 		struct epoch_tracker et;
 
 		CTR3(KTR_CXGBE, "%s: tid %u, excess rx (%d bytes)",
 		    __func__, tid, pdu_length);
 		m_freem(m);
 		SOCKBUF_UNLOCK(sb);
 		INP_WUNLOCK(inp);
 
 		CURVNET_SET(toep->vnet);
 		NET_EPOCH_ENTER(et);
 		INP_WLOCK(inp);
 		tp = tcp_drop(tp, ECONNRESET);
 		if (tp)
 			INP_WUNLOCK(inp);
 		NET_EPOCH_EXIT(et);
 		CURVNET_RESTORE();
 
 		return (0);
 	}
 
 	/*
 	 * Not all of the bytes on the wire are included in the socket buffer
 	 * (e.g. the MAC of the TLS record).  However, those bytes are included
 	 * in the TCP sequence space.
 	 */
 
 	/* receive buffer autosize */
 	MPASS(toep->vnet == so->so_vnet);
 	CURVNET_SET(toep->vnet);
 	if (sb->sb_flags & SB_AUTOSIZE &&
 	    V_tcp_do_autorcvbuf &&
 	    sb->sb_hiwat < V_tcp_autorcvbuf_max &&
 	    m->m_pkthdr.len > (sbspace(sb) / 8 * 7)) {
 		unsigned int hiwat = sb->sb_hiwat;
 		unsigned int newsize = min(hiwat + sc->tt.autorcvbuf_inc,
 		    V_tcp_autorcvbuf_max);
 
 		if (!sbreserve_locked(sb, newsize, so, NULL))
 			sb->sb_flags &= ~SB_AUTOSIZE;
 	}
 
 	sbappendstream_locked(sb, m, 0);
 	rx_credits = sbspace(sb) > tp->rcv_wnd ? sbspace(sb) - tp->rcv_wnd : 0;
 #ifdef VERBOSE_TRACES
 	CTR4(KTR_CXGBE, "%s: tid %u rx_credits %u rcv_wnd %u",
 	    __func__, tid, rx_credits, tp->rcv_wnd);
 #endif
 	if (rx_credits > 0 && sbused(sb) + tp->rcv_wnd < sb->sb_lowat) {
 		rx_credits = send_rx_credits(sc, toep, rx_credits);
 		tp->rcv_wnd += rx_credits;
 		tp->rcv_adv += rx_credits;
 	}
 
 	sorwakeup_locked(so);
 	SOCKBUF_UNLOCK_ASSERT(sb);
 
 	INP_WUNLOCK(inp);
 	CURVNET_RESTORE();
 	return (0);
 }
 
 void
 t4_tls_mod_load(void)
 {
 
 	mtx_init(&tls_handshake_lock, "t4tls handshake", NULL, MTX_DEF);
 	t4_register_cpl_handler(CPL_TLS_DATA, do_tls_data);
 	t4_register_cpl_handler(CPL_RX_TLS_CMP, do_rx_tls_cmp);
 }
 
 void
 t4_tls_mod_unload(void)
 {
 
 	t4_register_cpl_handler(CPL_TLS_DATA, NULL);
 	t4_register_cpl_handler(CPL_RX_TLS_CMP, NULL);
 	mtx_destroy(&tls_handshake_lock);
 }
 #endif	/* TCP_OFFLOAD */
Index: head/sys/dev/glxsb/glxsb.c
===================================================================
--- head/sys/dev/glxsb/glxsb.c	(revision 359373)
+++ head/sys/dev/glxsb/glxsb.c	(revision 359374)
@@ -1,873 +1,779 @@
 /* $OpenBSD: glxsb.c,v 1.7 2007/02/12 14:31:45 tom Exp $ */
 
 /*
  * Copyright (c) 2006 Tom Cosgrove <tom@openbsd.org>
  * Copyright (c) 2003, 2004 Theo de Raadt
  * Copyright (c) 2003 Jason Wright
  *
  * Permission to use, copy, modify, and distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
  * copyright notice and this permission notice appear in all copies.
  *
  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  */
 
 /*
  * Driver for the security block on the AMD Geode LX processors
  * http://www.amd.com/files/connectivitysolutions/geode/geode_lx/33234d_lx_ds.pdf
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/bus.h>
 #include <sys/errno.h>
 #include <sys/kernel.h>
 #include <sys/lock.h>
 #include <sys/malloc.h>
 #include <sys/mbuf.h>
 #include <sys/module.h>
 #include <sys/mutex.h>
 #include <sys/proc.h>
 #include <sys/random.h>
 #include <sys/rman.h>
 #include <sys/rwlock.h>
 #include <sys/sysctl.h>
 #include <sys/taskqueue.h>
 
 #include <machine/bus.h>
 #include <machine/cpufunc.h>
 #include <machine/resource.h>
 
 #include <dev/pci/pcivar.h>
 #include <dev/pci/pcireg.h>
 
 #include <opencrypto/cryptodev.h>
-#include <opencrypto/cryptosoft.h>
 #include <opencrypto/xform.h>
 
 #include "cryptodev_if.h"
 #include "glxsb.h"
 
 #define PCI_VENDOR_AMD			0x1022	/* AMD */
 #define PCI_PRODUCT_AMD_GEODE_LX_CRYPTO	0x2082	/* Geode LX Crypto */
 
 #define SB_GLD_MSR_CAP		0x58002000	/* RO - Capabilities */
 #define SB_GLD_MSR_CONFIG	0x58002001	/* RW - Master Config */
 #define SB_GLD_MSR_SMI		0x58002002	/* RW - SMI */
 #define SB_GLD_MSR_ERROR	0x58002003	/* RW - Error */
 #define SB_GLD_MSR_PM		0x58002004	/* RW - Power Mgmt */
 #define SB_GLD_MSR_DIAG		0x58002005	/* RW - Diagnostic */
 #define SB_GLD_MSR_CTRL		0x58002006	/* RW - Security Block Cntrl */
 
 						/* For GLD_MSR_CTRL: */
 #define SB_GMC_DIV0		0x0000		/* AES update divisor values */
 #define SB_GMC_DIV1		0x0001
 #define SB_GMC_DIV2		0x0002
 #define SB_GMC_DIV3		0x0003
 #define SB_GMC_DIV_MASK		0x0003
 #define SB_GMC_SBI		0x0004		/* AES swap bits */
 #define SB_GMC_SBY		0x0008		/* AES swap bytes */
 #define SB_GMC_TW		0x0010		/* Time write (EEPROM) */
 #define SB_GMC_T_SEL0		0x0000		/* RNG post-proc: none */
 #define SB_GMC_T_SEL1		0x0100		/* RNG post-proc: LFSR */
 #define SB_GMC_T_SEL2		0x0200		/* RNG post-proc: whitener */
 #define SB_GMC_T_SEL3		0x0300		/* RNG LFSR+whitener */
 #define SB_GMC_T_SEL_MASK	0x0300
 #define SB_GMC_T_NE		0x0400		/* Noise (generator) Enable */
 #define SB_GMC_T_TM		0x0800		/* RNG test mode */
 						/*     (deterministic) */
 
 /* Security Block configuration/control registers (offsets from base) */
 #define SB_CTL_A		0x0000		/* RW - SB Control A */
 #define SB_CTL_B		0x0004		/* RW - SB Control B */
 #define SB_AES_INT		0x0008		/* RW - SB AES Interrupt */
 #define SB_SOURCE_A		0x0010		/* RW - Source A */
 #define SB_DEST_A		0x0014		/* RW - Destination A */
 #define SB_LENGTH_A		0x0018		/* RW - Length A */
 #define SB_SOURCE_B		0x0020		/* RW - Source B */
 #define SB_DEST_B		0x0024		/* RW - Destination B */
 #define SB_LENGTH_B		0x0028		/* RW - Length B */
 #define SB_WKEY			0x0030		/* WO - Writable Key 0-3 */
 #define SB_WKEY_0		0x0030		/* WO - Writable Key 0 */
 #define SB_WKEY_1		0x0034		/* WO - Writable Key 1 */
 #define SB_WKEY_2		0x0038		/* WO - Writable Key 2 */
 #define SB_WKEY_3		0x003C		/* WO - Writable Key 3 */
 #define SB_CBC_IV		0x0040		/* RW - CBC IV 0-3 */
 #define SB_CBC_IV_0		0x0040		/* RW - CBC IV 0 */
 #define SB_CBC_IV_1		0x0044		/* RW - CBC IV 1 */
 #define SB_CBC_IV_2		0x0048		/* RW - CBC IV 2 */
 #define SB_CBC_IV_3		0x004C		/* RW - CBC IV 3 */
 #define SB_RANDOM_NUM		0x0050		/* RW - Random Number */
 #define SB_RANDOM_NUM_STATUS	0x0054		/* RW - Random Number Status */
 #define SB_EEPROM_COMM		0x0800		/* RW - EEPROM Command */
 #define SB_EEPROM_ADDR		0x0804		/* RW - EEPROM Address */
 #define SB_EEPROM_DATA		0x0808		/* RW - EEPROM Data */
 #define SB_EEPROM_SEC_STATE	0x080C		/* RW - EEPROM Security State */
 
 						/* For SB_CTL_A and _B */
 #define SB_CTL_ST		0x0001		/* Start operation (enc/dec) */
 #define SB_CTL_ENC		0x0002		/* Encrypt (0 is decrypt) */
 #define SB_CTL_DEC		0x0000		/* Decrypt */
 #define SB_CTL_WK		0x0004		/* Use writable key (we set) */
 #define SB_CTL_DC		0x0008		/* Destination coherent */
 #define SB_CTL_SC		0x0010		/* Source coherent */
 #define SB_CTL_CBC		0x0020		/* CBC (0 is ECB) */
 
 						/* For SB_AES_INT */
 #define SB_AI_DISABLE_AES_A	0x0001		/* Disable AES A compl int */
 #define SB_AI_ENABLE_AES_A	0x0000		/* Enable AES A compl int */
 #define SB_AI_DISABLE_AES_B	0x0002		/* Disable AES B compl int */
 #define SB_AI_ENABLE_AES_B	0x0000		/* Enable AES B compl int */
 #define SB_AI_DISABLE_EEPROM	0x0004		/* Disable EEPROM op comp int */
 #define SB_AI_ENABLE_EEPROM	0x0000		/* Enable EEPROM op compl int */
 #define SB_AI_AES_A_COMPLETE   0x10000		/* AES A operation complete */
 #define SB_AI_AES_B_COMPLETE   0x20000		/* AES B operation complete */
 #define SB_AI_EEPROM_COMPLETE  0x40000		/* EEPROM operation complete */
 
 #define SB_AI_CLEAR_INTR \
 	(SB_AI_DISABLE_AES_A | SB_AI_DISABLE_AES_B |\
 	SB_AI_DISABLE_EEPROM | SB_AI_AES_A_COMPLETE |\
 	SB_AI_AES_B_COMPLETE | SB_AI_EEPROM_COMPLETE)
 
 #define SB_RNS_TRNG_VALID	0x0001		/* in SB_RANDOM_NUM_STATUS */
 
 #define SB_MEM_SIZE		0x0810		/* Size of memory block */
 
 #define SB_AES_ALIGN		0x0010		/* Source and dest buffers */
 						/* must be 16-byte aligned */
 #define SB_AES_BLOCK_SIZE	0x0010
 
 /*
  * The Geode LX security block AES acceleration doesn't perform scatter-
  * gather: it just takes source and destination addresses.  Therefore the
  * plain- and ciphertexts need to be contiguous.  To this end, we allocate
  * a buffer for both, and accept the overhead of copying in and out.  If
  * the number of bytes in one operation is bigger than allowed for by the
  * buffer (buffer is twice the size of the max length, as it has both input
  * and output) then we have to perform multiple encryptions/decryptions.
  */
 
 #define GLXSB_MAX_AES_LEN	16384
 
 MALLOC_DEFINE(M_GLXSB, "glxsb_data", "Glxsb Data");
 
 struct glxsb_dma_map {
 	bus_dmamap_t		dma_map;	/* DMA map */
 	bus_dma_segment_t	dma_seg;	/* segments */
 	int			dma_nsegs;	/* #segments */
 	int			dma_size;	/* size */
 	caddr_t			dma_vaddr;	/* virtual address */
 	bus_addr_t		dma_paddr;	/* physical address */
 };
 
 struct glxsb_taskop {
 	struct glxsb_session	*to_ses;	/* crypto session */
 	struct cryptop		*to_crp;	/* cryptop to perfom */
-	struct cryptodesc	*to_enccrd;	/* enccrd to perform */
-	struct cryptodesc	*to_maccrd;	/* maccrd to perform */
 };
 
 struct glxsb_softc {
 	device_t		sc_dev;		/* device backpointer */
 	struct resource		*sc_sr;		/* resource */
 	int			sc_rid;		/* resource rid */
 	struct callout		sc_rngco;	/* RNG callout */
 	int			sc_rnghz;	/* RNG callout ticks */
 	bus_dma_tag_t		sc_dmat;	/* DMA tag */
 	struct glxsb_dma_map	sc_dma;		/* DMA map */
 	int32_t			sc_cid;		/* crypto tag */
 	struct mtx		sc_task_mtx;	/* task mutex */
 	struct taskqueue	*sc_tq;		/* task queue */
 	struct task		sc_cryptotask;	/* task */
 	struct glxsb_taskop	sc_to;		/* task's crypto operation */
 	int			sc_task_count;	/* tasks count */
 };
 
 static int glxsb_probe(device_t);
 static int glxsb_attach(device_t);
 static int glxsb_detach(device_t);
 
 static void glxsb_dmamap_cb(void *, bus_dma_segment_t *, int, int);
 static int  glxsb_dma_alloc(struct glxsb_softc *);
 static void glxsb_dma_pre_op(struct glxsb_softc *, struct glxsb_dma_map *);
 static void glxsb_dma_post_op(struct glxsb_softc *, struct glxsb_dma_map *);
 static void glxsb_dma_free(struct glxsb_softc *, struct glxsb_dma_map *);
 
 static void glxsb_rnd(void *);
 static int  glxsb_crypto_setup(struct glxsb_softc *);
-static int  glxsb_crypto_newsession(device_t, crypto_session_t, struct cryptoini *);
+static int  glxsb_crypto_probesession(device_t,
+	const struct crypto_session_params *);
+static int  glxsb_crypto_newsession(device_t, crypto_session_t,
+	const struct crypto_session_params *);
 static void glxsb_crypto_freesession(device_t, crypto_session_t);
 static int  glxsb_aes(struct glxsb_softc *, uint32_t, uint32_t,
-	uint32_t, void *, int, void *);
+	uint32_t, const void *, int, const void *);
 
-static int  glxsb_crypto_encdec(struct cryptop *, struct cryptodesc *,
-	struct glxsb_session *, struct glxsb_softc *);
+static int  glxsb_crypto_encdec(struct cryptop *, struct glxsb_session *,
+	struct glxsb_softc *);
 
 static void glxsb_crypto_task(void *, int);
 static int  glxsb_crypto_process(device_t, struct cryptop *, int);
 
 static device_method_t glxsb_methods[] = {
 	/* device interface */
 	DEVMETHOD(device_probe,		glxsb_probe),
 	DEVMETHOD(device_attach,	glxsb_attach),
 	DEVMETHOD(device_detach,	glxsb_detach),
 
 	/* crypto device methods */
+	DEVMETHOD(cryptodev_probesession,	glxsb_crypto_probesession),
 	DEVMETHOD(cryptodev_newsession,		glxsb_crypto_newsession),
 	DEVMETHOD(cryptodev_freesession,	glxsb_crypto_freesession),
 	DEVMETHOD(cryptodev_process,		glxsb_crypto_process),
 
 	{0,0}
 };
 
 static driver_t glxsb_driver = {
 	"glxsb",
 	glxsb_methods,
 	sizeof(struct glxsb_softc)
 };
 
 static devclass_t glxsb_devclass;
 
 DRIVER_MODULE(glxsb, pci, glxsb_driver, glxsb_devclass, 0, 0);
 MODULE_VERSION(glxsb, 1);
 MODULE_DEPEND(glxsb, crypto, 1, 1, 1);
 
 static int
 glxsb_probe(device_t dev)
 {
 
 	if (pci_get_vendor(dev) == PCI_VENDOR_AMD &&
 	    pci_get_device(dev) == PCI_PRODUCT_AMD_GEODE_LX_CRYPTO) {
 		device_set_desc(dev,
 		    "AMD Geode LX Security Block (AES-128-CBC, RNG)");
 		return (BUS_PROBE_DEFAULT);
 	}
 
 	return (ENXIO);
 }
 
 static int
 glxsb_attach(device_t dev)
 {
 	struct glxsb_softc *sc = device_get_softc(dev);
 	uint64_t msr;
 
 	sc->sc_dev = dev;
 	msr = rdmsr(SB_GLD_MSR_CAP);
 
 	if ((msr & 0xFFFF00) != 0x130400) {
 		device_printf(dev, "unknown ID 0x%x\n",
 		    (int)((msr & 0xFFFF00) >> 16));
 		return (ENXIO);
 	}
 
 	pci_enable_busmaster(dev);
 
 	/* Map in the security block configuration/control registers */
 	sc->sc_rid = PCIR_BAR(0);
 	sc->sc_sr = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->sc_rid,
 	    RF_ACTIVE);
 	if (sc->sc_sr == NULL) {
 		device_printf(dev, "cannot map register space\n");
 		return (ENXIO);
 	}
 
 	/*
 	 * Configure the Security Block.
 	 *
 	 * We want to enable the noise generator (T_NE), and enable the
 	 * linear feedback shift register and whitener post-processing
 	 * (T_SEL = 3).  Also ensure that test mode (deterministic values)
 	 * is disabled.
 	 */
 	msr = rdmsr(SB_GLD_MSR_CTRL);
 	msr &= ~(SB_GMC_T_TM | SB_GMC_T_SEL_MASK);
 	msr |= SB_GMC_T_NE | SB_GMC_T_SEL3;
 #if 0
 	msr |= SB_GMC_SBI | SB_GMC_SBY;		/* for AES, if necessary */
 #endif
 	wrmsr(SB_GLD_MSR_CTRL, msr);
 
 	/* Disable interrupts */
 	bus_write_4(sc->sc_sr, SB_AES_INT, SB_AI_CLEAR_INTR);
 
 	/* Allocate a contiguous DMA-able buffer to work in */
 	if (glxsb_dma_alloc(sc) != 0)
 		goto fail0;
 
 	/* Initialize our task queue */
 	sc->sc_tq = taskqueue_create("glxsb_taskq", M_NOWAIT | M_ZERO,
 	    taskqueue_thread_enqueue, &sc->sc_tq);
 	if (sc->sc_tq == NULL) {
 		device_printf(dev, "cannot create task queue\n");
 		goto fail0;
 	}
 	if (taskqueue_start_threads(&sc->sc_tq, 1, PI_NET, "%s taskq",
 	    device_get_nameunit(dev)) != 0) {
 		device_printf(dev, "cannot start task queue\n");
 		goto fail1;
 	}
 	TASK_INIT(&sc->sc_cryptotask, 0, glxsb_crypto_task, sc);
 
 	/* Initialize crypto */
 	if (glxsb_crypto_setup(sc) != 0)
 		goto fail1;
 
 	/* Install a periodic collector for the "true" (AMD's word) RNG */
 	if (hz > 100)
 		sc->sc_rnghz = hz / 100;
 	else
 		sc->sc_rnghz = 1;
 	callout_init(&sc->sc_rngco, 1);
 	glxsb_rnd(sc);
 
 	return (0);
 
 fail1:
 	taskqueue_free(sc->sc_tq);
 fail0:
 	bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rid, sc->sc_sr);
 	return (ENXIO);
 }
 
 static int
 glxsb_detach(device_t dev)
 {
 	struct glxsb_softc *sc = device_get_softc(dev);
 
 	crypto_unregister_all(sc->sc_cid);
 
 	callout_drain(&sc->sc_rngco);
 	taskqueue_drain(sc->sc_tq, &sc->sc_cryptotask);
 	bus_generic_detach(dev);
 	glxsb_dma_free(sc, &sc->sc_dma);
 	bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rid, sc->sc_sr);
 	taskqueue_free(sc->sc_tq);
 	mtx_destroy(&sc->sc_task_mtx);
 	return (0);
 }
 
 /*
  *	callback for bus_dmamap_load()
  */
 static void
 glxsb_dmamap_cb(void *arg, bus_dma_segment_t *seg, int nseg, int error)
 {
 
 	bus_addr_t *paddr = (bus_addr_t*) arg;
 	*paddr = seg[0].ds_addr;
 }
 
 static int
 glxsb_dma_alloc(struct glxsb_softc *sc)
 {
 	struct glxsb_dma_map *dma = &sc->sc_dma;
 	int rc;
 
 	dma->dma_nsegs = 1;
 	dma->dma_size = GLXSB_MAX_AES_LEN * 2;
 
 	/* Setup DMA descriptor area */
 	rc = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev),	/* parent */
 				SB_AES_ALIGN, 0,	/* alignments, bounds */
 				BUS_SPACE_MAXADDR_32BIT,/* lowaddr */
 				BUS_SPACE_MAXADDR,	/* highaddr */
 				NULL, NULL,		/* filter, filterarg */
 				dma->dma_size,		/* maxsize */
 				dma->dma_nsegs,		/* nsegments */
 				dma->dma_size,		/* maxsegsize */
 				BUS_DMA_ALLOCNOW,	/* flags */
 				NULL, NULL,		/* lockfunc, lockarg */
 				&sc->sc_dmat);
 	if (rc != 0) {
 		device_printf(sc->sc_dev,
 		    "cannot allocate DMA tag (%d)\n", rc);
 		return (rc);
 	}
 
 	rc = bus_dmamem_alloc(sc->sc_dmat, (void **)&dma->dma_vaddr,
 	    BUS_DMA_NOWAIT, &dma->dma_map);
 	if (rc != 0) {
 		device_printf(sc->sc_dev,
 		    "cannot allocate DMA memory of %d bytes (%d)\n",
 			dma->dma_size, rc);
 		goto fail0;
 	}
 
 	rc = bus_dmamap_load(sc->sc_dmat, dma->dma_map, dma->dma_vaddr,
 	    dma->dma_size, glxsb_dmamap_cb, &dma->dma_paddr, BUS_DMA_NOWAIT);
 	if (rc != 0) {
 		device_printf(sc->sc_dev,
 		    "cannot load DMA memory for %d bytes (%d)\n",
 		   dma->dma_size, rc);
 		goto fail1;
 	}
 
 	return (0);
 
 fail1:
 	bus_dmamem_free(sc->sc_dmat, dma->dma_vaddr, dma->dma_map);
 fail0:
 	bus_dma_tag_destroy(sc->sc_dmat);
 	return (rc);
 }
 
 static void
 glxsb_dma_pre_op(struct glxsb_softc *sc, struct glxsb_dma_map *dma)
 {
 
 	bus_dmamap_sync(sc->sc_dmat, dma->dma_map,
 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 }
 
 static void
 glxsb_dma_post_op(struct glxsb_softc *sc, struct glxsb_dma_map *dma)
 {
 
 	bus_dmamap_sync(sc->sc_dmat, dma->dma_map,
 	    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 }
 
 static void
 glxsb_dma_free(struct glxsb_softc *sc, struct glxsb_dma_map *dma)
 {
 
 	bus_dmamap_unload(sc->sc_dmat, dma->dma_map);
 	bus_dmamem_free(sc->sc_dmat, dma->dma_vaddr, dma->dma_map);
 	bus_dma_tag_destroy(sc->sc_dmat);
 }
 
 static void
 glxsb_rnd(void *v)
 {
 	struct glxsb_softc *sc = v;
 	uint32_t status, value;
 
 	status = bus_read_4(sc->sc_sr, SB_RANDOM_NUM_STATUS);
 	if (status & SB_RNS_TRNG_VALID) {
 		value = bus_read_4(sc->sc_sr, SB_RANDOM_NUM);
 		/* feed with one uint32 */
 		/* MarkM: FIX!! Check that this does not swamp the harvester! */
 		random_harvest_queue(&value, sizeof(value), RANDOM_PURE_GLXSB);
 	}
 
 	callout_reset(&sc->sc_rngco, sc->sc_rnghz, glxsb_rnd, sc);
 }
 
 static int
 glxsb_crypto_setup(struct glxsb_softc *sc)
 {
 
 	sc->sc_cid = crypto_get_driverid(sc->sc_dev,
 	    sizeof(struct glxsb_session), CRYPTOCAP_F_HARDWARE);
 
 	if (sc->sc_cid < 0) {
 		device_printf(sc->sc_dev, "cannot get crypto driver id\n");
 		return (ENOMEM);
 	}
 
 	mtx_init(&sc->sc_task_mtx, "glxsb_crypto_mtx", NULL, MTX_DEF);
 
-	if (crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0) != 0)
-		goto crypto_fail;
-	if (crypto_register(sc->sc_cid, CRYPTO_NULL_HMAC, 0, 0) != 0)
-		goto crypto_fail;
-	if (crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0) != 0)
-		goto crypto_fail;
-	if (crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0) != 0)
-		goto crypto_fail;
-	if (crypto_register(sc->sc_cid, CRYPTO_RIPEMD160_HMAC, 0, 0) != 0)
-		goto crypto_fail;
-	if (crypto_register(sc->sc_cid, CRYPTO_SHA2_256_HMAC, 0, 0) != 0)
-		goto crypto_fail;
-	if (crypto_register(sc->sc_cid, CRYPTO_SHA2_384_HMAC, 0, 0) != 0)
-		goto crypto_fail;
-	if (crypto_register(sc->sc_cid, CRYPTO_SHA2_512_HMAC, 0, 0) != 0)
-		goto crypto_fail;
-
 	return (0);
-
-crypto_fail:
-	device_printf(sc->sc_dev, "cannot register crypto\n");
-	crypto_unregister_all(sc->sc_cid);
-	mtx_destroy(&sc->sc_task_mtx);
-	return (ENOMEM);
 }
 
 static int
-glxsb_crypto_newsession(device_t dev, crypto_session_t cses,
-    struct cryptoini *cri)
+glxsb_crypto_probesession(device_t dev, const struct crypto_session_params *csp)
 {
-	struct glxsb_softc *sc = device_get_softc(dev);
-	struct glxsb_session *ses;
-	struct cryptoini *encini, *macini;
-	int error;
 
-	if (sc == NULL || cri == NULL)
+	if (csp->csp_flags != 0)
 		return (EINVAL);
 
-	encini = macini = NULL;
-	for (; cri != NULL; cri = cri->cri_next) {
-		switch(cri->cri_alg) {
+	/*
+	 * We only support HMAC algorithms to be able to work with
+	 * ipsec(4), so if we are asked only for authentication without
+	 * encryption, don't pretend we can accelerate it.
+	 */
+	switch (csp->csp_mode) {
+	case CSP_MODE_ETA:
+		switch (csp->csp_auth_alg) {
 		case CRYPTO_NULL_HMAC:
 		case CRYPTO_MD5_HMAC:
 		case CRYPTO_SHA1_HMAC:
 		case CRYPTO_RIPEMD160_HMAC:
 		case CRYPTO_SHA2_256_HMAC:
 		case CRYPTO_SHA2_384_HMAC:
 		case CRYPTO_SHA2_512_HMAC:
-			if (macini != NULL)
-				return (EINVAL);
-			macini = cri;
 			break;
+		default:
+			return (EINVAL);
+		}
+		/* FALLTHROUGH */
+	case CSP_MODE_CIPHER:
+		switch (csp->csp_cipher_alg) {
 		case CRYPTO_AES_CBC:
-			if (encini != NULL)
+			if (csp->csp_cipher_klen * 8 != 128)
 				return (EINVAL);
-			encini = cri;
 			break;
 		default:
 			return (EINVAL);
 		}
+	default:
+		return (EINVAL);
 	}
+	return (CRYPTODEV_PROBE_HARDWARE);
+}
 
-	/*
-	 * We only support HMAC algorithms to be able to work with
-	 * ipsec(4), so if we are asked only for authentication without
-	 * encryption, don't pretend we can accellerate it.
-	 */
-	if (encini == NULL)
-		return (EINVAL);
+static int
+glxsb_crypto_newsession(device_t dev, crypto_session_t cses,
+    const struct crypto_session_params *csp)
+{
+	struct glxsb_softc *sc = device_get_softc(dev);
+	struct glxsb_session *ses;
+	int error;
 
 	ses = crypto_get_driver_session(cses);
-	if (encini->cri_alg == CRYPTO_AES_CBC) {
-		if (encini->cri_klen != 128) {
-			glxsb_crypto_freesession(sc->sc_dev, cses);
-			return (EINVAL);
-		}
-		arc4rand(ses->ses_iv, sizeof(ses->ses_iv), 0);
-		ses->ses_klen = encini->cri_klen;
 
-		/* Copy the key (Geode LX wants the primary key only) */
-		bcopy(encini->cri_key, ses->ses_key, sizeof(ses->ses_key));
-	}
+	/* Copy the key (Geode LX wants the primary key only) */
+	if (csp->csp_cipher_key != NULL)
+		bcopy(csp->csp_cipher_key, ses->ses_key, sizeof(ses->ses_key));
 
-	if (macini != NULL) {
-		error = glxsb_hash_setup(ses, macini);
+	if (csp->csp_auth_alg != 0) {
+		error = glxsb_hash_setup(ses, csp);
 		if (error != 0) {
 			glxsb_crypto_freesession(sc->sc_dev, cses);
 			return (error);
 		}
 	}
 
 	return (0);
 }
 
 static void
 glxsb_crypto_freesession(device_t dev, crypto_session_t cses)
 {
-	struct glxsb_softc *sc = device_get_softc(dev);
 	struct glxsb_session *ses;
 
-	if (sc == NULL)
-		return;
-
 	ses = crypto_get_driver_session(cses);
 	glxsb_hash_free(ses);
 }
 
 static int
 glxsb_aes(struct glxsb_softc *sc, uint32_t control, uint32_t psrc,
-    uint32_t pdst, void *key, int len, void *iv)
+    uint32_t pdst, const void *key, int len, const void *iv)
 {
 	uint32_t status;
 	int i;
 
 	if (len & 0xF) {
 		device_printf(sc->sc_dev,
 		    "len must be a multiple of 16 (not %d)\n", len);
 		return (EINVAL);
 	}
 
 	/* Set the source */
 	bus_write_4(sc->sc_sr, SB_SOURCE_A, psrc);
 
 	/* Set the destination address */
 	bus_write_4(sc->sc_sr, SB_DEST_A, pdst);
 
 	/* Set the data length */
 	bus_write_4(sc->sc_sr, SB_LENGTH_A, len);
 
 	/* Set the IV */
 	if (iv != NULL) {
 		bus_write_region_4(sc->sc_sr, SB_CBC_IV, iv, 4);
 		control |= SB_CTL_CBC;
 	}
 
 	/* Set the key */
 	bus_write_region_4(sc->sc_sr, SB_WKEY, key, 4);
 
 	/* Ask the security block to do it */
 	bus_write_4(sc->sc_sr, SB_CTL_A,
 	    control | SB_CTL_WK | SB_CTL_DC | SB_CTL_SC | SB_CTL_ST);
 
 	/*
 	 * Now wait until it is done.
 	 *
 	 * We do a busy wait.  Obviously the number of iterations of
 	 * the loop required to perform the AES operation depends upon
 	 * the number of bytes to process.
 	 *
 	 * On a 500 MHz Geode LX we see
 	 *
 	 *	length (bytes)	typical max iterations
 	 *	    16		   12
 	 *	    64		   22
 	 *	   256		   59
 	 *	  1024		  212
 	 *	  8192		1,537
 	 *
 	 * Since we have a maximum size of operation defined in
 	 * GLXSB_MAX_AES_LEN, we use this constant to decide how long
 	 * to wait.  Allow an order of magnitude longer than it should
 	 * really take, just in case.
 	 */
 
 	for (i = 0; i < GLXSB_MAX_AES_LEN * 10; i++) {
 		status = bus_read_4(sc->sc_sr, SB_CTL_A);
 		if ((status & SB_CTL_ST) == 0)		/* Done */
 			return (0);
 	}
 
 	device_printf(sc->sc_dev, "operation failed to complete\n");
 	return (EIO);
 }
 
 static int
-glxsb_crypto_encdec(struct cryptop *crp, struct cryptodesc *crd,
-    struct glxsb_session *ses, struct glxsb_softc *sc)
+glxsb_crypto_encdec(struct cryptop *crp, struct glxsb_session *ses,
+    struct glxsb_softc *sc)
 {
 	char *op_src, *op_dst;
+	const void *key;
 	uint32_t op_psrc, op_pdst;
-	uint8_t op_iv[SB_AES_BLOCK_SIZE], *piv;
+	uint8_t op_iv[SB_AES_BLOCK_SIZE];
 	int error;
 	int len, tlen, xlen;
 	int offset;
 	uint32_t control;
 
-	if (crd == NULL || (crd->crd_len % SB_AES_BLOCK_SIZE) != 0)
+	if ((crp->crp_payload_length % SB_AES_BLOCK_SIZE) != 0)
 		return (EINVAL);
 
 	/* How much of our buffer will we need to use? */
-	xlen = crd->crd_len > GLXSB_MAX_AES_LEN ?
-	    GLXSB_MAX_AES_LEN : crd->crd_len;
+	xlen = crp->crp_payload_length > GLXSB_MAX_AES_LEN ?
+	    GLXSB_MAX_AES_LEN : crp->crp_payload_length;
 
 	/*
 	 * XXX Check if we can have input == output on Geode LX.
 	 * XXX In the meantime, use two separate (adjacent) buffers.
 	 */
 	op_src = sc->sc_dma.dma_vaddr;
 	op_dst = (char *)sc->sc_dma.dma_vaddr + xlen;
 
 	op_psrc = sc->sc_dma.dma_paddr;
 	op_pdst = sc->sc_dma.dma_paddr + xlen;
 
-	if (crd->crd_flags & CRD_F_ENCRYPT) {
+	if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
 		control = SB_CTL_ENC;
-		if (crd->crd_flags & CRD_F_IV_EXPLICIT)
-			bcopy(crd->crd_iv, op_iv, sizeof(op_iv));
-		else
-			bcopy(ses->ses_iv, op_iv, sizeof(op_iv));
-
-		if ((crd->crd_flags & CRD_F_IV_PRESENT) == 0) {
-			crypto_copyback(crp->crp_flags, crp->crp_buf,
-			    crd->crd_inject, sizeof(op_iv), op_iv);
-		}
-	} else {
+	else
 		control = SB_CTL_DEC;
-		if (crd->crd_flags & CRD_F_IV_EXPLICIT)
-			bcopy(crd->crd_iv, op_iv, sizeof(op_iv));
-		else {
-			crypto_copydata(crp->crp_flags, crp->crp_buf,
-			    crd->crd_inject, sizeof(op_iv), op_iv);
-		}
-	}
 
+	if (crp->crp_flags & CRYPTO_F_IV_GENERATE) {
+		arc4rand(op_iv, sizeof(op_iv), 0);
+		crypto_copyback(crp, crp->crp_iv_start, sizeof(op_iv), op_iv);
+	} else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+		memcpy(op_iv, crp->crp_iv, sizeof(op_iv));
+	else
+		crypto_copydata(crp, crp->crp_iv_start, sizeof(op_iv), op_iv);
+	
 	offset = 0;
-	tlen = crd->crd_len;
-	piv = op_iv;
+	tlen = crp->crp_payload_length;
 
+	if (crp->crp_cipher_key != NULL)
+		key = crp->crp_cipher_key;
+	else
+		key = ses->ses_key;
+
 	/* Process the data in GLXSB_MAX_AES_LEN chunks */
 	while (tlen > 0) {
 		len = (tlen > GLXSB_MAX_AES_LEN) ? GLXSB_MAX_AES_LEN : tlen;
-		crypto_copydata(crp->crp_flags, crp->crp_buf,
-		    crd->crd_skip + offset, len, op_src);
+		crypto_copydata(crp, crp->crp_payload_start + offset, len,
+		    op_src);
 
 		glxsb_dma_pre_op(sc, &sc->sc_dma);
 
-		error = glxsb_aes(sc, control, op_psrc, op_pdst, ses->ses_key,
-		    len, op_iv);
+		error = glxsb_aes(sc, control, op_psrc, op_pdst, key, len,
+		    op_iv);
 
 		glxsb_dma_post_op(sc, &sc->sc_dma);
 		if (error != 0)
 			return (error);
 
-		crypto_copyback(crp->crp_flags, crp->crp_buf,
-		    crd->crd_skip + offset, len, op_dst);
+		crypto_copyback(crp, crp->crp_payload_start + offset, len,
+		    op_dst);
 
 		offset += len;
 		tlen -= len;
 
-		if (tlen <= 0) {	/* Ideally, just == 0 */
-			/* Finished - put the IV in session IV */
-			piv = ses->ses_iv;
-		}
-
 		/*
-		 * Copy out last block for use as next iteration/session IV.
-		 *
-		 * piv is set to op_iv[] before the loop starts, but is
-		 * set to ses->ses_iv if we're going to exit the loop this
-		 * time.
+		 * Copy out last block for use as next iteration IV.
 		 */
-		if (crd->crd_flags & CRD_F_ENCRYPT)
-			bcopy(op_dst + len - sizeof(op_iv), piv, sizeof(op_iv));
-		else {
-			/* Decryption, only need this if another iteration */
-			if (tlen > 0) {
-				bcopy(op_src + len - sizeof(op_iv), piv,
-				    sizeof(op_iv));
-			}
-		}
+		if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
+			bcopy(op_dst + len - sizeof(op_iv), op_iv,
+			    sizeof(op_iv));
+		else
+			bcopy(op_src + len - sizeof(op_iv), op_iv,
+			    sizeof(op_iv));
 	} /* while */
 
 	/* All AES processing has now been done. */
 	bzero(sc->sc_dma.dma_vaddr, xlen * 2);
 
 	return (0);
 }
 
 static void
 glxsb_crypto_task(void *arg, int pending)
 {
 	struct glxsb_softc *sc = arg;
+	const struct crypto_session_params *csp;
 	struct glxsb_session *ses;
 	struct cryptop *crp;
-	struct cryptodesc *enccrd, *maccrd;
 	int error;
 
-	maccrd = sc->sc_to.to_maccrd;
-	enccrd = sc->sc_to.to_enccrd;
 	crp = sc->sc_to.to_crp;
 	ses = sc->sc_to.to_ses;
+	csp = crypto_get_params(crp->crp_session);
 
 	/* Perform data authentication if requested before encryption */
-	if (maccrd != NULL && maccrd->crd_next == enccrd) {
-		error = glxsb_hash_process(ses, maccrd, crp);
+	if (csp->csp_mode == CSP_MODE_ETA &&
+	    !CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
+		error = glxsb_hash_process(ses, csp, crp);
 		if (error != 0)
 			goto out;
 	}
 
-	error = glxsb_crypto_encdec(crp, enccrd, ses, sc);
+	error = glxsb_crypto_encdec(crp, ses, sc);
 	if (error != 0)
 		goto out;
 
 	/* Perform data authentication if requested after encryption */
-	if (maccrd != NULL && enccrd->crd_next == maccrd) {
-		error = glxsb_hash_process(ses, maccrd, crp);
+	if (csp->csp_mode == CSP_MODE_ETA &&
+	    CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
+		error = glxsb_hash_process(ses, csp, crp);
 		if (error != 0)
 			goto out;
 	}
 out:
 	mtx_lock(&sc->sc_task_mtx);
 	sc->sc_task_count--;
 	mtx_unlock(&sc->sc_task_mtx);
 
 	crp->crp_etype = error;
 	crypto_unblock(sc->sc_cid, CRYPTO_SYMQ);
 	crypto_done(crp);
 }
 
 static int
 glxsb_crypto_process(device_t dev, struct cryptop *crp, int hint)
 {
 	struct glxsb_softc *sc = device_get_softc(dev);
 	struct glxsb_session *ses;
-	struct cryptodesc *crd, *enccrd, *maccrd;
-	int error = 0;
 
-	enccrd = maccrd = NULL;
-
-	/* Sanity check. */
-	if (crp == NULL)
-		return (EINVAL);
-
-	if (crp->crp_callback == NULL || crp->crp_desc == NULL) {
-		error = EINVAL;
-		goto fail;
-	}
-
-	for (crd = crp->crp_desc; crd != NULL; crd = crd->crd_next) {
-		switch (crd->crd_alg) {
-		case CRYPTO_NULL_HMAC:
-		case CRYPTO_MD5_HMAC:
-		case CRYPTO_SHA1_HMAC:
-		case CRYPTO_RIPEMD160_HMAC:
-		case CRYPTO_SHA2_256_HMAC:
-		case CRYPTO_SHA2_384_HMAC:
-		case CRYPTO_SHA2_512_HMAC:
-			if (maccrd != NULL) {
-				error = EINVAL;
-				goto fail;
-			}
-			maccrd = crd;
-			break;
-		case CRYPTO_AES_CBC:
-			if (enccrd != NULL) {
-				error = EINVAL;
-				goto fail;
-			}
-			enccrd = crd;
-			break;
-		default:
-			error = EINVAL;
-			goto fail;
-		}
-	}
-
-	if (enccrd == NULL || enccrd->crd_len % AES_BLOCK_LEN != 0) {
-		error = EINVAL;
-		goto fail;
-	}
-
 	ses = crypto_get_driver_session(crp->crp_session);
 
 	mtx_lock(&sc->sc_task_mtx);
 	if (sc->sc_task_count != 0) {
 		mtx_unlock(&sc->sc_task_mtx);
 		return (ERESTART);
 	}
 	sc->sc_task_count++;
 
-	sc->sc_to.to_maccrd = maccrd;
-	sc->sc_to.to_enccrd = enccrd;
 	sc->sc_to.to_crp = crp;
 	sc->sc_to.to_ses = ses;
 	mtx_unlock(&sc->sc_task_mtx);
 
 	taskqueue_enqueue(sc->sc_tq, &sc->sc_cryptotask);
 	return(0);
-
-fail:
-	crp->crp_etype = error;
-	crypto_done(crp);
-	return (error);
 }
Index: head/sys/dev/glxsb/glxsb.h
===================================================================
--- head/sys/dev/glxsb/glxsb.h	(revision 359373)
+++ head/sys/dev/glxsb/glxsb.h	(revision 359374)
@@ -1,56 +1,54 @@
 /*-
  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
  *
  * Copyright (c) 2005-2006 Pawel Jakub Dawidek <pjd@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
  * $FreeBSD$
  */
 
 #ifndef _GLXSB_H_
 #define _GLXSB_H_
 
 #include <opencrypto/cryptodev.h>
 
 #define SB_AES_BLOCK_SIZE       0x0010
 
 struct glxsb_session {
 	uint32_t	ses_key[4];		/* key */
-	uint8_t		ses_iv[SB_AES_BLOCK_SIZE]; /* initialization vector */
-	int		ses_klen;		/* key len */
 	struct auth_hash *ses_axf;
 	uint8_t		*ses_ictx;
 	uint8_t		*ses_octx;
 	int		ses_mlen;
 };
 
 int glxsb_hash_setup(struct glxsb_session *ses,
-	    struct cryptoini *macini);
+    const struct crypto_session_params *csp);
 
 int glxsb_hash_process(struct glxsb_session *ses,
-	    struct cryptodesc *maccrd, struct cryptop *crp);
+    const struct crypto_session_params *csp, struct cryptop *crp);
 
 void glxsb_hash_free(struct glxsb_session *ses);
 
 #endif	/* !_GLXSB_H_ */
Index: head/sys/dev/glxsb/glxsb_hash.c
===================================================================
--- head/sys/dev/glxsb/glxsb_hash.c	(revision 359373)
+++ head/sys/dev/glxsb/glxsb_hash.c	(revision 359374)
@@ -1,184 +1,158 @@
 /*-
  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
  *
  * Copyright (c) 2006 Pawel Jakub Dawidek <pjd@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/malloc.h>
 
-#include <opencrypto/cryptosoft.h> /* for hmac_ipad_buffer and hmac_opad_buffer */
 #include <opencrypto/xform.h>
 
 #include "glxsb.h"
 
 /*
  * Implementation notes.
  *
  * The Geode LX Security Block provides AES-128-CBC acceleration.
  * We implement all HMAC algorithms provided by crypto(9) framework so glxsb can work
  * with ipsec(4)
  *
  * This code was stolen from crypto/via/padlock_hash.c
  */
 
 MALLOC_DECLARE(M_GLXSB);
 
 static void
-glxsb_hash_key_setup(struct glxsb_session *ses, caddr_t key, int klen)
+glxsb_hash_key_setup(struct glxsb_session *ses, const char *key, int klen)
 {
 	struct auth_hash *axf;
-	int i;
 
-	klen /= 8;
 	axf = ses->ses_axf;
-
-	for (i = 0; i < klen; i++)
-		key[i] ^= HMAC_IPAD_VAL;
-
-	axf->Init(ses->ses_ictx);
-	axf->Update(ses->ses_ictx, key, klen);
-	axf->Update(ses->ses_ictx, hmac_ipad_buffer, axf->blocksize - klen);
-
-	for (i = 0; i < klen; i++)
-		key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
-
-	axf->Init(ses->ses_octx);
-	axf->Update(ses->ses_octx, key, klen);
-	axf->Update(ses->ses_octx, hmac_opad_buffer, axf->blocksize - klen);
-
-	for (i = 0; i < klen; i++)
-		key[i] ^= HMAC_OPAD_VAL;
+	hmac_init_ipad(axf, key, klen, ses->ses_ictx);
+	hmac_init_opad(axf, key, klen, ses->ses_octx);
 }
 
 /*
  * Compute keyed-hash authenticator.
  */
 static int
-glxsb_authcompute(struct glxsb_session *ses, struct cryptodesc *crd,
-    caddr_t buf, int flags)
+glxsb_authcompute(struct glxsb_session *ses, struct cryptop *crp)
 {
-	u_char hash[HASH_MAX_LEN];
+	u_char hash[HASH_MAX_LEN], hash2[HASH_MAX_LEN];
 	struct auth_hash *axf;
 	union authctx ctx;
 	int error;
 
 	axf = ses->ses_axf;
 	bcopy(ses->ses_ictx, &ctx, axf->ctxsize);
-	error = crypto_apply(flags, buf, crd->crd_skip, crd->crd_len,
+	error = crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length,
 	    (int (*)(void *, void *, unsigned int))axf->Update, (caddr_t)&ctx);
 	if (error != 0)
 		return (error);
+	error = crypto_apply(crp, crp->crp_payload_start,
+	    crp->crp_payload_length, 
+	    (int (*)(void *, void *, unsigned int))axf->Update, (caddr_t)&ctx);
+	if (error != 0)
+		return (error);
+	
 	axf->Final(hash, &ctx);
 
 	bcopy(ses->ses_octx, &ctx, axf->ctxsize);
 	axf->Update(&ctx, hash, axf->hashsize);
 	axf->Final(hash, &ctx);
 
-	/* Inject the authentication data */
-	crypto_copyback(flags, buf, crd->crd_inject,
-	ses->ses_mlen == 0 ? axf->hashsize : ses->ses_mlen, hash);
+	/* Verify or inject the authentication data */
+	if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
+		crypto_copydata(crp, crp->crp_digest_start, ses->ses_mlen,
+		    hash2);
+		if (timingsafe_bcmp(hash, hash2, ses->ses_mlen) != 0)
+			return (EBADMSG);
+	} else
+		crypto_copyback(crp, crp->crp_digest_start, ses->ses_mlen,
+		    hash);
 	return (0);
 }
 
 int
-glxsb_hash_setup(struct glxsb_session *ses, struct cryptoini *macini)
+glxsb_hash_setup(struct glxsb_session *ses,
+    const struct crypto_session_params *csp)
 {
 
-	ses->ses_mlen = macini->cri_mlen;
+	ses->ses_axf = crypto_auth_hash(csp);
+	if (csp->csp_auth_mlen == 0)
+		ses->ses_mlen = ses->ses_axf->hashsize;
+	else
+		ses->ses_mlen = csp->csp_auth_mlen;
 
-	/* Find software structure which describes HMAC algorithm. */
-	switch (macini->cri_alg) {
-	case CRYPTO_NULL_HMAC:
-		ses->ses_axf = &auth_hash_null;
-		break;
-	case CRYPTO_MD5_HMAC:
-		ses->ses_axf = &auth_hash_hmac_md5;
-		break;
-	case CRYPTO_SHA1_HMAC:
-		ses->ses_axf = &auth_hash_hmac_sha1;
-		break;
-	case CRYPTO_RIPEMD160_HMAC:
-		ses->ses_axf = &auth_hash_hmac_ripemd_160;
-		break;
-	case CRYPTO_SHA2_256_HMAC:
-		ses->ses_axf = &auth_hash_hmac_sha2_256;
-		break;
-	case CRYPTO_SHA2_384_HMAC:
-		ses->ses_axf = &auth_hash_hmac_sha2_384;
-		break;
-	case CRYPTO_SHA2_512_HMAC:
-		ses->ses_axf = &auth_hash_hmac_sha2_512;
-		break;
-	}
-
 	/* Allocate memory for HMAC inner and outer contexts. */
 	ses->ses_ictx = malloc(ses->ses_axf->ctxsize, M_GLXSB,
 	    M_ZERO | M_NOWAIT);
 	ses->ses_octx = malloc(ses->ses_axf->ctxsize, M_GLXSB,
 	    M_ZERO | M_NOWAIT);
 	if (ses->ses_ictx == NULL || ses->ses_octx == NULL)
 		return (ENOMEM);
 
 	/* Setup key if given. */
-	if (macini->cri_key != NULL) {
-		glxsb_hash_key_setup(ses, macini->cri_key,
-		    macini->cri_klen);
+	if (csp->csp_auth_key != NULL) {
+		glxsb_hash_key_setup(ses, csp->csp_auth_key,
+		    csp->csp_auth_klen);
 	}
 	return (0);
 }
 
 int
-glxsb_hash_process(struct glxsb_session *ses, struct cryptodesc *maccrd,
-    struct cryptop *crp)
+glxsb_hash_process(struct glxsb_session *ses,
+    const struct crypto_session_params *csp, struct cryptop *crp)
 {
 	int error;
 
-	if ((maccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0)
-		glxsb_hash_key_setup(ses, maccrd->crd_key, maccrd->crd_klen);
+	if (crp->crp_auth_key != NULL)
+		glxsb_hash_key_setup(ses, crp->crp_auth_key,
+		    csp->csp_auth_klen);
 
-	error = glxsb_authcompute(ses, maccrd, crp->crp_buf, crp->crp_flags);
+	error = glxsb_authcompute(ses, crp);
 	return (error);
 }
 
 void
 glxsb_hash_free(struct glxsb_session *ses)
 {
 
 	if (ses->ses_ictx != NULL) {
 		bzero(ses->ses_ictx, ses->ses_axf->ctxsize);
 		free(ses->ses_ictx, M_GLXSB);
 		ses->ses_ictx = NULL;
 	}
 	if (ses->ses_octx != NULL) {
 		bzero(ses->ses_octx, ses->ses_axf->ctxsize);
 		free(ses->ses_octx, M_GLXSB);
 		ses->ses_octx = NULL;
 	}
 }
Index: head/sys/dev/hifn/hifn7751.c
===================================================================
--- head/sys/dev/hifn/hifn7751.c	(revision 359373)
+++ head/sys/dev/hifn/hifn7751.c	(revision 359374)
@@ -1,2863 +1,2825 @@
 /*	$OpenBSD: hifn7751.c,v 1.120 2002/05/17 00:33:34 deraadt Exp $	*/
 
 /*-
  * SPDX-License-Identifier: BSD-3-Clause
  *
  * Invertex AEON / Hifn 7751 driver
  * Copyright (c) 1999 Invertex Inc. All rights reserved.
  * Copyright (c) 1999 Theo de Raadt
  * Copyright (c) 2000-2001 Network Security Technologies, Inc.
  *			http://www.netsec.net
  * Copyright (c) 2003 Hifn Inc.
  *
  * This driver is based on a previous driver by Invertex, for which they
  * requested:  Please send any comments, feedback, bug-fixes, or feature
  * requests to software@invertex.com.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  *
  * 1. Redistributions of source code must retain the above copyright
  *   notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *   notice, this list of conditions and the following disclaimer in the
  *   documentation and/or other materials provided with the distribution.
  * 3. The name of the author may not be used to endorse or promote products
  *   derived from this software without specific prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  * Effort sponsored in part by the Defense Advanced Research Projects
  * Agency (DARPA) and Air Force Research Laboratory, Air Force
  * Materiel Command, USAF, under agreement number F30602-01-2-0537.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 /*
  * Driver for various Hifn encryption processors.
  */
 #include "opt_hifn.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/proc.h>
 #include <sys/errno.h>
 #include <sys/malloc.h>
 #include <sys/kernel.h>
 #include <sys/module.h>
 #include <sys/mbuf.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <sys/sysctl.h>
+#include <sys/uio.h>
 
 #include <vm/vm.h>
 #include <vm/pmap.h>
 
 #include <machine/bus.h>
 #include <machine/resource.h>
 #include <sys/bus.h>
 #include <sys/rman.h>
 
 #include <opencrypto/cryptodev.h>
+#include <opencrypto/xform_auth.h>
 #include <sys/random.h>
 #include <sys/kobj.h>
 
 #include "cryptodev_if.h"
 
 #include <dev/pci/pcivar.h>
 #include <dev/pci/pcireg.h>
 
 #ifdef HIFN_RNDTEST
 #include <dev/rndtest/rndtest.h>
 #endif
 #include <dev/hifn/hifn7751reg.h>
 #include <dev/hifn/hifn7751var.h>
 
 #ifdef HIFN_VULCANDEV
 #include <sys/conf.h>
 #include <sys/uio.h>
 
 static struct cdevsw vulcanpk_cdevsw; /* forward declaration */
 #endif
 
 /*
  * Prototypes and count for the pci_device structure
  */
 static	int hifn_probe(device_t);
 static	int hifn_attach(device_t);
 static	int hifn_detach(device_t);
 static	int hifn_suspend(device_t);
 static	int hifn_resume(device_t);
 static	int hifn_shutdown(device_t);
 
-static	int hifn_newsession(device_t, crypto_session_t, struct cryptoini *);
+static	int hifn_probesession(device_t, const struct crypto_session_params *);
+static	int hifn_newsession(device_t, crypto_session_t,
+    const struct crypto_session_params *);
 static	int hifn_process(device_t, struct cryptop *, int);
 
 static device_method_t hifn_methods[] = {
 	/* Device interface */
 	DEVMETHOD(device_probe,		hifn_probe),
 	DEVMETHOD(device_attach,	hifn_attach),
 	DEVMETHOD(device_detach,	hifn_detach),
 	DEVMETHOD(device_suspend,	hifn_suspend),
 	DEVMETHOD(device_resume,	hifn_resume),
 	DEVMETHOD(device_shutdown,	hifn_shutdown),
 
 	/* crypto device methods */
+	DEVMETHOD(cryptodev_probesession, hifn_probesession),
 	DEVMETHOD(cryptodev_newsession,	hifn_newsession),
 	DEVMETHOD(cryptodev_process,	hifn_process),
 
 	DEVMETHOD_END
 };
 static driver_t hifn_driver = {
 	"hifn",
 	hifn_methods,
 	sizeof (struct hifn_softc)
 };
 static devclass_t hifn_devclass;
 
 DRIVER_MODULE(hifn, pci, hifn_driver, hifn_devclass, 0, 0);
 MODULE_DEPEND(hifn, crypto, 1, 1, 1);
 #ifdef HIFN_RNDTEST
 MODULE_DEPEND(hifn, rndtest, 1, 1, 1);
 #endif
 
 static	void hifn_reset_board(struct hifn_softc *, int);
 static	void hifn_reset_puc(struct hifn_softc *);
 static	void hifn_puc_wait(struct hifn_softc *);
 static	int hifn_enable_crypto(struct hifn_softc *);
 static	void hifn_set_retry(struct hifn_softc *sc);
 static	void hifn_init_dma(struct hifn_softc *);
 static	void hifn_init_pci_registers(struct hifn_softc *);
 static	int hifn_sramsize(struct hifn_softc *);
 static	int hifn_dramsize(struct hifn_softc *);
 static	int hifn_ramtype(struct hifn_softc *);
 static	void hifn_sessions(struct hifn_softc *);
 static	void hifn_intr(void *);
 static	u_int hifn_write_command(struct hifn_command *, u_int8_t *);
 static	u_int32_t hifn_next_signature(u_int32_t a, u_int cnt);
 static	void hifn_callback(struct hifn_softc *, struct hifn_command *, u_int8_t *);
 static	int hifn_crypto(struct hifn_softc *, struct hifn_command *, struct cryptop *, int);
 static	int hifn_readramaddr(struct hifn_softc *, int, u_int8_t *);
 static	int hifn_writeramaddr(struct hifn_softc *, int, u_int8_t *);
 static	int hifn_dmamap_load_src(struct hifn_softc *, struct hifn_command *);
 static	int hifn_dmamap_load_dst(struct hifn_softc *, struct hifn_command *);
 static	int hifn_init_pubrng(struct hifn_softc *);
 static	void hifn_rng(void *);
 static	void hifn_tick(void *);
 static	void hifn_abort(struct hifn_softc *);
 static	void hifn_alloc_slot(struct hifn_softc *, int *, int *, int *, int *);
 
 static	void hifn_write_reg_0(struct hifn_softc *, bus_size_t, u_int32_t);
 static	void hifn_write_reg_1(struct hifn_softc *, bus_size_t, u_int32_t);
 
 static __inline u_int32_t
 READ_REG_0(struct hifn_softc *sc, bus_size_t reg)
 {
     u_int32_t v = bus_space_read_4(sc->sc_st0, sc->sc_sh0, reg);
     sc->sc_bar0_lastreg = (bus_size_t) -1;
     return (v);
 }
 #define	WRITE_REG_0(sc, reg, val)	hifn_write_reg_0(sc, reg, val)
 
 static __inline u_int32_t
 READ_REG_1(struct hifn_softc *sc, bus_size_t reg)
 {
     u_int32_t v = bus_space_read_4(sc->sc_st1, sc->sc_sh1, reg);
     sc->sc_bar1_lastreg = (bus_size_t) -1;
     return (v);
 }
 #define	WRITE_REG_1(sc, reg, val)	hifn_write_reg_1(sc, reg, val)
 
 static SYSCTL_NODE(_hw, OID_AUTO, hifn, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
     "Hifn driver parameters");
 
 #ifdef HIFN_DEBUG
 static	int hifn_debug = 0;
 SYSCTL_INT(_hw_hifn, OID_AUTO, debug, CTLFLAG_RW, &hifn_debug,
 	    0, "control debugging msgs");
 #endif
 
 static	struct hifn_stats hifnstats;
 SYSCTL_STRUCT(_hw_hifn, OID_AUTO, stats, CTLFLAG_RD, &hifnstats,
 	    hifn_stats, "driver statistics");
 static	int hifn_maxbatch = 1;
 SYSCTL_INT(_hw_hifn, OID_AUTO, maxbatch, CTLFLAG_RW, &hifn_maxbatch,
 	    0, "max ops to batch w/o interrupt");
 
 /*
  * Probe for a supported device.  The PCI vendor and device
  * IDs are used to detect devices we know how to handle.
  */
 static int
 hifn_probe(device_t dev)
 {
 	if (pci_get_vendor(dev) == PCI_VENDOR_INVERTEX &&
 	    pci_get_device(dev) == PCI_PRODUCT_INVERTEX_AEON)
 		return (BUS_PROBE_DEFAULT);
 	if (pci_get_vendor(dev) == PCI_VENDOR_HIFN &&
 	    (pci_get_device(dev) == PCI_PRODUCT_HIFN_7751 ||
 	     pci_get_device(dev) == PCI_PRODUCT_HIFN_7951 ||
 	     pci_get_device(dev) == PCI_PRODUCT_HIFN_7955 ||
 	     pci_get_device(dev) == PCI_PRODUCT_HIFN_7956 ||
 	     pci_get_device(dev) == PCI_PRODUCT_HIFN_7811))
 		return (BUS_PROBE_DEFAULT);
 	if (pci_get_vendor(dev) == PCI_VENDOR_NETSEC &&
 	    pci_get_device(dev) == PCI_PRODUCT_NETSEC_7751)
 		return (BUS_PROBE_DEFAULT);
 	return (ENXIO);
 }
 
 static void
 hifn_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 {
 	bus_addr_t *paddr = (bus_addr_t*) arg;
 	*paddr = segs->ds_addr;
 }
 
 static const char*
 hifn_partname(struct hifn_softc *sc)
 {
 	/* XXX sprintf numbers when not decoded */
 	switch (pci_get_vendor(sc->sc_dev)) {
 	case PCI_VENDOR_HIFN:
 		switch (pci_get_device(sc->sc_dev)) {
 		case PCI_PRODUCT_HIFN_6500:	return "Hifn 6500";
 		case PCI_PRODUCT_HIFN_7751:	return "Hifn 7751";
 		case PCI_PRODUCT_HIFN_7811:	return "Hifn 7811";
 		case PCI_PRODUCT_HIFN_7951:	return "Hifn 7951";
 		case PCI_PRODUCT_HIFN_7955:	return "Hifn 7955";
 		case PCI_PRODUCT_HIFN_7956:	return "Hifn 7956";
 		}
 		return "Hifn unknown-part";
 	case PCI_VENDOR_INVERTEX:
 		switch (pci_get_device(sc->sc_dev)) {
 		case PCI_PRODUCT_INVERTEX_AEON:	return "Invertex AEON";
 		}
 		return "Invertex unknown-part";
 	case PCI_VENDOR_NETSEC:
 		switch (pci_get_device(sc->sc_dev)) {
 		case PCI_PRODUCT_NETSEC_7751:	return "NetSec 7751";
 		}
 		return "NetSec unknown-part";
 	}
 	return "Unknown-vendor unknown-part";
 }
 
 static void
 default_harvest(struct rndtest_state *rsp, void *buf, u_int count)
 {
 	/* MarkM: FIX!! Check that this does not swamp the harvester! */
 	random_harvest_queue(buf, count, RANDOM_PURE_HIFN);
 }
 
 static u_int
 checkmaxmin(device_t dev, const char *what, u_int v, u_int min, u_int max)
 {
 	if (v > max) {
 		device_printf(dev, "Warning, %s %u out of range, "
 			"using max %u\n", what, v, max);
 		v = max;
 	} else if (v < min) {
 		device_printf(dev, "Warning, %s %u out of range, "
 			"using min %u\n", what, v, min);
 		v = min;
 	}
 	return v;
 }
 
 /*
  * Select PLL configuration for 795x parts.  This is complicated in
  * that we cannot determine the optimal parameters without user input.
  * The reference clock is derived from an external clock through a
  * multiplier.  The external clock is either the host bus (i.e. PCI)
  * or an external clock generator.  When using the PCI bus we assume
  * the clock is either 33 or 66 MHz; for an external source we cannot
  * tell the speed.
  *
  * PLL configuration is done with a string: "pci" for PCI bus, or "ext"
  * for an external source, followed by the frequency.  We calculate
  * the appropriate multiplier and PLL register contents accordingly.
  * When no configuration is given we default to "pci66" since that
  * always will allow the card to work.  If a card is using the PCI
  * bus clock and in a 33MHz slot then it will be operating at half
  * speed until the correct information is provided.
  *
  * We use a default setting of "ext66" because according to Mike Ham
  * of HiFn, almost every board in existence has an external crystal
  * populated at 66Mhz. Using PCI can be a problem on modern motherboards,
  * because PCI33 can have clocks from 0 to 33Mhz, and some have
  * non-PCI-compliant spread-spectrum clocks, which can confuse the pll.
  */
 static void
 hifn_getpllconfig(device_t dev, u_int *pll)
 {
 	const char *pllspec;
 	u_int freq, mul, fl, fh;
 	u_int32_t pllconfig;
 	char *nxt;
 
 	if (resource_string_value("hifn", device_get_unit(dev),
 	    "pllconfig", &pllspec))
 		pllspec = "ext66";
 	fl = 33, fh = 66;
 	pllconfig = 0;
 	if (strncmp(pllspec, "ext", 3) == 0) {
 		pllspec += 3;
 		pllconfig |= HIFN_PLL_REF_SEL;
 		switch (pci_get_device(dev)) {
 		case PCI_PRODUCT_HIFN_7955:
 		case PCI_PRODUCT_HIFN_7956:
 			fl = 20, fh = 100;
 			break;
 #ifdef notyet
 		case PCI_PRODUCT_HIFN_7954:
 			fl = 20, fh = 66;
 			break;
 #endif
 		}
 	} else if (strncmp(pllspec, "pci", 3) == 0)
 		pllspec += 3;
 	freq = strtoul(pllspec, &nxt, 10);
 	if (nxt == pllspec)
 		freq = 66;
 	else
 		freq = checkmaxmin(dev, "frequency", freq, fl, fh);
 	/*
 	 * Calculate multiplier.  We target a Fck of 266 MHz,
 	 * allowing only even values, possibly rounded down.
 	 * Multipliers > 8 must set the charge pump current.
 	 */
 	mul = checkmaxmin(dev, "PLL divisor", (266 / freq) &~ 1, 2, 12);
 	pllconfig |= (mul / 2 - 1) << HIFN_PLL_ND_SHIFT;
 	if (mul > 8)
 		pllconfig |= HIFN_PLL_IS;
 	*pll = pllconfig;
 }
 
 /*
  * Attach an interface that successfully probed.
  */
 static int 
 hifn_attach(device_t dev)
 {
 	struct hifn_softc *sc = device_get_softc(dev);
 	caddr_t kva;
 	int rseg, rid;
 	char rbase;
-	u_int16_t ena, rev;
+	uint16_t rev;
 
 	sc->sc_dev = dev;
 
 	mtx_init(&sc->sc_mtx, device_get_nameunit(dev), "hifn driver", MTX_DEF);
 
 	/* XXX handle power management */
 
 	/*
 	 * The 7951 and 795x have a random number generator and
 	 * public key support; note this.
 	 */
 	if (pci_get_vendor(dev) == PCI_VENDOR_HIFN &&
 	    (pci_get_device(dev) == PCI_PRODUCT_HIFN_7951 ||
 	     pci_get_device(dev) == PCI_PRODUCT_HIFN_7955 ||
 	     pci_get_device(dev) == PCI_PRODUCT_HIFN_7956))
 		sc->sc_flags = HIFN_HAS_RNG | HIFN_HAS_PUBLIC;
 	/*
 	 * The 7811 has a random number generator and
 	 * we also note it's identity 'cuz of some quirks.
 	 */
 	if (pci_get_vendor(dev) == PCI_VENDOR_HIFN &&
 	    pci_get_device(dev) == PCI_PRODUCT_HIFN_7811)
 		sc->sc_flags |= HIFN_IS_7811 | HIFN_HAS_RNG;
 
 	/*
 	 * The 795x parts support AES.
 	 */
 	if (pci_get_vendor(dev) == PCI_VENDOR_HIFN &&
 	    (pci_get_device(dev) == PCI_PRODUCT_HIFN_7955 ||
 	     pci_get_device(dev) == PCI_PRODUCT_HIFN_7956)) {
 		sc->sc_flags |= HIFN_IS_7956 | HIFN_HAS_AES;
 		/*
 		 * Select PLL configuration.  This depends on the
 		 * bus and board design and must be manually configured
 		 * if the default setting is unacceptable.
 		 */
 		hifn_getpllconfig(dev, &sc->sc_pllconfig);
 	}
 
 	/*
 	 * Setup PCI resources. Note that we record the bus
 	 * tag and handle for each register mapping, this is
 	 * used by the READ_REG_0, WRITE_REG_0, READ_REG_1,
 	 * and WRITE_REG_1 macros throughout the driver.
 	 */
 	pci_enable_busmaster(dev);
 
 	rid = HIFN_BAR0;
 	sc->sc_bar0res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 			 			RF_ACTIVE);
 	if (sc->sc_bar0res == NULL) {
 		device_printf(dev, "cannot map bar%d register space\n", 0);
 		goto fail_pci;
 	}
 	sc->sc_st0 = rman_get_bustag(sc->sc_bar0res);
 	sc->sc_sh0 = rman_get_bushandle(sc->sc_bar0res);
 	sc->sc_bar0_lastreg = (bus_size_t) -1;
 
 	rid = HIFN_BAR1;
 	sc->sc_bar1res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 						RF_ACTIVE);
 	if (sc->sc_bar1res == NULL) {
 		device_printf(dev, "cannot map bar%d register space\n", 1);
 		goto fail_io0;
 	}
 	sc->sc_st1 = rman_get_bustag(sc->sc_bar1res);
 	sc->sc_sh1 = rman_get_bushandle(sc->sc_bar1res);
 	sc->sc_bar1_lastreg = (bus_size_t) -1;
 
 	hifn_set_retry(sc);
 
 	/*
 	 * Setup the area where the Hifn DMA's descriptors
 	 * and associated data structures.
 	 */
 	if (bus_dma_tag_create(bus_get_dma_tag(dev),	/* PCI parent */
 			       1, 0,			/* alignment,boundary */
 			       BUS_SPACE_MAXADDR_32BIT,	/* lowaddr */
 			       BUS_SPACE_MAXADDR,	/* highaddr */
 			       NULL, NULL,		/* filter, filterarg */
 			       HIFN_MAX_DMALEN,		/* maxsize */
 			       MAX_SCATTER,		/* nsegments */
 			       HIFN_MAX_SEGLEN,		/* maxsegsize */
 			       BUS_DMA_ALLOCNOW,	/* flags */
 			       NULL,			/* lockfunc */
 			       NULL,			/* lockarg */
 			       &sc->sc_dmat)) {
 		device_printf(dev, "cannot allocate DMA tag\n");
 		goto fail_io1;
 	}
 	if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &sc->sc_dmamap)) {
 		device_printf(dev, "cannot create dma map\n");
 		bus_dma_tag_destroy(sc->sc_dmat);
 		goto fail_io1;
 	}
 	if (bus_dmamem_alloc(sc->sc_dmat, (void**) &kva, BUS_DMA_NOWAIT, &sc->sc_dmamap)) {
 		device_printf(dev, "cannot alloc dma buffer\n");
 		bus_dmamap_destroy(sc->sc_dmat, sc->sc_dmamap);
 		bus_dma_tag_destroy(sc->sc_dmat);
 		goto fail_io1;
 	}
 	if (bus_dmamap_load(sc->sc_dmat, sc->sc_dmamap, kva,
 			     sizeof (*sc->sc_dma),
 			     hifn_dmamap_cb, &sc->sc_dma_physaddr,
 			     BUS_DMA_NOWAIT)) {
 		device_printf(dev, "cannot load dma map\n");
 		bus_dmamem_free(sc->sc_dmat, kva, sc->sc_dmamap);
 		bus_dma_tag_destroy(sc->sc_dmat);
 		goto fail_io1;
 	}
 	sc->sc_dma = (struct hifn_dma *)kva;
 	bzero(sc->sc_dma, sizeof(*sc->sc_dma));
 
 	KASSERT(sc->sc_st0 != 0, ("hifn_attach: null bar0 tag!"));
 	KASSERT(sc->sc_sh0 != 0, ("hifn_attach: null bar0 handle!"));
 	KASSERT(sc->sc_st1 != 0, ("hifn_attach: null bar1 tag!"));
 	KASSERT(sc->sc_sh1 != 0, ("hifn_attach: null bar1 handle!"));
 
 	/*
 	 * Reset the board and do the ``secret handshake''
 	 * to enable the crypto support.  Then complete the
 	 * initialization procedure by setting up the interrupt
 	 * and hooking in to the system crypto support so we'll
 	 * get used for system services like the crypto device,
 	 * IPsec, RNG device, etc.
 	 */
 	hifn_reset_board(sc, 0);
 
 	if (hifn_enable_crypto(sc) != 0) {
 		device_printf(dev, "crypto enabling failed\n");
 		goto fail_mem;
 	}
 	hifn_reset_puc(sc);
 
 	hifn_init_dma(sc);
 	hifn_init_pci_registers(sc);
 
 	/* XXX can't dynamically determine ram type for 795x; force dram */
 	if (sc->sc_flags & HIFN_IS_7956)
 		sc->sc_drammodel = 1;
 	else if (hifn_ramtype(sc))
 		goto fail_mem;
 
 	if (sc->sc_drammodel == 0)
 		hifn_sramsize(sc);
 	else
 		hifn_dramsize(sc);
 
 	/*
 	 * Workaround for NetSec 7751 rev A: half ram size because two
 	 * of the address lines were left floating
 	 */
 	if (pci_get_vendor(dev) == PCI_VENDOR_NETSEC &&
 	    pci_get_device(dev) == PCI_PRODUCT_NETSEC_7751 &&
 	    pci_get_revid(dev) == 0x61)	/*XXX???*/
 		sc->sc_ramsize >>= 1;
 
 	/*
 	 * Arrange the interrupt line.
 	 */
 	rid = 0;
 	sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
 					    RF_SHAREABLE|RF_ACTIVE);
 	if (sc->sc_irq == NULL) {
 		device_printf(dev, "could not map interrupt\n");
 		goto fail_mem;
 	}
 	/*
 	 * NB: Network code assumes we are blocked with splimp()
 	 *     so make sure the IRQ is marked appropriately.
 	 */
 	if (bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET | INTR_MPSAFE,
 			   NULL, hifn_intr, sc, &sc->sc_intrhand)) {
 		device_printf(dev, "could not setup interrupt\n");
 		goto fail_intr2;
 	}
 
 	hifn_sessions(sc);
 
 	/*
 	 * NB: Keep only the low 16 bits; this masks the chip id
 	 *     from the 7951.
 	 */
 	rev = READ_REG_1(sc, HIFN_1_REVID) & 0xffff;
 
 	rseg = sc->sc_ramsize / 1024;
 	rbase = 'K';
 	if (sc->sc_ramsize >= (1024 * 1024)) {
 		rbase = 'M';
 		rseg /= 1024;
 	}
 	device_printf(sc->sc_dev, "%s, rev %u, %d%cB %cram",
 		hifn_partname(sc), rev,
 		rseg, rbase, sc->sc_drammodel ? 'd' : 's');
 	if (sc->sc_flags & HIFN_IS_7956)
 		printf(", pll=0x%x<%s clk, %ux mult>",
 			sc->sc_pllconfig,
 			sc->sc_pllconfig & HIFN_PLL_REF_SEL ? "ext" : "pci",
 			2 + 2*((sc->sc_pllconfig & HIFN_PLL_ND) >> 11));
 	printf("\n");
 
-	sc->sc_cid = crypto_get_driverid(dev, sizeof(struct hifn_session),
-	    CRYPTOCAP_F_HARDWARE);
-	if (sc->sc_cid < 0) {
-		device_printf(dev, "could not get crypto driver id\n");
-		goto fail_intr;
-	}
-
 	WRITE_REG_0(sc, HIFN_0_PUCNFG,
 	    READ_REG_0(sc, HIFN_0_PUCNFG) | HIFN_PUCNFG_CHIPID);
-	ena = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA;
+	sc->sc_ena = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA;
 
-	switch (ena) {
+	switch (sc->sc_ena) {
 	case HIFN_PUSTAT_ENA_2:
-		crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
-		crypto_register(sc->sc_cid, CRYPTO_ARC4, 0, 0);
-		if (sc->sc_flags & HIFN_HAS_AES)
-			crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
-		/*FALLTHROUGH*/
 	case HIFN_PUSTAT_ENA_1:
-		crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0);
-		crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0);
-		crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
-		crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
-		crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
+		sc->sc_cid = crypto_get_driverid(dev,
+		    sizeof(struct hifn_session), CRYPTOCAP_F_HARDWARE);
+		if (sc->sc_cid < 0) {
+			device_printf(dev, "could not get crypto driver id\n");
+			goto fail_intr;
+		}
 		break;
 	}
-
+		
 	bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 
 	if (sc->sc_flags & (HIFN_HAS_PUBLIC | HIFN_HAS_RNG))
 		hifn_init_pubrng(sc);
 
 	callout_init(&sc->sc_tickto, 1);
 	callout_reset(&sc->sc_tickto, hz, hifn_tick, sc);
 
 	return (0);
 
 fail_intr:
 	bus_teardown_intr(dev, sc->sc_irq, sc->sc_intrhand);
 fail_intr2:
 	/* XXX don't store rid */
 	bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
 fail_mem:
 	bus_dmamap_unload(sc->sc_dmat, sc->sc_dmamap);
 	bus_dmamem_free(sc->sc_dmat, sc->sc_dma, sc->sc_dmamap);
 	bus_dma_tag_destroy(sc->sc_dmat);
 
 	/* Turn off DMA polling */
 	WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
 	    HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
 fail_io1:
 	bus_release_resource(dev, SYS_RES_MEMORY, HIFN_BAR1, sc->sc_bar1res);
 fail_io0:
 	bus_release_resource(dev, SYS_RES_MEMORY, HIFN_BAR0, sc->sc_bar0res);
 fail_pci:
 	mtx_destroy(&sc->sc_mtx);
 	return (ENXIO);
 }
 
 /*
  * Detach an interface that successfully probed.
  */
 static int 
 hifn_detach(device_t dev)
 {
 	struct hifn_softc *sc = device_get_softc(dev);
 
 	KASSERT(sc != NULL, ("hifn_detach: null software carrier!"));
 
 	/* disable interrupts */
 	WRITE_REG_1(sc, HIFN_1_DMA_IER, 0);
 
 	/*XXX other resources */
 	callout_stop(&sc->sc_tickto);
 	callout_stop(&sc->sc_rngto);
 #ifdef HIFN_RNDTEST
 	if (sc->sc_rndtest)
 		rndtest_detach(sc->sc_rndtest);
 #endif
 
 	/* Turn off DMA polling */
 	WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
 	    HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
 
 	crypto_unregister_all(sc->sc_cid);
 
 	bus_generic_detach(dev);	/*XXX should be no children, right? */
 
 	bus_teardown_intr(dev, sc->sc_irq, sc->sc_intrhand);
 	/* XXX don't store rid */
 	bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
 
 	bus_dmamap_unload(sc->sc_dmat, sc->sc_dmamap);
 	bus_dmamem_free(sc->sc_dmat, sc->sc_dma, sc->sc_dmamap);
 	bus_dma_tag_destroy(sc->sc_dmat);
 
 	bus_release_resource(dev, SYS_RES_MEMORY, HIFN_BAR1, sc->sc_bar1res);
 	bus_release_resource(dev, SYS_RES_MEMORY, HIFN_BAR0, sc->sc_bar0res);
 
 	mtx_destroy(&sc->sc_mtx);
 
 	return (0);
 }
 
 /*
  * Stop all chip I/O so that the kernel's probe routines don't
  * get confused by errant DMAs when rebooting.
  */
 static int
 hifn_shutdown(device_t dev)
 {
 #ifdef notyet
 	hifn_stop(device_get_softc(dev));
 #endif
 	return (0);
 }
 
 /*
  * Device suspend routine.  Stop the interface and save some PCI
  * settings in case the BIOS doesn't restore them properly on
  * resume.
  */
 static int
 hifn_suspend(device_t dev)
 {
 	struct hifn_softc *sc = device_get_softc(dev);
 #ifdef notyet
 	hifn_stop(sc);
 #endif
 	sc->sc_suspended = 1;
 
 	return (0);
 }
 
 /*
  * Device resume routine.  Restore some PCI settings in case the BIOS
  * doesn't, re-enable busmastering, and restart the interface if
  * appropriate.
  */
 static int
 hifn_resume(device_t dev)
 {
 	struct hifn_softc *sc = device_get_softc(dev);
 #ifdef notyet
         /* reinitialize interface if necessary */
         if (ifp->if_flags & IFF_UP)
                 rl_init(sc);
 #endif
 	sc->sc_suspended = 0;
 
 	return (0);
 }
 
 static int
 hifn_init_pubrng(struct hifn_softc *sc)
 {
 	u_int32_t r;
 	int i;
 
 #ifdef HIFN_RNDTEST
 	sc->sc_rndtest = rndtest_attach(sc->sc_dev);
 	if (sc->sc_rndtest)
 		sc->sc_harvest = rndtest_harvest;
 	else
 		sc->sc_harvest = default_harvest;
 #else
 	sc->sc_harvest = default_harvest;
 #endif
 	if ((sc->sc_flags & HIFN_IS_7811) == 0) {
 		/* Reset 7951 public key/rng engine */
 		WRITE_REG_1(sc, HIFN_1_PUB_RESET,
 		    READ_REG_1(sc, HIFN_1_PUB_RESET) | HIFN_PUBRST_RESET);
 
 		for (i = 0; i < 100; i++) {
 			DELAY(1000);
 			if ((READ_REG_1(sc, HIFN_1_PUB_RESET) &
 			    HIFN_PUBRST_RESET) == 0)
 				break;
 		}
 
 		if (i == 100) {
 			device_printf(sc->sc_dev, "public key init failed\n");
 			return (1);
 		}
 	}
 
 	/* Enable the rng, if available */
 	if (sc->sc_flags & HIFN_HAS_RNG) {
 		if (sc->sc_flags & HIFN_IS_7811) {
 			r = READ_REG_1(sc, HIFN_1_7811_RNGENA);
 			if (r & HIFN_7811_RNGENA_ENA) {
 				r &= ~HIFN_7811_RNGENA_ENA;
 				WRITE_REG_1(sc, HIFN_1_7811_RNGENA, r);
 			}
 			WRITE_REG_1(sc, HIFN_1_7811_RNGCFG,
 			    HIFN_7811_RNGCFG_DEFL);
 			r |= HIFN_7811_RNGENA_ENA;
 			WRITE_REG_1(sc, HIFN_1_7811_RNGENA, r);
 		} else
 			WRITE_REG_1(sc, HIFN_1_RNG_CONFIG,
 			    READ_REG_1(sc, HIFN_1_RNG_CONFIG) |
 			    HIFN_RNGCFG_ENA);
 
 		sc->sc_rngfirst = 1;
 		if (hz >= 100)
 			sc->sc_rnghz = hz / 100;
 		else
 			sc->sc_rnghz = 1;
 		callout_init(&sc->sc_rngto, 1);
 		callout_reset(&sc->sc_rngto, sc->sc_rnghz, hifn_rng, sc);
 	}
 
 	/* Enable public key engine, if available */
 	if (sc->sc_flags & HIFN_HAS_PUBLIC) {
 		WRITE_REG_1(sc, HIFN_1_PUB_IEN, HIFN_PUBIEN_DONE);
 		sc->sc_dmaier |= HIFN_DMAIER_PUBDONE;
 		WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
 #ifdef HIFN_VULCANDEV
 		sc->sc_pkdev = make_dev(&vulcanpk_cdevsw, 0, 
 					UID_ROOT, GID_WHEEL, 0666,
 					"vulcanpk");
 		sc->sc_pkdev->si_drv1 = sc;
 #endif
 	}
 
 	return (0);
 }
 
 static void
 hifn_rng(void *vsc)
 {
 #define	RANDOM_BITS(n)	(n)*sizeof (u_int32_t), (n)*sizeof (u_int32_t)*NBBY, 0
 	struct hifn_softc *sc = vsc;
 	u_int32_t sts, num[2];
 	int i;
 
 	if (sc->sc_flags & HIFN_IS_7811) {
 		/* ONLY VALID ON 7811!!!! */
 		for (i = 0; i < 5; i++) {
 			sts = READ_REG_1(sc, HIFN_1_7811_RNGSTS);
 			if (sts & HIFN_7811_RNGSTS_UFL) {
 				device_printf(sc->sc_dev,
 					      "RNG underflow: disabling\n");
 				return;
 			}
 			if ((sts & HIFN_7811_RNGSTS_RDY) == 0)
 				break;
 
 			/*
 			 * There are at least two words in the RNG FIFO
 			 * at this point.
 			 */
 			num[0] = READ_REG_1(sc, HIFN_1_7811_RNGDAT);
 			num[1] = READ_REG_1(sc, HIFN_1_7811_RNGDAT);
 			/* NB: discard first data read */
 			if (sc->sc_rngfirst)
 				sc->sc_rngfirst = 0;
 			else
 				(*sc->sc_harvest)(sc->sc_rndtest,
 					num, sizeof (num));
 		}
 	} else {
 		num[0] = READ_REG_1(sc, HIFN_1_RNG_DATA);
 
 		/* NB: discard first data read */
 		if (sc->sc_rngfirst)
 			sc->sc_rngfirst = 0;
 		else
 			(*sc->sc_harvest)(sc->sc_rndtest,
 				num, sizeof (num[0]));
 	}
 
 	callout_reset(&sc->sc_rngto, sc->sc_rnghz, hifn_rng, sc);
 #undef RANDOM_BITS
 }
 
 static void
 hifn_puc_wait(struct hifn_softc *sc)
 {
 	int i;
 	int reg = HIFN_0_PUCTRL;
 
 	if (sc->sc_flags & HIFN_IS_7956) {
 		reg = HIFN_0_PUCTRL2;
 	}
 
 	for (i = 5000; i > 0; i--) {
 		DELAY(1);
 		if (!(READ_REG_0(sc, reg) & HIFN_PUCTRL_RESET))
 			break;
 	}
 	if (!i)
 		device_printf(sc->sc_dev, "proc unit did not reset\n");
 }
 
 /*
  * Reset the processing unit.
  */
 static void
 hifn_reset_puc(struct hifn_softc *sc)
 {
 	/* Reset processing unit */
 	int reg = HIFN_0_PUCTRL;
 
 	if (sc->sc_flags & HIFN_IS_7956) {
 		reg = HIFN_0_PUCTRL2;
 	}
 	WRITE_REG_0(sc, reg, HIFN_PUCTRL_DMAENA);
 
 	hifn_puc_wait(sc);
 }
 
 /*
  * Set the Retry and TRDY registers; note that we set them to
  * zero because the 7811 locks up when forced to retry (section
  * 3.6 of "Specification Update SU-0014-04".  Not clear if we
  * should do this for all Hifn parts, but it doesn't seem to hurt.
  */
 static void
 hifn_set_retry(struct hifn_softc *sc)
 {
 	/* NB: RETRY only responds to 8-bit reads/writes */
 	pci_write_config(sc->sc_dev, HIFN_RETRY_TIMEOUT, 0, 1);
 	pci_write_config(sc->sc_dev, HIFN_TRDY_TIMEOUT, 0, 1);
 }
 
 /*
  * Resets the board.  Values in the regesters are left as is
  * from the reset (i.e. initial values are assigned elsewhere).
  */
 static void
 hifn_reset_board(struct hifn_softc *sc, int full)
 {
 	u_int32_t reg;
 
 	/*
 	 * Set polling in the DMA configuration register to zero.  0x7 avoids
 	 * resetting the board and zeros out the other fields.
 	 */
 	WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
 	    HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
 
 	/*
 	 * Now that polling has been disabled, we have to wait 1 ms
 	 * before resetting the board.
 	 */
 	DELAY(1000);
 
 	/* Reset the DMA unit */
 	if (full) {
 		WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE);
 		DELAY(1000);
 	} else {
 		WRITE_REG_1(sc, HIFN_1_DMA_CNFG,
 		    HIFN_DMACNFG_MODE | HIFN_DMACNFG_MSTRESET);
 		hifn_reset_puc(sc);
 	}
 
 	KASSERT(sc->sc_dma != NULL, ("hifn_reset_board: null DMA tag!"));
 	bzero(sc->sc_dma, sizeof(*sc->sc_dma));
 
 	/* Bring dma unit out of reset */
 	WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
 	    HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
 
 	hifn_puc_wait(sc);
 	hifn_set_retry(sc);
 
 	if (sc->sc_flags & HIFN_IS_7811) {
 		for (reg = 0; reg < 1000; reg++) {
 			if (READ_REG_1(sc, HIFN_1_7811_MIPSRST) &
 			    HIFN_MIPSRST_CRAMINIT)
 				break;
 			DELAY(1000);
 		}
 		if (reg == 1000)
 			printf(": cram init timeout\n");
 	} else {
 	  /* set up DMA configuration register #2 */
 	  /* turn off all PK and BAR0 swaps */
 	  WRITE_REG_1(sc, HIFN_1_DMA_CNFG2,
 		      (3 << HIFN_DMACNFG2_INIT_WRITE_BURST_SHIFT)|
 		      (3 << HIFN_DMACNFG2_INIT_READ_BURST_SHIFT)|
 		      (2 << HIFN_DMACNFG2_TGT_WRITE_BURST_SHIFT)|
 		      (2 << HIFN_DMACNFG2_TGT_READ_BURST_SHIFT));
 	}
 		      
 }
 
 static u_int32_t
 hifn_next_signature(u_int32_t a, u_int cnt)
 {
 	int i;
 	u_int32_t v;
 
 	for (i = 0; i < cnt; i++) {
 
 		/* get the parity */
 		v = a & 0x80080125;
 		v ^= v >> 16;
 		v ^= v >> 8;
 		v ^= v >> 4;
 		v ^= v >> 2;
 		v ^= v >> 1;
 
 		a = (v & 1) ^ (a << 1);
 	}
 
 	return a;
 }
 
 struct pci2id {
 	u_short		pci_vendor;
 	u_short		pci_prod;
 	char		card_id[13];
 };
 static struct pci2id pci2id[] = {
 	{
 		PCI_VENDOR_HIFN,
 		PCI_PRODUCT_HIFN_7951,
 		{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 		  0x00, 0x00, 0x00, 0x00, 0x00 }
 	}, {
 		PCI_VENDOR_HIFN,
 		PCI_PRODUCT_HIFN_7955,
 		{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 		  0x00, 0x00, 0x00, 0x00, 0x00 }
 	}, {
 		PCI_VENDOR_HIFN,
 		PCI_PRODUCT_HIFN_7956,
 		{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 		  0x00, 0x00, 0x00, 0x00, 0x00 }
 	}, {
 		PCI_VENDOR_NETSEC,
 		PCI_PRODUCT_NETSEC_7751,
 		{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 		  0x00, 0x00, 0x00, 0x00, 0x00 }
 	}, {
 		PCI_VENDOR_INVERTEX,
 		PCI_PRODUCT_INVERTEX_AEON,
 		{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 		  0x00, 0x00, 0x00, 0x00, 0x00 }
 	}, {
 		PCI_VENDOR_HIFN,
 		PCI_PRODUCT_HIFN_7811,
 		{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 		  0x00, 0x00, 0x00, 0x00, 0x00 }
 	}, {
 		/*
 		 * Other vendors share this PCI ID as well, such as
 		 * http://www.powercrypt.com, and obviously they also
 		 * use the same key.
 		 */
 		PCI_VENDOR_HIFN,
 		PCI_PRODUCT_HIFN_7751,
 		{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 		  0x00, 0x00, 0x00, 0x00, 0x00 }
 	},
 };
 
 /*
  * Checks to see if crypto is already enabled.  If crypto isn't enable,
  * "hifn_enable_crypto" is called to enable it.  The check is important,
  * as enabling crypto twice will lock the board.
  */
 static int 
 hifn_enable_crypto(struct hifn_softc *sc)
 {
 	u_int32_t dmacfg, ramcfg, encl, addr, i;
 	char *offtbl = NULL;
 
 	for (i = 0; i < nitems(pci2id); i++) {
 		if (pci2id[i].pci_vendor == pci_get_vendor(sc->sc_dev) &&
 		    pci2id[i].pci_prod == pci_get_device(sc->sc_dev)) {
 			offtbl = pci2id[i].card_id;
 			break;
 		}
 	}
 	if (offtbl == NULL) {
 		device_printf(sc->sc_dev, "Unknown card!\n");
 		return (1);
 	}
 
 	ramcfg = READ_REG_0(sc, HIFN_0_PUCNFG);
 	dmacfg = READ_REG_1(sc, HIFN_1_DMA_CNFG);
 
 	/*
 	 * The RAM config register's encrypt level bit needs to be set before
 	 * every read performed on the encryption level register.
 	 */
 	WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg | HIFN_PUCNFG_CHIPID);
 
 	encl = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA;
 
 	/*
 	 * Make sure we don't re-unlock.  Two unlocks kills chip until the
 	 * next reboot.
 	 */
 	if (encl == HIFN_PUSTAT_ENA_1 || encl == HIFN_PUSTAT_ENA_2) {
 #ifdef HIFN_DEBUG
 		if (hifn_debug)
 			device_printf(sc->sc_dev,
 			    "Strong crypto already enabled!\n");
 #endif
 		goto report;
 	}
 
 	if (encl != 0 && encl != HIFN_PUSTAT_ENA_0) {
 #ifdef HIFN_DEBUG
 		if (hifn_debug)
 			device_printf(sc->sc_dev,
 			      "Unknown encryption level 0x%x\n", encl);
 #endif
 		return 1;
 	}
 
 	WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_UNLOCK |
 	    HIFN_DMACNFG_MSTRESET | HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
 	DELAY(1000);
 	addr = READ_REG_1(sc, HIFN_UNLOCK_SECRET1);
 	DELAY(1000);
 	WRITE_REG_1(sc, HIFN_UNLOCK_SECRET2, 0);
 	DELAY(1000);
 
 	for (i = 0; i <= 12; i++) {
 		addr = hifn_next_signature(addr, offtbl[i] + 0x101);
 		WRITE_REG_1(sc, HIFN_UNLOCK_SECRET2, addr);
 
 		DELAY(1000);
 	}
 
 	WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg | HIFN_PUCNFG_CHIPID);
 	encl = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA;
 
 #ifdef HIFN_DEBUG
 	if (hifn_debug) {
 		if (encl != HIFN_PUSTAT_ENA_1 && encl != HIFN_PUSTAT_ENA_2)
 			device_printf(sc->sc_dev, "Engine is permanently "
 				"locked until next system reset!\n");
 		else
 			device_printf(sc->sc_dev, "Engine enabled "
 				"successfully!\n");
 	}
 #endif
 
 report:
 	WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg);
 	WRITE_REG_1(sc, HIFN_1_DMA_CNFG, dmacfg);
 
 	switch (encl) {
 	case HIFN_PUSTAT_ENA_1:
 	case HIFN_PUSTAT_ENA_2:
 		break;
 	case HIFN_PUSTAT_ENA_0:
 	default:
 		device_printf(sc->sc_dev, "disabled");
 		break;
 	}
 
 	return 0;
 }
 
 /*
  * Give initial values to the registers listed in the "Register Space"
  * section of the HIFN Software Development reference manual.
  */
 static void 
 hifn_init_pci_registers(struct hifn_softc *sc)
 {
 	/* write fixed values needed by the Initialization registers */
 	WRITE_REG_0(sc, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA);
 	WRITE_REG_0(sc, HIFN_0_FIFOCNFG, HIFN_FIFOCNFG_THRESHOLD);
 	WRITE_REG_0(sc, HIFN_0_PUIER, HIFN_PUIER_DSTOVER);
 
 	/* write all 4 ring address registers */
 	WRITE_REG_1(sc, HIFN_1_DMA_CRAR, sc->sc_dma_physaddr +
 	    offsetof(struct hifn_dma, cmdr[0]));
 	WRITE_REG_1(sc, HIFN_1_DMA_SRAR, sc->sc_dma_physaddr +
 	    offsetof(struct hifn_dma, srcr[0]));
 	WRITE_REG_1(sc, HIFN_1_DMA_DRAR, sc->sc_dma_physaddr +
 	    offsetof(struct hifn_dma, dstr[0]));
 	WRITE_REG_1(sc, HIFN_1_DMA_RRAR, sc->sc_dma_physaddr +
 	    offsetof(struct hifn_dma, resr[0]));
 
 	DELAY(2000);
 
 	/* write status register */
 	WRITE_REG_1(sc, HIFN_1_DMA_CSR,
 	    HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS |
 	    HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS |
 	    HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST |
 	    HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER |
 	    HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST |
 	    HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER |
 	    HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST |
 	    HIFN_DMACSR_S_WAIT |
 	    HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST |
 	    HIFN_DMACSR_C_WAIT |
 	    HIFN_DMACSR_ENGINE |
 	    ((sc->sc_flags & HIFN_HAS_PUBLIC) ?
 		HIFN_DMACSR_PUBDONE : 0) |
 	    ((sc->sc_flags & HIFN_IS_7811) ?
 		HIFN_DMACSR_ILLW | HIFN_DMACSR_ILLR : 0));
 
 	sc->sc_d_busy = sc->sc_r_busy = sc->sc_s_busy = sc->sc_c_busy = 0;
 	sc->sc_dmaier |= HIFN_DMAIER_R_DONE | HIFN_DMAIER_C_ABORT |
 	    HIFN_DMAIER_D_OVER | HIFN_DMAIER_R_OVER |
 	    HIFN_DMAIER_S_ABORT | HIFN_DMAIER_D_ABORT | HIFN_DMAIER_R_ABORT |
 	    ((sc->sc_flags & HIFN_IS_7811) ?
 		HIFN_DMAIER_ILLW | HIFN_DMAIER_ILLR : 0);
 	sc->sc_dmaier &= ~HIFN_DMAIER_C_WAIT;
 	WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
 
 
 	if (sc->sc_flags & HIFN_IS_7956) {
 		u_int32_t pll;
 
 		WRITE_REG_0(sc, HIFN_0_PUCNFG, HIFN_PUCNFG_COMPSING |
 		    HIFN_PUCNFG_TCALLPHASES |
 		    HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32);
 
 		/* turn off the clocks and insure bypass is set */
 		pll = READ_REG_1(sc, HIFN_1_PLL);
 		pll = (pll &~ (HIFN_PLL_PK_CLK_SEL | HIFN_PLL_PE_CLK_SEL))
 		  | HIFN_PLL_BP | HIFN_PLL_MBSET;
 		WRITE_REG_1(sc, HIFN_1_PLL, pll);
 		DELAY(10*1000);		/* 10ms */
 
 		/* change configuration */
 		pll = (pll &~ HIFN_PLL_CONFIG) | sc->sc_pllconfig;
 		WRITE_REG_1(sc, HIFN_1_PLL, pll);
 		DELAY(10*1000);		/* 10ms */
 
 		/* disable bypass */
 		pll &= ~HIFN_PLL_BP;
 		WRITE_REG_1(sc, HIFN_1_PLL, pll);
 		/* enable clocks with new configuration */
 		pll |= HIFN_PLL_PK_CLK_SEL | HIFN_PLL_PE_CLK_SEL;
 		WRITE_REG_1(sc, HIFN_1_PLL, pll);
 	} else {
 		WRITE_REG_0(sc, HIFN_0_PUCNFG, HIFN_PUCNFG_COMPSING |
 		    HIFN_PUCNFG_DRFR_128 | HIFN_PUCNFG_TCALLPHASES |
 		    HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32 |
 		    (sc->sc_drammodel ? HIFN_PUCNFG_DRAM : HIFN_PUCNFG_SRAM));
 	}
 
 	WRITE_REG_0(sc, HIFN_0_PUISR, HIFN_PUISR_DSTOVER);
 	WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
 	    HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE | HIFN_DMACNFG_LAST |
 	    ((HIFN_POLL_FREQUENCY << 16 ) & HIFN_DMACNFG_POLLFREQ) |
 	    ((HIFN_POLL_SCALAR << 8) & HIFN_DMACNFG_POLLINVAL));
 }
 
 /*
  * The maximum number of sessions supported by the card
  * is dependent on the amount of context ram, which
  * encryption algorithms are enabled, and how compression
  * is configured.  This should be configured before this
  * routine is called.
  */
 static void
 hifn_sessions(struct hifn_softc *sc)
 {
 	u_int32_t pucnfg;
 	int ctxsize;
 
 	pucnfg = READ_REG_0(sc, HIFN_0_PUCNFG);
 
 	if (pucnfg & HIFN_PUCNFG_COMPSING) {
 		if (pucnfg & HIFN_PUCNFG_ENCCNFG)
 			ctxsize = 128;
 		else
 			ctxsize = 512;
 		/*
 		 * 7955/7956 has internal context memory of 32K
 		 */
 		if (sc->sc_flags & HIFN_IS_7956)
 			sc->sc_maxses = 32768 / ctxsize;
 		else
 			sc->sc_maxses = 1 +
 			    ((sc->sc_ramsize - 32768) / ctxsize);
 	} else
 		sc->sc_maxses = sc->sc_ramsize / 16384;
 
 	if (sc->sc_maxses > 2048)
 		sc->sc_maxses = 2048;
 }
 
 /*
  * Determine ram type (sram or dram).  Board should be just out of a reset
  * state when this is called.
  */
 static int
 hifn_ramtype(struct hifn_softc *sc)
 {
 	u_int8_t data[8], dataexpect[8];
 	int i;
 
 	for (i = 0; i < sizeof(data); i++)
 		data[i] = dataexpect[i] = 0x55;
 	if (hifn_writeramaddr(sc, 0, data))
 		return (-1);
 	if (hifn_readramaddr(sc, 0, data))
 		return (-1);
 	if (bcmp(data, dataexpect, sizeof(data)) != 0) {
 		sc->sc_drammodel = 1;
 		return (0);
 	}
 
 	for (i = 0; i < sizeof(data); i++)
 		data[i] = dataexpect[i] = 0xaa;
 	if (hifn_writeramaddr(sc, 0, data))
 		return (-1);
 	if (hifn_readramaddr(sc, 0, data))
 		return (-1);
 	if (bcmp(data, dataexpect, sizeof(data)) != 0) {
 		sc->sc_drammodel = 1;
 		return (0);
 	}
 
 	return (0);
 }
 
 #define	HIFN_SRAM_MAX		(32 << 20)
 #define	HIFN_SRAM_STEP_SIZE	16384
 #define	HIFN_SRAM_GRANULARITY	(HIFN_SRAM_MAX / HIFN_SRAM_STEP_SIZE)
 
 static int
 hifn_sramsize(struct hifn_softc *sc)
 {
 	u_int32_t a;
 	u_int8_t data[8];
 	u_int8_t dataexpect[sizeof(data)];
 	int32_t i;
 
 	for (i = 0; i < sizeof(data); i++)
 		data[i] = dataexpect[i] = i ^ 0x5a;
 
 	for (i = HIFN_SRAM_GRANULARITY - 1; i >= 0; i--) {
 		a = i * HIFN_SRAM_STEP_SIZE;
 		bcopy(&i, data, sizeof(i));
 		hifn_writeramaddr(sc, a, data);
 	}
 
 	for (i = 0; i < HIFN_SRAM_GRANULARITY; i++) {
 		a = i * HIFN_SRAM_STEP_SIZE;
 		bcopy(&i, dataexpect, sizeof(i));
 		if (hifn_readramaddr(sc, a, data) < 0)
 			return (0);
 		if (bcmp(data, dataexpect, sizeof(data)) != 0)
 			return (0);
 		sc->sc_ramsize = a + HIFN_SRAM_STEP_SIZE;
 	}
 
 	return (0);
 }
 
 /*
  * XXX For dram boards, one should really try all of the
  * HIFN_PUCNFG_DSZ_*'s.  This just assumes that PUCNFG
  * is already set up correctly.
  */
 static int
 hifn_dramsize(struct hifn_softc *sc)
 {
 	u_int32_t cnfg;
 
 	if (sc->sc_flags & HIFN_IS_7956) {
 		/*
 		 * 7955/7956 have a fixed internal ram of only 32K.
 		 */
 		sc->sc_ramsize = 32768;
 	} else {
 		cnfg = READ_REG_0(sc, HIFN_0_PUCNFG) &
 		    HIFN_PUCNFG_DRAMMASK;
 		sc->sc_ramsize = 1 << ((cnfg >> 13) + 18);
 	}
 	return (0);
 }
 
 static void
 hifn_alloc_slot(struct hifn_softc *sc, int *cmdp, int *srcp, int *dstp, int *resp)
 {
 	struct hifn_dma *dma = sc->sc_dma;
 
 	if (sc->sc_cmdi == HIFN_D_CMD_RSIZE) {
 		sc->sc_cmdi = 0;
 		dma->cmdr[HIFN_D_CMD_RSIZE].l = htole32(HIFN_D_VALID |
 		    HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
 		HIFN_CMDR_SYNC(sc, HIFN_D_CMD_RSIZE,
 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
 	}
 	*cmdp = sc->sc_cmdi++;
 	sc->sc_cmdk = sc->sc_cmdi;
 
 	if (sc->sc_srci == HIFN_D_SRC_RSIZE) {
 		sc->sc_srci = 0;
 		dma->srcr[HIFN_D_SRC_RSIZE].l = htole32(HIFN_D_VALID |
 		    HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
 		HIFN_SRCR_SYNC(sc, HIFN_D_SRC_RSIZE,
 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
 	}
 	*srcp = sc->sc_srci++;
 	sc->sc_srck = sc->sc_srci;
 
 	if (sc->sc_dsti == HIFN_D_DST_RSIZE) {
 		sc->sc_dsti = 0;
 		dma->dstr[HIFN_D_DST_RSIZE].l = htole32(HIFN_D_VALID |
 		    HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
 		HIFN_DSTR_SYNC(sc, HIFN_D_DST_RSIZE,
 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
 	}
 	*dstp = sc->sc_dsti++;
 	sc->sc_dstk = sc->sc_dsti;
 
 	if (sc->sc_resi == HIFN_D_RES_RSIZE) {
 		sc->sc_resi = 0;
 		dma->resr[HIFN_D_RES_RSIZE].l = htole32(HIFN_D_VALID |
 		    HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
 		HIFN_RESR_SYNC(sc, HIFN_D_RES_RSIZE,
 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
 	}
 	*resp = sc->sc_resi++;
 	sc->sc_resk = sc->sc_resi;
 }
 
 static int
 hifn_writeramaddr(struct hifn_softc *sc, int addr, u_int8_t *data)
 {
 	struct hifn_dma *dma = sc->sc_dma;
 	hifn_base_command_t wc;
 	const u_int32_t masks = HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ;
 	int r, cmdi, resi, srci, dsti;
 
 	wc.masks = htole16(3 << 13);
 	wc.session_num = htole16(addr >> 14);
 	wc.total_source_count = htole16(8);
 	wc.total_dest_count = htole16(addr & 0x3fff);
 
 	hifn_alloc_slot(sc, &cmdi, &srci, &dsti, &resi);
 
 	WRITE_REG_1(sc, HIFN_1_DMA_CSR,
 	    HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA |
 	    HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA);
 
 	/* build write command */
 	bzero(dma->command_bufs[cmdi], HIFN_MAX_COMMAND);
 	*(hifn_base_command_t *)dma->command_bufs[cmdi] = wc;
 	bcopy(data, &dma->test_src, sizeof(dma->test_src));
 
 	dma->srcr[srci].p = htole32(sc->sc_dma_physaddr
 	    + offsetof(struct hifn_dma, test_src));
 	dma->dstr[dsti].p = htole32(sc->sc_dma_physaddr
 	    + offsetof(struct hifn_dma, test_dst));
 
 	dma->cmdr[cmdi].l = htole32(16 | masks);
 	dma->srcr[srci].l = htole32(8 | masks);
 	dma->dstr[dsti].l = htole32(4 | masks);
 	dma->resr[resi].l = htole32(4 | masks);
 
 	bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 
 	for (r = 10000; r >= 0; r--) {
 		DELAY(10);
 		bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
 		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 		if ((dma->resr[resi].l & htole32(HIFN_D_VALID)) == 0)
 			break;
 		bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
 		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 	}
 	if (r == 0) {
 		device_printf(sc->sc_dev, "writeramaddr -- "
 		    "result[%d](addr %d) still valid\n", resi, addr);
 		r = -1;
 		return (-1);
 	} else
 		r = 0;
 
 	WRITE_REG_1(sc, HIFN_1_DMA_CSR,
 	    HIFN_DMACSR_C_CTRL_DIS | HIFN_DMACSR_S_CTRL_DIS |
 	    HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS);
 
 	return (r);
 }
 
 static int
 hifn_readramaddr(struct hifn_softc *sc, int addr, u_int8_t *data)
 {
 	struct hifn_dma *dma = sc->sc_dma;
 	hifn_base_command_t rc;
 	const u_int32_t masks = HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ;
 	int r, cmdi, srci, dsti, resi;
 
 	rc.masks = htole16(2 << 13);
 	rc.session_num = htole16(addr >> 14);
 	rc.total_source_count = htole16(addr & 0x3fff);
 	rc.total_dest_count = htole16(8);
 
 	hifn_alloc_slot(sc, &cmdi, &srci, &dsti, &resi);
 
 	WRITE_REG_1(sc, HIFN_1_DMA_CSR,
 	    HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA |
 	    HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA);
 
 	bzero(dma->command_bufs[cmdi], HIFN_MAX_COMMAND);
 	*(hifn_base_command_t *)dma->command_bufs[cmdi] = rc;
 
 	dma->srcr[srci].p = htole32(sc->sc_dma_physaddr +
 	    offsetof(struct hifn_dma, test_src));
 	dma->test_src = 0;
 	dma->dstr[dsti].p =  htole32(sc->sc_dma_physaddr +
 	    offsetof(struct hifn_dma, test_dst));
 	dma->test_dst = 0;
 	dma->cmdr[cmdi].l = htole32(8 | masks);
 	dma->srcr[srci].l = htole32(8 | masks);
 	dma->dstr[dsti].l = htole32(8 | masks);
 	dma->resr[resi].l = htole32(HIFN_MAX_RESULT | masks);
 
 	bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 
 	for (r = 10000; r >= 0; r--) {
 		DELAY(10);
 		bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
 		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 		if ((dma->resr[resi].l & htole32(HIFN_D_VALID)) == 0)
 			break;
 		bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
 		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 	}
 	if (r == 0) {
 		device_printf(sc->sc_dev, "readramaddr -- "
 		    "result[%d](addr %d) still valid\n", resi, addr);
 		r = -1;
 	} else {
 		r = 0;
 		bcopy(&dma->test_dst, data, sizeof(dma->test_dst));
 	}
 
 	WRITE_REG_1(sc, HIFN_1_DMA_CSR,
 	    HIFN_DMACSR_C_CTRL_DIS | HIFN_DMACSR_S_CTRL_DIS |
 	    HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS);
 
 	return (r);
 }
 
 /*
  * Initialize the descriptor rings.
  */
 static void 
 hifn_init_dma(struct hifn_softc *sc)
 {
 	struct hifn_dma *dma = sc->sc_dma;
 	int i;
 
 	hifn_set_retry(sc);
 
 	/* initialize static pointer values */
 	for (i = 0; i < HIFN_D_CMD_RSIZE; i++)
 		dma->cmdr[i].p = htole32(sc->sc_dma_physaddr +
 		    offsetof(struct hifn_dma, command_bufs[i][0]));
 	for (i = 0; i < HIFN_D_RES_RSIZE; i++)
 		dma->resr[i].p = htole32(sc->sc_dma_physaddr +
 		    offsetof(struct hifn_dma, result_bufs[i][0]));
 
 	dma->cmdr[HIFN_D_CMD_RSIZE].p =
 	    htole32(sc->sc_dma_physaddr + offsetof(struct hifn_dma, cmdr[0]));
 	dma->srcr[HIFN_D_SRC_RSIZE].p =
 	    htole32(sc->sc_dma_physaddr + offsetof(struct hifn_dma, srcr[0]));
 	dma->dstr[HIFN_D_DST_RSIZE].p =
 	    htole32(sc->sc_dma_physaddr + offsetof(struct hifn_dma, dstr[0]));
 	dma->resr[HIFN_D_RES_RSIZE].p =
 	    htole32(sc->sc_dma_physaddr + offsetof(struct hifn_dma, resr[0]));
 
 	sc->sc_cmdu = sc->sc_srcu = sc->sc_dstu = sc->sc_resu = 0;
 	sc->sc_cmdi = sc->sc_srci = sc->sc_dsti = sc->sc_resi = 0;
 	sc->sc_cmdk = sc->sc_srck = sc->sc_dstk = sc->sc_resk = 0;
 }
 
 /*
  * Writes out the raw command buffer space.  Returns the
  * command buffer size.
  */
 static u_int
 hifn_write_command(struct hifn_command *cmd, u_int8_t *buf)
 {
+	struct cryptop *crp;
 	u_int8_t *buf_pos;
 	hifn_base_command_t *base_cmd;
 	hifn_mac_command_t *mac_cmd;
 	hifn_crypt_command_t *cry_cmd;
 	int using_mac, using_crypt, len, ivlen;
 	u_int32_t dlen, slen;
 
+	crp = cmd->crp;
 	buf_pos = buf;
 	using_mac = cmd->base_masks & HIFN_BASE_CMD_MAC;
 	using_crypt = cmd->base_masks & HIFN_BASE_CMD_CRYPT;
 
 	base_cmd = (hifn_base_command_t *)buf_pos;
 	base_cmd->masks = htole16(cmd->base_masks);
 	slen = cmd->src_mapsize;
 	if (cmd->sloplen)
 		dlen = cmd->dst_mapsize - cmd->sloplen + sizeof(u_int32_t);
 	else
 		dlen = cmd->dst_mapsize;
 	base_cmd->total_source_count = htole16(slen & HIFN_BASE_CMD_LENMASK_LO);
 	base_cmd->total_dest_count = htole16(dlen & HIFN_BASE_CMD_LENMASK_LO);
 	dlen >>= 16;
 	slen >>= 16;
 	base_cmd->session_num = htole16(
 	    ((slen << HIFN_BASE_CMD_SRCLEN_S) & HIFN_BASE_CMD_SRCLEN_M) |
 	    ((dlen << HIFN_BASE_CMD_DSTLEN_S) & HIFN_BASE_CMD_DSTLEN_M));
 	buf_pos += sizeof(hifn_base_command_t);
 
 	if (using_mac) {
 		mac_cmd = (hifn_mac_command_t *)buf_pos;
-		dlen = cmd->maccrd->crd_len;
+		dlen = crp->crp_aad_length + crp->crp_payload_length;
 		mac_cmd->source_count = htole16(dlen & 0xffff);
 		dlen >>= 16;
 		mac_cmd->masks = htole16(cmd->mac_masks |
 		    ((dlen << HIFN_MAC_CMD_SRCLEN_S) & HIFN_MAC_CMD_SRCLEN_M));
-		mac_cmd->header_skip = htole16(cmd->maccrd->crd_skip);
+		if (crp->crp_aad_length != 0)
+			mac_cmd->header_skip = htole16(crp->crp_aad_start);
+		else
+			mac_cmd->header_skip = htole16(crp->crp_payload_start);
 		mac_cmd->reserved = 0;
 		buf_pos += sizeof(hifn_mac_command_t);
 	}
 
 	if (using_crypt) {
 		cry_cmd = (hifn_crypt_command_t *)buf_pos;
-		dlen = cmd->enccrd->crd_len;
+		dlen = crp->crp_payload_length;
 		cry_cmd->source_count = htole16(dlen & 0xffff);
 		dlen >>= 16;
 		cry_cmd->masks = htole16(cmd->cry_masks |
 		    ((dlen << HIFN_CRYPT_CMD_SRCLEN_S) & HIFN_CRYPT_CMD_SRCLEN_M));
-		cry_cmd->header_skip = htole16(cmd->enccrd->crd_skip);
+		cry_cmd->header_skip = htole16(crp->crp_payload_length);
 		cry_cmd->reserved = 0;
 		buf_pos += sizeof(hifn_crypt_command_t);
 	}
 
 	if (using_mac && cmd->mac_masks & HIFN_MAC_CMD_NEW_KEY) {
 		bcopy(cmd->mac, buf_pos, HIFN_MAC_KEY_LENGTH);
 		buf_pos += HIFN_MAC_KEY_LENGTH;
 	}
 
 	if (using_crypt && cmd->cry_masks & HIFN_CRYPT_CMD_NEW_KEY) {
 		switch (cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) {
 		case HIFN_CRYPT_CMD_ALG_3DES:
 			bcopy(cmd->ck, buf_pos, HIFN_3DES_KEY_LENGTH);
 			buf_pos += HIFN_3DES_KEY_LENGTH;
 			break;
 		case HIFN_CRYPT_CMD_ALG_DES:
 			bcopy(cmd->ck, buf_pos, HIFN_DES_KEY_LENGTH);
 			buf_pos += HIFN_DES_KEY_LENGTH;
 			break;
 		case HIFN_CRYPT_CMD_ALG_RC4:
 			len = 256;
 			do {
 				int clen;
 
 				clen = MIN(cmd->cklen, len);
 				bcopy(cmd->ck, buf_pos, clen);
 				len -= clen;
 				buf_pos += clen;
 			} while (len > 0);
 			bzero(buf_pos, 4);
 			buf_pos += 4;
 			break;
 		case HIFN_CRYPT_CMD_ALG_AES:
 			/*
 			 * AES keys are variable 128, 192 and
 			 * 256 bits (16, 24 and 32 bytes).
 			 */
 			bcopy(cmd->ck, buf_pos, cmd->cklen);
 			buf_pos += cmd->cklen;
 			break;
 		}
 	}
 
 	if (using_crypt && cmd->cry_masks & HIFN_CRYPT_CMD_NEW_IV) {
 		switch (cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) {
 		case HIFN_CRYPT_CMD_ALG_AES:
 			ivlen = HIFN_AES_IV_LENGTH;
 			break;
 		default:
 			ivlen = HIFN_IV_LENGTH;
 			break;
 		}
 		bcopy(cmd->iv, buf_pos, ivlen);
 		buf_pos += ivlen;
 	}
 
 	if ((cmd->base_masks & (HIFN_BASE_CMD_MAC|HIFN_BASE_CMD_CRYPT)) == 0) {
 		bzero(buf_pos, 8);
 		buf_pos += 8;
 	}
 
 	return (buf_pos - buf);
 }
 
 static int
 hifn_dmamap_aligned(struct hifn_operand *op)
 {
 	int i;
 
 	for (i = 0; i < op->nsegs; i++) {
 		if (op->segs[i].ds_addr & 3)
 			return (0);
 		if ((i != (op->nsegs - 1)) && (op->segs[i].ds_len & 3))
 			return (0);
 	}
 	return (1);
 }
 
 static __inline int
 hifn_dmamap_dstwrap(struct hifn_softc *sc, int idx)
 {
 	struct hifn_dma *dma = sc->sc_dma;
 
 	if (++idx == HIFN_D_DST_RSIZE) {
 		dma->dstr[idx].l = htole32(HIFN_D_VALID | HIFN_D_JUMP |
 		    HIFN_D_MASKDONEIRQ);
 		HIFN_DSTR_SYNC(sc, idx,
 		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 		idx = 0;
 	}
 	return (idx);
 }
 
 static int
 hifn_dmamap_load_dst(struct hifn_softc *sc, struct hifn_command *cmd)
 {
 	struct hifn_dma *dma = sc->sc_dma;
 	struct hifn_operand *dst = &cmd->dst;
 	u_int32_t p, l;
 	int idx, used = 0, i;
 
 	idx = sc->sc_dsti;
 	for (i = 0; i < dst->nsegs - 1; i++) {
 		dma->dstr[idx].p = htole32(dst->segs[i].ds_addr);
 		dma->dstr[idx].l = htole32(HIFN_D_VALID |
 		    HIFN_D_MASKDONEIRQ | dst->segs[i].ds_len);
 		HIFN_DSTR_SYNC(sc, idx,
 		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 		used++;
 
 		idx = hifn_dmamap_dstwrap(sc, idx);
 	}
 
 	if (cmd->sloplen == 0) {
 		p = dst->segs[i].ds_addr;
 		l = HIFN_D_VALID | HIFN_D_MASKDONEIRQ | HIFN_D_LAST |
 		    dst->segs[i].ds_len;
 	} else {
 		p = sc->sc_dma_physaddr +
 		    offsetof(struct hifn_dma, slop[cmd->slopidx]);
 		l = HIFN_D_VALID | HIFN_D_MASKDONEIRQ | HIFN_D_LAST |
 		    sizeof(u_int32_t);
 
 		if ((dst->segs[i].ds_len - cmd->sloplen) != 0) {
 			dma->dstr[idx].p = htole32(dst->segs[i].ds_addr);
 			dma->dstr[idx].l = htole32(HIFN_D_VALID |
 			    HIFN_D_MASKDONEIRQ |
 			    (dst->segs[i].ds_len - cmd->sloplen));
 			HIFN_DSTR_SYNC(sc, idx,
 			    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 			used++;
 
 			idx = hifn_dmamap_dstwrap(sc, idx);
 		}
 	}
 	dma->dstr[idx].p = htole32(p);
 	dma->dstr[idx].l = htole32(l);
 	HIFN_DSTR_SYNC(sc, idx, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 	used++;
 
 	idx = hifn_dmamap_dstwrap(sc, idx);
 
 	sc->sc_dsti = idx;
 	sc->sc_dstu += used;
 	return (idx);
 }
 
 static __inline int
 hifn_dmamap_srcwrap(struct hifn_softc *sc, int idx)
 {
 	struct hifn_dma *dma = sc->sc_dma;
 
 	if (++idx == HIFN_D_SRC_RSIZE) {
 		dma->srcr[idx].l = htole32(HIFN_D_VALID |
 		    HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
 		HIFN_SRCR_SYNC(sc, HIFN_D_SRC_RSIZE,
 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
 		idx = 0;
 	}
 	return (idx);
 }
 
 static int
 hifn_dmamap_load_src(struct hifn_softc *sc, struct hifn_command *cmd)
 {
 	struct hifn_dma *dma = sc->sc_dma;
 	struct hifn_operand *src = &cmd->src;
 	int idx, i;
 	u_int32_t last = 0;
 
 	idx = sc->sc_srci;
 	for (i = 0; i < src->nsegs; i++) {
 		if (i == src->nsegs - 1)
 			last = HIFN_D_LAST;
 
 		dma->srcr[idx].p = htole32(src->segs[i].ds_addr);
 		dma->srcr[idx].l = htole32(src->segs[i].ds_len |
 		    HIFN_D_VALID | HIFN_D_MASKDONEIRQ | last);
 		HIFN_SRCR_SYNC(sc, idx,
 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
 
 		idx = hifn_dmamap_srcwrap(sc, idx);
 	}
 	sc->sc_srci = idx;
 	sc->sc_srcu += src->nsegs;
 	return (idx);
 } 
 
+static bus_size_t
+hifn_crp_length(struct cryptop *crp)
+{
+
+	switch (crp->crp_buf_type) {
+	case CRYPTO_BUF_MBUF:
+		return (crp->crp_mbuf->m_pkthdr.len);
+	case CRYPTO_BUF_UIO:
+		return (crp->crp_uio->uio_resid);
+	case CRYPTO_BUF_CONTIG:
+		return (crp->crp_ilen);
+	default:
+		panic("bad crp buffer type");
+	}
+}
+
 static void
-hifn_op_cb(void* arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error)
+hifn_op_cb(void* arg, bus_dma_segment_t *seg, int nsegs, int error)
 {
 	struct hifn_operand *op = arg;
 
 	KASSERT(nsegs <= MAX_SCATTER,
 		("hifn_op_cb: too many DMA segments (%u > %u) "
 		 "returned when mapping operand", nsegs, MAX_SCATTER));
-	op->mapsize = mapsize;
 	op->nsegs = nsegs;
 	bcopy(seg, op->segs, nsegs * sizeof (seg[0]));
 }
 
 static int 
 hifn_crypto(
 	struct hifn_softc *sc,
 	struct hifn_command *cmd,
 	struct cryptop *crp,
 	int hint)
 {
 	struct	hifn_dma *dma = sc->sc_dma;
 	u_int32_t cmdlen, csr;
 	int cmdi, resi, err = 0;
 
 	/*
 	 * need 1 cmd, and 1 res
 	 *
 	 * NB: check this first since it's easy.
 	 */
 	HIFN_LOCK(sc);
 	if ((sc->sc_cmdu + 1) > HIFN_D_CMD_RSIZE ||
 	    (sc->sc_resu + 1) > HIFN_D_RES_RSIZE) {
 #ifdef HIFN_DEBUG
 		if (hifn_debug) {
 			device_printf(sc->sc_dev,
 				"cmd/result exhaustion, cmdu %u resu %u\n",
 				sc->sc_cmdu, sc->sc_resu);
 		}
 #endif
 		hifnstats.hst_nomem_cr++;
 		HIFN_UNLOCK(sc);
 		return (ERESTART);
 	}
 
 	if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &cmd->src_map)) {
 		hifnstats.hst_nomem_map++;
 		HIFN_UNLOCK(sc);
 		return (ENOMEM);
 	}
 
-	if (crp->crp_flags & CRYPTO_F_IMBUF) {
-		if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->src_map,
-		    cmd->src_m, hifn_op_cb, &cmd->src, BUS_DMA_NOWAIT)) {
-			hifnstats.hst_nomem_load++;
-			err = ENOMEM;
-			goto err_srcmap1;
-		}
-	} else if (crp->crp_flags & CRYPTO_F_IOV) {
-		if (bus_dmamap_load_uio(sc->sc_dmat, cmd->src_map,
-		    cmd->src_io, hifn_op_cb, &cmd->src, BUS_DMA_NOWAIT)) {
-			hifnstats.hst_nomem_load++;
-			err = ENOMEM;
-			goto err_srcmap1;
-		}
-	} else {
-		err = EINVAL;
+	if (bus_dmamap_load_crp(sc->sc_dmat, cmd->src_map, crp, hifn_op_cb,
+	    &cmd->src, BUS_DMA_NOWAIT)) {
+		hifnstats.hst_nomem_load++;
+		err = ENOMEM;
 		goto err_srcmap1;
 	}
+	cmd->src_mapsize = hifn_crp_length(crp);
 
 	if (hifn_dmamap_aligned(&cmd->src)) {
 		cmd->sloplen = cmd->src_mapsize & 3;
 		cmd->dst = cmd->src;
-	} else {
-		if (crp->crp_flags & CRYPTO_F_IOV) {
-			err = EINVAL;
-			goto err_srcmap;
-		} else if (crp->crp_flags & CRYPTO_F_IMBUF) {
-			int totlen, len;
-			struct mbuf *m, *m0, *mlast;
+	} else if (crp->crp_buf_type == CRYPTO_BUF_MBUF) {
+		int totlen, len;
+		struct mbuf *m, *m0, *mlast;
 
-			KASSERT(cmd->dst_m == cmd->src_m,
-				("hifn_crypto: dst_m initialized improperly"));
-			hifnstats.hst_unaligned++;
-			/*
-			 * Source is not aligned on a longword boundary.
-			 * Copy the data to insure alignment.  If we fail
-			 * to allocate mbufs or clusters while doing this
-			 * we return ERESTART so the operation is requeued
-			 * at the crypto later, but only if there are
-			 * ops already posted to the hardware; otherwise we
-			 * have no guarantee that we'll be re-entered.
-			 */
-			totlen = cmd->src_mapsize;
-			if (cmd->src_m->m_flags & M_PKTHDR) {
-				len = MHLEN;
-				MGETHDR(m0, M_NOWAIT, MT_DATA);
-				if (m0 && !m_dup_pkthdr(m0, cmd->src_m, M_NOWAIT)) {
-					m_free(m0);
-					m0 = NULL;
-				}
-			} else {
-				len = MLEN;
-				MGET(m0, M_NOWAIT, MT_DATA);
+		KASSERT(cmd->dst_m == NULL,
+		    ("hifn_crypto: dst_m initialized improperly"));
+		hifnstats.hst_unaligned++;
+
+		/*
+		 * Source is not aligned on a longword boundary.
+		 * Copy the data to insure alignment.  If we fail
+		 * to allocate mbufs or clusters while doing this
+		 * we return ERESTART so the operation is requeued
+		 * at the crypto later, but only if there are
+		 * ops already posted to the hardware; otherwise we
+		 * have no guarantee that we'll be re-entered.
+		 */
+		totlen = cmd->src_mapsize;
+		if (crp->crp_mbuf->m_flags & M_PKTHDR) {
+			len = MHLEN;
+			MGETHDR(m0, M_NOWAIT, MT_DATA);
+			if (m0 && !m_dup_pkthdr(m0, crp->crp_mbuf, M_NOWAIT)) {
+				m_free(m0);
+				m0 = NULL;
 			}
-			if (m0 == NULL) {
+		} else {
+			len = MLEN;
+			MGET(m0, M_NOWAIT, MT_DATA);
+		}
+		if (m0 == NULL) {
+			hifnstats.hst_nomem_mbuf++;
+			err = sc->sc_cmdu ? ERESTART : ENOMEM;
+			goto err_srcmap;
+		}
+		if (totlen >= MINCLSIZE) {
+			if (!(MCLGET(m0, M_NOWAIT))) {
+				hifnstats.hst_nomem_mcl++;
+				err = sc->sc_cmdu ? ERESTART : ENOMEM;
+				m_freem(m0);
+				goto err_srcmap;
+			}
+			len = MCLBYTES;
+		}
+		totlen -= len;
+		m0->m_pkthdr.len = m0->m_len = len;
+		mlast = m0;
+
+		while (totlen > 0) {
+			MGET(m, M_NOWAIT, MT_DATA);
+			if (m == NULL) {
 				hifnstats.hst_nomem_mbuf++;
 				err = sc->sc_cmdu ? ERESTART : ENOMEM;
+				m_freem(m0);
 				goto err_srcmap;
 			}
+			len = MLEN;
 			if (totlen >= MINCLSIZE) {
-				if (!(MCLGET(m0, M_NOWAIT))) {
+				if (!(MCLGET(m, M_NOWAIT))) {
 					hifnstats.hst_nomem_mcl++;
 					err = sc->sc_cmdu ? ERESTART : ENOMEM;
+					mlast->m_next = m;
 					m_freem(m0);
 					goto err_srcmap;
 				}
 				len = MCLBYTES;
 			}
+
+			m->m_len = len;
+			m0->m_pkthdr.len += len;
 			totlen -= len;
-			m0->m_pkthdr.len = m0->m_len = len;
-			mlast = m0;
 
-			while (totlen > 0) {
-				MGET(m, M_NOWAIT, MT_DATA);
-				if (m == NULL) {
-					hifnstats.hst_nomem_mbuf++;
-					err = sc->sc_cmdu ? ERESTART : ENOMEM;
-					m_freem(m0);
-					goto err_srcmap;
-				}
-				len = MLEN;
-				if (totlen >= MINCLSIZE) {
-					if (!(MCLGET(m, M_NOWAIT))) {
-						hifnstats.hst_nomem_mcl++;
-						err = sc->sc_cmdu ? ERESTART : ENOMEM;
-						mlast->m_next = m;
-						m_freem(m0);
-						goto err_srcmap;
-					}
-					len = MCLBYTES;
-				}
-
-				m->m_len = len;
-				m0->m_pkthdr.len += len;
-				totlen -= len;
-
-				mlast->m_next = m;
-				mlast = m;
-			}
-			cmd->dst_m = m0;
+			mlast->m_next = m;
+			mlast = m;
 		}
-	}
+		cmd->dst_m = m0;
 
-	if (cmd->dst_map == NULL) {
-		if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &cmd->dst_map)) {
+		if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT,
+		    &cmd->dst_map)) {
 			hifnstats.hst_nomem_map++;
 			err = ENOMEM;
 			goto err_srcmap;
 		}
-		if (crp->crp_flags & CRYPTO_F_IMBUF) {
-			if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->dst_map,
-			    cmd->dst_m, hifn_op_cb, &cmd->dst, BUS_DMA_NOWAIT)) {
-				hifnstats.hst_nomem_map++;
-				err = ENOMEM;
-				goto err_dstmap1;
-			}
-		} else if (crp->crp_flags & CRYPTO_F_IOV) {
-			if (bus_dmamap_load_uio(sc->sc_dmat, cmd->dst_map,
-			    cmd->dst_io, hifn_op_cb, &cmd->dst, BUS_DMA_NOWAIT)) {
-				hifnstats.hst_nomem_load++;
-				err = ENOMEM;
-				goto err_dstmap1;
-			}
+
+		if (bus_dmamap_load_mbuf_sg(sc->sc_dmat, cmd->dst_map, m0,
+		    cmd->dst_segs, &cmd->dst_nsegs, 0)) {
+			hifnstats.hst_nomem_map++;
+			err = ENOMEM;
+			goto err_dstmap1;
 		}
+		cmd->dst_mapsize = m0->m_pkthdr.len;
+	} else {
+		err = EINVAL;
+		goto err_srcmap;
 	}
 
 #ifdef HIFN_DEBUG
 	if (hifn_debug) {
 		device_printf(sc->sc_dev,
 		    "Entering cmd: stat %8x ien %8x u %d/%d/%d/%d n %d/%d\n",
 		    READ_REG_1(sc, HIFN_1_DMA_CSR),
 		    READ_REG_1(sc, HIFN_1_DMA_IER),
 		    sc->sc_cmdu, sc->sc_srcu, sc->sc_dstu, sc->sc_resu,
 		    cmd->src_nsegs, cmd->dst_nsegs);
 	}
 #endif
 
 	if (cmd->src_map == cmd->dst_map) {
 		bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
 		    BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD);
 	} else {
 		bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
 		    BUS_DMASYNC_PREWRITE);
 		bus_dmamap_sync(sc->sc_dmat, cmd->dst_map,
 		    BUS_DMASYNC_PREREAD);
 	}
 
 	/*
 	 * need N src, and N dst
 	 */
 	if ((sc->sc_srcu + cmd->src_nsegs) > HIFN_D_SRC_RSIZE ||
 	    (sc->sc_dstu + cmd->dst_nsegs + 1) > HIFN_D_DST_RSIZE) {
 #ifdef HIFN_DEBUG
 		if (hifn_debug) {
 			device_printf(sc->sc_dev,
 				"src/dst exhaustion, srcu %u+%u dstu %u+%u\n",
 				sc->sc_srcu, cmd->src_nsegs,
 				sc->sc_dstu, cmd->dst_nsegs);
 		}
 #endif
 		hifnstats.hst_nomem_sd++;
 		err = ERESTART;
 		goto err_dstmap;
 	}
 
 	if (sc->sc_cmdi == HIFN_D_CMD_RSIZE) {
 		sc->sc_cmdi = 0;
 		dma->cmdr[HIFN_D_CMD_RSIZE].l = htole32(HIFN_D_VALID |
 		    HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
 		HIFN_CMDR_SYNC(sc, HIFN_D_CMD_RSIZE,
 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
 	}
 	cmdi = sc->sc_cmdi++;
 	cmdlen = hifn_write_command(cmd, dma->command_bufs[cmdi]);
 	HIFN_CMD_SYNC(sc, cmdi, BUS_DMASYNC_PREWRITE);
 
 	/* .p for command/result already set */
 	dma->cmdr[cmdi].l = htole32(cmdlen | HIFN_D_VALID | HIFN_D_LAST |
 	    HIFN_D_MASKDONEIRQ);
 	HIFN_CMDR_SYNC(sc, cmdi,
 	    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
 	sc->sc_cmdu++;
 
 	/*
 	 * We don't worry about missing an interrupt (which a "command wait"
 	 * interrupt salvages us from), unless there is more than one command
 	 * in the queue.
 	 */
 	if (sc->sc_cmdu > 1) {
 		sc->sc_dmaier |= HIFN_DMAIER_C_WAIT;
 		WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
 	}
 
 	hifnstats.hst_ipackets++;
 	hifnstats.hst_ibytes += cmd->src_mapsize;
 
 	hifn_dmamap_load_src(sc, cmd);
 
 	/*
 	 * Unlike other descriptors, we don't mask done interrupt from
 	 * result descriptor.
 	 */
 #ifdef HIFN_DEBUG
 	if (hifn_debug)
 		printf("load res\n");
 #endif
 	if (sc->sc_resi == HIFN_D_RES_RSIZE) {
 		sc->sc_resi = 0;
 		dma->resr[HIFN_D_RES_RSIZE].l = htole32(HIFN_D_VALID |
 		    HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
 		HIFN_RESR_SYNC(sc, HIFN_D_RES_RSIZE,
 		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 	}
 	resi = sc->sc_resi++;
 	KASSERT(sc->sc_hifn_commands[resi] == NULL,
 		("hifn_crypto: command slot %u busy", resi));
 	sc->sc_hifn_commands[resi] = cmd;
 	HIFN_RES_SYNC(sc, resi, BUS_DMASYNC_PREREAD);
 	if ((hint & CRYPTO_HINT_MORE) && sc->sc_curbatch < hifn_maxbatch) {
 		dma->resr[resi].l = htole32(HIFN_MAX_RESULT |
 		    HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ);
 		sc->sc_curbatch++;
 		if (sc->sc_curbatch > hifnstats.hst_maxbatch)
 			hifnstats.hst_maxbatch = sc->sc_curbatch;
 		hifnstats.hst_totbatch++;
 	} else {
 		dma->resr[resi].l = htole32(HIFN_MAX_RESULT |
 		    HIFN_D_VALID | HIFN_D_LAST);
 		sc->sc_curbatch = 0;
 	}
 	HIFN_RESR_SYNC(sc, resi,
 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 	sc->sc_resu++;
 
 	if (cmd->sloplen)
 		cmd->slopidx = resi;
 
 	hifn_dmamap_load_dst(sc, cmd);
 
 	csr = 0;
 	if (sc->sc_c_busy == 0) {
 		csr |= HIFN_DMACSR_C_CTRL_ENA;
 		sc->sc_c_busy = 1;
 	}
 	if (sc->sc_s_busy == 0) {
 		csr |= HIFN_DMACSR_S_CTRL_ENA;
 		sc->sc_s_busy = 1;
 	}
 	if (sc->sc_r_busy == 0) {
 		csr |= HIFN_DMACSR_R_CTRL_ENA;
 		sc->sc_r_busy = 1;
 	}
 	if (sc->sc_d_busy == 0) {
 		csr |= HIFN_DMACSR_D_CTRL_ENA;
 		sc->sc_d_busy = 1;
 	}
 	if (csr)
 		WRITE_REG_1(sc, HIFN_1_DMA_CSR, csr);
 
 #ifdef HIFN_DEBUG
 	if (hifn_debug) {
 		device_printf(sc->sc_dev, "command: stat %8x ier %8x\n",
 		    READ_REG_1(sc, HIFN_1_DMA_CSR),
 		    READ_REG_1(sc, HIFN_1_DMA_IER));
 	}
 #endif
 
 	sc->sc_active = 5;
 	HIFN_UNLOCK(sc);
 	KASSERT(err == 0, ("hifn_crypto: success with error %u", err));
 	return (err);		/* success */
 
 err_dstmap:
 	if (cmd->src_map != cmd->dst_map)
 		bus_dmamap_unload(sc->sc_dmat, cmd->dst_map);
 err_dstmap1:
 	if (cmd->src_map != cmd->dst_map)
 		bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map);
 err_srcmap:
-	if (crp->crp_flags & CRYPTO_F_IMBUF) {
-		if (cmd->src_m != cmd->dst_m)
+	if (crp->crp_buf_type == CRYPTO_BUF_MBUF) {
+		if (cmd->dst_m != NULL)
 			m_freem(cmd->dst_m);
 	}
 	bus_dmamap_unload(sc->sc_dmat, cmd->src_map);
 err_srcmap1:
 	bus_dmamap_destroy(sc->sc_dmat, cmd->src_map);
 	HIFN_UNLOCK(sc);
 	return (err);
 }
 
 static void
 hifn_tick(void* vsc)
 {
 	struct hifn_softc *sc = vsc;
 
 	HIFN_LOCK(sc);
 	if (sc->sc_active == 0) {
 		u_int32_t r = 0;
 
 		if (sc->sc_cmdu == 0 && sc->sc_c_busy) {
 			sc->sc_c_busy = 0;
 			r |= HIFN_DMACSR_C_CTRL_DIS;
 		}
 		if (sc->sc_srcu == 0 && sc->sc_s_busy) {
 			sc->sc_s_busy = 0;
 			r |= HIFN_DMACSR_S_CTRL_DIS;
 		}
 		if (sc->sc_dstu == 0 && sc->sc_d_busy) {
 			sc->sc_d_busy = 0;
 			r |= HIFN_DMACSR_D_CTRL_DIS;
 		}
 		if (sc->sc_resu == 0 && sc->sc_r_busy) {
 			sc->sc_r_busy = 0;
 			r |= HIFN_DMACSR_R_CTRL_DIS;
 		}
 		if (r)
 			WRITE_REG_1(sc, HIFN_1_DMA_CSR, r);
 	} else
 		sc->sc_active--;
 	HIFN_UNLOCK(sc);
 	callout_reset(&sc->sc_tickto, hz, hifn_tick, sc);
 }
 
 static void 
 hifn_intr(void *arg)
 {
 	struct hifn_softc *sc = arg;
 	struct hifn_dma *dma;
 	u_int32_t dmacsr, restart;
 	int i, u;
 
 	dmacsr = READ_REG_1(sc, HIFN_1_DMA_CSR);
 
 	/* Nothing in the DMA unit interrupted */
 	if ((dmacsr & sc->sc_dmaier) == 0)
 		return;
 
 	HIFN_LOCK(sc);
 
 	dma = sc->sc_dma;
 
 #ifdef HIFN_DEBUG
 	if (hifn_debug) {
 		device_printf(sc->sc_dev,
 		    "irq: stat %08x ien %08x damier %08x i %d/%d/%d/%d k %d/%d/%d/%d u %d/%d/%d/%d\n",
 		    dmacsr, READ_REG_1(sc, HIFN_1_DMA_IER), sc->sc_dmaier,
 		    sc->sc_cmdi, sc->sc_srci, sc->sc_dsti, sc->sc_resi,
 		    sc->sc_cmdk, sc->sc_srck, sc->sc_dstk, sc->sc_resk,
 		    sc->sc_cmdu, sc->sc_srcu, sc->sc_dstu, sc->sc_resu);
 	}
 #endif
 
 	WRITE_REG_1(sc, HIFN_1_DMA_CSR, dmacsr & sc->sc_dmaier);
 
 	if ((sc->sc_flags & HIFN_HAS_PUBLIC) &&
 	    (dmacsr & HIFN_DMACSR_PUBDONE))
 		WRITE_REG_1(sc, HIFN_1_PUB_STATUS,
 		    READ_REG_1(sc, HIFN_1_PUB_STATUS) | HIFN_PUBSTS_DONE);
 
 	restart = dmacsr & (HIFN_DMACSR_D_OVER | HIFN_DMACSR_R_OVER);
 	if (restart)
 		device_printf(sc->sc_dev, "overrun %x\n", dmacsr);
 
 	if (sc->sc_flags & HIFN_IS_7811) {
 		if (dmacsr & HIFN_DMACSR_ILLR)
 			device_printf(sc->sc_dev, "illegal read\n");
 		if (dmacsr & HIFN_DMACSR_ILLW)
 			device_printf(sc->sc_dev, "illegal write\n");
 	}
 
 	restart = dmacsr & (HIFN_DMACSR_C_ABORT | HIFN_DMACSR_S_ABORT |
 	    HIFN_DMACSR_D_ABORT | HIFN_DMACSR_R_ABORT);
 	if (restart) {
 		device_printf(sc->sc_dev, "abort, resetting.\n");
 		hifnstats.hst_abort++;
 		hifn_abort(sc);
 		HIFN_UNLOCK(sc);
 		return;
 	}
 
 	if ((dmacsr & HIFN_DMACSR_C_WAIT) && (sc->sc_cmdu == 0)) {
 		/*
 		 * If no slots to process and we receive a "waiting on
 		 * command" interrupt, we disable the "waiting on command"
 		 * (by clearing it).
 		 */
 		sc->sc_dmaier &= ~HIFN_DMAIER_C_WAIT;
 		WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
 	}
 
 	/* clear the rings */
 	i = sc->sc_resk; u = sc->sc_resu;
 	while (u != 0) {
 		HIFN_RESR_SYNC(sc, i,
 		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 		if (dma->resr[i].l & htole32(HIFN_D_VALID)) {
 			HIFN_RESR_SYNC(sc, i,
 			    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 			break;
 		}
 
 		if (i != HIFN_D_RES_RSIZE) {
 			struct hifn_command *cmd;
 			u_int8_t *macbuf = NULL;
 
 			HIFN_RES_SYNC(sc, i, BUS_DMASYNC_POSTREAD);
 			cmd = sc->sc_hifn_commands[i];
 			KASSERT(cmd != NULL,
 				("hifn_intr: null command slot %u", i));
 			sc->sc_hifn_commands[i] = NULL;
 
 			if (cmd->base_masks & HIFN_BASE_CMD_MAC) {
 				macbuf = dma->result_bufs[i];
 				macbuf += 12;
 			}
 
 			hifn_callback(sc, cmd, macbuf);
 			hifnstats.hst_opackets++;
 			u--;
 		}
 
 		if (++i == (HIFN_D_RES_RSIZE + 1))
 			i = 0;
 	}
 	sc->sc_resk = i; sc->sc_resu = u;
 
 	i = sc->sc_srck; u = sc->sc_srcu;
 	while (u != 0) {
 		if (i == HIFN_D_SRC_RSIZE)
 			i = 0;
 		HIFN_SRCR_SYNC(sc, i,
 		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 		if (dma->srcr[i].l & htole32(HIFN_D_VALID)) {
 			HIFN_SRCR_SYNC(sc, i,
 			    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 			break;
 		}
 		i++, u--;
 	}
 	sc->sc_srck = i; sc->sc_srcu = u;
 
 	i = sc->sc_cmdk; u = sc->sc_cmdu;
 	while (u != 0) {
 		HIFN_CMDR_SYNC(sc, i,
 		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 		if (dma->cmdr[i].l & htole32(HIFN_D_VALID)) {
 			HIFN_CMDR_SYNC(sc, i,
 			    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 			break;
 		}
 		if (i != HIFN_D_CMD_RSIZE) {
 			u--;
 			HIFN_CMD_SYNC(sc, i, BUS_DMASYNC_POSTWRITE);
 		}
 		if (++i == (HIFN_D_CMD_RSIZE + 1))
 			i = 0;
 	}
 	sc->sc_cmdk = i; sc->sc_cmdu = u;
 
 	HIFN_UNLOCK(sc);
 
 	if (sc->sc_needwakeup) {		/* XXX check high watermark */
 		int wakeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ);
 #ifdef HIFN_DEBUG
 		if (hifn_debug)
 			device_printf(sc->sc_dev,
 				"wakeup crypto (%x) u %d/%d/%d/%d\n",
 				sc->sc_needwakeup,
 				sc->sc_cmdu, sc->sc_srcu, sc->sc_dstu, sc->sc_resu);
 #endif
 		sc->sc_needwakeup &= ~wakeup;
 		crypto_unblock(sc->sc_cid, wakeup);
 	}
 }
 
-/*
- * Allocate a new 'session' and return an encoded session id.  'sidp'
- * contains our registration id, and should contain an encoded session
- * id on successful allocation.
- */
-static int
-hifn_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
+static bool
+hifn_auth_supported(struct hifn_softc *sc,
+    const struct crypto_session_params *csp)
 {
-	struct hifn_softc *sc = device_get_softc(dev);
-	struct cryptoini *c;
-	int mac = 0, cry = 0;
-	struct hifn_session *ses;
 
-	KASSERT(sc != NULL, ("hifn_newsession: null softc"));
-	if (cri == NULL || sc == NULL)
-		return (EINVAL);
+	switch (sc->sc_ena) {
+	case HIFN_PUSTAT_ENA_2:
+	case HIFN_PUSTAT_ENA_1:
+		break;
+	default:
+		return (false);
+	}
+		
+	switch (csp->csp_auth_alg) {
+	case CRYPTO_MD5:
+	case CRYPTO_SHA1:
+		break;
+	case CRYPTO_MD5_HMAC:
+	case CRYPTO_SHA1_HMAC:
+		if (csp->csp_auth_klen > HIFN_MAC_KEY_LENGTH)
+			return (false);
+		break;
+	default:
+		return (false);
+	}
 
-	ses = crypto_get_driver_session(cses);
+	return (true);	
+}
 
-	for (c = cri; c != NULL; c = c->cri_next) {
-		switch (c->cri_alg) {
-		case CRYPTO_MD5:
-		case CRYPTO_SHA1:
-		case CRYPTO_MD5_HMAC:
-		case CRYPTO_SHA1_HMAC:
-			if (mac)
-				return (EINVAL);
-			mac = 1;
-			ses->hs_mlen = c->cri_mlen;
-			if (ses->hs_mlen == 0) {
-				switch (c->cri_alg) {
-				case CRYPTO_MD5:
-				case CRYPTO_MD5_HMAC:
-					ses->hs_mlen = 16;
-					break;
-				case CRYPTO_SHA1:
-				case CRYPTO_SHA1_HMAC:
-					ses->hs_mlen = 20;
-					break;
-				}
-			}
-			break;
-		case CRYPTO_DES_CBC:
+static bool
+hifn_cipher_supported(struct hifn_softc *sc,
+    const struct crypto_session_params *csp)
+{
+
+	if (csp->csp_cipher_klen == 0)
+		return (false);
+	if (csp->csp_ivlen > HIFN_MAX_IV_LENGTH)
+		return (false);
+	switch (sc->sc_ena) {
+	case HIFN_PUSTAT_ENA_2:
+		switch (csp->csp_cipher_alg) {
 		case CRYPTO_3DES_CBC:
-		case CRYPTO_AES_CBC:
-			/* XXX this may read fewer, does it matter? */
-			read_random(ses->hs_iv,
-				c->cri_alg == CRYPTO_AES_CBC ?
-					HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH);
-			/*FALLTHROUGH*/
 		case CRYPTO_ARC4:
-			if (cry)
-				return (EINVAL);
-			cry = 1;
 			break;
-		default:
-			return (EINVAL);
+		case CRYPTO_AES_CBC:
+			if ((sc->sc_flags & HIFN_HAS_AES) == 0)
+				return (false);
+			switch (csp->csp_cipher_klen) {
+			case 128:
+			case 192:
+			case 256:
+				break;
+			default:
+				return (false);
+			}
+			return (true);
 		}
+		/*FALLTHROUGH*/
+	case HIFN_PUSTAT_ENA_1:
+		switch (csp->csp_cipher_alg) {
+		case CRYPTO_DES_CBC:
+			return (true);
+		}
+		break;
 	}
-	if (mac == 0 && cry == 0)
+	return (false);
+}
+
+static int
+hifn_probesession(device_t dev, const struct crypto_session_params *csp)
+{
+	struct hifn_softc *sc;
+
+	sc = device_get_softc(dev);
+	if (csp->csp_flags != 0)
 		return (EINVAL);
+	switch (csp->csp_mode) {
+	case CSP_MODE_DIGEST:
+		if (!hifn_auth_supported(sc, csp))
+			return (EINVAL);
+		break;
+	case CSP_MODE_CIPHER:
+		if (!hifn_cipher_supported(sc, csp))
+			return (EINVAL);
+		break;
+	case CSP_MODE_ETA:
+		if (!hifn_auth_supported(sc, csp) ||
+		    !hifn_cipher_supported(sc, csp))
+			return (EINVAL);
+		break;
+	default:
+		return (EINVAL);
+	}
+
+	return (CRYPTODEV_PROBE_HARDWARE);
+}
+
+/*
+ * Allocate a new 'session'.
+ */
+static int
+hifn_newsession(device_t dev, crypto_session_t cses,
+    const struct crypto_session_params *csp)
+{
+	struct hifn_session *ses;
+
+	ses = crypto_get_driver_session(cses);
+
+	if (csp->csp_auth_alg != 0) {
+		if (csp->csp_auth_mlen == 0)
+			ses->hs_mlen = crypto_auth_hash(csp)->hashsize;
+		else
+			ses->hs_mlen = csp->csp_auth_mlen;
+	}
+
 	return (0);
 }
 
 /*
  * XXX freesession routine should run a zero'd mac/encrypt key into context
  * ram.  to blow away any keys already stored there.
  */
 
 static int
 hifn_process(device_t dev, struct cryptop *crp, int hint)
 {
+	const struct crypto_session_params *csp;
 	struct hifn_softc *sc = device_get_softc(dev);
 	struct hifn_command *cmd = NULL;
-	int err, ivlen;
-	struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
+	const void *mackey;
+	int err, ivlen, keylen;
 	struct hifn_session *ses;
 
-	if (crp == NULL || crp->crp_callback == NULL) {
-		hifnstats.hst_invalid++;
-		return (EINVAL);
-	}
-
 	ses = crypto_get_driver_session(crp->crp_session);
+
 	cmd = malloc(sizeof(struct hifn_command), M_DEVBUF, M_NOWAIT | M_ZERO);
 	if (cmd == NULL) {
 		hifnstats.hst_nomem++;
 		err = ENOMEM;
 		goto errout;
 	}
 
-	if (crp->crp_flags & CRYPTO_F_IMBUF) {
-		cmd->src_m = (struct mbuf *)crp->crp_buf;
-		cmd->dst_m = (struct mbuf *)crp->crp_buf;
-	} else if (crp->crp_flags & CRYPTO_F_IOV) {
-		cmd->src_io = (struct uio *)crp->crp_buf;
-		cmd->dst_io = (struct uio *)crp->crp_buf;
-	} else {
-		err = EINVAL;
-		goto errout;	/* XXX we don't handle contiguous buffers! */
-	}
+	csp = crypto_get_params(crp->crp_session);
 
-	crd1 = crp->crp_desc;
-	if (crd1 == NULL) {
+	/*
+	 * The driver only supports ETA requests where there is no
+	 * gap between the AAD and payload.
+	 */
+	if (csp->csp_mode == CSP_MODE_ETA && crp->crp_aad_length != 0 &&
+	    crp->crp_aad_start + crp->crp_aad_length !=
+	    crp->crp_payload_start) {
 		err = EINVAL;
 		goto errout;
 	}
-	crd2 = crd1->crd_next;
 
-	if (crd2 == NULL) {
-		if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
-		    crd1->crd_alg == CRYPTO_SHA1_HMAC ||
-		    crd1->crd_alg == CRYPTO_SHA1 ||
-		    crd1->crd_alg == CRYPTO_MD5) {
-			maccrd = crd1;
-			enccrd = NULL;
-		} else if (crd1->crd_alg == CRYPTO_DES_CBC ||
-		    crd1->crd_alg == CRYPTO_3DES_CBC ||
-		    crd1->crd_alg == CRYPTO_AES_CBC ||
-		    crd1->crd_alg == CRYPTO_ARC4) {
-			if ((crd1->crd_flags & CRD_F_ENCRYPT) == 0)
-				cmd->base_masks |= HIFN_BASE_CMD_DECODE;
-			maccrd = NULL;
-			enccrd = crd1;
-		} else {
-			err = EINVAL;
-			goto errout;
-		}
-	} else {
-		if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
-                     crd1->crd_alg == CRYPTO_SHA1_HMAC ||
-                     crd1->crd_alg == CRYPTO_MD5 ||
-                     crd1->crd_alg == CRYPTO_SHA1) &&
-		    (crd2->crd_alg == CRYPTO_DES_CBC ||
-		     crd2->crd_alg == CRYPTO_3DES_CBC ||
-		     crd2->crd_alg == CRYPTO_AES_CBC ||
-		     crd2->crd_alg == CRYPTO_ARC4) &&
-		    ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
-			cmd->base_masks = HIFN_BASE_CMD_DECODE;
-			maccrd = crd1;
-			enccrd = crd2;
-		} else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
-		     crd1->crd_alg == CRYPTO_ARC4 ||
-		     crd1->crd_alg == CRYPTO_3DES_CBC ||
-		     crd1->crd_alg == CRYPTO_AES_CBC) &&
-		    (crd2->crd_alg == CRYPTO_MD5_HMAC ||
-                     crd2->crd_alg == CRYPTO_SHA1_HMAC ||
-                     crd2->crd_alg == CRYPTO_MD5 ||
-                     crd2->crd_alg == CRYPTO_SHA1) &&
-		    (crd1->crd_flags & CRD_F_ENCRYPT)) {
-			enccrd = crd1;
-			maccrd = crd2;
-		} else {
-			/*
-			 * We cannot order the 7751 as requested
-			 */
-			err = EINVAL;
-			goto errout;
-		}
-	}
-
-	if (enccrd) {
-		cmd->enccrd = enccrd;
+	switch (csp->csp_mode) {
+	case CSP_MODE_CIPHER:
+	case CSP_MODE_ETA:
+		if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
+			cmd->base_masks |= HIFN_BASE_CMD_DECODE;
 		cmd->base_masks |= HIFN_BASE_CMD_CRYPT;
-		switch (enccrd->crd_alg) {
+		switch (csp->csp_cipher_alg) {
 		case CRYPTO_ARC4:
 			cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_RC4;
 			break;
 		case CRYPTO_DES_CBC:
 			cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_DES |
 			    HIFN_CRYPT_CMD_MODE_CBC |
 			    HIFN_CRYPT_CMD_NEW_IV;
 			break;
 		case CRYPTO_3DES_CBC:
 			cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_3DES |
 			    HIFN_CRYPT_CMD_MODE_CBC |
 			    HIFN_CRYPT_CMD_NEW_IV;
 			break;
 		case CRYPTO_AES_CBC:
 			cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_AES |
 			    HIFN_CRYPT_CMD_MODE_CBC |
 			    HIFN_CRYPT_CMD_NEW_IV;
 			break;
 		default:
 			err = EINVAL;
 			goto errout;
 		}
-		if (enccrd->crd_alg != CRYPTO_ARC4) {
-			ivlen = ((enccrd->crd_alg == CRYPTO_AES_CBC) ?
-				HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH);
-			if (enccrd->crd_flags & CRD_F_ENCRYPT) {
-				if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
-					bcopy(enccrd->crd_iv, cmd->iv, ivlen);
-				else
-					bcopy(ses->hs_iv, cmd->iv, ivlen);
-
-				if ((enccrd->crd_flags & CRD_F_IV_PRESENT)
-				    == 0) {
-					crypto_copyback(crp->crp_flags,
-					    crp->crp_buf, enccrd->crd_inject,
-					    ivlen, cmd->iv);
-				}
-			} else {
-				if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
-					bcopy(enccrd->crd_iv, cmd->iv, ivlen);
-				else {
-					crypto_copydata(crp->crp_flags,
-					    crp->crp_buf, enccrd->crd_inject,
-					    ivlen, cmd->iv);
-				}
-			}
+		if (csp->csp_cipher_alg != CRYPTO_ARC4) {
+			ivlen = csp->csp_ivlen;
+			if (crp->crp_flags & CRYPTO_F_IV_GENERATE) {
+				arc4rand(cmd->iv, ivlen, 0);
+				crypto_copyback(crp, crp->crp_iv_start, ivlen,
+				    cmd->iv);
+			} else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+				memcpy(cmd->iv, crp->crp_iv, ivlen);
+			else
+				crypto_copydata(crp, crp->crp_iv_start, ivlen,
+				    cmd->iv);
 		}
 
-		if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT)
-			cmd->cry_masks |= HIFN_CRYPT_CMD_NEW_KEY;
-		cmd->ck = enccrd->crd_key;
-		cmd->cklen = enccrd->crd_klen >> 3;
+		if (crp->crp_cipher_key != NULL)
+			cmd->ck = crp->crp_cipher_key;
+		else
+			cmd->ck = csp->csp_cipher_key;
+		cmd->cklen = csp->csp_cipher_klen;
 		cmd->cry_masks |= HIFN_CRYPT_CMD_NEW_KEY;
 
 		/* 
 		 * Need to specify the size for the AES key in the masks.
 		 */
 		if ((cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) ==
 		    HIFN_CRYPT_CMD_ALG_AES) {
 			switch (cmd->cklen) {
 			case 16:
 				cmd->cry_masks |= HIFN_CRYPT_CMD_KSZ_128;
 				break;
 			case 24:
 				cmd->cry_masks |= HIFN_CRYPT_CMD_KSZ_192;
 				break;
 			case 32:
 				cmd->cry_masks |= HIFN_CRYPT_CMD_KSZ_256;
 				break;
 			default:
 				err = EINVAL;
 				goto errout;
 			}
 		}
+		break;
 	}
 
-	if (maccrd) {
-		cmd->maccrd = maccrd;
+	switch (csp->csp_mode) {
+	case CSP_MODE_DIGEST:
+	case CSP_MODE_ETA:
 		cmd->base_masks |= HIFN_BASE_CMD_MAC;
 
-		switch (maccrd->crd_alg) {
+		switch (csp->csp_auth_alg) {
 		case CRYPTO_MD5:
 			cmd->mac_masks |= HIFN_MAC_CMD_ALG_MD5 |
 			    HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HASH |
 			    HIFN_MAC_CMD_POS_IPSEC;
                        break;
 		case CRYPTO_MD5_HMAC:
 			cmd->mac_masks |= HIFN_MAC_CMD_ALG_MD5 |
 			    HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HMAC |
 			    HIFN_MAC_CMD_POS_IPSEC | HIFN_MAC_CMD_TRUNC;
 			break;
 		case CRYPTO_SHA1:
 			cmd->mac_masks |= HIFN_MAC_CMD_ALG_SHA1 |
 			    HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HASH |
 			    HIFN_MAC_CMD_POS_IPSEC;
 			break;
 		case CRYPTO_SHA1_HMAC:
 			cmd->mac_masks |= HIFN_MAC_CMD_ALG_SHA1 |
 			    HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HMAC |
 			    HIFN_MAC_CMD_POS_IPSEC | HIFN_MAC_CMD_TRUNC;
 			break;
 		}
 
-		if (maccrd->crd_alg == CRYPTO_SHA1_HMAC ||
-		     maccrd->crd_alg == CRYPTO_MD5_HMAC) {
+		if (csp->csp_auth_alg == CRYPTO_SHA1_HMAC ||
+		    csp->csp_auth_alg == CRYPTO_MD5_HMAC) {
 			cmd->mac_masks |= HIFN_MAC_CMD_NEW_KEY;
-			bcopy(maccrd->crd_key, cmd->mac, maccrd->crd_klen >> 3);
-			bzero(cmd->mac + (maccrd->crd_klen >> 3),
-			    HIFN_MAC_KEY_LENGTH - (maccrd->crd_klen >> 3));
+			if (crp->crp_auth_key != NULL)
+				mackey = crp->crp_auth_key;
+			else
+				mackey = csp->csp_auth_key;
+			keylen = csp->csp_auth_klen;
+			bcopy(mackey, cmd->mac, keylen);
+			bzero(cmd->mac + keylen, HIFN_MAC_KEY_LENGTH - keylen);
 		}
 	}
 
 	cmd->crp = crp;
 	cmd->session = ses;
 	cmd->softc = sc;
 
 	err = hifn_crypto(sc, cmd, crp, hint);
 	if (!err) {
 		return 0;
 	} else if (err == ERESTART) {
 		/*
 		 * There weren't enough resources to dispatch the request
 		 * to the part.  Notify the caller so they'll requeue this
 		 * request and resubmit it again soon.
 		 */
 #ifdef HIFN_DEBUG
 		if (hifn_debug)
 			device_printf(sc->sc_dev, "requeue request\n");
 #endif
 		free(cmd, M_DEVBUF);
 		sc->sc_needwakeup |= CRYPTO_SYMQ;
 		return (err);
 	}
 
 errout:
 	if (cmd != NULL)
 		free(cmd, M_DEVBUF);
 	if (err == EINVAL)
 		hifnstats.hst_invalid++;
 	else
 		hifnstats.hst_nomem++;
 	crp->crp_etype = err;
 	crypto_done(crp);
 	return (err);
 }
 
 static void
 hifn_abort(struct hifn_softc *sc)
 {
 	struct hifn_dma *dma = sc->sc_dma;
 	struct hifn_command *cmd;
 	struct cryptop *crp;
 	int i, u;
 
 	i = sc->sc_resk; u = sc->sc_resu;
 	while (u != 0) {
 		cmd = sc->sc_hifn_commands[i];
 		KASSERT(cmd != NULL, ("hifn_abort: null command slot %u", i));
 		sc->sc_hifn_commands[i] = NULL;
 		crp = cmd->crp;
 
 		if ((dma->resr[i].l & htole32(HIFN_D_VALID)) == 0) {
 			/* Salvage what we can. */
 			u_int8_t *macbuf;
 
 			if (cmd->base_masks & HIFN_BASE_CMD_MAC) {
 				macbuf = dma->result_bufs[i];
 				macbuf += 12;
 			} else
 				macbuf = NULL;
 			hifnstats.hst_opackets++;
 			hifn_callback(sc, cmd, macbuf);
 		} else {
 			if (cmd->src_map == cmd->dst_map) {
 				bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
 				    BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
 			} else {
 				bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
 				    BUS_DMASYNC_POSTWRITE);
 				bus_dmamap_sync(sc->sc_dmat, cmd->dst_map,
 				    BUS_DMASYNC_POSTREAD);
 			}
 
-			if (cmd->src_m != cmd->dst_m) {
-				m_freem(cmd->src_m);
-				crp->crp_buf = (caddr_t)cmd->dst_m;
+			if (cmd->dst_m != NULL) {
+				m_freem(cmd->dst_m);
 			}
 
 			/* non-shared buffers cannot be restarted */
 			if (cmd->src_map != cmd->dst_map) {
 				/*
 				 * XXX should be EAGAIN, delayed until
 				 * after the reset.
 				 */
 				crp->crp_etype = ENOMEM;
 				bus_dmamap_unload(sc->sc_dmat, cmd->dst_map);
 				bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map);
 			} else
 				crp->crp_etype = ENOMEM;
 
 			bus_dmamap_unload(sc->sc_dmat, cmd->src_map);
 			bus_dmamap_destroy(sc->sc_dmat, cmd->src_map);
 
 			free(cmd, M_DEVBUF);
 			if (crp->crp_etype != EAGAIN)
 				crypto_done(crp);
 		}
 
 		if (++i == HIFN_D_RES_RSIZE)
 			i = 0;
 		u--;
 	}
 	sc->sc_resk = i; sc->sc_resu = u;
 
 	hifn_reset_board(sc, 1);
 	hifn_init_dma(sc);
 	hifn_init_pci_registers(sc);
 }
 
 static void
 hifn_callback(struct hifn_softc *sc, struct hifn_command *cmd, u_int8_t *macbuf)
 {
 	struct hifn_dma *dma = sc->sc_dma;
 	struct cryptop *crp = cmd->crp;
-	struct cryptodesc *crd;
+	uint8_t macbuf2[SHA1_HASH_LEN];
 	struct mbuf *m;
-	int totlen, i, u, ivlen;
+	int totlen, i, u;
 
 	if (cmd->src_map == cmd->dst_map) {
 		bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
 		    BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
 	} else {
 		bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
 		    BUS_DMASYNC_POSTWRITE);
 		bus_dmamap_sync(sc->sc_dmat, cmd->dst_map,
 		    BUS_DMASYNC_POSTREAD);
 	}
 
-	if (crp->crp_flags & CRYPTO_F_IMBUF) {
-		if (cmd->src_m != cmd->dst_m) {
-			crp->crp_buf = (caddr_t)cmd->dst_m;
+	if (crp->crp_buf_type == CRYPTO_BUF_MBUF) {
+		if (cmd->dst_m != NULL) {
 			totlen = cmd->src_mapsize;
 			for (m = cmd->dst_m; m != NULL; m = m->m_next) {
 				if (totlen < m->m_len) {
 					m->m_len = totlen;
 					totlen = 0;
 				} else
 					totlen -= m->m_len;
 			}
-			cmd->dst_m->m_pkthdr.len = cmd->src_m->m_pkthdr.len;
-			m_freem(cmd->src_m);
+			cmd->dst_m->m_pkthdr.len = crp->crp_mbuf->m_pkthdr.len;
+			m_freem(crp->crp_mbuf);
+			crp->crp_mbuf = cmd->dst_m;
 		}
 	}
 
 	if (cmd->sloplen != 0) {
-		crypto_copyback(crp->crp_flags, crp->crp_buf,
-		    cmd->src_mapsize - cmd->sloplen, cmd->sloplen,
-		    (caddr_t)&dma->slop[cmd->slopidx]);
+		crypto_copyback(crp, cmd->src_mapsize - cmd->sloplen,
+		    cmd->sloplen, &dma->slop[cmd->slopidx]);
 	}
 
 	i = sc->sc_dstk; u = sc->sc_dstu;
 	while (u != 0) {
 		if (i == HIFN_D_DST_RSIZE)
 			i = 0;
 		bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
 		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 		if (dma->dstr[i].l & htole32(HIFN_D_VALID)) {
 			bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
 			    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 			break;
 		}
 		i++, u--;
 	}
 	sc->sc_dstk = i; sc->sc_dstu = u;
 
 	hifnstats.hst_obytes += cmd->dst_mapsize;
 
-	if ((cmd->base_masks & (HIFN_BASE_CMD_CRYPT | HIFN_BASE_CMD_DECODE)) ==
-	    HIFN_BASE_CMD_CRYPT) {
-		for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
-			if (crd->crd_alg != CRYPTO_DES_CBC &&
-			    crd->crd_alg != CRYPTO_3DES_CBC &&
-			    crd->crd_alg != CRYPTO_AES_CBC)
-				continue;
-			ivlen = ((crd->crd_alg == CRYPTO_AES_CBC) ?
-				HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH);
-			crypto_copydata(crp->crp_flags, crp->crp_buf,
-			    crd->crd_skip + crd->crd_len - ivlen, ivlen,
-			    cmd->session->hs_iv);
-			break;
-		}
-	}
-
 	if (macbuf != NULL) {
-		for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
-                        int len;
-
-			if (crd->crd_alg != CRYPTO_MD5 &&
-			    crd->crd_alg != CRYPTO_SHA1 &&
-			    crd->crd_alg != CRYPTO_MD5_HMAC &&
-			    crd->crd_alg != CRYPTO_SHA1_HMAC) {
-				continue;
-			}
-			len = cmd->session->hs_mlen;
-			crypto_copyback(crp->crp_flags, crp->crp_buf,
-			    crd->crd_inject, len, macbuf);
-			break;
-		}
+		if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
+			crypto_copydata(crp, crp->crp_digest_start,
+			    cmd->session->hs_mlen, macbuf2);
+			if (timingsafe_bcmp(macbuf, macbuf2,
+			    cmd->session->hs_mlen) != 0)
+				crp->crp_etype = EBADMSG;
+		} else
+			crypto_copyback(crp, crp->crp_digest_start,
+			    cmd->session->hs_mlen, macbuf);
 	}
 
 	if (cmd->src_map != cmd->dst_map) {
 		bus_dmamap_unload(sc->sc_dmat, cmd->dst_map);
 		bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map);
 	}
 	bus_dmamap_unload(sc->sc_dmat, cmd->src_map);
 	bus_dmamap_destroy(sc->sc_dmat, cmd->src_map);
 	free(cmd, M_DEVBUF);
 	crypto_done(crp);
 }
 
 /*
  * 7811 PB3 rev/2 parts lock-up on burst writes to Group 0
  * and Group 1 registers; avoid conditions that could create
  * burst writes by doing a read in between the writes.
  *
  * NB: The read we interpose is always to the same register;
  *     we do this because reading from an arbitrary (e.g. last)
  *     register may not always work.
  */
 static void
 hifn_write_reg_0(struct hifn_softc *sc, bus_size_t reg, u_int32_t val)
 {
 	if (sc->sc_flags & HIFN_IS_7811) {
 		if (sc->sc_bar0_lastreg == reg - 4)
 			bus_space_read_4(sc->sc_st0, sc->sc_sh0, HIFN_0_PUCNFG);
 		sc->sc_bar0_lastreg = reg;
 	}
 	bus_space_write_4(sc->sc_st0, sc->sc_sh0, reg, val);
 }
 
 static void
 hifn_write_reg_1(struct hifn_softc *sc, bus_size_t reg, u_int32_t val)
 {
 	if (sc->sc_flags & HIFN_IS_7811) {
 		if (sc->sc_bar1_lastreg == reg - 4)
 			bus_space_read_4(sc->sc_st1, sc->sc_sh1, HIFN_1_REVID);
 		sc->sc_bar1_lastreg = reg;
 	}
 	bus_space_write_4(sc->sc_st1, sc->sc_sh1, reg, val);
 }
 
 #ifdef HIFN_VULCANDEV
 /*
  * this code provides support for mapping the PK engine's register
  * into a userspace program.
  *
  */
 static int
 vulcanpk_mmap(struct cdev *dev, vm_ooffset_t offset,
 	      vm_paddr_t *paddr, int nprot, vm_memattr_t *memattr)
 {
 	struct hifn_softc *sc;
 	vm_paddr_t pd;
 	void *b;
 
 	sc = dev->si_drv1;
 
 	pd = rman_get_start(sc->sc_bar1res);
 	b = rman_get_virtual(sc->sc_bar1res);
 
 #if 0
 	printf("vpk mmap: %p(%016llx) offset=%lld\n", b,
 	    (unsigned long long)pd, offset);
 	hexdump(b, HIFN_1_PUB_MEMEND, "vpk", 0);
 #endif
 
 	if (offset == 0) {
 		*paddr = pd;
 		return (0);
 	}
 	return (-1);
 }
 
 static struct cdevsw vulcanpk_cdevsw = {
 	.d_version =	D_VERSION,
 	.d_mmap =	vulcanpk_mmap,
 	.d_name =	"vulcanpk",
 };
 #endif /* HIFN_VULCANDEV */
Index: head/sys/dev/hifn/hifn7751var.h
===================================================================
--- head/sys/dev/hifn/hifn7751var.h	(revision 359373)
+++ head/sys/dev/hifn/hifn7751var.h	(revision 359374)
@@ -1,354 +1,347 @@
 /* $FreeBSD$ */
 /*	$OpenBSD: hifn7751var.h,v 1.42 2002/04/08 17:49:42 jason Exp $	*/
 
 /*-
  * SPDX-License-Identifier: BSD-3-Clause
  *
  * Invertex AEON / Hifn 7751 driver
  * Copyright (c) 1999 Invertex Inc. All rights reserved.
  * Copyright (c) 1999 Theo de Raadt
  * Copyright (c) 2000-2001 Network Security Technologies, Inc.
  *			http://www.netsec.net
  *
  * Please send any comments, feedback, bug-fixes, or feature requests to
  * software@invertex.com.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  *
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  * 3. The name of the author may not be used to endorse or promote products
  *    derived from this software without specific prior written permission.
  *
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  * Effort sponsored in part by the Defense Advanced Research Projects
  * Agency (DARPA) and Air Force Research Laboratory, Air Force
  * Materiel Command, USAF, under agreement number F30602-01-2-0537.
  *
  */
 
 #ifndef __HIFN7751VAR_H__
 #define __HIFN7751VAR_H__
 
 #ifdef _KERNEL
 
 /*
  * Some configurable values for the driver.  By default command+result
  * descriptor rings are the same size.  The src+dst descriptor rings
  * are sized at 3.5x the number of potential commands.  Slower parts
  * (e.g. 7951) tend to run out of src descriptors; faster parts (7811)
  * src+cmd/result descriptors.  It's not clear that increasing the size
  * of the descriptor rings helps performance significantly as other
  * factors tend to come into play (e.g. copying misaligned packets).
  */
 #define	HIFN_D_CMD_RSIZE	24	/* command descriptors */
 #define	HIFN_D_SRC_RSIZE	((HIFN_D_CMD_RSIZE * 7) / 2)	/* source descriptors */
 #define	HIFN_D_RES_RSIZE	HIFN_D_CMD_RSIZE	/* result descriptors */
 #define	HIFN_D_DST_RSIZE	HIFN_D_SRC_RSIZE	/* destination descriptors */
 
 /*
  *  Length values for cryptography
  */
 #define HIFN_DES_KEY_LENGTH		8
 #define HIFN_3DES_KEY_LENGTH		24
 #define HIFN_MAX_CRYPT_KEY_LENGTH	HIFN_3DES_KEY_LENGTH
 #define HIFN_IV_LENGTH			8
 #define	HIFN_AES_IV_LENGTH		16
 #define HIFN_MAX_IV_LENGTH		HIFN_AES_IV_LENGTH
 
 /*
  *  Length values for authentication
  */
 #define HIFN_MAC_KEY_LENGTH		64
 #define HIFN_MD5_LENGTH			16
 #define HIFN_SHA1_LENGTH		20
 #define HIFN_MAC_TRUNC_LENGTH		12
 
 #define MAX_SCATTER 64
 
 /*
  * Data structure to hold all 4 rings and any other ring related data
  * that should reside in DMA.
  */
 struct hifn_dma {
 	/*
 	 *  Descriptor rings.  We add +1 to the size to accomidate the
 	 *  jump descriptor.
 	 */
 	struct hifn_desc	cmdr[HIFN_D_CMD_RSIZE+1];
 	struct hifn_desc	srcr[HIFN_D_SRC_RSIZE+1];
 	struct hifn_desc	dstr[HIFN_D_DST_RSIZE+1];
 	struct hifn_desc	resr[HIFN_D_RES_RSIZE+1];
 
 
 	u_char			command_bufs[HIFN_D_CMD_RSIZE][HIFN_MAX_COMMAND];
 	u_char			result_bufs[HIFN_D_CMD_RSIZE][HIFN_MAX_RESULT];
 	u_int32_t		slop[HIFN_D_CMD_RSIZE];
 	u_int64_t		test_src, test_dst;
 } ;
 
 
 struct hifn_session {
-	u_int8_t hs_iv[HIFN_MAX_IV_LENGTH];
 	int hs_mlen;
 };
 
 #define	HIFN_RING_SYNC(sc, r, i, f)					\
 	bus_dmamap_sync((sc)->sc_dmat, (sc)->sc_dmamap, (f))
 
 #define	HIFN_CMDR_SYNC(sc, i, f)	HIFN_RING_SYNC((sc), cmdr, (i), (f))
 #define	HIFN_RESR_SYNC(sc, i, f)	HIFN_RING_SYNC((sc), resr, (i), (f))
 #define	HIFN_SRCR_SYNC(sc, i, f)	HIFN_RING_SYNC((sc), srcr, (i), (f))
 #define	HIFN_DSTR_SYNC(sc, i, f)	HIFN_RING_SYNC((sc), dstr, (i), (f))
 
 #define	HIFN_CMD_SYNC(sc, i, f)						\
 	bus_dmamap_sync((sc)->sc_dmat, (sc)->sc_dmamap, (f))
 
 #define	HIFN_RES_SYNC(sc, i, f)						\
 	bus_dmamap_sync((sc)->sc_dmat, (sc)->sc_dmamap, (f))
 
 /*
  * Holds data specific to a single HIFN board.
  */
 struct hifn_softc {
 	device_t		sc_dev;		/* device backpointer */
 	struct mtx		sc_mtx;		/* per-instance lock */
 	bus_dma_tag_t		sc_dmat;	/* parent DMA tag descriptor */
 	struct resource		*sc_bar0res;
 	bus_space_handle_t	sc_sh0;		/* bar0 bus space handle */
 	bus_space_tag_t		sc_st0;		/* bar0 bus space tag */
 	bus_size_t		sc_bar0_lastreg;/* bar0 last reg written */
 	struct resource		*sc_bar1res;
 	bus_space_handle_t	sc_sh1;		/* bar1 bus space handle */
 	bus_space_tag_t		sc_st1;		/* bar1 bus space tag */
 	bus_size_t		sc_bar1_lastreg;/* bar1 last reg written */
 	struct resource		*sc_irq;
 	void			*sc_intrhand;	/* interrupt handle */
 
 	u_int32_t		sc_dmaier;
 	u_int32_t		sc_drammodel;	/* 1=dram, 0=sram */
 	u_int32_t		sc_pllconfig;	/* 7954/7955/7956 PLL config */
 
 	struct hifn_dma		*sc_dma;
 	bus_dmamap_t		sc_dmamap;
 	bus_dma_segment_t 	sc_dmasegs[1];
 	bus_addr_t		sc_dma_physaddr;/* physical address of sc_dma */
 	int			sc_dmansegs;
 	struct hifn_command	*sc_hifn_commands[HIFN_D_RES_RSIZE];
 	/*
 	 *  Our current positions for insertion and removal from the desriptor
 	 *  rings. 
 	 */
 	int			sc_cmdi, sc_srci, sc_dsti, sc_resi;
 	volatile int		sc_cmdu, sc_srcu, sc_dstu, sc_resu;
 	int			sc_cmdk, sc_srck, sc_dstk, sc_resk;
 
 	int32_t			sc_cid;
+	uint16_t		sc_ena;
 	int			sc_maxses;
 	int			sc_ramsize;
 	int			sc_flags;
 #define	HIFN_HAS_RNG		0x1	/* includes random number generator */
 #define	HIFN_HAS_PUBLIC		0x2	/* includes public key support */
 #define	HIFN_HAS_AES		0x4	/* includes AES support */
 #define	HIFN_IS_7811		0x8	/* Hifn 7811 part */
 #define	HIFN_IS_7956		0x10	/* Hifn 7956/7955 don't have SDRAM */
 	struct callout		sc_rngto;	/* for polling RNG */
 	struct callout		sc_tickto;	/* for managing DMA */
 	int			sc_rngfirst;
 	int			sc_rnghz;	/* RNG polling frequency */
 	struct rndtest_state	*sc_rndtest;	/* RNG test state */
 	void			(*sc_harvest)(struct rndtest_state *,
 					void *, u_int);
 	int			sc_c_busy;	/* command ring busy */
 	int			sc_s_busy;	/* source data ring busy */
 	int			sc_d_busy;	/* destination data ring busy */
 	int			sc_r_busy;	/* result ring busy */
 	int			sc_active;	/* for initial countdown */
 	int			sc_needwakeup;	/* ops q'd wating on resources */
 	int			sc_curbatch;	/* # ops submitted w/o int */
 	int			sc_suspended;
 #ifdef HIFN_VULCANDEV
 	struct cdev            *sc_pkdev;
 #endif
 };
 
 #define	HIFN_LOCK(_sc)		mtx_lock(&(_sc)->sc_mtx)
 #define	HIFN_UNLOCK(_sc)	mtx_unlock(&(_sc)->sc_mtx)
 
 /*
  *  hifn_command_t
  *
  *  This is the control structure used to pass commands to hifn_encrypt().
  *
  *  flags
  *  -----
  *  Flags is the bitwise "or" values for command configuration.  A single
  *  encrypt direction needs to be set:
  *
  *	HIFN_ENCODE or HIFN_DECODE
  *
  *  To use cryptography, a single crypto algorithm must be included:
  *
  *	HIFN_CRYPT_3DES or HIFN_CRYPT_DES
  *
  *  To use authentication is used, a single MAC algorithm must be included:
  *
  *	HIFN_MAC_MD5 or HIFN_MAC_SHA1
  *
  *  By default MD5 uses a 16 byte hash and SHA-1 uses a 20 byte hash.
  *  If the value below is set, hash values are truncated or assumed
  *  truncated to 12 bytes:
  *
  *	HIFN_MAC_TRUNC
  *
  *  Keys for encryption and authentication can be sent as part of a command,
  *  or the last key value used with a particular session can be retrieved
  *  and used again if either of these flags are not specified.
  *
  *	HIFN_CRYPT_NEW_KEY, HIFN_MAC_NEW_KEY
  *
  *  session_num
  *  -----------
  *  A number between 0 and 2048 (for DRAM models) or a number between 
  *  0 and 768 (for SRAM models).  Those who don't want to use session
  *  numbers should leave value at zero and send a new crypt key and/or
  *  new MAC key on every command.  If you use session numbers and
  *  don't send a key with a command, the last key sent for that same
  *  session number will be used.
  *
  *  Warning:  Using session numbers and multiboard at the same time
  *            is currently broken.
  *
  *  mbuf
  *  ----
  *  Either fill in the mbuf pointer and npa=0 or
  *	 fill packp[] and packl[] and set npa to > 0
  * 
  *  mac_header_skip
  *  ---------------
  *  The number of bytes of the source_buf that are skipped over before
  *  authentication begins.  This must be a number between 0 and 2^16-1
  *  and can be used by IPsec implementers to skip over IP headers.
  *  *** Value ignored if authentication not used ***
  *
  *  crypt_header_skip
  *  -----------------
  *  The number of bytes of the source_buf that are skipped over before
  *  the cryptographic operation begins.  This must be a number between 0
  *  and 2^16-1.  For IPsec, this number will always be 8 bytes larger
  *  than the auth_header_skip (to skip over the ESP header).
  *  *** Value ignored if cryptography not used ***
  *
  */
 struct hifn_operand {
-	union {
-		struct mbuf *m;
-		struct uio *io;
-	} u;
 	bus_dmamap_t	map;
 	bus_size_t	mapsize;
 	int		nsegs;
 	bus_dma_segment_t segs[MAX_SCATTER];
 };
 struct hifn_command {
 	struct hifn_session *session;
 	u_int16_t base_masks, cry_masks, mac_masks;
-	u_int8_t iv[HIFN_MAX_IV_LENGTH], *ck, mac[HIFN_MAC_KEY_LENGTH];
+	u_int8_t iv[HIFN_MAX_IV_LENGTH], mac[HIFN_MAC_KEY_LENGTH];
+	const uint8_t *ck;
 	int cklen;
 	int sloplen, slopidx;
 
 	struct hifn_operand src;
 	struct hifn_operand dst;
+	struct mbuf *dst_m;
 
 	struct hifn_softc *softc;
 	struct cryptop *crp;
-	struct cryptodesc *enccrd, *maccrd;
 };
 
-#define	src_m		src.u.m
-#define	src_io		src.u.io
 #define	src_map		src.map
 #define	src_mapsize	src.mapsize
 #define	src_segs	src.segs
 #define	src_nsegs	src.nsegs
 
-#define	dst_m		dst.u.m
-#define	dst_io		dst.u.io
 #define	dst_map		dst.map
 #define	dst_mapsize	dst.mapsize
 #define	dst_segs	dst.segs
 #define	dst_nsegs	dst.nsegs
 
 /*
  *  Return values for hifn_crypto()
  */
 #define HIFN_CRYPTO_SUCCESS	0
 #define HIFN_CRYPTO_BAD_INPUT	(-1)
 #define HIFN_CRYPTO_RINGS_FULL	(-2)
 
 /**************************************************************************
  *
  *  Function:  hifn_crypto
  *
  *  Purpose:   Called by external drivers to begin an encryption on the
  *             HIFN board.
  *
  *  Blocking/Non-blocking Issues
  *  ============================
  *  The driver cannot block in hifn_crypto (no calls to tsleep) currently.
  *  hifn_crypto() returns HIFN_CRYPTO_RINGS_FULL if there is not enough
  *  room in any of the rings for the request to proceed.
  *
  *  Return Values
  *  =============
  *  0 for success, negative values on error
  *
  *  Defines for negative error codes are:
  *  
  *    HIFN_CRYPTO_BAD_INPUT  :  The passed in command had invalid settings.
  *    HIFN_CRYPTO_RINGS_FULL :  All DMA rings were full and non-blocking
  *                              behaviour was requested.
  *
  *************************************************************************/
 #endif /* _KERNEL */
 
 struct hifn_stats {
 	u_int64_t hst_ibytes;
 	u_int64_t hst_obytes;
 	u_int32_t hst_ipackets;
 	u_int32_t hst_opackets;
 	u_int32_t hst_invalid;
 	u_int32_t hst_nomem;		/* malloc or one of hst_nomem_* */
 	u_int32_t hst_abort;
 	u_int32_t hst_noirq;		/* IRQ for no reason */
 	u_int32_t hst_totbatch;		/* ops submitted w/o interrupt */
 	u_int32_t hst_maxbatch;		/* max ops submitted together */
 	u_int32_t hst_unaligned;	/* unaligned src caused copy */
 	/*
 	 * The following divides hst_nomem into more specific buckets.
 	 */
 	u_int32_t hst_nomem_map;	/* bus_dmamap_create failed */
 	u_int32_t hst_nomem_load;	/* bus_dmamap_load_* failed */
 	u_int32_t hst_nomem_mbuf;	/* MGET* failed */
 	u_int32_t hst_nomem_mcl;	/* MCLGET* failed */
 	u_int32_t hst_nomem_cr;		/* out of command/result descriptor */
 	u_int32_t hst_nomem_sd;		/* out of src/dst descriptors */
 };
 
 #endif /* __HIFN7751VAR_H__ */
Index: head/sys/dev/safe/safe.c
===================================================================
--- head/sys/dev/safe/safe.c	(revision 359373)
+++ head/sys/dev/safe/safe.c	(revision 359374)
@@ -1,2169 +1,2046 @@
 /*-
  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
  *
  * Copyright (c) 2003 Sam Leffler, Errno Consulting
  * Copyright (c) 2003 Global Technology Associates, Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 /*
  * SafeNet SafeXcel-1141 hardware crypto accelerator
  */
 #include "opt_safe.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/proc.h>
 #include <sys/errno.h>
 #include <sys/malloc.h>
 #include <sys/kernel.h>
 #include <sys/mbuf.h>
 #include <sys/module.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <sys/sysctl.h>
 #include <sys/endian.h>
+#include <sys/uio.h>
 
 #include <vm/vm.h>
 #include <vm/pmap.h>
 
 #include <machine/bus.h>
 #include <machine/resource.h>
 #include <sys/bus.h>
 #include <sys/rman.h>
 
-#include <crypto/sha1.h>
 #include <opencrypto/cryptodev.h>
-#include <opencrypto/cryptosoft.h>
-#include <sys/md5.h>
+#include <opencrypto/xform_auth.h>
 #include <sys/random.h>
 #include <sys/kobj.h>
 
 #include "cryptodev_if.h"
 
 #include <dev/pci/pcivar.h>
 #include <dev/pci/pcireg.h>
 
 #ifdef SAFE_RNDTEST
 #include <dev/rndtest/rndtest.h>
 #endif
 #include <dev/safe/safereg.h>
 #include <dev/safe/safevar.h>
 
 #ifndef bswap32
 #define	bswap32	NTOHL
 #endif
 
 /*
  * Prototypes and count for the pci_device structure
  */
 static	int safe_probe(device_t);
 static	int safe_attach(device_t);
 static	int safe_detach(device_t);
 static	int safe_suspend(device_t);
 static	int safe_resume(device_t);
 static	int safe_shutdown(device_t);
 
-static	int safe_newsession(device_t, crypto_session_t, struct cryptoini *);
+static	int safe_probesession(device_t, const struct crypto_session_params *);
+static	int safe_newsession(device_t, crypto_session_t,
+	    const struct crypto_session_params *);
 static	int safe_process(device_t, struct cryptop *, int);
 
 static device_method_t safe_methods[] = {
 	/* Device interface */
 	DEVMETHOD(device_probe,		safe_probe),
 	DEVMETHOD(device_attach,	safe_attach),
 	DEVMETHOD(device_detach,	safe_detach),
 	DEVMETHOD(device_suspend,	safe_suspend),
 	DEVMETHOD(device_resume,	safe_resume),
 	DEVMETHOD(device_shutdown,	safe_shutdown),
 
 	/* crypto device methods */
+	DEVMETHOD(cryptodev_probesession, safe_probesession),
 	DEVMETHOD(cryptodev_newsession,	safe_newsession),
 	DEVMETHOD(cryptodev_process,	safe_process),
 
 	DEVMETHOD_END
 };
 static driver_t safe_driver = {
 	"safe",
 	safe_methods,
 	sizeof (struct safe_softc)
 };
 static devclass_t safe_devclass;
 
 DRIVER_MODULE(safe, pci, safe_driver, safe_devclass, 0, 0);
 MODULE_DEPEND(safe, crypto, 1, 1, 1);
 #ifdef SAFE_RNDTEST
 MODULE_DEPEND(safe, rndtest, 1, 1, 1);
 #endif
 
 static	void safe_intr(void *);
 static	void safe_callback(struct safe_softc *, struct safe_ringentry *);
 static	void safe_feed(struct safe_softc *, struct safe_ringentry *);
 static	void safe_mcopy(struct mbuf *, struct mbuf *, u_int);
 #ifndef SAFE_NO_RNG
 static	void safe_rng_init(struct safe_softc *);
 static	void safe_rng(void *);
 #endif /* SAFE_NO_RNG */
 static	int safe_dma_malloc(struct safe_softc *, bus_size_t,
 	        struct safe_dma_alloc *, int);
 #define	safe_dma_sync(_dma, _flags) \
 	bus_dmamap_sync((_dma)->dma_tag, (_dma)->dma_map, (_flags))
 static	void safe_dma_free(struct safe_softc *, struct safe_dma_alloc *);
 static	int safe_dmamap_aligned(const struct safe_operand *);
 static	int safe_dmamap_uniform(const struct safe_operand *);
 
 static	void safe_reset_board(struct safe_softc *);
 static	void safe_init_board(struct safe_softc *);
 static	void safe_init_pciregs(device_t dev);
 static	void safe_cleanchip(struct safe_softc *);
 static	void safe_totalreset(struct safe_softc *);
 
 static	int safe_free_entry(struct safe_softc *, struct safe_ringentry *);
 
 static SYSCTL_NODE(_hw, OID_AUTO, safe, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
     "SafeNet driver parameters");
 
 #ifdef SAFE_DEBUG
 static	void safe_dump_dmastatus(struct safe_softc *, const char *);
 static	void safe_dump_ringstate(struct safe_softc *, const char *);
 static	void safe_dump_intrstate(struct safe_softc *, const char *);
 static	void safe_dump_request(struct safe_softc *, const char *,
 		struct safe_ringentry *);
 
 static	struct safe_softc *safec;		/* for use by hw.safe.dump */
 
 static	int safe_debug = 0;
 SYSCTL_INT(_hw_safe, OID_AUTO, debug, CTLFLAG_RW, &safe_debug,
 	    0, "control debugging msgs");
 #define	DPRINTF(_x)	if (safe_debug) printf _x
 #else
 #define	DPRINTF(_x)
 #endif
 
 #define	READ_REG(sc,r) \
 	bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r))
 
 #define WRITE_REG(sc,reg,val) \
 	bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val)
 
 struct safe_stats safestats;
 SYSCTL_STRUCT(_hw_safe, OID_AUTO, stats, CTLFLAG_RD, &safestats,
 	    safe_stats, "driver statistics");
 #ifndef SAFE_NO_RNG
 static	int safe_rnginterval = 1;		/* poll once a second */
 SYSCTL_INT(_hw_safe, OID_AUTO, rnginterval, CTLFLAG_RW, &safe_rnginterval,
 	    0, "RNG polling interval (secs)");
 static	int safe_rngbufsize = 16;		/* 64 bytes each poll  */
 SYSCTL_INT(_hw_safe, OID_AUTO, rngbufsize, CTLFLAG_RW, &safe_rngbufsize,
 	    0, "RNG polling buffer size (32-bit words)");
 static	int safe_rngmaxalarm = 8;		/* max alarms before reset */
 SYSCTL_INT(_hw_safe, OID_AUTO, rngmaxalarm, CTLFLAG_RW, &safe_rngmaxalarm,
 	    0, "RNG max alarms before reset");
 #endif /* SAFE_NO_RNG */
 
 static int
 safe_probe(device_t dev)
 {
 	if (pci_get_vendor(dev) == PCI_VENDOR_SAFENET &&
 	    pci_get_device(dev) == PCI_PRODUCT_SAFEXCEL)
 		return (BUS_PROBE_DEFAULT);
 	return (ENXIO);
 }
 
 static const char*
 safe_partname(struct safe_softc *sc)
 {
 	/* XXX sprintf numbers when not decoded */
 	switch (pci_get_vendor(sc->sc_dev)) {
 	case PCI_VENDOR_SAFENET:
 		switch (pci_get_device(sc->sc_dev)) {
 		case PCI_PRODUCT_SAFEXCEL: return "SafeNet SafeXcel-1141";
 		}
 		return "SafeNet unknown-part";
 	}
 	return "Unknown-vendor unknown-part";
 }
 
 #ifndef SAFE_NO_RNG
 static void
 default_harvest(struct rndtest_state *rsp, void *buf, u_int count)
 {
 	/* MarkM: FIX!! Check that this does not swamp the harvester! */
 	random_harvest_queue(buf, count, RANDOM_PURE_SAFE);
 }
 #endif /* SAFE_NO_RNG */
 
 static int
 safe_attach(device_t dev)
 {
 	struct safe_softc *sc = device_get_softc(dev);
 	u_int32_t raddr;
-	u_int32_t i, devinfo;
+	u_int32_t i;
 	int rid;
 
 	bzero(sc, sizeof (*sc));
 	sc->sc_dev = dev;
 
 	/* XXX handle power management */
 
 	pci_enable_busmaster(dev);
 
 	/* 
 	 * Setup memory-mapping of PCI registers.
 	 */
 	rid = BS_BAR;
 	sc->sc_sr = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 					   RF_ACTIVE);
 	if (sc->sc_sr == NULL) {
 		device_printf(dev, "cannot map register space\n");
 		goto bad;
 	}
 	sc->sc_st = rman_get_bustag(sc->sc_sr);
 	sc->sc_sh = rman_get_bushandle(sc->sc_sr);
 
 	/*
 	 * Arrange interrupt line.
 	 */
 	rid = 0;
 	sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
 					    RF_SHAREABLE|RF_ACTIVE);
 	if (sc->sc_irq == NULL) {
 		device_printf(dev, "could not map interrupt\n");
 		goto bad1;
 	}
 	/*
 	 * NB: Network code assumes we are blocked with splimp()
 	 *     so make sure the IRQ is mapped appropriately.
 	 */
 	if (bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET | INTR_MPSAFE,
 			   NULL, safe_intr, sc, &sc->sc_ih)) {
 		device_printf(dev, "could not establish interrupt\n");
 		goto bad2;
 	}
 
 	sc->sc_cid = crypto_get_driverid(dev, sizeof(struct safe_session),
 	    CRYPTOCAP_F_HARDWARE);
 	if (sc->sc_cid < 0) {
 		device_printf(dev, "could not get crypto driver id\n");
 		goto bad3;
 	}
 
 	sc->sc_chiprev = READ_REG(sc, SAFE_DEVINFO) &
 		(SAFE_DEVINFO_REV_MAJ | SAFE_DEVINFO_REV_MIN);
 
 	/*
 	 * Setup DMA descriptor area.
 	 */
 	if (bus_dma_tag_create(bus_get_dma_tag(dev),	/* parent */
 			       1,			/* alignment */
 			       SAFE_DMA_BOUNDARY,	/* boundary */
 			       BUS_SPACE_MAXADDR_32BIT,	/* lowaddr */
 			       BUS_SPACE_MAXADDR,	/* highaddr */
 			       NULL, NULL,		/* filter, filterarg */
 			       SAFE_MAX_DMA,		/* maxsize */
 			       SAFE_MAX_PART,		/* nsegments */
 			       SAFE_MAX_SSIZE,		/* maxsegsize */
 			       BUS_DMA_ALLOCNOW,	/* flags */
 			       NULL, NULL,		/* locking */
 			       &sc->sc_srcdmat)) {
 		device_printf(dev, "cannot allocate DMA tag\n");
 		goto bad4;
 	}
 	if (bus_dma_tag_create(bus_get_dma_tag(dev),	/* parent */
 			       1,			/* alignment */
 			       SAFE_MAX_DSIZE,		/* boundary */
 			       BUS_SPACE_MAXADDR_32BIT,	/* lowaddr */
 			       BUS_SPACE_MAXADDR,	/* highaddr */
 			       NULL, NULL,		/* filter, filterarg */
 			       SAFE_MAX_DMA,		/* maxsize */
 			       SAFE_MAX_PART,		/* nsegments */
 			       SAFE_MAX_DSIZE,		/* maxsegsize */
 			       BUS_DMA_ALLOCNOW,	/* flags */
 			       NULL, NULL,		/* locking */
 			       &sc->sc_dstdmat)) {
 		device_printf(dev, "cannot allocate DMA tag\n");
 		goto bad4;
 	}
 
 	/*
 	 * Allocate packet engine descriptors.
 	 */
 	if (safe_dma_malloc(sc,
 	    SAFE_MAX_NQUEUE * sizeof (struct safe_ringentry),
 	    &sc->sc_ringalloc, 0)) {
 		device_printf(dev, "cannot allocate PE descriptor ring\n");
 		bus_dma_tag_destroy(sc->sc_srcdmat);
 		goto bad4;
 	}
 	/*
 	 * Hookup the static portion of all our data structures.
 	 */
 	sc->sc_ring = (struct safe_ringentry *) sc->sc_ringalloc.dma_vaddr;
 	sc->sc_ringtop = sc->sc_ring + SAFE_MAX_NQUEUE;
 	sc->sc_front = sc->sc_ring;
 	sc->sc_back = sc->sc_ring;
 	raddr = sc->sc_ringalloc.dma_paddr;
 	bzero(sc->sc_ring, SAFE_MAX_NQUEUE * sizeof(struct safe_ringentry));
 	for (i = 0; i < SAFE_MAX_NQUEUE; i++) {
 		struct safe_ringentry *re = &sc->sc_ring[i];
 
 		re->re_desc.d_sa = raddr +
 			offsetof(struct safe_ringentry, re_sa);
 		re->re_sa.sa_staterec = raddr +
 			offsetof(struct safe_ringentry, re_sastate);
 
 		raddr += sizeof (struct safe_ringentry);
 	}
 	mtx_init(&sc->sc_ringmtx, device_get_nameunit(dev),
 		"packet engine ring", MTX_DEF);
 
 	/*
 	 * Allocate scatter and gather particle descriptors.
 	 */
 	if (safe_dma_malloc(sc, SAFE_TOTAL_SPART * sizeof (struct safe_pdesc),
 	    &sc->sc_spalloc, 0)) {
 		device_printf(dev, "cannot allocate source particle "
 			"descriptor ring\n");
 		mtx_destroy(&sc->sc_ringmtx);
 		safe_dma_free(sc, &sc->sc_ringalloc);
 		bus_dma_tag_destroy(sc->sc_srcdmat);
 		goto bad4;
 	}
 	sc->sc_spring = (struct safe_pdesc *) sc->sc_spalloc.dma_vaddr;
 	sc->sc_springtop = sc->sc_spring + SAFE_TOTAL_SPART;
 	sc->sc_spfree = sc->sc_spring;
 	bzero(sc->sc_spring, SAFE_TOTAL_SPART * sizeof(struct safe_pdesc));
 
 	if (safe_dma_malloc(sc, SAFE_TOTAL_DPART * sizeof (struct safe_pdesc),
 	    &sc->sc_dpalloc, 0)) {
 		device_printf(dev, "cannot allocate destination particle "
 			"descriptor ring\n");
 		mtx_destroy(&sc->sc_ringmtx);
 		safe_dma_free(sc, &sc->sc_spalloc);
 		safe_dma_free(sc, &sc->sc_ringalloc);
 		bus_dma_tag_destroy(sc->sc_dstdmat);
 		goto bad4;
 	}
 	sc->sc_dpring = (struct safe_pdesc *) sc->sc_dpalloc.dma_vaddr;
 	sc->sc_dpringtop = sc->sc_dpring + SAFE_TOTAL_DPART;
 	sc->sc_dpfree = sc->sc_dpring;
 	bzero(sc->sc_dpring, SAFE_TOTAL_DPART * sizeof(struct safe_pdesc));
 
 	device_printf(sc->sc_dev, "%s", safe_partname(sc));
 
-	devinfo = READ_REG(sc, SAFE_DEVINFO);
-	if (devinfo & SAFE_DEVINFO_RNG) {
+	sc->sc_devinfo = READ_REG(sc, SAFE_DEVINFO);
+	if (sc->sc_devinfo & SAFE_DEVINFO_RNG) {
 		sc->sc_flags |= SAFE_FLAGS_RNG;
 		printf(" rng");
 	}
-	if (devinfo & SAFE_DEVINFO_PKEY) {
+	if (sc->sc_devinfo & SAFE_DEVINFO_PKEY) {
 #if 0
 		printf(" key");
 		sc->sc_flags |= SAFE_FLAGS_KEY;
 		crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0);
 		crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0);
 #endif
 	}
-	if (devinfo & SAFE_DEVINFO_DES) {
+	if (sc->sc_devinfo & SAFE_DEVINFO_DES) {
 		printf(" des/3des");
-		crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
-		crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
 	}
-	if (devinfo & SAFE_DEVINFO_AES) {
+	if (sc->sc_devinfo & SAFE_DEVINFO_AES) {
 		printf(" aes");
-		crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
 	}
-	if (devinfo & SAFE_DEVINFO_MD5) {
+	if (sc->sc_devinfo & SAFE_DEVINFO_MD5) {
 		printf(" md5");
-		crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
 	}
-	if (devinfo & SAFE_DEVINFO_SHA1) {
+	if (sc->sc_devinfo & SAFE_DEVINFO_SHA1) {
 		printf(" sha1");
-		crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
 	}
-	printf(" null");
-	crypto_register(sc->sc_cid, CRYPTO_NULL_CBC, 0, 0);
-	crypto_register(sc->sc_cid, CRYPTO_NULL_HMAC, 0, 0);
 	/* XXX other supported algorithms */
 	printf("\n");
 
 	safe_reset_board(sc);		/* reset h/w */
 	safe_init_pciregs(dev);		/* init pci settings */
 	safe_init_board(sc);		/* init h/w */
 
 #ifndef SAFE_NO_RNG
 	if (sc->sc_flags & SAFE_FLAGS_RNG) {
 #ifdef SAFE_RNDTEST
 		sc->sc_rndtest = rndtest_attach(dev);
 		if (sc->sc_rndtest)
 			sc->sc_harvest = rndtest_harvest;
 		else
 			sc->sc_harvest = default_harvest;
 #else
 		sc->sc_harvest = default_harvest;
 #endif
 		safe_rng_init(sc);
 
 		callout_init(&sc->sc_rngto, 1);
 		callout_reset(&sc->sc_rngto, hz*safe_rnginterval, safe_rng, sc);
 	}
 #endif /* SAFE_NO_RNG */
 #ifdef SAFE_DEBUG
 	safec = sc;			/* for use by hw.safe.dump */
 #endif
 	return (0);
 bad4:
 	crypto_unregister_all(sc->sc_cid);
 bad3:
 	bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
 bad2:
 	bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
 bad1:
 	bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr);
 bad:
 	return (ENXIO);
 }
 
 /*
  * Detach a device that successfully probed.
  */
 static int
 safe_detach(device_t dev)
 {
 	struct safe_softc *sc = device_get_softc(dev);
 
 	/* XXX wait/abort active ops */
 
 	WRITE_REG(sc, SAFE_HI_MASK, 0);		/* disable interrupts */
 
 	callout_stop(&sc->sc_rngto);
 
 	crypto_unregister_all(sc->sc_cid);
 
 #ifdef SAFE_RNDTEST
 	if (sc->sc_rndtest)
 		rndtest_detach(sc->sc_rndtest);
 #endif
 
 	safe_cleanchip(sc);
 	safe_dma_free(sc, &sc->sc_dpalloc);
 	safe_dma_free(sc, &sc->sc_spalloc);
 	mtx_destroy(&sc->sc_ringmtx);
 	safe_dma_free(sc, &sc->sc_ringalloc);
 
 	bus_generic_detach(dev);
 	bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
 	bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
 
 	bus_dma_tag_destroy(sc->sc_srcdmat);
 	bus_dma_tag_destroy(sc->sc_dstdmat);
 	bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr);
 
 	return (0);
 }
 
 /*
  * Stop all chip i/o so that the kernel's probe routines don't
  * get confused by errant DMAs when rebooting.
  */
 static int
 safe_shutdown(device_t dev)
 {
 #ifdef notyet
 	safe_stop(device_get_softc(dev));
 #endif
 	return (0);
 }
 
 /*
  * Device suspend routine.
  */
 static int
 safe_suspend(device_t dev)
 {
 	struct safe_softc *sc = device_get_softc(dev);
 
 #ifdef notyet
 	/* XXX stop the device and save PCI settings */
 #endif
 	sc->sc_suspended = 1;
 
 	return (0);
 }
 
 static int
 safe_resume(device_t dev)
 {
 	struct safe_softc *sc = device_get_softc(dev);
 
 #ifdef notyet
 	/* XXX retore PCI settings and start the device */
 #endif
 	sc->sc_suspended = 0;
 	return (0);
 }
 
 /*
  * SafeXcel Interrupt routine
  */
 static void
 safe_intr(void *arg)
 {
 	struct safe_softc *sc = arg;
 	volatile u_int32_t stat;
 
 	stat = READ_REG(sc, SAFE_HM_STAT);
 	if (stat == 0)			/* shared irq, not for us */
 		return;
 
 	WRITE_REG(sc, SAFE_HI_CLR, stat);	/* IACK */
 
 	if ((stat & SAFE_INT_PE_DDONE)) {
 		/*
 		 * Descriptor(s) done; scan the ring and
 		 * process completed operations.
 		 */
 		mtx_lock(&sc->sc_ringmtx);
 		while (sc->sc_back != sc->sc_front) {
 			struct safe_ringentry *re = sc->sc_back;
 #ifdef SAFE_DEBUG
 			if (safe_debug) {
 				safe_dump_ringstate(sc, __func__);
 				safe_dump_request(sc, __func__, re);
 			}
 #endif
 			/*
 			 * safe_process marks ring entries that were allocated
 			 * but not used with a csr of zero.  This insures the
 			 * ring front pointer never needs to be set backwards
 			 * in the event that an entry is allocated but not used
 			 * because of a setup error.
 			 */
 			if (re->re_desc.d_csr != 0) {
 				if (!SAFE_PE_CSR_IS_DONE(re->re_desc.d_csr))
 					break;
 				if (!SAFE_PE_LEN_IS_DONE(re->re_desc.d_len))
 					break;
 				sc->sc_nqchip--;
 				safe_callback(sc, re);
 			}
 			if (++(sc->sc_back) == sc->sc_ringtop)
 				sc->sc_back = sc->sc_ring;
 		}
 		mtx_unlock(&sc->sc_ringmtx);
 	}
 
 	/*
 	 * Check to see if we got any DMA Error
 	 */
 	if (stat & SAFE_INT_PE_ERROR) {
 		DPRINTF(("dmaerr dmastat %08x\n",
 			READ_REG(sc, SAFE_PE_DMASTAT)));
 		safestats.st_dmaerr++;
 		safe_totalreset(sc);
 #if 0
 		safe_feed(sc);
 #endif
 	}
 
 	if (sc->sc_needwakeup) {		/* XXX check high watermark */
 		int wakeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ);
 		DPRINTF(("%s: wakeup crypto %x\n", __func__,
 			sc->sc_needwakeup));
 		sc->sc_needwakeup &= ~wakeup;
 		crypto_unblock(sc->sc_cid, wakeup);
 	}
 }
 
 /*
  * safe_feed() - post a request to chip
  */
 static void
 safe_feed(struct safe_softc *sc, struct safe_ringentry *re)
 {
 	bus_dmamap_sync(sc->sc_srcdmat, re->re_src_map, BUS_DMASYNC_PREWRITE);
 	if (re->re_dst_map != NULL)
 		bus_dmamap_sync(sc->sc_dstdmat, re->re_dst_map,
 			BUS_DMASYNC_PREREAD);
 	/* XXX have no smaller granularity */
 	safe_dma_sync(&sc->sc_ringalloc,
 		BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 	safe_dma_sync(&sc->sc_spalloc, BUS_DMASYNC_PREWRITE);
 	safe_dma_sync(&sc->sc_dpalloc, BUS_DMASYNC_PREWRITE);
 
 #ifdef SAFE_DEBUG
 	if (safe_debug) {
 		safe_dump_ringstate(sc, __func__);
 		safe_dump_request(sc, __func__, re);
 	}
 #endif
 	sc->sc_nqchip++;
 	if (sc->sc_nqchip > safestats.st_maxqchip)
 		safestats.st_maxqchip = sc->sc_nqchip;
 	/* poke h/w to check descriptor ring, any value can be written */
 	WRITE_REG(sc, SAFE_HI_RD_DESCR, 0);
 }
 
 #define	N(a)	(sizeof(a) / sizeof (a[0]))
 static void
-safe_setup_enckey(struct safe_session *ses, caddr_t key)
+safe_setup_enckey(struct safe_session *ses, const void *key)
 {
 	int i;
 
-	bcopy(key, ses->ses_key, ses->ses_klen / 8);
+	bcopy(key, ses->ses_key, ses->ses_klen);
 
 	/* PE is little-endian, insure proper byte order */
 	for (i = 0; i < N(ses->ses_key); i++)
 		ses->ses_key[i] = htole32(ses->ses_key[i]);
 }
 
 static void
-safe_setup_mackey(struct safe_session *ses, int algo, caddr_t key, int klen)
+safe_setup_mackey(struct safe_session *ses, int algo, const uint8_t *key,
+    int klen)
 {
 	MD5_CTX md5ctx;
 	SHA1_CTX sha1ctx;
 	int i;
 
-
-	for (i = 0; i < klen; i++)
-		key[i] ^= HMAC_IPAD_VAL;
-
 	if (algo == CRYPTO_MD5_HMAC) {
-		MD5Init(&md5ctx);
-		MD5Update(&md5ctx, key, klen);
-		MD5Update(&md5ctx, hmac_ipad_buffer, MD5_BLOCK_LEN - klen);
+		hmac_init_ipad(&auth_hash_hmac_md5, key, klen, &md5ctx);
 		bcopy(md5ctx.state, ses->ses_hminner, sizeof(md5ctx.state));
-	} else {
-		SHA1Init(&sha1ctx);
-		SHA1Update(&sha1ctx, key, klen);
-		SHA1Update(&sha1ctx, hmac_ipad_buffer,
-		    SHA1_BLOCK_LEN - klen);
-		bcopy(sha1ctx.h.b32, ses->ses_hminner, sizeof(sha1ctx.h.b32));
-	}
 
-	for (i = 0; i < klen; i++)
-		key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
-
-	if (algo == CRYPTO_MD5_HMAC) {
-		MD5Init(&md5ctx);
-		MD5Update(&md5ctx, key, klen);
-		MD5Update(&md5ctx, hmac_opad_buffer, MD5_BLOCK_LEN - klen);
+		hmac_init_opad(&auth_hash_hmac_md5, key, klen, &md5ctx);
 		bcopy(md5ctx.state, ses->ses_hmouter, sizeof(md5ctx.state));
+
+		explicit_bzero(&md5ctx, sizeof(md5ctx));
 	} else {
-		SHA1Init(&sha1ctx);
-		SHA1Update(&sha1ctx, key, klen);
-		SHA1Update(&sha1ctx, hmac_opad_buffer,
-		    SHA1_BLOCK_LEN - klen);
+		hmac_init_ipad(&auth_hash_hmac_sha1, key, klen, &sha1ctx);
+		bcopy(sha1ctx.h.b32, ses->ses_hminner, sizeof(sha1ctx.h.b32));
+
+		hmac_init_opad(&auth_hash_hmac_sha1, key, klen, &sha1ctx);
 		bcopy(sha1ctx.h.b32, ses->ses_hmouter, sizeof(sha1ctx.h.b32));
+
+		explicit_bzero(&sha1ctx, sizeof(sha1ctx));
 	}
 
-	for (i = 0; i < klen; i++)
-		key[i] ^= HMAC_OPAD_VAL;
-
 	/* PE is little-endian, insure proper byte order */
 	for (i = 0; i < N(ses->ses_hminner); i++) {
 		ses->ses_hminner[i] = htole32(ses->ses_hminner[i]);
 		ses->ses_hmouter[i] = htole32(ses->ses_hmouter[i]);
 	}
 }
 #undef N
 
-/*
- * Allocate a new 'session' and return an encoded session id.  'sidp'
- * contains our registration id, and should contain an encoded session
- * id on successful allocation.
- */
+static bool
+safe_auth_supported(struct safe_softc *sc,
+    const struct crypto_session_params *csp)
+{
+
+	switch (csp->csp_auth_alg) {
+	case CRYPTO_MD5_HMAC:
+		if ((sc->sc_devinfo & SAFE_DEVINFO_MD5) == 0)
+			return (false);
+		break;
+	case CRYPTO_SHA1_HMAC:
+		if ((sc->sc_devinfo & SAFE_DEVINFO_SHA1) == 0)
+			return (false);
+		break;
+	default:
+		return (false);
+	}
+	return (true);
+}
+
+static bool
+safe_cipher_supported(struct safe_softc *sc,
+    const struct crypto_session_params *csp)
+{
+
+	switch (csp->csp_cipher_alg) {
+	case CRYPTO_DES_CBC:
+	case CRYPTO_3DES_CBC:
+		if ((sc->sc_devinfo & SAFE_DEVINFO_DES) == 0)
+			return (false);
+		if (csp->csp_ivlen != 8)
+			return (false);
+		if (csp->csp_cipher_alg == CRYPTO_DES_CBC) {
+			if (csp->csp_cipher_klen != 8)
+				return (false);
+		} else {
+			if (csp->csp_cipher_klen != 24)
+				return (false);
+		}
+		break;
+	case CRYPTO_AES_CBC:
+		if ((sc->sc_devinfo & SAFE_DEVINFO_AES) == 0)
+			return (false);
+		if (csp->csp_ivlen != 16)
+			return (false);
+		if (csp->csp_cipher_klen != 16 &&
+		    csp->csp_cipher_klen != 24 &&
+		    csp->csp_cipher_klen != 32)
+			return (false);
+		break;
+	}
+	return (true);
+}
+
 static int
-safe_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
+safe_probesession(device_t dev, const struct crypto_session_params *csp)
 {
 	struct safe_softc *sc = device_get_softc(dev);
-	struct cryptoini *c, *encini = NULL, *macini = NULL;
-	struct safe_session *ses = NULL;
 
-	if (cri == NULL || sc == NULL)
+	if (csp->csp_flags != 0)
 		return (EINVAL);
-
-	for (c = cri; c != NULL; c = c->cri_next) {
-		if (c->cri_alg == CRYPTO_MD5_HMAC ||
-		    c->cri_alg == CRYPTO_SHA1_HMAC ||
-		    c->cri_alg == CRYPTO_NULL_HMAC) {
-			if (macini)
-				return (EINVAL);
-			macini = c;
-		} else if (c->cri_alg == CRYPTO_DES_CBC ||
-		    c->cri_alg == CRYPTO_3DES_CBC ||
-		    c->cri_alg == CRYPTO_AES_CBC ||
-		    c->cri_alg == CRYPTO_NULL_CBC) {
-			if (encini)
-				return (EINVAL);
-			encini = c;
-		} else
+	switch (csp->csp_mode) {
+	case CSP_MODE_DIGEST:
+		if (!safe_auth_supported(sc, csp))
 			return (EINVAL);
-	}
-	if (encini == NULL && macini == NULL)
+		break;
+	case CSP_MODE_CIPHER:
+		if (!safe_cipher_supported(sc, csp))
+			return (EINVAL);
+		break;
+	case CSP_MODE_ETA:
+		if (!safe_auth_supported(sc, csp) ||
+		    !safe_cipher_supported(sc, csp))
+			return (EINVAL);
+		break;
+	default:
 		return (EINVAL);
-	if (encini) {			/* validate key length */
-		switch (encini->cri_alg) {
-		case CRYPTO_DES_CBC:
-			if (encini->cri_klen != 64)
-				return (EINVAL);
-			break;
-		case CRYPTO_3DES_CBC:
-			if (encini->cri_klen != 192)
-				return (EINVAL);
-			break;
-		case CRYPTO_AES_CBC:
-			if (encini->cri_klen != 128 &&
-			    encini->cri_klen != 192 &&
-			    encini->cri_klen != 256)
-				return (EINVAL);
-			break;
-		}
 	}
 
-	ses = crypto_get_driver_session(cses);
-	if (encini) {
-		/* get an IV */
-		/* XXX may read fewer than requested */
-		read_random(ses->ses_iv, sizeof(ses->ses_iv));
+	return (CRYPTODEV_PROBE_HARDWARE);
+}
 
-		ses->ses_klen = encini->cri_klen;
-		if (encini->cri_key != NULL)
-			safe_setup_enckey(ses, encini->cri_key);
+/*
+ * Allocate a new 'session'.
+ */
+static int
+safe_newsession(device_t dev, crypto_session_t cses,
+    const struct crypto_session_params *csp)
+{
+	struct safe_session *ses;
+
+	ses = crypto_get_driver_session(cses);
+	if (csp->csp_cipher_alg != 0) {
+		ses->ses_klen = csp->csp_cipher_klen;
+		if (csp->csp_cipher_key != NULL)
+			safe_setup_enckey(ses, csp->csp_cipher_key);
 	}
 
-	if (macini) {
-		ses->ses_mlen = macini->cri_mlen;
+	if (csp->csp_auth_alg != 0) {
+		ses->ses_mlen = csp->csp_auth_mlen;
 		if (ses->ses_mlen == 0) {
-			if (macini->cri_alg == CRYPTO_MD5_HMAC)
+			if (csp->csp_auth_alg == CRYPTO_MD5_HMAC)
 				ses->ses_mlen = MD5_HASH_LEN;
 			else
 				ses->ses_mlen = SHA1_HASH_LEN;
 		}
 
-		if (macini->cri_key != NULL) {
-			safe_setup_mackey(ses, macini->cri_alg, macini->cri_key,
-			    macini->cri_klen / 8);
+		if (csp->csp_auth_key != NULL) {
+			safe_setup_mackey(ses, csp->csp_auth_alg,
+			    csp->csp_auth_key, csp->csp_auth_klen);
 		}
 	}
 
 	return (0);
 }
 
+static bus_size_t
+safe_crp_length(struct cryptop *crp)
+{
+
+	switch (crp->crp_buf_type) {
+	case CRYPTO_BUF_MBUF:
+		return (crp->crp_mbuf->m_pkthdr.len);
+	case CRYPTO_BUF_UIO:
+		return (crp->crp_uio->uio_resid);
+	case CRYPTO_BUF_CONTIG:
+		return (crp->crp_ilen);
+	default:
+		panic("bad crp buffer type");
+	}
+}
+
 static void
-safe_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error)
+safe_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, int error)
 {
 	struct safe_operand *op = arg;
 
-	DPRINTF(("%s: mapsize %u nsegs %d error %d\n", __func__,
-		(u_int) mapsize, nsegs, error));
+	DPRINTF(("%s: nsegs %d error %d\n", __func__,
+		nsegs, error));
 	if (error != 0)
 		return;
-	op->mapsize = mapsize;
 	op->nsegs = nsegs;
 	bcopy(seg, op->segs, nsegs * sizeof (seg[0]));
 }
 
 static int
 safe_process(device_t dev, struct cryptop *crp, int hint)
 {
-	struct safe_softc *sc = device_get_softc(dev);
+	struct safe_softc *sc = device_get_softc(dev);	
+	const struct crypto_session_params *csp;
 	int err = 0, i, nicealign, uniform;
-	struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
-	int bypass, oplen, ivsize;
-	caddr_t iv;
+	int bypass, oplen;
 	int16_t coffset;
 	struct safe_session *ses;
 	struct safe_ringentry *re;
 	struct safe_sarec *sa;
 	struct safe_pdesc *pd;
 	u_int32_t cmd0, cmd1, staterec;
 
-	if (crp == NULL || crp->crp_callback == NULL || sc == NULL) {
-		safestats.st_invalid++;
-		return (EINVAL);
-	}
-
 	mtx_lock(&sc->sc_ringmtx);
 	if (sc->sc_front == sc->sc_back && sc->sc_nqchip != 0) {
 		safestats.st_ringfull++;
 		sc->sc_needwakeup |= CRYPTO_SYMQ;
 		mtx_unlock(&sc->sc_ringmtx);
 		return (ERESTART);
 	}
 	re = sc->sc_front;
 
 	staterec = re->re_sa.sa_staterec;	/* save */
 	/* NB: zero everything but the PE descriptor */
 	bzero(&re->re_sa, sizeof(struct safe_ringentry) - sizeof(re->re_desc));
 	re->re_sa.sa_staterec = staterec;	/* restore */
 
 	re->re_crp = crp;
 
-	if (crp->crp_flags & CRYPTO_F_IMBUF) {
-		re->re_src_m = (struct mbuf *)crp->crp_buf;
-		re->re_dst_m = (struct mbuf *)crp->crp_buf;
-	} else if (crp->crp_flags & CRYPTO_F_IOV) {
-		re->re_src_io = (struct uio *)crp->crp_buf;
-		re->re_dst_io = (struct uio *)crp->crp_buf;
-	} else {
-		safestats.st_badflags++;
-		err = EINVAL;
-		goto errout;	/* XXX we don't handle contiguous blocks! */
-	}
-
 	sa = &re->re_sa;
 	ses = crypto_get_driver_session(crp->crp_session);
+	csp = crypto_get_params(crp->crp_session);
 
-	crd1 = crp->crp_desc;
-	if (crd1 == NULL) {
-		safestats.st_nodesc++;
-		err = EINVAL;
-		goto errout;
-	}
-	crd2 = crd1->crd_next;
-
 	cmd0 = SAFE_SA_CMD0_BASIC;		/* basic group operation */
 	cmd1 = 0;
-	if (crd2 == NULL) {
-		if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
-		    crd1->crd_alg == CRYPTO_SHA1_HMAC ||
-		    crd1->crd_alg == CRYPTO_NULL_HMAC) {
-			maccrd = crd1;
-			enccrd = NULL;
-			cmd0 |= SAFE_SA_CMD0_OP_HASH;
-		} else if (crd1->crd_alg == CRYPTO_DES_CBC ||
-		    crd1->crd_alg == CRYPTO_3DES_CBC ||
-		    crd1->crd_alg == CRYPTO_AES_CBC ||
-		    crd1->crd_alg == CRYPTO_NULL_CBC) {
-			maccrd = NULL;
-			enccrd = crd1;
-			cmd0 |= SAFE_SA_CMD0_OP_CRYPT;
-		} else {
-			safestats.st_badalg++;
-			err = EINVAL;
-			goto errout;
-		}
-	} else {
-		if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
-		    crd1->crd_alg == CRYPTO_SHA1_HMAC ||
-		    crd1->crd_alg == CRYPTO_NULL_HMAC) &&
-		    (crd2->crd_alg == CRYPTO_DES_CBC ||
-			crd2->crd_alg == CRYPTO_3DES_CBC ||
-		        crd2->crd_alg == CRYPTO_AES_CBC ||
-		        crd2->crd_alg == CRYPTO_NULL_CBC) &&
-		    ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
-			maccrd = crd1;
-			enccrd = crd2;
-		} else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
-		    crd1->crd_alg == CRYPTO_3DES_CBC ||
-		    crd1->crd_alg == CRYPTO_AES_CBC ||
-		    crd1->crd_alg == CRYPTO_NULL_CBC) &&
-		    (crd2->crd_alg == CRYPTO_MD5_HMAC ||
-			crd2->crd_alg == CRYPTO_SHA1_HMAC ||
-			crd2->crd_alg == CRYPTO_NULL_HMAC) &&
-		    (crd1->crd_flags & CRD_F_ENCRYPT)) {
-			enccrd = crd1;
-			maccrd = crd2;
-		} else {
-			safestats.st_badalg++;
-			err = EINVAL;
-			goto errout;
-		}
+	switch (csp->csp_mode) {
+	case CSP_MODE_DIGEST:
+		cmd0 |= SAFE_SA_CMD0_OP_HASH;
+		break;
+	case CSP_MODE_CIPHER:
+		cmd0 |= SAFE_SA_CMD0_OP_CRYPT;
+		break;
+	case CSP_MODE_ETA:
 		cmd0 |= SAFE_SA_CMD0_OP_BOTH;
+		break;
 	}
 
-	if (enccrd) {
-		if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT)
-			safe_setup_enckey(ses, enccrd->crd_key);
+	if (csp->csp_cipher_alg != 0) {
+		if (crp->crp_cipher_key != NULL)
+			safe_setup_enckey(ses, crp->crp_cipher_key);
 
-		if (enccrd->crd_alg == CRYPTO_DES_CBC) {
+		switch (csp->csp_cipher_alg) {
+		case CRYPTO_DES_CBC:
 			cmd0 |= SAFE_SA_CMD0_DES;
 			cmd1 |= SAFE_SA_CMD1_CBC;
-			ivsize = 2*sizeof(u_int32_t);
-		} else if (enccrd->crd_alg == CRYPTO_3DES_CBC) {
+			break;
+		case CRYPTO_3DES_CBC:
 			cmd0 |= SAFE_SA_CMD0_3DES;
 			cmd1 |= SAFE_SA_CMD1_CBC;
-			ivsize = 2*sizeof(u_int32_t);
-		} else if (enccrd->crd_alg == CRYPTO_AES_CBC) {
+			break;
+		case CRYPTO_AES_CBC:
 			cmd0 |= SAFE_SA_CMD0_AES;
 			cmd1 |= SAFE_SA_CMD1_CBC;
-			if (ses->ses_klen == 128)
+			if (ses->ses_klen * 8 == 128)
 			     cmd1 |=  SAFE_SA_CMD1_AES128;
-			else if (ses->ses_klen == 192)
+			else if (ses->ses_klen * 8 == 192)
 			     cmd1 |=  SAFE_SA_CMD1_AES192;
 			else
 			     cmd1 |=  SAFE_SA_CMD1_AES256;
-			ivsize = 4*sizeof(u_int32_t);
-		} else {
-			cmd0 |= SAFE_SA_CMD0_CRYPT_NULL;
-			ivsize = 0;
 		}
 
 		/*
 		 * Setup encrypt/decrypt state.  When using basic ops
 		 * we can't use an inline IV because hash/crypt offset
 		 * must be from the end of the IV to the start of the
 		 * crypt data and this leaves out the preceding header
 		 * from the hash calculation.  Instead we place the IV
 		 * in the state record and set the hash/crypt offset to
 		 * copy both the header+IV.
 		 */
-		if (enccrd->crd_flags & CRD_F_ENCRYPT) {
+		if (crp->crp_flags & CRYPTO_F_IV_GENERATE) {
+			arc4rand(re->re_sastate.sa_saved_iv, csp->csp_ivlen, 0);
+			crypto_copyback(crp, crp->crp_iv_start, csp->csp_ivlen,
+			    re->re_sastate.sa_saved_iv);
+		} else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+			memcpy(re->re_sastate.sa_saved_iv, crp->crp_iv,
+			    csp->csp_ivlen);
+		else
+			crypto_copydata(crp, crp->crp_iv_start, csp->csp_ivlen,
+			    re->re_sastate.sa_saved_iv);
+		cmd0 |= SAFE_SA_CMD0_IVLD_STATE;
+
+		if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
 			cmd0 |= SAFE_SA_CMD0_OUTBOUND;
 
-			if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
-				iv = enccrd->crd_iv;
-			else
-				iv = (caddr_t) ses->ses_iv;
-			if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
-				crypto_copyback(crp->crp_flags, crp->crp_buf,
-				    enccrd->crd_inject, ivsize, iv);
-			}
-			bcopy(iv, re->re_sastate.sa_saved_iv, ivsize);
-			cmd0 |= SAFE_SA_CMD0_IVLD_STATE | SAFE_SA_CMD0_SAVEIV;
-			re->re_flags |= SAFE_QFLAGS_COPYOUTIV;
+			/*
+			 * XXX: I suspect we don't need this since we
+			 * don't save the returned IV.
+			 */
+			cmd0 |= SAFE_SA_CMD0_SAVEIV;
 		} else {
 			cmd0 |= SAFE_SA_CMD0_INBOUND;
-
-			if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) {
-				bcopy(enccrd->crd_iv,
-					re->re_sastate.sa_saved_iv, ivsize);
-			} else {
-				crypto_copydata(crp->crp_flags, crp->crp_buf,
-				    enccrd->crd_inject, ivsize,
-				    (caddr_t)re->re_sastate.sa_saved_iv);
-			}
-			cmd0 |= SAFE_SA_CMD0_IVLD_STATE;
 		}
 		/*
 		 * For basic encryption use the zero pad algorithm.
 		 * This pads results to an 8-byte boundary and
 		 * suppresses padding verification for inbound (i.e.
 		 * decrypt) operations.
 		 *
 		 * NB: Not sure if the 8-byte pad boundary is a problem.
 		 */
 		cmd0 |= SAFE_SA_CMD0_PAD_ZERO;
 
 		/* XXX assert key bufs have the same size */
 		bcopy(ses->ses_key, sa->sa_key, sizeof(sa->sa_key));
 	}
 
-	if (maccrd) {
-		if (maccrd->crd_flags & CRD_F_KEY_EXPLICIT) {
-			safe_setup_mackey(ses, maccrd->crd_alg,
-			    maccrd->crd_key, maccrd->crd_klen / 8);
+	if (csp->csp_auth_alg != 0) {
+		if (crp->crp_auth_key != NULL) {
+			safe_setup_mackey(ses, csp->csp_auth_alg,
+			    crp->crp_auth_key, csp->csp_auth_klen);
 		}
 
-		if (maccrd->crd_alg == CRYPTO_MD5_HMAC) {
+		switch (csp->csp_auth_alg) {
+		case CRYPTO_MD5_HMAC:
 			cmd0 |= SAFE_SA_CMD0_MD5;
 			cmd1 |= SAFE_SA_CMD1_HMAC;	/* NB: enable HMAC */
-		} else if (maccrd->crd_alg == CRYPTO_SHA1_HMAC) {
+			break;
+		case CRYPTO_SHA1_HMAC:
 			cmd0 |= SAFE_SA_CMD0_SHA1;
 			cmd1 |= SAFE_SA_CMD1_HMAC;	/* NB: enable HMAC */
-		} else {
-			cmd0 |= SAFE_SA_CMD0_HASH_NULL;
+			break;
 		}
+
 		/*
 		 * Digest data is loaded from the SA and the hash
 		 * result is saved to the state block where we
 		 * retrieve it for return to the caller.
 		 */
 		/* XXX assert digest bufs have the same size */
 		bcopy(ses->ses_hminner, sa->sa_indigest,
 			sizeof(sa->sa_indigest));
 		bcopy(ses->ses_hmouter, sa->sa_outdigest,
 			sizeof(sa->sa_outdigest));
 
 		cmd0 |= SAFE_SA_CMD0_HSLD_SA | SAFE_SA_CMD0_SAVEHASH;
 		re->re_flags |= SAFE_QFLAGS_COPYOUTICV;
 	}
 
-	if (enccrd && maccrd) {
+	if (csp->csp_mode == CSP_MODE_ETA) {
 		/*
-		 * The offset from hash data to the start of
-		 * crypt data is the difference in the skips.
+		 * The driver only supports ETA requests where there
+		 * is no gap between the AAD and payload.
 		 */
-		bypass = maccrd->crd_skip;
-		coffset = enccrd->crd_skip - maccrd->crd_skip;
-		if (coffset < 0) {
-			DPRINTF(("%s: hash does not precede crypt; "
-				"mac skip %u enc skip %u\n",
-				__func__, maccrd->crd_skip, enccrd->crd_skip));
-			safestats.st_skipmismatch++;
-			err = EINVAL;
-			goto errout;
-		}
-		oplen = enccrd->crd_skip + enccrd->crd_len;
-		if (maccrd->crd_skip + maccrd->crd_len != oplen) {
-			DPRINTF(("%s: hash amount %u != crypt amount %u\n",
-				__func__, maccrd->crd_skip + maccrd->crd_len,
-				oplen));
+		if (crp->crp_aad_length != 0 &&
+		    crp->crp_aad_start + crp->crp_aad_length !=
+		    crp->crp_payload_start) {
 			safestats.st_lenmismatch++;
 			err = EINVAL;
 			goto errout;
 		}
+		if (crp->crp_aad_length != 0)
+			bypass = crp->crp_aad_start;
+		else
+			bypass = crp->crp_payload_start;
+		coffset = crp->crp_aad_length;
+		oplen = crp->crp_payload_start + crp->crp_payload_length;
 #ifdef SAFE_DEBUG
 		if (safe_debug) {
-			printf("mac: skip %d, len %d, inject %d\n",
-			    maccrd->crd_skip, maccrd->crd_len,
-			    maccrd->crd_inject);
-			printf("enc: skip %d, len %d, inject %d\n",
-			    enccrd->crd_skip, enccrd->crd_len,
-			    enccrd->crd_inject);
+			printf("AAD: skip %d, len %d, digest %d\n",
+			    crp->crp_aad_start, crp->crp_aad_length,
+			    crp->crp_digest_start);
+			printf("payload: skip %d, len %d, IV %d\n",
+			    crp->crp_payload_start, crp->crp_payload_length,
+			    crp->crp_iv_start);
 			printf("bypass %d coffset %d oplen %d\n",
 				bypass, coffset, oplen);
 		}
 #endif
 		if (coffset & 3) {	/* offset must be 32-bit aligned */
 			DPRINTF(("%s: coffset %u misaligned\n",
 				__func__, coffset));
 			safestats.st_coffmisaligned++;
 			err = EINVAL;
 			goto errout;
 		}
 		coffset >>= 2;
 		if (coffset > 255) {	/* offset must be <256 dwords */
 			DPRINTF(("%s: coffset %u too big\n",
 				__func__, coffset));
 			safestats.st_cofftoobig++;
 			err = EINVAL;
 			goto errout;
 		}
 		/*
 		 * Tell the hardware to copy the header to the output.
 		 * The header is defined as the data from the end of
 		 * the bypass to the start of data to be encrypted. 
 		 * Typically this is the inline IV.  Note that you need
 		 * to do this even if src+dst are the same; it appears
 		 * that w/o this bit the crypted data is written
 		 * immediately after the bypass data.
 		 */
 		cmd1 |= SAFE_SA_CMD1_HDRCOPY;
 		/*
 		 * Disable IP header mutable bit handling.  This is
 		 * needed to get correct HMAC calculations.
 		 */
 		cmd1 |= SAFE_SA_CMD1_MUTABLE;
 	} else {
-		if (enccrd) {
-			bypass = enccrd->crd_skip;
-			oplen = bypass + enccrd->crd_len;
-		} else {
-			bypass = maccrd->crd_skip;
-			oplen = bypass + maccrd->crd_len;
-		}
+		bypass = crp->crp_payload_start;
+		oplen = bypass + crp->crp_payload_length;
 		coffset = 0;
 	}
 	/* XXX verify multiple of 4 when using s/g */
 	if (bypass > 96) {		/* bypass offset must be <= 96 bytes */
 		DPRINTF(("%s: bypass %u too big\n", __func__, bypass));
 		safestats.st_bypasstoobig++;
 		err = EINVAL;
 		goto errout;
 	}
 
 	if (bus_dmamap_create(sc->sc_srcdmat, BUS_DMA_NOWAIT, &re->re_src_map)) {
 		safestats.st_nomap++;
 		err = ENOMEM;
 		goto errout;
 	}
-	if (crp->crp_flags & CRYPTO_F_IMBUF) {
-		if (bus_dmamap_load_mbuf(sc->sc_srcdmat, re->re_src_map,
-		    re->re_src_m, safe_op_cb,
-		    &re->re_src, BUS_DMA_NOWAIT) != 0) {
-			bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map);
-			re->re_src_map = NULL;
-			safestats.st_noload++;
-			err = ENOMEM;
-			goto errout;
-		}
-	} else if (crp->crp_flags & CRYPTO_F_IOV) {
-		if (bus_dmamap_load_uio(sc->sc_srcdmat, re->re_src_map,
-		    re->re_src_io, safe_op_cb,
-		    &re->re_src, BUS_DMA_NOWAIT) != 0) {
-			bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map);
-			re->re_src_map = NULL;
-			safestats.st_noload++;
-			err = ENOMEM;
-			goto errout;
-		}
+	if (bus_dmamap_load_crp(sc->sc_srcdmat, re->re_src_map, crp, safe_op_cb,
+	    &re->re_src, BUS_DMA_NOWAIT) != 0) {
+		bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map);
+		re->re_src_map = NULL;
+		safestats.st_noload++;
+		err = ENOMEM;
+		goto errout;
 	}
+	re->re_src_mapsize = safe_crp_length(crp);
 	nicealign = safe_dmamap_aligned(&re->re_src);
 	uniform = safe_dmamap_uniform(&re->re_src);
 
 	DPRINTF(("src nicealign %u uniform %u nsegs %u\n",
 		nicealign, uniform, re->re_src.nsegs));
 	if (re->re_src.nsegs > 1) {
 		re->re_desc.d_src = sc->sc_spalloc.dma_paddr +
 			((caddr_t) sc->sc_spfree - (caddr_t) sc->sc_spring);
 		for (i = 0; i < re->re_src_nsegs; i++) {
 			/* NB: no need to check if there's space */
 			pd = sc->sc_spfree;
 			if (++(sc->sc_spfree) == sc->sc_springtop)
 				sc->sc_spfree = sc->sc_spring;
 
 			KASSERT((pd->pd_flags&3) == 0 ||
 				(pd->pd_flags&3) == SAFE_PD_DONE,
 				("bogus source particle descriptor; flags %x",
 				pd->pd_flags));
 			pd->pd_addr = re->re_src_segs[i].ds_addr;
 			pd->pd_size = re->re_src_segs[i].ds_len;
 			pd->pd_flags = SAFE_PD_READY;
 		}
 		cmd0 |= SAFE_SA_CMD0_IGATHER;
 	} else {
 		/*
 		 * No need for gather, reference the operand directly.
 		 */
 		re->re_desc.d_src = re->re_src_segs[0].ds_addr;
 	}
 
-	if (enccrd == NULL && maccrd != NULL) {
+	if (csp->csp_mode == CSP_MODE_DIGEST) {
 		/*
 		 * Hash op; no destination needed.
 		 */
 	} else {
-		if (crp->crp_flags & CRYPTO_F_IOV) {
-			if (!nicealign) {
-				safestats.st_iovmisaligned++;
-				err = EINVAL;
+		if (nicealign && uniform == 1) {
+			/*
+			 * Source layout is suitable for direct
+			 * sharing of the DMA map and segment list.
+			 */
+			re->re_dst = re->re_src;
+		} else if (nicealign && uniform == 2) {
+			/*
+			 * The source is properly aligned but requires a
+			 * different particle list to handle DMA of the
+			 * result.  Create a new map and do the load to
+			 * create the segment list.  The particle
+			 * descriptor setup code below will handle the
+			 * rest.
+			 */
+			if (bus_dmamap_create(sc->sc_dstdmat, BUS_DMA_NOWAIT,
+			    &re->re_dst_map)) {
+				safestats.st_nomap++;
+				err = ENOMEM;
 				goto errout;
 			}
-			if (uniform != 1) {
-				/*
-				 * Source is not suitable for direct use as
-				 * the destination.  Create a new scatter/gather
-				 * list based on the destination requirements
-				 * and check if that's ok.
-				 */
-				if (bus_dmamap_create(sc->sc_dstdmat,
-				    BUS_DMA_NOWAIT, &re->re_dst_map)) {
-					safestats.st_nomap++;
-					err = ENOMEM;
-					goto errout;
+			if (bus_dmamap_load_crp(sc->sc_dstdmat, re->re_dst_map,
+			    crp, safe_op_cb, &re->re_dst, BUS_DMA_NOWAIT) !=
+			    0) {
+				bus_dmamap_destroy(sc->sc_dstdmat,
+				    re->re_dst_map);
+				re->re_dst_map = NULL;
+				safestats.st_noload++;
+				err = ENOMEM;
+				goto errout;
+			}
+		} else if (crp->crp_buf_type == CRYPTO_BUF_MBUF) {
+			int totlen, len;
+			struct mbuf *m, *top, **mp;
+
+			/*
+			 * DMA constraints require that we allocate a
+			 * new mbuf chain for the destination.  We
+			 * allocate an entire new set of mbufs of
+			 * optimal/required size and then tell the
+			 * hardware to copy any bits that are not
+			 * created as a byproduct of the operation.
+			 */
+			if (!nicealign)
+				safestats.st_unaligned++;
+			if (!uniform)
+				safestats.st_notuniform++;
+			totlen = re->re_src_mapsize;
+			if (crp->crp_mbuf->m_flags & M_PKTHDR) {
+				len = MHLEN;
+				MGETHDR(m, M_NOWAIT, MT_DATA);
+				if (m && !m_dup_pkthdr(m, crp->crp_mbuf,
+				    M_NOWAIT)) {
+					m_free(m);
+					m = NULL;
 				}
-				if (bus_dmamap_load_uio(sc->sc_dstdmat,
-				    re->re_dst_map, re->re_dst_io,
-				    safe_op_cb, &re->re_dst,
-				    BUS_DMA_NOWAIT) != 0) {
-					bus_dmamap_destroy(sc->sc_dstdmat,
-						re->re_dst_map);
-					re->re_dst_map = NULL;
-					safestats.st_noload++;
-					err = ENOMEM;
+			} else {
+				len = MLEN;
+				MGET(m, M_NOWAIT, MT_DATA);
+			}
+			if (m == NULL) {
+				safestats.st_nombuf++;
+				err = sc->sc_nqchip ? ERESTART : ENOMEM;
+				goto errout;
+			}
+			if (totlen >= MINCLSIZE) {
+				if (!(MCLGET(m, M_NOWAIT))) {
+					m_free(m);
+					safestats.st_nomcl++;
+					err = sc->sc_nqchip ?
+					    ERESTART : ENOMEM;
 					goto errout;
 				}
-				uniform = safe_dmamap_uniform(&re->re_dst);
-				if (!uniform) {
-					/*
-					 * There's no way to handle the DMA
-					 * requirements with this uio.  We
-					 * could create a separate DMA area for
-					 * the result and then copy it back,
-					 * but for now we just bail and return
-					 * an error.  Note that uio requests
-					 * > SAFE_MAX_DSIZE are handled because
-					 * the DMA map and segment list for the
-					 * destination wil result in a
-					 * destination particle list that does
-					 * the necessary scatter DMA.
-					 */ 
-					safestats.st_iovnotuniform++;
-					err = EINVAL;
-					goto errout;
-				}
-			} else
-				re->re_dst = re->re_src;
-		} else if (crp->crp_flags & CRYPTO_F_IMBUF) {
-			if (nicealign && uniform == 1) {
-				/*
-				 * Source layout is suitable for direct
-				 * sharing of the DMA map and segment list.
-				 */
-				re->re_dst = re->re_src;
-			} else if (nicealign && uniform == 2) {
-				/*
-				 * The source is properly aligned but requires a
-				 * different particle list to handle DMA of the
-				 * result.  Create a new map and do the load to
-				 * create the segment list.  The particle
-				 * descriptor setup code below will handle the
-				 * rest.
-				 */
-				if (bus_dmamap_create(sc->sc_dstdmat,
-				    BUS_DMA_NOWAIT, &re->re_dst_map)) {
-					safestats.st_nomap++;
-					err = ENOMEM;
-					goto errout;
-				}
-				if (bus_dmamap_load_mbuf(sc->sc_dstdmat,
-				    re->re_dst_map, re->re_dst_m,
-				    safe_op_cb, &re->re_dst,
-				    BUS_DMA_NOWAIT) != 0) {
-					bus_dmamap_destroy(sc->sc_dstdmat,
-						re->re_dst_map);
-					re->re_dst_map = NULL;
-					safestats.st_noload++;
-					err = ENOMEM;
-					goto errout;
-				}
-			} else {		/* !(aligned and/or uniform) */
-				int totlen, len;
-				struct mbuf *m, *top, **mp;
+				len = MCLBYTES;
+			}
+			m->m_len = len;
+			top = NULL;
+			mp = &top;
 
-				/*
-				 * DMA constraints require that we allocate a
-				 * new mbuf chain for the destination.  We
-				 * allocate an entire new set of mbufs of
-				 * optimal/required size and then tell the
-				 * hardware to copy any bits that are not
-				 * created as a byproduct of the operation.
-				 */
-				if (!nicealign)
-					safestats.st_unaligned++;
-				if (!uniform)
-					safestats.st_notuniform++;
-				totlen = re->re_src_mapsize;
-				if (re->re_src_m->m_flags & M_PKTHDR) {
-					len = MHLEN;
-					MGETHDR(m, M_NOWAIT, MT_DATA);
-					if (m && !m_dup_pkthdr(m, re->re_src_m,
-					    M_NOWAIT)) {
-						m_free(m);
-						m = NULL;
+			while (totlen > 0) {
+				if (top) {
+					MGET(m, M_NOWAIT, MT_DATA);
+					if (m == NULL) {
+						m_freem(top);
+						safestats.st_nombuf++;
+						err = sc->sc_nqchip ?
+						    ERESTART : ENOMEM;
+						goto errout;
 					}
-				} else {
 					len = MLEN;
-					MGET(m, M_NOWAIT, MT_DATA);
 				}
-				if (m == NULL) {
-					safestats.st_nombuf++;
-					err = sc->sc_nqchip ? ERESTART : ENOMEM;
-					goto errout;
-				}
-				if (totlen >= MINCLSIZE) {
+				if (top && totlen >= MINCLSIZE) {
 					if (!(MCLGET(m, M_NOWAIT))) {
-						m_free(m);
+						*mp = m;
+						m_freem(top);
 						safestats.st_nomcl++;
 						err = sc->sc_nqchip ?
-							ERESTART : ENOMEM;
+						    ERESTART : ENOMEM;
 						goto errout;
 					}
 					len = MCLBYTES;
 				}
-				m->m_len = len;
-				top = NULL;
-				mp = &top;
-
-				while (totlen > 0) {
-					if (top) {
-						MGET(m, M_NOWAIT, MT_DATA);
-						if (m == NULL) {
-							m_freem(top);
-							safestats.st_nombuf++;
-							err = sc->sc_nqchip ?
-							    ERESTART : ENOMEM;
-							goto errout;
-						}
-						len = MLEN;
-					}
-					if (top && totlen >= MINCLSIZE) {
-						if (!(MCLGET(m, M_NOWAIT))) {
-							*mp = m;
-							m_freem(top);
-							safestats.st_nomcl++;
-							err = sc->sc_nqchip ?
-							    ERESTART : ENOMEM;
-							goto errout;
-						}
-						len = MCLBYTES;
-					}
-					m->m_len = len = min(totlen, len);
-					totlen -= len;
-					*mp = m;
-					mp = &m->m_next;
-				}
-				re->re_dst_m = top;
-				if (bus_dmamap_create(sc->sc_dstdmat, 
-				    BUS_DMA_NOWAIT, &re->re_dst_map) != 0) {
-					safestats.st_nomap++;
-					err = ENOMEM;
-					goto errout;
-				}
-				if (bus_dmamap_load_mbuf(sc->sc_dstdmat,
-				    re->re_dst_map, re->re_dst_m,
-				    safe_op_cb, &re->re_dst,
-				    BUS_DMA_NOWAIT) != 0) {
-					bus_dmamap_destroy(sc->sc_dstdmat,
-					re->re_dst_map);
-					re->re_dst_map = NULL;
-					safestats.st_noload++;
-					err = ENOMEM;
-					goto errout;
-				}
-				if (re->re_src.mapsize > oplen) {
-					/*
-					 * There's data following what the
-					 * hardware will copy for us.  If this
-					 * isn't just the ICV (that's going to
-					 * be written on completion), copy it
-					 * to the new mbufs
-					 */
-					if (!(maccrd &&
-					    (re->re_src.mapsize-oplen) == 12 &&
-					    maccrd->crd_inject == oplen))
-						safe_mcopy(re->re_src_m,
-							   re->re_dst_m,
-							   oplen);
-					else
-						safestats.st_noicvcopy++;
-				}
+				m->m_len = len = min(totlen, len);
+				totlen -= len;
+				*mp = m;
+				mp = &m->m_next;
 			}
+			re->re_dst_m = top;
+			if (bus_dmamap_create(sc->sc_dstdmat,
+			    BUS_DMA_NOWAIT, &re->re_dst_map) != 0) {
+				safestats.st_nomap++;
+				err = ENOMEM;
+				goto errout;
+			}
+			if (bus_dmamap_load_mbuf_sg(sc->sc_dstdmat,
+			    re->re_dst_map, top, re->re_dst_segs,
+			    &re->re_dst_nsegs, 0) != 0) {
+				bus_dmamap_destroy(sc->sc_dstdmat,
+				    re->re_dst_map);
+				re->re_dst_map = NULL;
+				safestats.st_noload++;
+				err = ENOMEM;
+				goto errout;
+			}
+			re->re_dst_mapsize = re->re_src_mapsize;
+			if (re->re_src.mapsize > oplen) {
+				/*
+				 * There's data following what the
+				 * hardware will copy for us.  If this
+				 * isn't just the ICV (that's going to
+				 * be written on completion), copy it
+				 * to the new mbufs
+				 */
+				if (!(csp->csp_mode == CSP_MODE_ETA &&
+				    (re->re_src.mapsize-oplen) == ses->ses_mlen &&
+				    crp->crp_digest_start == oplen))
+					safe_mcopy(crp->crp_mbuf, re->re_dst_m,
+					    oplen);
+				else
+					safestats.st_noicvcopy++;
+			}
 		} else {
-			safestats.st_badflags++;
-			err = EINVAL;
-			goto errout;
+			if (!nicealign) {
+				safestats.st_iovmisaligned++;
+				err = EINVAL;
+				goto errout;
+			} else {
+				/*
+				 * There's no way to handle the DMA
+				 * requirements with this uio.  We
+				 * could create a separate DMA area for
+				 * the result and then copy it back,
+				 * but for now we just bail and return
+				 * an error.  Note that uio requests
+				 * > SAFE_MAX_DSIZE are handled because
+				 * the DMA map and segment list for the
+				 * destination wil result in a
+				 * destination particle list that does
+				 * the necessary scatter DMA.
+				 */
+				safestats.st_iovnotuniform++;
+				err = EINVAL;
+				goto errout;
+			}
 		}
 
 		if (re->re_dst.nsegs > 1) {
 			re->re_desc.d_dst = sc->sc_dpalloc.dma_paddr +
 			    ((caddr_t) sc->sc_dpfree - (caddr_t) sc->sc_dpring);
 			for (i = 0; i < re->re_dst_nsegs; i++) {
 				pd = sc->sc_dpfree;
 				KASSERT((pd->pd_flags&3) == 0 ||
 					(pd->pd_flags&3) == SAFE_PD_DONE,
 					("bogus dest particle descriptor; flags %x",
 						pd->pd_flags));
 				if (++(sc->sc_dpfree) == sc->sc_dpringtop)
 					sc->sc_dpfree = sc->sc_dpring;
 				pd->pd_addr = re->re_dst_segs[i].ds_addr;
 				pd->pd_flags = SAFE_PD_READY;
 			}
 			cmd0 |= SAFE_SA_CMD0_OSCATTER;
 		} else {
 			/*
 			 * No need for scatter, reference the operand directly.
 			 */
 			re->re_desc.d_dst = re->re_dst_segs[0].ds_addr;
 		}
 	}
 
 	/*
 	 * All done with setup; fillin the SA command words
 	 * and the packet engine descriptor.  The operation
 	 * is now ready for submission to the hardware.
 	 */
 	sa->sa_cmd0 = cmd0 | SAFE_SA_CMD0_IPCI | SAFE_SA_CMD0_OPCI;
 	sa->sa_cmd1 = cmd1
 		    | (coffset << SAFE_SA_CMD1_OFFSET_S)
 		    | SAFE_SA_CMD1_SAREV1	/* Rev 1 SA data structure */
 		    | SAFE_SA_CMD1_SRPCI
 		    ;
 	/*
 	 * NB: the order of writes is important here.  In case the
 	 * chip is scanning the ring because of an outstanding request
 	 * it might nab this one too.  In that case we need to make
 	 * sure the setup is complete before we write the length
 	 * field of the descriptor as it signals the descriptor is
 	 * ready for processing.
 	 */
 	re->re_desc.d_csr = SAFE_PE_CSR_READY | SAFE_PE_CSR_SAPCI;
-	if (maccrd)
+	if (csp->csp_auth_alg != 0)
 		re->re_desc.d_csr |= SAFE_PE_CSR_LOADSA | SAFE_PE_CSR_HASHFINAL;
 	re->re_desc.d_len = oplen
 			  | SAFE_PE_LEN_READY
 			  | (bypass << SAFE_PE_LEN_BYPASS_S)
 			  ;
 
 	safestats.st_ipackets++;
 	safestats.st_ibytes += oplen;
 
 	if (++(sc->sc_front) == sc->sc_ringtop)
 		sc->sc_front = sc->sc_ring;
 
 	/* XXX honor batching */
 	safe_feed(sc, re);
 	mtx_unlock(&sc->sc_ringmtx);
 	return (0);
 
 errout:
-	if ((re->re_dst_m != NULL) && (re->re_src_m != re->re_dst_m))
+	if (re->re_dst_m != NULL)
 		m_freem(re->re_dst_m);
 
 	if (re->re_dst_map != NULL && re->re_dst_map != re->re_src_map) {
 		bus_dmamap_unload(sc->sc_dstdmat, re->re_dst_map);
 		bus_dmamap_destroy(sc->sc_dstdmat, re->re_dst_map);
 	}
 	if (re->re_src_map != NULL) {
 		bus_dmamap_unload(sc->sc_srcdmat, re->re_src_map);
 		bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map);
 	}
 	mtx_unlock(&sc->sc_ringmtx);
 	if (err != ERESTART) {
 		crp->crp_etype = err;
 		crypto_done(crp);
 	} else {
 		sc->sc_needwakeup |= CRYPTO_SYMQ;
 	}
 	return (err);
 }
 
 static void
 safe_callback(struct safe_softc *sc, struct safe_ringentry *re)
 {
+	const struct crypto_session_params *csp;
 	struct cryptop *crp = (struct cryptop *)re->re_crp;
 	struct safe_session *ses;
-	struct cryptodesc *crd;
+	uint8_t hash[HASH_MAX_LEN];
 
 	ses = crypto_get_driver_session(crp->crp_session);
+	csp = crypto_get_params(crp->crp_session);
 
 	safestats.st_opackets++;
 	safestats.st_obytes += re->re_dst.mapsize;
 
 	safe_dma_sync(&sc->sc_ringalloc,
 		BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
 	if (re->re_desc.d_csr & SAFE_PE_CSR_STATUS) {
 		device_printf(sc->sc_dev, "csr 0x%x cmd0 0x%x cmd1 0x%x\n",
 			re->re_desc.d_csr,
 			re->re_sa.sa_cmd0, re->re_sa.sa_cmd1);
 		safestats.st_peoperr++;
 		crp->crp_etype = EIO;		/* something more meaningful? */
 	}
+
+	/* XXX: Should crp_mbuf be updated to re->re_dst_m if it is non-NULL? */
+
 	if (re->re_dst_map != NULL && re->re_dst_map != re->re_src_map) {
 		bus_dmamap_sync(sc->sc_dstdmat, re->re_dst_map,
 		    BUS_DMASYNC_POSTREAD);
 		bus_dmamap_unload(sc->sc_dstdmat, re->re_dst_map);
 		bus_dmamap_destroy(sc->sc_dstdmat, re->re_dst_map);
 	}
 	bus_dmamap_sync(sc->sc_srcdmat, re->re_src_map, BUS_DMASYNC_POSTWRITE);
 	bus_dmamap_unload(sc->sc_srcdmat, re->re_src_map);
 	bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map);
 
-	/* 
-	 * If result was written to a differet mbuf chain, swap
-	 * it in as the return value and reclaim the original.
-	 */
-	if ((crp->crp_flags & CRYPTO_F_IMBUF) && re->re_src_m != re->re_dst_m) {
-		m_freem(re->re_src_m);
-		crp->crp_buf = (caddr_t)re->re_dst_m;
-	}
-
-	if (re->re_flags & SAFE_QFLAGS_COPYOUTIV) {
-		/* copy out IV for future use */
-		for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
-			int ivsize;
-
-			if (crd->crd_alg == CRYPTO_DES_CBC ||
-			    crd->crd_alg == CRYPTO_3DES_CBC) {
-				ivsize = 2*sizeof(u_int32_t);
-			} else if (crd->crd_alg == CRYPTO_AES_CBC) {
-				ivsize = 4*sizeof(u_int32_t);
-			} else
-				continue;
-			crypto_copydata(crp->crp_flags, crp->crp_buf,
-			    crd->crd_skip + crd->crd_len - ivsize, ivsize,
-			    (caddr_t)ses->ses_iv);
-			break;
-		}
-	}
-
 	if (re->re_flags & SAFE_QFLAGS_COPYOUTICV) {
-		/* copy out ICV result */
-		for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
-			if (!(crd->crd_alg == CRYPTO_MD5_HMAC ||
-			    crd->crd_alg == CRYPTO_SHA1_HMAC ||
-			    crd->crd_alg == CRYPTO_NULL_HMAC))
-				continue;
-			if (crd->crd_alg == CRYPTO_SHA1_HMAC) {
-				/*
-				 * SHA-1 ICV's are byte-swapped; fix 'em up
-				 * before copy them to their destination.
-				 */
-				re->re_sastate.sa_saved_indigest[0] =
-				    bswap32(re->re_sastate.sa_saved_indigest[0]);
-				re->re_sastate.sa_saved_indigest[1] =
-				    bswap32(re->re_sastate.sa_saved_indigest[1]);
-				re->re_sastate.sa_saved_indigest[2] =
-				    bswap32(re->re_sastate.sa_saved_indigest[2]);
-			}
-			crypto_copyback(crp->crp_flags, crp->crp_buf,
-			    crd->crd_inject, ses->ses_mlen,
-			    (caddr_t)re->re_sastate.sa_saved_indigest);
-			break;
+		if (csp->csp_auth_alg == CRYPTO_SHA1_HMAC) {
+			/*
+			 * SHA-1 ICV's are byte-swapped; fix 'em up
+			 * before copying them to their destination.
+			 */
+			re->re_sastate.sa_saved_indigest[0] =
+			    bswap32(re->re_sastate.sa_saved_indigest[0]);
+			re->re_sastate.sa_saved_indigest[1] =
+			    bswap32(re->re_sastate.sa_saved_indigest[1]);
+			re->re_sastate.sa_saved_indigest[2] =
+			    bswap32(re->re_sastate.sa_saved_indigest[2]);
 		}
+
+		if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
+			crypto_copydata(crp, crp->crp_digest_start,
+			    ses->ses_mlen, hash);
+			if (timingsafe_bcmp(re->re_sastate.sa_saved_indigest,
+			    hash, ses->ses_mlen) != 0)
+				crp->crp_etype = EBADMSG;
+		} else
+			crypto_copyback(crp, crp->crp_digest_start,
+			    ses->ses_mlen, re->re_sastate.sa_saved_indigest);
 	}
 	crypto_done(crp);
 }
 
 /*
  * Copy all data past offset from srcm to dstm.
  */
 static void
 safe_mcopy(struct mbuf *srcm, struct mbuf *dstm, u_int offset)
 {
 	u_int j, dlen, slen;
 	caddr_t dptr, sptr;
 
 	/*
 	 * Advance src and dst to offset.
 	 */
 	j = offset;
 	while (j >= srcm->m_len) {
 		j -= srcm->m_len;
 		srcm = srcm->m_next;
 		if (srcm == NULL)
 			return;
 	}
 	sptr = mtod(srcm, caddr_t) + j;
 	slen = srcm->m_len - j;
 
 	j = offset;
 	while (j >= dstm->m_len) {
 		j -= dstm->m_len;
 		dstm = dstm->m_next;
 		if (dstm == NULL)
 			return;
 	}
 	dptr = mtod(dstm, caddr_t) + j;
 	dlen = dstm->m_len - j;
 
 	/*
 	 * Copy everything that remains.
 	 */
 	for (;;) {
 		j = min(slen, dlen);
 		bcopy(sptr, dptr, j);
 		if (slen == j) {
 			srcm = srcm->m_next;
 			if (srcm == NULL)
 				return;
 			sptr = srcm->m_data;
 			slen = srcm->m_len;
 		} else
 			sptr += j, slen -= j;
 		if (dlen == j) {
 			dstm = dstm->m_next;
 			if (dstm == NULL)
 				return;
 			dptr = dstm->m_data;
 			dlen = dstm->m_len;
 		} else
 			dptr += j, dlen -= j;
 	}
 }
 
 #ifndef SAFE_NO_RNG
 #define	SAFE_RNG_MAXWAIT	1000
 
 static void
 safe_rng_init(struct safe_softc *sc)
 {
 	u_int32_t w, v;
 	int i;
 
 	WRITE_REG(sc, SAFE_RNG_CTRL, 0);
 	/* use default value according to the manual */
 	WRITE_REG(sc, SAFE_RNG_CNFG, 0x834);	/* magic from SafeNet */
 	WRITE_REG(sc, SAFE_RNG_ALM_CNT, 0);
 
 	/*
 	 * There is a bug in rev 1.0 of the 1140 that when the RNG
 	 * is brought out of reset the ready status flag does not
 	 * work until the RNG has finished its internal initialization.
 	 *
 	 * So in order to determine the device is through its
 	 * initialization we must read the data register, using the
 	 * status reg in the read in case it is initialized.  Then read
 	 * the data register until it changes from the first read.
 	 * Once it changes read the data register until it changes
 	 * again.  At this time the RNG is considered initialized. 
 	 * This could take between 750ms - 1000ms in time.
 	 */
 	i = 0;
 	w = READ_REG(sc, SAFE_RNG_OUT);
 	do {
 		v = READ_REG(sc, SAFE_RNG_OUT);
 		if (v != w) {
 			w = v;
 			break;
 		}
 		DELAY(10);
 	} while (++i < SAFE_RNG_MAXWAIT);
 
 	/* Wait Until data changes again */
 	i = 0;
 	do {
 		v = READ_REG(sc, SAFE_RNG_OUT);
 		if (v != w)
 			break;
 		DELAY(10);
 	} while (++i < SAFE_RNG_MAXWAIT);
 }
 
 static __inline void
 safe_rng_disable_short_cycle(struct safe_softc *sc)
 {
 	WRITE_REG(sc, SAFE_RNG_CTRL,
 		READ_REG(sc, SAFE_RNG_CTRL) &~ SAFE_RNG_CTRL_SHORTEN);
 }
 
 static __inline void
 safe_rng_enable_short_cycle(struct safe_softc *sc)
 {
 	WRITE_REG(sc, SAFE_RNG_CTRL, 
 		READ_REG(sc, SAFE_RNG_CTRL) | SAFE_RNG_CTRL_SHORTEN);
 }
 
 static __inline u_int32_t
 safe_rng_read(struct safe_softc *sc)
 {
 	int i;
 
 	i = 0;
 	while (READ_REG(sc, SAFE_RNG_STAT) != 0 && ++i < SAFE_RNG_MAXWAIT)
 		;
 	return READ_REG(sc, SAFE_RNG_OUT);
 }
 
 static void
 safe_rng(void *arg)
 {
 	struct safe_softc *sc = arg;
 	u_int32_t buf[SAFE_RNG_MAXBUFSIZ];	/* NB: maybe move to softc */
 	u_int maxwords;
 	int i;
 
 	safestats.st_rng++;
 	/*
 	 * Fetch the next block of data.
 	 */
 	maxwords = safe_rngbufsize;
 	if (maxwords > SAFE_RNG_MAXBUFSIZ)
 		maxwords = SAFE_RNG_MAXBUFSIZ;
 retry:
 	for (i = 0; i < maxwords; i++)
 		buf[i] = safe_rng_read(sc);
 	/*
 	 * Check the comparator alarm count and reset the h/w if
 	 * it exceeds our threshold.  This guards against the
 	 * hardware oscillators resonating with external signals.
 	 */
 	if (READ_REG(sc, SAFE_RNG_ALM_CNT) > safe_rngmaxalarm) {
 		u_int32_t freq_inc, w;
 
 		DPRINTF(("%s: alarm count %u exceeds threshold %u\n", __func__,
 			READ_REG(sc, SAFE_RNG_ALM_CNT), safe_rngmaxalarm));
 		safestats.st_rngalarm++;
 		safe_rng_enable_short_cycle(sc);
 		freq_inc = 18;
 		for (i = 0; i < 64; i++) {
 			w = READ_REG(sc, SAFE_RNG_CNFG);
 			freq_inc = ((w + freq_inc) & 0x3fL);
 			w = ((w & ~0x3fL) | freq_inc);
 			WRITE_REG(sc, SAFE_RNG_CNFG, w);
 
 			WRITE_REG(sc, SAFE_RNG_ALM_CNT, 0);
 
 			(void) safe_rng_read(sc);
 			DELAY(25);
 
 			if (READ_REG(sc, SAFE_RNG_ALM_CNT) == 0) {
 				safe_rng_disable_short_cycle(sc);
 				goto retry;
 			}
 			freq_inc = 1;
 		}
 		safe_rng_disable_short_cycle(sc);
 	} else
 		WRITE_REG(sc, SAFE_RNG_ALM_CNT, 0);
 
 	(*sc->sc_harvest)(sc->sc_rndtest, buf, maxwords*sizeof (u_int32_t));
 	callout_reset(&sc->sc_rngto,
 		hz * (safe_rnginterval ? safe_rnginterval : 1), safe_rng, sc);
 }
 #endif /* SAFE_NO_RNG */
 
 static void
 safe_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 {
 	bus_addr_t *paddr = (bus_addr_t*) arg;
 	*paddr = segs->ds_addr;
 }
 
 static int
 safe_dma_malloc(
 	struct safe_softc *sc,
 	bus_size_t size,
 	struct safe_dma_alloc *dma,
 	int mapflags
 )
 {
 	int r;
 
 	r = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev),	/* parent */
 			       sizeof(u_int32_t), 0,	/* alignment, bounds */
 			       BUS_SPACE_MAXADDR_32BIT,	/* lowaddr */
 			       BUS_SPACE_MAXADDR,	/* highaddr */
 			       NULL, NULL,		/* filter, filterarg */
 			       size,			/* maxsize */
 			       1,			/* nsegments */
 			       size,			/* maxsegsize */
 			       BUS_DMA_ALLOCNOW,	/* flags */
 			       NULL, NULL,		/* locking */
 			       &dma->dma_tag);
 	if (r != 0) {
 		device_printf(sc->sc_dev, "safe_dma_malloc: "
 			"bus_dma_tag_create failed; error %u\n", r);
 		goto fail_0;
 	}
 
 	r = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
 			     BUS_DMA_NOWAIT, &dma->dma_map);
 	if (r != 0) {
 		device_printf(sc->sc_dev, "safe_dma_malloc: "
 			"bus_dmammem_alloc failed; size %ju, error %u\n",
 			(uintmax_t)size, r);
 		goto fail_1;
 	}
 
 	r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
 		            size,
 			    safe_dmamap_cb,
 			    &dma->dma_paddr,
 			    mapflags | BUS_DMA_NOWAIT);
 	if (r != 0) {
 		device_printf(sc->sc_dev, "safe_dma_malloc: "
 			"bus_dmamap_load failed; error %u\n", r);
 		goto fail_2;
 	}
 
 	dma->dma_size = size;
 	return (0);
 
 	bus_dmamap_unload(dma->dma_tag, dma->dma_map);
 fail_2:
 	bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
 fail_1:
 	bus_dma_tag_destroy(dma->dma_tag);
 fail_0:
 	dma->dma_tag = NULL;
 	return (r);
 }
 
 static void
 safe_dma_free(struct safe_softc *sc, struct safe_dma_alloc *dma)
 {
 	bus_dmamap_unload(dma->dma_tag, dma->dma_map);
 	bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
 	bus_dma_tag_destroy(dma->dma_tag);
 }
 
 /*
  * Resets the board.  Values in the regesters are left as is
  * from the reset (i.e. initial values are assigned elsewhere).
  */
 static void
 safe_reset_board(struct safe_softc *sc)
 {
 	u_int32_t v;
 	/*
 	 * Reset the device.  The manual says no delay
 	 * is needed between marking and clearing reset.
 	 */
 	v = READ_REG(sc, SAFE_PE_DMACFG) &~
 		(SAFE_PE_DMACFG_PERESET | SAFE_PE_DMACFG_PDRRESET |
 		 SAFE_PE_DMACFG_SGRESET);
 	WRITE_REG(sc, SAFE_PE_DMACFG, v
 				    | SAFE_PE_DMACFG_PERESET
 				    | SAFE_PE_DMACFG_PDRRESET
 				    | SAFE_PE_DMACFG_SGRESET);
 	WRITE_REG(sc, SAFE_PE_DMACFG, v);
 }
 
 /*
  * Initialize registers we need to touch only once.
  */
 static void
 safe_init_board(struct safe_softc *sc)
 {
 	u_int32_t v, dwords;
 
 	v = READ_REG(sc, SAFE_PE_DMACFG);
 	v &=~ SAFE_PE_DMACFG_PEMODE;
 	v |= SAFE_PE_DMACFG_FSENA		/* failsafe enable */
 	  |  SAFE_PE_DMACFG_GPRPCI		/* gather ring on PCI */
 	  |  SAFE_PE_DMACFG_SPRPCI		/* scatter ring on PCI */
 	  |  SAFE_PE_DMACFG_ESDESC		/* endian-swap descriptors */
 	  |  SAFE_PE_DMACFG_ESSA		/* endian-swap SA's */
 	  |  SAFE_PE_DMACFG_ESPDESC		/* endian-swap part. desc's */
 	  ;
 	WRITE_REG(sc, SAFE_PE_DMACFG, v);
 #if 0
 	/* XXX select byte swap based on host byte order */
 	WRITE_REG(sc, SAFE_ENDIAN, 0x1b);
 #endif
 	if (sc->sc_chiprev == SAFE_REV(1,0)) {
 		/*
 		 * Avoid large PCI DMA transfers.  Rev 1.0 has a bug where
 		 * "target mode transfers" done while the chip is DMA'ing
 		 * >1020 bytes cause the hardware to lockup.  To avoid this
 		 * we reduce the max PCI transfer size and use small source
 		 * particle descriptors (<= 256 bytes).
 		 */
 		WRITE_REG(sc, SAFE_DMA_CFG, 256);
 		device_printf(sc->sc_dev,
 			"Reduce max DMA size to %u words for rev %u.%u WAR\n",
 			(READ_REG(sc, SAFE_DMA_CFG)>>2) & 0xff,
 			SAFE_REV_MAJ(sc->sc_chiprev),
 			SAFE_REV_MIN(sc->sc_chiprev));
 	}
 
 	/* NB: operands+results are overlaid */
 	WRITE_REG(sc, SAFE_PE_PDRBASE, sc->sc_ringalloc.dma_paddr);
 	WRITE_REG(sc, SAFE_PE_RDRBASE, sc->sc_ringalloc.dma_paddr);
 	/*
 	 * Configure ring entry size and number of items in the ring.
 	 */
 	KASSERT((sizeof(struct safe_ringentry) % sizeof(u_int32_t)) == 0,
 		("PE ring entry not 32-bit aligned!"));
 	dwords = sizeof(struct safe_ringentry) / sizeof(u_int32_t);
 	WRITE_REG(sc, SAFE_PE_RINGCFG,
 		(dwords << SAFE_PE_RINGCFG_OFFSET_S) | SAFE_MAX_NQUEUE);
 	WRITE_REG(sc, SAFE_PE_RINGPOLL, 0);	/* disable polling */
 
 	WRITE_REG(sc, SAFE_PE_GRNGBASE, sc->sc_spalloc.dma_paddr);
 	WRITE_REG(sc, SAFE_PE_SRNGBASE, sc->sc_dpalloc.dma_paddr);
 	WRITE_REG(sc, SAFE_PE_PARTSIZE,
 		(SAFE_TOTAL_DPART<<16) | SAFE_TOTAL_SPART);
 	/*
 	 * NB: destination particles are fixed size.  We use
 	 *     an mbuf cluster and require all results go to
 	 *     clusters or smaller.
 	 */
 	WRITE_REG(sc, SAFE_PE_PARTCFG, SAFE_MAX_DSIZE);
 
 	/* it's now safe to enable PE mode, do it */
 	WRITE_REG(sc, SAFE_PE_DMACFG, v | SAFE_PE_DMACFG_PEMODE);
 
 	/*
 	 * Configure hardware to use level-triggered interrupts and
 	 * to interrupt after each descriptor is processed.
 	 */
 	WRITE_REG(sc, SAFE_HI_CFG, SAFE_HI_CFG_LEVEL);
 	WRITE_REG(sc, SAFE_HI_DESC_CNT, 1);
 	WRITE_REG(sc, SAFE_HI_MASK, SAFE_INT_PE_DDONE | SAFE_INT_PE_ERROR);
 }
 
 /*
  * Init PCI registers
  */
 static void
 safe_init_pciregs(device_t dev)
 {
 }
 
 /*
  * Clean up after a chip crash.
  * It is assumed that the caller in splimp()
  */
 static void
 safe_cleanchip(struct safe_softc *sc)
 {
 
 	if (sc->sc_nqchip != 0) {
 		struct safe_ringentry *re = sc->sc_back;
 
 		while (re != sc->sc_front) {
 			if (re->re_desc.d_csr != 0)
 				safe_free_entry(sc, re);
 			if (++re == sc->sc_ringtop)
 				re = sc->sc_ring;
 		}
 		sc->sc_back = re;
 		sc->sc_nqchip = 0;
 	}
 }
 
 /*
  * free a safe_q
  * It is assumed that the caller is within splimp().
  */
 static int
 safe_free_entry(struct safe_softc *sc, struct safe_ringentry *re)
 {
 	struct cryptop *crp;
 
 	/*
 	 * Free header MCR
 	 */
-	if ((re->re_dst_m != NULL) && (re->re_src_m != re->re_dst_m))
+	if (re->re_dst_m != NULL)
 		m_freem(re->re_dst_m);
 
 	crp = (struct cryptop *)re->re_crp;
 	
 	re->re_desc.d_csr = 0;
 	
 	crp->crp_etype = EFAULT;
 	crypto_done(crp);
 	return(0);
 }
 
 /*
  * Routine to reset the chip and clean up.
  * It is assumed that the caller is in splimp()
  */
 static void
 safe_totalreset(struct safe_softc *sc)
 {
 	safe_reset_board(sc);
 	safe_init_board(sc);
 	safe_cleanchip(sc);
 }
 
 /*
  * Is the operand suitable aligned for direct DMA.  Each
  * segment must be aligned on a 32-bit boundary and all
  * but the last segment must be a multiple of 4 bytes.
  */
 static int
 safe_dmamap_aligned(const struct safe_operand *op)
 {
 	int i;
 
 	for (i = 0; i < op->nsegs; i++) {
 		if (op->segs[i].ds_addr & 3)
 			return (0);
 		if (i != (op->nsegs - 1) && (op->segs[i].ds_len & 3))
 			return (0);
 	}
 	return (1);
 }
 
 /*
  * Is the operand suitable for direct DMA as the destination
  * of an operation.  The hardware requires that each ``particle''
  * but the last in an operation result have the same size.  We
  * fix that size at SAFE_MAX_DSIZE bytes.  This routine returns
  * 0 if some segment is not a multiple of of this size, 1 if all
  * segments are exactly this size, or 2 if segments are at worst
  * a multple of this size.
  */
 static int
 safe_dmamap_uniform(const struct safe_operand *op)
 {
 	int result = 1;
 
 	if (op->nsegs > 0) {
 		int i;
 
 		for (i = 0; i < op->nsegs-1; i++) {
 			if (op->segs[i].ds_len % SAFE_MAX_DSIZE)
 				return (0);
 			if (op->segs[i].ds_len != SAFE_MAX_DSIZE)
 				result = 2;
 		}
 	}
 	return (result);
 }
 
 #ifdef SAFE_DEBUG
 static void
 safe_dump_dmastatus(struct safe_softc *sc, const char *tag)
 {
 	printf("%s: ENDIAN 0x%x SRC 0x%x DST 0x%x STAT 0x%x\n"
 		, tag
 		, READ_REG(sc, SAFE_DMA_ENDIAN)
 		, READ_REG(sc, SAFE_DMA_SRCADDR)
 		, READ_REG(sc, SAFE_DMA_DSTADDR)
 		, READ_REG(sc, SAFE_DMA_STAT)
 	);
 }
 
 static void
 safe_dump_intrstate(struct safe_softc *sc, const char *tag)
 {
 	printf("%s: HI_CFG 0x%x HI_MASK 0x%x HI_DESC_CNT 0x%x HU_STAT 0x%x HM_STAT 0x%x\n"
 		, tag
 		, READ_REG(sc, SAFE_HI_CFG)
 		, READ_REG(sc, SAFE_HI_MASK)
 		, READ_REG(sc, SAFE_HI_DESC_CNT)
 		, READ_REG(sc, SAFE_HU_STAT)
 		, READ_REG(sc, SAFE_HM_STAT)
 	);
 }
 
 static void
 safe_dump_ringstate(struct safe_softc *sc, const char *tag)
 {
 	u_int32_t estat = READ_REG(sc, SAFE_PE_ERNGSTAT);
 
 	/* NB: assume caller has lock on ring */
 	printf("%s: ERNGSTAT %x (next %u) back %lu front %lu\n",
 		tag,
 		estat, (estat >> SAFE_PE_ERNGSTAT_NEXT_S),
 		(unsigned long)(sc->sc_back - sc->sc_ring),
 		(unsigned long)(sc->sc_front - sc->sc_ring));
 }
 
 static void
 safe_dump_request(struct safe_softc *sc, const char* tag, struct safe_ringentry *re)
 {
 	int ix, nsegs;
 
 	ix = re - sc->sc_ring;
 	printf("%s: %p (%u): csr %x src %x dst %x sa %x len %x\n"
 		, tag
 		, re, ix
 		, re->re_desc.d_csr
 		, re->re_desc.d_src
 		, re->re_desc.d_dst
 		, re->re_desc.d_sa
 		, re->re_desc.d_len
 	);
 	if (re->re_src.nsegs > 1) {
 		ix = (re->re_desc.d_src - sc->sc_spalloc.dma_paddr) /
 			sizeof(struct safe_pdesc);
 		for (nsegs = re->re_src.nsegs; nsegs; nsegs--) {
 			printf(" spd[%u] %p: %p size %u flags %x"
 				, ix, &sc->sc_spring[ix]
 				, (caddr_t)(uintptr_t) sc->sc_spring[ix].pd_addr
 				, sc->sc_spring[ix].pd_size
 				, sc->sc_spring[ix].pd_flags
 			);
 			if (sc->sc_spring[ix].pd_size == 0)
 				printf(" (zero!)");
 			printf("\n");
 			if (++ix == SAFE_TOTAL_SPART)
 				ix = 0;
 		}
 	}
 	if (re->re_dst.nsegs > 1) {
 		ix = (re->re_desc.d_dst - sc->sc_dpalloc.dma_paddr) /
 			sizeof(struct safe_pdesc);
 		for (nsegs = re->re_dst.nsegs; nsegs; nsegs--) {
 			printf(" dpd[%u] %p: %p flags %x\n"
 				, ix, &sc->sc_dpring[ix]
 				, (caddr_t)(uintptr_t) sc->sc_dpring[ix].pd_addr
 				, sc->sc_dpring[ix].pd_flags
 			);
 			if (++ix == SAFE_TOTAL_DPART)
 				ix = 0;
 		}
 	}
 	printf("sa: cmd0 %08x cmd1 %08x staterec %x\n",
 		re->re_sa.sa_cmd0, re->re_sa.sa_cmd1, re->re_sa.sa_staterec);
 	printf("sa: key %x %x %x %x %x %x %x %x\n"
 		, re->re_sa.sa_key[0]
 		, re->re_sa.sa_key[1]
 		, re->re_sa.sa_key[2]
 		, re->re_sa.sa_key[3]
 		, re->re_sa.sa_key[4]
 		, re->re_sa.sa_key[5]
 		, re->re_sa.sa_key[6]
 		, re->re_sa.sa_key[7]
 	);
 	printf("sa: indigest %x %x %x %x %x\n"
 		, re->re_sa.sa_indigest[0]
 		, re->re_sa.sa_indigest[1]
 		, re->re_sa.sa_indigest[2]
 		, re->re_sa.sa_indigest[3]
 		, re->re_sa.sa_indigest[4]
 	);
 	printf("sa: outdigest %x %x %x %x %x\n"
 		, re->re_sa.sa_outdigest[0]
 		, re->re_sa.sa_outdigest[1]
 		, re->re_sa.sa_outdigest[2]
 		, re->re_sa.sa_outdigest[3]
 		, re->re_sa.sa_outdigest[4]
 	);
 	printf("sr: iv %x %x %x %x\n"
 		, re->re_sastate.sa_saved_iv[0]
 		, re->re_sastate.sa_saved_iv[1]
 		, re->re_sastate.sa_saved_iv[2]
 		, re->re_sastate.sa_saved_iv[3]
 	);
 	printf("sr: hashbc %u indigest %x %x %x %x %x\n"
 		, re->re_sastate.sa_saved_hashbc
 		, re->re_sastate.sa_saved_indigest[0]
 		, re->re_sastate.sa_saved_indigest[1]
 		, re->re_sastate.sa_saved_indigest[2]
 		, re->re_sastate.sa_saved_indigest[3]
 		, re->re_sastate.sa_saved_indigest[4]
 	);
 }
 
 static void
 safe_dump_ring(struct safe_softc *sc, const char *tag)
 {
 	mtx_lock(&sc->sc_ringmtx);
 	printf("\nSafeNet Ring State:\n");
 	safe_dump_intrstate(sc, tag);
 	safe_dump_dmastatus(sc, tag);
 	safe_dump_ringstate(sc, tag);
 	if (sc->sc_nqchip) {
 		struct safe_ringentry *re = sc->sc_back;
 		do {
 			safe_dump_request(sc, tag, re);
 			if (++re == sc->sc_ringtop)
 				re = sc->sc_ring;
 		} while (re != sc->sc_front);
 	}
 	mtx_unlock(&sc->sc_ringmtx);
 }
 
 static int
 sysctl_hw_safe_dump(SYSCTL_HANDLER_ARGS)
 {
 	char dmode[64];
 	int error;
 
 	strncpy(dmode, "", sizeof(dmode) - 1);
 	dmode[sizeof(dmode) - 1] = '\0';
 	error = sysctl_handle_string(oidp, &dmode[0], sizeof(dmode), req);
 
 	if (error == 0 && req->newptr != NULL) {
 		struct safe_softc *sc = safec;
 
 		if (!sc)
 			return EINVAL;
 		if (strncmp(dmode, "dma", 3) == 0)
 			safe_dump_dmastatus(sc, "safe0");
 		else if (strncmp(dmode, "int", 3) == 0)
 			safe_dump_intrstate(sc, "safe0");
 		else if (strncmp(dmode, "ring", 4) == 0)
 			safe_dump_ring(sc, "safe0");
 		else
 			return EINVAL;
 	}
 	return error;
 }
 SYSCTL_PROC(_hw_safe, OID_AUTO, dump,
     CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 0, 0,
     sysctl_hw_safe_dump, "A",
     "Dump driver state");
 #endif /* SAFE_DEBUG */
Index: head/sys/dev/safe/safevar.h
===================================================================
--- head/sys/dev/safe/safevar.h	(revision 359373)
+++ head/sys/dev/safe/safevar.h	(revision 359374)
@@ -1,214 +1,206 @@
 /*-
  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
  *
  * Copyright (c) 2003 Sam Leffler, Errno Consulting
  * Copyright (c) 2003 Global Technology Associates, Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
  * $FreeBSD$
  */
 #ifndef _SAFE_SAFEVAR_H_
 #define	_SAFE_SAFEVAR_H_
 
 /* Maximum queue length */
 #ifndef SAFE_MAX_NQUEUE
 #define SAFE_MAX_NQUEUE	60
 #endif
 
 #define	SAFE_MAX_PART		64	/* Maximum scatter/gather depth */
 #define	SAFE_DMA_BOUNDARY	0	/* No boundary for source DMA ops */
 #define	SAFE_MAX_DSIZE		MCLBYTES /* Fixed scatter particle size */
 #define	SAFE_MAX_SSIZE		0x0ffff	/* Maximum gather particle size */
 #define	SAFE_MAX_DMA		0xfffff	/* Maximum PE operand size (20 bits) */
 /* total src+dst particle descriptors */
 #define	SAFE_TOTAL_DPART	(SAFE_MAX_NQUEUE * SAFE_MAX_PART)
 #define	SAFE_TOTAL_SPART	(SAFE_MAX_NQUEUE * SAFE_MAX_PART)
 
 #define	SAFE_RNG_MAXBUFSIZ	128	/* 32-bit words */
 
 #define SAFE_DEF_RTY		0xff	/* PCI Retry Timeout */
 #define SAFE_DEF_TOUT		0xff	/* PCI TRDY Timeout */
 #define SAFE_DEF_CACHELINE	0x01	/* Cache Line setting */
 
 #ifdef _KERNEL
 /*
  * State associated with the allocation of each chunk
  * of memory setup for DMA.
  */
 struct safe_dma_alloc {
 	u_int32_t		dma_paddr;	/* physical address */
 	caddr_t			dma_vaddr;	/* virtual address */
 	bus_dma_tag_t		dma_tag;	/* bus dma tag used */
 	bus_dmamap_t		dma_map;	/* associated map */
 	bus_dma_segment_t	dma_seg;
 	bus_size_t		dma_size;	/* mapped memory size (bytes) */
 	int			dma_nseg;	/* number of segments */
 };
 
 /*
  * Cryptographic operand state.  One of these exists for each
  * source and destination operand passed in from the crypto
  * subsystem.  When possible source and destination operands
  * refer to the same memory.  More often they are distinct.
  * We track the virtual address of each operand as well as
  * where each is mapped for DMA.
  */
 struct safe_operand {
-	union {
-		struct mbuf *m;
-		struct uio *io;
-	} u;
 	bus_dmamap_t		map;
 	bus_size_t		mapsize;
 	int			nsegs;
 	bus_dma_segment_t	segs[SAFE_MAX_PART];
 };
 
 /*
  * Packet engine ring entry and cryptographic operation state.
  * The packet engine requires a ring of descriptors that contain
  * pointers to various cryptographic state.  However the ring
  * configuration register allows you to specify an arbitrary size
  * for ring entries.  We use this feature to collect most of the
  * state for each cryptographic request into one spot.  Other than
  * ring entries only the ``particle descriptors'' (scatter/gather
  * lists) and the actual operand data are kept separate.  The
  * particle descriptors must also be organized in rings.  The
  * operand data can be located aribtrarily (modulo alignment constraints).
  *
  * Note that the descriptor ring is mapped onto the PCI bus so
  * the hardware can DMA data.  This means the entire ring must be
  * contiguous.
  */
 struct safe_ringentry {
 	struct safe_desc	re_desc;	/* command descriptor */
 	struct safe_sarec	re_sa;		/* SA record */
 	struct safe_sastate	re_sastate;	/* SA state record */
 	struct cryptop		*re_crp;	/* crypto operation */
 
 	struct safe_operand	re_src;		/* source operand */
 	struct safe_operand	re_dst;		/* destination operand */
+	struct mbuf		*re_dst_m;
 
 	int			unused;
 	int			re_flags;
-#define	SAFE_QFLAGS_COPYOUTIV	0x1		/* copy back on completion */
 #define	SAFE_QFLAGS_COPYOUTICV	0x2		/* copy back on completion */
 };
 
-#define	re_src_m	re_src.u.m
-#define	re_src_io	re_src.u.io
 #define	re_src_map	re_src.map
 #define	re_src_nsegs	re_src.nsegs
 #define	re_src_segs	re_src.segs
 #define	re_src_mapsize	re_src.mapsize
 
-#define	re_dst_m	re_dst.u.m
-#define	re_dst_io	re_dst.u.io
 #define	re_dst_map	re_dst.map
 #define	re_dst_nsegs	re_dst.nsegs
 #define	re_dst_segs	re_dst.segs
 #define	re_dst_mapsize	re_dst.mapsize
 
 struct rndstate_test;
 
 struct safe_session {
 	u_int32_t	ses_klen;		/* key length in bits */
 	u_int32_t	ses_key[8];		/* DES/3DES/AES key */
 	u_int32_t	ses_mlen;		/* hmac length in bytes */
 	u_int32_t	ses_hminner[5];		/* hmac inner state */
 	u_int32_t	ses_hmouter[5];		/* hmac outer state */
-	u_int32_t	ses_iv[4];		/* DES/3DES/AES iv */
 };
 
 struct safe_softc {
 	device_t		sc_dev;		/* device backpointer */
 	struct resource		*sc_irq;
 	void			*sc_ih;		/* interrupt handler cookie */
 	bus_space_handle_t	sc_sh;		/* memory handle */
 	bus_space_tag_t		sc_st;		/* memory tag */
 	struct resource		*sc_sr;		/* memory resource */
 	bus_dma_tag_t		sc_srcdmat;	/* source dma tag */
 	bus_dma_tag_t		sc_dstdmat;	/* destination dma tag */
 	u_int			sc_chiprev;	/* major/minor chip revision */
 	int			sc_flags;	/* device specific flags */
 #define	SAFE_FLAGS_KEY		0x01		/* has key accelerator */
 #define	SAFE_FLAGS_RNG		0x02		/* hardware rng */
 	int			sc_suspended;
 	int			sc_needwakeup;	/* notify crypto layer */
 	int32_t			sc_cid;		/* crypto tag */
+	uint32_t		sc_devinfo;
 	struct safe_dma_alloc	sc_ringalloc;	/* PE ring allocation state */
 	struct safe_ringentry	*sc_ring;	/* PE ring */
 	struct safe_ringentry	*sc_ringtop;	/* PE ring top */
 	struct safe_ringentry	*sc_front;	/* next free entry */
 	struct safe_ringentry	*sc_back;	/* next pending entry */
 	int			sc_nqchip;	/* # passed to chip */
 	struct mtx		sc_ringmtx;	/* PE ring lock */
 	struct safe_pdesc	*sc_spring;	/* src particle ring */
 	struct safe_pdesc	*sc_springtop;	/* src particle ring top */
 	struct safe_pdesc	*sc_spfree;	/* next free src particle */
 	struct safe_dma_alloc	sc_spalloc;	/* src particle ring state */
 	struct safe_pdesc	*sc_dpring;	/* dest particle ring */
 	struct safe_pdesc	*sc_dpringtop;	/* dest particle ring top */
 	struct safe_pdesc	*sc_dpfree;	/* next free dest particle */
 	struct safe_dma_alloc	sc_dpalloc;	/* dst particle ring state */
 
 	struct callout		sc_rngto;	/* rng timeout */
 	struct rndtest_state	*sc_rndtest;	/* RNG test state */
 	void			(*sc_harvest)(struct rndtest_state *,
 					void *, u_int);
 };
 #endif /* _KERNEL */
 
 struct safe_stats {
 	u_int64_t st_ibytes;
 	u_int64_t st_obytes;
 	u_int32_t st_ipackets;
 	u_int32_t st_opackets;
 	u_int32_t st_invalid;		/* invalid argument */
 	u_int32_t st_badsession;	/* invalid session id */
 	u_int32_t st_badflags;		/* flags indicate !(mbuf | uio) */
 	u_int32_t st_nodesc;		/* op submitted w/o descriptors */
 	u_int32_t st_badalg;		/* unsupported algorithm */
 	u_int32_t st_ringfull;		/* PE descriptor ring full */
 	u_int32_t st_peoperr;		/* PE marked error */
 	u_int32_t st_dmaerr;		/* PE DMA error */
 	u_int32_t st_bypasstoobig;	/* bypass > 96 bytes */
 	u_int32_t st_skipmismatch;	/* enc part begins before auth part */
 	u_int32_t st_lenmismatch;	/* enc length different auth length */
 	u_int32_t st_coffmisaligned;	/* crypto offset not 32-bit aligned */
 	u_int32_t st_cofftoobig;	/* crypto offset > 255 words */
 	u_int32_t st_iovmisaligned;	/* iov op not aligned */
 	u_int32_t st_iovnotuniform;	/* iov op not suitable */
 	u_int32_t st_unaligned;		/* unaligned src caused copy */
 	u_int32_t st_notuniform;	/* non-uniform src caused copy */
 	u_int32_t st_nomap;		/* bus_dmamap_create failed */
 	u_int32_t st_noload;		/* bus_dmamap_load_* failed */
 	u_int32_t st_nombuf;		/* MGET* failed */
 	u_int32_t st_nomcl;		/* MCLGET* failed */
 	u_int32_t st_maxqchip;		/* max mcr1 ops out for processing */
 	u_int32_t st_rng;		/* RNG requests */
 	u_int32_t st_rngalarm;		/* RNG alarm requests */
 	u_int32_t st_noicvcopy;		/* ICV data copies suppressed */
 };
 #endif /* _SAFE_SAFEVAR_H_ */
Index: head/sys/dev/sec/sec.c
===================================================================
--- head/sys/dev/sec/sec.c	(revision 359373)
+++ head/sys/dev/sec/sec.c	(revision 359374)
@@ -1,1785 +1,1671 @@
 /*-
  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
  *
  * Copyright (C) 2008-2009 Semihalf, Piotr Ziecik
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN
  * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
  * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
 /*
  * Freescale integrated Security Engine (SEC) driver. Currently SEC 2.0 and
  * 3.0 are supported.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/bus.h>
 #include <sys/endian.h>
 #include <sys/kernel.h>
 #include <sys/lock.h>
 #include <sys/malloc.h>
 #include <sys/mbuf.h>
 #include <sys/module.h>
 #include <sys/mutex.h>
 #include <sys/random.h>
 #include <sys/rman.h>
 
 #include <machine/_inttypes.h>
 #include <machine/bus.h>
 #include <machine/resource.h>
 
 #include <opencrypto/cryptodev.h>
+#include <opencrypto/xform_auth.h>
 #include "cryptodev_if.h"
 
 #include <dev/ofw/ofw_bus_subr.h>
 #include <dev/sec/sec.h>
 
 static int	sec_probe(device_t dev);
 static int	sec_attach(device_t dev);
 static int	sec_detach(device_t dev);
 static int	sec_suspend(device_t dev);
 static int	sec_resume(device_t dev);
 static int	sec_shutdown(device_t dev);
 static void	sec_primary_intr(void *arg);
 static void	sec_secondary_intr(void *arg);
 static int	sec_setup_intr(struct sec_softc *sc, struct resource **ires,
     void **ihand, int *irid, driver_intr_t handler, const char *iname);
 static void	sec_release_intr(struct sec_softc *sc, struct resource *ires,
     void *ihand, int irid, const char *iname);
 static int	sec_controller_reset(struct sec_softc *sc);
 static int	sec_channel_reset(struct sec_softc *sc, int channel, int full);
 static int	sec_init(struct sec_softc *sc);
 static int	sec_alloc_dma_mem(struct sec_softc *sc,
     struct sec_dma_mem *dma_mem, bus_size_t size);
 static int	sec_desc_map_dma(struct sec_softc *sc,
-    struct sec_dma_mem *dma_mem, void *mem, bus_size_t size, int type,
+    struct sec_dma_mem *dma_mem, struct cryptop *crp, bus_size_t size,
     struct sec_desc_map_info *sdmi);
 static void	sec_free_dma_mem(struct sec_dma_mem *dma_mem);
 static void	sec_enqueue(struct sec_softc *sc);
 static int	sec_enqueue_desc(struct sec_softc *sc, struct sec_desc *desc,
     int channel);
 static int	sec_eu_channel(struct sec_softc *sc, int eu);
 static int	sec_make_pointer(struct sec_softc *sc, struct sec_desc *desc,
-    u_int n, void *data, bus_size_t doffset, bus_size_t dsize, int dtype);
+    u_int n, struct cryptop *crp, bus_size_t doffset, bus_size_t dsize);
 static int	sec_make_pointer_direct(struct sec_softc *sc,
     struct sec_desc *desc, u_int n, bus_addr_t data, bus_size_t dsize);
+static int	sec_probesession(device_t dev,
+    const struct crypto_session_params *csp);
 static int	sec_newsession(device_t dev, crypto_session_t cses,
-    struct cryptoini *cri);
+    const struct crypto_session_params *csp);
 static int	sec_process(device_t dev, struct cryptop *crp, int hint);
-static int	sec_split_cri(struct cryptoini *cri, struct cryptoini **enc,
-    struct cryptoini **mac);
-static int	sec_split_crp(struct cryptop *crp, struct cryptodesc **enc,
-    struct cryptodesc **mac);
 static int	sec_build_common_ns_desc(struct sec_softc *sc,
-    struct sec_desc *desc, struct sec_session *ses, struct cryptop *crp,
-    struct cryptodesc *enc, int buftype);
+    struct sec_desc *desc, const struct crypto_session_params *csp,
+    struct cryptop *crp);
 static int	sec_build_common_s_desc(struct sec_softc *sc,
-    struct sec_desc *desc, struct sec_session *ses, struct cryptop *crp,
-    struct cryptodesc *enc, struct cryptodesc *mac, int buftype);
+    struct sec_desc *desc, const struct crypto_session_params *csp,
+    struct cryptop *crp);
 
 static struct sec_desc *sec_find_desc(struct sec_softc *sc, bus_addr_t paddr);
 
 /* AESU */
-static int	sec_aesu_newsession(struct sec_softc *sc,
-    struct sec_session *ses, struct cryptoini *enc, struct cryptoini *mac);
+static bool	sec_aesu_newsession(const struct crypto_session_params *csp);
 static int	sec_aesu_make_desc(struct sec_softc *sc,
-    struct sec_session *ses, struct sec_desc *desc, struct cryptop *crp,
-    int buftype);
+    const struct crypto_session_params *csp, struct sec_desc *desc,
+    struct cryptop *crp);
 
 /* DEU */
-static int	sec_deu_newsession(struct sec_softc *sc,
-    struct sec_session *ses, struct cryptoini *enc, struct cryptoini *mac);
+static bool	sec_deu_newsession(const struct crypto_session_params *csp);
 static int	sec_deu_make_desc(struct sec_softc *sc,
-    struct sec_session *ses, struct sec_desc *desc, struct cryptop *crp,
-    int buftype);
+    const struct crypto_session_params *csp, struct sec_desc *desc,
+    struct cryptop *crp);
 
 /* MDEU */
-static int	sec_mdeu_can_handle(u_int alg);
-static int	sec_mdeu_config(struct cryptodesc *crd,
+static bool	sec_mdeu_can_handle(u_int alg);
+static int	sec_mdeu_config(const struct crypto_session_params *csp,
     u_int *eu, u_int *mode, u_int *hashlen);
-static int	sec_mdeu_newsession(struct sec_softc *sc,
-    struct sec_session *ses, struct cryptoini *enc, struct cryptoini *mac);
+static bool	sec_mdeu_newsession(const struct crypto_session_params *csp);
 static int	sec_mdeu_make_desc(struct sec_softc *sc,
-    struct sec_session *ses, struct sec_desc *desc, struct cryptop *crp,
-    int buftype);
+    const struct crypto_session_params *csp, struct sec_desc *desc,
+    struct cryptop *crp);
 
 static device_method_t sec_methods[] = {
 	/* Device interface */
 	DEVMETHOD(device_probe,		sec_probe),
 	DEVMETHOD(device_attach,	sec_attach),
 	DEVMETHOD(device_detach,	sec_detach),
 
 	DEVMETHOD(device_suspend,	sec_suspend),
 	DEVMETHOD(device_resume,	sec_resume),
 	DEVMETHOD(device_shutdown,	sec_shutdown),
 
 	/* Crypto methods */
+	DEVMETHOD(cryptodev_probesession, sec_probesession),
 	DEVMETHOD(cryptodev_newsession,	sec_newsession),
 	DEVMETHOD(cryptodev_process,	sec_process),
 
 	DEVMETHOD_END
 };
 static driver_t sec_driver = {
 	"sec",
 	sec_methods,
 	sizeof(struct sec_softc),
 };
 
 static devclass_t sec_devclass;
 DRIVER_MODULE(sec, simplebus, sec_driver, sec_devclass, 0, 0);
 MODULE_DEPEND(sec, crypto, 1, 1, 1);
 
 static struct sec_eu_methods sec_eus[] = {
 	{
 		sec_aesu_newsession,
 		sec_aesu_make_desc,
 	},
 	{
 		sec_deu_newsession,
 		sec_deu_make_desc,
 	},
 	{
 		sec_mdeu_newsession,
 		sec_mdeu_make_desc,
 	},
 	{ NULL, NULL }
 };
 
 static inline void
 sec_sync_dma_mem(struct sec_dma_mem *dma_mem, bus_dmasync_op_t op)
 {
 
 	/* Sync only if dma memory is valid */
 	if (dma_mem->dma_vaddr != NULL)
 		bus_dmamap_sync(dma_mem->dma_tag, dma_mem->dma_map, op);
 }
 
 static inline void *
 sec_get_pointer_data(struct sec_desc *desc, u_int n)
 {
 
 	return (desc->sd_ptr_dmem[n].dma_vaddr);
 }
 
 static int
 sec_probe(device_t dev)
 {
 	struct sec_softc *sc;
 	uint64_t id;
 
 	if (!ofw_bus_status_okay(dev))
 		return (ENXIO);
 
 	if (!ofw_bus_is_compatible(dev, "fsl,sec2.0"))
 		return (ENXIO);
 
 	sc = device_get_softc(dev);
 
 	sc->sc_rrid = 0;
 	sc->sc_rres = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->sc_rrid,
 	    RF_ACTIVE);
 
 	if (sc->sc_rres == NULL)
 		return (ENXIO);
 
 	sc->sc_bas.bsh = rman_get_bushandle(sc->sc_rres);
 	sc->sc_bas.bst = rman_get_bustag(sc->sc_rres);
 
 	id = SEC_READ(sc, SEC_ID);
 
 	bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, sc->sc_rres);
 
 	switch (id) {
 	case SEC_20_ID:
 		device_set_desc(dev, "Freescale Security Engine 2.0");
 		sc->sc_version = 2;
 		break;
 	case SEC_30_ID:
 		device_set_desc(dev, "Freescale Security Engine 3.0");
 		sc->sc_version = 3;
 		break;
 	case SEC_31_ID:
 		device_set_desc(dev, "Freescale Security Engine 3.1");
 		sc->sc_version = 3;
 		break;
 	default:
 		device_printf(dev, "unknown SEC ID 0x%016"PRIx64"!\n", id);
 		return (ENXIO);
 	}
 
 	return (0);
 }
 
 static int
 sec_attach(device_t dev)
 {
 	struct sec_softc *sc;
 	struct sec_hw_lt *lt;
 	int error = 0;
 	int i;
 
 	sc = device_get_softc(dev);
 	sc->sc_dev = dev;
 	sc->sc_blocked = 0;
 	sc->sc_shutdown = 0;
 
 	sc->sc_cid = crypto_get_driverid(dev, sizeof(struct sec_session),
 	    CRYPTOCAP_F_HARDWARE);
 	if (sc->sc_cid < 0) {
 		device_printf(dev, "could not get crypto driver ID!\n");
 		return (ENXIO);
 	}
 
 	/* Init locks */
 	mtx_init(&sc->sc_controller_lock, device_get_nameunit(dev),
 	    "SEC Controller lock", MTX_DEF);
 	mtx_init(&sc->sc_descriptors_lock, device_get_nameunit(dev),
 	    "SEC Descriptors lock", MTX_DEF);
 
 	/* Allocate I/O memory for SEC registers */
 	sc->sc_rrid = 0;
 	sc->sc_rres = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->sc_rrid,
 	    RF_ACTIVE);
 
 	if (sc->sc_rres == NULL) {
 		device_printf(dev, "could not allocate I/O memory!\n");
 		goto fail1;
 	}
 
 	sc->sc_bas.bsh = rman_get_bushandle(sc->sc_rres);
 	sc->sc_bas.bst = rman_get_bustag(sc->sc_rres);
 
 	/* Setup interrupts */
 	sc->sc_pri_irid = 0;
 	error = sec_setup_intr(sc, &sc->sc_pri_ires, &sc->sc_pri_ihand,
 	    &sc->sc_pri_irid, sec_primary_intr, "primary");
 
 	if (error)
 		goto fail2;
 
 
 	if (sc->sc_version == 3) {
 		sc->sc_sec_irid = 1;
 		error = sec_setup_intr(sc, &sc->sc_sec_ires, &sc->sc_sec_ihand,
 		    &sc->sc_sec_irid, sec_secondary_intr, "secondary");
 
 		if (error)
 			goto fail3;
 	}
 
 	/* Alloc DMA memory for descriptors and link tables */
 	error = sec_alloc_dma_mem(sc, &(sc->sc_desc_dmem),
 	    SEC_DESCRIPTORS * sizeof(struct sec_hw_desc));
 
 	if (error)
 		goto fail4;
 
 	error = sec_alloc_dma_mem(sc, &(sc->sc_lt_dmem),
 	    (SEC_LT_ENTRIES + 1) * sizeof(struct sec_hw_lt));
 
 	if (error)
 		goto fail5;
 
 	/* Fill in descriptors and link tables */
 	for (i = 0; i < SEC_DESCRIPTORS; i++) {
 		sc->sc_desc[i].sd_desc =
 		    (struct sec_hw_desc*)(sc->sc_desc_dmem.dma_vaddr) + i;
 		sc->sc_desc[i].sd_desc_paddr = sc->sc_desc_dmem.dma_paddr +
 		    (i * sizeof(struct sec_hw_desc));
 	}
 
 	for (i = 0; i < SEC_LT_ENTRIES + 1; i++) {
 		sc->sc_lt[i].sl_lt =
 		    (struct sec_hw_lt*)(sc->sc_lt_dmem.dma_vaddr) + i;
 		sc->sc_lt[i].sl_lt_paddr = sc->sc_lt_dmem.dma_paddr +
 		    (i * sizeof(struct sec_hw_lt));
 	}
 
 	/* Last entry in link table is used to create a circle */
 	lt = sc->sc_lt[SEC_LT_ENTRIES].sl_lt;
 	lt->shl_length = 0;
 	lt->shl_r = 0;
 	lt->shl_n = 1;
 	lt->shl_ptr = sc->sc_lt[0].sl_lt_paddr;
 
 	/* Init descriptor and link table queues pointers */
 	SEC_CNT_INIT(sc, sc_free_desc_get_cnt, SEC_DESCRIPTORS);
 	SEC_CNT_INIT(sc, sc_free_desc_put_cnt, SEC_DESCRIPTORS);
 	SEC_CNT_INIT(sc, sc_ready_desc_get_cnt, SEC_DESCRIPTORS);
 	SEC_CNT_INIT(sc, sc_ready_desc_put_cnt, SEC_DESCRIPTORS);
 	SEC_CNT_INIT(sc, sc_queued_desc_get_cnt, SEC_DESCRIPTORS);
 	SEC_CNT_INIT(sc, sc_queued_desc_put_cnt, SEC_DESCRIPTORS);
 	SEC_CNT_INIT(sc, sc_lt_alloc_cnt, SEC_LT_ENTRIES);
 	SEC_CNT_INIT(sc, sc_lt_free_cnt, SEC_LT_ENTRIES);
 
 	/* Create masks for fast checks */
 	sc->sc_int_error_mask = 0;
 	for (i = 0; i < SEC_CHANNELS; i++)
 		sc->sc_int_error_mask |= (~0ULL & SEC_INT_CH_ERR(i));
 
 	switch (sc->sc_version) {
 	case 2:
 		sc->sc_channel_idle_mask =
 		    (SEC_CHAN_CSR2_FFLVL_M << SEC_CHAN_CSR2_FFLVL_S) |
 		    (SEC_CHAN_CSR2_MSTATE_M << SEC_CHAN_CSR2_MSTATE_S) |
 		    (SEC_CHAN_CSR2_PSTATE_M << SEC_CHAN_CSR2_PSTATE_S) |
 		    (SEC_CHAN_CSR2_GSTATE_M << SEC_CHAN_CSR2_GSTATE_S);
 		break;
 	case 3:
 		sc->sc_channel_idle_mask =
 		    (SEC_CHAN_CSR3_FFLVL_M << SEC_CHAN_CSR3_FFLVL_S) |
 		    (SEC_CHAN_CSR3_MSTATE_M << SEC_CHAN_CSR3_MSTATE_S) |
 		    (SEC_CHAN_CSR3_PSTATE_M << SEC_CHAN_CSR3_PSTATE_S) |
 		    (SEC_CHAN_CSR3_GSTATE_M << SEC_CHAN_CSR3_GSTATE_S);
 		break;
 	}
 
 	/* Init hardware */
 	error = sec_init(sc);
 
 	if (error)
 		goto fail6;
 
-	/* Register in OCF (AESU) */
-	crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
-
-	/* Register in OCF (DEU) */
-	crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
-	crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
-
-	/* Register in OCF (MDEU) */
-	crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0);
-	crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
-	crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0);
-	crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
-	crypto_register(sc->sc_cid, CRYPTO_SHA2_256_HMAC, 0, 0);
-	if (sc->sc_version >= 3) {
-		crypto_register(sc->sc_cid, CRYPTO_SHA2_384_HMAC, 0, 0);
-		crypto_register(sc->sc_cid, CRYPTO_SHA2_512_HMAC, 0, 0);
-	}
-
 	return (0);
 
 fail6:
 	sec_free_dma_mem(&(sc->sc_lt_dmem));
 fail5:
 	sec_free_dma_mem(&(sc->sc_desc_dmem));
 fail4:
 	sec_release_intr(sc, sc->sc_sec_ires, sc->sc_sec_ihand,
 	    sc->sc_sec_irid, "secondary");
 fail3:
 	sec_release_intr(sc, sc->sc_pri_ires, sc->sc_pri_ihand,
 	    sc->sc_pri_irid, "primary");
 fail2:
 	bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid, sc->sc_rres);
 fail1:
 	mtx_destroy(&sc->sc_controller_lock);
 	mtx_destroy(&sc->sc_descriptors_lock);
 
 	return (ENXIO);
 }
 
 static int
 sec_detach(device_t dev)
 {
 	struct sec_softc *sc = device_get_softc(dev);
 	int i, error, timeout = SEC_TIMEOUT;
 
 	/* Prepare driver to shutdown */
 	SEC_LOCK(sc, descriptors);
 	sc->sc_shutdown = 1;
 	SEC_UNLOCK(sc, descriptors);
 
 	/* Wait until all queued processing finishes */
 	while (1) {
 		SEC_LOCK(sc, descriptors);
 		i = SEC_READY_DESC_CNT(sc) + SEC_QUEUED_DESC_CNT(sc);
 		SEC_UNLOCK(sc, descriptors);
 
 		if (i == 0)
 			break;
 
 		if (timeout < 0) {
 			device_printf(dev, "queue flush timeout!\n");
 
 			/* DMA can be still active - stop it */
 			for (i = 0; i < SEC_CHANNELS; i++)
 				sec_channel_reset(sc, i, 1);
 
 			break;
 		}
 
 		timeout -= 1000;
 		DELAY(1000);
 	}
 
 	/* Disable interrupts */
 	SEC_WRITE(sc, SEC_IER, 0);
 
 	/* Unregister from OCF */
 	crypto_unregister_all(sc->sc_cid);
 
 	/* Free DMA memory */
 	for (i = 0; i < SEC_DESCRIPTORS; i++)
 		SEC_DESC_FREE_POINTERS(&(sc->sc_desc[i]));
 
 	sec_free_dma_mem(&(sc->sc_lt_dmem));
 	sec_free_dma_mem(&(sc->sc_desc_dmem));
 
 	/* Release interrupts */
 	sec_release_intr(sc, sc->sc_pri_ires, sc->sc_pri_ihand,
 	    sc->sc_pri_irid, "primary");
 	sec_release_intr(sc, sc->sc_sec_ires, sc->sc_sec_ihand,
 	    sc->sc_sec_irid, "secondary");
 
 	/* Release memory */
 	if (sc->sc_rres) {
 		error = bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_rrid,
 		    sc->sc_rres);
 		if (error)
 			device_printf(dev, "bus_release_resource() failed for"
 			    " I/O memory, error %d\n", error);
 
 		sc->sc_rres = NULL;
 	}
 
 	mtx_destroy(&sc->sc_controller_lock);
 	mtx_destroy(&sc->sc_descriptors_lock);
 
 	return (0);
 }
 
 static int
 sec_suspend(device_t dev)
 {
 
 	return (0);
 }
 
 static int
 sec_resume(device_t dev)
 {
 
 	return (0);
 }
 
 static int
 sec_shutdown(device_t dev)
 {
 
 	return (0);
 }
 
 static int
 sec_setup_intr(struct sec_softc *sc, struct resource **ires, void **ihand,
     int *irid, driver_intr_t handler, const char *iname)
 {
 	int error;
 
 	(*ires) = bus_alloc_resource_any(sc->sc_dev, SYS_RES_IRQ, irid,
 	    RF_ACTIVE);
 
 	if ((*ires) == NULL) {
 		device_printf(sc->sc_dev, "could not allocate %s IRQ\n", iname);
 		return (ENXIO);
 	}
 
 	error = bus_setup_intr(sc->sc_dev, *ires, INTR_MPSAFE | INTR_TYPE_NET,
 	    NULL, handler, sc, ihand);
 
 	if (error) {
 		device_printf(sc->sc_dev, "failed to set up %s IRQ\n", iname);
 		if (bus_release_resource(sc->sc_dev, SYS_RES_IRQ, *irid, *ires))
 			device_printf(sc->sc_dev, "could not release %s IRQ\n",
 			    iname);
 
 		(*ires) = NULL;
 		return (error);
 	}
 
 	return (0);
 }
 
 static void
 sec_release_intr(struct sec_softc *sc, struct resource *ires, void *ihand,
     int irid, const char *iname)
 {
 	int error;
 
 	if (ires == NULL)
 		return;
 
 	error = bus_teardown_intr(sc->sc_dev, ires, ihand);
 	if (error)
 		device_printf(sc->sc_dev, "bus_teardown_intr() failed for %s"
 		    " IRQ, error %d\n", iname, error);
 
 	error = bus_release_resource(sc->sc_dev, SYS_RES_IRQ, irid, ires);
 	if (error)
 		device_printf(sc->sc_dev, "bus_release_resource() failed for %s"
 		    " IRQ, error %d\n", iname, error);
 }
 
 static void
 sec_primary_intr(void *arg)
 {
+	struct sec_session *ses;
 	struct sec_softc *sc = arg;
 	struct sec_desc *desc;
+	struct cryptop *crp;
 	uint64_t isr;
+	uint8_t hash[HASH_MAX_LEN];
 	int i, wakeup = 0;
 
 	SEC_LOCK(sc, controller);
 
 	/* Check for errors */
 	isr = SEC_READ(sc, SEC_ISR);
 	if (isr & sc->sc_int_error_mask) {
 		/* Check each channel for error */
 		for (i = 0; i < SEC_CHANNELS; i++) {
 			if ((isr & SEC_INT_CH_ERR(i)) == 0)
 				continue;
 
 			device_printf(sc->sc_dev,
 			    "I/O error on channel %i!\n", i);
 
 			/* Find and mark problematic descriptor */
 			desc = sec_find_desc(sc, SEC_READ(sc,
 			    SEC_CHAN_CDPR(i)));
 
 			if (desc != NULL)
 				desc->sd_error = EIO;
 
 			/* Do partial channel reset */
 			sec_channel_reset(sc, i, 0);
 		}
 	}
 
 	/* ACK interrupt */
 	SEC_WRITE(sc, SEC_ICR, 0xFFFFFFFFFFFFFFFFULL);
 
 	SEC_UNLOCK(sc, controller);
 	SEC_LOCK(sc, descriptors);
 
 	/* Handle processed descriptors */
 	SEC_DESC_SYNC(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 
 	while (SEC_QUEUED_DESC_CNT(sc) > 0) {
 		desc = SEC_GET_QUEUED_DESC(sc);
 
 		if (desc->sd_desc->shd_done != 0xFF && desc->sd_error == 0) {
 			SEC_PUT_BACK_QUEUED_DESC(sc);
 			break;
 		}
 
 		SEC_DESC_SYNC_POINTERS(desc, BUS_DMASYNC_PREREAD |
 		    BUS_DMASYNC_PREWRITE);
 
-		desc->sd_crp->crp_etype = desc->sd_error;
+		crp = desc->sd_crp;
+		crp->crp_etype = desc->sd_error;
+		if (crp->crp_etype == 0) {
+			ses = crypto_get_driver_session(crp->crp_session);
+			if (ses->ss_mlen != 0) {
+				if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
+					crypto_copydata(crp,
+					    crp->crp_digest_start,
+					    ses->ss_mlen, hash);
+					if (timingsafe_bcmp(
+					    desc->sd_desc->shd_digest,
+					    hash, ses->ss_mlen) != 0)
+						crp->crp_etype = EBADMSG;
+				} else
+					crypto_copyback(crp,
+					    crp->crp_digest_start,
+					    ses->ss_mlen,
+					    desc->sd_desc->shd_digest);
+			}
+		}
 		crypto_done(desc->sd_crp);
 
 		SEC_DESC_FREE_POINTERS(desc);
 		SEC_DESC_FREE_LT(sc, desc);
 		SEC_DESC_QUEUED2FREE(sc);
 	}
 
 	SEC_DESC_SYNC(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 
 	if (!sc->sc_shutdown) {
 		wakeup = sc->sc_blocked;
 		sc->sc_blocked = 0;
 	}
 
 	SEC_UNLOCK(sc, descriptors);
 
 	/* Enqueue ready descriptors in hardware */
 	sec_enqueue(sc);
 
 	if (wakeup)
 		crypto_unblock(sc->sc_cid, wakeup);
 }
 
 static void
 sec_secondary_intr(void *arg)
 {
 	struct sec_softc *sc = arg;
 
 	device_printf(sc->sc_dev, "spurious secondary interrupt!\n");
 	sec_primary_intr(arg);
 }
 
 static int
 sec_controller_reset(struct sec_softc *sc)
 {
 	int timeout = SEC_TIMEOUT;
 
 	/* Reset Controller */
 	SEC_WRITE(sc, SEC_MCR, SEC_MCR_SWR);
 
 	while (SEC_READ(sc, SEC_MCR) & SEC_MCR_SWR) {
 		DELAY(1000);
 		timeout -= 1000;
 
 		if (timeout < 0) {
 			device_printf(sc->sc_dev, "timeout while waiting for "
 			    "device reset!\n");
 			return (ETIMEDOUT);
 		}
 	}
 
 	return (0);
 }
 
 static int
 sec_channel_reset(struct sec_softc *sc, int channel, int full)
 {
 	int timeout = SEC_TIMEOUT;
 	uint64_t bit = (full) ? SEC_CHAN_CCR_R : SEC_CHAN_CCR_CON;
 	uint64_t reg;
 
 	/* Reset Channel */
 	reg = SEC_READ(sc, SEC_CHAN_CCR(channel));
 	SEC_WRITE(sc, SEC_CHAN_CCR(channel), reg | bit);
 
 	while (SEC_READ(sc, SEC_CHAN_CCR(channel)) & bit) {
 		DELAY(1000);
 		timeout -= 1000;
 
 		if (timeout < 0) {
 			device_printf(sc->sc_dev, "timeout while waiting for "
 			    "channel reset!\n");
 			return (ETIMEDOUT);
 		}
 	}
 
 	if (full) {
 		reg = SEC_CHAN_CCR_CDIE | SEC_CHAN_CCR_NT | SEC_CHAN_CCR_BS;
 
 		switch(sc->sc_version) {
 		case 2:
 			reg |= SEC_CHAN_CCR_CDWE;
 			break;
 		case 3:
 			reg |= SEC_CHAN_CCR_AWSE | SEC_CHAN_CCR_WGN;
 			break;
 		}
 
 		SEC_WRITE(sc, SEC_CHAN_CCR(channel), reg);
 	}
 
 	return (0);
 }
 
 static int
 sec_init(struct sec_softc *sc)
 {
 	uint64_t reg;
 	int error, i;
 
 	/* Reset controller twice to clear all pending interrupts */
 	error = sec_controller_reset(sc);
 	if (error)
 		return (error);
 
 	error = sec_controller_reset(sc);
 	if (error)
 		return (error);
 
 	/* Reset channels */
 	for (i = 0; i < SEC_CHANNELS; i++) {
 		error = sec_channel_reset(sc, i, 1);
 		if (error)
 			return (error);
 	}
 
 	/* Enable Interrupts */
 	reg = SEC_INT_ITO;
 	for (i = 0; i < SEC_CHANNELS; i++)
 		reg |= SEC_INT_CH_DN(i) | SEC_INT_CH_ERR(i);
 
 	SEC_WRITE(sc, SEC_IER, reg);
 
 	return (error);
 }
 
 static void
 sec_alloc_dma_mem_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 {
 	struct sec_dma_mem *dma_mem = arg;
 
 	if (error)
 		return;
 
 	KASSERT(nseg == 1, ("Wrong number of segments, should be 1"));
 	dma_mem->dma_paddr = segs->ds_addr;
 }
 
 static void
 sec_dma_map_desc_cb(void *arg, bus_dma_segment_t *segs, int nseg,
     int error)
 {
 	struct sec_desc_map_info *sdmi = arg;
 	struct sec_softc *sc = sdmi->sdmi_sc;
 	struct sec_lt *lt = NULL;
 	bus_addr_t addr;
 	bus_size_t size;
 	int i;
 
 	SEC_LOCK_ASSERT(sc, descriptors);
 
 	if (error)
 		return;
 
 	for (i = 0; i < nseg; i++) {
 		addr = segs[i].ds_addr;
 		size = segs[i].ds_len;
 
 		/* Skip requested offset */
 		if (sdmi->sdmi_offset >= size) {
 			sdmi->sdmi_offset -= size;
 			continue;
 		}
 
 		addr += sdmi->sdmi_offset;
 		size -= sdmi->sdmi_offset;
 		sdmi->sdmi_offset = 0;
 
 		/* Do not link more than requested */
 		if (sdmi->sdmi_size < size)
 			size = sdmi->sdmi_size;
 
 		lt = SEC_ALLOC_LT_ENTRY(sc);
 		lt->sl_lt->shl_length = size;
 		lt->sl_lt->shl_r = 0;
 		lt->sl_lt->shl_n = 0;
 		lt->sl_lt->shl_ptr = addr;
 
 		if (sdmi->sdmi_lt_first == NULL)
 			sdmi->sdmi_lt_first = lt;
 
 		sdmi->sdmi_lt_used += 1;
 
 		if ((sdmi->sdmi_size -= size) == 0)
 			break;
 	}
 
 	sdmi->sdmi_lt_last = lt;
 }
 
-static void
-sec_dma_map_desc_cb2(void *arg, bus_dma_segment_t *segs, int nseg,
-    bus_size_t size, int error)
-{
-
-	sec_dma_map_desc_cb(arg, segs, nseg, error);
-}
-
 static int
 sec_alloc_dma_mem(struct sec_softc *sc, struct sec_dma_mem *dma_mem,
     bus_size_t size)
 {
 	int error;
 
 	if (dma_mem->dma_vaddr != NULL)
 		return (EBUSY);
 
 	error = bus_dma_tag_create(NULL,	/* parent */
 		SEC_DMA_ALIGNMENT, 0,		/* alignment, boundary */
 		BUS_SPACE_MAXADDR_32BIT,	/* lowaddr */
 		BUS_SPACE_MAXADDR,		/* highaddr */
 		NULL, NULL,			/* filtfunc, filtfuncarg */
 		size, 1,			/* maxsize, nsegments */
 		size, 0,			/* maxsegsz, flags */
 		NULL, NULL,			/* lockfunc, lockfuncarg */
 		&(dma_mem->dma_tag));		/* dmat */
 
 	if (error) {
 		device_printf(sc->sc_dev, "failed to allocate busdma tag, error"
 		    " %i!\n", error);
 		goto err1;
 	}
 
 	error = bus_dmamem_alloc(dma_mem->dma_tag, &(dma_mem->dma_vaddr),
 	    BUS_DMA_NOWAIT | BUS_DMA_ZERO, &(dma_mem->dma_map));
 
 	if (error) {
 		device_printf(sc->sc_dev, "failed to allocate DMA safe"
 		    " memory, error %i!\n", error);
 		goto err2;
 	}
 
 	error = bus_dmamap_load(dma_mem->dma_tag, dma_mem->dma_map,
 		    dma_mem->dma_vaddr, size, sec_alloc_dma_mem_cb, dma_mem,
 		    BUS_DMA_NOWAIT);
 
 	if (error) {
 		device_printf(sc->sc_dev, "cannot get address of the DMA"
 		    " memory, error %i\n", error);
 		goto err3;
 	}
 
 	dma_mem->dma_is_map = 0;
 	return (0);
 
 err3:
 	bus_dmamem_free(dma_mem->dma_tag, dma_mem->dma_vaddr, dma_mem->dma_map);
 err2:
 	bus_dma_tag_destroy(dma_mem->dma_tag);
 err1:
 	dma_mem->dma_vaddr = NULL;
 	return(error);
 }
 
 static int
-sec_desc_map_dma(struct sec_softc *sc, struct sec_dma_mem *dma_mem, void *mem,
-    bus_size_t size, int type, struct sec_desc_map_info *sdmi)
+sec_desc_map_dma(struct sec_softc *sc, struct sec_dma_mem *dma_mem,
+    struct cryptop *crp, bus_size_t size, struct sec_desc_map_info *sdmi)
 {
 	int error;
 
 	if (dma_mem->dma_vaddr != NULL)
 		return (EBUSY);
 
-	switch (type) {
-	case SEC_MEMORY:
+	switch (crp->crp_buf_type) {
+	case CRYPTO_BUF_CONTIG:
 		break;
-	case SEC_UIO:
+	case CRYPTO_BUF_UIO:
 		size = SEC_FREE_LT_CNT(sc) * SEC_MAX_DMA_BLOCK_SIZE;
 		break;
-	case SEC_MBUF:
-		size = m_length((struct mbuf*)mem, NULL);
+	case CRYPTO_BUF_MBUF:
+		size = m_length(crp->crp_mbuf, NULL);
 		break;
 	default:
 		return (EINVAL);
 	}
 
 	error = bus_dma_tag_create(NULL,	/* parent */
 		SEC_DMA_ALIGNMENT, 0,		/* alignment, boundary */
 		BUS_SPACE_MAXADDR_32BIT,	/* lowaddr */
 		BUS_SPACE_MAXADDR,		/* highaddr */
 		NULL, NULL,			/* filtfunc, filtfuncarg */
 		size,				/* maxsize */
 		SEC_FREE_LT_CNT(sc),		/* nsegments */
 		SEC_MAX_DMA_BLOCK_SIZE, 0,	/* maxsegsz, flags */
 		NULL, NULL,			/* lockfunc, lockfuncarg */
 		&(dma_mem->dma_tag));		/* dmat */
 
 	if (error) {
 		device_printf(sc->sc_dev, "failed to allocate busdma tag, error"
 		    " %i!\n", error);
 		dma_mem->dma_vaddr = NULL;
 		return (error);
 	}
 
 	error = bus_dmamap_create(dma_mem->dma_tag, 0, &(dma_mem->dma_map));
 
 	if (error) {
 		device_printf(sc->sc_dev, "failed to create DMA map, error %i!"
 		    "\n", error);
 		bus_dma_tag_destroy(dma_mem->dma_tag);
 		return (error);
 	}
 
-	switch (type) {
-	case SEC_MEMORY:
-		error = bus_dmamap_load(dma_mem->dma_tag, dma_mem->dma_map,
-		    mem, size, sec_dma_map_desc_cb, sdmi, BUS_DMA_NOWAIT);
-		break;
-	case SEC_UIO:
-		error = bus_dmamap_load_uio(dma_mem->dma_tag, dma_mem->dma_map,
-		    mem, sec_dma_map_desc_cb2, sdmi, BUS_DMA_NOWAIT);
-		break;
-	case SEC_MBUF:
-		error = bus_dmamap_load_mbuf(dma_mem->dma_tag, dma_mem->dma_map,
-		    mem, sec_dma_map_desc_cb2, sdmi, BUS_DMA_NOWAIT);
-		break;
-	}
+	error = bus_dmamap_load_crp(dma_mem->dma_tag, dma_mem->dma_map, crp,
+	    sec_dma_map_desc_cb, sdmi, BUS_DMA_NOWAIT);
 
 	if (error) {
 		device_printf(sc->sc_dev, "cannot get address of the DMA"
 		    " memory, error %i!\n", error);
 		bus_dmamap_destroy(dma_mem->dma_tag, dma_mem->dma_map);
 		bus_dma_tag_destroy(dma_mem->dma_tag);
 		return (error);
 	}
 
 	dma_mem->dma_is_map = 1;
-	dma_mem->dma_vaddr = mem;
+	dma_mem->dma_vaddr = crp;
 
 	return (0);
 }
 
 static void
 sec_free_dma_mem(struct sec_dma_mem *dma_mem)
 {
 
 	/* Check for double free */
 	if (dma_mem->dma_vaddr == NULL)
 		return;
 
 	bus_dmamap_unload(dma_mem->dma_tag, dma_mem->dma_map);
 
 	if (dma_mem->dma_is_map)
 		bus_dmamap_destroy(dma_mem->dma_tag, dma_mem->dma_map);
 	else
 		bus_dmamem_free(dma_mem->dma_tag, dma_mem->dma_vaddr,
 		    dma_mem->dma_map);
 
 	bus_dma_tag_destroy(dma_mem->dma_tag);
 	dma_mem->dma_vaddr = NULL;
 }
 
 static int
 sec_eu_channel(struct sec_softc *sc, int eu)
 {
 	uint64_t reg;
 	int channel = 0;
 
 	SEC_LOCK_ASSERT(sc, controller);
 
 	reg = SEC_READ(sc, SEC_EUASR);
 
 	switch (eu) {
 	case SEC_EU_AFEU:
 		channel = SEC_EUASR_AFEU(reg);
 		break;
 	case SEC_EU_DEU:
 		channel = SEC_EUASR_DEU(reg);
 		break;
 	case SEC_EU_MDEU_A:
 	case SEC_EU_MDEU_B:
 		channel = SEC_EUASR_MDEU(reg);
 		break;
 	case SEC_EU_RNGU:
 		channel = SEC_EUASR_RNGU(reg);
 		break;
 	case SEC_EU_PKEU:
 		channel = SEC_EUASR_PKEU(reg);
 		break;
 	case SEC_EU_AESU:
 		channel = SEC_EUASR_AESU(reg);
 		break;
 	case SEC_EU_KEU:
 		channel = SEC_EUASR_KEU(reg);
 		break;
 	case SEC_EU_CRCU:
 		channel = SEC_EUASR_CRCU(reg);
 		break;
 	}
 
 	return (channel - 1);
 }
 
 static int
 sec_enqueue_desc(struct sec_softc *sc, struct sec_desc *desc, int channel)
 {
 	u_int fflvl = SEC_MAX_FIFO_LEVEL;
 	uint64_t reg;
 	int i;
 
 	SEC_LOCK_ASSERT(sc, controller);
 
 	/* Find free channel if have not got one */
 	if (channel < 0) {
 		for (i = 0; i < SEC_CHANNELS; i++) {
 			reg = SEC_READ(sc, SEC_CHAN_CSR(channel));
 
 			if ((reg & sc->sc_channel_idle_mask) == 0) {
 				channel = i;
 				break;
 			}
 		}
 	}
 
 	/* There is no free channel */
 	if (channel < 0)
 		return (-1);
 
 	/* Check FIFO level on selected channel */
 	reg = SEC_READ(sc, SEC_CHAN_CSR(channel));
 
 	switch(sc->sc_version) {
 	case 2:
 		fflvl = (reg >> SEC_CHAN_CSR2_FFLVL_S) & SEC_CHAN_CSR2_FFLVL_M;
 		break;
 	case 3:
 		fflvl = (reg >> SEC_CHAN_CSR3_FFLVL_S) & SEC_CHAN_CSR3_FFLVL_M;
 		break;
 	}
 
 	if (fflvl >= SEC_MAX_FIFO_LEVEL)
 		return (-1);
 
 	/* Enqueue descriptor in channel */
 	SEC_WRITE(sc, SEC_CHAN_FF(channel), desc->sd_desc_paddr);
 
 	return (channel);
 }
 
 static void
 sec_enqueue(struct sec_softc *sc)
 {
 	struct sec_desc *desc;
 	int ch0, ch1;
 
 	SEC_LOCK(sc, descriptors);
 	SEC_LOCK(sc, controller);
 
 	while (SEC_READY_DESC_CNT(sc) > 0) {
 		desc = SEC_GET_READY_DESC(sc);
 
 		ch0 = sec_eu_channel(sc, desc->sd_desc->shd_eu_sel0);
 		ch1 = sec_eu_channel(sc, desc->sd_desc->shd_eu_sel1);
 
 		/*
 		 * Both EU are used by the same channel.
 		 * Enqueue descriptor in channel used by busy EUs.
 		 */
 		if (ch0 >= 0 && ch0 == ch1) {
 			if (sec_enqueue_desc(sc, desc, ch0) >= 0) {
 				SEC_DESC_READY2QUEUED(sc);
 				continue;
 			}
 		}
 
 		/*
 		 * Only one EU is free.
 		 * Enqueue descriptor in channel used by busy EU.
 		 */
 		if ((ch0 >= 0 && ch1 < 0) || (ch1 >= 0 && ch0 < 0)) {
 			if (sec_enqueue_desc(sc, desc, (ch0 >= 0) ? ch0 : ch1)
 			    >= 0) {
 				SEC_DESC_READY2QUEUED(sc);
 				continue;
 			}
 		}
 
 		/*
 		 * Both EU are free.
 		 * Enqueue descriptor in first free channel.
 		 */
 		if (ch0 < 0 && ch1 < 0) {
 			if (sec_enqueue_desc(sc, desc, -1) >= 0) {
 				SEC_DESC_READY2QUEUED(sc);
 				continue;
 			}
 		}
 
 		/* Current descriptor can not be queued at the moment */
 		SEC_PUT_BACK_READY_DESC(sc);
 		break;
 	}
 
 	SEC_UNLOCK(sc, controller);
 	SEC_UNLOCK(sc, descriptors);
 }
 
 static struct sec_desc *
 sec_find_desc(struct sec_softc *sc, bus_addr_t paddr)
 {
 	struct sec_desc *desc = NULL;
 	int i;
 
 	SEC_LOCK_ASSERT(sc, descriptors);
 
 	for (i = 0; i < SEC_CHANNELS; i++) {
 		if (sc->sc_desc[i].sd_desc_paddr == paddr) {
 			desc = &(sc->sc_desc[i]);
 			break;
 		}
 	}
 
 	return (desc);
 }
 
 static int
 sec_make_pointer_direct(struct sec_softc *sc, struct sec_desc *desc, u_int n,
     bus_addr_t data, bus_size_t dsize)
 {
 	struct sec_hw_desc_ptr *ptr;
 
 	SEC_LOCK_ASSERT(sc, descriptors);
 
 	ptr = &(desc->sd_desc->shd_pointer[n]);
 	ptr->shdp_length = dsize;
 	ptr->shdp_extent = 0;
 	ptr->shdp_j = 0;
 	ptr->shdp_ptr = data;
 
 	return (0);
 }
 
 static int
 sec_make_pointer(struct sec_softc *sc, struct sec_desc *desc,
-    u_int n, void *data, bus_size_t doffset, bus_size_t dsize, int dtype)
+    u_int n, struct cryptop *crp, bus_size_t doffset, bus_size_t dsize)
 {
 	struct sec_desc_map_info sdmi = { sc, dsize, doffset, NULL, NULL, 0 };
 	struct sec_hw_desc_ptr *ptr;
 	int error;
 
 	SEC_LOCK_ASSERT(sc, descriptors);
 
-	/* For flat memory map only requested region */
-	if (dtype == SEC_MEMORY) {
-		 data = (uint8_t*)(data) + doffset;
-		 sdmi.sdmi_offset = 0;
-	}
+	error = sec_desc_map_dma(sc, &(desc->sd_ptr_dmem[n]), crp, dsize,
+	    &sdmi);
 
-	error = sec_desc_map_dma(sc, &(desc->sd_ptr_dmem[n]), data, dsize,
-	    dtype, &sdmi);
-
 	if (error)
 		return (error);
 
 	sdmi.sdmi_lt_last->sl_lt->shl_r = 1;
 	desc->sd_lt_used += sdmi.sdmi_lt_used;
 
 	ptr = &(desc->sd_desc->shd_pointer[n]);
 	ptr->shdp_length = dsize;
 	ptr->shdp_extent = 0;
 	ptr->shdp_j = 1;
 	ptr->shdp_ptr = sdmi.sdmi_lt_first->sl_lt_paddr;
 
 	return (0);
 }
 
-static int
-sec_split_cri(struct cryptoini *cri, struct cryptoini **enc,
-    struct cryptoini **mac)
+static bool
+sec_cipher_supported(const struct crypto_session_params *csp)
 {
-	struct cryptoini *e, *m;
 
-	e = cri;
-	m = cri->cri_next;
-
-	/* We can haldle only two operations */
-	if (m && m->cri_next)
-		return (EINVAL);
-
-	if (sec_mdeu_can_handle(e->cri_alg)) {
-		cri = m;
-		m = e;
-		e = cri;
+	switch (csp->csp_cipher_alg) {
+	case CRYPTO_AES_CBC:
+		/* AESU */
+		if (csp->csp_ivlen != AES_BLOCK_LEN)
+			return (false);
+		break;
+	case CRYPTO_DES_CBC:
+	case CRYPTO_3DES_CBC:
+		/* DEU */
+		if (csp->csp_ivlen != DES_BLOCK_LEN)
+			return (false);
+		break;
+	default:
+		return (false);
 	}
 
-	if (m && !sec_mdeu_can_handle(m->cri_alg))
-		return (EINVAL);
+	if (csp->csp_cipher_klen == 0 || csp->csp_cipher_klen > SEC_MAX_KEY_LEN)
+		return (false);
 
-	*enc = e;
-	*mac = m;
+	return (true);
+}
 
-	return (0);
+static bool
+sec_auth_supported(struct sec_softc *sc,
+    const struct crypto_session_params *csp)
+{
+
+	switch (csp->csp_auth_alg) {
+	case CRYPTO_SHA2_384_HMAC:
+	case CRYPTO_SHA2_512_HMAC:
+		if (sc->sc_version < 3)
+			return (false);
+		/* FALLTHROUGH */
+	case CRYPTO_MD5_HMAC:
+	case CRYPTO_SHA1_HMAC:
+	case CRYPTO_SHA2_256_HMAC:
+		if (csp->csp_auth_klen > SEC_MAX_KEY_LEN)
+			return (false);
+		break;
+	case CRYPTO_MD5:
+	case CRYPTO_SHA1:
+		break;
+	default:
+		return (false);
+	}
+	return (true);
 }
 
 static int
-sec_split_crp(struct cryptop *crp, struct cryptodesc **enc,
-    struct cryptodesc **mac)
+sec_probesession(device_t dev, const struct crypto_session_params *csp)
 {
-	struct cryptodesc *e, *m, *t;
+	struct sec_softc *sc = device_get_softc(dev);
 
-	e = crp->crp_desc;
-	m = e->crd_next;
-
-	/* We can haldle only two operations */
-	if (m && m->crd_next)
+	if (csp->csp_flags != 0)
 		return (EINVAL);
-
-	if (sec_mdeu_can_handle(e->crd_alg)) {
-		t = m;
-		m = e;
-		e = t;
-	}
-
-	if (m && !sec_mdeu_can_handle(m->crd_alg))
+	switch (csp->csp_mode) {
+	case CSP_MODE_DIGEST:
+		if (!sec_auth_supported(sc, csp))
+			return (EINVAL);
+		break;
+	case CSP_MODE_CIPHER:
+		if (!sec_cipher_supported(csp))
+			return (EINVAL);
+		break;
+	case CSP_MODE_ETA:
+		if (!sec_auth_supported(sc, csp) || !sec_cipher_supported(csp))
+			return (EINVAL);
+		break;
+	default:
 		return (EINVAL);
-
-	*enc = e;
-	*mac = m;
-
-	return (0);
+	}
+	return (CRYPTODEV_PROBE_HARDWARE);
 }
 
 static int
-sec_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
+sec_newsession(device_t dev, crypto_session_t cses,
+    const struct crypto_session_params *csp)
 {
-	struct sec_softc *sc = device_get_softc(dev);
 	struct sec_eu_methods *eu = sec_eus;
-	struct cryptoini *enc = NULL;
-	struct cryptoini *mac = NULL;
 	struct sec_session *ses;
-	int error = -1;
 
-	error = sec_split_cri(cri, &enc, &mac);
-	if (error)
-		return (error);
-
-	/* Check key lengths */
-	if (enc && enc->cri_key && (enc->cri_klen / 8) > SEC_MAX_KEY_LEN)
-		return (E2BIG);
-
-	if (mac && mac->cri_key && (mac->cri_klen / 8) > SEC_MAX_KEY_LEN)
-		return (E2BIG);
-
-	/* Only SEC 3.0 supports digests larger than 256 bits */
-	if (sc->sc_version < 3 && mac && mac->cri_klen > 256)
-		return (E2BIG);
-
 	ses = crypto_get_driver_session(cses);
 
 	/* Find EU for this session */
 	while (eu->sem_make_desc != NULL) {
-		error = eu->sem_newsession(sc, ses, enc, mac);
-		if (error >= 0)
+		if (eu->sem_newsession(csp))
 			break;
-
 		eu++;
 	}
+	KASSERT(eu->sem_make_desc != NULL, ("failed to find eu for session"));
 
-	/* If not found, return EINVAL */
-	if (error < 0)
-		return (EINVAL);
-
 	/* Save cipher key */
-	if (enc && enc->cri_key) {
-		ses->ss_klen = enc->cri_klen / 8;
-		memcpy(ses->ss_key, enc->cri_key, ses->ss_klen);
-	}
+	if (csp->csp_cipher_key != NULL)
+		memcpy(ses->ss_key, csp->csp_cipher_key, csp->csp_cipher_klen);
 
 	/* Save digest key */
-	if (mac && mac->cri_key) {
-		ses->ss_mklen = mac->cri_klen / 8;
-		memcpy(ses->ss_mkey, mac->cri_key, ses->ss_mklen);
+	if (csp->csp_auth_key != NULL)
+		memcpy(ses->ss_mkey, csp->csp_auth_key, csp->csp_auth_klen);
+
+	if (csp->csp_auth_alg != 0) {
+		if (csp->csp_auth_mlen == 0)
+			ses->ss_mlen = crypto_auth_hash(csp)->hashsize;
+		else
+			ses->ss_mlen = csp->csp_auth_mlen;
 	}
 
-	ses->ss_eu = eu;
 	return (0);
 }
 
 static int
 sec_process(device_t dev, struct cryptop *crp, int hint)
 {
 	struct sec_softc *sc = device_get_softc(dev);
 	struct sec_desc *desc = NULL;
-	struct cryptodesc *mac, *enc;
+	const struct crypto_session_params *csp;
 	struct sec_session *ses;
-	int buftype, error = 0;
+	int error = 0;
 
 	ses = crypto_get_driver_session(crp->crp_session);
+	csp = crypto_get_params(crp->crp_session);
 
 	/* Check for input length */
 	if (crp->crp_ilen > SEC_MAX_DMA_BLOCK_SIZE) {
 		crp->crp_etype = E2BIG;
 		crypto_done(crp);
 		return (0);
 	}
 
-	/* Get descriptors */
-	if (sec_split_crp(crp, &enc, &mac)) {
-		crp->crp_etype = EINVAL;
-		crypto_done(crp);
-		return (0);
-	}
-
 	SEC_LOCK(sc, descriptors);
 	SEC_DESC_SYNC(sc, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 
 	/* Block driver if there is no free descriptors or we are going down */
 	if (SEC_FREE_DESC_CNT(sc) == 0 || sc->sc_shutdown) {
 		sc->sc_blocked |= CRYPTO_SYMQ;
 		SEC_UNLOCK(sc, descriptors);
 		return (ERESTART);
 	}
 
 	/* Prepare descriptor */
 	desc = SEC_GET_FREE_DESC(sc);
 	desc->sd_lt_used = 0;
 	desc->sd_error = 0;
 	desc->sd_crp = crp;
 
-	if (crp->crp_flags & CRYPTO_F_IOV)
-		buftype = SEC_UIO;
-	else if (crp->crp_flags & CRYPTO_F_IMBUF)
-		buftype = SEC_MBUF;
-	else
-		buftype = SEC_MEMORY;
-
-	if (enc && enc->crd_flags & CRD_F_ENCRYPT) {
-		if (enc->crd_flags & CRD_F_IV_EXPLICIT)
-			memcpy(desc->sd_desc->shd_iv, enc->crd_iv,
-			    ses->ss_ivlen);
-		else
-			arc4rand(desc->sd_desc->shd_iv, ses->ss_ivlen, 0);
-
-		if ((enc->crd_flags & CRD_F_IV_PRESENT) == 0)
-			crypto_copyback(crp->crp_flags, crp->crp_buf,
-			    enc->crd_inject, ses->ss_ivlen,
+	if (csp->csp_cipher_alg != 0) {
+		if (crp->crp_flags & CRYPTO_F_IV_GENERATE) {
+			arc4rand(desc->sd_desc->shd_iv, csp->csp_ivlen, 0);
+			crypto_copyback(crp, crp->crp_iv_start, csp->csp_ivlen,
 			    desc->sd_desc->shd_iv);
-	} else if (enc) {
-		if (enc->crd_flags & CRD_F_IV_EXPLICIT)
-			memcpy(desc->sd_desc->shd_iv, enc->crd_iv,
-			    ses->ss_ivlen);
+		} else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+			memcpy(desc->sd_desc->shd_iv, crp->crp_iv,
+			    csp->csp_ivlen);
 		else
-			crypto_copydata(crp->crp_flags, crp->crp_buf,
-			    enc->crd_inject, ses->ss_ivlen,
+			crypto_copydata(crp, crp->crp_iv_start, csp->csp_ivlen,
 			    desc->sd_desc->shd_iv);
 	}
 
-	if (enc && enc->crd_flags & CRD_F_KEY_EXPLICIT) {
-		if ((enc->crd_klen / 8) <= SEC_MAX_KEY_LEN) {
-			ses->ss_klen = enc->crd_klen / 8;
-			memcpy(ses->ss_key, enc->crd_key, ses->ss_klen);
-		} else
-			error = E2BIG;
-	}
+	if (crp->crp_cipher_key != NULL)
+		memcpy(ses->ss_key, crp->crp_cipher_key, csp->csp_cipher_klen);
 
-	if (!error && mac && mac->crd_flags & CRD_F_KEY_EXPLICIT) {
-		if ((mac->crd_klen / 8) <= SEC_MAX_KEY_LEN) {
-			ses->ss_mklen = mac->crd_klen / 8;
-			memcpy(ses->ss_mkey, mac->crd_key, ses->ss_mklen);
-		} else
-			error = E2BIG;
-	}
+	if (crp->crp_auth_key != NULL)
+		memcpy(ses->ss_mkey, crp->crp_auth_key, csp->csp_auth_klen);
 
-	if (!error) {
-		memcpy(desc->sd_desc->shd_key, ses->ss_key, ses->ss_klen);
-		memcpy(desc->sd_desc->shd_mkey, ses->ss_mkey, ses->ss_mklen);
+	memcpy(desc->sd_desc->shd_key, ses->ss_key, csp->csp_cipher_klen);
+	memcpy(desc->sd_desc->shd_mkey, ses->ss_mkey, csp->csp_auth_klen);
 
-		error = ses->ss_eu->sem_make_desc(sc, ses, desc, crp, buftype);
-	}
+	error = ses->ss_eu->sem_make_desc(sc, csp, desc, crp);
 
 	if (error) {
 		SEC_DESC_FREE_POINTERS(desc);
 		SEC_DESC_PUT_BACK_LT(sc, desc);
 		SEC_PUT_BACK_FREE_DESC(sc);
 		SEC_UNLOCK(sc, descriptors);
 		crp->crp_etype = error;
 		crypto_done(crp);
 		return (0);
 	}
 
 	/*
 	 * Skip DONE interrupt if this is not last request in burst, but only
 	 * if we are running on SEC 3.X. On SEC 2.X we have to enable DONE
 	 * signaling on each descriptor.
 	 */
 	if ((hint & CRYPTO_HINT_MORE) && sc->sc_version == 3)
 		desc->sd_desc->shd_dn = 0;
 	else
 		desc->sd_desc->shd_dn = 1;
 
 	SEC_DESC_SYNC(sc, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 	SEC_DESC_SYNC_POINTERS(desc, BUS_DMASYNC_POSTREAD |
 	    BUS_DMASYNC_POSTWRITE);
 	SEC_DESC_FREE2READY(sc);
 	SEC_UNLOCK(sc, descriptors);
 
 	/* Enqueue ready descriptors in hardware */
 	sec_enqueue(sc);
 
 	return (0);
 }
 
 static int
 sec_build_common_ns_desc(struct sec_softc *sc, struct sec_desc *desc,
-    struct sec_session *ses, struct cryptop *crp, struct cryptodesc *enc,
-    int buftype)
+    const struct crypto_session_params *csp, struct cryptop *crp)
 {
 	struct sec_hw_desc *hd = desc->sd_desc;
 	int error;
 
 	hd->shd_desc_type = SEC_DT_COMMON_NONSNOOP;
 	hd->shd_eu_sel1 = SEC_EU_NONE;
 	hd->shd_mode1 = 0;
 
 	/* Pointer 0: NULL */
 	error = sec_make_pointer_direct(sc, desc, 0, 0, 0);
 	if (error)
 		return (error);
 
 	/* Pointer 1: IV IN */
 	error = sec_make_pointer_direct(sc, desc, 1, desc->sd_desc_paddr +
-	    offsetof(struct sec_hw_desc, shd_iv), ses->ss_ivlen);
+	    offsetof(struct sec_hw_desc, shd_iv), csp->csp_ivlen);
 	if (error)
 		return (error);
 
 	/* Pointer 2: Cipher Key */
 	error = sec_make_pointer_direct(sc, desc, 2, desc->sd_desc_paddr +
-	    offsetof(struct sec_hw_desc, shd_key), ses->ss_klen);
+	    offsetof(struct sec_hw_desc, shd_key), csp->csp_cipher_klen);
  	if (error)
 		return (error);
 
 	/* Pointer 3: Data IN */
-	error = sec_make_pointer(sc, desc, 3, crp->crp_buf, enc->crd_skip,
-	    enc->crd_len, buftype);
+	error = sec_make_pointer(sc, desc, 3, crp, crp->crp_payload_start,
+	    crp->crp_payload_length);
 	if (error)
 		return (error);
 
 	/* Pointer 4: Data OUT */
-	error = sec_make_pointer(sc, desc, 4, crp->crp_buf, enc->crd_skip,
-	    enc->crd_len, buftype);
+	error = sec_make_pointer(sc, desc, 4, crp, crp->crp_payload_start,
+	    crp->crp_payload_length);
 	if (error)
 		return (error);
 
 	/* Pointer 5: IV OUT (Not used: NULL) */
 	error = sec_make_pointer_direct(sc, desc, 5, 0, 0);
 	if (error)
 		return (error);
 
 	/* Pointer 6: NULL */
 	error = sec_make_pointer_direct(sc, desc, 6, 0, 0);
 
 	return (error);
 }
 
 static int
 sec_build_common_s_desc(struct sec_softc *sc, struct sec_desc *desc,
-    struct sec_session *ses, struct cryptop *crp, struct cryptodesc *enc,
-    struct cryptodesc *mac, int buftype)
+    const struct crypto_session_params *csp, struct cryptop *crp)
 {
 	struct sec_hw_desc *hd = desc->sd_desc;
 	u_int eu, mode, hashlen;
 	int error;
 
-	if (mac->crd_len < enc->crd_len)
-		return (EINVAL);
-
-	if (mac->crd_skip + mac->crd_len != enc->crd_skip + enc->crd_len)
-		return (EINVAL);
-
-	error = sec_mdeu_config(mac, &eu, &mode, &hashlen);
+	error = sec_mdeu_config(csp, &eu, &mode, &hashlen);
 	if (error)
 		return (error);
 
 	hd->shd_desc_type = SEC_DT_HMAC_SNOOP;
 	hd->shd_eu_sel1 = eu;
 	hd->shd_mode1 = mode;
 
 	/* Pointer 0: HMAC Key */
 	error = sec_make_pointer_direct(sc, desc, 0, desc->sd_desc_paddr +
-	    offsetof(struct sec_hw_desc, shd_mkey), ses->ss_mklen);
+	    offsetof(struct sec_hw_desc, shd_mkey), csp->csp_auth_klen);
 	if (error)
 		return (error);
 
 	/* Pointer 1: HMAC-Only Data IN */
-	error = sec_make_pointer(sc, desc, 1, crp->crp_buf, mac->crd_skip,
-	    mac->crd_len - enc->crd_len, buftype);
+	error = sec_make_pointer(sc, desc, 1, crp, crp->crp_aad_start,
+	    crp->crp_aad_length);
 	if (error)
 		return (error);
 
 	/* Pointer 2: Cipher Key */
 	error = sec_make_pointer_direct(sc, desc, 2, desc->sd_desc_paddr +
-	    offsetof(struct sec_hw_desc, shd_key), ses->ss_klen);
+	    offsetof(struct sec_hw_desc, shd_key), csp->csp_cipher_klen);
  	if (error)
 		return (error);
 
 	/* Pointer 3: IV IN */
 	error = sec_make_pointer_direct(sc, desc, 3, desc->sd_desc_paddr +
-	    offsetof(struct sec_hw_desc, shd_iv), ses->ss_ivlen);
+	    offsetof(struct sec_hw_desc, shd_iv), csp->csp_ivlen);
 	if (error)
 		return (error);
 
 	/* Pointer 4: Data IN */
-	error = sec_make_pointer(sc, desc, 4, crp->crp_buf, enc->crd_skip,
-	    enc->crd_len, buftype);
+	error = sec_make_pointer(sc, desc, 4, crp, crp->crp_payload_start,
+	    crp->crp_payload_length);
 	if (error)
 		return (error);
 
 	/* Pointer 5: Data OUT */
-	error = sec_make_pointer(sc, desc, 5, crp->crp_buf, enc->crd_skip,
-	    enc->crd_len, buftype);
+	error = sec_make_pointer(sc, desc, 5, crp, crp->crp_payload_start,
+	    crp->crp_payload_length);
 	if (error)
 		return (error);
 
 	/* Pointer 6: HMAC OUT */
-	error = sec_make_pointer(sc, desc, 6, crp->crp_buf, mac->crd_inject,
-	    hashlen, buftype);
+	error = sec_make_pointer_direct(sc, desc, 6, desc->sd_desc_paddr +
+	    offsetof(struct sec_hw_desc, shd_digest), hashlen);
 
 	return (error);
 }
 
 /* AESU */
 
-static int
-sec_aesu_newsession(struct sec_softc *sc, struct sec_session *ses,
-    struct cryptoini *enc, struct cryptoini *mac)
+static bool
+sec_aesu_newsession(const struct crypto_session_params *csp)
 {
 
-	if (enc == NULL)
-		return (-1);
-
-	if (enc->cri_alg != CRYPTO_AES_CBC)
-		return (-1);
-
-	ses->ss_ivlen = AES_BLOCK_LEN;
-
-	return (0);
+	return (csp->csp_cipher_alg == CRYPTO_AES_CBC);
 }
 
 static int
-sec_aesu_make_desc(struct sec_softc *sc, struct sec_session *ses,
-    struct sec_desc *desc, struct cryptop *crp, int buftype)
+sec_aesu_make_desc(struct sec_softc *sc,
+    const struct crypto_session_params *csp, struct sec_desc *desc,
+    struct cryptop *crp)
 {
 	struct sec_hw_desc *hd = desc->sd_desc;
-	struct cryptodesc *enc, *mac;
 	int error;
 
-	error = sec_split_crp(crp, &enc, &mac);
-	if (error)
-		return (error);
-
-	if (!enc)
-		return (EINVAL);
-
 	hd->shd_eu_sel0 = SEC_EU_AESU;
 	hd->shd_mode0 = SEC_AESU_MODE_CBC;
 
-	if (enc->crd_alg != CRYPTO_AES_CBC)
-		return (EINVAL);
-
-	if (enc->crd_flags & CRD_F_ENCRYPT) {
+	if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
 		hd->shd_mode0 |= SEC_AESU_MODE_ED;
 		hd->shd_dir = 0;
 	} else
 		hd->shd_dir = 1;
 
-	if (mac)
-		error = sec_build_common_s_desc(sc, desc, ses, crp, enc, mac,
-		    buftype);
+	if (csp->csp_mode == CSP_MODE_ETA)
+		error = sec_build_common_s_desc(sc, desc, csp, crp);
 	else
-		error = sec_build_common_ns_desc(sc, desc, ses, crp, enc,
-		    buftype);
+		error = sec_build_common_ns_desc(sc, desc, csp, crp);
 
 	return (error);
 }
 
 /* DEU */
 
-static int
-sec_deu_newsession(struct sec_softc *sc, struct sec_session *ses,
-    struct cryptoini *enc, struct cryptoini *mac)
+static bool
+sec_deu_newsession(const struct crypto_session_params *csp)
 {
 
-	if (enc == NULL)
-		return (-1);
-
-	switch (enc->cri_alg) {
+	switch (csp->csp_cipher_alg) {
 	case CRYPTO_DES_CBC:
 	case CRYPTO_3DES_CBC:
-		break;
+		return (true);
 	default:
-		return (-1);
+		return (false);
 	}
-
-	ses->ss_ivlen = DES_BLOCK_LEN;
-
-	return (0);
 }
 
 static int
-sec_deu_make_desc(struct sec_softc *sc, struct sec_session *ses,
-    struct sec_desc *desc, struct cryptop *crp, int buftype)
+sec_deu_make_desc(struct sec_softc *sc, const struct crypto_session_params *csp,
+    struct sec_desc *desc, struct cryptop *crp)
 {
 	struct sec_hw_desc *hd = desc->sd_desc;
-	struct cryptodesc *enc, *mac;
 	int error;
 
-	error = sec_split_crp(crp, &enc, &mac);
-	if (error)
-		return (error);
-
-	if (!enc)
-		return (EINVAL);
-
 	hd->shd_eu_sel0 = SEC_EU_DEU;
 	hd->shd_mode0 = SEC_DEU_MODE_CBC;
 
-	switch (enc->crd_alg) {
+	switch (csp->csp_cipher_alg) {
 	case CRYPTO_3DES_CBC:
 		hd->shd_mode0 |= SEC_DEU_MODE_TS;
 		break;
 	case CRYPTO_DES_CBC:
 		break;
 	default:
 		return (EINVAL);
 	}
 
-	if (enc->crd_flags & CRD_F_ENCRYPT) {
+	if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
 		hd->shd_mode0 |= SEC_DEU_MODE_ED;
 		hd->shd_dir = 0;
 	} else
 		hd->shd_dir = 1;
 
-	if (mac)
-		error = sec_build_common_s_desc(sc, desc, ses, crp, enc, mac,
-		    buftype);
+	if (csp->csp_mode == CSP_MODE_ETA)
+		error = sec_build_common_s_desc(sc, desc, csp, crp);
 	else
-		error = sec_build_common_ns_desc(sc, desc, ses, crp, enc,
-		    buftype);
+		error = sec_build_common_ns_desc(sc, desc, csp, crp);
 
 	return (error);
 }
 
 /* MDEU */
 
-static int
+static bool
 sec_mdeu_can_handle(u_int alg)
 {
 	switch (alg) {
 	case CRYPTO_MD5:
 	case CRYPTO_SHA1:
 	case CRYPTO_MD5_HMAC:
 	case CRYPTO_SHA1_HMAC:
 	case CRYPTO_SHA2_256_HMAC:
 	case CRYPTO_SHA2_384_HMAC:
 	case CRYPTO_SHA2_512_HMAC:
-		return (1);
+		return (true);
 	default:
-		return (0);
+		return (false);
 	}
 }
 
 static int
-sec_mdeu_config(struct cryptodesc *crd, u_int *eu, u_int *mode, u_int *hashlen)
+sec_mdeu_config(const struct crypto_session_params *csp, u_int *eu, u_int *mode,
+    u_int *hashlen)
 {
 
 	*mode = SEC_MDEU_MODE_PD | SEC_MDEU_MODE_INIT;
 	*eu = SEC_EU_NONE;
 
-	switch (crd->crd_alg) {
+	switch (csp->csp_auth_alg) {
 	case CRYPTO_MD5_HMAC:
 		*mode |= SEC_MDEU_MODE_HMAC;
 		/* FALLTHROUGH */
 	case CRYPTO_MD5:
 		*eu = SEC_EU_MDEU_A;
 		*mode |= SEC_MDEU_MODE_MD5;
 		*hashlen = MD5_HASH_LEN;
 		break;
 	case CRYPTO_SHA1_HMAC:
 		*mode |= SEC_MDEU_MODE_HMAC;
 		/* FALLTHROUGH */
 	case CRYPTO_SHA1:
 		*eu = SEC_EU_MDEU_A;
 		*mode |= SEC_MDEU_MODE_SHA1;
 		*hashlen = SHA1_HASH_LEN;
 		break;
 	case CRYPTO_SHA2_256_HMAC:
 		*mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA256;
 		*eu = SEC_EU_MDEU_A;
 		break;
 	case CRYPTO_SHA2_384_HMAC:
 		*mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA384;
 		*eu = SEC_EU_MDEU_B;
 		break;
 	case CRYPTO_SHA2_512_HMAC:
 		*mode |= SEC_MDEU_MODE_HMAC | SEC_MDEU_MODE_SHA512;
 		*eu = SEC_EU_MDEU_B;
 		break;
 	default:
 		return (EINVAL);
 	}
 
 	if (*mode & SEC_MDEU_MODE_HMAC)
 		*hashlen = SEC_HMAC_HASH_LEN;
 
 	return (0);
 }
 
-static int
-sec_mdeu_newsession(struct sec_softc *sc, struct sec_session *ses,
-    struct cryptoini *enc, struct cryptoini *mac)
+static bool
+sec_mdeu_newsession(const struct crypto_session_params *csp)
 {
 
-	if (mac && sec_mdeu_can_handle(mac->cri_alg))
-		return (0);
-
-	return (-1);
+	return (sec_mdeu_can_handle(csp->csp_auth_alg));
 }
 
 static int
-sec_mdeu_make_desc(struct sec_softc *sc, struct sec_session *ses,
-    struct sec_desc *desc, struct cryptop *crp, int buftype)
+sec_mdeu_make_desc(struct sec_softc *sc,
+    const struct crypto_session_params *csp,
+    struct sec_desc *desc, struct cryptop *crp)
 {
-	struct cryptodesc *enc, *mac;
 	struct sec_hw_desc *hd = desc->sd_desc;
 	u_int eu, mode, hashlen;
 	int error;
 
-	error = sec_split_crp(crp, &enc, &mac);
+	error = sec_mdeu_config(csp, &eu, &mode, &hashlen);
 	if (error)
 		return (error);
 
-	if (enc)
-		return (EINVAL);
-
-	error = sec_mdeu_config(mac, &eu, &mode, &hashlen);
-	if (error)
-		return (error);
-
 	hd->shd_desc_type = SEC_DT_COMMON_NONSNOOP;
 	hd->shd_eu_sel0 = eu;
 	hd->shd_mode0 = mode;
 	hd->shd_eu_sel1 = SEC_EU_NONE;
 	hd->shd_mode1 = 0;
 
 	/* Pointer 0: NULL */
 	error = sec_make_pointer_direct(sc, desc, 0, 0, 0);
 	if (error)
 		return (error);
 
 	/* Pointer 1: Context In (Not used: NULL) */
 	error = sec_make_pointer_direct(sc, desc, 1, 0, 0);
 	if (error)
 		return (error);
 
 	/* Pointer 2: HMAC Key (or NULL, depending on digest type) */
 	if (hd->shd_mode0 & SEC_MDEU_MODE_HMAC)
 		error = sec_make_pointer_direct(sc, desc, 2,
 		    desc->sd_desc_paddr + offsetof(struct sec_hw_desc,
-		    shd_mkey), ses->ss_mklen);
+		    shd_mkey), csp->csp_auth_klen);
 	else
 		error = sec_make_pointer_direct(sc, desc, 2, 0, 0);
 
 	if (error)
 		return (error);
 
 	/* Pointer 3: Input Data */
-	error = sec_make_pointer(sc, desc, 3, crp->crp_buf, mac->crd_skip,
-	    mac->crd_len, buftype);
+	error = sec_make_pointer(sc, desc, 3, crp, crp->crp_payload_start,
+	    crp->crp_payload_length);
 	if (error)
 		return (error);
 
 	/* Pointer 4: NULL */
 	error = sec_make_pointer_direct(sc, desc, 4, 0, 0);
 	if (error)
 		return (error);
 
 	/* Pointer 5: Hash out */
-	error = sec_make_pointer(sc, desc, 5, crp->crp_buf,
-	    mac->crd_inject, hashlen, buftype);
+	error = sec_make_pointer_direct(sc, desc, 5, desc->sd_desc_paddr +
+	    offsetof(struct sec_hw_desc, shd_digest), hashlen);
 	if (error)
 		return (error);
 
 	/* Pointer 6: NULL */
 	error = sec_make_pointer_direct(sc, desc, 6, 0, 0);
 
 	return (0);
 }
Index: head/sys/dev/sec/sec.h
===================================================================
--- head/sys/dev/sec/sec.h	(revision 359373)
+++ head/sys/dev/sec/sec.h	(revision 359374)
@@ -1,425 +1,417 @@
 /*-
  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
  *
  * Copyright (C) 2008-2009 Semihalf, Piotr Ziecik
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN
  * NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
  * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
  * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  * $FreeBSD$
  */
 
 #ifndef	_SEC_H
 #define _SEC_H
 
 /*
  * Each SEC channel can hold up to 24 descriptors. All 4 channels can be
  * simultaneously active holding 96 descriptors. Each descriptor can use 0 or
  * more link table entries depending of size and granulation of input/output
  * data. One link table entry is needed for each 65535 bytes of data.
  */
 
 /* Driver settings */
 #define SEC_TIMEOUT			100000
 #define SEC_MAX_SESSIONS		256
 #define SEC_DESCRIPTORS			256	/* Must be power of 2 */
 #define SEC_LT_ENTRIES			1024	/* Must be power of 2 */
 #define SEC_MAX_IV_LEN			16
 #define SEC_MAX_KEY_LEN			64
 
 /* SEC information */
 #define SEC_20_ID			0x0000000000000040ULL
 #define SEC_30_ID			0x0030030000000000ULL
 #define SEC_31_ID			0x0030030100000000ULL
 #define SEC_CHANNELS			4
 #define SEC_POINTERS			7
 #define SEC_MAX_DMA_BLOCK_SIZE		0xFFFF
 #define SEC_MAX_FIFO_LEVEL		24
 #define SEC_DMA_ALIGNMENT		8
 
 #define __packed__			__attribute__ ((__packed__))
 
 struct sec_softc;
 struct sec_session;
 
 /* SEC descriptor definition */
 struct sec_hw_desc_ptr {
 	u_int		shdp_length		: 16;
 	u_int		shdp_j			: 1;
 	u_int		shdp_extent		: 7;
 	u_int		__padding0		: 4;
 	uint64_t	shdp_ptr		: 36;
 } __packed__;
 
 struct sec_hw_desc {
 	union __packed__ {
 		struct __packed__ {
 			u_int	eu_sel0		: 4;
 			u_int	mode0		: 8;
 			u_int	eu_sel1		: 4;
 			u_int	mode1		: 8;
 			u_int	desc_type	: 5;
 			u_int	__padding0	: 1;
 			u_int	dir		: 1;
 			u_int	dn		: 1;
 			u_int	__padding1	: 32;
 		} request;
 		struct __packed__ {
 			u_int	done		: 8;
 			u_int	__padding0	: 27;
 			u_int	iccr0		: 2;
 			u_int	__padding1	: 6;
 			u_int	iccr1		: 2;
 			u_int	__padding2	: 19;
 		} feedback;
 	} shd_control;
 
 	struct sec_hw_desc_ptr	shd_pointer[SEC_POINTERS];
 
 	/* Data below is mapped to descriptor pointers */
 	uint8_t			shd_iv[SEC_MAX_IV_LEN];
 	uint8_t			shd_key[SEC_MAX_KEY_LEN];
 	uint8_t			shd_mkey[SEC_MAX_KEY_LEN];
+	uint8_t			shd_digest[HASH_MAX_LEN];
 } __packed__;
 
 #define shd_eu_sel0		shd_control.request.eu_sel0
 #define shd_mode0		shd_control.request.mode0
 #define shd_eu_sel1		shd_control.request.eu_sel1
 #define shd_mode1		shd_control.request.mode1
 #define shd_desc_type		shd_control.request.desc_type
 #define shd_dir			shd_control.request.dir
 #define shd_dn			shd_control.request.dn
 #define shd_done		shd_control.feedback.done
 #define shd_iccr0		shd_control.feedback.iccr0
 #define shd_iccr1		shd_control.feedback.iccr1
 
 /* SEC link table entries definition */
 struct sec_hw_lt {
 	u_int			shl_length	: 16;
 	u_int			__padding0	: 6;
 	u_int			shl_r		: 1;
 	u_int			shl_n		: 1;
 	u_int			__padding1	: 4;
 	uint64_t		shl_ptr		: 36;
 } __packed__;
 
 struct sec_dma_mem {
 	void			*dma_vaddr;
 	bus_addr_t		dma_paddr;
 	bus_dma_tag_t		dma_tag;
 	bus_dmamap_t		dma_map;
 	u_int			dma_is_map;
 };
 
 struct sec_desc {
 	struct sec_hw_desc	*sd_desc;
 	bus_addr_t		sd_desc_paddr;
 	struct sec_dma_mem	sd_ptr_dmem[SEC_POINTERS];
 	struct cryptop		*sd_crp;
 	u_int			sd_lt_used;
 	u_int			sd_error;
 };
 
 struct sec_lt {
 	struct sec_hw_lt	*sl_lt;
 	bus_addr_t		sl_lt_paddr;
 };
 
 struct sec_eu_methods {
-	int	(*sem_newsession)(struct sec_softc *sc,
-	    struct sec_session *ses, struct cryptoini *enc,
-	    struct cryptoini *mac);
+	bool	(*sem_newsession)(const struct crypto_session_params *csp);
 	int	(*sem_make_desc)(struct sec_softc *sc,
-	    struct sec_session *ses, struct sec_desc *desc,
-	    struct cryptop *crp, int buftype);
+	    const struct crypto_session_params *csp, struct sec_desc *desc,
+	    struct cryptop *crp);
 };
 
 struct sec_session {
 	struct sec_eu_methods	*ss_eu;
 	uint8_t			ss_key[SEC_MAX_KEY_LEN];
 	uint8_t			ss_mkey[SEC_MAX_KEY_LEN];
-	u_int			ss_klen;
-	u_int			ss_mklen;
-	u_int			ss_ivlen;
+	int			ss_mlen;
 };
 
 struct sec_desc_map_info {
 	struct sec_softc	*sdmi_sc;
 	bus_size_t		sdmi_size;
 	bus_size_t		sdmi_offset;
 	struct sec_lt		*sdmi_lt_first;
 	struct sec_lt		*sdmi_lt_last;
 	u_int			sdmi_lt_used;
 };
 
 struct sec_softc {
 	device_t		sc_dev;
 	int32_t			sc_cid;
 	int			sc_blocked;
 	int			sc_shutdown;
 	u_int			sc_version;
 
 	uint64_t		sc_int_error_mask;
 	uint64_t		sc_channel_idle_mask;
 
 	struct mtx		sc_controller_lock;
 	struct mtx		sc_descriptors_lock;
 
 	struct sec_desc		sc_desc[SEC_DESCRIPTORS];
 	u_int			sc_free_desc_get_cnt;
 	u_int			sc_free_desc_put_cnt;
 	u_int			sc_ready_desc_get_cnt;
 	u_int			sc_ready_desc_put_cnt;
 	u_int			sc_queued_desc_get_cnt;
 	u_int			sc_queued_desc_put_cnt;
 
 	struct sec_lt		sc_lt[SEC_LT_ENTRIES + 1];
 	u_int			sc_lt_alloc_cnt;
 	u_int			sc_lt_free_cnt;
 
 	struct sec_dma_mem	sc_desc_dmem;	/* descriptors DMA memory */
 	struct sec_dma_mem	sc_lt_dmem;	/* link tables DMA memory */
 
 	struct resource		*sc_rres;	/* register resource */
         int			sc_rrid;	/* register rid */
 	struct {
 		bus_space_tag_t	bst;
 		bus_space_handle_t bsh;
 	} sc_bas;
 
 	struct resource		*sc_pri_ires;	/* primary irq resource */
 	void			*sc_pri_ihand;	/* primary irq handler */
 	int			sc_pri_irid;	/* primary irq resource id */
 
 	struct resource		*sc_sec_ires;	/* secondary irq resource */
 	void			*sc_sec_ihand;	/* secondary irq handler */
 	int			sc_sec_irid;	/* secondary irq resource id */
 };
 
 /* Locking macros */
 #define SEC_LOCK(sc, what)						\
 	mtx_lock(&(sc)->sc_ ## what ## _lock)
 #define SEC_UNLOCK(sc, what)						\
 	mtx_unlock(&(sc)->sc_ ## what ## _lock)
 #define SEC_LOCK_ASSERT(sc, what)					\
 	mtx_assert(&(sc)->sc_ ## what ## _lock, MA_OWNED)
 
 /* Read/Write definitions */
 #define SEC_READ(sc, reg)						\
 	bus_space_read_8((sc)->sc_bas.bst, (sc)->sc_bas.bsh, (reg))
 #define SEC_WRITE(sc, reg, val)						\
 	bus_space_write_8((sc)->sc_bas.bst, (sc)->sc_bas.bsh, (reg), (val))
 
 /* Base allocation macros (warning: wrap must be 2^n) */
 #define SEC_CNT_INIT(sc, cnt, wrap)					\
 	(((sc)->cnt) = ((wrap) - 1))
 #define SEC_ADD(sc, cnt, wrap, val)					\
 	((sc)->cnt = (((sc)->cnt) + (val)) & ((wrap) - 1))
 #define SEC_INC(sc, cnt, wrap)						\
 	SEC_ADD(sc, cnt, wrap, 1)
 #define SEC_DEC(sc, cnt, wrap)						\
 	SEC_ADD(sc, cnt, wrap, -1)
 #define SEC_GET_GENERIC(sc, tab, cnt, wrap)				\
 	((sc)->tab[SEC_INC(sc, cnt, wrap)])
 #define SEC_PUT_GENERIC(sc, tab, cnt, wrap, val)			\
 	((sc)->tab[SEC_INC(sc, cnt, wrap)] = val)
 
 /* Interface for descriptors */
 #define SEC_GET_FREE_DESC(sc)						\
 	&SEC_GET_GENERIC(sc, sc_desc, sc_free_desc_get_cnt, SEC_DESCRIPTORS)
 
 #define SEC_PUT_BACK_FREE_DESC(sc)					\
 	SEC_DEC(sc, sc_free_desc_get_cnt, SEC_DESCRIPTORS)
 
 #define SEC_DESC_FREE2READY(sc)						\
 	SEC_INC(sc, sc_ready_desc_put_cnt, SEC_DESCRIPTORS)
 
 #define SEC_GET_READY_DESC(sc)						\
 	&SEC_GET_GENERIC(sc, sc_desc, sc_ready_desc_get_cnt, SEC_DESCRIPTORS)
 
 #define SEC_PUT_BACK_READY_DESC(sc)					\
 	SEC_DEC(sc, sc_ready_desc_get_cnt, SEC_DESCRIPTORS)
 
 #define SEC_DESC_READY2QUEUED(sc)					\
 	SEC_INC(sc, sc_queued_desc_put_cnt, SEC_DESCRIPTORS)
 
 #define SEC_GET_QUEUED_DESC(sc)						\
 	&SEC_GET_GENERIC(sc, sc_desc, sc_queued_desc_get_cnt, SEC_DESCRIPTORS)
 
 #define SEC_PUT_BACK_QUEUED_DESC(sc)					\
 	SEC_DEC(sc, sc_queued_desc_get_cnt, SEC_DESCRIPTORS)
 
 #define SEC_DESC_QUEUED2FREE(sc)					\
 	SEC_INC(sc, sc_free_desc_put_cnt, SEC_DESCRIPTORS)
 
 #define SEC_FREE_DESC_CNT(sc)						\
 	(((sc)->sc_free_desc_put_cnt - (sc)->sc_free_desc_get_cnt - 1)	\
 	& (SEC_DESCRIPTORS - 1))
 
 #define SEC_READY_DESC_CNT(sc)						\
 	(((sc)->sc_ready_desc_put_cnt - (sc)->sc_ready_desc_get_cnt) &	\
 	(SEC_DESCRIPTORS - 1))
 
 #define SEC_QUEUED_DESC_CNT(sc)						\
 	(((sc)->sc_queued_desc_put_cnt - (sc)->sc_queued_desc_get_cnt)	\
 	& (SEC_DESCRIPTORS - 1))
 
 #define SEC_DESC_SYNC(sc, mode) do {					\
 	sec_sync_dma_mem(&((sc)->sc_desc_dmem), (mode));		\
 	sec_sync_dma_mem(&((sc)->sc_lt_dmem), (mode));			\
 } while (0)
 
 #define SEC_DESC_SYNC_POINTERS(desc, mode) do {				\
 	u_int i;							\
 	for (i = 0; i < SEC_POINTERS; i++)				\
 		sec_sync_dma_mem(&((desc)->sd_ptr_dmem[i]), (mode));	\
 } while (0)
 
 #define SEC_DESC_FREE_POINTERS(desc) do {				\
 	u_int i;							\
 	for (i = 0; i < SEC_POINTERS; i++)				\
 		sec_free_dma_mem(&(desc)->sd_ptr_dmem[i]);		\
 } while (0);
 
 #define SEC_DESC_PUT_BACK_LT(sc, desc)					\
 	SEC_PUT_BACK_LT(sc, (desc)->sd_lt_used)
 
 #define SEC_DESC_FREE_LT(sc, desc)					\
 	SEC_FREE_LT(sc, (desc)->sd_lt_used)
 
 /* Interface for link tables */
 #define SEC_ALLOC_LT_ENTRY(sc)						\
 	&SEC_GET_GENERIC(sc, sc_lt, sc_lt_alloc_cnt, SEC_LT_ENTRIES)
 
 #define SEC_PUT_BACK_LT(sc, num)					\
 	SEC_ADD(sc, sc_lt_alloc_cnt, SEC_LT_ENTRIES, -(num))
 
 #define SEC_FREE_LT(sc, num)						\
 	SEC_ADD(sc, sc_lt_free_cnt, SEC_LT_ENTRIES, num)
 
 #define SEC_FREE_LT_CNT(sc)						\
 	(((sc)->sc_lt_free_cnt - (sc)->sc_lt_alloc_cnt - 1)		\
 	& (SEC_LT_ENTRIES - 1))
-
-/* DMA Maping defines */
-#define SEC_MEMORY		0
-#define SEC_UIO			1
-#define SEC_MBUF		2
 
 /* Size of SEC registers area */
 #define SEC_IO_SIZE		0x10000
 
 /* SEC Controller registers */
 #define SEC_IER			0x1008
 #define SEC_INT_CH_DN(n)	(1ULL << (((n) * 2) + 32))
 #define SEC_INT_CH_ERR(n)	(1ULL << (((n) * 2) + 33))
 #define SEC_INT_ITO		(1ULL << 55)
 
 #define SEC_ISR			0x1010
 #define SEC_ICR			0x1018
 #define SEC_ID			0x1020
 
 #define SEC_EUASR		0x1028
 #define SEC_EUASR_RNGU(r)	(((r) >> 0) & 0xF)
 #define SEC_EUASR_PKEU(r)	(((r) >> 8) & 0xF)
 #define SEC_EUASR_KEU(r)	(((r) >> 16) & 0xF)
 #define SEC_EUASR_CRCU(r)	(((r) >> 20) & 0xF)
 #define SEC_EUASR_DEU(r)	(((r) >> 32) & 0xF)
 #define SEC_EUASR_AESU(r)	(((r) >> 40) & 0xF)
 #define SEC_EUASR_MDEU(r)	(((r) >> 48) & 0xF)
 #define SEC_EUASR_AFEU(r)	(((r) >> 56) & 0xF)
 
 #define SEC_MCR			0x1030
 #define SEC_MCR_SWR		(1ULL << 32)
 
 /* SEC Channel registers */
 #define SEC_CHAN_CCR(n)		(((n) * 0x100) + 0x1108)
 #define SEC_CHAN_CCR_CDIE	(1ULL << 1)
 #define SEC_CHAN_CCR_NT		(1ULL << 2)
 #define SEC_CHAN_CCR_AWSE	(1ULL << 3)
 #define SEC_CHAN_CCR_CDWE	(1ULL << 4)
 #define SEC_CHAN_CCR_BS		(1ULL << 8)
 #define SEC_CHAN_CCR_WGN	(1ULL << 13)
 #define SEC_CHAN_CCR_R		(1ULL << 32)
 #define SEC_CHAN_CCR_CON	(1ULL << 33)
 
 #define SEC_CHAN_CSR(n)		(((n) * 0x100) + 0x1110)
 #define SEC_CHAN_CSR2_FFLVL_M	0x1FULL
 #define SEC_CHAN_CSR2_FFLVL_S	56
 #define SEC_CHAN_CSR2_GSTATE_M	0x0FULL
 #define SEC_CHAN_CSR2_GSTATE_S	48
 #define SEC_CHAN_CSR2_PSTATE_M	0x0FULL
 #define SEC_CHAN_CSR2_PSTATE_S	40
 #define SEC_CHAN_CSR2_MSTATE_M	0x3FULL
 #define SEC_CHAN_CSR2_MSTATE_S	32
 #define SEC_CHAN_CSR3_FFLVL_M	0x1FULL
 #define SEC_CHAN_CSR3_FFLVL_S	24
 #define SEC_CHAN_CSR3_MSTATE_M	0x1FFULL
 #define SEC_CHAN_CSR3_MSTATE_S	32
 #define SEC_CHAN_CSR3_PSTATE_M	0x7FULL
 #define SEC_CHAN_CSR3_PSTATE_S	48
 #define SEC_CHAN_CSR3_GSTATE_M	0x7FULL
 #define SEC_CHAN_CSR3_GSTATE_S	56
 
 #define SEC_CHAN_CDPR(n)	(((n) * 0x100) + 0x1140)
 #define SEC_CHAN_FF(n)		(((n) * 0x100) + 0x1148)
 
 /* SEC Execution Units numbers */
 #define SEC_EU_NONE		0x0
 #define SEC_EU_AFEU		0x1
 #define SEC_EU_DEU		0x2
 #define SEC_EU_MDEU_A		0x3
 #define SEC_EU_MDEU_B		0xB
 #define SEC_EU_RNGU		0x4
 #define SEC_EU_PKEU		0x5
 #define SEC_EU_AESU		0x6
 #define SEC_EU_KEU		0x7
 #define SEC_EU_CRCU		0x8
 
 /* SEC descriptor types */
 #define SEC_DT_COMMON_NONSNOOP	0x02
 #define SEC_DT_HMAC_SNOOP	0x04
 
 /* SEC AESU declarations and definitions */
 #define SEC_AESU_MODE_ED	(1ULL << 0)
 #define SEC_AESU_MODE_CBC	(1ULL << 1)
 
 /* SEC DEU declarations and definitions */
 #define SEC_DEU_MODE_ED		(1ULL << 0)
 #define SEC_DEU_MODE_TS		(1ULL << 1)
 #define SEC_DEU_MODE_CBC	(1ULL << 2)
 
 /* SEC MDEU declarations and definitions */
 #define SEC_HMAC_HASH_LEN	12
 #define SEC_MDEU_MODE_SHA1	0x00	/* MDEU A */
 #define SEC_MDEU_MODE_SHA384	0x00	/* MDEU B */
 #define SEC_MDEU_MODE_SHA256	0x01
 #define SEC_MDEU_MODE_MD5	0x02	/* MDEU A */
 #define SEC_MDEU_MODE_SHA512	0x02	/* MDEU B */
 #define SEC_MDEU_MODE_SHA224	0x03
 #define SEC_MDEU_MODE_PD	(1ULL << 2)
 #define SEC_MDEU_MODE_HMAC	(1ULL << 3)
 #define SEC_MDEU_MODE_INIT	(1ULL << 4)
 #define SEC_MDEU_MODE_SMAC	(1ULL << 5)
 #define SEC_MDEU_MODE_CICV	(1ULL << 6)
 #define SEC_MDEU_MODE_CONT	(1ULL << 7)
 
 #endif
Index: head/sys/dev/ubsec/ubsec.c
===================================================================
--- head/sys/dev/ubsec/ubsec.c	(revision 359373)
+++ head/sys/dev/ubsec/ubsec.c	(revision 359374)
@@ -1,2798 +1,2707 @@
 /*	$OpenBSD: ubsec.c,v 1.115 2002/09/24 18:33:26 jason Exp $	*/
 
 /*-
  * SPDX-License-Identifier: BSD-4-Clause
  *
  * Copyright (c) 2000 Jason L. Wright (jason@thought.net)
  * Copyright (c) 2000 Theo de Raadt (deraadt@openbsd.org)
  * Copyright (c) 2001 Patrik Lindergren (patrik@ipunplugged.com)
  *
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  * 3. All advertising materials mentioning features or use of this software
  *    must display the following acknowledgement:
  *	This product includes software developed by Jason L. Wright
  * 4. The name of the author may not be used to endorse or promote products
  *    derived from this software without specific prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  * DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
  * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  *
  * Effort sponsored in part by the Defense Advanced Research Projects
  * Agency (DARPA) and Air Force Research Laboratory, Air Force
  * Materiel Command, USAF, under agreement number F30602-01-2-0537.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 /*
  * uBsec 5[56]01, 58xx hardware crypto accelerator
  */
 
 #include "opt_ubsec.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/proc.h>
 #include <sys/errno.h>
 #include <sys/malloc.h>
 #include <sys/kernel.h>
 #include <sys/module.h>
 #include <sys/mbuf.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <sys/sysctl.h>
 #include <sys/endian.h>
+#include <sys/uio.h>
 
 #include <vm/vm.h>
 #include <vm/pmap.h>
 
 #include <machine/bus.h>
 #include <machine/resource.h>
 #include <sys/bus.h>
 #include <sys/rman.h>
 
-#include <crypto/sha1.h>
 #include <opencrypto/cryptodev.h>
-#include <opencrypto/cryptosoft.h>
-#include <sys/md5.h>
+#include <opencrypto/xform_auth.h>
 #include <sys/random.h>
 #include <sys/kobj.h>
 
 #include "cryptodev_if.h"
 
 #include <dev/pci/pcivar.h>
 #include <dev/pci/pcireg.h>
 
 /* grr, #defines for gratuitous incompatibility in queue.h */
 #define	SIMPLEQ_HEAD		STAILQ_HEAD
 #define	SIMPLEQ_ENTRY		STAILQ_ENTRY
 #define	SIMPLEQ_INIT		STAILQ_INIT
 #define	SIMPLEQ_INSERT_TAIL	STAILQ_INSERT_TAIL
 #define	SIMPLEQ_EMPTY		STAILQ_EMPTY
 #define	SIMPLEQ_FIRST		STAILQ_FIRST
 #define	SIMPLEQ_REMOVE_HEAD	STAILQ_REMOVE_HEAD
 #define	SIMPLEQ_FOREACH		STAILQ_FOREACH
 /* ditto for endian.h */
 #define	letoh16(x)		le16toh(x)
 #define	letoh32(x)		le32toh(x)
 
 #ifdef UBSEC_RNDTEST
 #include <dev/rndtest/rndtest.h>
 #endif
 #include <dev/ubsec/ubsecreg.h>
 #include <dev/ubsec/ubsecvar.h>
 
 /*
  * Prototypes and count for the pci_device structure
  */
 static	int ubsec_probe(device_t);
 static	int ubsec_attach(device_t);
 static	int ubsec_detach(device_t);
 static	int ubsec_suspend(device_t);
 static	int ubsec_resume(device_t);
 static	int ubsec_shutdown(device_t);
 
-static	int ubsec_newsession(device_t, crypto_session_t, struct cryptoini *);
+static	int ubsec_probesession(device_t, const struct crypto_session_params *);
+static	int ubsec_newsession(device_t, crypto_session_t,
+	    const struct crypto_session_params *);
 static	int ubsec_process(device_t, struct cryptop *, int);
 static	int ubsec_kprocess(device_t, struct cryptkop *, int);
 
 static device_method_t ubsec_methods[] = {
 	/* Device interface */
 	DEVMETHOD(device_probe,		ubsec_probe),
 	DEVMETHOD(device_attach,	ubsec_attach),
 	DEVMETHOD(device_detach,	ubsec_detach),
 	DEVMETHOD(device_suspend,	ubsec_suspend),
 	DEVMETHOD(device_resume,	ubsec_resume),
 	DEVMETHOD(device_shutdown,	ubsec_shutdown),
 
 	/* crypto device methods */
+	DEVMETHOD(cryptodev_probesession, ubsec_probesession),
 	DEVMETHOD(cryptodev_newsession,	ubsec_newsession),
 	DEVMETHOD(cryptodev_process,	ubsec_process),
 	DEVMETHOD(cryptodev_kprocess,	ubsec_kprocess),
 
 	DEVMETHOD_END
 };
 static driver_t ubsec_driver = {
 	"ubsec",
 	ubsec_methods,
 	sizeof (struct ubsec_softc)
 };
 static devclass_t ubsec_devclass;
 
 DRIVER_MODULE(ubsec, pci, ubsec_driver, ubsec_devclass, 0, 0);
 MODULE_DEPEND(ubsec, crypto, 1, 1, 1);
 #ifdef UBSEC_RNDTEST
 MODULE_DEPEND(ubsec, rndtest, 1, 1, 1);
 #endif
 
 static	void ubsec_intr(void *);
 static	void ubsec_callback(struct ubsec_softc *, struct ubsec_q *);
 static	void ubsec_feed(struct ubsec_softc *);
 static	void ubsec_mcopy(struct mbuf *, struct mbuf *, int, int);
 static	void ubsec_callback2(struct ubsec_softc *, struct ubsec_q2 *);
 static	int ubsec_feed2(struct ubsec_softc *);
 static	void ubsec_rng(void *);
 static	int ubsec_dma_malloc(struct ubsec_softc *, bus_size_t,
 			     struct ubsec_dma_alloc *, int);
 #define	ubsec_dma_sync(_dma, _flags) \
 	bus_dmamap_sync((_dma)->dma_tag, (_dma)->dma_map, (_flags))
 static	void ubsec_dma_free(struct ubsec_softc *, struct ubsec_dma_alloc *);
 static	int ubsec_dmamap_aligned(struct ubsec_operand *op);
 
 static	void ubsec_reset_board(struct ubsec_softc *sc);
 static	void ubsec_init_board(struct ubsec_softc *sc);
 static	void ubsec_init_pciregs(device_t dev);
 static	void ubsec_totalreset(struct ubsec_softc *sc);
 
 static	int ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q);
 
 static	int ubsec_kprocess_modexp_hw(struct ubsec_softc *, struct cryptkop *, int);
 static	int ubsec_kprocess_modexp_sw(struct ubsec_softc *, struct cryptkop *, int);
 static	int ubsec_kprocess_rsapriv(struct ubsec_softc *, struct cryptkop *, int);
 static	void ubsec_kfree(struct ubsec_softc *, struct ubsec_q2 *);
 static	int ubsec_ksigbits(struct crparam *);
 static	void ubsec_kshift_r(u_int, u_int8_t *, u_int, u_int8_t *, u_int);
 static	void ubsec_kshift_l(u_int, u_int8_t *, u_int, u_int8_t *, u_int);
 
 static SYSCTL_NODE(_hw, OID_AUTO, ubsec, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
     "Broadcom driver parameters");
 
 #ifdef UBSEC_DEBUG
 static	void ubsec_dump_pb(volatile struct ubsec_pktbuf *);
 static	void ubsec_dump_mcr(struct ubsec_mcr *);
 static	void ubsec_dump_ctx2(struct ubsec_ctx_keyop *);
 
 static	int ubsec_debug = 0;
 SYSCTL_INT(_hw_ubsec, OID_AUTO, debug, CTLFLAG_RW, &ubsec_debug,
 	    0, "control debugging msgs");
 #endif
 
 #define	READ_REG(sc,r) \
 	bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r))
 
 #define WRITE_REG(sc,reg,val) \
 	bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val)
 
 #define	SWAP32(x) (x) = htole32(ntohl((x)))
 #define	HTOLE32(x) (x) = htole32(x)
 
 struct ubsec_stats ubsecstats;
 SYSCTL_STRUCT(_hw_ubsec, OID_AUTO, stats, CTLFLAG_RD, &ubsecstats,
 	    ubsec_stats, "driver statistics");
 
 static int
 ubsec_probe(device_t dev)
 {
 	if (pci_get_vendor(dev) == PCI_VENDOR_SUN &&
 	    (pci_get_device(dev) == PCI_PRODUCT_SUN_5821 ||
 	     pci_get_device(dev) == PCI_PRODUCT_SUN_SCA1K))
 		return (BUS_PROBE_DEFAULT);
 	if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL &&
 	    (pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5501 ||
 	     pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601))
 		return (BUS_PROBE_DEFAULT);
 	if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
 	    (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5801 ||
 	     pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5802 ||
 	     pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805 ||
 	     pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820 ||
 	     pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 ||
 	     pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822 ||
 	     pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5823 ||
 	     pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5825
 	     ))
 		return (BUS_PROBE_DEFAULT);
 	return (ENXIO);
 }
 
 static const char*
 ubsec_partname(struct ubsec_softc *sc)
 {
 	/* XXX sprintf numbers when not decoded */
 	switch (pci_get_vendor(sc->sc_dev)) {
 	case PCI_VENDOR_BROADCOM:
 		switch (pci_get_device(sc->sc_dev)) {
 		case PCI_PRODUCT_BROADCOM_5801:	return "Broadcom 5801";
 		case PCI_PRODUCT_BROADCOM_5802:	return "Broadcom 5802";
 		case PCI_PRODUCT_BROADCOM_5805:	return "Broadcom 5805";
 		case PCI_PRODUCT_BROADCOM_5820:	return "Broadcom 5820";
 		case PCI_PRODUCT_BROADCOM_5821:	return "Broadcom 5821";
 		case PCI_PRODUCT_BROADCOM_5822:	return "Broadcom 5822";
 		case PCI_PRODUCT_BROADCOM_5823:	return "Broadcom 5823";
 		case PCI_PRODUCT_BROADCOM_5825:	return "Broadcom 5825";
 		}
 		return "Broadcom unknown-part";
 	case PCI_VENDOR_BLUESTEEL:
 		switch (pci_get_device(sc->sc_dev)) {
 		case PCI_PRODUCT_BLUESTEEL_5601: return "Bluesteel 5601";
 		}
 		return "Bluesteel unknown-part";
 	case PCI_VENDOR_SUN:
 		switch (pci_get_device(sc->sc_dev)) {
 		case PCI_PRODUCT_SUN_5821: return "Sun Crypto 5821";
 		case PCI_PRODUCT_SUN_SCA1K: return "Sun Crypto 1K";
 		}
 		return "Sun unknown-part";
 	}
 	return "Unknown-vendor unknown-part";
 }
 
 static void
 default_harvest(struct rndtest_state *rsp, void *buf, u_int count)
 {
 	/* MarkM: FIX!! Check that this does not swamp the harvester! */
 	random_harvest_queue(buf, count, RANDOM_PURE_UBSEC);
 }
 
 static int
 ubsec_attach(device_t dev)
 {
 	struct ubsec_softc *sc = device_get_softc(dev);
 	struct ubsec_dma *dmap;
 	u_int32_t i;
 	int rid;
 
 	bzero(sc, sizeof (*sc));
 	sc->sc_dev = dev;
 
 	SIMPLEQ_INIT(&sc->sc_queue);
 	SIMPLEQ_INIT(&sc->sc_qchip);
 	SIMPLEQ_INIT(&sc->sc_queue2);
 	SIMPLEQ_INIT(&sc->sc_qchip2);
 	SIMPLEQ_INIT(&sc->sc_q2free);
 
 	/* XXX handle power management */
 
 	sc->sc_statmask = BS_STAT_MCR1_DONE | BS_STAT_DMAERR;
 
 	if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL &&
 	    pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601)
 		sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG;
 
 	if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
 	    (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5802 ||
 	     pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805))
 		sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG;
 
 	if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
 	    pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820)
 		sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG |
 		    UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY;
 
 	if ((pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
 	     (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 ||
 	      pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822 ||
 	      pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5823 ||
 	      pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5825)) ||
 	    (pci_get_vendor(dev) == PCI_VENDOR_SUN &&
 	     (pci_get_device(dev) == PCI_PRODUCT_SUN_SCA1K ||
 	      pci_get_device(dev) == PCI_PRODUCT_SUN_5821))) {
 		/* NB: the 5821/5822 defines some additional status bits */
 		sc->sc_statmask |= BS_STAT_MCR1_ALLEMPTY |
 		    BS_STAT_MCR2_ALLEMPTY;
 		sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG |
 		    UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY;
 	}
 
 	pci_enable_busmaster(dev);
 
 	/*
 	 * Setup memory-mapping of PCI registers.
 	 */
 	rid = BS_BAR;
 	sc->sc_sr = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 					   RF_ACTIVE);
 	if (sc->sc_sr == NULL) {
 		device_printf(dev, "cannot map register space\n");
 		goto bad;
 	}
 	sc->sc_st = rman_get_bustag(sc->sc_sr);
 	sc->sc_sh = rman_get_bushandle(sc->sc_sr);
 
 	/*
 	 * Arrange interrupt line.
 	 */
 	rid = 0;
 	sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
 					    RF_SHAREABLE|RF_ACTIVE);
 	if (sc->sc_irq == NULL) {
 		device_printf(dev, "could not map interrupt\n");
 		goto bad1;
 	}
 	/*
 	 * NB: Network code assumes we are blocked with splimp()
 	 *     so make sure the IRQ is mapped appropriately.
 	 */
 	if (bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET | INTR_MPSAFE,
 			   NULL, ubsec_intr, sc, &sc->sc_ih)) {
 		device_printf(dev, "could not establish interrupt\n");
 		goto bad2;
 	}
 
-	sc->sc_cid = crypto_get_driverid(dev, sizeof(struct ubsec_session),
-	    CRYPTOCAP_F_HARDWARE);
-	if (sc->sc_cid < 0) {
-		device_printf(dev, "could not get crypto driver id\n");
-		goto bad3;
-	}
-
 	/*
 	 * Setup DMA descriptor area.
 	 */
 	if (bus_dma_tag_create(bus_get_dma_tag(dev),	/* parent */
 			       1, 0,			/* alignment, bounds */
 			       BUS_SPACE_MAXADDR_32BIT,	/* lowaddr */
 			       BUS_SPACE_MAXADDR,	/* highaddr */
 			       NULL, NULL,		/* filter, filterarg */
 			       0x3ffff,			/* maxsize */
 			       UBS_MAX_SCATTER,		/* nsegments */
 			       0xffff,			/* maxsegsize */
 			       BUS_DMA_ALLOCNOW,	/* flags */
 			       NULL, NULL,		/* lockfunc, lockarg */
 			       &sc->sc_dmat)) {
 		device_printf(dev, "cannot allocate DMA tag\n");
-		goto bad4;
+		goto bad3;
 	}
 	SIMPLEQ_INIT(&sc->sc_freequeue);
 	dmap = sc->sc_dmaa;
 	for (i = 0; i < UBS_MAX_NQUEUE; i++, dmap++) {
 		struct ubsec_q *q;
 
 		q = (struct ubsec_q *)malloc(sizeof(struct ubsec_q),
 		    M_DEVBUF, M_NOWAIT);
 		if (q == NULL) {
 			device_printf(dev, "cannot allocate queue buffers\n");
 			break;
 		}
 
 		if (ubsec_dma_malloc(sc, sizeof(struct ubsec_dmachunk),
 		    &dmap->d_alloc, 0)) {
 			device_printf(dev, "cannot allocate dma buffers\n");
 			free(q, M_DEVBUF);
 			break;
 		}
 		dmap->d_dma = (struct ubsec_dmachunk *)dmap->d_alloc.dma_vaddr;
 
 		q->q_dma = dmap;
 		sc->sc_queuea[i] = q;
 
 		SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
 	}
 	mtx_init(&sc->sc_mcr1lock, device_get_nameunit(dev),
 		"mcr1 operations", MTX_DEF);
 	mtx_init(&sc->sc_freeqlock, device_get_nameunit(dev),
 		"mcr1 free q", MTX_DEF);
 
 	device_printf(sc->sc_dev, "%s\n", ubsec_partname(sc));
 
-	crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
-	crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
-	crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
-	crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
-
 	/*
 	 * Reset Broadcom chip
 	 */
 	ubsec_reset_board(sc);
 
 	/*
 	 * Init Broadcom specific PCI settings
 	 */
 	ubsec_init_pciregs(dev);
 
 	/*
 	 * Init Broadcom chip
 	 */
 	ubsec_init_board(sc);
 
+	sc->sc_cid = crypto_get_driverid(dev, sizeof(struct ubsec_session),
+	    CRYPTOCAP_F_HARDWARE);
+	if (sc->sc_cid < 0) {
+		device_printf(dev, "could not get crypto driver id\n");
+		goto bad4;
+	}
+
 #ifndef UBSEC_NO_RNG
 	if (sc->sc_flags & UBS_FLAGS_RNG) {
 		sc->sc_statmask |= BS_STAT_MCR2_DONE;
 #ifdef UBSEC_RNDTEST
 		sc->sc_rndtest = rndtest_attach(dev);
 		if (sc->sc_rndtest)
 			sc->sc_harvest = rndtest_harvest;
 		else
 			sc->sc_harvest = default_harvest;
 #else
 		sc->sc_harvest = default_harvest;
 #endif
 
 		if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
 		    &sc->sc_rng.rng_q.q_mcr, 0))
 			goto skip_rng;
 
 		if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rngbypass),
 		    &sc->sc_rng.rng_q.q_ctx, 0)) {
 			ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
 			goto skip_rng;
 		}
 
 		if (ubsec_dma_malloc(sc, sizeof(u_int32_t) *
 		    UBSEC_RNG_BUFSIZ, &sc->sc_rng.rng_buf, 0)) {
 			ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx);
 			ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
 			goto skip_rng;
 		}
 
 		if (hz >= 100)
 			sc->sc_rnghz = hz / 100;
 		else
 			sc->sc_rnghz = 1;
 		callout_init(&sc->sc_rngto, 1);
 		callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc);
 skip_rng:
 	;
 	}
 #endif /* UBSEC_NO_RNG */
 	mtx_init(&sc->sc_mcr2lock, device_get_nameunit(dev),
 		"mcr2 operations", MTX_DEF);
 
 	if (sc->sc_flags & UBS_FLAGS_KEY) {
 		sc->sc_statmask |= BS_STAT_MCR2_DONE;
 
 		crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0);
 #if 0
 		crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0);
 #endif
 	}
 	return (0);
 bad4:
-	crypto_unregister_all(sc->sc_cid);
+	while (!SIMPLEQ_EMPTY(&sc->sc_freequeue)) {
+		struct ubsec_q *q;
+
+		q = SIMPLEQ_FIRST(&sc->sc_freequeue);
+		SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q_next);
+		ubsec_dma_free(sc, &q->q_dma->d_alloc);
+		free(q, M_DEVBUF);
+	}
+	bus_dma_tag_destroy(sc->sc_dmat);
 bad3:
 	bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
 bad2:
 	bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
 bad1:
 	bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr);
 bad:
 	return (ENXIO);
 }
 
 /*
  * Detach a device that successfully probed.
  */
 static int
 ubsec_detach(device_t dev)
 {
 	struct ubsec_softc *sc = device_get_softc(dev);
 
 	/* XXX wait/abort active ops */
 
+	crypto_unregister_all(sc->sc_cid);
+
 	/* disable interrupts */
 	WRITE_REG(sc, BS_CTRL, READ_REG(sc, BS_CTRL) &~
 		(BS_CTRL_MCR2INT | BS_CTRL_MCR1INT | BS_CTRL_DMAERR));
 
 	callout_stop(&sc->sc_rngto);
+	bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
 
-	crypto_unregister_all(sc->sc_cid);
-
 #ifdef UBSEC_RNDTEST
 	if (sc->sc_rndtest)
 		rndtest_detach(sc->sc_rndtest);
 #endif
 
 	while (!SIMPLEQ_EMPTY(&sc->sc_freequeue)) {
 		struct ubsec_q *q;
 
 		q = SIMPLEQ_FIRST(&sc->sc_freequeue);
 		SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q_next);
 		ubsec_dma_free(sc, &q->q_dma->d_alloc);
 		free(q, M_DEVBUF);
 	}
 	mtx_destroy(&sc->sc_mcr1lock);
 	mtx_destroy(&sc->sc_freeqlock);
 #ifndef UBSEC_NO_RNG
 	if (sc->sc_flags & UBS_FLAGS_RNG) {
 		ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
 		ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx);
 		ubsec_dma_free(sc, &sc->sc_rng.rng_buf);
 	}
 #endif /* UBSEC_NO_RNG */
 	mtx_destroy(&sc->sc_mcr2lock);
 
 	bus_generic_detach(dev);
-	bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
 	bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
 
 	bus_dma_tag_destroy(sc->sc_dmat);
 	bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr);
 
 	return (0);
 }
 
 /*
  * Stop all chip i/o so that the kernel's probe routines don't
  * get confused by errant DMAs when rebooting.
  */
 static int
 ubsec_shutdown(device_t dev)
 {
 #ifdef notyet
 	ubsec_stop(device_get_softc(dev));
 #endif
 	return (0);
 }
 
 /*
  * Device suspend routine.
  */
 static int
 ubsec_suspend(device_t dev)
 {
 	struct ubsec_softc *sc = device_get_softc(dev);
 
 #ifdef notyet
 	/* XXX stop the device and save PCI settings */
 #endif
 	sc->sc_suspended = 1;
 
 	return (0);
 }
 
 static int
 ubsec_resume(device_t dev)
 {
 	struct ubsec_softc *sc = device_get_softc(dev);
 
 #ifdef notyet
 	/* XXX retore PCI settings and start the device */
 #endif
 	sc->sc_suspended = 0;
 	return (0);
 }
 
 /*
  * UBSEC Interrupt routine
  */
 static void
 ubsec_intr(void *arg)
 {
 	struct ubsec_softc *sc = arg;
 	volatile u_int32_t stat;
 	struct ubsec_q *q;
 	struct ubsec_dma *dmap;
 	int npkts = 0, i;
 
 	stat = READ_REG(sc, BS_STAT);
 	stat &= sc->sc_statmask;
 	if (stat == 0)
 		return;
 
 	WRITE_REG(sc, BS_STAT, stat);		/* IACK */
 
 	/*
 	 * Check to see if we have any packets waiting for us
 	 */
 	if ((stat & BS_STAT_MCR1_DONE)) {
 		mtx_lock(&sc->sc_mcr1lock);
 		while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) {
 			q = SIMPLEQ_FIRST(&sc->sc_qchip);
 			dmap = q->q_dma;
 
 			if ((dmap->d_dma->d_mcr.mcr_flags & htole16(UBS_MCR_DONE)) == 0)
 				break;
 
 			SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q_next);
 
 			npkts = q->q_nstacked_mcrs;
 			sc->sc_nqchip -= 1+npkts;
 			/*
 			 * search for further sc_qchip ubsec_q's that share
 			 * the same MCR, and complete them too, they must be
 			 * at the top.
 			 */
 			for (i = 0; i < npkts; i++) {
 				if(q->q_stacked_mcr[i]) {
 					ubsec_callback(sc, q->q_stacked_mcr[i]);
 				} else {
 					break;
 				}
 			}
 			ubsec_callback(sc, q);
 		}
 		/*
 		 * Don't send any more packet to chip if there has been
 		 * a DMAERR.
 		 */
 		if (!(stat & BS_STAT_DMAERR))
 			ubsec_feed(sc);
 		mtx_unlock(&sc->sc_mcr1lock);
 	}
 
 	/*
 	 * Check to see if we have any key setups/rng's waiting for us
 	 */
 	if ((sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) &&
 	    (stat & BS_STAT_MCR2_DONE)) {
 		struct ubsec_q2 *q2;
 		struct ubsec_mcr *mcr;
 
 		mtx_lock(&sc->sc_mcr2lock);
 		while (!SIMPLEQ_EMPTY(&sc->sc_qchip2)) {
 			q2 = SIMPLEQ_FIRST(&sc->sc_qchip2);
 
 			ubsec_dma_sync(&q2->q_mcr,
 			    BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
 
 			mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr;
 			if ((mcr->mcr_flags & htole16(UBS_MCR_DONE)) == 0) {
 				ubsec_dma_sync(&q2->q_mcr,
 				    BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
 				break;
 			}
 			SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip2, q_next);
 			ubsec_callback2(sc, q2);
 			/*
 			 * Don't send any more packet to chip if there has been
 			 * a DMAERR.
 			 */
 			if (!(stat & BS_STAT_DMAERR))
 				ubsec_feed2(sc);
 		}
 		mtx_unlock(&sc->sc_mcr2lock);
 	}
 
 	/*
 	 * Check to see if we got any DMA Error
 	 */
 	if (stat & BS_STAT_DMAERR) {
 #ifdef UBSEC_DEBUG
 		if (ubsec_debug) {
 			volatile u_int32_t a = READ_REG(sc, BS_ERR);
 
 			printf("dmaerr %s@%08x\n",
 			    (a & BS_ERR_READ) ? "read" : "write",
 			    a & BS_ERR_ADDR);
 		}
 #endif /* UBSEC_DEBUG */
 		ubsecstats.hst_dmaerr++;
 		mtx_lock(&sc->sc_mcr1lock);
 		ubsec_totalreset(sc);
 		ubsec_feed(sc);
 		mtx_unlock(&sc->sc_mcr1lock);
 	}
 
 	if (sc->sc_needwakeup) {		/* XXX check high watermark */
 		int wakeup;
 
 		mtx_lock(&sc->sc_freeqlock);
 		wakeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ);
 #ifdef UBSEC_DEBUG
 		if (ubsec_debug)
 			device_printf(sc->sc_dev, "wakeup crypto (%x)\n",
 				sc->sc_needwakeup);
 #endif /* UBSEC_DEBUG */
 		sc->sc_needwakeup &= ~wakeup;
 		mtx_unlock(&sc->sc_freeqlock);
 		crypto_unblock(sc->sc_cid, wakeup);
 	}
 }
 
 /*
  * ubsec_feed() - aggregate and post requests to chip
  */
 static void
 ubsec_feed(struct ubsec_softc *sc)
 {
 	struct ubsec_q *q, *q2;
 	int npkts, i;
 	void *v;
 	u_int32_t stat;
 
 	/*
 	 * Decide how many ops to combine in a single MCR.  We cannot
 	 * aggregate more than UBS_MAX_AGGR because this is the number
 	 * of slots defined in the data structure.  Note that
 	 * aggregation only happens if ops are marked batch'able.
 	 * Aggregating ops reduces the number of interrupts to the host
 	 * but also (potentially) increases the latency for processing
 	 * completed ops as we only get an interrupt when all aggregated
 	 * ops have completed.
 	 */
 	if (sc->sc_nqueue == 0)
 		return;
 	if (sc->sc_nqueue > 1) {
 		npkts = 0;
 		SIMPLEQ_FOREACH(q, &sc->sc_queue, q_next) {
 			npkts++;
 			if ((q->q_crp->crp_flags & CRYPTO_F_BATCH) == 0)
 				break;
 		}
 	} else
 		npkts = 1;
 	/*
 	 * Check device status before going any further.
 	 */
 	if ((stat = READ_REG(sc, BS_STAT)) & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) {
 		if (stat & BS_STAT_DMAERR) {
 			ubsec_totalreset(sc);
 			ubsecstats.hst_dmaerr++;
 		} else
 			ubsecstats.hst_mcr1full++;
 		return;
 	}
 	if (sc->sc_nqueue > ubsecstats.hst_maxqueue)
 		ubsecstats.hst_maxqueue = sc->sc_nqueue;
 	if (npkts > UBS_MAX_AGGR)
 		npkts = UBS_MAX_AGGR;
 	if (npkts < 2)				/* special case 1 op */
 		goto feed1;
 
 	ubsecstats.hst_totbatch += npkts-1;
 #ifdef UBSEC_DEBUG
 	if (ubsec_debug)
 		printf("merging %d records\n", npkts);
 #endif /* UBSEC_DEBUG */
 
 	q = SIMPLEQ_FIRST(&sc->sc_queue);
 	SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q_next);
 	--sc->sc_nqueue;
 
 	bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_PREWRITE);
 	if (q->q_dst_map != NULL)
 		bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, BUS_DMASYNC_PREREAD);
 
 	q->q_nstacked_mcrs = npkts - 1;		/* Number of packets stacked */
 
 	for (i = 0; i < q->q_nstacked_mcrs; i++) {
 		q2 = SIMPLEQ_FIRST(&sc->sc_queue);
 		bus_dmamap_sync(sc->sc_dmat, q2->q_src_map,
 		    BUS_DMASYNC_PREWRITE);
 		if (q2->q_dst_map != NULL)
 			bus_dmamap_sync(sc->sc_dmat, q2->q_dst_map,
 			    BUS_DMASYNC_PREREAD);
 		SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q_next);
 		--sc->sc_nqueue;
 
 		v = (void*)(((char *)&q2->q_dma->d_dma->d_mcr) + sizeof(struct ubsec_mcr) -
 		    sizeof(struct ubsec_mcr_add));
 		bcopy(v, &q->q_dma->d_dma->d_mcradd[i], sizeof(struct ubsec_mcr_add));
 		q->q_stacked_mcr[i] = q2;
 	}
 	q->q_dma->d_dma->d_mcr.mcr_pkts = htole16(npkts);
 	SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next);
 	sc->sc_nqchip += npkts;
 	if (sc->sc_nqchip > ubsecstats.hst_maxqchip)
 		ubsecstats.hst_maxqchip = sc->sc_nqchip;
 	ubsec_dma_sync(&q->q_dma->d_alloc,
 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 	WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr +
 	    offsetof(struct ubsec_dmachunk, d_mcr));
 	return;
 feed1:
 	q = SIMPLEQ_FIRST(&sc->sc_queue);
 
 	bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_PREWRITE);
 	if (q->q_dst_map != NULL)
 		bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, BUS_DMASYNC_PREREAD);
 	ubsec_dma_sync(&q->q_dma->d_alloc,
 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 
 	WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr +
 	    offsetof(struct ubsec_dmachunk, d_mcr));
 #ifdef UBSEC_DEBUG
 	if (ubsec_debug)
 		printf("feed1: q->chip %p %08x stat %08x\n",
 		      q, (u_int32_t)vtophys(&q->q_dma->d_dma->d_mcr),
 		      stat);
 #endif /* UBSEC_DEBUG */
 	SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q_next);
 	--sc->sc_nqueue;
 	SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next);
 	sc->sc_nqchip++;
 	if (sc->sc_nqchip > ubsecstats.hst_maxqchip)
 		ubsecstats.hst_maxqchip = sc->sc_nqchip;
 	return;
 }
 
 static void
-ubsec_setup_enckey(struct ubsec_session *ses, int algo, caddr_t key)
+ubsec_setup_enckey(struct ubsec_session *ses, int algo, const void *key)
 {
 
 	/* Go ahead and compute key in ubsec's byte order */
 	if (algo == CRYPTO_DES_CBC) {
 		bcopy(key, &ses->ses_deskey[0], 8);
 		bcopy(key, &ses->ses_deskey[2], 8);
 		bcopy(key, &ses->ses_deskey[4], 8);
 	} else
 		bcopy(key, ses->ses_deskey, 24);
 
 	SWAP32(ses->ses_deskey[0]);
 	SWAP32(ses->ses_deskey[1]);
 	SWAP32(ses->ses_deskey[2]);
 	SWAP32(ses->ses_deskey[3]);
 	SWAP32(ses->ses_deskey[4]);
 	SWAP32(ses->ses_deskey[5]);
 }
 
 static void
-ubsec_setup_mackey(struct ubsec_session *ses, int algo, caddr_t key, int klen)
+ubsec_setup_mackey(struct ubsec_session *ses, int algo, const char *key,
+    int klen)
 {
 	MD5_CTX md5ctx;
 	SHA1_CTX sha1ctx;
-	int i;
 
-	for (i = 0; i < klen; i++)
-		key[i] ^= HMAC_IPAD_VAL;
-
 	if (algo == CRYPTO_MD5_HMAC) {
-		MD5Init(&md5ctx);
-		MD5Update(&md5ctx, key, klen);
-		MD5Update(&md5ctx, hmac_ipad_buffer, MD5_BLOCK_LEN - klen);
+		hmac_init_ipad(&auth_hash_hmac_md5, key, klen, &md5ctx);
 		bcopy(md5ctx.state, ses->ses_hminner, sizeof(md5ctx.state));
+
+		hmac_init_opad(&auth_hash_hmac_md5, key, klen, &md5ctx);
+		bcopy(md5ctx.state, ses->ses_hmouter, sizeof(md5ctx.state));
+
+		explicit_bzero(&md5ctx, sizeof(md5ctx));
 	} else {
-		SHA1Init(&sha1ctx);
-		SHA1Update(&sha1ctx, key, klen);
-		SHA1Update(&sha1ctx, hmac_ipad_buffer,
-		    SHA1_BLOCK_LEN - klen);
+		hmac_init_ipad(&auth_hash_hmac_sha1, key, klen, &sha1ctx);
 		bcopy(sha1ctx.h.b32, ses->ses_hminner, sizeof(sha1ctx.h.b32));
+
+		hmac_init_opad(&auth_hash_hmac_sha1, key, klen, &sha1ctx);
+		bcopy(sha1ctx.h.b32, ses->ses_hmouter, sizeof(sha1ctx.h.b32));
+
+		explicit_bzero(&sha1ctx, sizeof(sha1ctx));
 	}
+}
 
-	for (i = 0; i < klen; i++)
-		key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
+static bool
+ubsec_auth_supported(const struct crypto_session_params *csp)
+{
 
-	if (algo == CRYPTO_MD5_HMAC) {
-		MD5Init(&md5ctx);
-		MD5Update(&md5ctx, key, klen);
-		MD5Update(&md5ctx, hmac_opad_buffer, MD5_BLOCK_LEN - klen);
-		bcopy(md5ctx.state, ses->ses_hmouter, sizeof(md5ctx.state));
-	} else {
-		SHA1Init(&sha1ctx);
-		SHA1Update(&sha1ctx, key, klen);
-		SHA1Update(&sha1ctx, hmac_opad_buffer,
-		    SHA1_BLOCK_LEN - klen);
-		bcopy(sha1ctx.h.b32, ses->ses_hmouter, sizeof(sha1ctx.h.b32));
+	switch (csp->csp_auth_alg) {
+	case CRYPTO_MD5_HMAC:
+	case CRYPTO_SHA1_HMAC:
+		return (true);
+	default:
+		return (false);
 	}
+}
 
-	for (i = 0; i < klen; i++)
-		key[i] ^= HMAC_OPAD_VAL;
+static bool
+ubsec_cipher_supported(const struct crypto_session_params *csp)
+{
+
+	switch (csp->csp_cipher_alg) {
+	case CRYPTO_DES_CBC:
+	case CRYPTO_3DES_CBC:
+		return (csp->csp_ivlen == 8);
+	default:
+		return (false);
+	}
 }
 
-/*
- * Allocate a new 'session' and return an encoded session id.  'sidp'
- * contains our registration id, and should contain an encoded session
- * id on successful allocation.
- */
 static int
-ubsec_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
+ubsec_probesession(device_t dev, const struct crypto_session_params *csp)
 {
-	struct ubsec_softc *sc = device_get_softc(dev);
-	struct cryptoini *c, *encini = NULL, *macini = NULL;
-	struct ubsec_session *ses = NULL;
 
-	if (cri == NULL || sc == NULL)
+	if (csp->csp_flags != 0)
 		return (EINVAL);
-
-	for (c = cri; c != NULL; c = c->cri_next) {
-		if (c->cri_alg == CRYPTO_MD5_HMAC ||
-		    c->cri_alg == CRYPTO_SHA1_HMAC) {
-			if (macini)
-				return (EINVAL);
-			macini = c;
-		} else if (c->cri_alg == CRYPTO_DES_CBC ||
-		    c->cri_alg == CRYPTO_3DES_CBC) {
-			if (encini)
-				return (EINVAL);
-			encini = c;
-		} else
+	switch (csp->csp_mode) {
+	case CSP_MODE_DIGEST:
+		if (!ubsec_auth_supported(csp))
 			return (EINVAL);
-	}
-	if (encini == NULL && macini == NULL)
+		break;
+	case CSP_MODE_CIPHER:
+		if (!ubsec_cipher_supported(csp))
+			return (EINVAL);
+		break;
+	case CSP_MODE_ETA:
+		if (!ubsec_auth_supported(csp) ||
+		    !ubsec_cipher_supported(csp))
+			return (EINVAL);
+		break;
+	default:
 		return (EINVAL);
+	}
 
-	ses = crypto_get_driver_session(cses);
-	if (encini) {
-		/* get an IV, network byte order */
-		/* XXX may read fewer than requested */
-		read_random(ses->ses_iv, sizeof(ses->ses_iv));
+	return (CRYPTODEV_PROBE_HARDWARE);
+}
 
-		if (encini->cri_key != NULL) {
-			ubsec_setup_enckey(ses, encini->cri_alg,
-			    encini->cri_key);
-		}
-	}
+/*
+ * Allocate a new 'session'.
+ */
+static int
+ubsec_newsession(device_t dev, crypto_session_t cses,
+    const struct crypto_session_params *csp)
+{
+	struct ubsec_session *ses;
 
-	if (macini) {
-		ses->ses_mlen = macini->cri_mlen;
+	ses = crypto_get_driver_session(cses);
+	if (csp->csp_cipher_alg != 0 && csp->csp_cipher_key != NULL)
+		ubsec_setup_enckey(ses, csp->csp_cipher_alg,
+		    csp->csp_cipher_key);
+
+	if (csp->csp_auth_alg != 0) {
+		ses->ses_mlen = csp->csp_auth_mlen;
 		if (ses->ses_mlen == 0) {
-			if (macini->cri_alg == CRYPTO_MD5_HMAC)
+			if (csp->csp_auth_alg == CRYPTO_MD5_HMAC)
 				ses->ses_mlen = MD5_HASH_LEN;
 			else
 				ses->ses_mlen = SHA1_HASH_LEN;
 		}
 
-		if (macini->cri_key != NULL) {
-			ubsec_setup_mackey(ses, macini->cri_alg,
-			    macini->cri_key, macini->cri_klen / 8);
+		if (csp->csp_auth_key != NULL) {
+			ubsec_setup_mackey(ses, csp->csp_auth_alg,
+			    csp->csp_auth_key, csp->csp_auth_klen);
 		}
 	}
 
 	return (0);
 }
 
+static bus_size_t
+ubsec_crp_length(struct cryptop *crp)
+{
+
+	switch (crp->crp_buf_type) {
+	case CRYPTO_BUF_MBUF:
+		return (crp->crp_mbuf->m_pkthdr.len);
+	case CRYPTO_BUF_UIO:
+		return (crp->crp_uio->uio_resid);
+	case CRYPTO_BUF_CONTIG:
+		return (crp->crp_ilen);
+	default:
+		panic("bad crp buffer type");
+	}
+}
+
 static void
-ubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error)
+ubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, int error)
 {
 	struct ubsec_operand *op = arg;
 
 	KASSERT(nsegs <= UBS_MAX_SCATTER,
 		("Too many DMA segments returned when mapping operand"));
 #ifdef UBSEC_DEBUG
 	if (ubsec_debug)
-		printf("ubsec_op_cb: mapsize %u nsegs %d error %d\n",
-			(u_int) mapsize, nsegs, error);
+		printf("ubsec_op_cb: nsegs %d error %d\n",
+			nsegs, error);
 #endif
 	if (error != 0)
 		return;
-	op->mapsize = mapsize;
 	op->nsegs = nsegs;
 	bcopy(seg, op->segs, nsegs * sizeof (seg[0]));
 }
 
 static int
 ubsec_process(device_t dev, struct cryptop *crp, int hint)
 {
+	const struct crypto_session_params *csp;
 	struct ubsec_softc *sc = device_get_softc(dev);
 	struct ubsec_q *q = NULL;
 	int err = 0, i, j, nicealign;
-	struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
-	int encoffset = 0, macoffset = 0, cpskip, cpoffset;
+	int cpskip, cpoffset;
 	int sskip, dskip, stheend, dtheend;
 	int16_t coffset;
 	struct ubsec_session *ses;
 	struct ubsec_pktctx ctx;
 	struct ubsec_dma *dmap = NULL;
 
-	if (crp == NULL || crp->crp_callback == NULL || sc == NULL) {
-		ubsecstats.hst_invalid++;
-		return (EINVAL);
-	}
-
 	mtx_lock(&sc->sc_freeqlock);
 	if (SIMPLEQ_EMPTY(&sc->sc_freequeue)) {
 		ubsecstats.hst_queuefull++;
 		sc->sc_needwakeup |= CRYPTO_SYMQ;
 		mtx_unlock(&sc->sc_freeqlock);
 		return (ERESTART);
 	}
 	q = SIMPLEQ_FIRST(&sc->sc_freequeue);
 	SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q_next);
 	mtx_unlock(&sc->sc_freeqlock);
 
 	dmap = q->q_dma; /* Save dma pointer */
 	bzero(q, sizeof(struct ubsec_q));
 	bzero(&ctx, sizeof(ctx));
 
 	q->q_dma = dmap;
 	ses = crypto_get_driver_session(crp->crp_session);
 
-	if (crp->crp_flags & CRYPTO_F_IMBUF) {
-		q->q_src_m = (struct mbuf *)crp->crp_buf;
-		q->q_dst_m = (struct mbuf *)crp->crp_buf;
-	} else if (crp->crp_flags & CRYPTO_F_IOV) {
-		q->q_src_io = (struct uio *)crp->crp_buf;
-		q->q_dst_io = (struct uio *)crp->crp_buf;
-	} else {
-		ubsecstats.hst_badflags++;
-		err = EINVAL;
-		goto errout;	/* XXX we don't handle contiguous blocks! */
-	}
-
 	bzero(&dmap->d_dma->d_mcr, sizeof(struct ubsec_mcr));
 
 	dmap->d_dma->d_mcr.mcr_pkts = htole16(1);
 	dmap->d_dma->d_mcr.mcr_flags = 0;
 	q->q_crp = crp;
 
-	crd1 = crp->crp_desc;
-	if (crd1 == NULL) {
-		ubsecstats.hst_nodesc++;
-		err = EINVAL;
-		goto errout;
-	}
-	crd2 = crd1->crd_next;
+	csp = crypto_get_params(crp->crp_session);
 
-	if (crd2 == NULL) {
-		if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
-		    crd1->crd_alg == CRYPTO_SHA1_HMAC) {
-			maccrd = crd1;
-			enccrd = NULL;
-		} else if (crd1->crd_alg == CRYPTO_DES_CBC ||
-		    crd1->crd_alg == CRYPTO_3DES_CBC) {
-			maccrd = NULL;
-			enccrd = crd1;
-		} else {
-			ubsecstats.hst_badalg++;
-			err = EINVAL;
-			goto errout;
+	if (csp->csp_cipher_alg != 0) {
+		if (crp->crp_cipher_key != NULL) {
+			ubsec_setup_enckey(ses, csp->csp_cipher_alg,
+			    crp->crp_cipher_key);
 		}
-	} else {
-		if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
-		    crd1->crd_alg == CRYPTO_SHA1_HMAC) &&
-		    (crd2->crd_alg == CRYPTO_DES_CBC ||
-			crd2->crd_alg == CRYPTO_3DES_CBC) &&
-		    ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
-			maccrd = crd1;
-			enccrd = crd2;
-		} else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
-		    crd1->crd_alg == CRYPTO_3DES_CBC) &&
-		    (crd2->crd_alg == CRYPTO_MD5_HMAC ||
-			crd2->crd_alg == CRYPTO_SHA1_HMAC) &&
-		    (crd1->crd_flags & CRD_F_ENCRYPT)) {
-			enccrd = crd1;
-			maccrd = crd2;
-		} else {
-			/*
-			 * We cannot order the ubsec as requested
-			 */
-			ubsecstats.hst_badalg++;
-			err = EINVAL;
-			goto errout;
-		}
-	}
 
-	if (enccrd) {
-		if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT) {
-			ubsec_setup_enckey(ses, enccrd->crd_alg,
-			    enccrd->crd_key);
-		}
-
-		encoffset = enccrd->crd_skip;
 		ctx.pc_flags |= htole16(UBS_PKTCTX_ENC_3DES);
 
-		if (enccrd->crd_flags & CRD_F_ENCRYPT) {
-			q->q_flags |= UBSEC_QFLAGS_COPYOUTIV;
+		if (crp->crp_flags & CRYPTO_F_IV_GENERATE) {
+			arc4rand(ctx.pc_iv, csp->csp_ivlen, 0);
+			crypto_copyback(crp, crp->crp_iv_start,
+			    csp->csp_ivlen, ctx.pc_iv);
+		} else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+			memcpy(ctx.pc_iv, crp->crp_iv, csp->csp_ivlen);
+		else
+			crypto_copydata(crp, crp->crp_iv_start, csp->csp_ivlen,
+			    ctx.pc_iv);
 
-			if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
-				bcopy(enccrd->crd_iv, ctx.pc_iv, 8);
-			else {
-				ctx.pc_iv[0] = ses->ses_iv[0];
-				ctx.pc_iv[1] = ses->ses_iv[1];
-			}
-
-			if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
-				crypto_copyback(crp->crp_flags, crp->crp_buf,
-				    enccrd->crd_inject, 8, (caddr_t)ctx.pc_iv);
-			}
-		} else {
+		if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
 			ctx.pc_flags |= htole16(UBS_PKTCTX_INBOUND);
-
-			if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
-				bcopy(enccrd->crd_iv, ctx.pc_iv, 8);
-			else {
-				crypto_copydata(crp->crp_flags, crp->crp_buf,
-				    enccrd->crd_inject, 8, (caddr_t)ctx.pc_iv);
-			}
 		}
 
 		ctx.pc_deskey[0] = ses->ses_deskey[0];
 		ctx.pc_deskey[1] = ses->ses_deskey[1];
 		ctx.pc_deskey[2] = ses->ses_deskey[2];
 		ctx.pc_deskey[3] = ses->ses_deskey[3];
 		ctx.pc_deskey[4] = ses->ses_deskey[4];
 		ctx.pc_deskey[5] = ses->ses_deskey[5];
 		SWAP32(ctx.pc_iv[0]);
 		SWAP32(ctx.pc_iv[1]);
 	}
 
-	if (maccrd) {
-		if (maccrd->crd_flags & CRD_F_KEY_EXPLICIT) {
-			ubsec_setup_mackey(ses, maccrd->crd_alg,
-			    maccrd->crd_key, maccrd->crd_klen / 8);
+	if (csp->csp_auth_alg != 0) {
+		if (crp->crp_auth_key != NULL) {
+			ubsec_setup_mackey(ses, csp->csp_auth_alg,
+			    crp->crp_auth_key, csp->csp_auth_klen);
 		}
 
-		macoffset = maccrd->crd_skip;
-
-		if (maccrd->crd_alg == CRYPTO_MD5_HMAC)
+		if (csp->csp_auth_alg == CRYPTO_MD5_HMAC)
 			ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_MD5);
 		else
 			ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_SHA1);
 
 		for (i = 0; i < 5; i++) {
 			ctx.pc_hminner[i] = ses->ses_hminner[i];
 			ctx.pc_hmouter[i] = ses->ses_hmouter[i];
 
 			HTOLE32(ctx.pc_hminner[i]);
 			HTOLE32(ctx.pc_hmouter[i]);
 		}
 	}
 
-	if (enccrd && maccrd) {
+	if (csp->csp_mode == CSP_MODE_ETA) {
 		/*
-		 * ubsec cannot handle packets where the end of encryption
-		 * and authentication are not the same, or where the
-		 * encrypted part begins before the authenticated part.
+		 * ubsec only supports ETA requests where there is no
+		 * gap between the AAD and payload.
 		 */
-		if ((encoffset + enccrd->crd_len) !=
-		    (macoffset + maccrd->crd_len)) {
+		if (crp->crp_aad_length != 0 &&
+		    crp->crp_aad_start + crp->crp_aad_length !=
+		    crp->crp_payload_start) {
 			ubsecstats.hst_lenmismatch++;
 			err = EINVAL;
 			goto errout;
 		}
-		if (enccrd->crd_skip < maccrd->crd_skip) {
-			ubsecstats.hst_skipmismatch++;
-			err = EINVAL;
-			goto errout;
+
+		if (crp->crp_aad_length != 0) {
+			sskip = crp->crp_aad_start;
+		} else {
+			sskip = crp->crp_payload_start;
 		}
-		sskip = maccrd->crd_skip;
-		cpskip = dskip = enccrd->crd_skip;
-		stheend = maccrd->crd_len;
-		dtheend = enccrd->crd_len;
-		coffset = enccrd->crd_skip - maccrd->crd_skip;
+		cpskip = dskip = crp->crp_payload_start;
+		stheend = crp->crp_aad_length + crp->crp_payload_length;
+		dtheend = crp->crp_payload_length;
+		coffset = crp->crp_aad_length;
 		cpoffset = cpskip + dtheend;
 #ifdef UBSEC_DEBUG
 		if (ubsec_debug) {
-			printf("mac: skip %d, len %d, inject %d\n",
-			    maccrd->crd_skip, maccrd->crd_len, maccrd->crd_inject);
-			printf("enc: skip %d, len %d, inject %d\n",
-			    enccrd->crd_skip, enccrd->crd_len, enccrd->crd_inject);
+			printf("AAD: start %d, len %d, digest %d\n",
+			    crp->crp_aad_start, crp->crp_aad_length,
+			    crp->crp_digest_start);
+			printf("payload: start %d, len %d, IV %d\n",
+			    crp->crp_payload_start, crp->crp_payload_length,
+			    crp->crp_iv_start);
 			printf("src: skip %d, len %d\n", sskip, stheend);
 			printf("dst: skip %d, len %d\n", dskip, dtheend);
 			printf("ubs: coffset %d, pktlen %d, cpskip %d, cpoffset %d\n",
 			    coffset, stheend, cpskip, cpoffset);
 		}
 #endif
 	} else {
-		cpskip = dskip = sskip = macoffset + encoffset;
-		dtheend = stheend = (enccrd)?enccrd->crd_len:maccrd->crd_len;
+		cpskip = dskip = sskip = crp->crp_payload_start;
+		dtheend = stheend = crp->crp_payload_length;
 		cpoffset = cpskip + dtheend;
 		coffset = 0;
 	}
 	ctx.pc_offset = htole16(coffset >> 2);
 
 	if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &q->q_src_map)) {
 		ubsecstats.hst_nomap++;
 		err = ENOMEM;
 		goto errout;
 	}
-	if (crp->crp_flags & CRYPTO_F_IMBUF) {
-		if (bus_dmamap_load_mbuf(sc->sc_dmat, q->q_src_map,
-		    q->q_src_m, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) {
-			bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
-			q->q_src_map = NULL;
-			ubsecstats.hst_noload++;
-			err = ENOMEM;
-			goto errout;
-		}
-	} else if (crp->crp_flags & CRYPTO_F_IOV) {
-		if (bus_dmamap_load_uio(sc->sc_dmat, q->q_src_map,
-		    q->q_src_io, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) {
-			bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
-			q->q_src_map = NULL;
-			ubsecstats.hst_noload++;
-			err = ENOMEM;
-			goto errout;
-		}
+	if (bus_dmamap_load_crp(sc->sc_dmat, q->q_src_map, crp, ubsec_op_cb,
+	    &q->q_src, BUS_DMA_NOWAIT) != 0) {
+		bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
+		q->q_src_map = NULL;
+		ubsecstats.hst_noload++;
+		err = ENOMEM;
+		goto errout;
 	}
+	q->q_src_mapsize = ubsec_crp_length(crp);
 	nicealign = ubsec_dmamap_aligned(&q->q_src);
 
 	dmap->d_dma->d_mcr.mcr_pktlen = htole16(stheend);
 
 #ifdef UBSEC_DEBUG
 	if (ubsec_debug)
 		printf("src skip: %d nicealign: %u\n", sskip, nicealign);
 #endif
 	for (i = j = 0; i < q->q_src_nsegs; i++) {
 		struct ubsec_pktbuf *pb;
 		bus_size_t packl = q->q_src_segs[i].ds_len;
 		bus_addr_t packp = q->q_src_segs[i].ds_addr;
 
 		if (sskip >= packl) {
 			sskip -= packl;
 			continue;
 		}
 
 		packl -= sskip;
 		packp += sskip;
 		sskip = 0;
 
 		if (packl > 0xfffc) {
 			err = EIO;
 			goto errout;
 		}
 
 		if (j == 0)
 			pb = &dmap->d_dma->d_mcr.mcr_ipktbuf;
 		else
 			pb = &dmap->d_dma->d_sbuf[j - 1];
 
 		pb->pb_addr = htole32(packp);
 
 		if (stheend) {
 			if (packl > stheend) {
 				pb->pb_len = htole32(stheend);
 				stheend = 0;
 			} else {
 				pb->pb_len = htole32(packl);
 				stheend -= packl;
 			}
 		} else
 			pb->pb_len = htole32(packl);
 
 		if ((i + 1) == q->q_src_nsegs)
 			pb->pb_next = 0;
 		else
 			pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
 			    offsetof(struct ubsec_dmachunk, d_sbuf[j]));
 		j++;
 	}
 
-	if (enccrd == NULL && maccrd != NULL) {
+	if (csp->csp_mode == CSP_MODE_DIGEST) {
 		dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr = 0;
 		dmap->d_dma->d_mcr.mcr_opktbuf.pb_len = 0;
 		dmap->d_dma->d_mcr.mcr_opktbuf.pb_next = htole32(dmap->d_alloc.dma_paddr +
 		    offsetof(struct ubsec_dmachunk, d_macbuf[0]));
 #ifdef UBSEC_DEBUG
 		if (ubsec_debug)
 			printf("opkt: %x %x %x\n",
 			    dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr,
 			    dmap->d_dma->d_mcr.mcr_opktbuf.pb_len,
 			    dmap->d_dma->d_mcr.mcr_opktbuf.pb_next);
 #endif
 	} else {
-		if (crp->crp_flags & CRYPTO_F_IOV) {
-			if (!nicealign) {
-				ubsecstats.hst_iovmisaligned++;
-				err = EINVAL;
-				goto errout;
+		if (nicealign) {
+			q->q_dst = q->q_src;
+		} else if (crp->crp_buf_type == CRYPTO_BUF_MBUF) {
+			int totlen, len;
+			struct mbuf *m, *top, **mp;
+
+			ubsecstats.hst_unaligned++;
+			totlen = q->q_src_mapsize;
+			if (totlen >= MINCLSIZE) {
+				m = m_getcl(M_NOWAIT, MT_DATA,
+				    crp->crp_mbuf->m_flags & M_PKTHDR);
+				len = MCLBYTES;
+			} else if (crp->crp_mbuf->m_flags & M_PKTHDR) {
+				m = m_gethdr(M_NOWAIT, MT_DATA);
+				len = MHLEN;
+			} else {
+				m = m_get(M_NOWAIT, MT_DATA);
+				len = MLEN;
 			}
-			if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT,
-			     &q->q_dst_map)) {
-				ubsecstats.hst_nomap++;
-				err = ENOMEM;
-				goto errout;
+			if (m && crp->crp_mbuf->m_flags & M_PKTHDR &&
+			    !m_dup_pkthdr(m, crp->crp_mbuf, M_NOWAIT)) {
+				m_free(m);
+				m = NULL;
 			}
-			if (bus_dmamap_load_uio(sc->sc_dmat, q->q_dst_map,
-			    q->q_dst_io, ubsec_op_cb, &q->q_dst, BUS_DMA_NOWAIT) != 0) {
-				bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
-				q->q_dst_map = NULL;
-				ubsecstats.hst_noload++;
-				err = ENOMEM;
+			if (m == NULL) {
+				ubsecstats.hst_nombuf++;
+				err = sc->sc_nqueue ? ERESTART : ENOMEM;
 				goto errout;
 			}
-		} else if (crp->crp_flags & CRYPTO_F_IMBUF) {
-			if (nicealign) {
-				q->q_dst = q->q_src;
-			} else {
-				int totlen, len;
-				struct mbuf *m, *top, **mp;
+			m->m_len = len = min(totlen, len);
+			totlen -= len;
+			top = m;
+			mp = &top;
 
-				ubsecstats.hst_unaligned++;
-				totlen = q->q_src_mapsize;
+			while (totlen > 0) {
 				if (totlen >= MINCLSIZE) {
-					m = m_getcl(M_NOWAIT, MT_DATA,
-					    q->q_src_m->m_flags & M_PKTHDR);
+					m = m_getcl(M_NOWAIT, MT_DATA, 0);
 					len = MCLBYTES;
-				} else if (q->q_src_m->m_flags & M_PKTHDR) {
-					m = m_gethdr(M_NOWAIT, MT_DATA);
-					len = MHLEN;
 				} else {
 					m = m_get(M_NOWAIT, MT_DATA);
 					len = MLEN;
 				}
-				if (m && q->q_src_m->m_flags & M_PKTHDR &&
-				    !m_dup_pkthdr(m, q->q_src_m, M_NOWAIT)) {
-					m_free(m);
-					m = NULL;
-				}
 				if (m == NULL) {
+					m_freem(top);
 					ubsecstats.hst_nombuf++;
 					err = sc->sc_nqueue ? ERESTART : ENOMEM;
 					goto errout;
 				}
 				m->m_len = len = min(totlen, len);
 				totlen -= len;
-				top = m;
-				mp = &top;
-
-				while (totlen > 0) {
-					if (totlen >= MINCLSIZE) {
-						m = m_getcl(M_NOWAIT,
-						    MT_DATA, 0);
-						len = MCLBYTES;
-					} else {
-						m = m_get(M_NOWAIT, MT_DATA);
-						len = MLEN;
-					}
-					if (m == NULL) {
-						m_freem(top);
-						ubsecstats.hst_nombuf++;
-						err = sc->sc_nqueue ? ERESTART : ENOMEM;
-						goto errout;
-					}
-					m->m_len = len = min(totlen, len);
-					totlen -= len;
-					*mp = m;
-					mp = &m->m_next;
-				}
-				q->q_dst_m = top;
-				ubsec_mcopy(q->q_src_m, q->q_dst_m,
-				    cpskip, cpoffset);
-				if (bus_dmamap_create(sc->sc_dmat,
-				    BUS_DMA_NOWAIT, &q->q_dst_map) != 0) {
-					ubsecstats.hst_nomap++;
-					err = ENOMEM;
-					goto errout;
-				}
-				if (bus_dmamap_load_mbuf(sc->sc_dmat,
-				    q->q_dst_map, q->q_dst_m,
-				    ubsec_op_cb, &q->q_dst,
-				    BUS_DMA_NOWAIT) != 0) {
-					bus_dmamap_destroy(sc->sc_dmat,
-					q->q_dst_map);
-					q->q_dst_map = NULL;
-					ubsecstats.hst_noload++;
-					err = ENOMEM;
-					goto errout;
-				}
+				*mp = m;
+				mp = &m->m_next;
 			}
+			q->q_dst_m = top;
+			ubsec_mcopy(crp->crp_mbuf, q->q_dst_m, cpskip, cpoffset);
+			if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT,
+			    &q->q_dst_map) != 0) {
+				ubsecstats.hst_nomap++;
+				err = ENOMEM;
+				goto errout;
+			}
+			if (bus_dmamap_load_mbuf_sg(sc->sc_dmat,
+			    q->q_dst_map, q->q_dst_m, q->q_dst_segs,
+			    &q->q_dst_nsegs, 0) != 0) {
+				bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
+				q->q_dst_map = NULL;
+				ubsecstats.hst_noload++;
+				err = ENOMEM;
+				goto errout;
+			}
+			q->q_dst_mapsize = q->q_src_mapsize;
 		} else {
-			ubsecstats.hst_badflags++;
+			ubsecstats.hst_iovmisaligned++;
 			err = EINVAL;
 			goto errout;
 		}
 
 #ifdef UBSEC_DEBUG
 		if (ubsec_debug)
 			printf("dst skip: %d\n", dskip);
 #endif
 		for (i = j = 0; i < q->q_dst_nsegs; i++) {
 			struct ubsec_pktbuf *pb;
 			bus_size_t packl = q->q_dst_segs[i].ds_len;
 			bus_addr_t packp = q->q_dst_segs[i].ds_addr;
 
 			if (dskip >= packl) {
 				dskip -= packl;
 				continue;
 			}
 
 			packl -= dskip;
 			packp += dskip;
 			dskip = 0;
 
 			if (packl > 0xfffc) {
 				err = EIO;
 				goto errout;
 			}
 
 			if (j == 0)
 				pb = &dmap->d_dma->d_mcr.mcr_opktbuf;
 			else
 				pb = &dmap->d_dma->d_dbuf[j - 1];
 
 			pb->pb_addr = htole32(packp);
 
 			if (dtheend) {
 				if (packl > dtheend) {
 					pb->pb_len = htole32(dtheend);
 					dtheend = 0;
 				} else {
 					pb->pb_len = htole32(packl);
 					dtheend -= packl;
 				}
 			} else
 				pb->pb_len = htole32(packl);
 
 			if ((i + 1) == q->q_dst_nsegs) {
-				if (maccrd)
+				if (csp->csp_auth_alg != 0)
 					pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
 					    offsetof(struct ubsec_dmachunk, d_macbuf[0]));
 				else
 					pb->pb_next = 0;
 			} else
 				pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
 				    offsetof(struct ubsec_dmachunk, d_dbuf[j]));
 			j++;
 		}
 	}
 
 	dmap->d_dma->d_mcr.mcr_cmdctxp = htole32(dmap->d_alloc.dma_paddr +
 	    offsetof(struct ubsec_dmachunk, d_ctx));
 
 	if (sc->sc_flags & UBS_FLAGS_LONGCTX) {
 		struct ubsec_pktctx_long *ctxl;
 
 		ctxl = (struct ubsec_pktctx_long *)(dmap->d_alloc.dma_vaddr +
 		    offsetof(struct ubsec_dmachunk, d_ctx));
 
 		/* transform small context into long context */
 		ctxl->pc_len = htole16(sizeof(struct ubsec_pktctx_long));
 		ctxl->pc_type = htole16(UBS_PKTCTX_TYPE_IPSEC);
 		ctxl->pc_flags = ctx.pc_flags;
 		ctxl->pc_offset = ctx.pc_offset;
 		for (i = 0; i < 6; i++)
 			ctxl->pc_deskey[i] = ctx.pc_deskey[i];
 		for (i = 0; i < 5; i++)
 			ctxl->pc_hminner[i] = ctx.pc_hminner[i];
 		for (i = 0; i < 5; i++)
 			ctxl->pc_hmouter[i] = ctx.pc_hmouter[i];
 		ctxl->pc_iv[0] = ctx.pc_iv[0];
 		ctxl->pc_iv[1] = ctx.pc_iv[1];
 	} else
 		bcopy(&ctx, dmap->d_alloc.dma_vaddr +
 		    offsetof(struct ubsec_dmachunk, d_ctx),
 		    sizeof(struct ubsec_pktctx));
 
 	mtx_lock(&sc->sc_mcr1lock);
 	SIMPLEQ_INSERT_TAIL(&sc->sc_queue, q, q_next);
 	sc->sc_nqueue++;
 	ubsecstats.hst_ipackets++;
 	ubsecstats.hst_ibytes += dmap->d_alloc.dma_size;
 	if ((hint & CRYPTO_HINT_MORE) == 0 || sc->sc_nqueue >= UBS_MAX_AGGR)
 		ubsec_feed(sc);
 	mtx_unlock(&sc->sc_mcr1lock);
 	return (0);
 
 errout:
 	if (q != NULL) {
-		if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m))
+		if (q->q_dst_m != NULL)
 			m_freem(q->q_dst_m);
 
 		if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) {
 			bus_dmamap_unload(sc->sc_dmat, q->q_dst_map);
 			bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
 		}
 		if (q->q_src_map != NULL) {
 			bus_dmamap_unload(sc->sc_dmat, q->q_src_map);
 			bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
 		}
 	}
 	if (q != NULL || err == ERESTART) {
 		mtx_lock(&sc->sc_freeqlock);
 		if (q != NULL)
 			SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
 		if (err == ERESTART)
 			sc->sc_needwakeup |= CRYPTO_SYMQ;
 		mtx_unlock(&sc->sc_freeqlock);
 	}
 	if (err != ERESTART) {
 		crp->crp_etype = err;
 		crypto_done(crp);
 	}
 	return (err);
 }
 
 static void
 ubsec_callback(struct ubsec_softc *sc, struct ubsec_q *q)
 {
+	const struct crypto_session_params *csp;
 	struct cryptop *crp = (struct cryptop *)q->q_crp;
 	struct ubsec_session *ses;
-	struct cryptodesc *crd;
 	struct ubsec_dma *dmap = q->q_dma;
+	char hash[SHA1_HASH_LEN];
 
 	ses = crypto_get_driver_session(crp->crp_session);
+	csp = crypto_get_params(crp->crp_session);
 
 	ubsecstats.hst_opackets++;
 	ubsecstats.hst_obytes += dmap->d_alloc.dma_size;
 
 	ubsec_dma_sync(&dmap->d_alloc,
 	    BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
 	if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) {
 		bus_dmamap_sync(sc->sc_dmat, q->q_dst_map,
 		    BUS_DMASYNC_POSTREAD);
 		bus_dmamap_unload(sc->sc_dmat, q->q_dst_map);
 		bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
 	}
 	bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_POSTWRITE);
 	bus_dmamap_unload(sc->sc_dmat, q->q_src_map);
 	bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
 
-	if ((crp->crp_flags & CRYPTO_F_IMBUF) && (q->q_src_m != q->q_dst_m)) {
-		m_freem(q->q_src_m);
-		crp->crp_buf = (caddr_t)q->q_dst_m;
+	if (q->q_dst_m != NULL) {
+		m_freem(crp->crp_mbuf);
+		crp->crp_mbuf = q->q_dst_m;
 	}
 
-	/* copy out IV for future use */
-	if (q->q_flags & UBSEC_QFLAGS_COPYOUTIV) {
-		for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
-			if (crd->crd_alg != CRYPTO_DES_CBC &&
-			    crd->crd_alg != CRYPTO_3DES_CBC)
-				continue;
-			crypto_copydata(crp->crp_flags, crp->crp_buf,
-			    crd->crd_skip + crd->crd_len - 8, 8,
-			    (caddr_t)ses->ses_iv);
-			break;
-		}
+	if (csp->csp_auth_alg != 0) {
+		if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
+			crypto_copydata(crp, crp->crp_digest_start,
+			    ses->ses_mlen, hash);
+			if (timingsafe_bcmp(dmap->d_dma->d_macbuf, hash,
+			    ses->ses_mlen) != 0)
+				crp->crp_etype = EBADMSG;
+		} else
+			crypto_copyback(crp, crp->crp_digest_start,
+			    ses->ses_mlen, dmap->d_dma->d_macbuf);
 	}
-
-	for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
-		if (crd->crd_alg != CRYPTO_MD5_HMAC &&
-		    crd->crd_alg != CRYPTO_SHA1_HMAC)
-			continue;
-		crypto_copyback(crp->crp_flags, crp->crp_buf, crd->crd_inject,
-		    ses->ses_mlen, (caddr_t)dmap->d_dma->d_macbuf);
-		break;
-	}
 	mtx_lock(&sc->sc_freeqlock);
 	SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
 	mtx_unlock(&sc->sc_freeqlock);
 	crypto_done(crp);
 }
 
 static void
 ubsec_mcopy(struct mbuf *srcm, struct mbuf *dstm, int hoffset, int toffset)
 {
 	int i, j, dlen, slen;
 	caddr_t dptr, sptr;
 
 	j = 0;
 	sptr = srcm->m_data;
 	slen = srcm->m_len;
 	dptr = dstm->m_data;
 	dlen = dstm->m_len;
 
 	while (1) {
 		for (i = 0; i < min(slen, dlen); i++) {
 			if (j < hoffset || j >= toffset)
 				*dptr++ = *sptr++;
 			slen--;
 			dlen--;
 			j++;
 		}
 		if (slen == 0) {
 			srcm = srcm->m_next;
 			if (srcm == NULL)
 				return;
 			sptr = srcm->m_data;
 			slen = srcm->m_len;
 		}
 		if (dlen == 0) {
 			dstm = dstm->m_next;
 			if (dstm == NULL)
 				return;
 			dptr = dstm->m_data;
 			dlen = dstm->m_len;
 		}
 	}
 }
 
 /*
  * feed the key generator, must be called at splimp() or higher.
  */
 static int
 ubsec_feed2(struct ubsec_softc *sc)
 {
 	struct ubsec_q2 *q;
 
 	while (!SIMPLEQ_EMPTY(&sc->sc_queue2)) {
 		if (READ_REG(sc, BS_STAT) & BS_STAT_MCR2_FULL)
 			break;
 		q = SIMPLEQ_FIRST(&sc->sc_queue2);
 
 		ubsec_dma_sync(&q->q_mcr,
 		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 		ubsec_dma_sync(&q->q_ctx, BUS_DMASYNC_PREWRITE);
 
 		WRITE_REG(sc, BS_MCR2, q->q_mcr.dma_paddr);
 		SIMPLEQ_REMOVE_HEAD(&sc->sc_queue2, q_next);
 		--sc->sc_nqueue2;
 		SIMPLEQ_INSERT_TAIL(&sc->sc_qchip2, q, q_next);
 	}
 	return (0);
 }
 
 /*
  * Callback for handling random numbers
  */
 static void
 ubsec_callback2(struct ubsec_softc *sc, struct ubsec_q2 *q)
 {
 	struct cryptkop *krp;
 	struct ubsec_ctx_keyop *ctx;
 
 	ctx = (struct ubsec_ctx_keyop *)q->q_ctx.dma_vaddr;
 	ubsec_dma_sync(&q->q_ctx, BUS_DMASYNC_POSTWRITE);
 
 	switch (q->q_type) {
 #ifndef UBSEC_NO_RNG
 	case UBS_CTXOP_RNGBYPASS: {
 		struct ubsec_q2_rng *rng = (struct ubsec_q2_rng *)q;
 
 		ubsec_dma_sync(&rng->rng_buf, BUS_DMASYNC_POSTREAD);
 		(*sc->sc_harvest)(sc->sc_rndtest,
 			rng->rng_buf.dma_vaddr,
 			UBSEC_RNG_BUFSIZ*sizeof (u_int32_t));
 		rng->rng_used = 0;
 		callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc);
 		break;
 	}
 #endif
 	case UBS_CTXOP_MODEXP: {
 		struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q;
 		u_int rlen, clen;
 
 		krp = me->me_krp;
 		rlen = (me->me_modbits + 7) / 8;
 		clen = (krp->krp_param[krp->krp_iparams].crp_nbits + 7) / 8;
 
 		ubsec_dma_sync(&me->me_M, BUS_DMASYNC_POSTWRITE);
 		ubsec_dma_sync(&me->me_E, BUS_DMASYNC_POSTWRITE);
 		ubsec_dma_sync(&me->me_C, BUS_DMASYNC_POSTREAD);
 		ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_POSTWRITE);
 
 		if (clen < rlen)
 			krp->krp_status = E2BIG;
 		else {
 			if (sc->sc_flags & UBS_FLAGS_HWNORM) {
 				bzero(krp->krp_param[krp->krp_iparams].crp_p,
 				    (krp->krp_param[krp->krp_iparams].crp_nbits
 					+ 7) / 8);
 				bcopy(me->me_C.dma_vaddr,
 				    krp->krp_param[krp->krp_iparams].crp_p,
 				    (me->me_modbits + 7) / 8);
 			} else
 				ubsec_kshift_l(me->me_shiftbits,
 				    me->me_C.dma_vaddr, me->me_normbits,
 				    krp->krp_param[krp->krp_iparams].crp_p,
 				    krp->krp_param[krp->krp_iparams].crp_nbits);
 		}
 
 		crypto_kdone(krp);
 
 		/* bzero all potentially sensitive data */
 		bzero(me->me_E.dma_vaddr, me->me_E.dma_size);
 		bzero(me->me_M.dma_vaddr, me->me_M.dma_size);
 		bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
 		bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size);
 
 		/* Can't free here, so put us on the free list. */
 		SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &me->me_q, q_next);
 		break;
 	}
 	case UBS_CTXOP_RSAPRIV: {
 		struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q;
 		u_int len;
 
 		krp = rp->rpr_krp;
 		ubsec_dma_sync(&rp->rpr_msgin, BUS_DMASYNC_POSTWRITE);
 		ubsec_dma_sync(&rp->rpr_msgout, BUS_DMASYNC_POSTREAD);
 
 		len = (krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_nbits + 7) / 8;
 		bcopy(rp->rpr_msgout.dma_vaddr,
 		    krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_p, len);
 
 		crypto_kdone(krp);
 
 		bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size);
 		bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size);
 		bzero(rp->rpr_q.q_ctx.dma_vaddr, rp->rpr_q.q_ctx.dma_size);
 
 		/* Can't free here, so put us on the free list. */
 		SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &rp->rpr_q, q_next);
 		break;
 	}
 	default:
 		device_printf(sc->sc_dev, "unknown ctx op: %x\n",
 		    letoh16(ctx->ctx_op));
 		break;
 	}
 }
 
 #ifndef UBSEC_NO_RNG
 static void
 ubsec_rng(void *vsc)
 {
 	struct ubsec_softc *sc = vsc;
 	struct ubsec_q2_rng *rng = &sc->sc_rng;
 	struct ubsec_mcr *mcr;
 	struct ubsec_ctx_rngbypass *ctx;
 
 	mtx_lock(&sc->sc_mcr2lock);
 	if (rng->rng_used) {
 		mtx_unlock(&sc->sc_mcr2lock);
 		return;
 	}
 	sc->sc_nqueue2++;
 	if (sc->sc_nqueue2 >= UBS_MAX_NQUEUE)
 		goto out;
 
 	mcr = (struct ubsec_mcr *)rng->rng_q.q_mcr.dma_vaddr;
 	ctx = (struct ubsec_ctx_rngbypass *)rng->rng_q.q_ctx.dma_vaddr;
 
 	mcr->mcr_pkts = htole16(1);
 	mcr->mcr_flags = 0;
 	mcr->mcr_cmdctxp = htole32(rng->rng_q.q_ctx.dma_paddr);
 	mcr->mcr_ipktbuf.pb_addr = mcr->mcr_ipktbuf.pb_next = 0;
 	mcr->mcr_ipktbuf.pb_len = 0;
 	mcr->mcr_reserved = mcr->mcr_pktlen = 0;
 	mcr->mcr_opktbuf.pb_addr = htole32(rng->rng_buf.dma_paddr);
 	mcr->mcr_opktbuf.pb_len = htole32(((sizeof(u_int32_t) * UBSEC_RNG_BUFSIZ)) &
 	    UBS_PKTBUF_LEN);
 	mcr->mcr_opktbuf.pb_next = 0;
 
 	ctx->rbp_len = htole16(sizeof(struct ubsec_ctx_rngbypass));
 	ctx->rbp_op = htole16(UBS_CTXOP_RNGBYPASS);
 	rng->rng_q.q_type = UBS_CTXOP_RNGBYPASS;
 
 	ubsec_dma_sync(&rng->rng_buf, BUS_DMASYNC_PREREAD);
 
 	SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rng->rng_q, q_next);
 	rng->rng_used = 1;
 	ubsec_feed2(sc);
 	ubsecstats.hst_rng++;
 	mtx_unlock(&sc->sc_mcr2lock);
 
 	return;
 
 out:
 	/*
 	 * Something weird happened, generate our own call back.
 	 */
 	sc->sc_nqueue2--;
 	mtx_unlock(&sc->sc_mcr2lock);
 	callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc);
 }
 #endif /* UBSEC_NO_RNG */
 
 static void
 ubsec_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 {
 	bus_addr_t *paddr = (bus_addr_t*) arg;
 	*paddr = segs->ds_addr;
 }
 
 static int
 ubsec_dma_malloc(
 	struct ubsec_softc *sc,
 	bus_size_t size,
 	struct ubsec_dma_alloc *dma,
 	int mapflags
 )
 {
 	int r;
 
 	/* XXX could specify sc_dmat as parent but that just adds overhead */
 	r = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev),	/* parent */
 			       1, 0,			/* alignment, bounds */
 			       BUS_SPACE_MAXADDR_32BIT,	/* lowaddr */
 			       BUS_SPACE_MAXADDR,	/* highaddr */
 			       NULL, NULL,		/* filter, filterarg */
 			       size,			/* maxsize */
 			       1,			/* nsegments */
 			       size,			/* maxsegsize */
 			       BUS_DMA_ALLOCNOW,	/* flags */
 			       NULL, NULL,		/* lockfunc, lockarg */
 			       &dma->dma_tag);
 	if (r != 0) {
 		device_printf(sc->sc_dev, "ubsec_dma_malloc: "
 			"bus_dma_tag_create failed; error %u\n", r);
 		goto fail_1;
 	}
 
 	r = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
 			     BUS_DMA_NOWAIT, &dma->dma_map);
 	if (r != 0) {
 		device_printf(sc->sc_dev, "ubsec_dma_malloc: "
 			"bus_dmammem_alloc failed; size %ju, error %u\n",
 			(intmax_t)size, r);
 		goto fail_2;
 	}
 
 	r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
 		            size,
 			    ubsec_dmamap_cb,
 			    &dma->dma_paddr,
 			    mapflags | BUS_DMA_NOWAIT);
 	if (r != 0) {
 		device_printf(sc->sc_dev, "ubsec_dma_malloc: "
 			"bus_dmamap_load failed; error %u\n", r);
 		goto fail_3;
 	}
 
 	dma->dma_size = size;
 	return (0);
 
 fail_3:
 	bus_dmamap_unload(dma->dma_tag, dma->dma_map);
 fail_2:
 	bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
 fail_1:
 	bus_dma_tag_destroy(dma->dma_tag);
 	dma->dma_tag = NULL;
 	return (r);
 }
 
 static void
 ubsec_dma_free(struct ubsec_softc *sc, struct ubsec_dma_alloc *dma)
 {
 	bus_dmamap_unload(dma->dma_tag, dma->dma_map);
 	bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
 	bus_dma_tag_destroy(dma->dma_tag);
 }
 
 /*
  * Resets the board.  Values in the regesters are left as is
  * from the reset (i.e. initial values are assigned elsewhere).
  */
 static void
 ubsec_reset_board(struct ubsec_softc *sc)
 {
     volatile u_int32_t ctrl;
 
     ctrl = READ_REG(sc, BS_CTRL);
     ctrl |= BS_CTRL_RESET;
     WRITE_REG(sc, BS_CTRL, ctrl);
 
     /*
      * Wait aprox. 30 PCI clocks = 900 ns = 0.9 us
      */
     DELAY(10);
 }
 
 /*
  * Init Broadcom registers
  */
 static void
 ubsec_init_board(struct ubsec_softc *sc)
 {
 	u_int32_t ctrl;
 
 	ctrl = READ_REG(sc, BS_CTRL);
 	ctrl &= ~(BS_CTRL_BE32 | BS_CTRL_BE64);
 	ctrl |= BS_CTRL_LITTLE_ENDIAN | BS_CTRL_MCR1INT;
 
 	if (sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG))
 		ctrl |= BS_CTRL_MCR2INT;
 	else
 		ctrl &= ~BS_CTRL_MCR2INT;
 
 	if (sc->sc_flags & UBS_FLAGS_HWNORM)
 		ctrl &= ~BS_CTRL_SWNORM;
 
 	WRITE_REG(sc, BS_CTRL, ctrl);
 }
 
 /*
  * Init Broadcom PCI registers
  */
 static void
 ubsec_init_pciregs(device_t dev)
 {
 #if 0
 	u_int32_t misc;
 
 	misc = pci_conf_read(pc, pa->pa_tag, BS_RTY_TOUT);
 	misc = (misc & ~(UBS_PCI_RTY_MASK << UBS_PCI_RTY_SHIFT))
 	    | ((UBS_DEF_RTY & 0xff) << UBS_PCI_RTY_SHIFT);
 	misc = (misc & ~(UBS_PCI_TOUT_MASK << UBS_PCI_TOUT_SHIFT))
 	    | ((UBS_DEF_TOUT & 0xff) << UBS_PCI_TOUT_SHIFT);
 	pci_conf_write(pc, pa->pa_tag, BS_RTY_TOUT, misc);
 #endif
 
 	/*
 	 * This will set the cache line size to 1, this will
 	 * force the BCM58xx chip just to do burst read/writes.
 	 * Cache line read/writes are to slow
 	 */
 	pci_write_config(dev, PCIR_CACHELNSZ, UBS_DEF_CACHELINE, 1);
 }
 
 /*
  * Clean up after a chip crash.
  * It is assumed that the caller in splimp()
  */
 static void
 ubsec_cleanchip(struct ubsec_softc *sc)
 {
 	struct ubsec_q *q;
 
 	while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) {
 		q = SIMPLEQ_FIRST(&sc->sc_qchip);
 		SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q_next);
 		ubsec_free_q(sc, q);
 	}
 	sc->sc_nqchip = 0;
 }
 
 /*
  * free a ubsec_q
  * It is assumed that the caller is within splimp().
  */
 static int
 ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q)
 {
 	struct ubsec_q *q2;
 	struct cryptop *crp;
 	int npkts;
 	int i;
 
 	npkts = q->q_nstacked_mcrs;
 
 	for (i = 0; i < npkts; i++) {
 		if(q->q_stacked_mcr[i]) {
 			q2 = q->q_stacked_mcr[i];
 
-			if ((q2->q_dst_m != NULL) && (q2->q_src_m != q2->q_dst_m))
+			if (q2->q_dst_m != NULL)
 				m_freem(q2->q_dst_m);
 
 			crp = (struct cryptop *)q2->q_crp;
 
 			SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q2, q_next);
 
 			crp->crp_etype = EFAULT;
 			crypto_done(crp);
 		} else {
 			break;
 		}
 	}
 
 	/*
 	 * Free header MCR
 	 */
-	if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m))
+	if (q->q_dst_m != NULL)
 		m_freem(q->q_dst_m);
 
 	crp = (struct cryptop *)q->q_crp;
 
 	SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
 
 	crp->crp_etype = EFAULT;
 	crypto_done(crp);
 	return(0);
 }
 
 /*
  * Routine to reset the chip and clean up.
  * It is assumed that the caller is in splimp()
  */
 static void
 ubsec_totalreset(struct ubsec_softc *sc)
 {
 	ubsec_reset_board(sc);
 	ubsec_init_board(sc);
 	ubsec_cleanchip(sc);
 }
 
 static int
 ubsec_dmamap_aligned(struct ubsec_operand *op)
 {
 	int i;
 
 	for (i = 0; i < op->nsegs; i++) {
 		if (op->segs[i].ds_addr & 3)
 			return (0);
 		if ((i != (op->nsegs - 1)) &&
 		    (op->segs[i].ds_len & 3))
 			return (0);
 	}
 	return (1);
 }
 
 static void
 ubsec_kfree(struct ubsec_softc *sc, struct ubsec_q2 *q)
 {
 	switch (q->q_type) {
 	case UBS_CTXOP_MODEXP: {
 		struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q;
 
 		ubsec_dma_free(sc, &me->me_q.q_mcr);
 		ubsec_dma_free(sc, &me->me_q.q_ctx);
 		ubsec_dma_free(sc, &me->me_M);
 		ubsec_dma_free(sc, &me->me_E);
 		ubsec_dma_free(sc, &me->me_C);
 		ubsec_dma_free(sc, &me->me_epb);
 		free(me, M_DEVBUF);
 		break;
 	}
 	case UBS_CTXOP_RSAPRIV: {
 		struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q;
 
 		ubsec_dma_free(sc, &rp->rpr_q.q_mcr);
 		ubsec_dma_free(sc, &rp->rpr_q.q_ctx);
 		ubsec_dma_free(sc, &rp->rpr_msgin);
 		ubsec_dma_free(sc, &rp->rpr_msgout);
 		free(rp, M_DEVBUF);
 		break;
 	}
 	default:
 		device_printf(sc->sc_dev, "invalid kfree 0x%x\n", q->q_type);
 		break;
 	}
 }
 
 static int
 ubsec_kprocess(device_t dev, struct cryptkop *krp, int hint)
 {
 	struct ubsec_softc *sc = device_get_softc(dev);
 	int r;
 
 	if (krp == NULL || krp->krp_callback == NULL)
 		return (EINVAL);
 
 	while (!SIMPLEQ_EMPTY(&sc->sc_q2free)) {
 		struct ubsec_q2 *q;
 
 		q = SIMPLEQ_FIRST(&sc->sc_q2free);
 		SIMPLEQ_REMOVE_HEAD(&sc->sc_q2free, q_next);
 		ubsec_kfree(sc, q);
 	}
 
 	switch (krp->krp_op) {
 	case CRK_MOD_EXP:
 		if (sc->sc_flags & UBS_FLAGS_HWNORM)
 			r = ubsec_kprocess_modexp_hw(sc, krp, hint);
 		else
 			r = ubsec_kprocess_modexp_sw(sc, krp, hint);
 		break;
 	case CRK_MOD_EXP_CRT:
 		return (ubsec_kprocess_rsapriv(sc, krp, hint));
 	default:
 		device_printf(sc->sc_dev, "kprocess: invalid op 0x%x\n",
 		    krp->krp_op);
 		krp->krp_status = EOPNOTSUPP;
 		crypto_kdone(krp);
 		return (0);
 	}
 	return (0);			/* silence compiler */
 }
 
 /*
  * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (sw normalization)
  */
 static int
 ubsec_kprocess_modexp_sw(struct ubsec_softc *sc, struct cryptkop *krp, int hint)
 {
 	struct ubsec_q2_modexp *me;
 	struct ubsec_mcr *mcr;
 	struct ubsec_ctx_modexp *ctx;
 	struct ubsec_pktbuf *epb;
 	int err = 0;
 	u_int nbits, normbits, mbits, shiftbits, ebits;
 
 	me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT);
 	if (me == NULL) {
 		err = ENOMEM;
 		goto errout;
 	}
 	bzero(me, sizeof *me);
 	me->me_krp = krp;
 	me->me_q.q_type = UBS_CTXOP_MODEXP;
 
 	nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]);
 	if (nbits <= 512)
 		normbits = 512;
 	else if (nbits <= 768)
 		normbits = 768;
 	else if (nbits <= 1024)
 		normbits = 1024;
 	else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536)
 		normbits = 1536;
 	else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048)
 		normbits = 2048;
 	else {
 		err = E2BIG;
 		goto errout;
 	}
 
 	shiftbits = normbits - nbits;
 
 	me->me_modbits = nbits;
 	me->me_shiftbits = shiftbits;
 	me->me_normbits = normbits;
 
 	/* Sanity check: result bits must be >= true modulus bits. */
 	if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) {
 		err = ERANGE;
 		goto errout;
 	}
 
 	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
 	    &me->me_q.q_mcr, 0)) {
 		err = ENOMEM;
 		goto errout;
 	}
 	mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr;
 
 	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp),
 	    &me->me_q.q_ctx, 0)) {
 		err = ENOMEM;
 		goto errout;
 	}
 
 	mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]);
 	if (mbits > nbits) {
 		err = E2BIG;
 		goto errout;
 	}
 	if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) {
 		err = ENOMEM;
 		goto errout;
 	}
 	ubsec_kshift_r(shiftbits,
 	    krp->krp_param[UBS_MODEXP_PAR_M].crp_p, mbits,
 	    me->me_M.dma_vaddr, normbits);
 
 	if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) {
 		err = ENOMEM;
 		goto errout;
 	}
 	bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
 
 	ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]);
 	if (ebits > nbits) {
 		err = E2BIG;
 		goto errout;
 	}
 	if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) {
 		err = ENOMEM;
 		goto errout;
 	}
 	ubsec_kshift_r(shiftbits,
 	    krp->krp_param[UBS_MODEXP_PAR_E].crp_p, ebits,
 	    me->me_E.dma_vaddr, normbits);
 
 	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf),
 	    &me->me_epb, 0)) {
 		err = ENOMEM;
 		goto errout;
 	}
 	epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr;
 	epb->pb_addr = htole32(me->me_E.dma_paddr);
 	epb->pb_next = 0;
 	epb->pb_len = htole32(normbits / 8);
 
 #ifdef UBSEC_DEBUG
 	if (ubsec_debug) {
 		printf("Epb ");
 		ubsec_dump_pb(epb);
 	}
 #endif
 
 	mcr->mcr_pkts = htole16(1);
 	mcr->mcr_flags = 0;
 	mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr);
 	mcr->mcr_reserved = 0;
 	mcr->mcr_pktlen = 0;
 
 	mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr);
 	mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8);
 	mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr);
 
 	mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr);
 	mcr->mcr_opktbuf.pb_next = 0;
 	mcr->mcr_opktbuf.pb_len = htole32(normbits / 8);
 
 #ifdef DIAGNOSTIC
 	/* Misaligned output buffer will hang the chip. */
 	if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0)
 		panic("%s: modexp invalid addr 0x%x\n",
 		    device_get_nameunit(sc->sc_dev),
 		    letoh32(mcr->mcr_opktbuf.pb_addr));
 	if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0)
 		panic("%s: modexp invalid len 0x%x\n",
 		    device_get_nameunit(sc->sc_dev),
 		    letoh32(mcr->mcr_opktbuf.pb_len));
 #endif
 
 	ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr;
 	bzero(ctx, sizeof(*ctx));
 	ubsec_kshift_r(shiftbits,
 	    krp->krp_param[UBS_MODEXP_PAR_N].crp_p, nbits,
 	    ctx->me_N, normbits);
 	ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t)));
 	ctx->me_op = htole16(UBS_CTXOP_MODEXP);
 	ctx->me_E_len = htole16(nbits);
 	ctx->me_N_len = htole16(nbits);
 
 #ifdef UBSEC_DEBUG
 	if (ubsec_debug) {
 		ubsec_dump_mcr(mcr);
 		ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx);
 	}
 #endif
 
 	/*
 	 * ubsec_feed2 will sync mcr and ctx, we just need to sync
 	 * everything else.
 	 */
 	ubsec_dma_sync(&me->me_M, BUS_DMASYNC_PREWRITE);
 	ubsec_dma_sync(&me->me_E, BUS_DMASYNC_PREWRITE);
 	ubsec_dma_sync(&me->me_C, BUS_DMASYNC_PREREAD);
 	ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_PREWRITE);
 
 	/* Enqueue and we're done... */
 	mtx_lock(&sc->sc_mcr2lock);
 	SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next);
 	ubsec_feed2(sc);
 	ubsecstats.hst_modexp++;
 	mtx_unlock(&sc->sc_mcr2lock);
 
 	return (0);
 
 errout:
 	if (me != NULL) {
 		if (me->me_q.q_mcr.dma_tag != NULL)
 			ubsec_dma_free(sc, &me->me_q.q_mcr);
 		if (me->me_q.q_ctx.dma_tag != NULL) {
 			bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size);
 			ubsec_dma_free(sc, &me->me_q.q_ctx);
 		}
 		if (me->me_M.dma_tag != NULL) {
 			bzero(me->me_M.dma_vaddr, me->me_M.dma_size);
 			ubsec_dma_free(sc, &me->me_M);
 		}
 		if (me->me_E.dma_tag != NULL) {
 			bzero(me->me_E.dma_vaddr, me->me_E.dma_size);
 			ubsec_dma_free(sc, &me->me_E);
 		}
 		if (me->me_C.dma_tag != NULL) {
 			bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
 			ubsec_dma_free(sc, &me->me_C);
 		}
 		if (me->me_epb.dma_tag != NULL)
 			ubsec_dma_free(sc, &me->me_epb);
 		free(me, M_DEVBUF);
 	}
 	krp->krp_status = err;
 	crypto_kdone(krp);
 	return (0);
 }
 
 /*
  * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (hw normalization)
  */
 static int
 ubsec_kprocess_modexp_hw(struct ubsec_softc *sc, struct cryptkop *krp, int hint)
 {
 	struct ubsec_q2_modexp *me;
 	struct ubsec_mcr *mcr;
 	struct ubsec_ctx_modexp *ctx;
 	struct ubsec_pktbuf *epb;
 	int err = 0;
 	u_int nbits, normbits, mbits, shiftbits, ebits;
 
 	me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT);
 	if (me == NULL) {
 		err = ENOMEM;
 		goto errout;
 	}
 	bzero(me, sizeof *me);
 	me->me_krp = krp;
 	me->me_q.q_type = UBS_CTXOP_MODEXP;
 
 	nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]);
 	if (nbits <= 512)
 		normbits = 512;
 	else if (nbits <= 768)
 		normbits = 768;
 	else if (nbits <= 1024)
 		normbits = 1024;
 	else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536)
 		normbits = 1536;
 	else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048)
 		normbits = 2048;
 	else {
 		err = E2BIG;
 		goto errout;
 	}
 
 	shiftbits = normbits - nbits;
 
 	/* XXX ??? */
 	me->me_modbits = nbits;
 	me->me_shiftbits = shiftbits;
 	me->me_normbits = normbits;
 
 	/* Sanity check: result bits must be >= true modulus bits. */
 	if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) {
 		err = ERANGE;
 		goto errout;
 	}
 
 	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
 	    &me->me_q.q_mcr, 0)) {
 		err = ENOMEM;
 		goto errout;
 	}
 	mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr;
 
 	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp),
 	    &me->me_q.q_ctx, 0)) {
 		err = ENOMEM;
 		goto errout;
 	}
 
 	mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]);
 	if (mbits > nbits) {
 		err = E2BIG;
 		goto errout;
 	}
 	if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) {
 		err = ENOMEM;
 		goto errout;
 	}
 	bzero(me->me_M.dma_vaddr, normbits / 8);
 	bcopy(krp->krp_param[UBS_MODEXP_PAR_M].crp_p,
 	    me->me_M.dma_vaddr, (mbits + 7) / 8);
 
 	if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) {
 		err = ENOMEM;
 		goto errout;
 	}
 	bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
 
 	ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]);
 	if (ebits > nbits) {
 		err = E2BIG;
 		goto errout;
 	}
 	if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) {
 		err = ENOMEM;
 		goto errout;
 	}
 	bzero(me->me_E.dma_vaddr, normbits / 8);
 	bcopy(krp->krp_param[UBS_MODEXP_PAR_E].crp_p,
 	    me->me_E.dma_vaddr, (ebits + 7) / 8);
 
 	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf),
 	    &me->me_epb, 0)) {
 		err = ENOMEM;
 		goto errout;
 	}
 	epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr;
 	epb->pb_addr = htole32(me->me_E.dma_paddr);
 	epb->pb_next = 0;
 	epb->pb_len = htole32((ebits + 7) / 8);
 
 #ifdef UBSEC_DEBUG
 	if (ubsec_debug) {
 		printf("Epb ");
 		ubsec_dump_pb(epb);
 	}
 #endif
 
 	mcr->mcr_pkts = htole16(1);
 	mcr->mcr_flags = 0;
 	mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr);
 	mcr->mcr_reserved = 0;
 	mcr->mcr_pktlen = 0;
 
 	mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr);
 	mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8);
 	mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr);
 
 	mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr);
 	mcr->mcr_opktbuf.pb_next = 0;
 	mcr->mcr_opktbuf.pb_len = htole32(normbits / 8);
 
 #ifdef DIAGNOSTIC
 	/* Misaligned output buffer will hang the chip. */
 	if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0)
 		panic("%s: modexp invalid addr 0x%x\n",
 		    device_get_nameunit(sc->sc_dev),
 		    letoh32(mcr->mcr_opktbuf.pb_addr));
 	if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0)
 		panic("%s: modexp invalid len 0x%x\n",
 		    device_get_nameunit(sc->sc_dev),
 		    letoh32(mcr->mcr_opktbuf.pb_len));
 #endif
 
 	ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr;
 	bzero(ctx, sizeof(*ctx));
 	bcopy(krp->krp_param[UBS_MODEXP_PAR_N].crp_p, ctx->me_N,
 	    (nbits + 7) / 8);
 	ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t)));
 	ctx->me_op = htole16(UBS_CTXOP_MODEXP);
 	ctx->me_E_len = htole16(ebits);
 	ctx->me_N_len = htole16(nbits);
 
 #ifdef UBSEC_DEBUG
 	if (ubsec_debug) {
 		ubsec_dump_mcr(mcr);
 		ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx);
 	}
 #endif
 
 	/*
 	 * ubsec_feed2 will sync mcr and ctx, we just need to sync
 	 * everything else.
 	 */
 	ubsec_dma_sync(&me->me_M, BUS_DMASYNC_PREWRITE);
 	ubsec_dma_sync(&me->me_E, BUS_DMASYNC_PREWRITE);
 	ubsec_dma_sync(&me->me_C, BUS_DMASYNC_PREREAD);
 	ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_PREWRITE);
 
 	/* Enqueue and we're done... */
 	mtx_lock(&sc->sc_mcr2lock);
 	SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next);
 	ubsec_feed2(sc);
 	mtx_unlock(&sc->sc_mcr2lock);
 
 	return (0);
 
 errout:
 	if (me != NULL) {
 		if (me->me_q.q_mcr.dma_tag != NULL)
 			ubsec_dma_free(sc, &me->me_q.q_mcr);
 		if (me->me_q.q_ctx.dma_tag != NULL) {
 			bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size);
 			ubsec_dma_free(sc, &me->me_q.q_ctx);
 		}
 		if (me->me_M.dma_tag != NULL) {
 			bzero(me->me_M.dma_vaddr, me->me_M.dma_size);
 			ubsec_dma_free(sc, &me->me_M);
 		}
 		if (me->me_E.dma_tag != NULL) {
 			bzero(me->me_E.dma_vaddr, me->me_E.dma_size);
 			ubsec_dma_free(sc, &me->me_E);
 		}
 		if (me->me_C.dma_tag != NULL) {
 			bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
 			ubsec_dma_free(sc, &me->me_C);
 		}
 		if (me->me_epb.dma_tag != NULL)
 			ubsec_dma_free(sc, &me->me_epb);
 		free(me, M_DEVBUF);
 	}
 	krp->krp_status = err;
 	crypto_kdone(krp);
 	return (0);
 }
 
 static int
 ubsec_kprocess_rsapriv(struct ubsec_softc *sc, struct cryptkop *krp, int hint)
 {
 	struct ubsec_q2_rsapriv *rp = NULL;
 	struct ubsec_mcr *mcr;
 	struct ubsec_ctx_rsapriv *ctx;
 	int err = 0;
 	u_int padlen, msglen;
 
 	msglen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_P]);
 	padlen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_Q]);
 	if (msglen > padlen)
 		padlen = msglen;
 
 	if (padlen <= 256)
 		padlen = 256;
 	else if (padlen <= 384)
 		padlen = 384;
 	else if (padlen <= 512)
 		padlen = 512;
 	else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 768)
 		padlen = 768;
 	else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 1024)
 		padlen = 1024;
 	else {
 		err = E2BIG;
 		goto errout;
 	}
 
 	if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DP]) > padlen) {
 		err = E2BIG;
 		goto errout;
 	}
 
 	if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DQ]) > padlen) {
 		err = E2BIG;
 		goto errout;
 	}
 
 	if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_PINV]) > padlen) {
 		err = E2BIG;
 		goto errout;
 	}
 
 	rp = (struct ubsec_q2_rsapriv *)malloc(sizeof *rp, M_DEVBUF, M_NOWAIT);
 	if (rp == NULL)
 		return (ENOMEM);
 	bzero(rp, sizeof *rp);
 	rp->rpr_krp = krp;
 	rp->rpr_q.q_type = UBS_CTXOP_RSAPRIV;
 
 	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
 	    &rp->rpr_q.q_mcr, 0)) {
 		err = ENOMEM;
 		goto errout;
 	}
 	mcr = (struct ubsec_mcr *)rp->rpr_q.q_mcr.dma_vaddr;
 
 	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rsapriv),
 	    &rp->rpr_q.q_ctx, 0)) {
 		err = ENOMEM;
 		goto errout;
 	}
 	ctx = (struct ubsec_ctx_rsapriv *)rp->rpr_q.q_ctx.dma_vaddr;
 	bzero(ctx, sizeof *ctx);
 
 	/* Copy in p */
 	bcopy(krp->krp_param[UBS_RSAPRIV_PAR_P].crp_p,
 	    &ctx->rpr_buf[0 * (padlen / 8)],
 	    (krp->krp_param[UBS_RSAPRIV_PAR_P].crp_nbits + 7) / 8);
 
 	/* Copy in q */
 	bcopy(krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_p,
 	    &ctx->rpr_buf[1 * (padlen / 8)],
 	    (krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_nbits + 7) / 8);
 
 	/* Copy in dp */
 	bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_p,
 	    &ctx->rpr_buf[2 * (padlen / 8)],
 	    (krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_nbits + 7) / 8);
 
 	/* Copy in dq */
 	bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_p,
 	    &ctx->rpr_buf[3 * (padlen / 8)],
 	    (krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_nbits + 7) / 8);
 
 	/* Copy in pinv */
 	bcopy(krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_p,
 	    &ctx->rpr_buf[4 * (padlen / 8)],
 	    (krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_nbits + 7) / 8);
 
 	msglen = padlen * 2;
 
 	/* Copy in input message (aligned buffer/length). */
 	if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGIN]) > msglen) {
 		/* Is this likely? */
 		err = E2BIG;
 		goto errout;
 	}
 	if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgin, 0)) {
 		err = ENOMEM;
 		goto errout;
 	}
 	bzero(rp->rpr_msgin.dma_vaddr, (msglen + 7) / 8);
 	bcopy(krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_p,
 	    rp->rpr_msgin.dma_vaddr,
 	    (krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_nbits + 7) / 8);
 
 	/* Prepare space for output message (aligned buffer/length). */
 	if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT]) < msglen) {
 		/* Is this likely? */
 		err = E2BIG;
 		goto errout;
 	}
 	if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgout, 0)) {
 		err = ENOMEM;
 		goto errout;
 	}
 	bzero(rp->rpr_msgout.dma_vaddr, (msglen + 7) / 8);
 
 	mcr->mcr_pkts = htole16(1);
 	mcr->mcr_flags = 0;
 	mcr->mcr_cmdctxp = htole32(rp->rpr_q.q_ctx.dma_paddr);
 	mcr->mcr_ipktbuf.pb_addr = htole32(rp->rpr_msgin.dma_paddr);
 	mcr->mcr_ipktbuf.pb_next = 0;
 	mcr->mcr_ipktbuf.pb_len = htole32(rp->rpr_msgin.dma_size);
 	mcr->mcr_reserved = 0;
 	mcr->mcr_pktlen = htole16(msglen);
 	mcr->mcr_opktbuf.pb_addr = htole32(rp->rpr_msgout.dma_paddr);
 	mcr->mcr_opktbuf.pb_next = 0;
 	mcr->mcr_opktbuf.pb_len = htole32(rp->rpr_msgout.dma_size);
 
 #ifdef DIAGNOSTIC
 	if (rp->rpr_msgin.dma_paddr & 3 || rp->rpr_msgin.dma_size & 3) {
 		panic("%s: rsapriv: invalid msgin %x(0x%jx)",
 		    device_get_nameunit(sc->sc_dev),
 		    rp->rpr_msgin.dma_paddr, (uintmax_t)rp->rpr_msgin.dma_size);
 	}
 	if (rp->rpr_msgout.dma_paddr & 3 || rp->rpr_msgout.dma_size & 3) {
 		panic("%s: rsapriv: invalid msgout %x(0x%jx)",
 		    device_get_nameunit(sc->sc_dev),
 		    rp->rpr_msgout.dma_paddr, (uintmax_t)rp->rpr_msgout.dma_size);
 	}
 #endif
 
 	ctx->rpr_len = (sizeof(u_int16_t) * 4) + (5 * (padlen / 8));
 	ctx->rpr_op = htole16(UBS_CTXOP_RSAPRIV);
 	ctx->rpr_q_len = htole16(padlen);
 	ctx->rpr_p_len = htole16(padlen);
 
 	/*
 	 * ubsec_feed2 will sync mcr and ctx, we just need to sync
 	 * everything else.
 	 */
 	ubsec_dma_sync(&rp->rpr_msgin, BUS_DMASYNC_PREWRITE);
 	ubsec_dma_sync(&rp->rpr_msgout, BUS_DMASYNC_PREREAD);
 
 	/* Enqueue and we're done... */
 	mtx_lock(&sc->sc_mcr2lock);
 	SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rp->rpr_q, q_next);
 	ubsec_feed2(sc);
 	ubsecstats.hst_modexpcrt++;
 	mtx_unlock(&sc->sc_mcr2lock);
 	return (0);
 
 errout:
 	if (rp != NULL) {
 		if (rp->rpr_q.q_mcr.dma_tag != NULL)
 			ubsec_dma_free(sc, &rp->rpr_q.q_mcr);
 		if (rp->rpr_msgin.dma_tag != NULL) {
 			bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size);
 			ubsec_dma_free(sc, &rp->rpr_msgin);
 		}
 		if (rp->rpr_msgout.dma_tag != NULL) {
 			bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size);
 			ubsec_dma_free(sc, &rp->rpr_msgout);
 		}
 		free(rp, M_DEVBUF);
 	}
 	krp->krp_status = err;
 	crypto_kdone(krp);
 	return (0);
 }
 
 #ifdef UBSEC_DEBUG
 static void
 ubsec_dump_pb(volatile struct ubsec_pktbuf *pb)
 {
 	printf("addr 0x%x (0x%x) next 0x%x\n",
 	    pb->pb_addr, pb->pb_len, pb->pb_next);
 }
 
 static void
 ubsec_dump_ctx2(struct ubsec_ctx_keyop *c)
 {
 	printf("CTX (0x%x):\n", c->ctx_len);
 	switch (letoh16(c->ctx_op)) {
 	case UBS_CTXOP_RNGBYPASS:
 	case UBS_CTXOP_RNGSHA1:
 		break;
 	case UBS_CTXOP_MODEXP:
 	{
 		struct ubsec_ctx_modexp *cx = (void *)c;
 		int i, len;
 
 		printf(" Elen %u, Nlen %u\n",
 		    letoh16(cx->me_E_len), letoh16(cx->me_N_len));
 		len = (cx->me_N_len + 7)/8;
 		for (i = 0; i < len; i++)
 			printf("%s%02x", (i == 0) ? " N: " : ":", cx->me_N[i]);
 		printf("\n");
 		break;
 	}
 	default:
 		printf("unknown context: %x\n", c->ctx_op);
 	}
 	printf("END CTX\n");
 }
 
 static void
 ubsec_dump_mcr(struct ubsec_mcr *mcr)
 {
 	volatile struct ubsec_mcr_add *ma;
 	int i;
 
 	printf("MCR:\n");
 	printf(" pkts: %u, flags 0x%x\n",
 	    letoh16(mcr->mcr_pkts), letoh16(mcr->mcr_flags));
 	ma = (volatile struct ubsec_mcr_add *)&mcr->mcr_cmdctxp;
 	for (i = 0; i < letoh16(mcr->mcr_pkts); i++) {
 		printf(" %d: ctx 0x%x len 0x%x rsvd 0x%x\n", i,
 		    letoh32(ma->mcr_cmdctxp), letoh16(ma->mcr_pktlen),
 		    letoh16(ma->mcr_reserved));
 		printf(" %d: ipkt ", i);
 		ubsec_dump_pb(&ma->mcr_ipktbuf);
 		printf(" %d: opkt ", i);
 		ubsec_dump_pb(&ma->mcr_opktbuf);
 		ma++;
 	}
 	printf("END MCR\n");
 }
 #endif /* UBSEC_DEBUG */
 
 /*
  * Return the number of significant bits of a big number.
  */
 static int
 ubsec_ksigbits(struct crparam *cr)
 {
 	u_int plen = (cr->crp_nbits + 7) / 8;
 	int i, sig = plen * 8;
 	u_int8_t c, *p = cr->crp_p;
 
 	for (i = plen - 1; i >= 0; i--) {
 		c = p[i];
 		if (c != 0) {
 			while ((c & 0x80) == 0) {
 				sig--;
 				c <<= 1;
 			}
 			break;
 		}
 		sig -= 8;
 	}
 	return (sig);
 }
 
 static void
 ubsec_kshift_r(
 	u_int shiftbits,
 	u_int8_t *src, u_int srcbits,
 	u_int8_t *dst, u_int dstbits)
 {
 	u_int slen, dlen;
 	int i, si, di, n;
 
 	slen = (srcbits + 7) / 8;
 	dlen = (dstbits + 7) / 8;
 
 	for (i = 0; i < slen; i++)
 		dst[i] = src[i];
 	for (i = 0; i < dlen - slen; i++)
 		dst[slen + i] = 0;
 
 	n = shiftbits / 8;
 	if (n != 0) {
 		si = dlen - n - 1;
 		di = dlen - 1;
 		while (si >= 0)
 			dst[di--] = dst[si--];
 		while (di >= 0)
 			dst[di--] = 0;
 	}
 
 	n = shiftbits % 8;
 	if (n != 0) {
 		for (i = dlen - 1; i > 0; i--)
 			dst[i] = (dst[i] << n) |
 			    (dst[i - 1] >> (8 - n));
 		dst[0] = dst[0] << n;
 	}
 }
 
 static void
 ubsec_kshift_l(
 	u_int shiftbits,
 	u_int8_t *src, u_int srcbits,
 	u_int8_t *dst, u_int dstbits)
 {
 	int slen, dlen, i, n;
 
 	slen = (srcbits + 7) / 8;
 	dlen = (dstbits + 7) / 8;
 
 	n = shiftbits / 8;
 	for (i = 0; i < slen; i++)
 		dst[i] = src[i + n];
 	for (i = 0; i < dlen - slen; i++)
 		dst[slen + i] = 0;
 
 	n = shiftbits % 8;
 	if (n != 0) {
 		for (i = 0; i < (dlen - 1); i++)
 			dst[i] = (dst[i] >> n) | (dst[i + 1] << (8 - n));
 		dst[dlen - 1] = dst[dlen - 1] >> n;
 	}
 }
Index: head/sys/dev/ubsec/ubsecvar.h
===================================================================
--- head/sys/dev/ubsec/ubsecvar.h	(revision 359373)
+++ head/sys/dev/ubsec/ubsecvar.h	(revision 359374)
@@ -1,254 +1,246 @@
 /* $FreeBSD$ */
 /*	$OpenBSD: ubsecvar.h,v 1.35 2002/09/24 18:33:26 jason Exp $	*/
 
 /*-
  * SPDX-License-Identifier: BSD-3-Clause
  *
  * Copyright (c) 2000 Theo de Raadt
  * Copyright (c) 2001 Patrik Lindergren (patrik@ipunplugged.com)
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  *
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  * 3. The name of the author may not be used to endorse or promote products
  *    derived from this software without specific prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  * Effort sponsored in part by the Defense Advanced Research Projects
  * Agency (DARPA) and Air Force Research Laboratory, Air Force
  * Materiel Command, USAF, under agreement number F30602-01-2-0537.
  *
  */
 
 /* Maximum queue length */
 #ifndef UBS_MAX_NQUEUE
 #define UBS_MAX_NQUEUE		60
 #endif
 
 #define	UBS_MAX_SCATTER		64	/* Maximum scatter/gather depth */
 
 #ifndef UBS_MAX_AGGR
 #define	UBS_MAX_AGGR		5	/* Maximum aggregation count */
 #endif
 
 #define UBS_DEF_RTY		0xff	/* PCI Retry Timeout */
 #define UBS_DEF_TOUT		0xff	/* PCI TRDY Timeout */
 #define UBS_DEF_CACHELINE	0x01	/* Cache Line setting */
 
 #ifdef _KERNEL
 
 struct ubsec_dma_alloc {
 	u_int32_t		dma_paddr;
 	caddr_t			dma_vaddr;
 	bus_dma_tag_t		dma_tag;
 	bus_dmamap_t		dma_map;
 	bus_dma_segment_t	dma_seg;
 	bus_size_t		dma_size;
 	int			dma_nseg;
 };
 
 struct ubsec_q2 {
 	SIMPLEQ_ENTRY(ubsec_q2)		q_next;
 	struct ubsec_dma_alloc		q_mcr;
 	struct ubsec_dma_alloc		q_ctx;
 	u_int				q_type;
 };
 
 struct ubsec_q2_rng {
 	struct ubsec_q2			rng_q;
 	struct ubsec_dma_alloc		rng_buf;
 	int				rng_used;
 };
 
 /* C = (M ^ E) mod N */
 #define	UBS_MODEXP_PAR_M	0
 #define	UBS_MODEXP_PAR_E	1
 #define	UBS_MODEXP_PAR_N	2
 #define	UBS_MODEXP_PAR_C	3
 struct ubsec_q2_modexp {
 	struct ubsec_q2			me_q;
 	struct cryptkop *		me_krp;
 	struct ubsec_dma_alloc		me_M;
 	struct ubsec_dma_alloc		me_E;
 	struct ubsec_dma_alloc		me_C;
 	struct ubsec_dma_alloc		me_epb;
 	int				me_modbits;
 	int				me_shiftbits;
 	int				me_normbits;
 };
 
 #define	UBS_RSAPRIV_PAR_P	0
 #define	UBS_RSAPRIV_PAR_Q	1
 #define	UBS_RSAPRIV_PAR_DP	2
 #define	UBS_RSAPRIV_PAR_DQ	3
 #define	UBS_RSAPRIV_PAR_PINV	4
 #define	UBS_RSAPRIV_PAR_MSGIN	5
 #define	UBS_RSAPRIV_PAR_MSGOUT	6
 struct ubsec_q2_rsapriv {
 	struct ubsec_q2			rpr_q;
 	struct cryptkop *		rpr_krp;
 	struct ubsec_dma_alloc		rpr_msgin;
 	struct ubsec_dma_alloc		rpr_msgout;
 };
 
 #define	UBSEC_RNG_BUFSIZ	16		/* measured in 32bit words */
 
 struct ubsec_dmachunk {
 	struct ubsec_mcr	d_mcr;
 	struct ubsec_mcr_add	d_mcradd[UBS_MAX_AGGR-1];
 	struct ubsec_pktbuf	d_sbuf[UBS_MAX_SCATTER-1];
 	struct ubsec_pktbuf	d_dbuf[UBS_MAX_SCATTER-1];
 	u_int32_t		d_macbuf[5];
 	union {
 		struct ubsec_pktctx_long	ctxl;
 		struct ubsec_pktctx		ctx;
 	} d_ctx;
 };
 
 struct ubsec_dma {
 	SIMPLEQ_ENTRY(ubsec_dma)	d_next;
 	struct ubsec_dmachunk		*d_dma;
 	struct ubsec_dma_alloc		d_alloc;
 };
 
 #define	UBS_FLAGS_KEY		0x01		/* has key accelerator */
 #define	UBS_FLAGS_LONGCTX	0x02		/* uses long ipsec ctx */
 #define	UBS_FLAGS_BIGKEY	0x04		/* 2048bit keys */
 #define	UBS_FLAGS_HWNORM	0x08		/* hardware normalization */
 #define	UBS_FLAGS_RNG		0x10		/* hardware rng */
 
 struct ubsec_operand {
-	union {
-		struct mbuf *m;
-		struct uio *io;
-	} u;
 	bus_dmamap_t		map;
 	bus_size_t		mapsize;
 	int			nsegs;
 	bus_dma_segment_t	segs[UBS_MAX_SCATTER];
 };
 
 struct ubsec_q {
 	SIMPLEQ_ENTRY(ubsec_q)		q_next;
 	int				q_nstacked_mcrs;
 	struct ubsec_q			*q_stacked_mcr[UBS_MAX_AGGR-1];
 	struct cryptop			*q_crp;
 	struct ubsec_dma		*q_dma;
 
 	struct ubsec_operand		q_src;
 	struct ubsec_operand		q_dst;
+	struct mbuf			*q_dst_m;
 
 	int				q_flags;
 };
 
-#define	q_src_m		q_src.u.m
-#define	q_src_io	q_src.u.io
 #define	q_src_map	q_src.map
 #define	q_src_nsegs	q_src.nsegs
 #define	q_src_segs	q_src.segs
 #define	q_src_mapsize	q_src.mapsize
 
-#define	q_dst_m		q_dst.u.m
-#define	q_dst_io	q_dst.u.io
 #define	q_dst_map	q_dst.map
 #define	q_dst_nsegs	q_dst.nsegs
 #define	q_dst_segs	q_dst.segs
 #define	q_dst_mapsize	q_dst.mapsize
 
 struct rndstate_test;
 
 struct ubsec_softc {
 	device_t		sc_dev;		/* device backpointer */
 	struct resource		*sc_irq;
 	void			*sc_ih;		/* interrupt handler cookie */
 	bus_space_handle_t	sc_sh;		/* memory handle */
 	bus_space_tag_t		sc_st;		/* memory tag */
 	struct resource		*sc_sr;		/* memory resource */
 	bus_dma_tag_t		sc_dmat;	/* dma tag */
 	int			sc_flags;	/* device specific flags */
 	int			sc_suspended;
 	int			sc_needwakeup;	/* notify crypto layer */
 	u_int32_t		sc_statmask;	/* interrupt status mask */
 	int32_t			sc_cid;		/* crypto tag */
 	struct mtx		sc_mcr1lock;	/* mcr1 operation lock */
 	SIMPLEQ_HEAD(,ubsec_q)	sc_queue;	/* packet queue, mcr1 */
 	int			sc_nqueue;	/* count enqueued, mcr1 */
 	SIMPLEQ_HEAD(,ubsec_q)	sc_qchip;	/* on chip, mcr1 */
 	int			sc_nqchip;	/* count on chip, mcr1 */
 	struct mtx		sc_freeqlock;	/* freequeue lock */
 	SIMPLEQ_HEAD(,ubsec_q)	sc_freequeue;	/* list of free queue elements */
 	struct mtx		sc_mcr2lock;	/* mcr2 operation lock */
 	SIMPLEQ_HEAD(,ubsec_q2)	sc_queue2;	/* packet queue, mcr2 */
 	int			sc_nqueue2;	/* count enqueued, mcr2 */
 	SIMPLEQ_HEAD(,ubsec_q2)	sc_qchip2;	/* on chip, mcr2 */
 	struct callout		sc_rngto;	/* rng timeout */
 	int			sc_rnghz;	/* rng poll time */
 	struct ubsec_q2_rng	sc_rng;
 	struct rndtest_state	*sc_rndtest;	/* RNG test state */
 	void			(*sc_harvest)(struct rndtest_state *,
 					void *, u_int);
 	struct ubsec_dma	sc_dmaa[UBS_MAX_NQUEUE];
 	struct ubsec_q		*sc_queuea[UBS_MAX_NQUEUE];
 	SIMPLEQ_HEAD(,ubsec_q2)	sc_q2free;	/* free list */
 };
 
 #define	UBSEC_QFLAGS_COPYOUTIV		0x1
 
 struct ubsec_session {
 	u_int32_t	ses_deskey[6];		/* 3DES key */
 	u_int32_t	ses_mlen;		/* hmac length */
 	u_int32_t	ses_hminner[5];		/* hmac inner state */
 	u_int32_t	ses_hmouter[5];		/* hmac outer state */
-	u_int32_t	ses_iv[2];		/* [3]DES iv */
 };
 #endif /* _KERNEL */
 
 struct ubsec_stats {
 	u_int64_t hst_ibytes;
 	u_int64_t hst_obytes;
 	u_int32_t hst_ipackets;
 	u_int32_t hst_opackets;
 	u_int32_t hst_invalid;		/* invalid argument */
 	u_int32_t hst_badsession;	/* invalid session id */
 	u_int32_t hst_badflags;		/* flags indicate !(mbuf | uio) */
 	u_int32_t hst_nodesc;		/* op submitted w/o descriptors */
 	u_int32_t hst_badalg;		/* unsupported algorithm */
 	u_int32_t hst_nomem;
 	u_int32_t hst_queuefull;
 	u_int32_t hst_dmaerr;
 	u_int32_t hst_mcrerr;
 	u_int32_t hst_nodmafree;
 	u_int32_t hst_lenmismatch;	/* enc/auth lengths different */
 	u_int32_t hst_skipmismatch;	/* enc part begins before auth part */
 	u_int32_t hst_iovmisaligned;	/* iov op not aligned */
 	u_int32_t hst_noirq;		/* IRQ for no reason */
 	u_int32_t hst_unaligned;	/* unaligned src caused copy */
 	u_int32_t hst_nomap;		/* bus_dmamap_create failed */
 	u_int32_t hst_noload;		/* bus_dmamap_load_* failed */
 	u_int32_t hst_nombuf;		/* MGET* failed */
 	u_int32_t hst_nomcl;		/* MCLGET* failed */
 	u_int32_t hst_totbatch;		/* ops submitted w/o interrupt */
 	u_int32_t hst_maxbatch;		/* max ops submitted together */
 	u_int32_t hst_maxqueue;		/* max ops queued for submission */
 	u_int32_t hst_maxqchip;		/* max mcr1 ops out for processing */
 	u_int32_t hst_mcr1full;		/* MCR1 too busy to take ops */
 	u_int32_t hst_rng;		/* RNG requests */
 	u_int32_t hst_modexp;		/* MOD EXP requests */
 	u_int32_t hst_modexpcrt;	/* MOD EXP CRT requests */
 };
Index: head/sys/geom/eli/g_eli.c
===================================================================
--- head/sys/geom/eli/g_eli.c	(revision 359373)
+++ head/sys/geom/eli/g_eli.c	(revision 359374)
@@ -1,1443 +1,1456 @@
 /*-
  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
  *
  * Copyright (c) 2005-2019 Pawel Jakub Dawidek <pawel@dawidek.net>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/cons.h>
 #include <sys/kernel.h>
 #include <sys/linker.h>
 #include <sys/module.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <sys/bio.h>
 #include <sys/sbuf.h>
 #include <sys/sysctl.h>
 #include <sys/malloc.h>
 #include <sys/eventhandler.h>
 #include <sys/kthread.h>
 #include <sys/proc.h>
 #include <sys/sched.h>
 #include <sys/smp.h>
 #include <sys/uio.h>
 #include <sys/vnode.h>
 
 #include <vm/uma.h>
 
 #include <geom/geom.h>
 #include <geom/geom_dbg.h>
 #include <geom/eli/g_eli.h>
 #include <geom/eli/pkcs5v2.h>
 
 #include <crypto/intake.h>
 
 FEATURE(geom_eli, "GEOM crypto module");
 
 MALLOC_DEFINE(M_ELI, "eli data", "GEOM_ELI Data");
 
 SYSCTL_DECL(_kern_geom);
 SYSCTL_NODE(_kern_geom, OID_AUTO, eli, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
     "GEOM_ELI stuff");
 static int g_eli_version = G_ELI_VERSION;
 SYSCTL_INT(_kern_geom_eli, OID_AUTO, version, CTLFLAG_RD, &g_eli_version, 0,
     "GELI version");
 int g_eli_debug = 0;
 SYSCTL_INT(_kern_geom_eli, OID_AUTO, debug, CTLFLAG_RWTUN, &g_eli_debug, 0,
     "Debug level");
 static u_int g_eli_tries = 3;
 SYSCTL_UINT(_kern_geom_eli, OID_AUTO, tries, CTLFLAG_RWTUN, &g_eli_tries, 0,
     "Number of tries for entering the passphrase");
 static u_int g_eli_visible_passphrase = GETS_NOECHO;
 SYSCTL_UINT(_kern_geom_eli, OID_AUTO, visible_passphrase, CTLFLAG_RWTUN,
     &g_eli_visible_passphrase, 0,
     "Visibility of passphrase prompt (0 = invisible, 1 = visible, 2 = asterisk)");
 u_int g_eli_overwrites = G_ELI_OVERWRITES;
 SYSCTL_UINT(_kern_geom_eli, OID_AUTO, overwrites, CTLFLAG_RWTUN, &g_eli_overwrites,
     0, "Number of times on-disk keys should be overwritten when destroying them");
 static u_int g_eli_threads = 0;
 SYSCTL_UINT(_kern_geom_eli, OID_AUTO, threads, CTLFLAG_RWTUN, &g_eli_threads, 0,
     "Number of threads doing crypto work");
 u_int g_eli_batch = 0;
 SYSCTL_UINT(_kern_geom_eli, OID_AUTO, batch, CTLFLAG_RWTUN, &g_eli_batch, 0,
     "Use crypto operations batching");
 
 /*
  * Passphrase cached during boot, in order to be more user-friendly if
  * there are multiple providers using the same passphrase.
  */
 static char cached_passphrase[256];
 static u_int g_eli_boot_passcache = 1;
 TUNABLE_INT("kern.geom.eli.boot_passcache", &g_eli_boot_passcache);
 SYSCTL_UINT(_kern_geom_eli, OID_AUTO, boot_passcache, CTLFLAG_RD,
     &g_eli_boot_passcache, 0,
     "Passphrases are cached during boot process for possible reuse");
 static void
 fetch_loader_passphrase(void * dummy)
 {
 	char * env_passphrase;
 
 	KASSERT(dynamic_kenv, ("need dynamic kenv"));
 
 	if ((env_passphrase = kern_getenv("kern.geom.eli.passphrase")) != NULL) {
 		/* Extract passphrase from the environment. */
 		strlcpy(cached_passphrase, env_passphrase,
 		    sizeof(cached_passphrase));
 		freeenv(env_passphrase);
 
 		/* Wipe the passphrase from the environment. */
 		kern_unsetenv("kern.geom.eli.passphrase");
 	}
 }
 SYSINIT(geli_fetch_loader_passphrase, SI_SUB_KMEM + 1, SI_ORDER_ANY,
     fetch_loader_passphrase, NULL);
 
 static void
 zero_boot_passcache(void)
 {
 
         explicit_bzero(cached_passphrase, sizeof(cached_passphrase));
 }
 
 static void
 zero_geli_intake_keys(void)
 {
         struct keybuf *keybuf;
         int i;
 
         if ((keybuf = get_keybuf()) != NULL) {
                 /* Scan the key buffer, clear all GELI keys. */
                 for (i = 0; i < keybuf->kb_nents; i++) {
                          if (keybuf->kb_ents[i].ke_type == KEYBUF_TYPE_GELI) {
                                  explicit_bzero(keybuf->kb_ents[i].ke_data,
                                      sizeof(keybuf->kb_ents[i].ke_data));
                                  keybuf->kb_ents[i].ke_type = KEYBUF_TYPE_NONE;
                          }
                 }
         }
 }
 
 static void
 zero_intake_passcache(void *dummy)
 {
         zero_boot_passcache();
         zero_geli_intake_keys();
 }
 EVENTHANDLER_DEFINE(mountroot, zero_intake_passcache, NULL, 0);
 
 static eventhandler_tag g_eli_pre_sync = NULL;
 
 static int g_eli_read_metadata_offset(struct g_class *mp, struct g_provider *pp,
     off_t offset, struct g_eli_metadata *md);
 
 static int g_eli_destroy_geom(struct gctl_req *req, struct g_class *mp,
     struct g_geom *gp);
 static void g_eli_init(struct g_class *mp);
 static void g_eli_fini(struct g_class *mp);
 
 static g_taste_t g_eli_taste;
 static g_dumpconf_t g_eli_dumpconf;
 
 struct g_class g_eli_class = {
 	.name = G_ELI_CLASS_NAME,
 	.version = G_VERSION,
 	.ctlreq = g_eli_config,
 	.taste = g_eli_taste,
 	.destroy_geom = g_eli_destroy_geom,
 	.init = g_eli_init,
 	.fini = g_eli_fini
 };
 
 
 /*
  * Code paths:
  * BIO_READ:
  *	g_eli_start -> g_eli_crypto_read -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> g_eli_crypto_read_done -> g_io_deliver
  * BIO_WRITE:
  *	g_eli_start -> g_eli_crypto_run -> g_eli_crypto_write_done -> g_io_request -> g_eli_write_done -> g_io_deliver
  */
 
 
 /*
  * EAGAIN from crypto(9) means, that we were probably balanced to another crypto
  * accelerator or something like this.
  * The function updates the SID and rerun the operation.
  */
 int
 g_eli_crypto_rerun(struct cryptop *crp)
 {
 	struct g_eli_softc *sc;
 	struct g_eli_worker *wr;
 	struct bio *bp;
 	int error;
 
 	bp = (struct bio *)crp->crp_opaque;
 	sc = bp->bio_to->geom->softc;
 	LIST_FOREACH(wr, &sc->sc_workers, w_next) {
 		if (wr->w_number == bp->bio_pflags)
 			break;
 	}
 	KASSERT(wr != NULL, ("Invalid worker (%u).", bp->bio_pflags));
 	G_ELI_DEBUG(1, "Rerunning crypto %s request (sid: %p -> %p).",
 	    bp->bio_cmd == BIO_READ ? "READ" : "WRITE", wr->w_sid,
 	    crp->crp_session);
 	wr->w_sid = crp->crp_session;
 	crp->crp_etype = 0;
 	error = crypto_dispatch(crp);
 	if (error == 0)
 		return (0);
 	G_ELI_DEBUG(1, "%s: crypto_dispatch() returned %d.", __func__, error);
 	crp->crp_etype = error;
 	return (error);
 }
 
 static void
 g_eli_getattr_done(struct bio *bp)
 {
 	if (bp->bio_error == 0 && 
 	    !strcmp(bp->bio_attribute, "GEOM::physpath")) {
 		strlcat(bp->bio_data, "/eli", bp->bio_length);
 	}
 	g_std_done(bp);
 }
 
 /*
  * The function is called afer reading encrypted data from the provider.
  *
  * g_eli_start -> g_eli_crypto_read -> g_io_request -> G_ELI_READ_DONE -> g_eli_crypto_run -> g_eli_crypto_read_done -> g_io_deliver
  */
 void
 g_eli_read_done(struct bio *bp)
 {
 	struct g_eli_softc *sc;
 	struct bio *pbp;
 
 	G_ELI_LOGREQ(2, bp, "Request done.");
 	pbp = bp->bio_parent;
 	if (pbp->bio_error == 0 && bp->bio_error != 0)
 		pbp->bio_error = bp->bio_error;
 	g_destroy_bio(bp);
 	/*
 	 * Do we have all sectors already?
 	 */
 	pbp->bio_inbed++;
 	if (pbp->bio_inbed < pbp->bio_children)
 		return;
 	sc = pbp->bio_to->geom->softc;
 	if (pbp->bio_error != 0) {
 		G_ELI_LOGREQ(0, pbp, "%s() failed (error=%d)", __func__,
 		    pbp->bio_error);
 		pbp->bio_completed = 0;
 		if (pbp->bio_driver2 != NULL) {
 			free(pbp->bio_driver2, M_ELI);
 			pbp->bio_driver2 = NULL;
 		}
 		g_io_deliver(pbp, pbp->bio_error);
 		if (sc != NULL)
 			atomic_subtract_int(&sc->sc_inflight, 1);
 		return;
 	}
 	mtx_lock(&sc->sc_queue_mtx);
 	bioq_insert_tail(&sc->sc_queue, pbp);
 	mtx_unlock(&sc->sc_queue_mtx);
 	wakeup(sc);
 }
 
 /*
  * The function is called after we encrypt and write data.
  *
  * g_eli_start -> g_eli_crypto_run -> g_eli_crypto_write_done -> g_io_request -> G_ELI_WRITE_DONE -> g_io_deliver
  */
 void
 g_eli_write_done(struct bio *bp)
 {
 	struct g_eli_softc *sc;
 	struct bio *pbp;
 
 	G_ELI_LOGREQ(2, bp, "Request done.");
 	pbp = bp->bio_parent;
 	if (pbp->bio_error == 0 && bp->bio_error != 0)
 		pbp->bio_error = bp->bio_error;
 	g_destroy_bio(bp);
 	/*
 	 * Do we have all sectors already?
 	 */
 	pbp->bio_inbed++;
 	if (pbp->bio_inbed < pbp->bio_children)
 		return;
 	free(pbp->bio_driver2, M_ELI);
 	pbp->bio_driver2 = NULL;
 	if (pbp->bio_error != 0) {
 		G_ELI_LOGREQ(0, pbp, "%s() failed (error=%d)", __func__,
 		    pbp->bio_error);
 		pbp->bio_completed = 0;
 	} else
 		pbp->bio_completed = pbp->bio_length;
 
 	/*
 	 * Write is finished, send it up.
 	 */
 	sc = pbp->bio_to->geom->softc;
 	g_io_deliver(pbp, pbp->bio_error);
 	if (sc != NULL)
 		atomic_subtract_int(&sc->sc_inflight, 1);
 }
 
 /*
  * This function should never be called, but GEOM made as it set ->orphan()
  * method for every geom.
  */
 static void
 g_eli_orphan_spoil_assert(struct g_consumer *cp)
 {
 
 	panic("Function %s() called for %s.", __func__, cp->geom->name);
 }
 
 static void
 g_eli_orphan(struct g_consumer *cp)
 {
 	struct g_eli_softc *sc;
 
 	g_topology_assert();
 	sc = cp->geom->softc;
 	if (sc == NULL)
 		return;
 	g_eli_destroy(sc, TRUE);
 }
 
 static void
 g_eli_resize(struct g_consumer *cp)
 {
 	struct g_eli_softc *sc;
 	struct g_provider *epp, *pp;
 	off_t oldsize;
 
 	g_topology_assert();
 	sc = cp->geom->softc;
 	if (sc == NULL)
 		return;
 
 	if ((sc->sc_flags & G_ELI_FLAG_AUTORESIZE) == 0) {
 		G_ELI_DEBUG(0, "Autoresize is turned off, old size: %jd.",
 		    (intmax_t)sc->sc_provsize);
 		return;
 	}
 
 	pp = cp->provider;
 
 	if ((sc->sc_flags & G_ELI_FLAG_ONETIME) == 0) {
 		struct g_eli_metadata md;
 		u_char *sector;
 		int error;
 
 		sector = NULL;
 
 		error = g_eli_read_metadata_offset(cp->geom->class, pp,
 		    sc->sc_provsize - pp->sectorsize, &md);
 		if (error != 0) {
 			G_ELI_DEBUG(0, "Cannot read metadata from %s (error=%d).",
 			    pp->name, error);
 			goto iofail;
 		}
 
 		md.md_provsize = pp->mediasize;
 
 		sector = malloc(pp->sectorsize, M_ELI, M_WAITOK | M_ZERO);
 		eli_metadata_encode(&md, sector);
 		error = g_write_data(cp, pp->mediasize - pp->sectorsize, sector,
 		    pp->sectorsize);
 		if (error != 0) {
 			G_ELI_DEBUG(0, "Cannot store metadata on %s (error=%d).",
 			    pp->name, error);
 			goto iofail;
 		}
 		explicit_bzero(sector, pp->sectorsize);
 		error = g_write_data(cp, sc->sc_provsize - pp->sectorsize,
 		    sector, pp->sectorsize);
 		if (error != 0) {
 			G_ELI_DEBUG(0, "Cannot clear old metadata from %s (error=%d).",
 			    pp->name, error);
 			goto iofail;
 		}
 iofail:
 		explicit_bzero(&md, sizeof(md));
 		if (sector != NULL) {
 			explicit_bzero(sector, pp->sectorsize);
 			free(sector, M_ELI);
 		}
 	}
 
 	oldsize = sc->sc_mediasize;
 	sc->sc_mediasize = eli_mediasize(sc, pp->mediasize, pp->sectorsize);
 	g_eli_key_resize(sc);
 	sc->sc_provsize = pp->mediasize;
 
 	epp = LIST_FIRST(&sc->sc_geom->provider);
 	g_resize_provider(epp, sc->sc_mediasize);
 	G_ELI_DEBUG(0, "Device %s size changed from %jd to %jd.", epp->name,
 	    (intmax_t)oldsize, (intmax_t)sc->sc_mediasize);
 }
 
 /*
  * BIO_READ:
  *	G_ELI_START -> g_eli_crypto_read -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> g_eli_crypto_read_done -> g_io_deliver
  * BIO_WRITE:
  *	G_ELI_START -> g_eli_crypto_run -> g_eli_crypto_write_done -> g_io_request -> g_eli_write_done -> g_io_deliver
  */
 static void
 g_eli_start(struct bio *bp)
 {
 	struct g_eli_softc *sc;
 	struct g_consumer *cp;
 	struct bio *cbp;
 
 	sc = bp->bio_to->geom->softc;
 	KASSERT(sc != NULL,
 	    ("Provider's error should be set (error=%d)(device=%s).",
 	    bp->bio_to->error, bp->bio_to->name));
 	G_ELI_LOGREQ(2, bp, "Request received.");
 
 	switch (bp->bio_cmd) {
 	case BIO_READ:
 	case BIO_WRITE:
 	case BIO_GETATTR:
 	case BIO_FLUSH:
 	case BIO_ZONE:
 	case BIO_SPEEDUP:
 		break;
 	case BIO_DELETE:
 		/*
 		 * If the user hasn't set the NODELETE flag, we just pass
 		 * it down the stack and let the layers beneath us do (or
 		 * not) whatever they do with it.  If they have, we
 		 * reject it.  A possible extension would be an
 		 * additional flag to take it as a hint to shred the data
 		 * with [multiple?] overwrites.
 		 */
 		if (!(sc->sc_flags & G_ELI_FLAG_NODELETE))
 			break;
 	default:
 		g_io_deliver(bp, EOPNOTSUPP);
 		return;
 	}
 	cbp = g_clone_bio(bp);
 	if (cbp == NULL) {
 		g_io_deliver(bp, ENOMEM);
 		return;
 	}
 	bp->bio_driver1 = cbp;
 	bp->bio_pflags = G_ELI_NEW_BIO;
 	switch (bp->bio_cmd) {
 	case BIO_READ:
 		if (!(sc->sc_flags & G_ELI_FLAG_AUTH)) {
 			g_eli_crypto_read(sc, bp, 0);
 			break;
 		}
 		/* FALLTHROUGH */
 	case BIO_WRITE:
 		mtx_lock(&sc->sc_queue_mtx);
 		bioq_insert_tail(&sc->sc_queue, bp);
 		mtx_unlock(&sc->sc_queue_mtx);
 		wakeup(sc);
 		break;
 	case BIO_GETATTR:
 	case BIO_FLUSH:
 	case BIO_DELETE:
 	case BIO_SPEEDUP:
 	case BIO_ZONE:
 		if (bp->bio_cmd == BIO_GETATTR)
 			cbp->bio_done = g_eli_getattr_done;
 		else
 			cbp->bio_done = g_std_done;
 		cp = LIST_FIRST(&sc->sc_geom->consumer);
 		cbp->bio_to = cp->provider;
 		G_ELI_LOGREQ(2, cbp, "Sending request.");
 		g_io_request(cbp, cp);
 		break;
 	}
 }
 
 static int
 g_eli_newsession(struct g_eli_worker *wr)
 {
 	struct g_eli_softc *sc;
-	struct cryptoini crie, cria;
+	struct crypto_session_params csp;
 	int error;
+	void *key;
 
 	sc = wr->w_softc;
 
-	bzero(&crie, sizeof(crie));
-	crie.cri_alg = sc->sc_ealgo;
-	crie.cri_klen = sc->sc_ekeylen;
+	memset(&csp, 0, sizeof(csp));
+	csp.csp_mode = CSP_MODE_CIPHER;
+	csp.csp_cipher_alg = sc->sc_ealgo;
+	csp.csp_ivlen = g_eli_ivlen(sc->sc_ealgo);
+	csp.csp_cipher_klen = sc->sc_ekeylen / 8;
 	if (sc->sc_ealgo == CRYPTO_AES_XTS)
-		crie.cri_klen <<= 1;
+		csp.csp_cipher_klen <<= 1;
 	if ((sc->sc_flags & G_ELI_FLAG_FIRST_KEY) != 0) {
-		crie.cri_key = g_eli_key_hold(sc, 0,
+		key = g_eli_key_hold(sc, 0,
 		    LIST_FIRST(&sc->sc_geom->consumer)->provider->sectorsize);
+		csp.csp_cipher_key = key;
 	} else {
-		crie.cri_key = sc->sc_ekey;
+		key = NULL;
+		csp.csp_cipher_key = sc->sc_ekey;
 	}
 	if (sc->sc_flags & G_ELI_FLAG_AUTH) {
-		bzero(&cria, sizeof(cria));
-		cria.cri_alg = sc->sc_aalgo;
-		cria.cri_klen = sc->sc_akeylen;
-		cria.cri_key = sc->sc_akey;
-		crie.cri_next = &cria;
+		csp.csp_mode = CSP_MODE_ETA;
+		csp.csp_auth_alg = sc->sc_aalgo;
+		csp.csp_auth_klen = G_ELI_AUTH_SECKEYLEN;
 	}
 
 	switch (sc->sc_crypto) {
 	case G_ELI_CRYPTO_SW:
-		error = crypto_newsession(&wr->w_sid, &crie,
+		error = crypto_newsession(&wr->w_sid, &csp,
 		    CRYPTOCAP_F_SOFTWARE);
 		break;
 	case G_ELI_CRYPTO_HW:
-		error = crypto_newsession(&wr->w_sid, &crie,
+		error = crypto_newsession(&wr->w_sid, &csp,
 		    CRYPTOCAP_F_HARDWARE);
 		break;
 	case G_ELI_CRYPTO_UNKNOWN:
-		error = crypto_newsession(&wr->w_sid, &crie,
+		error = crypto_newsession(&wr->w_sid, &csp,
 		    CRYPTOCAP_F_HARDWARE);
 		if (error == 0) {
 			mtx_lock(&sc->sc_queue_mtx);
 			if (sc->sc_crypto == G_ELI_CRYPTO_UNKNOWN)
 				sc->sc_crypto = G_ELI_CRYPTO_HW;
 			mtx_unlock(&sc->sc_queue_mtx);
 		} else {
-			error = crypto_newsession(&wr->w_sid, &crie,
+			error = crypto_newsession(&wr->w_sid, &csp,
 			    CRYPTOCAP_F_SOFTWARE);
 			mtx_lock(&sc->sc_queue_mtx);
 			if (sc->sc_crypto == G_ELI_CRYPTO_UNKNOWN)
 				sc->sc_crypto = G_ELI_CRYPTO_SW;
 			mtx_unlock(&sc->sc_queue_mtx);
 		}
 		break;
 	default:
 		panic("%s: invalid condition", __func__);
 	}
 
-	if ((sc->sc_flags & G_ELI_FLAG_FIRST_KEY) != 0)
-		g_eli_key_drop(sc, crie.cri_key);
+	if ((sc->sc_flags & G_ELI_FLAG_FIRST_KEY) != 0) {
+		if (error)
+			g_eli_key_drop(sc, key);
+		else
+			wr->w_first_key = key;
+	}
 
 	return (error);
 }
 
 static void
 g_eli_freesession(struct g_eli_worker *wr)
 {
+	struct g_eli_softc *sc;
 
 	crypto_freesession(wr->w_sid);
+	if (wr->w_first_key != NULL) {
+		sc = wr->w_softc;
+		g_eli_key_drop(sc, wr->w_first_key);
+		wr->w_first_key = NULL;
+	}
 }
 
 static void
 g_eli_cancel(struct g_eli_softc *sc)
 {
 	struct bio *bp;
 
 	mtx_assert(&sc->sc_queue_mtx, MA_OWNED);
 
 	while ((bp = bioq_takefirst(&sc->sc_queue)) != NULL) {
 		KASSERT(bp->bio_pflags == G_ELI_NEW_BIO,
 		    ("Not new bio when canceling (bp=%p).", bp));
 		g_io_deliver(bp, ENXIO);
 	}
 }
 
 static struct bio *
 g_eli_takefirst(struct g_eli_softc *sc)
 {
 	struct bio *bp;
 
 	mtx_assert(&sc->sc_queue_mtx, MA_OWNED);
 
 	if (!(sc->sc_flags & G_ELI_FLAG_SUSPEND))
 		return (bioq_takefirst(&sc->sc_queue));
 	/*
 	 * Device suspended, so we skip new I/O requests.
 	 */
 	TAILQ_FOREACH(bp, &sc->sc_queue.queue, bio_queue) {
 		if (bp->bio_pflags != G_ELI_NEW_BIO)
 			break;
 	}
 	if (bp != NULL)
 		bioq_remove(&sc->sc_queue, bp);
 	return (bp);
 }
 
 /*
  * This is the main function for kernel worker thread when we don't have
  * hardware acceleration and we have to do cryptography in software.
  * Dedicated thread is needed, so we don't slow down g_up/g_down GEOM
  * threads with crypto work.
  */
 static void
 g_eli_worker(void *arg)
 {
 	struct g_eli_softc *sc;
 	struct g_eli_worker *wr;
 	struct bio *bp;
 	int error;
 
 	wr = arg;
 	sc = wr->w_softc;
 #ifdef EARLY_AP_STARTUP
 	MPASS(!sc->sc_cpubind || smp_started);
 #elif defined(SMP)
 	/* Before sched_bind() to a CPU, wait for all CPUs to go on-line. */
 	if (sc->sc_cpubind) {
 		while (!smp_started)
 			tsleep(wr, 0, "geli:smp", hz / 4);
 	}
 #endif
 	thread_lock(curthread);
 	sched_prio(curthread, PUSER);
 	if (sc->sc_cpubind)
 		sched_bind(curthread, wr->w_number % mp_ncpus);
 	thread_unlock(curthread);
 
 	G_ELI_DEBUG(1, "Thread %s started.", curthread->td_proc->p_comm);
 
 	for (;;) {
 		mtx_lock(&sc->sc_queue_mtx);
 again:
 		bp = g_eli_takefirst(sc);
 		if (bp == NULL) {
 			if (sc->sc_flags & G_ELI_FLAG_DESTROY) {
 				g_eli_cancel(sc);
 				LIST_REMOVE(wr, w_next);
 				g_eli_freesession(wr);
 				free(wr, M_ELI);
 				G_ELI_DEBUG(1, "Thread %s exiting.",
 				    curthread->td_proc->p_comm);
 				wakeup(&sc->sc_workers);
 				mtx_unlock(&sc->sc_queue_mtx);
 				kproc_exit(0);
 			}
 			while (sc->sc_flags & G_ELI_FLAG_SUSPEND) {
 				if (sc->sc_inflight > 0) {
 					G_ELI_DEBUG(0, "inflight=%d",
 					    sc->sc_inflight);
 					/*
 					 * We still have inflight BIOs, so
 					 * sleep and retry.
 					 */
 					msleep(sc, &sc->sc_queue_mtx, PRIBIO,
 					    "geli:inf", hz / 5);
 					goto again;
 				}
 				/*
 				 * Suspend requested, mark the worker as
 				 * suspended and go to sleep.
 				 */
 				if (wr->w_active) {
 					g_eli_freesession(wr);
 					wr->w_active = FALSE;
 				}
 				wakeup(&sc->sc_workers);
 				msleep(sc, &sc->sc_queue_mtx, PRIBIO,
 				    "geli:suspend", 0);
 				if (!wr->w_active &&
 				    !(sc->sc_flags & G_ELI_FLAG_SUSPEND)) {
 					error = g_eli_newsession(wr);
 					KASSERT(error == 0,
 					    ("g_eli_newsession() failed on resume (error=%d)",
 					    error));
 					wr->w_active = TRUE;
 				}
 				goto again;
 			}
 			msleep(sc, &sc->sc_queue_mtx, PDROP, "geli:w", 0);
 			continue;
 		}
 		if (bp->bio_pflags == G_ELI_NEW_BIO)
 			atomic_add_int(&sc->sc_inflight, 1);
 		mtx_unlock(&sc->sc_queue_mtx);
 		if (bp->bio_pflags == G_ELI_NEW_BIO) {
 			bp->bio_pflags = 0;
 			if (sc->sc_flags & G_ELI_FLAG_AUTH) {
 				if (bp->bio_cmd == BIO_READ)
 					g_eli_auth_read(sc, bp);
 				else
 					g_eli_auth_run(wr, bp);
 			} else {
 				if (bp->bio_cmd == BIO_READ)
 					g_eli_crypto_read(sc, bp, 1);
 				else
 					g_eli_crypto_run(wr, bp);
 			}
 		} else {
 			if (sc->sc_flags & G_ELI_FLAG_AUTH)
 				g_eli_auth_run(wr, bp);
 			else
 				g_eli_crypto_run(wr, bp);
 		}
 	}
 }
 
 static int
 g_eli_read_metadata_offset(struct g_class *mp, struct g_provider *pp,
     off_t offset, struct g_eli_metadata *md)
 {
 	struct g_geom *gp;
 	struct g_consumer *cp;
 	u_char *buf = NULL;
 	int error;
 
 	g_topology_assert();
 
 	gp = g_new_geomf(mp, "eli:taste");
 	gp->start = g_eli_start;
 	gp->access = g_std_access;
 	/*
 	 * g_eli_read_metadata() is always called from the event thread.
 	 * Our geom is created and destroyed in the same event, so there
 	 * could be no orphan nor spoil event in the meantime.
 	 */
 	gp->orphan = g_eli_orphan_spoil_assert;
 	gp->spoiled = g_eli_orphan_spoil_assert;
 	cp = g_new_consumer(gp);
 	error = g_attach(cp, pp);
 	if (error != 0)
 		goto end;
 	error = g_access(cp, 1, 0, 0);
 	if (error != 0)
 		goto end;
 	g_topology_unlock();
 	buf = g_read_data(cp, offset, pp->sectorsize, &error);
 	g_topology_lock();
 	if (buf == NULL)
 		goto end;
 	error = eli_metadata_decode(buf, md);
 	if (error != 0)
 		goto end;
 	/* Metadata was read and decoded successfully. */
 end:
 	if (buf != NULL)
 		g_free(buf);
 	if (cp->provider != NULL) {
 		if (cp->acr == 1)
 			g_access(cp, -1, 0, 0);
 		g_detach(cp);
 	}
 	g_destroy_consumer(cp);
 	g_destroy_geom(gp);
 	return (error);
 }
 
 int
 g_eli_read_metadata(struct g_class *mp, struct g_provider *pp,
     struct g_eli_metadata *md)
 {
 
 	return (g_eli_read_metadata_offset(mp, pp,
 	    pp->mediasize - pp->sectorsize, md));
 }
 
 /*
  * The function is called when we had last close on provider and user requested
  * to close it when this situation occur.
  */
 static void
 g_eli_last_close(void *arg, int flags __unused)
 {
 	struct g_geom *gp;
 	char gpname[64];
 	int error;
 
 	g_topology_assert();
 	gp = arg;
 	strlcpy(gpname, gp->name, sizeof(gpname));
 	error = g_eli_destroy(gp->softc, TRUE);
 	KASSERT(error == 0, ("Cannot detach %s on last close (error=%d).",
 	    gpname, error));
 	G_ELI_DEBUG(0, "Detached %s on last close.", gpname);
 }
 
 int
 g_eli_access(struct g_provider *pp, int dr, int dw, int de)
 {
 	struct g_eli_softc *sc;
 	struct g_geom *gp;
 
 	gp = pp->geom;
 	sc = gp->softc;
 
 	if (dw > 0) {
 		if (sc->sc_flags & G_ELI_FLAG_RO) {
 			/* Deny write attempts. */
 			return (EROFS);
 		}
 		/* Someone is opening us for write, we need to remember that. */
 		sc->sc_flags |= G_ELI_FLAG_WOPEN;
 		return (0);
 	}
 	/* Is this the last close? */
 	if (pp->acr + dr > 0 || pp->acw + dw > 0 || pp->ace + de > 0)
 		return (0);
 
 	/*
 	 * Automatically detach on last close if requested.
 	 */
 	if ((sc->sc_flags & G_ELI_FLAG_RW_DETACH) ||
 	    (sc->sc_flags & G_ELI_FLAG_WOPEN)) {
 		g_post_event(g_eli_last_close, gp, M_WAITOK, NULL);
 	}
 	return (0);
 }
 
 static int
 g_eli_cpu_is_disabled(int cpu)
 {
 #ifdef SMP
 	return (CPU_ISSET(cpu, &hlt_cpus_mask));
 #else
 	return (0);
 #endif
 }
 
 struct g_geom *
 g_eli_create(struct gctl_req *req, struct g_class *mp, struct g_provider *bpp,
     const struct g_eli_metadata *md, const u_char *mkey, int nkey)
 {
 	struct g_eli_softc *sc;
 	struct g_eli_worker *wr;
 	struct g_geom *gp;
 	struct g_provider *pp;
 	struct g_consumer *cp;
 	u_int i, threads;
 	int dcw, error;
 
 	G_ELI_DEBUG(1, "Creating device %s%s.", bpp->name, G_ELI_SUFFIX);
 
 	gp = g_new_geomf(mp, "%s%s", bpp->name, G_ELI_SUFFIX);
 	sc = malloc(sizeof(*sc), M_ELI, M_WAITOK | M_ZERO);
 	gp->start = g_eli_start;
 	/*
 	 * Spoiling can happen even though we have the provider open
 	 * exclusively, e.g. through media change events.
 	 */
 	gp->spoiled = g_eli_orphan;
 	gp->orphan = g_eli_orphan;
 	gp->resize = g_eli_resize;
 	gp->dumpconf = g_eli_dumpconf;
 	/*
 	 * If detach-on-last-close feature is not enabled and we don't operate
 	 * on read-only provider, we can simply use g_std_access().
 	 */
 	if (md->md_flags & (G_ELI_FLAG_WO_DETACH | G_ELI_FLAG_RO))
 		gp->access = g_eli_access;
 	else
 		gp->access = g_std_access;
 
 	eli_metadata_softc(sc, md, bpp->sectorsize, bpp->mediasize);
 	sc->sc_nkey = nkey;
 
 	gp->softc = sc;
 	sc->sc_geom = gp;
 
 	bioq_init(&sc->sc_queue);
 	mtx_init(&sc->sc_queue_mtx, "geli:queue", NULL, MTX_DEF);
 	mtx_init(&sc->sc_ekeys_lock, "geli:ekeys", NULL, MTX_DEF);
 
 	pp = NULL;
 	cp = g_new_consumer(gp);
 	error = g_attach(cp, bpp);
 	if (error != 0) {
 		if (req != NULL) {
 			gctl_error(req, "Cannot attach to %s (error=%d).",
 			    bpp->name, error);
 		} else {
 			G_ELI_DEBUG(1, "Cannot attach to %s (error=%d).",
 			    bpp->name, error);
 		}
 		goto failed;
 	}
 	/*
 	 * Keep provider open all the time, so we can run critical tasks,
 	 * like Master Keys deletion, without wondering if we can open
 	 * provider or not.
 	 * We don't open provider for writing only when user requested read-only
 	 * access.
 	 */
 	dcw = (sc->sc_flags & G_ELI_FLAG_RO) ? 0 : 1;
 	error = g_access(cp, 1, dcw, 1);
 	if (error != 0) {
 		if (req != NULL) {
 			gctl_error(req, "Cannot access %s (error=%d).",
 			    bpp->name, error);
 		} else {
 			G_ELI_DEBUG(1, "Cannot access %s (error=%d).",
 			    bpp->name, error);
 		}
 		goto failed;
 	}
 
 	/*
 	 * Remember the keys in our softc structure.
 	 */
 	g_eli_mkey_propagate(sc, mkey);
 
 	LIST_INIT(&sc->sc_workers);
 
 	threads = g_eli_threads;
 	if (threads == 0)
 		threads = mp_ncpus;
 	sc->sc_cpubind = (mp_ncpus > 1 && threads == mp_ncpus);
 	for (i = 0; i < threads; i++) {
 		if (g_eli_cpu_is_disabled(i)) {
 			G_ELI_DEBUG(1, "%s: CPU %u disabled, skipping.",
 			    bpp->name, i);
 			continue;
 		}
 		wr = malloc(sizeof(*wr), M_ELI, M_WAITOK | M_ZERO);
 		wr->w_softc = sc;
 		wr->w_number = i;
 		wr->w_active = TRUE;
 
 		error = g_eli_newsession(wr);
 		if (error != 0) {
 			free(wr, M_ELI);
 			if (req != NULL) {
 				gctl_error(req, "Cannot set up crypto session "
 				    "for %s (error=%d).", bpp->name, error);
 			} else {
 				G_ELI_DEBUG(1, "Cannot set up crypto session "
 				    "for %s (error=%d).", bpp->name, error);
 			}
 			goto failed;
 		}
 
 		error = kproc_create(g_eli_worker, wr, &wr->w_proc, 0, 0,
 		    "g_eli[%u] %s", i, bpp->name);
 		if (error != 0) {
 			g_eli_freesession(wr);
 			free(wr, M_ELI);
 			if (req != NULL) {
 				gctl_error(req, "Cannot create kernel thread "
 				    "for %s (error=%d).", bpp->name, error);
 			} else {
 				G_ELI_DEBUG(1, "Cannot create kernel thread "
 				    "for %s (error=%d).", bpp->name, error);
 			}
 			goto failed;
 		}
 		LIST_INSERT_HEAD(&sc->sc_workers, wr, w_next);
 	}
 
 	/*
 	 * Create decrypted provider.
 	 */
 	pp = g_new_providerf(gp, "%s%s", bpp->name, G_ELI_SUFFIX);
 	pp->mediasize = sc->sc_mediasize;
 	pp->sectorsize = sc->sc_sectorsize;
 
 	g_error_provider(pp, 0);
 
 	G_ELI_DEBUG(0, "Device %s created.", pp->name);
 	G_ELI_DEBUG(0, "Encryption: %s %u", g_eli_algo2str(sc->sc_ealgo),
 	    sc->sc_ekeylen);
 	switch (sc->sc_ealgo) {
 	case CRYPTO_3DES_CBC:
 		gone_in(13,
 		    "support for GEOM_ELI volumes encrypted with 3des");
 		break;
 	case CRYPTO_BLF_CBC:
 		gone_in(13,
 		    "support for GEOM_ELI volumes encrypted with blowfish");
 		break;
 	}
 	if (sc->sc_flags & G_ELI_FLAG_AUTH) {
 		G_ELI_DEBUG(0, " Integrity: %s", g_eli_algo2str(sc->sc_aalgo));
 		switch (sc->sc_aalgo) {
 		case CRYPTO_MD5_HMAC:
 			gone_in(13,
 		    "support for GEOM_ELI volumes authenticated with hmac/md5");
 			break;
 		}
 	}
 	G_ELI_DEBUG(0, "    Crypto: %s",
 	    sc->sc_crypto == G_ELI_CRYPTO_SW ? "software" : "hardware");
 	return (gp);
 failed:
 	mtx_lock(&sc->sc_queue_mtx);
 	sc->sc_flags |= G_ELI_FLAG_DESTROY;
 	wakeup(sc);
 	/*
 	 * Wait for kernel threads self destruction.
 	 */
 	while (!LIST_EMPTY(&sc->sc_workers)) {
 		msleep(&sc->sc_workers, &sc->sc_queue_mtx, PRIBIO,
 		    "geli:destroy", 0);
 	}
 	mtx_destroy(&sc->sc_queue_mtx);
 	if (cp->provider != NULL) {
 		if (cp->acr == 1)
 			g_access(cp, -1, -dcw, -1);
 		g_detach(cp);
 	}
 	g_destroy_consumer(cp);
 	g_destroy_geom(gp);
 	g_eli_key_destroy(sc);
 	bzero(sc, sizeof(*sc));
 	free(sc, M_ELI);
 	return (NULL);
 }
 
 int
 g_eli_destroy(struct g_eli_softc *sc, boolean_t force)
 {
 	struct g_geom *gp;
 	struct g_provider *pp;
 
 	g_topology_assert();
 
 	if (sc == NULL)
 		return (ENXIO);
 
 	gp = sc->sc_geom;
 	pp = LIST_FIRST(&gp->provider);
 	if (pp != NULL && (pp->acr != 0 || pp->acw != 0 || pp->ace != 0)) {
 		if (force) {
 			G_ELI_DEBUG(1, "Device %s is still open, so it "
 			    "cannot be definitely removed.", pp->name);
 			sc->sc_flags |= G_ELI_FLAG_RW_DETACH;
 			gp->access = g_eli_access;
 			g_wither_provider(pp, ENXIO);
 			return (EBUSY);
 		} else {
 			G_ELI_DEBUG(1,
 			    "Device %s is still open (r%dw%de%d).", pp->name,
 			    pp->acr, pp->acw, pp->ace);
 			return (EBUSY);
 		}
 	}
 
 	mtx_lock(&sc->sc_queue_mtx);
 	sc->sc_flags |= G_ELI_FLAG_DESTROY;
 	wakeup(sc);
 	while (!LIST_EMPTY(&sc->sc_workers)) {
 		msleep(&sc->sc_workers, &sc->sc_queue_mtx, PRIBIO,
 		    "geli:destroy", 0);
 	}
 	mtx_destroy(&sc->sc_queue_mtx);
 	gp->softc = NULL;
 	g_eli_key_destroy(sc);
 	bzero(sc, sizeof(*sc));
 	free(sc, M_ELI);
 
 	G_ELI_DEBUG(0, "Device %s destroyed.", gp->name);
 	g_wither_geom_close(gp, ENXIO);
 
 	return (0);
 }
 
 static int
 g_eli_destroy_geom(struct gctl_req *req __unused,
     struct g_class *mp __unused, struct g_geom *gp)
 {
 	struct g_eli_softc *sc;
 
 	sc = gp->softc;
 	return (g_eli_destroy(sc, FALSE));
 }
 
 static int
 g_eli_keyfiles_load(struct hmac_ctx *ctx, const char *provider)
 {
 	u_char *keyfile, *data;
 	char *file, name[64];
 	size_t size;
 	int i;
 
 	for (i = 0; ; i++) {
 		snprintf(name, sizeof(name), "%s:geli_keyfile%d", provider, i);
 		keyfile = preload_search_by_type(name);
 		if (keyfile == NULL && i == 0) {
 			/*
 			 * If there is only one keyfile, allow simpler name.
 			 */
 			snprintf(name, sizeof(name), "%s:geli_keyfile", provider);
 			keyfile = preload_search_by_type(name);
 		}
 		if (keyfile == NULL)
 			return (i);	/* Return number of loaded keyfiles. */
 		data = preload_fetch_addr(keyfile);
 		if (data == NULL) {
 			G_ELI_DEBUG(0, "Cannot find key file data for %s.",
 			    name);
 			return (0);
 		}
 		size = preload_fetch_size(keyfile);
 		if (size == 0) {
 			G_ELI_DEBUG(0, "Cannot find key file size for %s.",
 			    name);
 			return (0);
 		}
 		file = preload_search_info(keyfile, MODINFO_NAME);
 		if (file == NULL) {
 			G_ELI_DEBUG(0, "Cannot find key file name for %s.",
 			    name);
 			return (0);
 		}
 		G_ELI_DEBUG(1, "Loaded keyfile %s for %s (type: %s).", file,
 		    provider, name);
 		g_eli_crypto_hmac_update(ctx, data, size);
 	}
 }
 
 static void
 g_eli_keyfiles_clear(const char *provider)
 {
 	u_char *keyfile, *data;
 	char name[64];
 	size_t size;
 	int i;
 
 	for (i = 0; ; i++) {
 		snprintf(name, sizeof(name), "%s:geli_keyfile%d", provider, i);
 		keyfile = preload_search_by_type(name);
 		if (keyfile == NULL)
 			return;
 		data = preload_fetch_addr(keyfile);
 		size = preload_fetch_size(keyfile);
 		if (data != NULL && size != 0)
 			bzero(data, size);
 	}
 }
 
 /*
  * Tasting is only made on boot.
  * We detect providers which should be attached before root is mounted.
  */
 static struct g_geom *
 g_eli_taste(struct g_class *mp, struct g_provider *pp, int flags __unused)
 {
 	struct g_eli_metadata md;
 	struct g_geom *gp;
 	struct hmac_ctx ctx;
 	char passphrase[256];
 	u_char key[G_ELI_USERKEYLEN], mkey[G_ELI_DATAIVKEYLEN];
 	u_int i, nkey, nkeyfiles, tries, showpass;
 	int error;
         struct keybuf *keybuf;
 
 	g_trace(G_T_TOPOLOGY, "%s(%s, %s)", __func__, mp->name, pp->name);
 	g_topology_assert();
 
 	if (root_mounted() || g_eli_tries == 0)
 		return (NULL);
 
 	G_ELI_DEBUG(3, "Tasting %s.", pp->name);
 
 	error = g_eli_read_metadata(mp, pp, &md);
 	if (error != 0)
 		return (NULL);
 	gp = NULL;
 
 	if (strcmp(md.md_magic, G_ELI_MAGIC) != 0)
 		return (NULL);
 	if (md.md_version > G_ELI_VERSION) {
 		printf("geom_eli.ko module is too old to handle %s.\n",
 		    pp->name);
 		return (NULL);
 	}
 	if (md.md_provsize != pp->mediasize)
 		return (NULL);
 	/* Should we attach it on boot? */
 	if (!(md.md_flags & G_ELI_FLAG_BOOT) &&
 	    !(md.md_flags & G_ELI_FLAG_GELIBOOT))
 		return (NULL);
 	if (md.md_keys == 0x00) {
 		G_ELI_DEBUG(0, "No valid keys on %s.", pp->name);
 		return (NULL);
 	}
 	if (md.md_iterations == -1) {
 		/* If there is no passphrase, we try only once. */
 		tries = 1;
 	} else {
 		/* Ask for the passphrase no more than g_eli_tries times. */
 		tries = g_eli_tries;
 	}
 
         if ((keybuf = get_keybuf()) != NULL) {
                 /* Scan the key buffer, try all GELI keys. */
                 for (i = 0; i < keybuf->kb_nents; i++) {
                          if (keybuf->kb_ents[i].ke_type == KEYBUF_TYPE_GELI) {
                                  memcpy(key, keybuf->kb_ents[i].ke_data,
                                      sizeof(key));
 
                                  if (g_eli_mkey_decrypt_any(&md, key,
                                      mkey, &nkey) == 0 ) {
                                          explicit_bzero(key, sizeof(key));
                                          goto have_key;
                                  }
                          }
                 }
         }
 
         for (i = 0; i <= tries; i++) {
                 g_eli_crypto_hmac_init(&ctx, NULL, 0);
 
                 /*
                  * Load all key files.
                  */
                 nkeyfiles = g_eli_keyfiles_load(&ctx, pp->name);
 
                 if (nkeyfiles == 0 && md.md_iterations == -1) {
                         /*
                          * No key files and no passphrase, something is
                          * definitely wrong here.
                          * geli(8) doesn't allow for such situation, so assume
                          * that there was really no passphrase and in that case
                          * key files are no properly defined in loader.conf.
                          */
                         G_ELI_DEBUG(0,
                             "Found no key files in loader.conf for %s.",
                             pp->name);
                         return (NULL);
                 }
 
                 /* Ask for the passphrase if defined. */
                 if (md.md_iterations >= 0) {
                         /* Try first with cached passphrase. */
                         if (i == 0) {
                                 if (!g_eli_boot_passcache)
                                         continue;
                                 memcpy(passphrase, cached_passphrase,
                                     sizeof(passphrase));
                         } else {
                                 printf("Enter passphrase for %s: ", pp->name);
 				showpass = g_eli_visible_passphrase;
 				if ((md.md_flags & G_ELI_FLAG_GELIDISPLAYPASS) != 0)
 					showpass = GETS_ECHOPASS;
                                 cngets(passphrase, sizeof(passphrase),
 				    showpass);
                                 memcpy(cached_passphrase, passphrase,
                                     sizeof(passphrase));
                         }
                 }
 
                 /*
                  * Prepare Derived-Key from the user passphrase.
                  */
                 if (md.md_iterations == 0) {
                         g_eli_crypto_hmac_update(&ctx, md.md_salt,
                             sizeof(md.md_salt));
                         g_eli_crypto_hmac_update(&ctx, passphrase,
                             strlen(passphrase));
                         explicit_bzero(passphrase, sizeof(passphrase));
                 } else if (md.md_iterations > 0) {
                         u_char dkey[G_ELI_USERKEYLEN];
 
                         pkcs5v2_genkey(dkey, sizeof(dkey), md.md_salt,
                             sizeof(md.md_salt), passphrase, md.md_iterations);
                         bzero(passphrase, sizeof(passphrase));
                         g_eli_crypto_hmac_update(&ctx, dkey, sizeof(dkey));
                         explicit_bzero(dkey, sizeof(dkey));
                 }
 
                 g_eli_crypto_hmac_final(&ctx, key, 0);
 
                 /*
                  * Decrypt Master-Key.
                  */
                 error = g_eli_mkey_decrypt_any(&md, key, mkey, &nkey);
                 bzero(key, sizeof(key));
                 if (error == -1) {
                         if (i == tries) {
                                 G_ELI_DEBUG(0,
                                     "Wrong key for %s. No tries left.",
                                     pp->name);
                                 g_eli_keyfiles_clear(pp->name);
                                 return (NULL);
                         }
                         if (i > 0) {
                                 G_ELI_DEBUG(0,
                                     "Wrong key for %s. Tries left: %u.",
                                     pp->name, tries - i);
                         }
                         /* Try again. */
                         continue;
                 } else if (error > 0) {
                         G_ELI_DEBUG(0,
                             "Cannot decrypt Master Key for %s (error=%d).",
                             pp->name, error);
                         g_eli_keyfiles_clear(pp->name);
                         return (NULL);
                 }
                 g_eli_keyfiles_clear(pp->name);
                 G_ELI_DEBUG(1, "Using Master Key %u for %s.", nkey, pp->name);
                 break;
         }
 have_key:
 
 	/*
 	 * We have correct key, let's attach provider.
 	 */
 	gp = g_eli_create(NULL, mp, pp, &md, mkey, nkey);
 	bzero(mkey, sizeof(mkey));
 	bzero(&md, sizeof(md));
 	if (gp == NULL) {
 		G_ELI_DEBUG(0, "Cannot create device %s%s.", pp->name,
 		    G_ELI_SUFFIX);
 		return (NULL);
 	}
 	return (gp);
 }
 
 static void
 g_eli_dumpconf(struct sbuf *sb, const char *indent, struct g_geom *gp,
     struct g_consumer *cp, struct g_provider *pp)
 {
 	struct g_eli_softc *sc;
 
 	g_topology_assert();
 	sc = gp->softc;
 	if (sc == NULL)
 		return;
 	if (pp != NULL || cp != NULL)
 		return;	/* Nothing here. */
 
 	sbuf_printf(sb, "%s<KeysTotal>%ju</KeysTotal>\n", indent,
 	    (uintmax_t)sc->sc_ekeys_total);
 	sbuf_printf(sb, "%s<KeysAllocated>%ju</KeysAllocated>\n", indent,
 	    (uintmax_t)sc->sc_ekeys_allocated);
 	sbuf_printf(sb, "%s<Flags>", indent);
 	if (sc->sc_flags == 0)
 		sbuf_cat(sb, "NONE");
 	else {
 		int first = 1;
 
 #define ADD_FLAG(flag, name)	do {					\
 	if (sc->sc_flags & (flag)) {					\
 		if (!first)						\
 			sbuf_cat(sb, ", ");				\
 		else							\
 			first = 0;					\
 		sbuf_cat(sb, name);					\
 	}								\
 } while (0)
 		ADD_FLAG(G_ELI_FLAG_SUSPEND, "SUSPEND");
 		ADD_FLAG(G_ELI_FLAG_SINGLE_KEY, "SINGLE-KEY");
 		ADD_FLAG(G_ELI_FLAG_NATIVE_BYTE_ORDER, "NATIVE-BYTE-ORDER");
 		ADD_FLAG(G_ELI_FLAG_ONETIME, "ONETIME");
 		ADD_FLAG(G_ELI_FLAG_BOOT, "BOOT");
 		ADD_FLAG(G_ELI_FLAG_WO_DETACH, "W-DETACH");
 		ADD_FLAG(G_ELI_FLAG_RW_DETACH, "RW-DETACH");
 		ADD_FLAG(G_ELI_FLAG_AUTH, "AUTH");
 		ADD_FLAG(G_ELI_FLAG_WOPEN, "W-OPEN");
 		ADD_FLAG(G_ELI_FLAG_DESTROY, "DESTROY");
 		ADD_FLAG(G_ELI_FLAG_RO, "READ-ONLY");
 		ADD_FLAG(G_ELI_FLAG_NODELETE, "NODELETE");
 		ADD_FLAG(G_ELI_FLAG_GELIBOOT, "GELIBOOT");
 		ADD_FLAG(G_ELI_FLAG_GELIDISPLAYPASS, "GELIDISPLAYPASS");
 		ADD_FLAG(G_ELI_FLAG_AUTORESIZE, "AUTORESIZE");
 #undef  ADD_FLAG
 	}
 	sbuf_cat(sb, "</Flags>\n");
 
 	if (!(sc->sc_flags & G_ELI_FLAG_ONETIME)) {
 		sbuf_printf(sb, "%s<UsedKey>%u</UsedKey>\n", indent,
 		    sc->sc_nkey);
 	}
 	sbuf_printf(sb, "%s<Version>%u</Version>\n", indent, sc->sc_version);
 	sbuf_printf(sb, "%s<Crypto>", indent);
 	switch (sc->sc_crypto) {
 	case G_ELI_CRYPTO_HW:
 		sbuf_cat(sb, "hardware");
 		break;
 	case G_ELI_CRYPTO_SW:
 		sbuf_cat(sb, "software");
 		break;
 	default:
 		sbuf_cat(sb, "UNKNOWN");
 		break;
 	}
 	sbuf_cat(sb, "</Crypto>\n");
 	if (sc->sc_flags & G_ELI_FLAG_AUTH) {
 		sbuf_printf(sb,
 		    "%s<AuthenticationAlgorithm>%s</AuthenticationAlgorithm>\n",
 		    indent, g_eli_algo2str(sc->sc_aalgo));
 	}
 	sbuf_printf(sb, "%s<KeyLength>%u</KeyLength>\n", indent,
 	    sc->sc_ekeylen);
 	sbuf_printf(sb, "%s<EncryptionAlgorithm>%s</EncryptionAlgorithm>\n",
 	    indent, g_eli_algo2str(sc->sc_ealgo));
 	sbuf_printf(sb, "%s<State>%s</State>\n", indent,
 	    (sc->sc_flags & G_ELI_FLAG_SUSPEND) ? "SUSPENDED" : "ACTIVE");
 }
 
 static void
 g_eli_shutdown_pre_sync(void *arg, int howto)
 {
 	struct g_class *mp;
 	struct g_geom *gp, *gp2;
 	struct g_provider *pp;
 	struct g_eli_softc *sc;
 	int error;
 
 	mp = arg;
 	g_topology_lock();
 	LIST_FOREACH_SAFE(gp, &mp->geom, geom, gp2) {
 		sc = gp->softc;
 		if (sc == NULL)
 			continue;
 		pp = LIST_FIRST(&gp->provider);
 		KASSERT(pp != NULL, ("No provider? gp=%p (%s)", gp, gp->name));
 		if (pp->acr + pp->acw + pp->ace == 0)
 			error = g_eli_destroy(sc, TRUE);
 		else {
 			sc->sc_flags |= G_ELI_FLAG_RW_DETACH;
 			gp->access = g_eli_access;
 		}
 	}
 	g_topology_unlock();
 }
 
 static void
 g_eli_init(struct g_class *mp)
 {
 
 	g_eli_pre_sync = EVENTHANDLER_REGISTER(shutdown_pre_sync,
 	    g_eli_shutdown_pre_sync, mp, SHUTDOWN_PRI_FIRST);
 	if (g_eli_pre_sync == NULL)
 		G_ELI_DEBUG(0, "Warning! Cannot register shutdown event.");
 }
 
 static void
 g_eli_fini(struct g_class *mp)
 {
 
 	if (g_eli_pre_sync != NULL)
 		EVENTHANDLER_DEREGISTER(shutdown_pre_sync, g_eli_pre_sync);
 }
 
 DECLARE_GEOM_CLASS(g_eli_class, g_eli);
 MODULE_DEPEND(g_eli, crypto, 1, 1, 1);
 MODULE_VERSION(geom_eli, 0);
Index: head/sys/geom/eli/g_eli.h
===================================================================
--- head/sys/geom/eli/g_eli.h	(revision 359373)
+++ head/sys/geom/eli/g_eli.h	(revision 359374)
@@ -1,723 +1,743 @@
 /*-
  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
  *
  * Copyright (c) 2005-2019 Pawel Jakub Dawidek <pawel@dawidek.net>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
  * $FreeBSD$
  */
 
 #ifndef	_G_ELI_H_
 #define	_G_ELI_H_
 
 #include <sys/endian.h>
 #include <sys/errno.h>
 #include <sys/malloc.h>
 #include <crypto/sha2/sha256.h>
 #include <crypto/sha2/sha512.h>
 #include <opencrypto/cryptodev.h>
 #ifdef _KERNEL
 #include <sys/bio.h>
 #include <sys/libkern.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <geom/geom.h>
 #include <crypto/intake.h>
 #else
 #include <assert.h>
 #include <stdio.h>
 #include <string.h>
 #include <strings.h>
 #endif
 #include <sys/queue.h>
 #include <sys/tree.h>
 #ifndef _OpenSSL_
 #include <sys/md5.h>
 #endif
 
 #define	G_ELI_CLASS_NAME	"ELI"
 #define	G_ELI_MAGIC		"GEOM::ELI"
 #define	G_ELI_SUFFIX		".eli"
 
 /*
  * Version history:
  * 0 - Initial version number.
  * 1 - Added data authentication support (md_aalgo field and
  *     G_ELI_FLAG_AUTH flag).
  * 2 - Added G_ELI_FLAG_READONLY.
  * 3 - Added 'configure' subcommand.
  * 4 - IV is generated from offset converted to little-endian
  *     (the G_ELI_FLAG_NATIVE_BYTE_ORDER flag will be set for older versions).
  * 5 - Added multiple encrypton keys and AES-XTS support.
  * 6 - Fixed usage of multiple keys for authenticated providers (the
  *     G_ELI_FLAG_FIRST_KEY flag will be set for older versions).
  * 7 - Encryption keys are now generated from the Data Key and not from the
  *     IV Key (the G_ELI_FLAG_ENC_IVKEY flag will be set for older versions).
  */
 #define	G_ELI_VERSION_00	0
 #define	G_ELI_VERSION_01	1
 #define	G_ELI_VERSION_02	2
 #define	G_ELI_VERSION_03	3
 #define	G_ELI_VERSION_04	4
 #define	G_ELI_VERSION_05	5
 #define	G_ELI_VERSION_06	6
 #define	G_ELI_VERSION_07	7
 #define	G_ELI_VERSION		G_ELI_VERSION_07
 
 /* ON DISK FLAGS. */
 /* Use random, onetime keys. */
 #define	G_ELI_FLAG_ONETIME		0x00000001
 /* Ask for the passphrase from the kernel, before mounting root. */
 #define	G_ELI_FLAG_BOOT			0x00000002
 /* Detach on last close, if we were open for writing. */
 #define	G_ELI_FLAG_WO_DETACH		0x00000004
 /* Detach on last close. */
 #define	G_ELI_FLAG_RW_DETACH		0x00000008
 /* Provide data authentication. */
 #define	G_ELI_FLAG_AUTH			0x00000010
 /* Provider is read-only, we should deny all write attempts. */
 #define	G_ELI_FLAG_RO			0x00000020
 /* Don't pass through BIO_DELETE requests. */
 #define	G_ELI_FLAG_NODELETE		0x00000040
 /* This GELI supports GELIBoot */
 #define	G_ELI_FLAG_GELIBOOT		0x00000080
 /* Hide passphrase length in GELIboot. */
 #define	G_ELI_FLAG_GELIDISPLAYPASS	0x00000100
 /* Expand provider automatically. */
 #define	G_ELI_FLAG_AUTORESIZE		0x00000200
 
 /* RUNTIME FLAGS. */
 /* Provider was open for writing. */
 #define	G_ELI_FLAG_WOPEN		0x00010000
 /* Destroy device. */
 #define	G_ELI_FLAG_DESTROY		0x00020000
 /* Provider uses native byte-order for IV generation. */
 #define	G_ELI_FLAG_NATIVE_BYTE_ORDER	0x00040000
 /* Provider uses single encryption key. */
 #define	G_ELI_FLAG_SINGLE_KEY		0x00080000
 /* Device suspended. */
 #define	G_ELI_FLAG_SUSPEND		0x00100000
 /* Provider uses first encryption key. */
 #define	G_ELI_FLAG_FIRST_KEY		0x00200000
 /* Provider uses IV-Key for encryption key generation. */
 #define	G_ELI_FLAG_ENC_IVKEY		0x00400000
 
 #define	G_ELI_NEW_BIO	255
 
 #define	SHA512_MDLEN		64
 #define	G_ELI_AUTH_SECKEYLEN	SHA256_DIGEST_LENGTH
 
 #define	G_ELI_MAXMKEYS		2
 #define	G_ELI_MAXKEYLEN		64
 #define	G_ELI_USERKEYLEN	G_ELI_MAXKEYLEN
 #define	G_ELI_DATAKEYLEN	G_ELI_MAXKEYLEN
 #define	G_ELI_AUTHKEYLEN	G_ELI_MAXKEYLEN
 #define	G_ELI_IVKEYLEN		G_ELI_MAXKEYLEN
 #define	G_ELI_SALTLEN		64
 #define	G_ELI_DATAIVKEYLEN	(G_ELI_DATAKEYLEN + G_ELI_IVKEYLEN)
 /* Data-Key, IV-Key, HMAC_SHA512(Derived-Key, Data-Key+IV-Key) */
 #define	G_ELI_MKEYLEN		(G_ELI_DATAIVKEYLEN + SHA512_MDLEN)
 #define	G_ELI_OVERWRITES	5
 /* Switch data encryption key every 2^20 blocks. */
 #define	G_ELI_KEY_SHIFT		20
 
 #define	G_ELI_CRYPTO_UNKNOWN	0
 #define	G_ELI_CRYPTO_HW		1
 #define	G_ELI_CRYPTO_SW		2
 
 #ifdef _KERNEL
 #if (MAX_KEY_BYTES < G_ELI_DATAIVKEYLEN)
 #error "MAX_KEY_BYTES is less than G_ELI_DATAKEYLEN"
 #endif
 
 extern int g_eli_debug;
 extern u_int g_eli_overwrites;
 extern u_int g_eli_batch;
 
 #define	G_ELI_DEBUG(lvl, ...) \
     _GEOM_DEBUG("GEOM_ELI", g_eli_debug, (lvl), NULL, __VA_ARGS__)
 #define	G_ELI_LOGREQ(lvl, bp, ...) \
     _GEOM_DEBUG("GEOM_ELI", g_eli_debug, (lvl), (bp), __VA_ARGS__)
 
 struct g_eli_worker {
 	struct g_eli_softc	*w_softc;
 	struct proc		*w_proc;
+	void			*w_first_key;
 	u_int			 w_number;
 	crypto_session_t	 w_sid;
 	boolean_t		 w_active;
 	LIST_ENTRY(g_eli_worker) w_next;
 };
 
 #endif	/* _KERNEL */
 
 struct g_eli_softc {
 	struct g_geom	*sc_geom;
 	u_int		 sc_version;
 	u_int		 sc_crypto;
 	uint8_t		 sc_mkey[G_ELI_DATAIVKEYLEN];
 	uint8_t		 sc_ekey[G_ELI_DATAKEYLEN];
 	TAILQ_HEAD(, g_eli_key) sc_ekeys_queue;
 	RB_HEAD(g_eli_key_tree, g_eli_key) sc_ekeys_tree;
 	struct mtx	 sc_ekeys_lock;
 	uint64_t	 sc_ekeys_total;
 	uint64_t	 sc_ekeys_allocated;
 	u_int		 sc_ealgo;
 	u_int		 sc_ekeylen;
 	uint8_t		 sc_akey[G_ELI_AUTHKEYLEN];
 	u_int		 sc_aalgo;
 	u_int		 sc_akeylen;
 	u_int		 sc_alen;
 	SHA256_CTX	 sc_akeyctx;
 	uint8_t		 sc_ivkey[G_ELI_IVKEYLEN];
 	SHA256_CTX	 sc_ivctx;
 	int		 sc_nkey;
 	uint32_t	 sc_flags;
 	int		 sc_inflight;
 	off_t		 sc_mediasize;
 	size_t		 sc_sectorsize;
 	off_t		 sc_provsize;
 	u_int		 sc_bytes_per_sector;
 	u_int		 sc_data_per_sector;
 #ifndef _KERNEL
 	int		 sc_cpubind;
 #else /* _KERNEL */
 	boolean_t	 sc_cpubind;
 
 	/* Only for software cryptography. */
 	struct bio_queue_head sc_queue;
 	struct mtx	 sc_queue_mtx;
 	LIST_HEAD(, g_eli_worker) sc_workers;
 #endif /* _KERNEL */
 };
 #define	sc_name		 sc_geom->name
 
 #define	G_ELI_KEY_MAGIC	0xe11341c
 
 struct g_eli_key {
 	/* Key value, must be first in the structure. */
 	uint8_t		gek_key[G_ELI_DATAKEYLEN];
 	/* Magic. */
 	int		gek_magic;
 	/* Key number. */
 	uint64_t	gek_keyno;
 	/* Reference counter. */
 	int		gek_count;
 	/* Keeps keys sorted by most recent use. */
 	TAILQ_ENTRY(g_eli_key) gek_next;
 	/* Keeps keys sorted by number. */
 	RB_ENTRY(g_eli_key) gek_link;
 };
 
 struct g_eli_metadata {
 	char		md_magic[16];	/* Magic value. */
 	uint32_t	md_version;	/* Version number. */
 	uint32_t	md_flags;	/* Additional flags. */
 	uint16_t	md_ealgo;	/* Encryption algorithm. */
 	uint16_t	md_keylen;	/* Key length. */
 	uint16_t	md_aalgo;	/* Authentication algorithm. */
 	uint64_t	md_provsize;	/* Provider's size. */
 	uint32_t	md_sectorsize;	/* Sector size. */
 	uint8_t		md_keys;	/* Available keys. */
 	int32_t		md_iterations;	/* Number of iterations for PKCS#5v2. */
 	uint8_t		md_salt[G_ELI_SALTLEN]; /* Salt. */
 			/* Encrypted master key (IV-key, Data-key, HMAC). */
 	uint8_t		md_mkeys[G_ELI_MAXMKEYS * G_ELI_MKEYLEN];
 	u_char		md_hash[16];	/* MD5 hash. */
 } __packed;
 #ifndef _OpenSSL_
 static __inline void
 eli_metadata_encode_v0(struct g_eli_metadata *md, u_char **datap)
 {
 	u_char *p;
 
 	p = *datap;
 	le32enc(p, md->md_flags);	p += sizeof(md->md_flags);
 	le16enc(p, md->md_ealgo);	p += sizeof(md->md_ealgo);
 	le16enc(p, md->md_keylen);	p += sizeof(md->md_keylen);
 	le64enc(p, md->md_provsize);	p += sizeof(md->md_provsize);
 	le32enc(p, md->md_sectorsize);	p += sizeof(md->md_sectorsize);
 	*p = md->md_keys;		p += sizeof(md->md_keys);
 	le32enc(p, md->md_iterations);	p += sizeof(md->md_iterations);
 	bcopy(md->md_salt, p, sizeof(md->md_salt)); p += sizeof(md->md_salt);
 	bcopy(md->md_mkeys, p, sizeof(md->md_mkeys)); p += sizeof(md->md_mkeys);
 	*datap = p;
 }
 static __inline void
 eli_metadata_encode_v1v2v3v4v5v6v7(struct g_eli_metadata *md, u_char **datap)
 {
 	u_char *p;
 
 	p = *datap;
 	le32enc(p, md->md_flags);	p += sizeof(md->md_flags);
 	le16enc(p, md->md_ealgo);	p += sizeof(md->md_ealgo);
 	le16enc(p, md->md_keylen);	p += sizeof(md->md_keylen);
 	le16enc(p, md->md_aalgo);	p += sizeof(md->md_aalgo);
 	le64enc(p, md->md_provsize);	p += sizeof(md->md_provsize);
 	le32enc(p, md->md_sectorsize);	p += sizeof(md->md_sectorsize);
 	*p = md->md_keys;		p += sizeof(md->md_keys);
 	le32enc(p, md->md_iterations);	p += sizeof(md->md_iterations);
 	bcopy(md->md_salt, p, sizeof(md->md_salt)); p += sizeof(md->md_salt);
 	bcopy(md->md_mkeys, p, sizeof(md->md_mkeys)); p += sizeof(md->md_mkeys);
 	*datap = p;
 }
 static __inline void
 eli_metadata_encode(struct g_eli_metadata *md, u_char *data)
 {
 	uint32_t hash[4];
 	MD5_CTX ctx;
 	u_char *p;
 
 	p = data;
 	bcopy(md->md_magic, p, sizeof(md->md_magic));
 	p += sizeof(md->md_magic);
 	le32enc(p, md->md_version);
 	p += sizeof(md->md_version);
 	switch (md->md_version) {
 	case G_ELI_VERSION_00:
 		eli_metadata_encode_v0(md, &p);
 		break;
 	case G_ELI_VERSION_01:
 	case G_ELI_VERSION_02:
 	case G_ELI_VERSION_03:
 	case G_ELI_VERSION_04:
 	case G_ELI_VERSION_05:
 	case G_ELI_VERSION_06:
 	case G_ELI_VERSION_07:
 		eli_metadata_encode_v1v2v3v4v5v6v7(md, &p);
 		break;
 	default:
 #ifdef _KERNEL
 		panic("%s: Unsupported version %u.", __func__,
 		    (u_int)md->md_version);
 #else
 		assert(!"Unsupported metadata version.");
 #endif
 	}
 	MD5Init(&ctx);
 	MD5Update(&ctx, data, p - data);
 	MD5Final((void *)hash, &ctx);
 	bcopy(hash, md->md_hash, sizeof(md->md_hash));
 	bcopy(md->md_hash, p, sizeof(md->md_hash));
 }
 static __inline int
 eli_metadata_decode_v0(const u_char *data, struct g_eli_metadata *md)
 {
 	uint32_t hash[4];
 	MD5_CTX ctx;
 	const u_char *p;
 
 	p = data + sizeof(md->md_magic) + sizeof(md->md_version);
 	md->md_flags = le32dec(p);	p += sizeof(md->md_flags);
 	md->md_ealgo = le16dec(p);	p += sizeof(md->md_ealgo);
 	md->md_keylen = le16dec(p);	p += sizeof(md->md_keylen);
 	md->md_provsize = le64dec(p);	p += sizeof(md->md_provsize);
 	md->md_sectorsize = le32dec(p);	p += sizeof(md->md_sectorsize);
 	md->md_keys = *p;		p += sizeof(md->md_keys);
 	md->md_iterations = le32dec(p);	p += sizeof(md->md_iterations);
 	bcopy(p, md->md_salt, sizeof(md->md_salt)); p += sizeof(md->md_salt);
 	bcopy(p, md->md_mkeys, sizeof(md->md_mkeys)); p += sizeof(md->md_mkeys);
 	MD5Init(&ctx);
 	MD5Update(&ctx, data, p - data);
 	MD5Final((void *)hash, &ctx);
 	bcopy(hash, md->md_hash, sizeof(md->md_hash));
 	if (bcmp(md->md_hash, p, 16) != 0)
 		return (EINVAL);
 	return (0);
 }
 
 static __inline int
 eli_metadata_decode_v1v2v3v4v5v6v7(const u_char *data, struct g_eli_metadata *md)
 {
 	uint32_t hash[4];
 	MD5_CTX ctx;
 	const u_char *p;
 
 	p = data + sizeof(md->md_magic) + sizeof(md->md_version);
 	md->md_flags = le32dec(p);	p += sizeof(md->md_flags);
 	md->md_ealgo = le16dec(p);	p += sizeof(md->md_ealgo);
 	md->md_keylen = le16dec(p);	p += sizeof(md->md_keylen);
 	md->md_aalgo = le16dec(p);	p += sizeof(md->md_aalgo);
 	md->md_provsize = le64dec(p);	p += sizeof(md->md_provsize);
 	md->md_sectorsize = le32dec(p);	p += sizeof(md->md_sectorsize);
 	md->md_keys = *p;		p += sizeof(md->md_keys);
 	md->md_iterations = le32dec(p);	p += sizeof(md->md_iterations);
 	bcopy(p, md->md_salt, sizeof(md->md_salt)); p += sizeof(md->md_salt);
 	bcopy(p, md->md_mkeys, sizeof(md->md_mkeys)); p += sizeof(md->md_mkeys);
 	MD5Init(&ctx);
 	MD5Update(&ctx, data, p - data);
 	MD5Final((void *)hash, &ctx);
 	bcopy(hash, md->md_hash, sizeof(md->md_hash));
 	if (bcmp(md->md_hash, p, 16) != 0)
 		return (EINVAL);
 	return (0);
 }
 static __inline int
 eli_metadata_decode(const u_char *data, struct g_eli_metadata *md)
 {
 	int error;
 
 	bcopy(data, md->md_magic, sizeof(md->md_magic));
 	if (strcmp(md->md_magic, G_ELI_MAGIC) != 0)
 		return (EINVAL);
 	md->md_version = le32dec(data + sizeof(md->md_magic));
 	switch (md->md_version) {
 	case G_ELI_VERSION_00:
 		error = eli_metadata_decode_v0(data, md);
 		break;
 	case G_ELI_VERSION_01:
 	case G_ELI_VERSION_02:
 	case G_ELI_VERSION_03:
 	case G_ELI_VERSION_04:
 	case G_ELI_VERSION_05:
 	case G_ELI_VERSION_06:
 	case G_ELI_VERSION_07:
 		error = eli_metadata_decode_v1v2v3v4v5v6v7(data, md);
 		break;
 	default:
 		error = EOPNOTSUPP;
 		break;
 	}
 	return (error);
 }
 #endif	/* !_OpenSSL */
 
 static __inline u_int
 g_eli_str2ealgo(const char *name)
 {
 
 	if (strcasecmp("null", name) == 0)
 		return (CRYPTO_NULL_CBC);
 	else if (strcasecmp("null-cbc", name) == 0)
 		return (CRYPTO_NULL_CBC);
 	else if (strcasecmp("aes", name) == 0)
 		return (CRYPTO_AES_XTS);
 	else if (strcasecmp("aes-cbc", name) == 0)
 		return (CRYPTO_AES_CBC);
 	else if (strcasecmp("aes-xts", name) == 0)
 		return (CRYPTO_AES_XTS);
 	else if (strcasecmp("blowfish", name) == 0)
 		return (CRYPTO_BLF_CBC);
 	else if (strcasecmp("blowfish-cbc", name) == 0)
 		return (CRYPTO_BLF_CBC);
 	else if (strcasecmp("camellia", name) == 0)
 		return (CRYPTO_CAMELLIA_CBC);
 	else if (strcasecmp("camellia-cbc", name) == 0)
 		return (CRYPTO_CAMELLIA_CBC);
 	else if (strcasecmp("3des", name) == 0)
 		return (CRYPTO_3DES_CBC);
 	else if (strcasecmp("3des-cbc", name) == 0)
 		return (CRYPTO_3DES_CBC);
 	return (CRYPTO_ALGORITHM_MIN - 1);
 }
 
 static __inline u_int
 g_eli_str2aalgo(const char *name)
 {
 
 	if (strcasecmp("hmac/md5", name) == 0)
 		return (CRYPTO_MD5_HMAC);
 	else if (strcasecmp("hmac/sha1", name) == 0)
 		return (CRYPTO_SHA1_HMAC);
 	else if (strcasecmp("hmac/ripemd160", name) == 0)
 		return (CRYPTO_RIPEMD160_HMAC);
 	else if (strcasecmp("hmac/sha256", name) == 0)
 		return (CRYPTO_SHA2_256_HMAC);
 	else if (strcasecmp("hmac/sha384", name) == 0)
 		return (CRYPTO_SHA2_384_HMAC);
 	else if (strcasecmp("hmac/sha512", name) == 0)
 		return (CRYPTO_SHA2_512_HMAC);
 	return (CRYPTO_ALGORITHM_MIN - 1);
 }
 
 static __inline const char *
 g_eli_algo2str(u_int algo)
 {
 
 	switch (algo) {
 	case CRYPTO_NULL_CBC:
 		return ("NULL");
 	case CRYPTO_AES_CBC:
 		return ("AES-CBC");
 	case CRYPTO_AES_XTS:
 		return ("AES-XTS");
 	case CRYPTO_BLF_CBC:
 		return ("Blowfish-CBC");
 	case CRYPTO_CAMELLIA_CBC:
 		return ("CAMELLIA-CBC");
 	case CRYPTO_3DES_CBC:
 		return ("3DES-CBC");
 	case CRYPTO_MD5_HMAC:
 		return ("HMAC/MD5");
 	case CRYPTO_SHA1_HMAC:
 		return ("HMAC/SHA1");
 	case CRYPTO_RIPEMD160_HMAC:
 		return ("HMAC/RIPEMD160");
 	case CRYPTO_SHA2_256_HMAC:
 		return ("HMAC/SHA256");
 	case CRYPTO_SHA2_384_HMAC:
 		return ("HMAC/SHA384");
 	case CRYPTO_SHA2_512_HMAC:
 		return ("HMAC/SHA512");
 	}
 	return ("unknown");
 }
 
 static __inline void
 eli_metadata_dump(const struct g_eli_metadata *md)
 {
 	static const char hex[] = "0123456789abcdef";
 	char str[sizeof(md->md_mkeys) * 2 + 1];
 	u_int i;
 
 	printf("     magic: %s\n", md->md_magic);
 	printf("   version: %u\n", (u_int)md->md_version);
 	printf("     flags: 0x%x\n", (u_int)md->md_flags);
 	printf("     ealgo: %s\n", g_eli_algo2str(md->md_ealgo));
 	printf("    keylen: %u\n", (u_int)md->md_keylen);
 	if (md->md_flags & G_ELI_FLAG_AUTH)
 		printf("     aalgo: %s\n", g_eli_algo2str(md->md_aalgo));
 	printf("  provsize: %ju\n", (uintmax_t)md->md_provsize);
 	printf("sectorsize: %u\n", (u_int)md->md_sectorsize);
 	printf("      keys: 0x%02x\n", (u_int)md->md_keys);
 	printf("iterations: %d\n", (int)md->md_iterations);
 	bzero(str, sizeof(str));
 	for (i = 0; i < sizeof(md->md_salt); i++) {
 		str[i * 2] = hex[md->md_salt[i] >> 4];
 		str[i * 2 + 1] = hex[md->md_salt[i] & 0x0f];
 	}
 	printf("      Salt: %s\n", str);
 	bzero(str, sizeof(str));
 	for (i = 0; i < sizeof(md->md_mkeys); i++) {
 		str[i * 2] = hex[md->md_mkeys[i] >> 4];
 		str[i * 2 + 1] = hex[md->md_mkeys[i] & 0x0f];
 	}
 	printf("Master Key: %s\n", str);
 	bzero(str, sizeof(str));
 	for (i = 0; i < 16; i++) {
 		str[i * 2] = hex[md->md_hash[i] >> 4];
 		str[i * 2 + 1] = hex[md->md_hash[i] & 0x0f];
 	}
 	printf("  MD5 hash: %s\n", str);
 }
 
 static __inline u_int
 g_eli_keylen(u_int algo, u_int keylen)
 {
 
 	switch (algo) {
 	case CRYPTO_NULL_CBC:
 		if (keylen == 0)
 			keylen = 64 * 8;
 		else {
 			if (keylen > 64 * 8)
 				keylen = 0;
 		}
 		return (keylen);
 	case CRYPTO_AES_CBC:
 	case CRYPTO_CAMELLIA_CBC:
 		switch (keylen) {
 		case 0:
 			return (128);
 		case 128:
 		case 192:
 		case 256:
 			return (keylen);
 		default:
 			return (0);
 		}
 	case CRYPTO_AES_XTS:
 		switch (keylen) {
 		case 0:
 			return (128);
 		case 128:
 		case 256:
 			return (keylen);
 		default:
 			return (0);
 		}
 	case CRYPTO_BLF_CBC:
 		if (keylen == 0)
 			return (128);
 		if (keylen < 128 || keylen > 448)
 			return (0);
 		if ((keylen % 32) != 0)
 			return (0);
 		return (keylen);
 	case CRYPTO_3DES_CBC:
 		if (keylen == 0 || keylen == 192)
 			return (192);
 		return (0);
 	default:
 		return (0);
 	}
+}
+
+static __inline u_int
+g_eli_ivlen(u_int algo)
+{
+
+	switch (algo) {
+	case CRYPTO_AES_XTS:
+		return (AES_XTS_IV_LEN);
+	case CRYPTO_AES_CBC:
+		return (AES_BLOCK_LEN);
+	case CRYPTO_BLF_CBC:
+		return (BLOWFISH_BLOCK_LEN);
+	case CRYPTO_CAMELLIA_CBC:
+		return (CAMELLIA_BLOCK_LEN);
+	case CRYPTO_3DES_CBC:
+		return (DES3_BLOCK_LEN);
+	}
+	return (0);
 }
 
 static __inline u_int
 g_eli_hashlen(u_int algo)
 {
 
 	switch (algo) {
 	case CRYPTO_MD5_HMAC:
 		return (16);
 	case CRYPTO_SHA1_HMAC:
 		return (20);
 	case CRYPTO_RIPEMD160_HMAC:
 		return (20);
 	case CRYPTO_SHA2_256_HMAC:
 		return (32);
 	case CRYPTO_SHA2_384_HMAC:
 		return (48);
 	case CRYPTO_SHA2_512_HMAC:
 		return (64);
 	}
 	return (0);
 }
 
 static __inline off_t
 eli_mediasize(const struct g_eli_softc *sc, off_t mediasize, u_int sectorsize)
 {
 
 	if ((sc->sc_flags & G_ELI_FLAG_ONETIME) == 0) {
 		mediasize -= sectorsize;
 	}
 	if ((sc->sc_flags & G_ELI_FLAG_AUTH) == 0) {
 		mediasize -= (mediasize % sc->sc_sectorsize);
 	} else {
 		mediasize /= sc->sc_bytes_per_sector;
 		mediasize *= sc->sc_sectorsize;
 	}
 
 	return (mediasize);
 }
 
 static __inline void
 eli_metadata_softc(struct g_eli_softc *sc, const struct g_eli_metadata *md,
     u_int sectorsize, off_t mediasize)
 {
 
 	sc->sc_version = md->md_version;
 	sc->sc_inflight = 0;
 	sc->sc_crypto = G_ELI_CRYPTO_UNKNOWN;
 	sc->sc_flags = md->md_flags;
 	/* Backward compatibility. */
 	if (md->md_version < G_ELI_VERSION_04)
 		sc->sc_flags |= G_ELI_FLAG_NATIVE_BYTE_ORDER;
 	if (md->md_version < G_ELI_VERSION_05)
 		sc->sc_flags |= G_ELI_FLAG_SINGLE_KEY;
 	if (md->md_version < G_ELI_VERSION_06 &&
 	    (sc->sc_flags & G_ELI_FLAG_AUTH) != 0) {
 		sc->sc_flags |= G_ELI_FLAG_FIRST_KEY;
 	}
 	if (md->md_version < G_ELI_VERSION_07)
 		sc->sc_flags |= G_ELI_FLAG_ENC_IVKEY;
 	sc->sc_ealgo = md->md_ealgo;
 
 	if (sc->sc_flags & G_ELI_FLAG_AUTH) {
 		sc->sc_akeylen = sizeof(sc->sc_akey) * 8;
 		sc->sc_aalgo = md->md_aalgo;
 		sc->sc_alen = g_eli_hashlen(sc->sc_aalgo);
 
 		sc->sc_data_per_sector = sectorsize - sc->sc_alen;
 		/*
 		 * Some hash functions (like SHA1 and RIPEMD160) generates hash
 		 * which length is not multiple of 128 bits, but we want data
 		 * length to be multiple of 128, so we can encrypt without
 		 * padding. The line below rounds down data length to multiple
 		 * of 128 bits.
 		 */
 		sc->sc_data_per_sector -= sc->sc_data_per_sector % 16;
 
 		sc->sc_bytes_per_sector =
 		    (md->md_sectorsize - 1) / sc->sc_data_per_sector + 1;
 		sc->sc_bytes_per_sector *= sectorsize;
 	}
 	sc->sc_provsize = mediasize;
 	sc->sc_sectorsize = md->md_sectorsize;
 	sc->sc_mediasize = eli_mediasize(sc, mediasize, sectorsize);
 	sc->sc_ekeylen = md->md_keylen;
 }
 
 #ifdef _KERNEL
 int g_eli_read_metadata(struct g_class *mp, struct g_provider *pp,
     struct g_eli_metadata *md);
 struct g_geom *g_eli_create(struct gctl_req *req, struct g_class *mp,
     struct g_provider *bpp, const struct g_eli_metadata *md,
     const u_char *mkey, int nkey);
 int g_eli_destroy(struct g_eli_softc *sc, boolean_t force);
 
 int g_eli_access(struct g_provider *pp, int dr, int dw, int de);
 void g_eli_config(struct gctl_req *req, struct g_class *mp, const char *verb);
 
 void g_eli_read_done(struct bio *bp);
 void g_eli_write_done(struct bio *bp);
 int g_eli_crypto_rerun(struct cryptop *crp);
 
 void g_eli_crypto_read(struct g_eli_softc *sc, struct bio *bp, boolean_t fromworker);
 void g_eli_crypto_run(struct g_eli_worker *wr, struct bio *bp);
 
 void g_eli_auth_read(struct g_eli_softc *sc, struct bio *bp);
 void g_eli_auth_run(struct g_eli_worker *wr, struct bio *bp);
 #endif
 void g_eli_crypto_ivgen(struct g_eli_softc *sc, off_t offset, u_char *iv,
     size_t size);
 
 void g_eli_mkey_hmac(unsigned char *mkey, const unsigned char *key);
 int g_eli_mkey_decrypt(const struct g_eli_metadata *md,
     const unsigned char *key, unsigned char *mkey, unsigned keyp);
 int g_eli_mkey_decrypt_any(const struct g_eli_metadata *md,
     const unsigned char *key, unsigned char *mkey, unsigned *nkeyp);
 int g_eli_mkey_encrypt(unsigned algo, const unsigned char *key, unsigned keylen,
     unsigned char *mkey);
 #ifdef _KERNEL
 void g_eli_mkey_propagate(struct g_eli_softc *sc, const unsigned char *mkey);
 #endif
 
 int g_eli_crypto_encrypt(u_int algo, u_char *data, size_t datasize,
     const u_char *key, size_t keysize);
 int g_eli_crypto_decrypt(u_int algo, u_char *data, size_t datasize,
     const u_char *key, size_t keysize);
 
 struct hmac_ctx {
 	SHA512_CTX	innerctx;
 	SHA512_CTX	outerctx;
 };
 
 void g_eli_crypto_hmac_init(struct hmac_ctx *ctx, const char *hkey,
     size_t hkeylen);
 void g_eli_crypto_hmac_update(struct hmac_ctx *ctx, const uint8_t *data,
     size_t datasize);
 void g_eli_crypto_hmac_final(struct hmac_ctx *ctx, uint8_t *md, size_t mdsize);
 void g_eli_crypto_hmac(const char *hkey, size_t hkeysize,
     const uint8_t *data, size_t datasize, uint8_t *md, size_t mdsize);
 
 void g_eli_key_fill(struct g_eli_softc *sc, struct g_eli_key *key,
     uint64_t keyno);
 #ifdef _KERNEL
 void g_eli_key_init(struct g_eli_softc *sc);
 void g_eli_key_destroy(struct g_eli_softc *sc);
 void g_eli_key_resize(struct g_eli_softc *sc);
 uint8_t *g_eli_key_hold(struct g_eli_softc *sc, off_t offset, size_t blocksize);
 void g_eli_key_drop(struct g_eli_softc *sc, uint8_t *rawkey);
 #endif
 #endif	/* !_G_ELI_H_ */
Index: head/sys/geom/eli/g_eli_crypto.c
===================================================================
--- head/sys/geom/eli/g_eli_crypto.c	(revision 359373)
+++ head/sys/geom/eli/g_eli_crypto.c	(revision 359374)
@@ -1,227 +1,217 @@
 /*-
  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
  *
  * Copyright (c) 2005-2010 Pawel Jakub Dawidek <pjd@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #ifdef _KERNEL
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/malloc.h>
 #else
 #include <stdint.h>
 #include <string.h>
 #include <strings.h>
 #include <errno.h>
 #include <assert.h>
 #include <openssl/evp.h>
 #define	_OpenSSL_
 #endif
 #include <geom/eli/g_eli.h>
 
 #ifdef _KERNEL
 MALLOC_DECLARE(M_ELI);
 
 static int
 g_eli_crypto_done(struct cryptop *crp)
 {
 
 	crp->crp_opaque = (void *)crp;
 	wakeup(crp);
 	return (0);
 }
 
 static int
 g_eli_crypto_cipher(u_int algo, int enc, u_char *data, size_t datasize,
     const u_char *key, size_t keysize)
 {
-	struct cryptoini cri;
+	struct crypto_session_params csp;
 	struct cryptop *crp;
-	struct cryptodesc *crd;
 	crypto_session_t sid;
-	u_char *p;
 	int error;
 
 	KASSERT(algo != CRYPTO_AES_XTS,
 	    ("%s: CRYPTO_AES_XTS unexpected here", __func__));
 
-	bzero(&cri, sizeof(cri));
-	cri.cri_alg = algo;
-	cri.cri_key = __DECONST(void *, key);
-	cri.cri_klen = keysize;
-	error = crypto_newsession(&sid, &cri, CRYPTOCAP_F_SOFTWARE);
+	memset(&csp, 0, sizeof(csp));
+	csp.csp_mode = CSP_MODE_CIPHER;
+	csp.csp_cipher_alg = algo;
+	csp.csp_ivlen = g_eli_ivlen(algo);
+	csp.csp_cipher_key = key;
+	csp.csp_cipher_klen = keysize / 8;
+	error = crypto_newsession(&sid, &csp, CRYPTOCAP_F_SOFTWARE);
 	if (error != 0)
 		return (error);
-	p = malloc(sizeof(*crp) + sizeof(*crd), M_ELI, M_NOWAIT | M_ZERO);
-	if (p == NULL) {
+	crp = crypto_getreq(sid, M_NOWAIT);
+	if (crp == NULL) {
 		crypto_freesession(sid);
 		return (ENOMEM);
 	}
-	crp = (struct cryptop *)p;	p += sizeof(*crp);
-	crd = (struct cryptodesc *)p;	p += sizeof(*crd);
 
-	crd->crd_skip = 0;
-	crd->crd_len = datasize;
-	crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
-	if (enc)
-		crd->crd_flags |= CRD_F_ENCRYPT;
-	crd->crd_alg = algo;
-	crd->crd_key = __DECONST(void *, key);
-	crd->crd_klen = keysize;
-	bzero(crd->crd_iv, sizeof(crd->crd_iv));
-	crd->crd_next = NULL;
+	crp->crp_payload_start = 0;
+	crp->crp_payload_length = datasize;
+	crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_IV_SEPARATE;
+	crp->crp_op = enc ? CRYPTO_OP_ENCRYPT : CRYPTO_OP_DECRYPT;
+	memset(crp->crp_iv, 0, sizeof(crp->crp_iv));
 
-	crp->crp_session = sid;
-	crp->crp_ilen = datasize;
-	crp->crp_olen = datasize;
 	crp->crp_opaque = NULL;
 	crp->crp_callback = g_eli_crypto_done;
+	crp->crp_buf_type = CRYPTO_BUF_CONTIG;
+	crp->crp_ilen = datasize;
 	crp->crp_buf = (void *)data;
-	crp->crp_flags = CRYPTO_F_CBIFSYNC;
-	crp->crp_desc = crd;
 
 	error = crypto_dispatch(crp);
 	if (error == 0) {
 		while (crp->crp_opaque == NULL)
 			tsleep(crp, PRIBIO, "geli", hz / 5);
 		error = crp->crp_etype;
 	}
 
-	free(crp, M_ELI);
+	crypto_freereq(crp);
 	crypto_freesession(sid);
 	return (error);
 }
 #else	/* !_KERNEL */
 static int
 g_eli_crypto_cipher(u_int algo, int enc, u_char *data, size_t datasize,
     const u_char *key, size_t keysize)
 {
 	EVP_CIPHER_CTX *ctx;
 	const EVP_CIPHER *type;
 	u_char iv[keysize];
 	int outsize;
 
 	assert(algo != CRYPTO_AES_XTS);
 
 	switch (algo) {
 	case CRYPTO_NULL_CBC:
 		type = EVP_enc_null();
 		break;
 	case CRYPTO_AES_CBC:
 		switch (keysize) {
 		case 128:
 			type = EVP_aes_128_cbc();
 			break;
 		case 192:
 			type = EVP_aes_192_cbc();
 			break;
 		case 256:
 			type = EVP_aes_256_cbc();
 			break;
 		default:
 			return (EINVAL);
 		}
 		break;
 	case CRYPTO_BLF_CBC:
 		type = EVP_bf_cbc();
 		break;
 #ifndef OPENSSL_NO_CAMELLIA
 	case CRYPTO_CAMELLIA_CBC:
 		switch (keysize) {
 		case 128:
 			type = EVP_camellia_128_cbc();
 			break;
 		case 192:
 			type = EVP_camellia_192_cbc();
 			break;
 		case 256:
 			type = EVP_camellia_256_cbc();
 			break;
 		default:
 			return (EINVAL);
 		}
 		break;
 #endif
 	case CRYPTO_3DES_CBC:
 		type = EVP_des_ede3_cbc();
 		break;
 	default:
 		return (EINVAL);
 	}
 
 	ctx = EVP_CIPHER_CTX_new();
 	if (ctx == NULL)
 		return (ENOMEM);
 
 	EVP_CipherInit_ex(ctx, type, NULL, NULL, NULL, enc);
 	EVP_CIPHER_CTX_set_key_length(ctx, keysize / 8);
 	EVP_CIPHER_CTX_set_padding(ctx, 0);
 	bzero(iv, sizeof(iv));
 	EVP_CipherInit_ex(ctx, NULL, NULL, key, iv, enc);
 
 	if (EVP_CipherUpdate(ctx, data, &outsize, data, datasize) == 0) {
 		EVP_CIPHER_CTX_free(ctx);
 		return (EINVAL);
 	}
 	assert(outsize == (int)datasize);
 
 	if (EVP_CipherFinal_ex(ctx, data + outsize, &outsize) == 0) {
 		EVP_CIPHER_CTX_free(ctx);
 		return (EINVAL);
 	}
 	assert(outsize == 0);
 
 	EVP_CIPHER_CTX_free(ctx);
 	return (0);
 }
 #endif	/* !_KERNEL */
 
 int
 g_eli_crypto_encrypt(u_int algo, u_char *data, size_t datasize,
     const u_char *key, size_t keysize)
 {
 
 	/* We prefer AES-CBC for metadata protection. */
 	if (algo == CRYPTO_AES_XTS)
 		algo = CRYPTO_AES_CBC;
 
 	return (g_eli_crypto_cipher(algo, 1, data, datasize, key, keysize));
 }
 
 int
 g_eli_crypto_decrypt(u_int algo, u_char *data, size_t datasize,
     const u_char *key, size_t keysize)
 {
 
 	/* We prefer AES-CBC for metadata protection. */
 	if (algo == CRYPTO_AES_XTS)
 		algo = CRYPTO_AES_CBC;
 
 	return (g_eli_crypto_cipher(algo, 0, data, datasize, key, keysize));
 }
Index: head/sys/geom/eli/g_eli_integrity.c
===================================================================
--- head/sys/geom/eli/g_eli_integrity.c	(revision 359373)
+++ head/sys/geom/eli/g_eli_integrity.c	(revision 359374)
@@ -1,541 +1,556 @@
 /*-
  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
  *
  * Copyright (c) 2005-2011 Pawel Jakub Dawidek <pawel@dawidek.net>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/linker.h>
 #include <sys/module.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <sys/bio.h>
 #include <sys/sysctl.h>
 #include <sys/malloc.h>
 #include <sys/kthread.h>
 #include <sys/proc.h>
 #include <sys/sched.h>
 #include <sys/smp.h>
 #include <sys/vnode.h>
 
 #include <vm/uma.h>
 
 #include <geom/geom.h>
 #include <geom/geom_dbg.h>
 #include <geom/eli/g_eli.h>
 #include <geom/eli/pkcs5v2.h>
 
 /*
  * The data layout description when integrity verification is configured.
  *
  * One of the most important assumption here is that authenticated data and its
  * HMAC has to be stored in the same place (namely in the same sector) to make
  * it work reliable.
  * The problem is that file systems work only with sectors that are multiple of
  * 512 bytes and a power of two number.
  * My idea to implement it is as follows.
  * Let's store HMAC in sector. This is a must. This leaves us 480 bytes for
  * data. We can't use that directly (ie. we can't create provider with 480 bytes
  * sector size). We need another sector from where we take only 32 bytes of data
  * and we store HMAC of this data as well. This takes two sectors from the
  * original provider at the input and leaves us one sector of authenticated data
  * at the output. Not very efficient, but you got the idea.
  * Now, let's assume, we want to create provider with 4096 bytes sector.
  * To output 4096 bytes of authenticated data we need 8x480 plus 1x256, so we
  * need nine 512-bytes sectors at the input to get one 4096-bytes sector at the
  * output. That's better. With 4096 bytes sector we can use 89% of size of the
  * original provider. I find it as an acceptable cost.
  * The reliability comes from the fact, that every HMAC stored inside the sector
  * is calculated only for the data in the same sector, so its impossible to
  * write new data and leave old HMAC or vice versa.
  *
  * And here is the picture:
  *
  * da0: +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+-----+
  *      |32b |480b| |32b |480b| |32b |480b| |32b |480b| |32b |480b| |32b |480b| |32b |480b| |32b |480b| |32b |256b |
  *      |HMAC|Data| |HMAC|Data| |HMAC|Data| |HMAC|Data| |HMAC|Data| |HMAC|Data| |HMAC|Data| |HMAC|Data| |HMAC|Data |
  *      +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+----+ +----+-----+
  *      |512 bytes| |512 bytes| |512 bytes| |512 bytes| |512 bytes| |512 bytes| |512 bytes| |512 bytes| |288 bytes |
  *      +---------+ +---------+ +---------+ +---------+ +---------+ +---------+ +---------+ +---------+ |224 unused|
  *                                                                                                      +----------+
  * da0.eli: +----+----+----+----+----+----+----+----+----+
  *          |480b|480b|480b|480b|480b|480b|480b|480b|256b|
  *          +----+----+----+----+----+----+----+----+----+
  *          |                 4096 bytes                 |
  *          +--------------------------------------------+
  *
  * PS. You can use any sector size with geli(8). My example is using 4kB,
  *     because it's most efficient. For 8kB sectors you need 2 extra sectors,
  *     so the cost is the same as for 4kB sectors.
  */
 
 /*
  * Code paths:
  * BIO_READ:
  *	g_eli_start -> g_eli_auth_read -> g_io_request -> g_eli_read_done -> g_eli_auth_run -> g_eli_auth_read_done -> g_io_deliver
  * BIO_WRITE:
  *	g_eli_start -> g_eli_auth_run -> g_eli_auth_write_done -> g_io_request -> g_eli_write_done -> g_io_deliver
  */
 
 MALLOC_DECLARE(M_ELI);
 
 /*
  * Here we generate key for HMAC. Every sector has its own HMAC key, so it is
  * not possible to copy sectors.
  * We cannot depend on fact, that every sector has its own IV, because different
  * IV doesn't change HMAC, when we use encrypt-then-authenticate method.
  */
 static void
 g_eli_auth_keygen(struct g_eli_softc *sc, off_t offset, u_char *key)
 {
 	SHA256_CTX ctx;
 
 	/* Copy precalculated SHA256 context. */
 	bcopy(&sc->sc_akeyctx, &ctx, sizeof(ctx));
 	SHA256_Update(&ctx, (uint8_t *)&offset, sizeof(offset));
 	SHA256_Final(key, &ctx);
 }
 
 /*
  * The function is called after we read and decrypt data.
  *
  * g_eli_start -> g_eli_auth_read -> g_io_request -> g_eli_read_done -> g_eli_auth_run -> G_ELI_AUTH_READ_DONE -> g_io_deliver
  */
 static int
 g_eli_auth_read_done(struct cryptop *crp)
 {
 	struct g_eli_softc *sc;
 	struct bio *bp;
 
 	if (crp->crp_etype == EAGAIN) {
 		if (g_eli_crypto_rerun(crp) == 0)
 			return (0);
 	}
 	bp = (struct bio *)crp->crp_opaque;
 	bp->bio_inbed++;
+	sc = bp->bio_to->geom->softc;
 	if (crp->crp_etype == 0) {
-		bp->bio_completed += crp->crp_olen;
-		G_ELI_DEBUG(3, "Crypto READ request done (%d/%d) (add=%jd completed=%jd).",
-		    bp->bio_inbed, bp->bio_children, (intmax_t)crp->crp_olen, (intmax_t)bp->bio_completed);
+		bp->bio_completed += crp->crp_payload_length;
+		G_ELI_DEBUG(3, "Crypto READ request done (%d/%d) (add=%d completed=%jd).",
+		    bp->bio_inbed, bp->bio_children, crp->crp_payload_length, (intmax_t)bp->bio_completed);
 	} else {
-		G_ELI_DEBUG(1, "Crypto READ request failed (%d/%d) error=%d.",
+		u_int nsec, decr_secsize, encr_secsize, rel_sec;
+		int *errorp;
+
+		/* Sectorsize of decrypted provider eg. 4096. */
+		decr_secsize = bp->bio_to->sectorsize;
+		/* The real sectorsize of encrypted provider, eg. 512. */
+		encr_secsize =
+		    LIST_FIRST(&sc->sc_geom->consumer)->provider->sectorsize;
+		/* Number of sectors from decrypted provider, eg. 2. */
+		nsec = bp->bio_length / decr_secsize;
+		/* Number of sectors from encrypted provider, eg. 18. */
+		nsec = (nsec * sc->sc_bytes_per_sector) / encr_secsize;
+		/* Which relative sector this request decrypted. */
+		rel_sec = ((crp->crp_buf + crp->crp_payload_start) -
+		    (char *)bp->bio_driver2) / encr_secsize;
+
+		errorp = (int *)((char *)bp->bio_driver2 + encr_secsize * nsec +
+		    sizeof(int) * rel_sec);
+		*errorp = crp->crp_etype;
+		G_ELI_DEBUG(1,
+		    "Crypto READ request failed (%d/%d) error=%d.",
 		    bp->bio_inbed, bp->bio_children, crp->crp_etype);
-		if (bp->bio_error == 0)
-			bp->bio_error = crp->crp_etype;
+		if (bp->bio_error == 0 || bp->bio_error == EINTEGRITY)
+			bp->bio_error = crp->crp_etype == EBADMSG ?
+			    EINTEGRITY : crp->crp_etype;
 	}
-	sc = bp->bio_to->geom->softc;
-	g_eli_key_drop(sc, crp->crp_desc->crd_next->crd_key);
+	if (crp->crp_cipher_key != NULL)
+		g_eli_key_drop(sc, __DECONST(void *, crp->crp_cipher_key));
+	crypto_freereq(crp);
 	/*
 	 * Do we have all sectors already?
 	 */
 	if (bp->bio_inbed < bp->bio_children)
 		return (0);
+
 	if (bp->bio_error == 0) {
 		u_int i, lsec, nsec, data_secsize, decr_secsize, encr_secsize;
-		u_char *srcdata, *dstdata, *auth;
-		off_t coroff, corsize;
+		u_char *srcdata, *dstdata;
 
-		/*
-		 * Verify data integrity based on calculated and read HMACs.
-		 */
 		/* Sectorsize of decrypted provider eg. 4096. */
 		decr_secsize = bp->bio_to->sectorsize;
 		/* The real sectorsize of encrypted provider, eg. 512. */
 		encr_secsize = LIST_FIRST(&sc->sc_geom->consumer)->provider->sectorsize;
 		/* Number of data bytes in one encrypted sector, eg. 480. */
 		data_secsize = sc->sc_data_per_sector;
 		/* Number of sectors from decrypted provider, eg. 2. */
 		nsec = bp->bio_length / decr_secsize;
 		/* Number of sectors from encrypted provider, eg. 18. */
 		nsec = (nsec * sc->sc_bytes_per_sector) / encr_secsize;
 		/* Last sector number in every big sector, eg. 9. */
 		lsec = sc->sc_bytes_per_sector / encr_secsize;
 
 		srcdata = bp->bio_driver2;
 		dstdata = bp->bio_data;
-		auth = srcdata + encr_secsize * nsec;
+
+		for (i = 1; i <= nsec; i++) {
+			data_secsize = sc->sc_data_per_sector;
+			if ((i % lsec) == 0)
+				data_secsize = decr_secsize % data_secsize;
+			bcopy(srcdata + sc->sc_alen, dstdata, data_secsize);
+			srcdata += encr_secsize;
+			dstdata += data_secsize;
+		}
+	} else if (bp->bio_error == EINTEGRITY) {
+		u_int i, lsec, nsec, data_secsize, decr_secsize, encr_secsize;
+		int *errorp;
+		off_t coroff, corsize, dstoff;
+
+		/* Sectorsize of decrypted provider eg. 4096. */
+		decr_secsize = bp->bio_to->sectorsize;
+		/* The real sectorsize of encrypted provider, eg. 512. */
+		encr_secsize = LIST_FIRST(&sc->sc_geom->consumer)->provider->sectorsize;
+		/* Number of data bytes in one encrypted sector, eg. 480. */
+		data_secsize = sc->sc_data_per_sector;
+		/* Number of sectors from decrypted provider, eg. 2. */
+		nsec = bp->bio_length / decr_secsize;
+		/* Number of sectors from encrypted provider, eg. 18. */
+		nsec = (nsec * sc->sc_bytes_per_sector) / encr_secsize;
+		/* Last sector number in every big sector, eg. 9. */
+		lsec = sc->sc_bytes_per_sector / encr_secsize;
+
+		errorp = (int *)((char *)bp->bio_driver2 + encr_secsize * nsec);
 		coroff = -1;
 		corsize = 0;
+		dstoff = bp->bio_offset;
 
 		for (i = 1; i <= nsec; i++) {
 			data_secsize = sc->sc_data_per_sector;
 			if ((i % lsec) == 0)
 				data_secsize = decr_secsize % data_secsize;
-			if (bcmp(srcdata, auth, sc->sc_alen) != 0) {
+			if (errorp[i - 1] == EBADMSG) {
 				/*
-				 * Curruption detected, remember the offset if
+				 * Corruption detected, remember the offset if
 				 * this is the first corrupted sector and
 				 * increase size.
 				 */
-				if (bp->bio_error == 0)
-					bp->bio_error = -1;
-				if (coroff == -1) {
-					coroff = bp->bio_offset +
-					    (dstdata - (u_char *)bp->bio_data);
-				}
+				if (coroff == -1)
+					coroff = dstoff;
 				corsize += data_secsize;
 			} else {
 				/*
-				 * No curruption, good.
+				 * No corruption, good.
 				 * Report previous corruption if there was one.
 				 */
 				if (coroff != -1) {
 					G_ELI_DEBUG(0, "%s: Failed to authenticate %jd "
 					    "bytes of data at offset %jd.",
 					    sc->sc_name, (intmax_t)corsize,
 					    (intmax_t)coroff);
 					coroff = -1;
 					corsize = 0;
 				}
-				bcopy(srcdata + sc->sc_alen, dstdata,
-				    data_secsize);
 			}
-			srcdata += encr_secsize;
-			dstdata += data_secsize;
-			auth += sc->sc_alen;
+			dstoff += data_secsize;
 		}
 		/* Report previous corruption if there was one. */
 		if (coroff != -1) {
 			G_ELI_DEBUG(0, "%s: Failed to authenticate %jd "
 			    "bytes of data at offset %jd.",
 			    sc->sc_name, (intmax_t)corsize, (intmax_t)coroff);
 		}
 	}
 	free(bp->bio_driver2, M_ELI);
 	bp->bio_driver2 = NULL;
 	if (bp->bio_error != 0) {
-		if (bp->bio_error == -1)
-			bp->bio_error = EINTEGRITY;
-		else {
+		if (bp->bio_error != EINTEGRITY) {
 			G_ELI_LOGREQ(0, bp,
 			    "Crypto READ request failed (error=%d).",
 			    bp->bio_error);
 		}
 		bp->bio_completed = 0;
 	}
 	/*
 	 * Read is finished, send it up.
 	 */
 	g_io_deliver(bp, bp->bio_error);
 	atomic_subtract_int(&sc->sc_inflight, 1);
 	return (0);
 }
 
 /*
  * The function is called after data encryption.
  *
  * g_eli_start -> g_eli_auth_run -> G_ELI_AUTH_WRITE_DONE -> g_io_request -> g_eli_write_done -> g_io_deliver
  */
 static int
 g_eli_auth_write_done(struct cryptop *crp)
 {
 	struct g_eli_softc *sc;
 	struct g_consumer *cp;
 	struct bio *bp, *cbp, *cbp2;
 	u_int nsec;
 
 	if (crp->crp_etype == EAGAIN) {
 		if (g_eli_crypto_rerun(crp) == 0)
 			return (0);
 	}
 	bp = (struct bio *)crp->crp_opaque;
 	bp->bio_inbed++;
 	if (crp->crp_etype == 0) {
 		G_ELI_DEBUG(3, "Crypto WRITE request done (%d/%d).",
 		    bp->bio_inbed, bp->bio_children);
 	} else {
 		G_ELI_DEBUG(1, "Crypto WRITE request failed (%d/%d) error=%d.",
 		    bp->bio_inbed, bp->bio_children, crp->crp_etype);
 		if (bp->bio_error == 0)
 			bp->bio_error = crp->crp_etype;
 	}
 	sc = bp->bio_to->geom->softc;
-	g_eli_key_drop(sc, crp->crp_desc->crd_key);
+	if (crp->crp_cipher_key != NULL)
+		g_eli_key_drop(sc, __DECONST(void *, crp->crp_cipher_key));
+	crypto_freereq(crp);
 	/*
 	 * All sectors are already encrypted?
 	 */
 	if (bp->bio_inbed < bp->bio_children)
 		return (0);
 	if (bp->bio_error != 0) {
 		G_ELI_LOGREQ(0, bp, "Crypto WRITE request failed (error=%d).",
 		    bp->bio_error);
 		free(bp->bio_driver2, M_ELI);
 		bp->bio_driver2 = NULL;
 		cbp = bp->bio_driver1;
 		bp->bio_driver1 = NULL;
 		g_destroy_bio(cbp);
 		g_io_deliver(bp, bp->bio_error);
 		atomic_subtract_int(&sc->sc_inflight, 1);
 		return (0);
 	}
 	cp = LIST_FIRST(&sc->sc_geom->consumer);
 	cbp = bp->bio_driver1;
 	bp->bio_driver1 = NULL;
 	cbp->bio_to = cp->provider;
 	cbp->bio_done = g_eli_write_done;
 
 	/* Number of sectors from decrypted provider, eg. 1. */
 	nsec = bp->bio_length / bp->bio_to->sectorsize;
 	/* Number of sectors from encrypted provider, eg. 9. */
 	nsec = (nsec * sc->sc_bytes_per_sector) / cp->provider->sectorsize;
 
 	cbp->bio_length = cp->provider->sectorsize * nsec;
 	cbp->bio_offset = (bp->bio_offset / bp->bio_to->sectorsize) * sc->sc_bytes_per_sector;
 	cbp->bio_data = bp->bio_driver2;
 
 	/*
 	 * We write more than what is requested, so we have to be ready to write
 	 * more than MAXPHYS.
 	 */
 	cbp2 = NULL;
 	if (cbp->bio_length > MAXPHYS) {
 		cbp2 = g_duplicate_bio(bp);
 		cbp2->bio_length = cbp->bio_length - MAXPHYS;
 		cbp2->bio_data = cbp->bio_data + MAXPHYS;
 		cbp2->bio_offset = cbp->bio_offset + MAXPHYS;
 		cbp2->bio_to = cp->provider;
 		cbp2->bio_done = g_eli_write_done;
 		cbp->bio_length = MAXPHYS;
 	}
 	/*
 	 * Send encrypted data to the provider.
 	 */
 	G_ELI_LOGREQ(2, cbp, "Sending request.");
 	bp->bio_inbed = 0;
 	bp->bio_children = (cbp2 != NULL ? 2 : 1);
 	g_io_request(cbp, cp);
 	if (cbp2 != NULL) {
 		G_ELI_LOGREQ(2, cbp2, "Sending request.");
 		g_io_request(cbp2, cp);
 	}
 	return (0);
 }
 
 void
 g_eli_auth_read(struct g_eli_softc *sc, struct bio *bp)
 {
 	struct g_consumer *cp;
 	struct bio *cbp, *cbp2;
 	size_t size;
 	off_t nsec;
 
 	bp->bio_pflags = 0;
 
 	cp = LIST_FIRST(&sc->sc_geom->consumer);
 	cbp = bp->bio_driver1;
 	bp->bio_driver1 = NULL;
 	cbp->bio_to = cp->provider;
 	cbp->bio_done = g_eli_read_done;
 
 	/* Number of sectors from decrypted provider, eg. 1. */
 	nsec = bp->bio_length / bp->bio_to->sectorsize;
 	/* Number of sectors from encrypted provider, eg. 9. */
 	nsec = (nsec * sc->sc_bytes_per_sector) / cp->provider->sectorsize;
 
 	cbp->bio_length = cp->provider->sectorsize * nsec;
 	size = cbp->bio_length;
-	size += sc->sc_alen * nsec;
-	size += sizeof(struct cryptop) * nsec;
-	size += sizeof(struct cryptodesc) * nsec * 2;
+	size += sizeof(int) * nsec;
 	size += G_ELI_AUTH_SECKEYLEN * nsec;
 	cbp->bio_offset = (bp->bio_offset / bp->bio_to->sectorsize) * sc->sc_bytes_per_sector;
 	bp->bio_driver2 = malloc(size, M_ELI, M_WAITOK);
 	cbp->bio_data = bp->bio_driver2;
 
+	/* Clear the error array. */
+	memset((char *)bp->bio_driver2 + cbp->bio_length, 0,
+	    sizeof(int) * nsec);
+
 	/*
 	 * We read more than what is requested, so we have to be ready to read
 	 * more than MAXPHYS.
 	 */
 	cbp2 = NULL;
 	if (cbp->bio_length > MAXPHYS) {
 		cbp2 = g_duplicate_bio(bp);
 		cbp2->bio_length = cbp->bio_length - MAXPHYS;
 		cbp2->bio_data = cbp->bio_data + MAXPHYS;
 		cbp2->bio_offset = cbp->bio_offset + MAXPHYS;
 		cbp2->bio_to = cp->provider;
 		cbp2->bio_done = g_eli_read_done;
 		cbp->bio_length = MAXPHYS;
 	}
 	/*
 	 * Read encrypted data from provider.
 	 */
 	G_ELI_LOGREQ(2, cbp, "Sending request.");
 	g_io_request(cbp, cp);
 	if (cbp2 != NULL) {
 		G_ELI_LOGREQ(2, cbp2, "Sending request.");
 		g_io_request(cbp2, cp);
 	}
 }
 
 /*
  * This is the main function responsible for cryptography (ie. communication
  * with crypto(9) subsystem).
  *
  * BIO_READ:
  *	g_eli_start -> g_eli_auth_read -> g_io_request -> g_eli_read_done -> G_ELI_AUTH_RUN -> g_eli_auth_read_done -> g_io_deliver
  * BIO_WRITE:
  *	g_eli_start -> G_ELI_AUTH_RUN -> g_eli_auth_write_done -> g_io_request -> g_eli_write_done -> g_io_deliver
  */
 void
 g_eli_auth_run(struct g_eli_worker *wr, struct bio *bp)
 {
 	struct g_eli_softc *sc;
 	struct cryptop *crp;
-	struct cryptodesc *crde, *crda;
 	u_int i, lsec, nsec, data_secsize, decr_secsize, encr_secsize;
 	off_t dstoff;
-	u_char *p, *data, *auth, *authkey, *plaindata;
+	u_char *p, *data, *authkey, *plaindata;
 	int error;
 
 	G_ELI_LOGREQ(3, bp, "%s", __func__);
 
 	bp->bio_pflags = wr->w_number;
 	sc = wr->w_softc;
 	/* Sectorsize of decrypted provider eg. 4096. */
 	decr_secsize = bp->bio_to->sectorsize;
 	/* The real sectorsize of encrypted provider, eg. 512. */
 	encr_secsize = LIST_FIRST(&sc->sc_geom->consumer)->provider->sectorsize;
 	/* Number of data bytes in one encrypted sector, eg. 480. */
 	data_secsize = sc->sc_data_per_sector;
 	/* Number of sectors from decrypted provider, eg. 2. */
 	nsec = bp->bio_length / decr_secsize;
 	/* Number of sectors from encrypted provider, eg. 18. */
 	nsec = (nsec * sc->sc_bytes_per_sector) / encr_secsize;
 	/* Last sector number in every big sector, eg. 9. */
 	lsec = sc->sc_bytes_per_sector / encr_secsize;
 	/* Destination offset, used for IV generation. */
 	dstoff = (bp->bio_offset / bp->bio_to->sectorsize) * sc->sc_bytes_per_sector;
 
-	auth = NULL;	/* Silence compiler warning. */
 	plaindata = bp->bio_data;
 	if (bp->bio_cmd == BIO_READ) {
 		data = bp->bio_driver2;
-		auth = data + encr_secsize * nsec;
-		p = auth + sc->sc_alen * nsec;
+		p = data + encr_secsize * nsec;
+		p += sizeof(int) * nsec;
 	} else {
 		size_t size;
 
 		size = encr_secsize * nsec;
-		size += sizeof(*crp) * nsec;
-		size += sizeof(*crde) * nsec;
-		size += sizeof(*crda) * nsec;
 		size += G_ELI_AUTH_SECKEYLEN * nsec;
 		size += sizeof(uintptr_t);	/* Space for alignment. */
 		data = malloc(size, M_ELI, M_WAITOK);
 		bp->bio_driver2 = data;
 		p = data + encr_secsize * nsec;
 	}
 	bp->bio_inbed = 0;
 	bp->bio_children = nsec;
 
 #if defined(__mips_n64) || defined(__mips_o64)
 	p = (char *)roundup((uintptr_t)p, sizeof(uintptr_t));
 #endif
 
 	for (i = 1; i <= nsec; i++, dstoff += encr_secsize) {
-		crp = (struct cryptop *)p;	p += sizeof(*crp);
-		crde = (struct cryptodesc *)p;	p += sizeof(*crde);
-		crda = (struct cryptodesc *)p;	p += sizeof(*crda);
+		crp = crypto_getreq(wr->w_sid, M_WAITOK);
 		authkey = (u_char *)p;		p += G_ELI_AUTH_SECKEYLEN;
 
 		data_secsize = sc->sc_data_per_sector;
 		if ((i % lsec) == 0) {
 			data_secsize = decr_secsize % data_secsize;
 			/*
 			 * Last encrypted sector of each decrypted sector is
 			 * only partially filled.
 			 */
 			if (bp->bio_cmd == BIO_WRITE)
 				memset(data + sc->sc_alen + data_secsize, 0,
 				    encr_secsize - sc->sc_alen - data_secsize);
 		}
 
-		if (bp->bio_cmd == BIO_READ) {
-			/* Remember read HMAC. */
-			bcopy(data, auth, sc->sc_alen);
-			auth += sc->sc_alen;
-			/* TODO: bzero(9) can be commented out later. */
-			bzero(data, sc->sc_alen);
-		} else {
+		if (bp->bio_cmd == BIO_WRITE) {
 			bcopy(plaindata, data + sc->sc_alen, data_secsize);
 			plaindata += data_secsize;
 		}
 
-		crp->crp_session = wr->w_sid;
 		crp->crp_ilen = sc->sc_alen + data_secsize;
-		crp->crp_olen = data_secsize;
 		crp->crp_opaque = (void *)bp;
+		crp->crp_buf_type = CRYPTO_BUF_CONTIG;
 		crp->crp_buf = (void *)data;
 		data += encr_secsize;
 		crp->crp_flags = CRYPTO_F_CBIFSYNC;
 		if (g_eli_batch)
 			crp->crp_flags |= CRYPTO_F_BATCH;
 		if (bp->bio_cmd == BIO_WRITE) {
 			crp->crp_callback = g_eli_auth_write_done;
-			crp->crp_desc = crde;
-			crde->crd_next = crda;
-			crda->crd_next = NULL;
+			crp->crp_op = CRYPTO_OP_ENCRYPT |
+			    CRYPTO_OP_COMPUTE_DIGEST;
 		} else {
 			crp->crp_callback = g_eli_auth_read_done;
-			crp->crp_desc = crda;
-			crda->crd_next = crde;
-			crde->crd_next = NULL;
+			crp->crp_op = CRYPTO_OP_DECRYPT |
+			    CRYPTO_OP_VERIFY_DIGEST;
 		}
 
-		crde->crd_skip = sc->sc_alen;
-		crde->crd_len = data_secsize;
-		crde->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
-		if ((sc->sc_flags & G_ELI_FLAG_FIRST_KEY) == 0)
-			crde->crd_flags |= CRD_F_KEY_EXPLICIT;
-		if (bp->bio_cmd == BIO_WRITE)
-			crde->crd_flags |= CRD_F_ENCRYPT;
-		crde->crd_alg = sc->sc_ealgo;
-		crde->crd_key = g_eli_key_hold(sc, dstoff, encr_secsize);
-		crde->crd_klen = sc->sc_ekeylen;
-		if (sc->sc_ealgo == CRYPTO_AES_XTS)
-			crde->crd_klen <<= 1;
-		g_eli_crypto_ivgen(sc, dstoff, crde->crd_iv,
-		    sizeof(crde->crd_iv));
+		crp->crp_digest_start = 0;
+		crp->crp_payload_start = sc->sc_alen;
+		crp->crp_payload_length = data_secsize;
+		crp->crp_flags |= CRYPTO_F_IV_SEPARATE;
+		if ((sc->sc_flags & G_ELI_FLAG_FIRST_KEY) == 0) {
+			crp->crp_cipher_key = g_eli_key_hold(sc, dstoff,
+			    encr_secsize);
+		}
+		g_eli_crypto_ivgen(sc, dstoff, crp->crp_iv,
+		    sizeof(crp->crp_iv));
 
-		crda->crd_skip = sc->sc_alen;
-		crda->crd_len = data_secsize;
-		crda->crd_inject = 0;
-		crda->crd_flags = CRD_F_KEY_EXPLICIT;
-		crda->crd_alg = sc->sc_aalgo;
 		g_eli_auth_keygen(sc, dstoff, authkey);
-		crda->crd_key = authkey;
-		crda->crd_klen = G_ELI_AUTH_SECKEYLEN * 8;
+		crp->crp_auth_key = authkey;
 
-		crp->crp_etype = 0;
 		error = crypto_dispatch(crp);
 		KASSERT(error == 0, ("crypto_dispatch() failed (error=%d)",
 		    error));
 	}
 }
Index: head/sys/geom/eli/g_eli_privacy.c
===================================================================
--- head/sys/geom/eli/g_eli_privacy.c	(revision 359373)
+++ head/sys/geom/eli/g_eli_privacy.c	(revision 359374)
@@ -1,319 +1,298 @@
 /*-
  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
  *
  * Copyright (c) 2005-2011 Pawel Jakub Dawidek <pawel@dawidek.net>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/linker.h>
 #include <sys/module.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <sys/bio.h>
 #include <sys/sysctl.h>
 #include <sys/malloc.h>
 #include <sys/kthread.h>
 #include <sys/proc.h>
 #include <sys/sched.h>
 #include <sys/smp.h>
 #include <sys/vnode.h>
 
 #include <vm/uma.h>
 
 #include <geom/geom.h>
 #include <geom/geom_dbg.h>
 #include <geom/eli/g_eli.h>
 #include <geom/eli/pkcs5v2.h>
 
 /*
  * Code paths:
  * BIO_READ:
  *	g_eli_start -> g_eli_crypto_read -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> g_eli_crypto_read_done -> g_io_deliver
  * BIO_WRITE:
  *	g_eli_start -> g_eli_crypto_run -> g_eli_crypto_write_done -> g_io_request -> g_eli_write_done -> g_io_deliver
  */
 
 MALLOC_DECLARE(M_ELI);
 
 /*
  * The function is called after we read and decrypt data.
  *
  * g_eli_start -> g_eli_crypto_read -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> G_ELI_CRYPTO_READ_DONE -> g_io_deliver
  */
 static int
 g_eli_crypto_read_done(struct cryptop *crp)
 {
 	struct g_eli_softc *sc;
 	struct bio *bp;
 
 	if (crp->crp_etype == EAGAIN) {
 		if (g_eli_crypto_rerun(crp) == 0)
 			return (0);
 	}
 	bp = (struct bio *)crp->crp_opaque;
 	bp->bio_inbed++;
 	if (crp->crp_etype == 0) {
 		G_ELI_DEBUG(3, "Crypto READ request done (%d/%d).",
 		    bp->bio_inbed, bp->bio_children);
-		bp->bio_completed += crp->crp_olen;
+		bp->bio_completed += crp->crp_ilen;
 	} else {
 		G_ELI_DEBUG(1, "Crypto READ request failed (%d/%d) error=%d.",
 		    bp->bio_inbed, bp->bio_children, crp->crp_etype);
 		if (bp->bio_error == 0)
 			bp->bio_error = crp->crp_etype;
 	}
 	sc = bp->bio_to->geom->softc;
-	if (sc != NULL)
-		g_eli_key_drop(sc, crp->crp_desc->crd_key);
+	if (sc != NULL && crp->crp_cipher_key != NULL)
+		g_eli_key_drop(sc, __DECONST(void *, crp->crp_cipher_key));
+	crypto_freereq(crp);
 	/*
 	 * Do we have all sectors already?
 	 */
 	if (bp->bio_inbed < bp->bio_children)
 		return (0);
 	free(bp->bio_driver2, M_ELI);
 	bp->bio_driver2 = NULL;
 	if (bp->bio_error != 0) {
 		G_ELI_LOGREQ(0, bp, "Crypto READ request failed (error=%d).",
 		    bp->bio_error);
 		bp->bio_completed = 0;
 	}
 	/*
 	 * Read is finished, send it up.
 	 */
 	g_io_deliver(bp, bp->bio_error);
 	if (sc != NULL)
 		atomic_subtract_int(&sc->sc_inflight, 1);
 	return (0);
 }
 
 /*
  * The function is called after data encryption.
  *
  * g_eli_start -> g_eli_crypto_run -> G_ELI_CRYPTO_WRITE_DONE -> g_io_request -> g_eli_write_done -> g_io_deliver
  */
 static int
 g_eli_crypto_write_done(struct cryptop *crp)
 {
 	struct g_eli_softc *sc;
 	struct g_geom *gp;
 	struct g_consumer *cp;
 	struct bio *bp, *cbp;
 
 	if (crp->crp_etype == EAGAIN) {
 		if (g_eli_crypto_rerun(crp) == 0)
 			return (0);
 	}
 	bp = (struct bio *)crp->crp_opaque;
 	bp->bio_inbed++;
 	if (crp->crp_etype == 0) {
 		G_ELI_DEBUG(3, "Crypto WRITE request done (%d/%d).",
 		    bp->bio_inbed, bp->bio_children);
 	} else {
 		G_ELI_DEBUG(1, "Crypto WRITE request failed (%d/%d) error=%d.",
 		    bp->bio_inbed, bp->bio_children, crp->crp_etype);
 		if (bp->bio_error == 0)
 			bp->bio_error = crp->crp_etype;
 	}
 	gp = bp->bio_to->geom;
 	sc = gp->softc;
-	g_eli_key_drop(sc, crp->crp_desc->crd_key);
+	if (crp->crp_cipher_key != NULL)
+		g_eli_key_drop(sc, __DECONST(void *, crp->crp_cipher_key));
+	crypto_freereq(crp);
 	/*
 	 * All sectors are already encrypted?
 	 */
 	if (bp->bio_inbed < bp->bio_children)
 		return (0);
 	bp->bio_inbed = 0;
 	bp->bio_children = 1;
 	cbp = bp->bio_driver1;
 	bp->bio_driver1 = NULL;
 	if (bp->bio_error != 0) {
 		G_ELI_LOGREQ(0, bp, "Crypto WRITE request failed (error=%d).",
 		    bp->bio_error);
 		free(bp->bio_driver2, M_ELI);
 		bp->bio_driver2 = NULL;
 		g_destroy_bio(cbp);
 		g_io_deliver(bp, bp->bio_error);
 		atomic_subtract_int(&sc->sc_inflight, 1);
 		return (0);
 	}
 	cbp->bio_data = bp->bio_driver2;
 	cbp->bio_done = g_eli_write_done;
 	cp = LIST_FIRST(&gp->consumer);
 	cbp->bio_to = cp->provider;
 	G_ELI_LOGREQ(2, cbp, "Sending request.");
 	/*
 	 * Send encrypted data to the provider.
 	 */
 	g_io_request(cbp, cp);
 	return (0);
 }
 
 /*
  * The function is called to read encrypted data.
  *
  * g_eli_start -> G_ELI_CRYPTO_READ -> g_io_request -> g_eli_read_done -> g_eli_crypto_run -> g_eli_crypto_read_done -> g_io_deliver
  */
 void
 g_eli_crypto_read(struct g_eli_softc *sc, struct bio *bp, boolean_t fromworker)
 {
 	struct g_consumer *cp;
 	struct bio *cbp;
 
 	if (!fromworker) {
 		/*
 		 * We are not called from the worker thread, so check if
 		 * device is suspended.
 		 */
 		mtx_lock(&sc->sc_queue_mtx);
 		if (sc->sc_flags & G_ELI_FLAG_SUSPEND) {
 			/*
 			 * If device is suspended, we place the request onto
 			 * the queue, so it can be handled after resume.
 			 */
 			G_ELI_DEBUG(0, "device suspended, move onto queue");
 			bioq_insert_tail(&sc->sc_queue, bp);
 			mtx_unlock(&sc->sc_queue_mtx);
 			wakeup(sc);
 			return;
 		}
 		atomic_add_int(&sc->sc_inflight, 1);
 		mtx_unlock(&sc->sc_queue_mtx);
 	}
 	bp->bio_pflags = 0;
 	bp->bio_driver2 = NULL;
 	cbp = bp->bio_driver1;
 	cbp->bio_done = g_eli_read_done;
 	cp = LIST_FIRST(&sc->sc_geom->consumer);
 	cbp->bio_to = cp->provider;
 	G_ELI_LOGREQ(2, cbp, "Sending request.");
 	/*
 	 * Read encrypted data from provider.
 	 */
 	g_io_request(cbp, cp);
 }
 
 /*
  * This is the main function responsible for cryptography (ie. communication
  * with crypto(9) subsystem).
  *
  * BIO_READ:
  *	g_eli_start -> g_eli_crypto_read -> g_io_request -> g_eli_read_done -> G_ELI_CRYPTO_RUN -> g_eli_crypto_read_done -> g_io_deliver
  * BIO_WRITE:
  *	g_eli_start -> G_ELI_CRYPTO_RUN -> g_eli_crypto_write_done -> g_io_request -> g_eli_write_done -> g_io_deliver
  */
 void
 g_eli_crypto_run(struct g_eli_worker *wr, struct bio *bp)
 {
 	struct g_eli_softc *sc;
 	struct cryptop *crp;
-	struct cryptodesc *crd;
 	u_int i, nsec, secsize;
 	off_t dstoff;
-	size_t size;
-	u_char *p, *data;
+	u_char *data;
 	int error;
 
 	G_ELI_LOGREQ(3, bp, "%s", __func__);
 
 	bp->bio_pflags = wr->w_number;
 	sc = wr->w_softc;
 	secsize = LIST_FIRST(&sc->sc_geom->provider)->sectorsize;
 	nsec = bp->bio_length / secsize;
 
+	bp->bio_inbed = 0;
+	bp->bio_children = nsec;
+
 	/*
-	 * Calculate how much memory do we need.
-	 * We need separate crypto operation for every single sector.
-	 * It is much faster to calculate total amount of needed memory here and
-	 * do the allocation once instead of allocating memory in pieces (many,
-	 * many pieces).
-	 */
-	size = sizeof(*crp) * nsec;
-	size += sizeof(*crd) * nsec;
-	/*
 	 * If we write the data we cannot destroy current bio_data content,
 	 * so we need to allocate more memory for encrypted data.
 	 */
-	if (bp->bio_cmd == BIO_WRITE)
-		size += bp->bio_length;
-	p = malloc(size, M_ELI, M_WAITOK);
-
-	bp->bio_inbed = 0;
-	bp->bio_children = nsec;
-	bp->bio_driver2 = p;
-
-	if (bp->bio_cmd == BIO_READ)
-		data = bp->bio_data;
-	else {
-		data = p;
-		p += bp->bio_length;
+	if (bp->bio_cmd == BIO_WRITE) {
+		data = malloc(bp->bio_length, M_ELI, M_WAITOK);
+		bp->bio_driver2 = data;
 		bcopy(bp->bio_data, data, bp->bio_length);
-	}
+	} else
+		data = bp->bio_data;
 
 	for (i = 0, dstoff = bp->bio_offset; i < nsec; i++, dstoff += secsize) {
-		crp = (struct cryptop *)p;	p += sizeof(*crp);
-		crd = (struct cryptodesc *)p;	p += sizeof(*crd);
+		crp = crypto_getreq(wr->w_sid, M_WAITOK);
 
-		crp->crp_session = wr->w_sid;
 		crp->crp_ilen = secsize;
-		crp->crp_olen = secsize;
 		crp->crp_opaque = (void *)bp;
+		crp->crp_buf_type = CRYPTO_BUF_CONTIG;
 		crp->crp_buf = (void *)data;
 		data += secsize;
-		if (bp->bio_cmd == BIO_WRITE)
+		if (bp->bio_cmd == BIO_WRITE) {
+			crp->crp_op = CRYPTO_OP_ENCRYPT;
 			crp->crp_callback = g_eli_crypto_write_done;
-		else /* if (bp->bio_cmd == BIO_READ) */
+		} else /* if (bp->bio_cmd == BIO_READ) */ {
+			crp->crp_op = CRYPTO_OP_DECRYPT;
 			crp->crp_callback = g_eli_crypto_read_done;
+		}
 		crp->crp_flags = CRYPTO_F_CBIFSYNC;
 		if (g_eli_batch)
 			crp->crp_flags |= CRYPTO_F_BATCH;
-		crp->crp_desc = crd;
 
-		crd->crd_skip = 0;
-		crd->crd_len = secsize;
-		crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
-		if ((sc->sc_flags & G_ELI_FLAG_SINGLE_KEY) == 0)
-			crd->crd_flags |= CRD_F_KEY_EXPLICIT;
-		if (bp->bio_cmd == BIO_WRITE)
-			crd->crd_flags |= CRD_F_ENCRYPT;
-		crd->crd_alg = sc->sc_ealgo;
-		crd->crd_key = g_eli_key_hold(sc, dstoff, secsize);
-		crd->crd_klen = sc->sc_ekeylen;
-		if (sc->sc_ealgo == CRYPTO_AES_XTS)
-			crd->crd_klen <<= 1;
-		g_eli_crypto_ivgen(sc, dstoff, crd->crd_iv,
-		    sizeof(crd->crd_iv));
-		crd->crd_next = NULL;
+		crp->crp_payload_start = 0;
+		crp->crp_payload_length = secsize;
+		crp->crp_flags |= CRYPTO_F_IV_SEPARATE;
+		if ((sc->sc_flags & G_ELI_FLAG_SINGLE_KEY) == 0) {
+			crp->crp_cipher_key = g_eli_key_hold(sc, dstoff,
+			    secsize);
+		}
+		g_eli_crypto_ivgen(sc, dstoff, crp->crp_iv,
+		    sizeof(crp->crp_iv));
 
-		crp->crp_etype = 0;
 		error = crypto_dispatch(crp);
 		KASSERT(error == 0, ("crypto_dispatch() failed (error=%d)",
 		    error));
 	}
 }
Index: head/sys/kern/subr_bus_dma.c
===================================================================
--- head/sys/kern/subr_bus_dma.c	(revision 359373)
+++ head/sys/kern/subr_bus_dma.c	(revision 359374)
@@ -1,637 +1,688 @@
 /*-
  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
  *
  * Copyright (c) 2012 EMC Corp.
  * All rights reserved.
  *
  * Copyright (c) 1997, 1998 Justin T. Gibbs.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include "opt_bus.h"
 
 #include <sys/param.h>
 #include <sys/conf.h>
 #include <sys/systm.h>
 #include <sys/bio.h>
 #include <sys/bus.h>
 #include <sys/callout.h>
 #include <sys/ktr.h>
 #include <sys/mbuf.h>
 #include <sys/memdesc.h>
 #include <sys/proc.h>
 #include <sys/uio.h>
 
 #include <vm/vm.h>
 #include <vm/vm_page.h>
 #include <vm/vm_map.h>
 #include <vm/pmap.h>
 
 #include <cam/cam.h>
 #include <cam/cam_ccb.h>
 
+#include <opencrypto/cryptodev.h>
+
 #include <machine/bus.h>
 
 /*
  * Load up data starting at offset within a region specified by a
  * list of virtual address ranges until either length or the region
  * are exhausted.
  */
 static int
 _bus_dmamap_load_vlist(bus_dma_tag_t dmat, bus_dmamap_t map,
     bus_dma_segment_t *list, int sglist_cnt, struct pmap *pmap, int *nsegs,
     int flags, size_t offset, size_t length)
 {
 	int error;
 
 	error = 0;
 	for (; sglist_cnt > 0 && length != 0; sglist_cnt--, list++) {
 		char *addr;
 		size_t ds_len;
 
 		KASSERT((offset < list->ds_len),
 		    ("Invalid mid-segment offset"));
 		addr = (char *)(uintptr_t)list->ds_addr + offset;
 		ds_len = list->ds_len - offset;
 		offset = 0;
 		if (ds_len > length)
 			ds_len = length;
 		length -= ds_len;
 		KASSERT((ds_len != 0), ("Segment length is zero"));
 		error = _bus_dmamap_load_buffer(dmat, map, addr, ds_len, pmap,
 		    flags, NULL, nsegs);
 		if (error)
 			break;
 	}
 	return (error);
 }
 
 /*
  * Load a list of physical addresses.
  */
 static int
 _bus_dmamap_load_plist(bus_dma_tag_t dmat, bus_dmamap_t map,
     bus_dma_segment_t *list, int sglist_cnt, int *nsegs, int flags)
 {
 	int error;
 
 	error = 0;
 	for (; sglist_cnt > 0; sglist_cnt--, list++) {
 		error = _bus_dmamap_load_phys(dmat, map,
 		    (vm_paddr_t)list->ds_addr, list->ds_len, flags, NULL,
 		    nsegs);
 		if (error)
 			break;
 	}
 	return (error);
 }
 
 /*
  * Load an unmapped mbuf
  */
 static int
 _bus_dmamap_load_unmapped_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map,
     struct mbuf *m, bus_dma_segment_t *segs, int *nsegs, int flags)
 {
 	struct mbuf_ext_pgs *ext_pgs;
 	int error, i, off, len, pglen, pgoff, seglen, segoff;
 
 	MBUF_EXT_PGS_ASSERT(m);
 	ext_pgs = m->m_ext.ext_pgs;
 
 	len = m->m_len;
 	error = 0;
 
 	/* Skip over any data removed from the front. */
 	off = mtod(m, vm_offset_t);
 
 	if (ext_pgs->hdr_len != 0) {
 		if (off >= ext_pgs->hdr_len) {
 			off -= ext_pgs->hdr_len;
 		} else {
 			seglen = ext_pgs->hdr_len - off;
 			segoff = off;
 			seglen = min(seglen, len);
 			off = 0;
 			len -= seglen;
 			error = _bus_dmamap_load_buffer(dmat, map,
 			    &ext_pgs->hdr[segoff], seglen, kernel_pmap,
 			    flags, segs, nsegs);
 		}
 	}
 	pgoff = ext_pgs->first_pg_off;
 	for (i = 0; i < ext_pgs->npgs && error == 0 && len > 0; i++) {
 		pglen = mbuf_ext_pg_len(ext_pgs, i, pgoff);
 		if (off >= pglen) {
 			off -= pglen;
 			pgoff = 0;
 			continue;
 		}
 		seglen = pglen - off;
 		segoff = pgoff + off;
 		off = 0;
 		seglen = min(seglen, len);
 		len -= seglen;
 		error = _bus_dmamap_load_phys(dmat, map,
 		    ext_pgs->pa[i] + segoff, seglen, flags, segs, nsegs);
 		pgoff = 0;
 	};
 	if (len != 0 && error == 0) {
 		KASSERT((off + len) <= ext_pgs->trail_len,
 		    ("off + len > trail (%d + %d > %d)", off, len,
 		    ext_pgs->trail_len));
 		error = _bus_dmamap_load_buffer(dmat, map,
 		    &ext_pgs->trail[off], len, kernel_pmap, flags, segs,
 		    nsegs);
 	}
 	return (error);
 }
 
 /*
  * Load an mbuf chain.
  */
 static int
 _bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map,
     struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs, int flags)
 {
 	struct mbuf *m;
 	int error;
 
 	error = 0;
 	for (m = m0; m != NULL && error == 0; m = m->m_next) {
 		if (m->m_len > 0) {
 			if ((m->m_flags & M_NOMAP) != 0)
 				error = _bus_dmamap_load_unmapped_mbuf_sg(dmat,
 				    map, m, segs, nsegs, flags);
 			else
 				error = _bus_dmamap_load_buffer(dmat, map,
 				    m->m_data, m->m_len, kernel_pmap,
 				    flags | BUS_DMA_LOAD_MBUF, segs, nsegs);
 		}
 	}
 	CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
 	    __func__, dmat, flags, error, *nsegs);
 	return (error);
 }
 
 /*
  * Load from block io.
  */
 static int
 _bus_dmamap_load_bio(bus_dma_tag_t dmat, bus_dmamap_t map, struct bio *bio,
     int *nsegs, int flags)
 {
 
 	if ((bio->bio_flags & BIO_VLIST) != 0) {
 		bus_dma_segment_t *segs = (bus_dma_segment_t *)bio->bio_data;
 		return (_bus_dmamap_load_vlist(dmat, map, segs, bio->bio_ma_n,
 		    kernel_pmap, nsegs, flags, bio->bio_ma_offset,
 		    bio->bio_bcount));
 	}
 
 	if ((bio->bio_flags & BIO_UNMAPPED) != 0)
 		return (_bus_dmamap_load_ma(dmat, map, bio->bio_ma,
 		    bio->bio_bcount, bio->bio_ma_offset, flags, NULL, nsegs));
 
 	return (_bus_dmamap_load_buffer(dmat, map, bio->bio_data,
 	    bio->bio_bcount, kernel_pmap, flags, NULL, nsegs));
 }
 
 int
 bus_dmamap_load_ma_triv(bus_dma_tag_t dmat, bus_dmamap_t map,
     struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags,
     bus_dma_segment_t *segs, int *segp)
 {
 	vm_paddr_t paddr;
 	bus_size_t len;
 	int error, i;
 
 	error = 0;
 	for (i = 0; tlen > 0; i++, tlen -= len) {
 		len = min(PAGE_SIZE - ma_offs, tlen);
 		paddr = VM_PAGE_TO_PHYS(ma[i]) + ma_offs;
 		error = _bus_dmamap_load_phys(dmat, map, paddr, len,
 		    flags, segs, segp);
 		if (error != 0)
 			break;
 		ma_offs = 0;
 	}
 	return (error);
 }
 
 /*
  * Load a cam control block.
  */
 static int
 _bus_dmamap_load_ccb(bus_dma_tag_t dmat, bus_dmamap_t map, union ccb *ccb,
 		    int *nsegs, int flags)
 {
 	struct ccb_hdr *ccb_h;
 	void *data_ptr;
 	int error;
 	uint32_t dxfer_len;
 	uint16_t sglist_cnt;
 
 	error = 0;
 	ccb_h = &ccb->ccb_h;
 	switch (ccb_h->func_code) {
 	case XPT_SCSI_IO: {
 		struct ccb_scsiio *csio;
 
 		csio = &ccb->csio;
 		data_ptr = csio->data_ptr;
 		dxfer_len = csio->dxfer_len;
 		sglist_cnt = csio->sglist_cnt;
 		break;
 	}
 	case XPT_CONT_TARGET_IO: {
 		struct ccb_scsiio *ctio;
 
 		ctio = &ccb->ctio;
 		data_ptr = ctio->data_ptr;
 		dxfer_len = ctio->dxfer_len;
 		sglist_cnt = ctio->sglist_cnt;
 		break;
 	}
 	case XPT_ATA_IO: {
 		struct ccb_ataio *ataio;
 
 		ataio = &ccb->ataio;
 		data_ptr = ataio->data_ptr;
 		dxfer_len = ataio->dxfer_len;
 		sglist_cnt = 0;
 		break;
 	}
 	case XPT_NVME_IO:
 	case XPT_NVME_ADMIN: {
 		struct ccb_nvmeio *nvmeio;
 
 		nvmeio = &ccb->nvmeio;
 		data_ptr = nvmeio->data_ptr;
 		dxfer_len = nvmeio->dxfer_len;
 		sglist_cnt = nvmeio->sglist_cnt;
 		break;
 	}
 	default:
 		panic("_bus_dmamap_load_ccb: Unsupported func code %d",
 		    ccb_h->func_code);
 	}
 
 	switch ((ccb_h->flags & CAM_DATA_MASK)) {
 	case CAM_DATA_VADDR:
 		error = _bus_dmamap_load_buffer(dmat, map, data_ptr, dxfer_len,
 		    kernel_pmap, flags, NULL, nsegs);
 		break;
 	case CAM_DATA_PADDR:
 		error = _bus_dmamap_load_phys(dmat, map,
 		    (vm_paddr_t)(uintptr_t)data_ptr, dxfer_len, flags, NULL,
 		    nsegs);
 		break;
 	case CAM_DATA_SG:
 		error = _bus_dmamap_load_vlist(dmat, map,
 		    (bus_dma_segment_t *)data_ptr, sglist_cnt, kernel_pmap,
 		    nsegs, flags, 0, dxfer_len);
 		break;
 	case CAM_DATA_SG_PADDR:
 		error = _bus_dmamap_load_plist(dmat, map,
 		    (bus_dma_segment_t *)data_ptr, sglist_cnt, nsegs, flags);
 		break;
 	case CAM_DATA_BIO:
 		error = _bus_dmamap_load_bio(dmat, map, (struct bio *)data_ptr,
 		    nsegs, flags);
 		break;
 	default:
 		panic("_bus_dmamap_load_ccb: flags 0x%X unimplemented",
 		    ccb_h->flags);
 	}
 	return (error);
 }
 
 /*
  * Load a uio.
  */
 static int
 _bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map, struct uio *uio,
     int *nsegs, int flags)
 {
 	bus_size_t resid;
 	bus_size_t minlen;
 	struct iovec *iov;
 	pmap_t pmap;
 	caddr_t addr;
 	int error, i;
 
 	if (uio->uio_segflg == UIO_USERSPACE) {
 		KASSERT(uio->uio_td != NULL,
 			("bus_dmamap_load_uio: USERSPACE but no proc"));
 		pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace);
 	} else
 		pmap = kernel_pmap;
 	resid = uio->uio_resid;
 	iov = uio->uio_iov;
 	error = 0;
 
 	for (i = 0; i < uio->uio_iovcnt && resid != 0 && !error; i++) {
 		/*
 		 * Now at the first iovec to load.  Load each iovec
 		 * until we have exhausted the residual count.
 		 */
 
 		addr = (caddr_t) iov[i].iov_base;
 		minlen = resid < iov[i].iov_len ? resid : iov[i].iov_len;
 		if (minlen > 0) {
 			error = _bus_dmamap_load_buffer(dmat, map, addr,
 			    minlen, pmap, flags, NULL, nsegs);
 			resid -= minlen;
 		}
 	}
 
 	return (error);
 }
 
 /*
  * Map the buffer buf into bus space using the dmamap map.
  */
 int
 bus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,
     bus_size_t buflen, bus_dmamap_callback_t *callback,
     void *callback_arg, int flags)
 {
 	bus_dma_segment_t *segs;
 	struct memdesc mem;
 	int error;
 	int nsegs;
 
 	if ((flags & BUS_DMA_NOWAIT) == 0) {
 		mem = memdesc_vaddr(buf, buflen);
 		_bus_dmamap_waitok(dmat, map, &mem, callback, callback_arg);
 	}
 
 	nsegs = -1;
 	error = _bus_dmamap_load_buffer(dmat, map, buf, buflen, kernel_pmap,
 	    flags, NULL, &nsegs);
 	nsegs++;
 
 	CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
 	    __func__, dmat, flags, error, nsegs);
 
 	if (error == EINPROGRESS)
 		return (error);
 
 	segs = _bus_dmamap_complete(dmat, map, NULL, nsegs, error);
 	if (error)
 		(*callback)(callback_arg, segs, 0, error);
 	else
 		(*callback)(callback_arg, segs, nsegs, 0);
 
 	/*
 	 * Return ENOMEM to the caller so that it can pass it up the stack.
 	 * This error only happens when NOWAIT is set, so deferral is disabled.
 	 */
 	if (error == ENOMEM)
 		return (error);
 
 	return (0);
 }
 
 int
 bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m0,
     bus_dmamap_callback2_t *callback, void *callback_arg, int flags)
 {
 	bus_dma_segment_t *segs;
 	int nsegs, error;
 
 	M_ASSERTPKTHDR(m0);
 
 	flags |= BUS_DMA_NOWAIT;
 	nsegs = -1;
 	error = _bus_dmamap_load_mbuf_sg(dmat, map, m0, NULL, &nsegs, flags);
 	++nsegs;
 
 	segs = _bus_dmamap_complete(dmat, map, NULL, nsegs, error);
 	if (error)
 		(*callback)(callback_arg, segs, 0, 0, error);
 	else
 		(*callback)(callback_arg, segs, nsegs, m0->m_pkthdr.len, error);
 
 	CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
 	    __func__, dmat, flags, error, nsegs);
 	return (error);
 }
 
 int
 bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m0,
     bus_dma_segment_t *segs, int *nsegs, int flags)
 {
 	int error;
 
 	flags |= BUS_DMA_NOWAIT;
 	*nsegs = -1;
 	error = _bus_dmamap_load_mbuf_sg(dmat, map, m0, segs, nsegs, flags);
 	++*nsegs;
 	_bus_dmamap_complete(dmat, map, segs, *nsegs, error);
 	return (error);
 }
 
 int
 bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map, struct uio *uio,
     bus_dmamap_callback2_t *callback, void *callback_arg, int flags)
 {
 	bus_dma_segment_t *segs;
 	int nsegs, error;
 
 	flags |= BUS_DMA_NOWAIT;
 	nsegs = -1;
 	error = _bus_dmamap_load_uio(dmat, map, uio, &nsegs, flags);
 	nsegs++;
 
 	segs = _bus_dmamap_complete(dmat, map, NULL, nsegs, error);
 	if (error)
 		(*callback)(callback_arg, segs, 0, 0, error);
 	else
 		(*callback)(callback_arg, segs, nsegs, uio->uio_resid, error);
 
 	CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
 	    __func__, dmat, flags, error, nsegs);
 	return (error);
 }
 
 int
 bus_dmamap_load_ccb(bus_dma_tag_t dmat, bus_dmamap_t map, union ccb *ccb,
 		    bus_dmamap_callback_t *callback, void *callback_arg,
 		    int flags)
 {
 	bus_dma_segment_t *segs;
 	struct ccb_hdr *ccb_h;
 	struct memdesc mem;
 	int error;
 	int nsegs;
 
 	ccb_h = &ccb->ccb_h;
 	if ((ccb_h->flags & CAM_DIR_MASK) == CAM_DIR_NONE) {
 		callback(callback_arg, NULL, 0, 0);
 		return (0);
 	}
 	if ((flags & BUS_DMA_NOWAIT) == 0) {
 		mem = memdesc_ccb(ccb);
 		_bus_dmamap_waitok(dmat, map, &mem, callback, callback_arg);
 	}
 	nsegs = -1;
 	error = _bus_dmamap_load_ccb(dmat, map, ccb, &nsegs, flags);
 	nsegs++;
 
 	CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
 	    __func__, dmat, flags, error, nsegs);
 
 	if (error == EINPROGRESS)
 		return (error);
 
 	segs = _bus_dmamap_complete(dmat, map, NULL, nsegs, error);
 	if (error)
 		(*callback)(callback_arg, segs, 0, error);
 	else
 		(*callback)(callback_arg, segs, nsegs, error);
 	/*
 	 * Return ENOMEM to the caller so that it can pass it up the stack.
 	 * This error only happens when NOWAIT is set, so deferral is disabled.
 	 */
 	if (error == ENOMEM)
 		return (error);
 
 	return (0);
 }
 
 int
 bus_dmamap_load_bio(bus_dma_tag_t dmat, bus_dmamap_t map, struct bio *bio,
 		    bus_dmamap_callback_t *callback, void *callback_arg,
 		    int flags)
 {
 	bus_dma_segment_t *segs;
 	struct memdesc mem;
 	int error;
 	int nsegs;
 
 	if ((flags & BUS_DMA_NOWAIT) == 0) {
 		mem = memdesc_bio(bio);
 		_bus_dmamap_waitok(dmat, map, &mem, callback, callback_arg);
 	}
 	nsegs = -1;
 	error = _bus_dmamap_load_bio(dmat, map, bio, &nsegs, flags);
 	nsegs++;
 
 	CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
 	    __func__, dmat, flags, error, nsegs);
 
 	if (error == EINPROGRESS)
 		return (error);
 
 	segs = _bus_dmamap_complete(dmat, map, NULL, nsegs, error);
 	if (error)
 		(*callback)(callback_arg, segs, 0, error);
 	else
 		(*callback)(callback_arg, segs, nsegs, error);
 	/*
 	 * Return ENOMEM to the caller so that it can pass it up the stack.
 	 * This error only happens when NOWAIT is set, so deferral is disabled.
 	 */
 	if (error == ENOMEM)
 		return (error);
 
 	return (0);
 }
 
 int
 bus_dmamap_load_mem(bus_dma_tag_t dmat, bus_dmamap_t map,
     struct memdesc *mem, bus_dmamap_callback_t *callback,
     void *callback_arg, int flags)
 {
 	bus_dma_segment_t *segs;
 	int error;
 	int nsegs;
 
 	if ((flags & BUS_DMA_NOWAIT) == 0)
 		_bus_dmamap_waitok(dmat, map, mem, callback, callback_arg);
 
 	nsegs = -1;
 	error = 0;
 	switch (mem->md_type) {
 	case MEMDESC_VADDR:
 		error = _bus_dmamap_load_buffer(dmat, map, mem->u.md_vaddr,
 		    mem->md_opaque, kernel_pmap, flags, NULL, &nsegs);
 		break;
 	case MEMDESC_PADDR:
 		error = _bus_dmamap_load_phys(dmat, map, mem->u.md_paddr,
 		    mem->md_opaque, flags, NULL, &nsegs);
 		break;
 	case MEMDESC_VLIST:
 		error = _bus_dmamap_load_vlist(dmat, map, mem->u.md_list,
 		    mem->md_opaque, kernel_pmap, &nsegs, flags, 0, SIZE_T_MAX);
 		break;
 	case MEMDESC_PLIST:
 		error = _bus_dmamap_load_plist(dmat, map, mem->u.md_list,
 		    mem->md_opaque, &nsegs, flags);
 		break;
 	case MEMDESC_BIO:
 		error = _bus_dmamap_load_bio(dmat, map, mem->u.md_bio,
 		    &nsegs, flags);
 		break;
 	case MEMDESC_UIO:
 		error = _bus_dmamap_load_uio(dmat, map, mem->u.md_uio,
 		    &nsegs, flags);
 		break;
 	case MEMDESC_MBUF:
 		error = _bus_dmamap_load_mbuf_sg(dmat, map, mem->u.md_mbuf,
 		    NULL, &nsegs, flags);
 		break;
 	case MEMDESC_CCB:
 		error = _bus_dmamap_load_ccb(dmat, map, mem->u.md_ccb, &nsegs,
+		    flags);
+		break;
+	}
+	nsegs++;
+
+	CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
+	    __func__, dmat, flags, error, nsegs);
+
+	if (error == EINPROGRESS)
+		return (error);
+
+	segs = _bus_dmamap_complete(dmat, map, NULL, nsegs, error);
+	if (error)
+		(*callback)(callback_arg, segs, 0, error);
+	else
+		(*callback)(callback_arg, segs, nsegs, 0);
+
+	/*
+	 * Return ENOMEM to the caller so that it can pass it up the stack.
+	 * This error only happens when NOWAIT is set, so deferral is disabled.
+	 */
+	if (error == ENOMEM)
+		return (error);
+
+	return (0);
+}
+
+int
+bus_dmamap_load_crp(bus_dma_tag_t dmat, bus_dmamap_t map, struct cryptop *crp,
+    bus_dmamap_callback_t *callback, void *callback_arg, int flags)
+{
+	bus_dma_segment_t *segs;
+	int error;
+	int nsegs;
+
+	flags |= BUS_DMA_NOWAIT;
+	nsegs = -1;
+	error = 0;
+	switch (crp->crp_buf_type) {
+	case CRYPTO_BUF_CONTIG:
+		error = _bus_dmamap_load_buffer(dmat, map, crp->crp_buf,
+		    crp->crp_ilen, kernel_pmap, flags, NULL, &nsegs);
+		break;
+	case CRYPTO_BUF_MBUF:
+		error = _bus_dmamap_load_mbuf_sg(dmat, map, crp->crp_mbuf,
+		    NULL, &nsegs, flags);
+		break;
+	case CRYPTO_BUF_UIO:
+		error = _bus_dmamap_load_uio(dmat, map, crp->crp_uio, &nsegs,
 		    flags);
 		break;
 	}
 	nsegs++;
 
 	CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d",
 	    __func__, dmat, flags, error, nsegs);
 
 	if (error == EINPROGRESS)
 		return (error);
 
 	segs = _bus_dmamap_complete(dmat, map, NULL, nsegs, error);
 	if (error)
 		(*callback)(callback_arg, segs, 0, error);
 	else
 		(*callback)(callback_arg, segs, nsegs, 0);
 
 	/*
 	 * Return ENOMEM to the caller so that it can pass it up the stack.
 	 * This error only happens when NOWAIT is set, so deferral is disabled.
 	 */
 	if (error == ENOMEM)
 		return (error);
 
 	return (0);
 }
Index: head/sys/kern/uipc_ktls.c
===================================================================
--- head/sys/kern/uipc_ktls.c	(revision 359373)
+++ head/sys/kern/uipc_ktls.c	(revision 359374)
@@ -1,1584 +1,1587 @@
 /*-
  * SPDX-License-Identifier: BSD-2-Clause
  *
  * Copyright (c) 2014-2019 Netflix Inc.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include "opt_inet.h"
 #include "opt_inet6.h"
 #include "opt_rss.h"
 
 #include <sys/param.h>
 #include <sys/kernel.h>
 #include <sys/ktls.h>
 #include <sys/lock.h>
 #include <sys/mbuf.h>
 #include <sys/mutex.h>
 #include <sys/rmlock.h>
 #include <sys/proc.h>
 #include <sys/protosw.h>
 #include <sys/refcount.h>
 #include <sys/smp.h>
 #include <sys/socket.h>
 #include <sys/socketvar.h>
 #include <sys/sysctl.h>
 #include <sys/taskqueue.h>
 #include <sys/kthread.h>
 #include <sys/uio.h>
 #include <sys/vmmeter.h>
 #if defined(__aarch64__) || defined(__amd64__) || defined(__i386__)
 #include <machine/pcb.h>
 #endif
 #include <machine/vmparam.h>
 #include <net/if.h>
 #include <net/if_var.h>
 #ifdef RSS
 #include <net/netisr.h>
 #include <net/rss_config.h>
 #endif
 #if defined(INET) || defined(INET6)
 #include <netinet/in.h>
 #include <netinet/in_pcb.h>
 #endif
 #include <netinet/tcp_var.h>
 #ifdef TCP_OFFLOAD
 #include <netinet/tcp_offload.h>
 #endif
 #include <opencrypto/xform.h>
 #include <vm/uma_dbg.h>
 #include <vm/vm.h>
 #include <vm/vm_pageout.h>
 #include <vm/vm_page.h>
 
 struct ktls_wq {
 	struct mtx	mtx;
 	STAILQ_HEAD(, mbuf_ext_pgs) head;
 	bool		running;
 } __aligned(CACHE_LINE_SIZE);
 
 static struct ktls_wq *ktls_wq;
 static struct proc *ktls_proc;
 LIST_HEAD(, ktls_crypto_backend) ktls_backends;
 static struct rmlock ktls_backends_lock;
 static uma_zone_t ktls_session_zone;
 static uint16_t ktls_cpuid_lookup[MAXCPU];
 
 SYSCTL_NODE(_kern_ipc, OID_AUTO, tls, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
     "Kernel TLS offload");
 SYSCTL_NODE(_kern_ipc_tls, OID_AUTO, stats, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
     "Kernel TLS offload stats");
 
 static int ktls_allow_unload;
 SYSCTL_INT(_kern_ipc_tls, OID_AUTO, allow_unload, CTLFLAG_RDTUN,
     &ktls_allow_unload, 0, "Allow software crypto modules to unload");
 
 #ifdef RSS
 static int ktls_bind_threads = 1;
 #else
 static int ktls_bind_threads;
 #endif
 SYSCTL_INT(_kern_ipc_tls, OID_AUTO, bind_threads, CTLFLAG_RDTUN,
     &ktls_bind_threads, 0,
     "Bind crypto threads to cores or domains at boot");
 
 static u_int ktls_maxlen = 16384;
 SYSCTL_UINT(_kern_ipc_tls, OID_AUTO, maxlen, CTLFLAG_RWTUN,
     &ktls_maxlen, 0, "Maximum TLS record size");
 
 static int ktls_number_threads;
 SYSCTL_INT(_kern_ipc_tls_stats, OID_AUTO, threads, CTLFLAG_RD,
     &ktls_number_threads, 0,
     "Number of TLS threads in thread-pool");
 
 static bool ktls_offload_enable;
 SYSCTL_BOOL(_kern_ipc_tls, OID_AUTO, enable, CTLFLAG_RW,
     &ktls_offload_enable, 0,
     "Enable support for kernel TLS offload");
 
 static bool ktls_cbc_enable = true;
 SYSCTL_BOOL(_kern_ipc_tls, OID_AUTO, cbc_enable, CTLFLAG_RW,
     &ktls_cbc_enable, 1,
     "Enable Support of AES-CBC crypto for kernel TLS");
 
 static counter_u64_t ktls_tasks_active;
 SYSCTL_COUNTER_U64(_kern_ipc_tls, OID_AUTO, tasks_active, CTLFLAG_RD,
     &ktls_tasks_active, "Number of active tasks");
 
 static counter_u64_t ktls_cnt_on;
 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, so_inqueue, CTLFLAG_RD,
     &ktls_cnt_on, "Number of TLS records in queue to tasks for SW crypto");
 
 static counter_u64_t ktls_offload_total;
 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, offload_total,
     CTLFLAG_RD, &ktls_offload_total,
     "Total successful TLS setups (parameters set)");
 
 static counter_u64_t ktls_offload_enable_calls;
 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, enable_calls,
     CTLFLAG_RD, &ktls_offload_enable_calls,
     "Total number of TLS enable calls made");
 
 static counter_u64_t ktls_offload_active;
 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, active, CTLFLAG_RD,
     &ktls_offload_active, "Total Active TLS sessions");
 
 static counter_u64_t ktls_offload_failed_crypto;
 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, failed_crypto, CTLFLAG_RD,
     &ktls_offload_failed_crypto, "Total TLS crypto failures");
 
 static counter_u64_t ktls_switch_to_ifnet;
 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, switch_to_ifnet, CTLFLAG_RD,
     &ktls_switch_to_ifnet, "TLS sessions switched from SW to ifnet");
 
 static counter_u64_t ktls_switch_to_sw;
 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, switch_to_sw, CTLFLAG_RD,
     &ktls_switch_to_sw, "TLS sessions switched from ifnet to SW");
 
 static counter_u64_t ktls_switch_failed;
 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats, OID_AUTO, switch_failed, CTLFLAG_RD,
     &ktls_switch_failed, "TLS sessions unable to switch between SW and ifnet");
 
 SYSCTL_NODE(_kern_ipc_tls, OID_AUTO, sw, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
     "Software TLS session stats");
 SYSCTL_NODE(_kern_ipc_tls, OID_AUTO, ifnet, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
     "Hardware (ifnet) TLS session stats");
 #ifdef TCP_OFFLOAD
 SYSCTL_NODE(_kern_ipc_tls, OID_AUTO, toe, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
     "TOE TLS session stats");
 #endif
 
 static counter_u64_t ktls_sw_cbc;
 SYSCTL_COUNTER_U64(_kern_ipc_tls_sw, OID_AUTO, cbc, CTLFLAG_RD, &ktls_sw_cbc,
     "Active number of software TLS sessions using AES-CBC");
 
 static counter_u64_t ktls_sw_gcm;
 SYSCTL_COUNTER_U64(_kern_ipc_tls_sw, OID_AUTO, gcm, CTLFLAG_RD, &ktls_sw_gcm,
     "Active number of software TLS sessions using AES-GCM");
 
 static counter_u64_t ktls_ifnet_cbc;
 SYSCTL_COUNTER_U64(_kern_ipc_tls_ifnet, OID_AUTO, cbc, CTLFLAG_RD,
     &ktls_ifnet_cbc,
     "Active number of ifnet TLS sessions using AES-CBC");
 
 static counter_u64_t ktls_ifnet_gcm;
 SYSCTL_COUNTER_U64(_kern_ipc_tls_ifnet, OID_AUTO, gcm, CTLFLAG_RD,
     &ktls_ifnet_gcm,
     "Active number of ifnet TLS sessions using AES-GCM");
 
 static counter_u64_t ktls_ifnet_reset;
 SYSCTL_COUNTER_U64(_kern_ipc_tls_ifnet, OID_AUTO, reset, CTLFLAG_RD,
     &ktls_ifnet_reset, "TLS sessions updated to a new ifnet send tag");
 
 static counter_u64_t ktls_ifnet_reset_dropped;
 SYSCTL_COUNTER_U64(_kern_ipc_tls_ifnet, OID_AUTO, reset_dropped, CTLFLAG_RD,
     &ktls_ifnet_reset_dropped,
     "TLS sessions dropped after failing to update ifnet send tag");
 
 static counter_u64_t ktls_ifnet_reset_failed;
 SYSCTL_COUNTER_U64(_kern_ipc_tls_ifnet, OID_AUTO, reset_failed, CTLFLAG_RD,
     &ktls_ifnet_reset_failed,
     "TLS sessions that failed to allocate a new ifnet send tag");
 
 static int ktls_ifnet_permitted;
 SYSCTL_UINT(_kern_ipc_tls_ifnet, OID_AUTO, permitted, CTLFLAG_RWTUN,
     &ktls_ifnet_permitted, 1,
     "Whether to permit hardware (ifnet) TLS sessions");
 
 #ifdef TCP_OFFLOAD
 static counter_u64_t ktls_toe_cbc;
 SYSCTL_COUNTER_U64(_kern_ipc_tls_toe, OID_AUTO, cbc, CTLFLAG_RD,
     &ktls_toe_cbc,
     "Active number of TOE TLS sessions using AES-CBC");
 
 static counter_u64_t ktls_toe_gcm;
 SYSCTL_COUNTER_U64(_kern_ipc_tls_toe, OID_AUTO, gcm, CTLFLAG_RD,
     &ktls_toe_gcm,
     "Active number of TOE TLS sessions using AES-GCM");
 #endif
 
 static MALLOC_DEFINE(M_KTLS, "ktls", "Kernel TLS");
 
 static void ktls_cleanup(struct ktls_session *tls);
 #if defined(INET) || defined(INET6)
 static void ktls_reset_send_tag(void *context, int pending);
 #endif
 static void ktls_work_thread(void *ctx);
 
 int
 ktls_crypto_backend_register(struct ktls_crypto_backend *be)
 {
 	struct ktls_crypto_backend *curr_be, *tmp;
 
 	if (be->api_version != KTLS_API_VERSION) {
 		printf("KTLS: API version mismatch (%d vs %d) for %s\n",
 		    be->api_version, KTLS_API_VERSION,
 		    be->name);
 		return (EINVAL);
 	}
 
 	rm_wlock(&ktls_backends_lock);
 	printf("KTLS: Registering crypto method %s with prio %d\n",
 	       be->name, be->prio);
 	if (LIST_EMPTY(&ktls_backends)) {
 		LIST_INSERT_HEAD(&ktls_backends, be, next);
 	} else {
 		LIST_FOREACH_SAFE(curr_be, &ktls_backends, next, tmp) {
 			if (curr_be->prio < be->prio) {
 				LIST_INSERT_BEFORE(curr_be, be, next);
 				break;
 			}
 			if (LIST_NEXT(curr_be, next) == NULL) {
 				LIST_INSERT_AFTER(curr_be, be, next);
 				break;
 			}
 		}
 	}
 	rm_wunlock(&ktls_backends_lock);
 	return (0);
 }
 
 int
 ktls_crypto_backend_deregister(struct ktls_crypto_backend *be)
 {
 	struct ktls_crypto_backend *tmp;
 
 	/*
 	 * Don't error if the backend isn't registered.  This permits
 	 * MOD_UNLOAD handlers to use this function unconditionally.
 	 */
 	rm_wlock(&ktls_backends_lock);
 	LIST_FOREACH(tmp, &ktls_backends, next) {
 		if (tmp == be)
 			break;
 	}
 	if (tmp == NULL) {
 		rm_wunlock(&ktls_backends_lock);
 		return (0);
 	}
 
 	if (!ktls_allow_unload) {
 		rm_wunlock(&ktls_backends_lock);
 		printf(
 		    "KTLS: Deregistering crypto method %s is not supported\n",
 		    be->name);
 		return (EBUSY);
 	}
 
 	if (be->use_count) {
 		rm_wunlock(&ktls_backends_lock);
 		return (EBUSY);
 	}
 
 	LIST_REMOVE(be, next);
 	rm_wunlock(&ktls_backends_lock);
 	return (0);
 }
 
 #if defined(INET) || defined(INET6)
 static u_int
 ktls_get_cpu(struct socket *so)
 {
 	struct inpcb *inp;
 	u_int cpuid;
 
 	inp = sotoinpcb(so);
 #ifdef RSS
 	cpuid = rss_hash2cpuid(inp->inp_flowid, inp->inp_flowtype);
 	if (cpuid != NETISR_CPUID_NONE)
 		return (cpuid);
 #endif
 	/*
 	 * Just use the flowid to shard connections in a repeatable
 	 * fashion.  Note that some crypto backends rely on the
 	 * serialization provided by having the same connection use
 	 * the same queue.
 	 */
 	cpuid = ktls_cpuid_lookup[inp->inp_flowid % ktls_number_threads];
 	return (cpuid);
 }
 #endif
 
 static void
 ktls_init(void *dummy __unused)
 {
 	struct thread *td;
 	struct pcpu *pc;
 	cpuset_t mask;
 	int error, i;
 
 	ktls_tasks_active = counter_u64_alloc(M_WAITOK);
 	ktls_cnt_on = counter_u64_alloc(M_WAITOK);
 	ktls_offload_total = counter_u64_alloc(M_WAITOK);
 	ktls_offload_enable_calls = counter_u64_alloc(M_WAITOK);
 	ktls_offload_active = counter_u64_alloc(M_WAITOK);
 	ktls_offload_failed_crypto = counter_u64_alloc(M_WAITOK);
 	ktls_switch_to_ifnet = counter_u64_alloc(M_WAITOK);
 	ktls_switch_to_sw = counter_u64_alloc(M_WAITOK);
 	ktls_switch_failed = counter_u64_alloc(M_WAITOK);
 	ktls_sw_cbc = counter_u64_alloc(M_WAITOK);
 	ktls_sw_gcm = counter_u64_alloc(M_WAITOK);
 	ktls_ifnet_cbc = counter_u64_alloc(M_WAITOK);
 	ktls_ifnet_gcm = counter_u64_alloc(M_WAITOK);
 	ktls_ifnet_reset = counter_u64_alloc(M_WAITOK);
 	ktls_ifnet_reset_dropped = counter_u64_alloc(M_WAITOK);
 	ktls_ifnet_reset_failed = counter_u64_alloc(M_WAITOK);
 #ifdef TCP_OFFLOAD
 	ktls_toe_cbc = counter_u64_alloc(M_WAITOK);
 	ktls_toe_gcm = counter_u64_alloc(M_WAITOK);
 #endif
 
 	rm_init(&ktls_backends_lock, "ktls backends");
 	LIST_INIT(&ktls_backends);
 
 	ktls_wq = malloc(sizeof(*ktls_wq) * (mp_maxid + 1), M_KTLS,
 	    M_WAITOK | M_ZERO);
 
 	ktls_session_zone = uma_zcreate("ktls_session",
 	    sizeof(struct ktls_session),
 	    NULL, NULL, NULL, NULL,
 	    UMA_ALIGN_CACHE, 0);
 
 	/*
 	 * Initialize the workqueues to run the TLS work.  We create a
 	 * work queue for each CPU.
 	 */
 	CPU_FOREACH(i) {
 		STAILQ_INIT(&ktls_wq[i].head);
 		mtx_init(&ktls_wq[i].mtx, "ktls work queue", NULL, MTX_DEF);
 		error = kproc_kthread_add(ktls_work_thread, &ktls_wq[i],
 		    &ktls_proc, &td, 0, 0, "KTLS", "thr_%d", i);
 		if (error)
 			panic("Can't add KTLS thread %d error %d", i, error);
 
 		/*
 		 * Bind threads to cores.  If ktls_bind_threads is >
 		 * 1, then we bind to the NUMA domain.
 		 */
 		if (ktls_bind_threads) {
 			if (ktls_bind_threads > 1) {
 				pc = pcpu_find(i);
 				CPU_COPY(&cpuset_domain[pc->pc_domain], &mask);
 			} else {
 				CPU_SETOF(i, &mask);
 			}
 			error = cpuset_setthread(td->td_tid, &mask);
 			if (error)
 				panic(
 			    "Unable to bind KTLS thread for CPU %d error %d",
 				     i, error);
 		}
 		ktls_cpuid_lookup[ktls_number_threads] = i;
 		ktls_number_threads++;
 	}
 	printf("KTLS: Initialized %d threads\n", ktls_number_threads);
 }
 SYSINIT(ktls, SI_SUB_SMP + 1, SI_ORDER_ANY, ktls_init, NULL);
 
 #if defined(INET) || defined(INET6)
 static int
 ktls_create_session(struct socket *so, struct tls_enable *en,
     struct ktls_session **tlsp)
 {
 	struct ktls_session *tls;
 	int error;
 
 	/* Only TLS 1.0 - 1.3 are supported. */
 	if (en->tls_vmajor != TLS_MAJOR_VER_ONE)
 		return (EINVAL);
 	if (en->tls_vminor < TLS_MINOR_VER_ZERO ||
 	    en->tls_vminor > TLS_MINOR_VER_THREE)
 		return (EINVAL);
 
 	if (en->auth_key_len < 0 || en->auth_key_len > TLS_MAX_PARAM_SIZE)
 		return (EINVAL);
 	if (en->cipher_key_len < 0 || en->cipher_key_len > TLS_MAX_PARAM_SIZE)
 		return (EINVAL);
 	if (en->iv_len < 0 || en->iv_len > sizeof(tls->params.iv))
 		return (EINVAL);
 
 	/* All supported algorithms require a cipher key. */
 	if (en->cipher_key_len == 0)
 		return (EINVAL);
 
 	/* No flags are currently supported. */
 	if (en->flags != 0)
 		return (EINVAL);
 
 	/* Common checks for supported algorithms. */
 	switch (en->cipher_algorithm) {
 	case CRYPTO_AES_NIST_GCM_16:
 		/*
 		 * auth_algorithm isn't used, but permit GMAC values
 		 * for compatibility.
 		 */
 		switch (en->auth_algorithm) {
 		case 0:
+#ifdef COMPAT_FREEBSD12
+		/* XXX: Really 13.0-current COMPAT. */
 		case CRYPTO_AES_128_NIST_GMAC:
 		case CRYPTO_AES_192_NIST_GMAC:
 		case CRYPTO_AES_256_NIST_GMAC:
+#endif
 			break;
 		default:
 			return (EINVAL);
 		}
 		if (en->auth_key_len != 0)
 			return (EINVAL);
 		if ((en->tls_vminor == TLS_MINOR_VER_TWO &&
 			en->iv_len != TLS_AEAD_GCM_LEN) ||
 		    (en->tls_vminor == TLS_MINOR_VER_THREE &&
 			en->iv_len != TLS_1_3_GCM_IV_LEN))
 			return (EINVAL);
 		break;
 	case CRYPTO_AES_CBC:
 		switch (en->auth_algorithm) {
 		case CRYPTO_SHA1_HMAC:
 			/*
 			 * TLS 1.0 requires an implicit IV.  TLS 1.1+
 			 * all use explicit IVs.
 			 */
 			if (en->tls_vminor == TLS_MINOR_VER_ZERO) {
 				if (en->iv_len != TLS_CBC_IMPLICIT_IV_LEN)
 					return (EINVAL);
 				break;
 			}
 
 			/* FALLTHROUGH */
 		case CRYPTO_SHA2_256_HMAC:
 		case CRYPTO_SHA2_384_HMAC:
 			/* Ignore any supplied IV. */
 			en->iv_len = 0;
 			break;
 		default:
 			return (EINVAL);
 		}
 		if (en->auth_key_len == 0)
 			return (EINVAL);
 		break;
 	default:
 		return (EINVAL);
 	}
 
 	tls = uma_zalloc(ktls_session_zone, M_WAITOK | M_ZERO);
 
 	counter_u64_add(ktls_offload_active, 1);
 
 	refcount_init(&tls->refcount, 1);
 	TASK_INIT(&tls->reset_tag_task, 0, ktls_reset_send_tag, tls);
 
 	tls->wq_index = ktls_get_cpu(so);
 
 	tls->params.cipher_algorithm = en->cipher_algorithm;
 	tls->params.auth_algorithm = en->auth_algorithm;
 	tls->params.tls_vmajor = en->tls_vmajor;
 	tls->params.tls_vminor = en->tls_vminor;
 	tls->params.flags = en->flags;
 	tls->params.max_frame_len = min(TLS_MAX_MSG_SIZE_V10_2, ktls_maxlen);
 
 	/* Set the header and trailer lengths. */
 	tls->params.tls_hlen = sizeof(struct tls_record_layer);
 	switch (en->cipher_algorithm) {
 	case CRYPTO_AES_NIST_GCM_16:
 		/*
 		 * TLS 1.2 uses a 4 byte implicit IV with an explicit 8 byte
 		 * nonce.  TLS 1.3 uses a 12 byte implicit IV.
 		 */
 		if (en->tls_vminor < TLS_MINOR_VER_THREE)
 			tls->params.tls_hlen += sizeof(uint64_t);
 		tls->params.tls_tlen = AES_GMAC_HASH_LEN;
 
 		/*
 		 * TLS 1.3 includes optional padding which we
 		 * do not support, and also puts the "real" record
 		 * type at the end of the encrypted data.
 		 */
 		if (en->tls_vminor == TLS_MINOR_VER_THREE)
 			tls->params.tls_tlen += sizeof(uint8_t);
 
 		tls->params.tls_bs = 1;
 		break;
 	case CRYPTO_AES_CBC:
 		switch (en->auth_algorithm) {
 		case CRYPTO_SHA1_HMAC:
 			if (en->tls_vminor == TLS_MINOR_VER_ZERO) {
 				/* Implicit IV, no nonce. */
 			} else {
 				tls->params.tls_hlen += AES_BLOCK_LEN;
 			}
 			tls->params.tls_tlen = AES_BLOCK_LEN +
 			    SHA1_HASH_LEN;
 			break;
 		case CRYPTO_SHA2_256_HMAC:
 			tls->params.tls_hlen += AES_BLOCK_LEN;
 			tls->params.tls_tlen = AES_BLOCK_LEN +
 			    SHA2_256_HASH_LEN;
 			break;
 		case CRYPTO_SHA2_384_HMAC:
 			tls->params.tls_hlen += AES_BLOCK_LEN;
 			tls->params.tls_tlen = AES_BLOCK_LEN +
 			    SHA2_384_HASH_LEN;
 			break;
 		default:
 			panic("invalid hmac");
 		}
 		tls->params.tls_bs = AES_BLOCK_LEN;
 		break;
 	default:
 		panic("invalid cipher");
 	}
 
 	KASSERT(tls->params.tls_hlen <= MBUF_PEXT_HDR_LEN,
 	    ("TLS header length too long: %d", tls->params.tls_hlen));
 	KASSERT(tls->params.tls_tlen <= MBUF_PEXT_TRAIL_LEN,
 	    ("TLS trailer length too long: %d", tls->params.tls_tlen));
 
 	if (en->auth_key_len != 0) {
 		tls->params.auth_key_len = en->auth_key_len;
 		tls->params.auth_key = malloc(en->auth_key_len, M_KTLS,
 		    M_WAITOK);
 		error = copyin(en->auth_key, tls->params.auth_key,
 		    en->auth_key_len);
 		if (error)
 			goto out;
 	}
 
 	tls->params.cipher_key_len = en->cipher_key_len;
 	tls->params.cipher_key = malloc(en->cipher_key_len, M_KTLS, M_WAITOK);
 	error = copyin(en->cipher_key, tls->params.cipher_key,
 	    en->cipher_key_len);
 	if (error)
 		goto out;
 
 	/*
 	 * This holds the implicit portion of the nonce for GCM and
 	 * the initial implicit IV for TLS 1.0.  The explicit portions
 	 * of the IV are generated in ktls_frame().
 	 */
 	if (en->iv_len != 0) {
 		tls->params.iv_len = en->iv_len;
 		error = copyin(en->iv, tls->params.iv, en->iv_len);
 		if (error)
 			goto out;
 
 		/*
 		 * For TLS 1.2, generate an 8-byte nonce as a counter
 		 * to generate unique explicit IVs.
 		 *
 		 * Store this counter in the last 8 bytes of the IV
 		 * array so that it is 8-byte aligned.
 		 */
 		if (en->cipher_algorithm == CRYPTO_AES_NIST_GCM_16 &&
 		    en->tls_vminor == TLS_MINOR_VER_TWO)
 			arc4rand(tls->params.iv + 8, sizeof(uint64_t), 0);
 	}
 
 	*tlsp = tls;
 	return (0);
 
 out:
 	ktls_cleanup(tls);
 	return (error);
 }
 
 static struct ktls_session *
 ktls_clone_session(struct ktls_session *tls)
 {
 	struct ktls_session *tls_new;
 
 	tls_new = uma_zalloc(ktls_session_zone, M_WAITOK | M_ZERO);
 
 	counter_u64_add(ktls_offload_active, 1);
 
 	refcount_init(&tls_new->refcount, 1);
 
 	/* Copy fields from existing session. */
 	tls_new->params = tls->params;
 	tls_new->wq_index = tls->wq_index;
 
 	/* Deep copy keys. */
 	if (tls_new->params.auth_key != NULL) {
 		tls_new->params.auth_key = malloc(tls->params.auth_key_len,
 		    M_KTLS, M_WAITOK);
 		memcpy(tls_new->params.auth_key, tls->params.auth_key,
 		    tls->params.auth_key_len);
 	}
 
 	tls_new->params.cipher_key = malloc(tls->params.cipher_key_len, M_KTLS,
 	    M_WAITOK);
 	memcpy(tls_new->params.cipher_key, tls->params.cipher_key,
 	    tls->params.cipher_key_len);
 
 	return (tls_new);
 }
 #endif
 
 static void
 ktls_cleanup(struct ktls_session *tls)
 {
 
 	counter_u64_add(ktls_offload_active, -1);
 	switch (tls->mode) {
 	case TCP_TLS_MODE_SW:
 		MPASS(tls->be != NULL);
 		switch (tls->params.cipher_algorithm) {
 		case CRYPTO_AES_CBC:
 			counter_u64_add(ktls_sw_cbc, -1);
 			break;
 		case CRYPTO_AES_NIST_GCM_16:
 			counter_u64_add(ktls_sw_gcm, -1);
 			break;
 		}
 		tls->free(tls);
 		break;
 	case TCP_TLS_MODE_IFNET:
 		switch (tls->params.cipher_algorithm) {
 		case CRYPTO_AES_CBC:
 			counter_u64_add(ktls_ifnet_cbc, -1);
 			break;
 		case CRYPTO_AES_NIST_GCM_16:
 			counter_u64_add(ktls_ifnet_gcm, -1);
 			break;
 		}
 		m_snd_tag_rele(tls->snd_tag);
 		break;
 #ifdef TCP_OFFLOAD
 	case TCP_TLS_MODE_TOE:
 		switch (tls->params.cipher_algorithm) {
 		case CRYPTO_AES_CBC:
 			counter_u64_add(ktls_toe_cbc, -1);
 			break;
 		case CRYPTO_AES_NIST_GCM_16:
 			counter_u64_add(ktls_toe_gcm, -1);
 			break;
 		}
 		break;
 #endif
 	}
 	if (tls->params.auth_key != NULL) {
 		explicit_bzero(tls->params.auth_key, tls->params.auth_key_len);
 		free(tls->params.auth_key, M_KTLS);
 		tls->params.auth_key = NULL;
 		tls->params.auth_key_len = 0;
 	}
 	if (tls->params.cipher_key != NULL) {
 		explicit_bzero(tls->params.cipher_key,
 		    tls->params.cipher_key_len);
 		free(tls->params.cipher_key, M_KTLS);
 		tls->params.cipher_key = NULL;
 		tls->params.cipher_key_len = 0;
 	}
 	explicit_bzero(tls->params.iv, sizeof(tls->params.iv));
 }
 
 #if defined(INET) || defined(INET6)
 
 #ifdef TCP_OFFLOAD
 static int
 ktls_try_toe(struct socket *so, struct ktls_session *tls)
 {
 	struct inpcb *inp;
 	struct tcpcb *tp;
 	int error;
 
 	inp = so->so_pcb;
 	INP_WLOCK(inp);
 	if (inp->inp_flags2 & INP_FREED) {
 		INP_WUNLOCK(inp);
 		return (ECONNRESET);
 	}
 	if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
 		INP_WUNLOCK(inp);
 		return (ECONNRESET);
 	}
 	if (inp->inp_socket == NULL) {
 		INP_WUNLOCK(inp);
 		return (ECONNRESET);
 	}
 	tp = intotcpcb(inp);
 	if (tp->tod == NULL) {
 		INP_WUNLOCK(inp);
 		return (EOPNOTSUPP);
 	}
 
 	error = tcp_offload_alloc_tls_session(tp, tls);
 	INP_WUNLOCK(inp);
 	if (error == 0) {
 		tls->mode = TCP_TLS_MODE_TOE;
 		switch (tls->params.cipher_algorithm) {
 		case CRYPTO_AES_CBC:
 			counter_u64_add(ktls_toe_cbc, 1);
 			break;
 		case CRYPTO_AES_NIST_GCM_16:
 			counter_u64_add(ktls_toe_gcm, 1);
 			break;
 		}
 	}
 	return (error);
 }
 #endif
 
 /*
  * Common code used when first enabling ifnet TLS on a connection or
  * when allocating a new ifnet TLS session due to a routing change.
  * This function allocates a new TLS send tag on whatever interface
  * the connection is currently routed over.
  */
 static int
 ktls_alloc_snd_tag(struct inpcb *inp, struct ktls_session *tls, bool force,
     struct m_snd_tag **mstp)
 {
 	union if_snd_tag_alloc_params params;
 	struct ifnet *ifp;
 	struct rtentry *rt;
 	struct tcpcb *tp;
 	int error;
 
 	INP_RLOCK(inp);
 	if (inp->inp_flags2 & INP_FREED) {
 		INP_RUNLOCK(inp);
 		return (ECONNRESET);
 	}
 	if (inp->inp_flags & (INP_TIMEWAIT | INP_DROPPED)) {
 		INP_RUNLOCK(inp);
 		return (ECONNRESET);
 	}
 	if (inp->inp_socket == NULL) {
 		INP_RUNLOCK(inp);
 		return (ECONNRESET);
 	}
 	tp = intotcpcb(inp);
 
 	/*
 	 * Check administrative controls on ifnet TLS to determine if
 	 * ifnet TLS should be denied.
 	 *
 	 * - Always permit 'force' requests.
 	 * - ktls_ifnet_permitted == 0: always deny.
 	 */
 	if (!force && ktls_ifnet_permitted == 0) {
 		INP_RUNLOCK(inp);
 		return (ENXIO);
 	}
 
 	/*
 	 * XXX: Use the cached route in the inpcb to find the
 	 * interface.  This should perhaps instead use
 	 * rtalloc1_fib(dst, 0, 0, fibnum).  Since KTLS is only
 	 * enabled after a connection has completed key negotiation in
 	 * userland, the cached route will be present in practice.
 	 */
 	rt = inp->inp_route.ro_rt;
 	if (rt == NULL || rt->rt_ifp == NULL) {
 		INP_RUNLOCK(inp);
 		return (ENXIO);
 	}
 	ifp = rt->rt_ifp;
 	if_ref(ifp);
 
 	params.hdr.type = IF_SND_TAG_TYPE_TLS;
 	params.hdr.flowid = inp->inp_flowid;
 	params.hdr.flowtype = inp->inp_flowtype;
 	params.hdr.numa_domain = inp->inp_numa_domain;
 	params.tls.inp = inp;
 	params.tls.tls = tls;
 	INP_RUNLOCK(inp);
 
 	if (ifp->if_snd_tag_alloc == NULL) {
 		error = EOPNOTSUPP;
 		goto out;
 	}
 	if ((ifp->if_capenable & IFCAP_NOMAP) == 0) {	
 		error = EOPNOTSUPP;
 		goto out;
 	}
 	if (inp->inp_vflag & INP_IPV6) {
 		if ((ifp->if_capenable & IFCAP_TXTLS6) == 0) {
 			error = EOPNOTSUPP;
 			goto out;
 		}
 	} else {
 		if ((ifp->if_capenable & IFCAP_TXTLS4) == 0) {
 			error = EOPNOTSUPP;
 			goto out;
 		}
 	}
 	error = ifp->if_snd_tag_alloc(ifp, &params, mstp);
 out:
 	if_rele(ifp);
 	return (error);
 }
 
 static int
 ktls_try_ifnet(struct socket *so, struct ktls_session *tls, bool force)
 {
 	struct m_snd_tag *mst;
 	int error;
 
 	error = ktls_alloc_snd_tag(so->so_pcb, tls, force, &mst);
 	if (error == 0) {
 		tls->mode = TCP_TLS_MODE_IFNET;
 		tls->snd_tag = mst;
 		switch (tls->params.cipher_algorithm) {
 		case CRYPTO_AES_CBC:
 			counter_u64_add(ktls_ifnet_cbc, 1);
 			break;
 		case CRYPTO_AES_NIST_GCM_16:
 			counter_u64_add(ktls_ifnet_gcm, 1);
 			break;
 		}
 	}
 	return (error);
 }
 
 static int
 ktls_try_sw(struct socket *so, struct ktls_session *tls)
 {
 	struct rm_priotracker prio;
 	struct ktls_crypto_backend *be;
 
 	/*
 	 * Choose the best software crypto backend.  Backends are
 	 * stored in sorted priority order (larget value == most
 	 * important at the head of the list), so this just stops on
 	 * the first backend that claims the session by returning
 	 * success.
 	 */
 	if (ktls_allow_unload)
 		rm_rlock(&ktls_backends_lock, &prio);
 	LIST_FOREACH(be, &ktls_backends, next) {
 		if (be->try(so, tls) == 0)
 			break;
 		KASSERT(tls->cipher == NULL,
 		    ("ktls backend leaked a cipher pointer"));
 	}
 	if (be != NULL) {
 		if (ktls_allow_unload)
 			be->use_count++;
 		tls->be = be;
 	}
 	if (ktls_allow_unload)
 		rm_runlock(&ktls_backends_lock, &prio);
 	if (be == NULL)
 		return (EOPNOTSUPP);
 	tls->mode = TCP_TLS_MODE_SW;
 	switch (tls->params.cipher_algorithm) {
 	case CRYPTO_AES_CBC:
 		counter_u64_add(ktls_sw_cbc, 1);
 		break;
 	case CRYPTO_AES_NIST_GCM_16:
 		counter_u64_add(ktls_sw_gcm, 1);
 		break;
 	}
 	return (0);
 }
 
 int
 ktls_enable_tx(struct socket *so, struct tls_enable *en)
 {
 	struct ktls_session *tls;
 	int error;
 
 	if (!ktls_offload_enable)
 		return (ENOTSUP);
 
 	counter_u64_add(ktls_offload_enable_calls, 1);
 
 	/*
 	 * This should always be true since only the TCP socket option
 	 * invokes this function.
 	 */
 	if (so->so_proto->pr_protocol != IPPROTO_TCP)
 		return (EINVAL);
 
 	/*
 	 * XXX: Don't overwrite existing sessions.  We should permit
 	 * this to support rekeying in the future.
 	 */
 	if (so->so_snd.sb_tls_info != NULL)
 		return (EALREADY);
 
 	if (en->cipher_algorithm == CRYPTO_AES_CBC && !ktls_cbc_enable)
 		return (ENOTSUP);
 
 	/* TLS requires ext pgs */
 	if (mb_use_ext_pgs == 0)
 		return (ENXIO);
 
 	error = ktls_create_session(so, en, &tls);
 	if (error)
 		return (error);
 
 	/* Prefer TOE -> ifnet TLS -> software TLS. */
 #ifdef TCP_OFFLOAD
 	error = ktls_try_toe(so, tls);
 	if (error)
 #endif
 		error = ktls_try_ifnet(so, tls, false);
 	if (error)
 		error = ktls_try_sw(so, tls);
 
 	if (error) {
 		ktls_cleanup(tls);
 		return (error);
 	}
 
 	error = sblock(&so->so_snd, SBL_WAIT);
 	if (error) {
 		ktls_cleanup(tls);
 		return (error);
 	}
 
 	SOCKBUF_LOCK(&so->so_snd);
 	so->so_snd.sb_tls_info = tls;
 	if (tls->mode != TCP_TLS_MODE_SW)
 		so->so_snd.sb_flags |= SB_TLS_IFNET;
 	SOCKBUF_UNLOCK(&so->so_snd);
 	sbunlock(&so->so_snd);
 
 	counter_u64_add(ktls_offload_total, 1);
 
 	return (0);
 }
 
 int
 ktls_get_tx_mode(struct socket *so)
 {
 	struct ktls_session *tls;
 	struct inpcb *inp;
 	int mode;
 
 	inp = so->so_pcb;
 	INP_WLOCK_ASSERT(inp);
 	SOCKBUF_LOCK(&so->so_snd);
 	tls = so->so_snd.sb_tls_info;
 	if (tls == NULL)
 		mode = TCP_TLS_MODE_NONE;
 	else
 		mode = tls->mode;
 	SOCKBUF_UNLOCK(&so->so_snd);
 	return (mode);
 }
 
 /*
  * Switch between SW and ifnet TLS sessions as requested.
  */
 int
 ktls_set_tx_mode(struct socket *so, int mode)
 {
 	struct ktls_session *tls, *tls_new;
 	struct inpcb *inp;
 	int error;
 
 	switch (mode) {
 	case TCP_TLS_MODE_SW:
 	case TCP_TLS_MODE_IFNET:
 		break;
 	default:
 		return (EINVAL);
 	}
 
 	inp = so->so_pcb;
 	INP_WLOCK_ASSERT(inp);
 	SOCKBUF_LOCK(&so->so_snd);
 	tls = so->so_snd.sb_tls_info;
 	if (tls == NULL) {
 		SOCKBUF_UNLOCK(&so->so_snd);
 		return (0);
 	}
 
 	if (tls->mode == mode) {
 		SOCKBUF_UNLOCK(&so->so_snd);
 		return (0);
 	}
 
 	tls = ktls_hold(tls);
 	SOCKBUF_UNLOCK(&so->so_snd);
 	INP_WUNLOCK(inp);
 
 	tls_new = ktls_clone_session(tls);
 
 	if (mode == TCP_TLS_MODE_IFNET)
 		error = ktls_try_ifnet(so, tls_new, true);
 	else
 		error = ktls_try_sw(so, tls_new);
 	if (error) {
 		counter_u64_add(ktls_switch_failed, 1);
 		ktls_free(tls_new);
 		ktls_free(tls);
 		INP_WLOCK(inp);
 		return (error);
 	}
 
 	error = sblock(&so->so_snd, SBL_WAIT);
 	if (error) {
 		counter_u64_add(ktls_switch_failed, 1);
 		ktls_free(tls_new);
 		ktls_free(tls);
 		INP_WLOCK(inp);
 		return (error);
 	}
 
 	/*
 	 * If we raced with another session change, keep the existing
 	 * session.
 	 */
 	if (tls != so->so_snd.sb_tls_info) {
 		counter_u64_add(ktls_switch_failed, 1);
 		sbunlock(&so->so_snd);
 		ktls_free(tls_new);
 		ktls_free(tls);
 		INP_WLOCK(inp);
 		return (EBUSY);
 	}
 
 	SOCKBUF_LOCK(&so->so_snd);
 	so->so_snd.sb_tls_info = tls_new;
 	if (tls_new->mode != TCP_TLS_MODE_SW)
 		so->so_snd.sb_flags |= SB_TLS_IFNET;
 	SOCKBUF_UNLOCK(&so->so_snd);
 	sbunlock(&so->so_snd);
 
 	/*
 	 * Drop two references on 'tls'.  The first is for the
 	 * ktls_hold() above.  The second drops the reference from the
 	 * socket buffer.
 	 */
 	KASSERT(tls->refcount >= 2, ("too few references on old session"));
 	ktls_free(tls);
 	ktls_free(tls);
 
 	if (mode == TCP_TLS_MODE_IFNET)
 		counter_u64_add(ktls_switch_to_ifnet, 1);
 	else
 		counter_u64_add(ktls_switch_to_sw, 1);
 
 	INP_WLOCK(inp);
 	return (0);
 }
 
 /*
  * Try to allocate a new TLS send tag.  This task is scheduled when
  * ip_output detects a route change while trying to transmit a packet
  * holding a TLS record.  If a new tag is allocated, replace the tag
  * in the TLS session.  Subsequent packets on the connection will use
  * the new tag.  If a new tag cannot be allocated, drop the
  * connection.
  */
 static void
 ktls_reset_send_tag(void *context, int pending)
 {
 	struct epoch_tracker et;
 	struct ktls_session *tls;
 	struct m_snd_tag *old, *new;
 	struct inpcb *inp;
 	struct tcpcb *tp;
 	int error;
 
 	MPASS(pending == 1);
 
 	tls = context;
 	inp = tls->inp;
 
 	/*
 	 * Free the old tag first before allocating a new one.
 	 * ip[6]_output_send() will treat a NULL send tag the same as
 	 * an ifp mismatch and drop packets until a new tag is
 	 * allocated.
 	 *
 	 * Write-lock the INP when changing tls->snd_tag since
 	 * ip[6]_output_send() holds a read-lock when reading the
 	 * pointer.
 	 */
 	INP_WLOCK(inp);
 	old = tls->snd_tag;
 	tls->snd_tag = NULL;
 	INP_WUNLOCK(inp);
 	if (old != NULL)
 		m_snd_tag_rele(old);
 
 	error = ktls_alloc_snd_tag(inp, tls, true, &new);
 
 	if (error == 0) {
 		INP_WLOCK(inp);
 		tls->snd_tag = new;
 		mtx_pool_lock(mtxpool_sleep, tls);
 		tls->reset_pending = false;
 		mtx_pool_unlock(mtxpool_sleep, tls);
 		if (!in_pcbrele_wlocked(inp))
 			INP_WUNLOCK(inp);
 
 		counter_u64_add(ktls_ifnet_reset, 1);
 
 		/*
 		 * XXX: Should we kick tcp_output explicitly now that
 		 * the send tag is fixed or just rely on timers?
 		 */
 	} else {
 		NET_EPOCH_ENTER(et);
 		INP_WLOCK(inp);
 		if (!in_pcbrele_wlocked(inp)) {
 			if (!(inp->inp_flags & INP_TIMEWAIT) &&
 			    !(inp->inp_flags & INP_DROPPED)) {
 				tp = intotcpcb(inp);
 				CURVNET_SET(tp->t_vnet);
 				tp = tcp_drop(tp, ECONNABORTED);
 				CURVNET_RESTORE();
 				if (tp != NULL)
 					INP_WUNLOCK(inp);
 				counter_u64_add(ktls_ifnet_reset_dropped, 1);
 			} else
 				INP_WUNLOCK(inp);
 		}
 		NET_EPOCH_EXIT(et);
 
 		counter_u64_add(ktls_ifnet_reset_failed, 1);
 
 		/*
 		 * Leave reset_pending true to avoid future tasks while
 		 * the socket goes away.
 		 */
 	}
 
 	ktls_free(tls);
 }
 
 int
 ktls_output_eagain(struct inpcb *inp, struct ktls_session *tls)
 {
 
 	if (inp == NULL)
 		return (ENOBUFS);
 
 	INP_LOCK_ASSERT(inp);
 
 	/*
 	 * See if we should schedule a task to update the send tag for
 	 * this session.
 	 */
 	mtx_pool_lock(mtxpool_sleep, tls);
 	if (!tls->reset_pending) {
 		(void) ktls_hold(tls);
 		in_pcbref(inp);
 		tls->inp = inp;
 		tls->reset_pending = true;
 		taskqueue_enqueue(taskqueue_thread, &tls->reset_tag_task);
 	}
 	mtx_pool_unlock(mtxpool_sleep, tls);
 	return (ENOBUFS);
 }
 #endif
 
 void
 ktls_destroy(struct ktls_session *tls)
 {
 	struct rm_priotracker prio;
 
 	ktls_cleanup(tls);
 	if (tls->be != NULL && ktls_allow_unload) {
 		rm_rlock(&ktls_backends_lock, &prio);
 		tls->be->use_count--;
 		rm_runlock(&ktls_backends_lock, &prio);
 	}
 	uma_zfree(ktls_session_zone, tls);
 }
 
 void
 ktls_seq(struct sockbuf *sb, struct mbuf *m)
 {
 	struct mbuf_ext_pgs *pgs;
 
 	for (; m != NULL; m = m->m_next) {
 		KASSERT((m->m_flags & M_NOMAP) != 0,
 		    ("ktls_seq: mapped mbuf %p", m));
 
 		pgs = m->m_ext.ext_pgs;
 		pgs->seqno = sb->sb_tls_seqno;
 		sb->sb_tls_seqno++;
 	}
 }
 
 /*
  * Add TLS framing (headers and trailers) to a chain of mbufs.  Each
  * mbuf in the chain must be an unmapped mbuf.  The payload of the
  * mbuf must be populated with the payload of each TLS record.
  *
  * The record_type argument specifies the TLS record type used when
  * populating the TLS header.
  *
  * The enq_count argument on return is set to the number of pages of
  * payload data for this entire chain that need to be encrypted via SW
  * encryption.  The returned value should be passed to ktls_enqueue
  * when scheduling encryption of this chain of mbufs.
  */
 void
 ktls_frame(struct mbuf *top, struct ktls_session *tls, int *enq_cnt,
     uint8_t record_type)
 {
 	struct tls_record_layer *tlshdr;
 	struct mbuf *m;
 	struct mbuf_ext_pgs *pgs;
 	uint64_t *noncep;
 	uint16_t tls_len;
 	int maxlen;
 
 	maxlen = tls->params.max_frame_len;
 	*enq_cnt = 0;
 	for (m = top; m != NULL; m = m->m_next) {
 		/*
 		 * All mbufs in the chain should be non-empty TLS
 		 * records whose payload does not exceed the maximum
 		 * frame length.
 		 */
 		KASSERT(m->m_len <= maxlen && m->m_len > 0,
 		    ("ktls_frame: m %p len %d\n", m, m->m_len));
 		/*
 		 * TLS frames require unmapped mbufs to store session
 		 * info.
 		 */
 		KASSERT((m->m_flags & M_NOMAP) != 0,
 		    ("ktls_frame: mapped mbuf %p (top = %p)\n", m, top));
 
 		tls_len = m->m_len;
 		pgs = m->m_ext.ext_pgs;
 
 		/* Save a reference to the session. */
 		pgs->tls = ktls_hold(tls);
 
 		pgs->hdr_len = tls->params.tls_hlen;
 		pgs->trail_len = tls->params.tls_tlen;
 		if (tls->params.cipher_algorithm == CRYPTO_AES_CBC) {
 			int bs, delta;
 
 			/*
 			 * AES-CBC pads messages to a multiple of the
 			 * block size.  Note that the padding is
 			 * applied after the digest and the encryption
 			 * is done on the "plaintext || mac || padding".
 			 * At least one byte of padding is always
 			 * present.
 			 *
 			 * Compute the final trailer length assuming
 			 * at most one block of padding.
 			 * tls->params.sb_tls_tlen is the maximum
 			 * possible trailer length (padding + digest).
 			 * delta holds the number of excess padding
 			 * bytes if the maximum were used.  Those
 			 * extra bytes are removed.
 			 */
 			bs = tls->params.tls_bs;
 			delta = (tls_len + tls->params.tls_tlen) & (bs - 1);
 			pgs->trail_len -= delta;
 		}
 		m->m_len += pgs->hdr_len + pgs->trail_len;
 
 		/* Populate the TLS header. */
 		tlshdr = (void *)pgs->hdr;
 		tlshdr->tls_vmajor = tls->params.tls_vmajor;
 
 		/*
 		 * TLS 1.3 masquarades as TLS 1.2 with a record type
 		 * of TLS_RLTYPE_APP.
 		 */
 		if (tls->params.tls_vminor == TLS_MINOR_VER_THREE &&
 		    tls->params.tls_vmajor == TLS_MAJOR_VER_ONE) {
 			tlshdr->tls_vminor = TLS_MINOR_VER_TWO;
 			tlshdr->tls_type = TLS_RLTYPE_APP;
 			/* save the real record type for later */
 			pgs->record_type = record_type;
 		} else {
 			tlshdr->tls_vminor = tls->params.tls_vminor;
 			tlshdr->tls_type = record_type;
 		}
 		tlshdr->tls_length = htons(m->m_len - sizeof(*tlshdr));
 
 		/*
 		 * Store nonces / explicit IVs after the end of the
 		 * TLS header.
 		 *
 		 * For GCM with TLS 1.2, an 8 byte nonce is copied
 		 * from the end of the IV.  The nonce is then
 		 * incremented for use by the next record.
 		 *
 		 * For CBC, a random nonce is inserted for TLS 1.1+.
 		 */
 		if (tls->params.cipher_algorithm == CRYPTO_AES_NIST_GCM_16 &&
 		    tls->params.tls_vminor == TLS_MINOR_VER_TWO) {
 			noncep = (uint64_t *)(tls->params.iv + 8);
 			be64enc(tlshdr + 1, *noncep);
 			(*noncep)++;
 		} else if (tls->params.cipher_algorithm == CRYPTO_AES_CBC &&
 		    tls->params.tls_vminor >= TLS_MINOR_VER_ONE)
 			arc4rand(tlshdr + 1, AES_BLOCK_LEN, 0);
 
 		/*
 		 * When using SW encryption, mark the mbuf not ready.
 		 * It will be marked ready via sbready() after the
 		 * record has been encrypted.
 		 *
 		 * When using ifnet TLS, unencrypted TLS records are
 		 * sent down the stack to the NIC.
 		 */
 		if (tls->mode == TCP_TLS_MODE_SW) {
 			m->m_flags |= M_NOTREADY;
 			pgs->nrdy = pgs->npgs;
 			*enq_cnt += pgs->npgs;
 		}
 	}
 }
 
 void
 ktls_enqueue_to_free(struct mbuf_ext_pgs *pgs)
 {
 	struct ktls_wq *wq;
 	bool running;
 
 	/* Mark it for freeing. */
 	pgs->mbuf = NULL;
 	wq = &ktls_wq[pgs->tls->wq_index];
 	mtx_lock(&wq->mtx);
 	STAILQ_INSERT_TAIL(&wq->head, pgs, stailq);
 	running = wq->running;
 	mtx_unlock(&wq->mtx);
 	if (!running)
 		wakeup(wq);
 }
 
 void
 ktls_enqueue(struct mbuf *m, struct socket *so, int page_count)
 {
 	struct mbuf_ext_pgs *pgs;
 	struct ktls_wq *wq;
 	bool running;
 
 	KASSERT(((m->m_flags & (M_NOMAP | M_NOTREADY)) ==
 	    (M_NOMAP | M_NOTREADY)),
 	    ("ktls_enqueue: %p not unready & nomap mbuf\n", m));
 	KASSERT(page_count != 0, ("enqueueing TLS mbuf with zero page count"));
 
 	pgs = m->m_ext.ext_pgs;
 
 	KASSERT(pgs->tls->mode == TCP_TLS_MODE_SW, ("!SW TLS mbuf"));
 
 	pgs->enc_cnt = page_count;
 	pgs->mbuf = m;
 
 	/*
 	 * Save a pointer to the socket.  The caller is responsible
 	 * for taking an additional reference via soref().
 	 */
 	pgs->so = so;
 
 	wq = &ktls_wq[pgs->tls->wq_index];
 	mtx_lock(&wq->mtx);
 	STAILQ_INSERT_TAIL(&wq->head, pgs, stailq);
 	running = wq->running;
 	mtx_unlock(&wq->mtx);
 	if (!running)
 		wakeup(wq);
 	counter_u64_add(ktls_cnt_on, 1);
 }
 
 static __noinline void
 ktls_encrypt(struct mbuf_ext_pgs *pgs)
 {
 	struct ktls_session *tls;
 	struct socket *so;
 	struct mbuf *m, *top;
 	vm_paddr_t parray[1 + btoc(TLS_MAX_MSG_SIZE_V10_2)];
 	struct iovec src_iov[1 + btoc(TLS_MAX_MSG_SIZE_V10_2)];
 	struct iovec dst_iov[1 + btoc(TLS_MAX_MSG_SIZE_V10_2)];
 	vm_page_t pg;
 	int error, i, len, npages, off, total_pages;
 	bool is_anon;
 
 	so = pgs->so;
 	tls = pgs->tls;
 	top = pgs->mbuf;
 	KASSERT(tls != NULL, ("tls = NULL, top = %p, pgs = %p\n", top, pgs));
 	KASSERT(so != NULL, ("so = NULL, top = %p, pgs = %p\n", top, pgs));
 #ifdef INVARIANTS
 	pgs->so = NULL;
 	pgs->mbuf = NULL;
 #endif
 	total_pages = pgs->enc_cnt;
 	npages = 0;
 
 	/*
 	 * Encrypt the TLS records in the chain of mbufs starting with
 	 * 'top'.  'total_pages' gives us a total count of pages and is
 	 * used to know when we have finished encrypting the TLS
 	 * records originally queued with 'top'.
 	 *
 	 * NB: These mbufs are queued in the socket buffer and
 	 * 'm_next' is traversing the mbufs in the socket buffer.  The
 	 * socket buffer lock is not held while traversing this chain.
 	 * Since the mbufs are all marked M_NOTREADY their 'm_next'
 	 * pointers should be stable.  However, the 'm_next' of the
 	 * last mbuf encrypted is not necessarily NULL.  It can point
 	 * to other mbufs appended while 'top' was on the TLS work
 	 * queue.
 	 *
 	 * Each mbuf holds an entire TLS record.
 	 */
 	error = 0;
 	for (m = top; npages != total_pages; m = m->m_next) {
 		pgs = m->m_ext.ext_pgs;
 
 		KASSERT(pgs->tls == tls,
 		    ("different TLS sessions in a single mbuf chain: %p vs %p",
 		    tls, pgs->tls));
 		KASSERT((m->m_flags & (M_NOMAP | M_NOTREADY)) ==
 		    (M_NOMAP | M_NOTREADY),
 		    ("%p not unready & nomap mbuf (top = %p)\n", m, top));
 		KASSERT(npages + pgs->npgs <= total_pages,
 		    ("page count mismatch: top %p, total_pages %d, m %p", top,
 		    total_pages, m));
 
 		/*
 		 * Generate source and destination ivoecs to pass to
 		 * the SW encryption backend.  For writable mbufs, the
 		 * destination iovec is a copy of the source and
 		 * encryption is done in place.  For file-backed mbufs
 		 * (from sendfile), anonymous wired pages are
 		 * allocated and assigned to the destination iovec.
 		 */
 		is_anon = (pgs->flags & MBUF_PEXT_FLAG_ANON) != 0;
 
 		off = pgs->first_pg_off;
 		for (i = 0; i < pgs->npgs; i++, off = 0) {
 			len = mbuf_ext_pg_len(pgs, i, off);
 			src_iov[i].iov_len = len;
 			src_iov[i].iov_base =
 			    (char *)(void *)PHYS_TO_DMAP(pgs->pa[i]) + off;
 
 			if (is_anon) {
 				dst_iov[i].iov_base = src_iov[i].iov_base;
 				dst_iov[i].iov_len = src_iov[i].iov_len;
 				continue;
 			}
 retry_page:
 			pg = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL |
 			    VM_ALLOC_NOOBJ | VM_ALLOC_NODUMP | VM_ALLOC_WIRED);
 			if (pg == NULL) {
 				vm_wait(NULL);
 				goto retry_page;
 			}
 			parray[i] = VM_PAGE_TO_PHYS(pg);
 			dst_iov[i].iov_base =
 			    (char *)(void *)PHYS_TO_DMAP(parray[i]) + off;
 			dst_iov[i].iov_len = len;
 		}
 
 		npages += i;
 
 		error = (*tls->sw_encrypt)(tls,
 		    (const struct tls_record_layer *)pgs->hdr,
 		    pgs->trail, src_iov, dst_iov, i, pgs->seqno,
 		    pgs->record_type);
 		if (error) {
 			counter_u64_add(ktls_offload_failed_crypto, 1);
 			break;
 		}
 
 		/*
 		 * For file-backed mbufs, release the file-backed
 		 * pages and replace them in the ext_pgs array with
 		 * the anonymous wired pages allocated above.
 		 */
 		if (!is_anon) {
 			/* Free the old pages. */
 			m->m_ext.ext_free(m);
 
 			/* Replace them with the new pages. */
 			for (i = 0; i < pgs->npgs; i++)
 				pgs->pa[i] = parray[i];
 
 			/* Use the basic free routine. */
 			m->m_ext.ext_free = mb_free_mext_pgs;
 
 			/* Pages are now writable. */
 			pgs->flags |= MBUF_PEXT_FLAG_ANON;
 		}
 
 		/*
 		 * Drop a reference to the session now that it is no
 		 * longer needed.  Existing code depends on encrypted
 		 * records having no associated session vs
 		 * yet-to-be-encrypted records having an associated
 		 * session.
 		 */
 		pgs->tls = NULL;
 		ktls_free(tls);
 	}
 
 	CURVNET_SET(so->so_vnet);
 	if (error == 0) {
 		(void)(*so->so_proto->pr_usrreqs->pru_ready)(so, top, npages);
 	} else {
 		so->so_proto->pr_usrreqs->pru_abort(so);
 		so->so_error = EIO;
 		mb_free_notready(top, total_pages);
 	}
 
 	SOCK_LOCK(so);
 	sorele(so);
 	CURVNET_RESTORE();
 }
 
 static void
 ktls_work_thread(void *ctx)
 {
 	struct ktls_wq *wq = ctx;
 	struct mbuf_ext_pgs *p, *n;
 	struct ktls_session *tls;
 	STAILQ_HEAD(, mbuf_ext_pgs) local_head;
 
 #if defined(__aarch64__) || defined(__amd64__) || defined(__i386__)
 	fpu_kern_thread(0);
 #endif
 	for (;;) {
 		mtx_lock(&wq->mtx);
 		while (STAILQ_EMPTY(&wq->head)) {
 			wq->running = false;
 			mtx_sleep(wq, &wq->mtx, 0, "-", 0);
 			wq->running = true;
 		}
 
 		STAILQ_INIT(&local_head);
 		STAILQ_CONCAT(&local_head, &wq->head);
 		mtx_unlock(&wq->mtx);
 
 		STAILQ_FOREACH_SAFE(p, &local_head, stailq, n) {
 			if (p->mbuf != NULL) {
 				ktls_encrypt(p);
 				counter_u64_add(ktls_cnt_on, -1);
 			} else {
 				tls = p->tls;
 				ktls_free(tls);
 				uma_zfree(zone_extpgs, p);
 			}
 		}
 	}
 }
Index: head/sys/kgssapi/krb5/kcrypto_aes.c
===================================================================
--- head/sys/kgssapi/krb5/kcrypto_aes.c	(revision 359373)
+++ head/sys/kgssapi/krb5/kcrypto_aes.c	(revision 359374)
@@ -1,392 +1,381 @@
 /*-
  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
  *
  * Copyright (c) 2008 Isilon Inc http://www.isilon.com/
  * Authors: Doug Rabson <dfr@rabson.org>
  * Developed with Red Inc: Alfred Perlstein <alfred@freebsd.org>
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/lock.h>
 #include <sys/malloc.h>
 #include <sys/mutex.h>
 #include <sys/kobj.h>
 #include <sys/mbuf.h>
 #include <opencrypto/cryptodev.h>
 
 #include <kgssapi/gssapi.h>
 #include <kgssapi/gssapi_impl.h>
 
 #include "kcrypto.h"
 
 struct aes_state {
 	struct mtx	as_lock;
 	crypto_session_t as_session_aes;
 	crypto_session_t as_session_sha1;
 };
 
 static void
 aes_init(struct krb5_key_state *ks)
 {
 	struct aes_state *as;
 
 	as = malloc(sizeof(struct aes_state), M_GSSAPI, M_WAITOK|M_ZERO);
 	mtx_init(&as->as_lock, "gss aes lock", NULL, MTX_DEF);
 	ks->ks_priv = as;
 }
 
 static void
 aes_destroy(struct krb5_key_state *ks)
 {
 	struct aes_state *as = ks->ks_priv;
 
 	if (as->as_session_aes != 0)
 		crypto_freesession(as->as_session_aes);
 	if (as->as_session_sha1 != 0)
 		crypto_freesession(as->as_session_sha1);
 	mtx_destroy(&as->as_lock);
 	free(ks->ks_priv, M_GSSAPI);
 }
 
 static void
 aes_set_key(struct krb5_key_state *ks, const void *in)
 {
 	void *kp = ks->ks_key;
 	struct aes_state *as = ks->ks_priv;
-	struct cryptoini cri;
+	struct crypto_session_params csp;
 
 	if (kp != in)
 		bcopy(in, kp, ks->ks_class->ec_keylen);
 
 	if (as->as_session_aes != 0)
 		crypto_freesession(as->as_session_aes);
 	if (as->as_session_sha1 != 0)
 		crypto_freesession(as->as_session_sha1);
 
 	/*
 	 * We only want the first 96 bits of the HMAC.
 	 */
-	bzero(&cri, sizeof(cri));
-	cri.cri_alg = CRYPTO_SHA1_HMAC;
-	cri.cri_klen = ks->ks_class->ec_keybits;
-	cri.cri_mlen = 12;
-	cri.cri_key = ks->ks_key;
-	cri.cri_next = NULL;
-	crypto_newsession(&as->as_session_sha1, &cri,
+	memset(&csp, 0, sizeof(csp));
+	csp.csp_mode = CSP_MODE_DIGEST;
+	csp.csp_auth_alg = CRYPTO_SHA1_HMAC;
+	csp.csp_auth_klen = ks->ks_class->ec_keybits / 8;
+	csp.csp_auth_mlen = 12;
+	csp.csp_auth_key = ks->ks_key;
+	crypto_newsession(&as->as_session_sha1, &csp,
 	    CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE);
 
-	bzero(&cri, sizeof(cri));
-	cri.cri_alg = CRYPTO_AES_CBC;
-	cri.cri_klen = ks->ks_class->ec_keybits;
-	cri.cri_mlen = 0;
-	cri.cri_key = ks->ks_key;
-	cri.cri_next = NULL;
-	crypto_newsession(&as->as_session_aes, &cri,
+	memset(&csp, 0, sizeof(csp));
+	csp.csp_mode = CSP_MODE_CIPHER;
+	csp.csp_cipher_alg = CRYPTO_AES_CBC;
+	csp.csp_cipher_klen = ks->ks_class->ec_keybits / 8;
+	csp.csp_cipher_key = ks->ks_key;
+	csp.csp_ivlen = 16;
+	crypto_newsession(&as->as_session_aes, &csp,
 	    CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE);
 }
 
 static void
 aes_random_to_key(struct krb5_key_state *ks, const void *in)
 {
 
 	aes_set_key(ks, in);
 }
 
 static int
 aes_crypto_cb(struct cryptop *crp)
 {
 	int error;
 	struct aes_state *as = (struct aes_state *) crp->crp_opaque;
 	
 	if (crypto_ses2caps(crp->crp_session) & CRYPTOCAP_F_SYNC)
 		return (0);
 
 	error = crp->crp_etype;
 	if (error == EAGAIN)
 		error = crypto_dispatch(crp);
 	mtx_lock(&as->as_lock);
 	if (error || (crp->crp_flags & CRYPTO_F_DONE))
 		wakeup(crp);
 	mtx_unlock(&as->as_lock);
 
 	return (0);
 }
 
 static void
 aes_encrypt_1(const struct krb5_key_state *ks, int buftype, void *buf,
-    size_t skip, size_t len, void *ivec, int encdec)
+    size_t skip, size_t len, void *ivec, bool encrypt)
 {
 	struct aes_state *as = ks->ks_priv;
 	struct cryptop *crp;
-	struct cryptodesc *crd;
 	int error;
 
-	crp = crypto_getreq(1);
-	crd = crp->crp_desc;
+	crp = crypto_getreq(as->as_session_aes, M_WAITOK);
 
-	crd->crd_skip = skip;
-	crd->crd_len = len;
-	crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT | encdec;
+	crp->crp_payload_start = skip;
+	crp->crp_payload_length = len;
+	crp->crp_op = encrypt ? CRYPTO_OP_ENCRYPT : CRYPTO_OP_DECRYPT;
+	crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_IV_SEPARATE;
 	if (ivec) {
-		bcopy(ivec, crd->crd_iv, 16);
+		memcpy(crp->crp_iv, ivec, 16);
 	} else {
-		bzero(crd->crd_iv, 16);
+		memset(crp->crp_iv, 0, 16);
 	}
-	crd->crd_next = NULL;
-	crd->crd_alg = CRYPTO_AES_CBC;
 
-	crp->crp_session = as->as_session_aes;
-	crp->crp_flags = buftype | CRYPTO_F_CBIFSYNC;
+	crp->crp_buf_type = buftype;
 	crp->crp_buf = buf;
-	crp->crp_opaque = (void *) as;
+	crp->crp_opaque = as;
 	crp->crp_callback = aes_crypto_cb;
 
 	error = crypto_dispatch(crp);
 
 	if ((crypto_ses2caps(as->as_session_aes) & CRYPTOCAP_F_SYNC) == 0) {
 		mtx_lock(&as->as_lock);
 		if (!error && !(crp->crp_flags & CRYPTO_F_DONE))
 			error = msleep(crp, &as->as_lock, 0, "gssaes", 0);
 		mtx_unlock(&as->as_lock);
 	}
 
 	crypto_freereq(crp);
 }
 
 static void
 aes_encrypt(const struct krb5_key_state *ks, struct mbuf *inout,
     size_t skip, size_t len, void *ivec, size_t ivlen)
 {
 	size_t blocklen = 16, plen;
 	struct {
 		uint8_t cn_1[16], cn[16];
 	} last2;
 	int i, off;
 
 	/*
 	 * AES encryption with cyphertext stealing:
 	 *
 	 * CTSencrypt(P[0], ..., P[n], IV, K):
 	 *	len = length(P[n])
 	 *	(C[0], ..., C[n-2], E[n-1]) =
 	 *		CBCencrypt(P[0], ..., P[n-1], IV, K)
 	 *	P = pad(P[n], 0, blocksize)
 	 *	E[n] = CBCencrypt(P, E[n-1], K);
 	 *	C[n-1] = E[n]
 	 *	C[n] = E[n-1]{0..len-1}
 	 */
 	plen = len % blocklen;
 	if (len == blocklen) {
 		/*
 		 * Note: caller will ensure len >= blocklen.
 		 */
-		aes_encrypt_1(ks, CRYPTO_F_IMBUF, inout, skip, len, ivec,
-		    CRD_F_ENCRYPT);
+		aes_encrypt_1(ks, CRYPTO_BUF_MBUF, inout, skip, len, ivec,
+		    true);
 	} else if (plen == 0) {
 		/*
 		 * This is equivalent to CBC mode followed by swapping
 		 * the last two blocks. We assume that neither of the
 		 * last two blocks cross iov boundaries.
 		 */
-		aes_encrypt_1(ks, CRYPTO_F_IMBUF, inout, skip, len, ivec,
-		    CRD_F_ENCRYPT);
+		aes_encrypt_1(ks, CRYPTO_BUF_MBUF, inout, skip, len, ivec,
+		    true);
 		off = skip + len - 2 * blocklen;
 		m_copydata(inout, off, 2 * blocklen, (void*) &last2);
 		m_copyback(inout, off, blocklen, last2.cn);
 		m_copyback(inout, off + blocklen, blocklen, last2.cn_1);
 	} else {
 		/*
 		 * This is the difficult case. We encrypt all but the
 		 * last partial block first. We then create a padded
 		 * copy of the last block and encrypt that using the
 		 * second to last encrypted block as IV. Once we have
 		 * the encrypted versions of the last two blocks, we
 		 * reshuffle to create the final result.
 		 */
-		aes_encrypt_1(ks, CRYPTO_F_IMBUF, inout, skip, len - plen,
-		    ivec, CRD_F_ENCRYPT);
+		aes_encrypt_1(ks, CRYPTO_BUF_MBUF, inout, skip, len - plen,
+		    ivec, true);
 
 		/*
 		 * Copy out the last two blocks, pad the last block
 		 * and encrypt it. Rearrange to get the final
 		 * result. The cyphertext for cn_1 is in cn. The
 		 * cyphertext for cn is the first plen bytes of what
 		 * is in cn_1 now.
 		 */
 		off = skip + len - blocklen - plen;
 		m_copydata(inout, off, blocklen + plen, (void*) &last2);
 		for (i = plen; i < blocklen; i++)
 			last2.cn[i] = 0;
-		aes_encrypt_1(ks, 0, last2.cn, 0, blocklen, last2.cn_1,
-		    CRD_F_ENCRYPT);
+		aes_encrypt_1(ks, CRYPTO_BUF_CONTIG, last2.cn, 0, blocklen,
+		    last2.cn_1, true);
 		m_copyback(inout, off, blocklen, last2.cn);
 		m_copyback(inout, off + blocklen, plen, last2.cn_1);
 	}
 }
 
 static void
 aes_decrypt(const struct krb5_key_state *ks, struct mbuf *inout,
     size_t skip, size_t len, void *ivec, size_t ivlen)
 {
 	size_t blocklen = 16, plen;
 	struct {
 		uint8_t cn_1[16], cn[16];
 	} last2;
 	int i, off, t;
 
 	/*
 	 * AES decryption with cyphertext stealing:
 	 *
 	 * CTSencrypt(C[0], ..., C[n], IV, K):
 	 *	len = length(C[n])
 	 *	E[n] = C[n-1]
 	 *	X = decrypt(E[n], K)
 	 *	P[n] = (X ^ C[n]){0..len-1}
 	 *	E[n-1] = {C[n,0],...,C[n,len-1],X[len],...,X[blocksize-1]}
 	 *	(P[0],...,P[n-1]) = CBCdecrypt(C[0],...,C[n-2],E[n-1], IV, K)
 	 */
 	plen = len % blocklen;
 	if (len == blocklen) {
 		/*
 		 * Note: caller will ensure len >= blocklen.
 		 */
-		aes_encrypt_1(ks, CRYPTO_F_IMBUF, inout, skip, len, ivec, 0);
+		aes_encrypt_1(ks, CRYPTO_BUF_MBUF, inout, skip, len, ivec,
+		    false);
 	} else if (plen == 0) {
 		/*
 		 * This is equivalent to CBC mode followed by swapping
 		 * the last two blocks.
 		 */
 		off = skip + len - 2 * blocklen;
 		m_copydata(inout, off, 2 * blocklen, (void*) &last2);
 		m_copyback(inout, off, blocklen, last2.cn);
 		m_copyback(inout, off + blocklen, blocklen, last2.cn_1);
-		aes_encrypt_1(ks, CRYPTO_F_IMBUF, inout, skip, len, ivec, 0);
+		aes_encrypt_1(ks, CRYPTO_BUF_MBUF, inout, skip, len, ivec,
+		    false);
 	} else {
 		/*
 		 * This is the difficult case. We first decrypt the
 		 * second to last block with a zero IV to make X. The
 		 * plaintext for the last block is the XOR of X and
 		 * the last cyphertext block.
 		 *
 		 * We derive a new cypher text for the second to last
 		 * block by mixing the unused bytes of X with the last
 		 * cyphertext block. The result of that can be
 		 * decrypted with the rest in CBC mode.
 		 */
 		off = skip + len - plen - blocklen;
-		aes_encrypt_1(ks, CRYPTO_F_IMBUF, inout, off, blocklen,
-		    NULL, 0);
+		aes_encrypt_1(ks, CRYPTO_BUF_MBUF, inout, off, blocklen,
+		    NULL, false);
 		m_copydata(inout, off, blocklen + plen, (void*) &last2);
 
 		for (i = 0; i < plen; i++) {
 			t = last2.cn[i];
 			last2.cn[i] ^= last2.cn_1[i];
 			last2.cn_1[i] = t;
 		}
 
 		m_copyback(inout, off, blocklen + plen, (void*) &last2);
-		aes_encrypt_1(ks, CRYPTO_F_IMBUF, inout, skip, len - plen,
-		    ivec, 0);
+		aes_encrypt_1(ks, CRYPTO_BUF_MBUF, inout, skip, len - plen,
+		    ivec, false);
 	}
 
 }
 
 static void
 aes_checksum(const struct krb5_key_state *ks, int usage,
     struct mbuf *inout, size_t skip, size_t inlen, size_t outlen)
 {
 	struct aes_state *as = ks->ks_priv;
 	struct cryptop *crp;
-	struct cryptodesc *crd;
 	int error;
 
-	crp = crypto_getreq(1);
-	crd = crp->crp_desc;
+	crp = crypto_getreq(as->as_session_sha1, M_WAITOK);
 
-	crd->crd_skip = skip;
-	crd->crd_len = inlen;
-	crd->crd_inject = skip + inlen;
-	crd->crd_flags = 0;
-	crd->crd_next = NULL;
-	crd->crd_alg = CRYPTO_SHA1_HMAC;
-
-	crp->crp_session = as->as_session_sha1;
-	crp->crp_ilen = inlen;
-	crp->crp_olen = 12;
-	crp->crp_etype = 0;
-	crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_CBIFSYNC;
-	crp->crp_buf = (void *) inout;
-	crp->crp_opaque = (void *) as;
+	crp->crp_payload_start = skip;
+	crp->crp_payload_length = inlen;
+	crp->crp_digest_start = skip + inlen;
+	crp->crp_flags = CRYPTO_F_CBIFSYNC;
+	crp->crp_buf_type = CRYPTO_BUF_MBUF;
+	crp->crp_mbuf = inout;
+	crp->crp_opaque = as;
 	crp->crp_callback = aes_crypto_cb;
 
 	error = crypto_dispatch(crp);
 
 	if ((crypto_ses2caps(as->as_session_sha1) & CRYPTOCAP_F_SYNC) == 0) {
 		mtx_lock(&as->as_lock);
 		if (!error && !(crp->crp_flags & CRYPTO_F_DONE))
 			error = msleep(crp, &as->as_lock, 0, "gssaes", 0);
 		mtx_unlock(&as->as_lock);
 	}
 
 	crypto_freereq(crp);
 }
 
 struct krb5_encryption_class krb5_aes128_encryption_class = {
 	"aes128-cts-hmac-sha1-96", /* name */
 	ETYPE_AES128_CTS_HMAC_SHA1_96, /* etype */
 	EC_DERIVED_KEYS,	/* flags */
 	16,			/* blocklen */
 	1,			/* msgblocklen */
 	12,			/* checksumlen */
 	128,			/* keybits */
 	16,			/* keylen */
 	aes_init,
 	aes_destroy,
 	aes_set_key,
 	aes_random_to_key,
 	aes_encrypt,
 	aes_decrypt,
 	aes_checksum
 };
 
 struct krb5_encryption_class krb5_aes256_encryption_class = {
 	"aes256-cts-hmac-sha1-96", /* name */
 	ETYPE_AES256_CTS_HMAC_SHA1_96, /* etype */
 	EC_DERIVED_KEYS,	/* flags */
 	16,			/* blocklen */
 	1,			/* msgblocklen */
 	12,			/* checksumlen */
 	256,			/* keybits */
 	32,			/* keylen */
 	aes_init,
 	aes_destroy,
 	aes_set_key,
 	aes_random_to_key,
 	aes_encrypt,
 	aes_decrypt,
 	aes_checksum
 };
Index: head/sys/kgssapi/krb5/kcrypto_des.c
===================================================================
--- head/sys/kgssapi/krb5/kcrypto_des.c	(revision 359373)
+++ head/sys/kgssapi/krb5/kcrypto_des.c	(revision 359374)
@@ -1,267 +1,260 @@
 /*-
  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
  *
  * Copyright (c) 2008 Isilon Inc http://www.isilon.com/
  * Authors: Doug Rabson <dfr@rabson.org>
  * Developed with Red Inc: Alfred Perlstein <alfred@freebsd.org>
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/lock.h>
 #include <sys/kobj.h>
 #include <sys/malloc.h>
 #include <sys/md5.h>
 #include <sys/mutex.h>
 #include <sys/mbuf.h>
 #include <crypto/des/des.h>
 #include <opencrypto/cryptodev.h>
 
 #include <kgssapi/gssapi.h>
 #include <kgssapi/gssapi_impl.h>
 
 #include "kcrypto.h"
 
 struct des1_state {
 	struct mtx	ds_lock;
 	crypto_session_t ds_session;
 };
 
 static void
 des1_init(struct krb5_key_state *ks)
 {
 	static struct timeval lastwarn;
 	struct des1_state *ds;
 
 	ds = malloc(sizeof(struct des1_state), M_GSSAPI, M_WAITOK|M_ZERO);
 	mtx_init(&ds->ds_lock, "gss des lock", NULL, MTX_DEF);
 	ks->ks_priv = ds;
 	if (ratecheck(&lastwarn, &krb5_warn_interval))
 		gone_in(13, "DES cipher for Kerberos GSS");
 }
 
 static void
 des1_destroy(struct krb5_key_state *ks)
 {
 	struct des1_state *ds = ks->ks_priv;
 
 	if (ds->ds_session)
 		crypto_freesession(ds->ds_session);
 	mtx_destroy(&ds->ds_lock);
 	free(ks->ks_priv, M_GSSAPI);
 
 }
 
 static void
 des1_set_key(struct krb5_key_state *ks, const void *in)
 {
+	struct crypto_session_params csp;
 	void *kp = ks->ks_key;
 	struct des1_state *ds = ks->ks_priv;
-	struct cryptoini cri[1];
 
-	if (kp != in)
-		bcopy(in, kp, ks->ks_class->ec_keylen);
-
 	if (ds->ds_session)
 		crypto_freesession(ds->ds_session);
 
-	bzero(cri, sizeof(cri));
+	if (kp != in)
+		bcopy(in, kp, ks->ks_class->ec_keylen);
 
-	cri[0].cri_alg = CRYPTO_DES_CBC;
-	cri[0].cri_klen = 64;
-	cri[0].cri_mlen = 0;
-	cri[0].cri_key = ks->ks_key;
-	cri[0].cri_next = NULL;
+	memset(&csp, 0, sizeof(csp));
+	csp.csp_mode = CSP_MODE_CIPHER;
+	csp.csp_ivlen = 8;
+	csp.csp_cipher_alg = CRYPTO_DES_CBC;
+	csp.csp_cipher_klen = 8;
+	csp.csp_cipher_key = ks->ks_key;
 
-	crypto_newsession(&ds->ds_session, cri,
+	crypto_newsession(&ds->ds_session, &csp,
 	    CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE);
 }
 
 static void
 des1_random_to_key(struct krb5_key_state *ks, const void *in)
 {
 	uint8_t *outkey = ks->ks_key;
 	const uint8_t *inkey = in;
 
 	/*
 	 * Expand 56 bits of random data to 64 bits as follows
 	 * (in the example, bit number 1 is the MSB of the 56
 	 * bits of random data):
 	 *
 	 * expanded = 
 	 *	 1  2  3  4  5  6  7  p
 	 *	 9 10 11 12 13 14 15  p
 	 *	17 18 19 20 21 22 23  p
 	 *	25 26 27 28 29 30 31  p
 	 *	33 34 35 36 37 38 39  p
 	 *	41 42 43 44 45 46 47  p
 	 *	49 50 51 52 53 54 55  p
 	 *	56 48 40 32 24 16  8  p
 	 */
 	outkey[0] = inkey[0];
 	outkey[1] = inkey[1];
 	outkey[2] = inkey[2];
 	outkey[3] = inkey[3];
 	outkey[4] = inkey[4];
 	outkey[5] = inkey[5];
 	outkey[6] = inkey[6];
 	outkey[7] = (((inkey[0] & 1) << 1)
 	    | ((inkey[1] & 1) << 2)
 	    | ((inkey[2] & 1) << 3)
 	    | ((inkey[3] & 1) << 4)
 	    | ((inkey[4] & 1) << 5)
 	    | ((inkey[5] & 1) << 6)
 	    | ((inkey[6] & 1) << 7));
 	des_set_odd_parity(outkey);
 	if (des_is_weak_key(outkey))
 		outkey[7] ^= 0xf0;
 
 	des1_set_key(ks, ks->ks_key);
 }
 
 static int
 des1_crypto_cb(struct cryptop *crp)
 {
 	int error;
 	struct des1_state *ds = (struct des1_state *) crp->crp_opaque;
 	
 	if (crypto_ses2caps(ds->ds_session) & CRYPTOCAP_F_SYNC)
 		return (0);
 
 	error = crp->crp_etype;
 	if (error == EAGAIN)
 		error = crypto_dispatch(crp);
 	mtx_lock(&ds->ds_lock);
 	if (error || (crp->crp_flags & CRYPTO_F_DONE))
 		wakeup(crp);
 	mtx_unlock(&ds->ds_lock);
 
 	return (0);
 }
 
 static void
-des1_encrypt_1(const struct krb5_key_state *ks, int buftype, void *buf,
-    size_t skip, size_t len, void *ivec, int encdec)
+des1_encrypt_1(const struct krb5_key_state *ks, int buf_type, void *buf,
+    size_t skip, size_t len, void *ivec, bool encrypt)
 {
 	struct des1_state *ds = ks->ks_priv;
 	struct cryptop *crp;
-	struct cryptodesc *crd;
 	int error;
 
-	crp = crypto_getreq(1);
-	crd = crp->crp_desc;
+	crp = crypto_getreq(ds->ds_session, M_WAITOK);
 
-	crd->crd_skip = skip;
-	crd->crd_len = len;
-	crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT | encdec;
+	crp->crp_payload_start = skip;
+	crp->crp_payload_length = len;
+	crp->crp_op = encrypt ? CRYPTO_OP_ENCRYPT : CRYPTO_OP_DECRYPT;
+	crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_IV_SEPARATE;
 	if (ivec) {
-		bcopy(ivec, crd->crd_iv, 8);
+		memcpy(crp->crp_iv, ivec, 8);
 	} else {
-		bzero(crd->crd_iv, 8);
+		memset(crp->crp_iv, 0, 8);
 	}
-	crd->crd_next = NULL;
-	crd->crd_alg = CRYPTO_DES_CBC;
-
-	crp->crp_session = ds->ds_session;
-	crp->crp_flags = buftype | CRYPTO_F_CBIFSYNC;
+	crp->crp_buf_type = buf_type;
 	crp->crp_buf = buf;
-	crp->crp_opaque = (void *) ds;
+	crp->crp_opaque = ds;
 	crp->crp_callback = des1_crypto_cb;
 
 	error = crypto_dispatch(crp);
 
 	if ((crypto_ses2caps(ds->ds_session) & CRYPTOCAP_F_SYNC) == 0) {
 		mtx_lock(&ds->ds_lock);
 		if (!error && !(crp->crp_flags & CRYPTO_F_DONE))
 			error = msleep(crp, &ds->ds_lock, 0, "gssdes", 0);
 		mtx_unlock(&ds->ds_lock);
 	}
 
 	crypto_freereq(crp);
 }
 
 static void
 des1_encrypt(const struct krb5_key_state *ks, struct mbuf *inout,
     size_t skip, size_t len, void *ivec, size_t ivlen)
 {
 
-	des1_encrypt_1(ks, CRYPTO_F_IMBUF, inout, skip, len, ivec,
-	    CRD_F_ENCRYPT);
+	des1_encrypt_1(ks, CRYPTO_BUF_MBUF, inout, skip, len, ivec, true);
 }
 
 static void
 des1_decrypt(const struct krb5_key_state *ks, struct mbuf *inout,
     size_t skip, size_t len, void *ivec, size_t ivlen)
 {
 
-	des1_encrypt_1(ks, CRYPTO_F_IMBUF, inout, skip, len, ivec, 0);
+	des1_encrypt_1(ks, CRYPTO_BUF_MBUF, inout, skip, len, ivec, false);
 }
 
 static int
 MD5Update_int(void *ctx, void *buf, u_int len)
 {
 
 	MD5Update(ctx, buf, len);
 	return (0);
 }
 
 static void
 des1_checksum(const struct krb5_key_state *ks, int usage,
     struct mbuf *inout, size_t skip, size_t inlen, size_t outlen)
 {
 	char hash[16];
 	MD5_CTX md5;
 
 	/*
 	 * This checksum is specifically for GSS-API. First take the
 	 * MD5 checksum of the message, then calculate the CBC mode
 	 * checksum of that MD5 checksum using a zero IV.
 	 */
 	MD5Init(&md5);
 	m_apply(inout, skip, inlen, MD5Update_int, &md5);
 	MD5Final(hash, &md5);
 
-	des1_encrypt_1(ks, 0, hash, 0, 16, NULL, CRD_F_ENCRYPT);
+	des1_encrypt_1(ks, CRYPTO_BUF_CONTIG, hash, 0, 16, NULL, true);
 	m_copyback(inout, skip + inlen, outlen, hash + 8);
 }
 
 struct krb5_encryption_class krb5_des_encryption_class = {
 	"des-cbc-md5",		/* name */
 	ETYPE_DES_CBC_CRC,	/* etype */
 	0,			/* flags */
 	8,			/* blocklen */
 	8,			/* msgblocklen */
 	8,			/* checksumlen */
 	56,			/* keybits */
 	8,			/* keylen */
 	des1_init,
 	des1_destroy,
 	des1_set_key,
 	des1_random_to_key,
 	des1_encrypt,
 	des1_decrypt,
 	des1_checksum
 };
Index: head/sys/kgssapi/krb5/kcrypto_des3.c
===================================================================
--- head/sys/kgssapi/krb5/kcrypto_des3.c	(revision 359373)
+++ head/sys/kgssapi/krb5/kcrypto_des3.c	(revision 359374)
@@ -1,407 +1,401 @@
 /*-
  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
  *
  * Copyright (c) 2008 Isilon Inc http://www.isilon.com/
  * Authors: Doug Rabson <dfr@rabson.org>
  * Developed with Red Inc: Alfred Perlstein <alfred@freebsd.org>
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/lock.h>
 #include <sys/malloc.h>
 #include <sys/mutex.h>
 #include <sys/kobj.h>
 #include <sys/mbuf.h>
 #include <crypto/des/des.h>
 #include <opencrypto/cryptodev.h>
 
 #include <kgssapi/gssapi.h>
 #include <kgssapi/gssapi_impl.h>
 
 #include "kcrypto.h"
 
 #define DES3_FLAGS	(CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)
 
 struct des3_state {
 	struct mtx	ds_lock;
-	crypto_session_t ds_session;
+	crypto_session_t ds_cipher_session;
+	crypto_session_t ds_hmac_session;
 };
 
 static void
 des3_init(struct krb5_key_state *ks)
 {
 	static struct timeval lastwarn;
 	struct des3_state *ds;
 
 	ds = malloc(sizeof(struct des3_state), M_GSSAPI, M_WAITOK|M_ZERO);
 	mtx_init(&ds->ds_lock, "gss des3 lock", NULL, MTX_DEF);
 	ks->ks_priv = ds;
 	if (ratecheck(&lastwarn, &krb5_warn_interval))
 		gone_in(13, "DES3 cipher for Kerberos GSS");
 }
 
 static void
 des3_destroy(struct krb5_key_state *ks)
 {
 	struct des3_state *ds = ks->ks_priv;
 
-	if (ds->ds_session)
-		crypto_freesession(ds->ds_session);
+	if (ds->ds_cipher_session) {
+		crypto_freesession(ds->ds_cipher_session);
+		crypto_freesession(ds->ds_hmac_session);
+	}
 	mtx_destroy(&ds->ds_lock);
 	free(ks->ks_priv, M_GSSAPI);
 }
 
 static void
 des3_set_key(struct krb5_key_state *ks, const void *in)
 {
+	struct crypto_session_params csp;
 	void *kp = ks->ks_key;
 	struct des3_state *ds = ks->ks_priv;
-	struct cryptoini cri[2];
 
+	if (ds->ds_cipher_session) {
+		crypto_freesession(ds->ds_cipher_session);
+		crypto_freesession(ds->ds_hmac_session);
+	}
+
 	if (kp != in)
 		bcopy(in, kp, ks->ks_class->ec_keylen);
 
-	if (ds->ds_session)
-		crypto_freesession(ds->ds_session);
+	memset(&csp, 0, sizeof(csp));
+	csp.csp_mode = CSP_MODE_DIGEST;
+	csp.csp_auth_alg = CRYPTO_SHA1_HMAC;
+	csp.csp_auth_klen = 24;
+	csp.csp_auth_key = ks->ks_key;
 
-	bzero(cri, sizeof(cri));
+	crypto_newsession(&ds->ds_hmac_session, &csp,
+	    CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE);
 
-	cri[0].cri_alg = CRYPTO_SHA1_HMAC;
-	cri[0].cri_klen = 192;
-	cri[0].cri_mlen = 0;
-	cri[0].cri_key = ks->ks_key;
-	cri[0].cri_next = &cri[1];
+	memset(&csp, 0, sizeof(csp));
+	csp.csp_mode = CSP_MODE_CIPHER;
+	csp.csp_cipher_alg = CRYPTO_3DES_CBC;
+	csp.csp_cipher_klen = 24;
+	csp.csp_cipher_key = ks->ks_key;
+	csp.csp_ivlen = 8;
 
-	cri[1].cri_alg = CRYPTO_3DES_CBC;
-	cri[1].cri_klen = 192;
-	cri[1].cri_mlen = 0;
-	cri[1].cri_key = ks->ks_key;
-	cri[1].cri_next = NULL;
-
-	crypto_newsession(&ds->ds_session, cri,
+	crypto_newsession(&ds->ds_cipher_session, &csp,
 	    CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE);
 }
 
 static void
 des3_random_to_key(struct krb5_key_state *ks, const void *in)
 {
 	uint8_t *outkey;
 	const uint8_t *inkey;
 	int subkey;
 
 	for (subkey = 0, outkey = ks->ks_key, inkey = in; subkey < 3;
 	     subkey++, outkey += 8, inkey += 7) {
 		/*
 		 * Expand 56 bits of random data to 64 bits as follows
 		 * (in the example, bit number 1 is the MSB of the 56
 		 * bits of random data):
 		 *
 		 * expanded = 
 		 *	 1  2  3  4  5  6  7  p
 		 *	 9 10 11 12 13 14 15  p
 		 *	17 18 19 20 21 22 23  p
 		 *	25 26 27 28 29 30 31  p
 		 *	33 34 35 36 37 38 39  p
 		 *	41 42 43 44 45 46 47  p
 		 *	49 50 51 52 53 54 55  p
 		 *	56 48 40 32 24 16  8  p
 		 */
 		outkey[0] = inkey[0];
 		outkey[1] = inkey[1];
 		outkey[2] = inkey[2];
 		outkey[3] = inkey[3];
 		outkey[4] = inkey[4];
 		outkey[5] = inkey[5];
 		outkey[6] = inkey[6];
 		outkey[7] = (((inkey[0] & 1) << 1)
 		    | ((inkey[1] & 1) << 2)
 		    | ((inkey[2] & 1) << 3)
 		    | ((inkey[3] & 1) << 4)
 		    | ((inkey[4] & 1) << 5)
 		    | ((inkey[5] & 1) << 6)
 		    | ((inkey[6] & 1) << 7));
 		des_set_odd_parity(outkey);
 		if (des_is_weak_key(outkey))
 			outkey[7] ^= 0xf0;
 	}
 
 	des3_set_key(ks, ks->ks_key);
 }
 
 static int
 des3_crypto_cb(struct cryptop *crp)
 {
 	int error;
 	struct des3_state *ds = (struct des3_state *) crp->crp_opaque;
 	
-	if (crypto_ses2caps(ds->ds_session) & CRYPTOCAP_F_SYNC)
+	if (crypto_ses2caps(crp->crp_session) & CRYPTOCAP_F_SYNC)
 		return (0);
 
 	error = crp->crp_etype;
 	if (error == EAGAIN)
 		error = crypto_dispatch(crp);
 	mtx_lock(&ds->ds_lock);
 	if (error || (crp->crp_flags & CRYPTO_F_DONE))
 		wakeup(crp);
 	mtx_unlock(&ds->ds_lock);
 
 	return (0);
 }
 
 static void
 des3_encrypt_1(const struct krb5_key_state *ks, struct mbuf *inout,
-    size_t skip, size_t len, void *ivec, int encdec)
+    size_t skip, size_t len, void *ivec, bool encrypt)
 {
 	struct des3_state *ds = ks->ks_priv;
 	struct cryptop *crp;
-	struct cryptodesc *crd;
 	int error;
 
-	crp = crypto_getreq(1);
-	crd = crp->crp_desc;
+	crp = crypto_getreq(ds->ds_cipher_session, M_WAITOK);
 
-	crd->crd_skip = skip;
-	crd->crd_len = len;
-	crd->crd_flags = CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT | encdec;
+	crp->crp_payload_start = skip;
+	crp->crp_payload_length = len;
+	crp->crp_op = encrypt ? CRYPTO_OP_ENCRYPT : CRYPTO_OP_DECRYPT;
+	crp->crp_flags = CRYPTO_F_CBIFSYNC | CRYPTO_F_IV_SEPARATE;
 	if (ivec) {
-		bcopy(ivec, crd->crd_iv, 8);
+		memcpy(crp->crp_iv, ivec, 8);
 	} else {
-		bzero(crd->crd_iv, 8);
+		memset(crp->crp_iv, 0, 8);
 	}
-	crd->crd_next = NULL;
-	crd->crd_alg = CRYPTO_3DES_CBC;
-
-	crp->crp_session = ds->ds_session;
-	crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_CBIFSYNC;
-	crp->crp_buf = (void *) inout;
-	crp->crp_opaque = (void *) ds;
+	crp->crp_buf_type = CRYPTO_BUF_MBUF;
+	crp->crp_mbuf = inout;
+	crp->crp_opaque = ds;
 	crp->crp_callback = des3_crypto_cb;
 
 	error = crypto_dispatch(crp);
 
-	if ((crypto_ses2caps(ds->ds_session) & CRYPTOCAP_F_SYNC) == 0) {
+	if ((crypto_ses2caps(ds->ds_cipher_session) & CRYPTOCAP_F_SYNC) == 0) {
 		mtx_lock(&ds->ds_lock);
 		if (!error && !(crp->crp_flags & CRYPTO_F_DONE))
 			error = msleep(crp, &ds->ds_lock, 0, "gssdes3", 0);
 		mtx_unlock(&ds->ds_lock);
 	}
 
 	crypto_freereq(crp);
 }
 
 static void
 des3_encrypt(const struct krb5_key_state *ks, struct mbuf *inout,
     size_t skip, size_t len, void *ivec, size_t ivlen)
 {
 
-	des3_encrypt_1(ks, inout, skip, len, ivec, CRD_F_ENCRYPT);
+	des3_encrypt_1(ks, inout, skip, len, ivec, true);
 }
 
 static void
 des3_decrypt(const struct krb5_key_state *ks, struct mbuf *inout,
     size_t skip, size_t len, void *ivec, size_t ivlen)
 {
 
-	des3_encrypt_1(ks, inout, skip, len, ivec, 0);
+	des3_encrypt_1(ks, inout, skip, len, ivec, false);
 }
 
 static void
 des3_checksum(const struct krb5_key_state *ks, int usage,
     struct mbuf *inout, size_t skip, size_t inlen, size_t outlen)
 {
 	struct des3_state *ds = ks->ks_priv;
 	struct cryptop *crp;
-	struct cryptodesc *crd;
 	int error;
 
-	crp = crypto_getreq(1);
-	crd = crp->crp_desc;
+	crp = crypto_getreq(ds->ds_hmac_session, M_WAITOK);
 
-	crd->crd_skip = skip;
-	crd->crd_len = inlen;
-	crd->crd_inject = skip + inlen;
-	crd->crd_flags = 0;
-	crd->crd_next = NULL;
-	crd->crd_alg = CRYPTO_SHA1_HMAC;
-
-	crp->crp_session = ds->ds_session;
-	crp->crp_ilen = inlen;
-	crp->crp_olen = 20;
-	crp->crp_etype = 0;
-	crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_CBIFSYNC;
-	crp->crp_buf = (void *) inout;
-	crp->crp_opaque = (void *) ds;
+	crp->crp_payload_start = skip;
+	crp->crp_payload_length = inlen;
+	crp->crp_digest_start = skip + inlen;
+	crp->crp_op = CRYPTO_OP_COMPUTE_DIGEST;
+	crp->crp_flags = CRYPTO_F_CBIFSYNC;
+	crp->crp_buf_type = CRYPTO_BUF_MBUF;
+	crp->crp_mbuf = inout;
+	crp->crp_opaque = ds;
 	crp->crp_callback = des3_crypto_cb;
 
 	error = crypto_dispatch(crp);
 
-	if ((crypto_ses2caps(ds->ds_session) & CRYPTOCAP_F_SYNC) == 0) {
+	if ((crypto_ses2caps(ds->ds_hmac_session) & CRYPTOCAP_F_SYNC) == 0) {
 		mtx_lock(&ds->ds_lock);
 		if (!error && !(crp->crp_flags & CRYPTO_F_DONE))
 			error = msleep(crp, &ds->ds_lock, 0, "gssdes3", 0);
 		mtx_unlock(&ds->ds_lock);
 	}
 
 	crypto_freereq(crp);
 }
 
 struct krb5_encryption_class krb5_des3_encryption_class = {
 	"des3-cbc-sha1",	/* name */
 	ETYPE_DES3_CBC_SHA1,	/* etype */
 	EC_DERIVED_KEYS,	/* flags */
 	8,			/* blocklen */
 	8,			/* msgblocklen */
 	20,			/* checksumlen */
 	168,			/* keybits */
 	24,			/* keylen */
 	des3_init,
 	des3_destroy,
 	des3_set_key,
 	des3_random_to_key,
 	des3_encrypt,
 	des3_decrypt,
 	des3_checksum
 };
 
 #if 0
 struct des3_dk_test {
 	uint8_t key[24];
 	uint8_t usage[8];
 	size_t usagelen;
 	uint8_t dk[24];
 };
 struct des3_dk_test tests[] = {
 	{{0xdc, 0xe0, 0x6b, 0x1f, 0x64, 0xc8, 0x57, 0xa1, 0x1c, 0x3d, 0xb5,
 	  0x7c, 0x51, 0x89, 0x9b, 0x2c, 0xc1, 0x79, 0x10, 0x08, 0xce, 0x97,
 	  0x3b, 0x92},
 	 {0x00, 0x00, 0x00, 0x01, 0x55}, 5,
 	 {0x92, 0x51, 0x79, 0xd0, 0x45, 0x91, 0xa7, 0x9b, 0x5d, 0x31, 0x92,
 	  0xc4, 0xa7, 0xe9, 0xc2, 0x89, 0xb0, 0x49, 0xc7, 0x1f, 0x6e, 0xe6,
 	  0x04, 0xcd}},
 
 	{{0x5e, 0x13, 0xd3, 0x1c, 0x70, 0xef, 0x76, 0x57, 0x46, 0x57, 0x85,
 	  0x31, 0xcb, 0x51, 0xc1, 0x5b, 0xf1, 0x1c, 0xa8, 0x2c, 0x97, 0xce,
 	  0xe9, 0xf2},
 	 {0x00, 0x00, 0x00, 0x01, 0xaa}, 5,
 	 {0x9e, 0x58, 0xe5, 0xa1, 0x46, 0xd9, 0x94, 0x2a, 0x10, 0x1c, 0x46,
 	  0x98, 0x45, 0xd6, 0x7a, 0x20, 0xe3, 0xc4, 0x25, 0x9e, 0xd9, 0x13,
 	  0xf2, 0x07}},
 
 	{{0x98, 0xe6, 0xfd, 0x8a, 0x04, 0xa4, 0xb6, 0x85, 0x9b, 0x75, 0xa1,
 	  0x76, 0x54, 0x0b, 0x97, 0x52, 0xba, 0xd3, 0xec, 0xd6, 0x10, 0xa2,
 	  0x52, 0xbc},
 	 {0x00, 0x00, 0x00, 0x01, 0x55}, 5,
 	 {0x13, 0xfe, 0xf8, 0x0d, 0x76, 0x3e, 0x94, 0xec, 0x6d, 0x13, 0xfd,
 	  0x2c, 0xa1, 0xd0, 0x85, 0x07, 0x02, 0x49, 0xda, 0xd3, 0x98, 0x08,
 	  0xea, 0xbf}},
 
 	{{0x62, 0x2a, 0xec, 0x25, 0xa2, 0xfe, 0x2c, 0xad, 0x70, 0x94, 0x68,
 	  0x0b, 0x7c, 0x64, 0x94, 0x02, 0x80, 0x08, 0x4c, 0x1a, 0x7c, 0xec,
 	  0x92, 0xb5},
 	 {0x00, 0x00, 0x00, 0x01, 0xaa}, 5,
 	 {0xf8, 0xdf, 0xbf, 0x04, 0xb0, 0x97, 0xe6, 0xd9, 0xdc, 0x07, 0x02,
 	  0x68, 0x6b, 0xcb, 0x34, 0x89, 0xd9, 0x1f, 0xd9, 0xa4, 0x51, 0x6b,
 	  0x70, 0x3e}},
 
 	{{0xd3, 0xf8, 0x29, 0x8c, 0xcb, 0x16, 0x64, 0x38, 0xdc, 0xb9, 0xb9,
 	  0x3e, 0xe5, 0xa7, 0x62, 0x92, 0x86, 0xa4, 0x91, 0xf8, 0x38, 0xf8,
 	  0x02, 0xfb},
 	 {0x6b, 0x65, 0x72, 0x62, 0x65, 0x72, 0x6f, 0x73}, 8,
 	 {0x23, 0x70, 0xda, 0x57, 0x5d, 0x2a, 0x3d, 0xa8, 0x64, 0xce, 0xbf,
 	  0xdc, 0x52, 0x04, 0xd5, 0x6d, 0xf7, 0x79, 0xa7, 0xdf, 0x43, 0xd9,
 	  0xda, 0x43}},
 
 	{{0xc1, 0x08, 0x16, 0x49, 0xad, 0xa7, 0x43, 0x62, 0xe6, 0xa1, 0x45,
 	  0x9d, 0x01, 0xdf, 0xd3, 0x0d, 0x67, 0xc2, 0x23, 0x4c, 0x94, 0x07,
 	  0x04, 0xda},
 	 {0x00, 0x00, 0x00, 0x01, 0x55}, 5,
 	 {0x34, 0x80, 0x57, 0xec, 0x98, 0xfd, 0xc4, 0x80, 0x16, 0x16, 0x1c,
 	  0x2a, 0x4c, 0x7a, 0x94, 0x3e, 0x92, 0xae, 0x49, 0x2c, 0x98, 0x91,
 	  0x75, 0xf7}},
 
 	{{0x5d, 0x15, 0x4a, 0xf2, 0x38, 0xf4, 0x67, 0x13, 0x15, 0x57, 0x19,
 	  0xd5, 0x5e, 0x2f, 0x1f, 0x79, 0x0d, 0xd6, 0x61, 0xf2, 0x79, 0xa7,
 	  0x91, 0x7c},
 	 {0x00, 0x00, 0x00, 0x01, 0xaa}, 5,
 	 {0xa8, 0x80, 0x8a, 0xc2, 0x67, 0xda, 0xda, 0x3d, 0xcb, 0xe9, 0xa7,
 	  0xc8, 0x46, 0x26, 0xfb, 0xc7, 0x61, 0xc2, 0x94, 0xb0, 0x13, 0x15,
 	  0xe5, 0xc1}},
 
 	{{0x79, 0x85, 0x62, 0xe0, 0x49, 0x85, 0x2f, 0x57, 0xdc, 0x8c, 0x34,
 	  0x3b, 0xa1, 0x7f, 0x2c, 0xa1, 0xd9, 0x73, 0x94, 0xef, 0xc8, 0xad,
 	  0xc4, 0x43},
 	 {0x00, 0x00, 0x00, 0x01, 0x55}, 5,
 	 {0xc8, 0x13, 0xf8, 0x8a, 0x3b, 0xe3, 0xb3, 0x34, 0xf7, 0x54, 0x25,
 	  0xce, 0x91, 0x75, 0xfb, 0xe3, 0xc8, 0x49, 0x3b, 0x89, 0xc8, 0x70,
 	  0x3b, 0x49}},
 
 	{{0x26, 0xdc, 0xe3, 0x34, 0xb5, 0x45, 0x29, 0x2f, 0x2f, 0xea, 0xb9,
 	  0xa8, 0x70, 0x1a, 0x89, 0xa4, 0xb9, 0x9e, 0xb9, 0x94, 0x2c, 0xec,
 	  0xd0, 0x16},
 	 {0x00, 0x00, 0x00, 0x01, 0xaa}, 5,
 	 {0xf4, 0x8f, 0xfd, 0x6e, 0x83, 0xf8, 0x3e, 0x73, 0x54, 0xe6, 0x94,
 	  0xfd, 0x25, 0x2c, 0xf8, 0x3b, 0xfe, 0x58, 0xf7, 0xd5, 0xba, 0x37,
 	  0xec, 0x5d}},
 };
 #define N_TESTS		(sizeof(tests) / sizeof(tests[0]))
 
 int
 main(int argc, char **argv)
 {
 	struct krb5_key_state *key, *dk;
 	uint8_t *dkp;
 	int j, i;
 
 	for (j = 0; j < N_TESTS; j++) {
 		struct des3_dk_test *t = &tests[j];
 		key = krb5_create_key(&des3_encryption_class);
 		krb5_set_key(key, t->key);
 		dk = krb5_derive_key(key, t->usage, t->usagelen);
 		krb5_free_key(key);
 		if (memcmp(dk->ks_key, t->dk, 24)) {
 			printf("DES3 dk(");
 			for (i = 0; i < 24; i++)
 				printf("%02x", t->key[i]);
 			printf(", ");
 			for (i = 0; i < t->usagelen; i++)
 				printf("%02x", t->usage[i]);
 			printf(") failed\n");
 			printf("should be: ");
 			for (i = 0; i < 24; i++)
 				printf("%02x", t->dk[i]);
 			printf("\n result was: ");
 			dkp = dk->ks_key;
 			for (i = 0; i < 24; i++)
 				printf("%02x", dkp[i]);
 			printf("\n");
 		}
 		krb5_free_key(dk);
 	}
 
 	return (0);
 }
 #endif
Index: head/sys/mips/cavium/cryptocteon/cavium_crypto.c
===================================================================
--- head/sys/mips/cavium/cryptocteon/cavium_crypto.c	(revision 359373)
+++ head/sys/mips/cavium/cryptocteon/cavium_crypto.c	(revision 359374)
@@ -1,2138 +1,2098 @@
 /*
  * vim:sw=4 ts=8
  */
 /*-
  * SPDX-License-Identifier: BSD-4-Clause
  *
  * Copyright (c) 2009 David McCullough <david.mccullough@securecomputing.com>
  *
  * Copyright (c) 2003-2007 Cavium Networks (support@cavium.com). All rights
  * reserved.
  * 
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions are met:
  * 1. Redistributions of source code must retain the above copyright notice,
  * this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright notice,
  * this list of conditions and the following disclaimer in the documentation
  * and/or other materials provided with the distribution.
  * 3. All advertising materials mentioning features or use of this software
  * must display the following acknowledgement:
  * This product includes software developed by Cavium Networks
  * 4. Cavium Networks' name may not be used to endorse or promote products
  * derived from this software without specific prior written permission.
  * 
  * This Software, including technical data, may be subject to U.S. export
  * control laws, including the U.S. Export Administration Act and its
  * associated regulations, and may be subject to export or import regulations
  * in other countries. You warrant that You will comply strictly in all
  * respects with all such regulations and acknowledge that you have the
  * responsibility to obtain licenses to export, re-export or import the
  * Software.
  * 
  * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS" AND
  * WITH ALL FAULTS AND CAVIUM MAKES NO PROMISES, REPRESENTATIONS OR WARRANTIES,
  * EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO THE
  * SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION OR
  * DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM
  * SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE,
  * MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF
  * VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR
  * CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR
  * PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
 */
 /****************************************************************************/
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/module.h>
 #include <sys/malloc.h>
 #include <sys/uio.h>
 
 #include <opencrypto/cryptodev.h>
 
 #include <contrib/octeon-sdk/cvmx.h>
 
 #include <mips/cavium/cryptocteon/cryptocteonvar.h>
 
 /****************************************************************************/
 
 #define	IOV_INIT(iov, ptr, idx, len)					\
 	do {								\
 	    (idx) = 0;							\
 	    (ptr) = (iov)[(idx)].iov_base;				\
 	    (len) = (iov)[(idx)].iov_len;				\
 	} while (0)
 
 /*
  * XXX
  * It would be better if this were an IOV_READ/IOV_WRITE macro instead so
  * that we could detect overflow before it happens rather than right after,
  * which is especially bad since there is usually no IOV_CONSUME after the
  * final read or write.
  */
 #define	IOV_CONSUME(iov, ptr, idx, len)					\
 	do {								\
 	    if ((len) > sizeof *(ptr)) {				\
 		(len) -= sizeof *(ptr);					\
 		(ptr)++;						\
 	    } else {							\
 		if ((len) != sizeof *(ptr))				\
 			panic("%s: went past end of iovec.", __func__);	\
 		(idx)++;						\
 		(ptr) = (iov)[(idx)].iov_base;				\
 		(len) = (iov)[(idx)].iov_len;				\
 	    }								\
 	} while (0)
 
 #define ESP_HEADER_LENGTH     8
 #define DES_CBC_IV_LENGTH     8
 #define AES_CBC_IV_LENGTH     16
 #define ESP_HMAC_LEN          12
 
 #define ESP_HEADER_LENGTH 8
 #define DES_CBC_IV_LENGTH 8
 
 /****************************************************************************/
 
 #define CVM_LOAD_SHA_UNIT(dat, next)  { \
    if (next == 0) {                     \
       next = 1;                         \
       CVMX_MT_HSH_DAT (dat, 0);         \
    } else if (next == 1) {              \
       next = 2;                         \
       CVMX_MT_HSH_DAT (dat, 1);         \
    } else if (next == 2) {              \
       next = 3;                    \
       CVMX_MT_HSH_DAT (dat, 2);         \
    } else if (next == 3) {              \
       next = 4;                         \
       CVMX_MT_HSH_DAT (dat, 3);         \
    } else if (next == 4) {              \
       next = 5;                           \
       CVMX_MT_HSH_DAT (dat, 4);         \
    } else if (next == 5) {              \
       next = 6;                         \
       CVMX_MT_HSH_DAT (dat, 5);         \
    } else if (next == 6) {              \
       next = 7;                         \
       CVMX_MT_HSH_DAT (dat, 6);         \
    } else {                             \
      CVMX_MT_HSH_STARTSHA (dat);        \
      next = 0;                          \
    }                                    \
 }
 
 #define CVM_LOAD2_SHA_UNIT(dat1, dat2, next)  { \
    if (next == 0) {                      \
       CVMX_MT_HSH_DAT (dat1, 0);         \
       CVMX_MT_HSH_DAT (dat2, 1);         \
       next = 2;                          \
    } else if (next == 1) {               \
       CVMX_MT_HSH_DAT (dat1, 1);         \
       CVMX_MT_HSH_DAT (dat2, 2);         \
       next = 3;                          \
    } else if (next == 2) {               \
       CVMX_MT_HSH_DAT (dat1, 2);         \
       CVMX_MT_HSH_DAT (dat2, 3);         \
       next = 4;                          \
    } else if (next == 3) {               \
       CVMX_MT_HSH_DAT (dat1, 3);         \
       CVMX_MT_HSH_DAT (dat2, 4);         \
       next = 5;                          \
    } else if (next == 4) {               \
       CVMX_MT_HSH_DAT (dat1, 4);         \
       CVMX_MT_HSH_DAT (dat2, 5);         \
       next = 6;                          \
    } else if (next == 5) {               \
       CVMX_MT_HSH_DAT (dat1, 5);         \
       CVMX_MT_HSH_DAT (dat2, 6);         \
       next = 7;                          \
    } else if (next == 6) {               \
       CVMX_MT_HSH_DAT (dat1, 6);         \
       CVMX_MT_HSH_STARTSHA (dat2);       \
       next = 0;                          \
    } else {                              \
      CVMX_MT_HSH_STARTSHA (dat1);        \
      CVMX_MT_HSH_DAT (dat2, 0);          \
      next = 1;                           \
    }                                     \
 }
 
 /****************************************************************************/
 
 #define CVM_LOAD_MD5_UNIT(dat, next)  { \
    if (next == 0) {                     \
       next = 1;                         \
       CVMX_MT_HSH_DAT (dat, 0);         \
    } else if (next == 1) {              \
       next = 2;                         \
       CVMX_MT_HSH_DAT (dat, 1);         \
    } else if (next == 2) {              \
       next = 3;                    \
       CVMX_MT_HSH_DAT (dat, 2);         \
    } else if (next == 3) {              \
       next = 4;                         \
       CVMX_MT_HSH_DAT (dat, 3);         \
    } else if (next == 4) {              \
       next = 5;                           \
       CVMX_MT_HSH_DAT (dat, 4);         \
    } else if (next == 5) {              \
       next = 6;                         \
       CVMX_MT_HSH_DAT (dat, 5);         \
    } else if (next == 6) {              \
       next = 7;                         \
       CVMX_MT_HSH_DAT (dat, 6);         \
    } else {                             \
      CVMX_MT_HSH_STARTMD5 (dat);        \
      next = 0;                          \
    }                                    \
 }
 
 #define CVM_LOAD2_MD5_UNIT(dat1, dat2, next)  { \
    if (next == 0) {                      \
       CVMX_MT_HSH_DAT (dat1, 0);         \
       CVMX_MT_HSH_DAT (dat2, 1);         \
       next = 2;                          \
    } else if (next == 1) {               \
       CVMX_MT_HSH_DAT (dat1, 1);         \
       CVMX_MT_HSH_DAT (dat2, 2);         \
       next = 3;                          \
    } else if (next == 2) {               \
       CVMX_MT_HSH_DAT (dat1, 2);         \
       CVMX_MT_HSH_DAT (dat2, 3);         \
       next = 4;                          \
    } else if (next == 3) {               \
       CVMX_MT_HSH_DAT (dat1, 3);         \
       CVMX_MT_HSH_DAT (dat2, 4);         \
       next = 5;                          \
    } else if (next == 4) {               \
       CVMX_MT_HSH_DAT (dat1, 4);         \
       CVMX_MT_HSH_DAT (dat2, 5);         \
       next = 6;                          \
    } else if (next == 5) {               \
       CVMX_MT_HSH_DAT (dat1, 5);         \
       CVMX_MT_HSH_DAT (dat2, 6);         \
       next = 7;                          \
    } else if (next == 6) {               \
       CVMX_MT_HSH_DAT (dat1, 6);         \
       CVMX_MT_HSH_STARTMD5 (dat2);       \
       next = 0;                          \
    } else {                              \
      CVMX_MT_HSH_STARTMD5 (dat1);        \
      CVMX_MT_HSH_DAT (dat2, 0);          \
      next = 1;                           \
    }                                     \
 }
 
 /****************************************************************************/
 
 void
 octo_calc_hash(uint8_t auth, unsigned char *key, uint64_t *inner, uint64_t *outer)
 {
     uint8_t hash_key[64];
     uint64_t *key1;
     register uint64_t xor1 = 0x3636363636363636ULL;
     register uint64_t xor2 = 0x5c5c5c5c5c5c5c5cULL;
 
     dprintf("%s()\n", __func__);
 
     memset(hash_key, 0, sizeof(hash_key));
     memcpy(hash_key, (uint8_t *) key, (auth ? 20 : 16));
     key1 = (uint64_t *) hash_key;
     if (auth) {
        CVMX_MT_HSH_IV(0x67452301EFCDAB89ULL, 0);
        CVMX_MT_HSH_IV(0x98BADCFE10325476ULL, 1);
        CVMX_MT_HSH_IV(0xC3D2E1F000000000ULL, 2);
     } else {
        CVMX_MT_HSH_IV(0x0123456789ABCDEFULL, 0);
        CVMX_MT_HSH_IV(0xFEDCBA9876543210ULL, 1);
     }
 
     CVMX_MT_HSH_DAT((*key1 ^ xor1), 0);
     key1++;
     CVMX_MT_HSH_DAT((*key1 ^ xor1), 1);
     key1++;
     CVMX_MT_HSH_DAT((*key1 ^ xor1), 2);
     key1++;
     CVMX_MT_HSH_DAT((*key1 ^ xor1), 3);
     key1++;
     CVMX_MT_HSH_DAT((*key1 ^ xor1), 4);
     key1++;
     CVMX_MT_HSH_DAT((*key1 ^ xor1), 5);
     key1++;
     CVMX_MT_HSH_DAT((*key1 ^ xor1), 6);
     key1++;
     if (auth)
 		CVMX_MT_HSH_STARTSHA((*key1 ^ xor1));
     else
 		CVMX_MT_HSH_STARTMD5((*key1 ^ xor1));
 
     CVMX_MF_HSH_IV(inner[0], 0);
     CVMX_MF_HSH_IV(inner[1], 1);
     if (auth) {
 		inner[2] = 0;
 		CVMX_MF_HSH_IV(((uint64_t *) inner)[2], 2);
     }
 
     memset(hash_key, 0, sizeof(hash_key));
     memcpy(hash_key, (uint8_t *) key, (auth ? 20 : 16));
     key1 = (uint64_t *) hash_key;
     if (auth) {
       CVMX_MT_HSH_IV(0x67452301EFCDAB89ULL, 0);
       CVMX_MT_HSH_IV(0x98BADCFE10325476ULL, 1);
       CVMX_MT_HSH_IV(0xC3D2E1F000000000ULL, 2);
     } else {
       CVMX_MT_HSH_IV(0x0123456789ABCDEFULL, 0);
       CVMX_MT_HSH_IV(0xFEDCBA9876543210ULL, 1);
     }
 
     CVMX_MT_HSH_DAT((*key1 ^ xor2), 0);
     key1++;
     CVMX_MT_HSH_DAT((*key1 ^ xor2), 1);
     key1++;
     CVMX_MT_HSH_DAT((*key1 ^ xor2), 2);
     key1++;
     CVMX_MT_HSH_DAT((*key1 ^ xor2), 3);
     key1++;
     CVMX_MT_HSH_DAT((*key1 ^ xor2), 4);
     key1++;
     CVMX_MT_HSH_DAT((*key1 ^ xor2), 5);
     key1++;
     CVMX_MT_HSH_DAT((*key1 ^ xor2), 6);
     key1++;
     if (auth)
        CVMX_MT_HSH_STARTSHA((*key1 ^ xor2));
     else 
        CVMX_MT_HSH_STARTMD5((*key1 ^ xor2));
 
     CVMX_MF_HSH_IV(outer[0], 0);
     CVMX_MF_HSH_IV(outer[1], 1);
     if (auth) {
       outer[2] = 0;
       CVMX_MF_HSH_IV(outer[2], 2);
     }
     return;
 }
 
 /****************************************************************************/
 /* DES functions */
 
 int
 octo_des_cbc_encrypt(
     struct octo_sess *od,
     struct iovec *iov, size_t iovcnt, size_t iovlen,
     int auth_off, int auth_len,
     int crypt_off, int crypt_len,
-    int icv_off, uint8_t *ivp)
+    uint8_t *icv, uint8_t *ivp)
 {
     uint64_t *data;
     int data_i, data_l;
 
     dprintf("%s()\n", __func__);
 
     if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
 	    (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) {
 	dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
 		"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
-		"icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
-		auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+		"icv=%p ivp=%p\n", __func__, od, iov, iovlen,
+		auth_off, auth_len, crypt_off, crypt_len, icv, ivp);
 	return -EINVAL;
     }
 
     IOV_INIT(iov, data, data_i, data_l);
 
     CVMX_PREFETCH0(ivp);
     CVMX_PREFETCH0(od->octo_enckey);
 
 
     /* load 3DES Key */
     CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
     if (od->octo_encklen == 24) {
 	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
 	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
     } else if (od->octo_encklen == 8) {
 	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
 	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
     } else {
 	dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
 	return -EINVAL;
     }
 
     CVMX_MT_3DES_IV(* (uint64_t *) ivp);
 
     while (crypt_off > 0) {
 	IOV_CONSUME(iov, data, data_i, data_l);
 	crypt_off -= 8;
     }
 
     while (crypt_len > 0) {
 	CVMX_MT_3DES_ENC_CBC(*data);
 	CVMX_MF_3DES_RESULT(*data);
 	IOV_CONSUME(iov, data, data_i, data_l);
 	crypt_len -= 8;
     }
 
     return 0;
 }
 
 
 int
 octo_des_cbc_decrypt(
     struct octo_sess *od,
     struct iovec *iov, size_t iovcnt, size_t iovlen,
     int auth_off, int auth_len,
     int crypt_off, int crypt_len,
-    int icv_off, uint8_t *ivp)
+    uint8_t *icv, uint8_t *ivp)
 {
     uint64_t *data;
     int data_i, data_l;
 
     dprintf("%s()\n", __func__);
 
     if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
 	    (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) {
 	dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
 		"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
-		"icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
-		auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+		"icv=%p ivp=%p\n", __func__, od, iov, iovlen,
+		auth_off, auth_len, crypt_off, crypt_len, icv, ivp);
 	return -EINVAL;
     }
 
     IOV_INIT(iov, data, data_i, data_l);
 
     CVMX_PREFETCH0(ivp);
     CVMX_PREFETCH0(od->octo_enckey);
 
     /* load 3DES Key */
     CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
     if (od->octo_encklen == 24) {
 	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
 	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
     } else if (od->octo_encklen == 8) {
 	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
 	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
     } else {
 	dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
 	return -EINVAL;
     }
 
     CVMX_MT_3DES_IV(* (uint64_t *) ivp);
 
     while (crypt_off > 0) {
 	IOV_CONSUME(iov, data, data_i, data_l);
 	crypt_off -= 8;
     }
 
     while (crypt_len > 0) {
 	CVMX_MT_3DES_DEC_CBC(*data);
 	CVMX_MF_3DES_RESULT(*data);
 	IOV_CONSUME(iov, data, data_i, data_l);
 	crypt_len -= 8;
     }
 
     return 0;
 }
 
 /****************************************************************************/
 /* AES functions */
 
 int
 octo_aes_cbc_encrypt(
     struct octo_sess *od,
     struct iovec *iov, size_t iovcnt, size_t iovlen,
     int auth_off, int auth_len,
     int crypt_off, int crypt_len,
-    int icv_off, uint8_t *ivp)
+    uint8_t *icv, uint8_t *ivp)
 {
     uint64_t *data, *pdata;
     int data_i, data_l;
 
     dprintf("%s()\n", __func__);
 
     if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
 	    (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) {
 	dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
 		"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
-		"icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
-		auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+		"icv=%p ivp=%p\n", __func__, od, iov, iovlen,
+		auth_off, auth_len, crypt_off, crypt_len, icv, ivp);
 	return -EINVAL;
     }
 
     IOV_INIT(iov, data, data_i, data_l);
 
     CVMX_PREFETCH0(ivp);
     CVMX_PREFETCH0(od->octo_enckey);
 
     /* load AES Key */
     CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
     CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
 
     if (od->octo_encklen == 16) {
 	CVMX_MT_AES_KEY(0x0, 2);
 	CVMX_MT_AES_KEY(0x0, 3);
     } else if (od->octo_encklen == 24) {
 	CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
 	CVMX_MT_AES_KEY(0x0, 3);
     } else if (od->octo_encklen == 32) {
 	CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
 	CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
     } else {
 	dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
 	return -EINVAL;
     }
     CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
 
     CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
     CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
 
     while (crypt_off > 0) {
 	IOV_CONSUME(iov, data, data_i, data_l);
 	crypt_off -= 8;
     }
 
     while (crypt_len > 0) {
 	pdata = data;
 	CVMX_MT_AES_ENC_CBC0(*data);
 	IOV_CONSUME(iov, data, data_i, data_l);
 	CVMX_MT_AES_ENC_CBC1(*data);
 	CVMX_MF_AES_RESULT(*pdata, 0);
 	CVMX_MF_AES_RESULT(*data, 1);
 	IOV_CONSUME(iov, data, data_i, data_l);
 	crypt_len -= 16;
     }
 
     return 0;
 }
 
 
 int
 octo_aes_cbc_decrypt(
     struct octo_sess *od,
     struct iovec *iov, size_t iovcnt, size_t iovlen,
     int auth_off, int auth_len,
     int crypt_off, int crypt_len,
-    int icv_off, uint8_t *ivp)
+    uint8_t *icv, uint8_t *ivp)
 {
     uint64_t *data, *pdata;
     int data_i, data_l;
 
     dprintf("%s()\n", __func__);
 
     if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
 	    (crypt_off & 0x7) || (crypt_off + crypt_len > iovlen))) {
 	dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
 		"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
-		"icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
-		auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+		"icv=%p ivp=%p\n", __func__, od, iov, iovlen,
+		auth_off, auth_len, crypt_off, crypt_len, icv, ivp);
 	return -EINVAL;
     }
 
     IOV_INIT(iov, data, data_i, data_l);
 
     CVMX_PREFETCH0(ivp);
     CVMX_PREFETCH0(od->octo_enckey);
 
     /* load AES Key */
     CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
     CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
 
     if (od->octo_encklen == 16) {
 	CVMX_MT_AES_KEY(0x0, 2);
 	CVMX_MT_AES_KEY(0x0, 3);
     } else if (od->octo_encklen == 24) {
 	CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
 	CVMX_MT_AES_KEY(0x0, 3);
     } else if (od->octo_encklen == 32) {
 	CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
 	CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
     } else {
 	dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
 	return -EINVAL;
     }
     CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
 
     CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
     CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
 
     while (crypt_off > 0) {
 	IOV_CONSUME(iov, data, data_i, data_l);
 	crypt_off -= 8;
     }
 
     while (crypt_len > 0) {
 	pdata = data;
 	CVMX_MT_AES_DEC_CBC0(*data);
 	IOV_CONSUME(iov, data, data_i, data_l);
 	CVMX_MT_AES_DEC_CBC1(*data);
 	CVMX_MF_AES_RESULT(*pdata, 0);
 	CVMX_MF_AES_RESULT(*data, 1);
 	IOV_CONSUME(iov, data, data_i, data_l);
 	crypt_len -= 16;
     }
 
     return 0;
 }
 
 /****************************************************************************/
 /* MD5 */
 
 int
 octo_null_md5_encrypt(
     struct octo_sess *od,
     struct iovec *iov, size_t iovcnt, size_t iovlen,
     int auth_off, int auth_len,
     int crypt_off, int crypt_len,
-    int icv_off, uint8_t *ivp)
+    uint8_t *icv, uint8_t *ivp)
 {
     int next = 0;
     uint64_t *data;
     uint64_t tmp1, tmp2;
     int data_i, data_l, alen = auth_len;
 
     dprintf("%s()\n", __func__);
 
     if (__predict_false(od == NULL || iov==NULL || iovlen==0 ||
 	    (auth_off & 0x7) || (auth_off + auth_len > iovlen))) {
 	dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
 		"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
-		"icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
-		auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+		"icv=%p ivp=%p\n", __func__, od, iov, iovlen,
+		auth_off, auth_len, crypt_off, crypt_len, icv, ivp);
 	return -EINVAL;
     }
 
     IOV_INIT(iov, data, data_i, data_l);
 
     /* Load MD5 IV */
     CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
     CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
 
     while (auth_off > 0) {
 	IOV_CONSUME(iov, data, data_i, data_l);
 	auth_off -= 8;
     }
 
     while (auth_len > 0) {
 	CVM_LOAD_MD5_UNIT(*data, next);
 	auth_len -= 8;
 	IOV_CONSUME(iov, data, data_i, data_l);
     }
 
     /* finish the hash */
     CVMX_PREFETCH0(od->octo_hmouter);
 #if 0
     if (__predict_false(inplen)) {
 	uint64_t tmp = 0;
 	uint8_t *p = (uint8_t *) & tmp;
 	p[inplen] = 0x80;
 	do {
 	    inplen--;
 	    p[inplen] = ((uint8_t *) data)[inplen];
 	} while (inplen);
 	CVM_LOAD_MD5_UNIT(tmp, next);
     } else {
 	CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
     }
 #else
     CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
 #endif
 
     /* Finish Inner hash */
     while (next != 7) {
 	CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next);
     }
     CVMX_ES64(tmp1, ((alen + 64) << 3));
     CVM_LOAD_MD5_UNIT(tmp1, next);
 
     /* Get the inner hash of HMAC */
     CVMX_MF_HSH_IV(tmp1, 0);
     CVMX_MF_HSH_IV(tmp2, 1);
 
     /* Initialize hash unit */
     CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
     CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
 
     CVMX_MT_HSH_DAT(tmp1, 0);
     CVMX_MT_HSH_DAT(tmp2, 1);
     CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2);
     CVMX_MT_HSH_DATZ(3);
     CVMX_MT_HSH_DATZ(4);
     CVMX_MT_HSH_DATZ(5);
     CVMX_MT_HSH_DATZ(6);
     CVMX_ES64(tmp1, ((64 + 16) << 3));
     CVMX_MT_HSH_STARTMD5(tmp1);
 
     /* save the HMAC */
-    IOV_INIT(iov, data, data_i, data_l);
-    while (icv_off > 0) {
-	IOV_CONSUME(iov, data, data_i, data_l);
-	icv_off -= 8;
-    }
+    data = (uint64_t *)icv;
     CVMX_MF_HSH_IV(*data, 0);
-    IOV_CONSUME(iov, data, data_i, data_l);
+    data++;
     CVMX_MF_HSH_IV(tmp1, 1);
     *(uint32_t *)data = (uint32_t) (tmp1 >> 32);
 
     return 0;
 }
 
 /****************************************************************************/
 /* SHA1 */
 
 int
 octo_null_sha1_encrypt(
     struct octo_sess *od,
     struct iovec *iov, size_t iovcnt, size_t iovlen,
     int auth_off, int auth_len,
     int crypt_off, int crypt_len,
-    int icv_off, uint8_t *ivp)
+    uint8_t *icv, uint8_t *ivp)
 {
     int next = 0;
     uint64_t *data;
     uint64_t tmp1, tmp2, tmp3;
     int data_i, data_l, alen = auth_len;
 
     dprintf("%s()\n", __func__);
 
     if (__predict_false(od == NULL || iov==NULL || iovlen==0 ||
 	    (auth_off & 0x7) || (auth_off + auth_len > iovlen))) {
 	dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
 		"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
-		"icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
-		auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+		"icv=%p ivp=%p\n", __func__, od, iov, iovlen,
+		auth_off, auth_len, crypt_off, crypt_len, icv, ivp);
 	return -EINVAL;
     }
 
     IOV_INIT(iov, data, data_i, data_l);
 
     /* Load SHA1 IV */
     CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
     CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
     CVMX_MT_HSH_IV(od->octo_hminner[2], 2);
 
     while (auth_off > 0) {
 	IOV_CONSUME(iov, data, data_i, data_l);
 	auth_off -= 8;
     }
 
     while (auth_len > 0) {
 	CVM_LOAD_SHA_UNIT(*data, next);
 	auth_len -= 8;
 	IOV_CONSUME(iov, data, data_i, data_l);
     }
 
     /* finish the hash */
     CVMX_PREFETCH0(od->octo_hmouter);
 #if 0
     if (__predict_false(inplen)) {
 	uint64_t tmp = 0;
 	uint8_t *p = (uint8_t *) & tmp;
 	p[inplen] = 0x80;
 	do {
 	    inplen--;
 	    p[inplen] = ((uint8_t *) data)[inplen];
 	} while (inplen);
 	CVM_LOAD_MD5_UNIT(tmp, next);
     } else {
 	CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
     }
 #else
     CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
 #endif
 
     /* Finish Inner hash */
     while (next != 7) {
 	CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next);
     }
 	CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next);
 
     /* Get the inner hash of HMAC */
     CVMX_MF_HSH_IV(tmp1, 0);
     CVMX_MF_HSH_IV(tmp2, 1);
     tmp3 = 0;
     CVMX_MF_HSH_IV(tmp3, 2);
 
     /* Initialize hash unit */
     CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
     CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
     CVMX_MT_HSH_IV(od->octo_hmouter[2], 2);
 
     CVMX_MT_HSH_DAT(tmp1, 0);
     CVMX_MT_HSH_DAT(tmp2, 1);
     tmp3 |= 0x0000000080000000;
     CVMX_MT_HSH_DAT(tmp3, 2);
     CVMX_MT_HSH_DATZ(3);
     CVMX_MT_HSH_DATZ(4);
     CVMX_MT_HSH_DATZ(5);
     CVMX_MT_HSH_DATZ(6);
     CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3));
 
     /* save the HMAC */
-    IOV_INIT(iov, data, data_i, data_l);
-    while (icv_off > 0) {
-	IOV_CONSUME(iov, data, data_i, data_l);
-	icv_off -= 8;
-    }
+    data = (uint64_t *)icv;
     CVMX_MF_HSH_IV(*data, 0);
-    IOV_CONSUME(iov, data, data_i, data_l);
+    data++;
     CVMX_MF_HSH_IV(tmp1, 1);
     *(uint32_t *)data = (uint32_t) (tmp1 >> 32);
 
     return 0;
 }
 
 /****************************************************************************/
 /* DES MD5 */
 
 int
 octo_des_cbc_md5_encrypt(
     struct octo_sess *od,
     struct iovec *iov, size_t iovcnt, size_t iovlen,
     int auth_off, int auth_len,
     int crypt_off, int crypt_len,
-    int icv_off, uint8_t *ivp)
+    uint8_t *icv, uint8_t *ivp)
 {
     int next = 0;
     union {
 	uint32_t data32[2];
 	uint64_t data64[1];
     } mydata;
     uint64_t *data = &mydata.data64[0];
     uint32_t *data32;
     uint64_t tmp1, tmp2;
     int data_i, data_l, alen = auth_len;
 
     dprintf("%s()\n", __func__);
 
     if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
 	    (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
 	    (crypt_len  & 0x7) ||
 	    (auth_len  & 0x7) ||
 	    (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
 	dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
 		"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
-		"icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
-		auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+		"icv=%p ivp=%p\n", __func__, od, iov, iovlen,
+		auth_off, auth_len, crypt_off, crypt_len, icv, ivp);
 	return -EINVAL;
     }
 
     IOV_INIT(iov, data32, data_i, data_l);
 
     CVMX_PREFETCH0(ivp);
     CVMX_PREFETCH0(od->octo_enckey);
 
     /* load 3DES Key */
     CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
     if (od->octo_encklen == 24) {
 	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
 	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
     } else if (od->octo_encklen == 8) {
 	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
 	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
     } else {
 	dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
 	return -EINVAL;
     }
 
     CVMX_MT_3DES_IV(* (uint64_t *) ivp);
 
     /* Load MD5 IV */
     CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
     CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
 
     while (crypt_off > 0 && auth_off > 0) {
 	IOV_CONSUME(iov, data32, data_i, data_l);
 	crypt_off -= 4;
 	auth_off -= 4;
     }
 
     while (crypt_len > 0 || auth_len > 0) {
     	uint32_t *first = data32;
 	mydata.data32[0] = *first;
 	IOV_CONSUME(iov, data32, data_i, data_l);
 	mydata.data32[1] = *data32;
     	if (crypt_off <= 0) {
 	    if (crypt_len > 0) {
 		CVMX_MT_3DES_ENC_CBC(*data);
 		CVMX_MF_3DES_RESULT(*data);
 		crypt_len -= 8;
 	    }
 	} else
 	    crypt_off -= 8;
     	if (auth_off <= 0) {
 	    if (auth_len > 0) {
 		CVM_LOAD_MD5_UNIT(*data, next);
 		auth_len -= 8;
 	    }
 	} else
 	    auth_off -= 8;
 	*first = mydata.data32[0];
 	*data32 = mydata.data32[1];
 	IOV_CONSUME(iov, data32, data_i, data_l);
     }
 
     /* finish the hash */
     CVMX_PREFETCH0(od->octo_hmouter);
 #if 0
     if (__predict_false(inplen)) {
 	uint64_t tmp = 0;
 	uint8_t *p = (uint8_t *) & tmp;
 	p[inplen] = 0x80;
 	do {
 	    inplen--;
 	    p[inplen] = ((uint8_t *) data)[inplen];
 	} while (inplen);
 	CVM_LOAD_MD5_UNIT(tmp, next);
     } else {
 	CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
     }
 #else
     CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
 #endif
 
     /* Finish Inner hash */
     while (next != 7) {
 	CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next);
     }
     CVMX_ES64(tmp1, ((alen + 64) << 3));
     CVM_LOAD_MD5_UNIT(tmp1, next);
 
     /* Get the inner hash of HMAC */
     CVMX_MF_HSH_IV(tmp1, 0);
     CVMX_MF_HSH_IV(tmp2, 1);
 
     /* Initialize hash unit */
     CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
     CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
 
     CVMX_MT_HSH_DAT(tmp1, 0);
     CVMX_MT_HSH_DAT(tmp2, 1);
     CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2);
     CVMX_MT_HSH_DATZ(3);
     CVMX_MT_HSH_DATZ(4);
     CVMX_MT_HSH_DATZ(5);
     CVMX_MT_HSH_DATZ(6);
     CVMX_ES64(tmp1, ((64 + 16) << 3));
     CVMX_MT_HSH_STARTMD5(tmp1);
 
     /* save the HMAC */
-    IOV_INIT(iov, data32, data_i, data_l);
-    while (icv_off > 0) {
-	IOV_CONSUME(iov, data32, data_i, data_l);
-	icv_off -= 4;
-    }
+    data32 = (uint32_t *)icv;
     CVMX_MF_HSH_IV(tmp1, 0);
     *data32 = (uint32_t) (tmp1 >> 32);
-    IOV_CONSUME(iov, data32, data_i, data_l);
+    data32++;
     *data32 = (uint32_t) tmp1;
-    IOV_CONSUME(iov, data32, data_i, data_l);
+    data32++;
     CVMX_MF_HSH_IV(tmp1, 1);
     *data32 = (uint32_t) (tmp1 >> 32);
 
     return 0;
 }
 
 int
 octo_des_cbc_md5_decrypt(
     struct octo_sess *od,
     struct iovec *iov, size_t iovcnt, size_t iovlen,
     int auth_off, int auth_len,
     int crypt_off, int crypt_len,
-    int icv_off, uint8_t *ivp)
+    uint8_t *icv, uint8_t *ivp)
 {
     int next = 0;
     union {
 	uint32_t data32[2];
 	uint64_t data64[1];
     } mydata;
     uint64_t *data = &mydata.data64[0];
     uint32_t *data32;
     uint64_t tmp1, tmp2;
     int data_i, data_l, alen = auth_len;
 
     dprintf("%s()\n", __func__);
 
     if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
 	    (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
 	    (crypt_len  & 0x7) ||
 	    (auth_len  & 0x7) ||
 	    (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
 	dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
 		"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
-		"icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
-		auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+		"icv=%p ivp=%p\n", __func__, od, iov, iovlen,
+		auth_off, auth_len, crypt_off, crypt_len, icv, ivp);
 	return -EINVAL;
     }
 
     IOV_INIT(iov, data32, data_i, data_l);
 
     CVMX_PREFETCH0(ivp);
     CVMX_PREFETCH0(od->octo_enckey);
 
     /* load 3DES Key */
     CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
     if (od->octo_encklen == 24) {
 	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
 	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
     } else if (od->octo_encklen == 8) {
 	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
 	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
     } else {
 	dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
 	return -EINVAL;
     }
 
     CVMX_MT_3DES_IV(* (uint64_t *) ivp);
 
     /* Load MD5 IV */
     CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
     CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
 
     while (crypt_off > 0 && auth_off > 0) {
 	IOV_CONSUME(iov, data32, data_i, data_l);
 	crypt_off -= 4;
 	auth_off -= 4;
     }
 
     while (crypt_len > 0 || auth_len > 0) {
     	uint32_t *first = data32;
 	mydata.data32[0] = *first;
 	IOV_CONSUME(iov, data32, data_i, data_l);
 	mydata.data32[1] = *data32;
     	if (auth_off <= 0) {
 	    if (auth_len > 0) {
 		CVM_LOAD_MD5_UNIT(*data, next);
 		auth_len -= 8;
 	    }
 	} else
 	    auth_off -= 8;
     	if (crypt_off <= 0) {
 	    if (crypt_len > 0) {
 		CVMX_MT_3DES_DEC_CBC(*data);
 		CVMX_MF_3DES_RESULT(*data);
 		crypt_len -= 8;
 	    }
 	} else
 	    crypt_off -= 8;
 	*first = mydata.data32[0];
 	*data32 = mydata.data32[1];
 	IOV_CONSUME(iov, data32, data_i, data_l);
     }
 
     /* finish the hash */
     CVMX_PREFETCH0(od->octo_hmouter);
 #if 0
     if (__predict_false(inplen)) {
 	uint64_t tmp = 0;
 	uint8_t *p = (uint8_t *) & tmp;
 	p[inplen] = 0x80;
 	do {
 	    inplen--;
 	    p[inplen] = ((uint8_t *) data)[inplen];
 	} while (inplen);
 	CVM_LOAD_MD5_UNIT(tmp, next);
     } else {
 	CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
     }
 #else
     CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
 #endif
 
     /* Finish Inner hash */
     while (next != 7) {
 	CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next);
     }
     CVMX_ES64(tmp1, ((alen + 64) << 3));
     CVM_LOAD_MD5_UNIT(tmp1, next);
 
     /* Get the inner hash of HMAC */
     CVMX_MF_HSH_IV(tmp1, 0);
     CVMX_MF_HSH_IV(tmp2, 1);
 
     /* Initialize hash unit */
     CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
     CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
 
     CVMX_MT_HSH_DAT(tmp1, 0);
     CVMX_MT_HSH_DAT(tmp2, 1);
     CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2);
     CVMX_MT_HSH_DATZ(3);
     CVMX_MT_HSH_DATZ(4);
     CVMX_MT_HSH_DATZ(5);
     CVMX_MT_HSH_DATZ(6);
     CVMX_ES64(tmp1, ((64 + 16) << 3));
     CVMX_MT_HSH_STARTMD5(tmp1);
 
     /* save the HMAC */
-    IOV_INIT(iov, data32, data_i, data_l);
-    while (icv_off > 0) {
-	IOV_CONSUME(iov, data32, data_i, data_l);
-	icv_off -= 4;
-    }
+    data32 = (uint32_t *)icv;
     CVMX_MF_HSH_IV(tmp1, 0);
     *data32 = (uint32_t) (tmp1 >> 32);
-    IOV_CONSUME(iov, data32, data_i, data_l);
+    data32++;
     *data32 = (uint32_t) tmp1;
-    IOV_CONSUME(iov, data32, data_i, data_l);
+    data32++;
     CVMX_MF_HSH_IV(tmp1, 1);
     *data32 = (uint32_t) (tmp1 >> 32);
 
     return 0;
 }
 
 /****************************************************************************/
 /* DES SHA */
 
 int
 octo_des_cbc_sha1_encrypt(
     struct octo_sess *od,
     struct iovec *iov, size_t iovcnt, size_t iovlen,
     int auth_off, int auth_len,
     int crypt_off, int crypt_len,
-    int icv_off, uint8_t *ivp)
+    uint8_t *icv, uint8_t *ivp)
 {
     int next = 0;
     union {
 	uint32_t data32[2];
 	uint64_t data64[1];
     } mydata;
     uint64_t *data = &mydata.data64[0];
     uint32_t *data32;
     uint64_t tmp1, tmp2, tmp3;
     int data_i, data_l, alen = auth_len;
 
     dprintf("%s()\n", __func__);
 
     if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
 	    (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
 	    (crypt_len  & 0x7) ||
 	    (auth_len  & 0x7) ||
 	    (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
 	dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
 		"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
-		"icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
-		auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+		"icv=%p ivp=%p\n", __func__, od, iov, iovlen,
+		auth_off, auth_len, crypt_off, crypt_len, icv, ivp);
 	return -EINVAL;
     }
 
     IOV_INIT(iov, data32, data_i, data_l);
 
     CVMX_PREFETCH0(ivp);
     CVMX_PREFETCH0(od->octo_enckey);
 
     /* load 3DES Key */
     CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
     if (od->octo_encklen == 24) {
 	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
 	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
     } else if (od->octo_encklen == 8) {
 	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
 	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
     } else {
 	dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
 	return -EINVAL;
     }
 
     CVMX_MT_3DES_IV(* (uint64_t *) ivp);
 
     /* Load SHA1 IV */
     CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
     CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
     CVMX_MT_HSH_IV(od->octo_hminner[2], 2);
 
     while (crypt_off > 0 && auth_off > 0) {
 	IOV_CONSUME(iov, data32, data_i, data_l);
 	crypt_off -= 4;
 	auth_off -= 4;
     }
 
     while (crypt_len > 0 || auth_len > 0) {
     	uint32_t *first = data32;
 	mydata.data32[0] = *first;
 	IOV_CONSUME(iov, data32, data_i, data_l);
 	mydata.data32[1] = *data32;
     	if (crypt_off <= 0) {
 	    if (crypt_len > 0) {
 		CVMX_MT_3DES_ENC_CBC(*data);
 		CVMX_MF_3DES_RESULT(*data);
 		crypt_len -= 8;
 	    }
 	} else
 	    crypt_off -= 8;
     	if (auth_off <= 0) {
 	    if (auth_len > 0) {
 		CVM_LOAD_SHA_UNIT(*data, next);
 		auth_len -= 8;
 	    }
 	} else
 	    auth_off -= 8;
 	*first = mydata.data32[0];
 	*data32 = mydata.data32[1];
 	IOV_CONSUME(iov, data32, data_i, data_l);
     }
 
     /* finish the hash */
     CVMX_PREFETCH0(od->octo_hmouter);
 #if 0
     if (__predict_false(inplen)) {
 	uint64_t tmp = 0;
 	uint8_t *p = (uint8_t *) & tmp;
 	p[inplen] = 0x80;
 	do {
 	    inplen--;
 	    p[inplen] = ((uint8_t *) data)[inplen];
 	} while (inplen);
 	CVM_LOAD_SHA_UNIT(tmp, next);
     } else {
 	CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
     }
 #else
     CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
 #endif
 
     /* Finish Inner hash */
     while (next != 7) {
 	CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next);
     }
 	CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next);
 
     /* Get the inner hash of HMAC */
     CVMX_MF_HSH_IV(tmp1, 0);
     CVMX_MF_HSH_IV(tmp2, 1);
     tmp3 = 0;
     CVMX_MF_HSH_IV(tmp3, 2);
 
     /* Initialize hash unit */
     CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
     CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
     CVMX_MT_HSH_IV(od->octo_hmouter[2], 2);
 
     CVMX_MT_HSH_DAT(tmp1, 0);
     CVMX_MT_HSH_DAT(tmp2, 1);
     tmp3 |= 0x0000000080000000;
     CVMX_MT_HSH_DAT(tmp3, 2);
     CVMX_MT_HSH_DATZ(3);
     CVMX_MT_HSH_DATZ(4);
     CVMX_MT_HSH_DATZ(5);
     CVMX_MT_HSH_DATZ(6);
     CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3));
 
     /* save the HMAC */
-    IOV_INIT(iov, data32, data_i, data_l);
-    while (icv_off > 0) {
-	IOV_CONSUME(iov, data32, data_i, data_l);
-	icv_off -= 4;
-    }
+    data32 = (uint32_t *)icv;
     CVMX_MF_HSH_IV(tmp1, 0);
     *data32 = (uint32_t) (tmp1 >> 32);
-    IOV_CONSUME(iov, data32, data_i, data_l);
+    data32++;
     *data32 = (uint32_t) tmp1;
-    IOV_CONSUME(iov, data32, data_i, data_l);
+    data32++;
     CVMX_MF_HSH_IV(tmp1, 1);
     *data32 = (uint32_t) (tmp1 >> 32);
 
     return 0;
 }
 
 int
 octo_des_cbc_sha1_decrypt(
     struct octo_sess *od,
     struct iovec *iov, size_t iovcnt, size_t iovlen,
     int auth_off, int auth_len,
     int crypt_off, int crypt_len,
-    int icv_off, uint8_t *ivp)
+    uint8_t *icv, uint8_t *ivp)
 {
     int next = 0;
     union {
 	uint32_t data32[2];
 	uint64_t data64[1];
     } mydata;
     uint64_t *data = &mydata.data64[0];
     uint32_t *data32;
     uint64_t tmp1, tmp2, tmp3;
     int data_i, data_l, alen = auth_len;
 
     dprintf("%s()\n", __func__);
 
     if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
 	    (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
 	    (crypt_len  & 0x7) ||
 	    (auth_len  & 0x7) ||
 	    (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
 	dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
 		"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
-		"icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
-		auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+		"icv=%p ivp=%p\n", __func__, od, iov, iovlen,
+		auth_off, auth_len, crypt_off, crypt_len, icv, ivp);
 	return -EINVAL;
     }
 
     IOV_INIT(iov, data32, data_i, data_l);
 
     CVMX_PREFETCH0(ivp);
     CVMX_PREFETCH0(od->octo_enckey);
 
     /* load 3DES Key */
     CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
     if (od->octo_encklen == 24) {
 	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
 	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
     } else if (od->octo_encklen == 8) {
 	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 1);
 	CVMX_MT_3DES_KEY(((uint64_t *) od->octo_enckey)[0], 2);
     } else {
 	dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
 	return -EINVAL;
     }
 
     CVMX_MT_3DES_IV(* (uint64_t *) ivp);
 
     /* Load SHA1 IV */
     CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
     CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
     CVMX_MT_HSH_IV(od->octo_hminner[2], 2);
 
     while (crypt_off > 0 && auth_off > 0) {
 	IOV_CONSUME(iov, data32, data_i, data_l);
 	crypt_off -= 4;
 	auth_off -= 4;
     }
 
     while (crypt_len > 0 || auth_len > 0) {
     	uint32_t *first = data32;
 	mydata.data32[0] = *first;
 	IOV_CONSUME(iov, data32, data_i, data_l);
 	mydata.data32[1] = *data32;
     	if (auth_off <= 0) {
 	    if (auth_len > 0) {
 		CVM_LOAD_SHA_UNIT(*data, next);
 		auth_len -= 8;
 	    }
 	} else
 	    auth_off -= 8;
     	if (crypt_off <= 0) {
 	    if (crypt_len > 0) {
 		CVMX_MT_3DES_DEC_CBC(*data);
 		CVMX_MF_3DES_RESULT(*data);
 		crypt_len -= 8;
 	    }
 	} else
 	    crypt_off -= 8;
 	*first = mydata.data32[0];
 	*data32 = mydata.data32[1];
 	IOV_CONSUME(iov, data32, data_i, data_l);
     }
 
     /* finish the hash */
     CVMX_PREFETCH0(od->octo_hmouter);
 #if 0
     if (__predict_false(inplen)) {
 	uint64_t tmp = 0;
 	uint8_t *p = (uint8_t *) & tmp;
 	p[inplen] = 0x80;
 	do {
 	    inplen--;
 	    p[inplen] = ((uint8_t *) data)[inplen];
 	} while (inplen);
 	CVM_LOAD_SHA_UNIT(tmp, next);
     } else {
 	CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
     }
 #else
     CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
 #endif
 
     /* Finish Inner hash */
     while (next != 7) {
 	CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next);
     }
 	CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next);
 
     /* Get the inner hash of HMAC */
     CVMX_MF_HSH_IV(tmp1, 0);
     CVMX_MF_HSH_IV(tmp2, 1);
     tmp3 = 0;
     CVMX_MF_HSH_IV(tmp3, 2);
 
     /* Initialize hash unit */
     CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
     CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
     CVMX_MT_HSH_IV(od->octo_hmouter[2], 2);
 
     CVMX_MT_HSH_DAT(tmp1, 0);
     CVMX_MT_HSH_DAT(tmp2, 1);
     tmp3 |= 0x0000000080000000;
     CVMX_MT_HSH_DAT(tmp3, 2);
     CVMX_MT_HSH_DATZ(3);
     CVMX_MT_HSH_DATZ(4);
     CVMX_MT_HSH_DATZ(5);
     CVMX_MT_HSH_DATZ(6);
     CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3));
     /* save the HMAC */
-    IOV_INIT(iov, data32, data_i, data_l);
-    while (icv_off > 0) {
-	IOV_CONSUME(iov, data32, data_i, data_l);
-	icv_off -= 4;
-    }
+    data32 = (uint32_t *)icv;
     CVMX_MF_HSH_IV(tmp1, 0);
     *data32 = (uint32_t) (tmp1 >> 32);
-    IOV_CONSUME(iov, data32, data_i, data_l);
+    data32++;
     *data32 = (uint32_t) tmp1;
-    IOV_CONSUME(iov, data32, data_i, data_l);
+    data32++;
     CVMX_MF_HSH_IV(tmp1, 1);
     *data32 = (uint32_t) (tmp1 >> 32);
 
     return 0;
 }
 
 /****************************************************************************/
 /* AES MD5 */
 
 int
 octo_aes_cbc_md5_encrypt(
     struct octo_sess *od,
     struct iovec *iov, size_t iovcnt, size_t iovlen,
     int auth_off, int auth_len,
     int crypt_off, int crypt_len,
-    int icv_off, uint8_t *ivp)
+    uint8_t *icv, uint8_t *ivp)
 {
     int next = 0;
     union {
 	uint32_t data32[2];
 	uint64_t data64[1];
     } mydata[2];
     uint64_t *pdata = &mydata[0].data64[0];
     uint64_t *data =  &mydata[1].data64[0];
     uint32_t *data32;
     uint64_t tmp1, tmp2;
     int data_i, data_l, alen = auth_len;
 
     dprintf("%s()\n", __func__);
 
     if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
 	    (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
 	    (crypt_len  & 0x7) ||
 	    (auth_len  & 0x7) ||
 	    (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
 	dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
 		"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
-		"icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
-		auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+		"icv=%p ivp=%p\n", __func__, od, iov, iovlen,
+		auth_off, auth_len, crypt_off, crypt_len, icv, ivp);
 	return -EINVAL;
     }
 
     IOV_INIT(iov, data32, data_i, data_l);
 
     CVMX_PREFETCH0(ivp);
     CVMX_PREFETCH0(od->octo_enckey);
 
     /* load AES Key */
     CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
     CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
 
     if (od->octo_encklen == 16) {
 	CVMX_MT_AES_KEY(0x0, 2);
 	CVMX_MT_AES_KEY(0x0, 3);
     } else if (od->octo_encklen == 24) {
 	CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
 	CVMX_MT_AES_KEY(0x0, 3);
     } else if (od->octo_encklen == 32) {
 	CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
 	CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
     } else {
 	dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
 	return -EINVAL;
     }
     CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
 
     CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
     CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
 
     /* Load MD5 IV */
     CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
     CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
 
     while (crypt_off > 0 && auth_off > 0) {
 	IOV_CONSUME(iov, data32, data_i, data_l);
 	crypt_off -= 4;
 	auth_off -= 4;
     }
 
     while (crypt_len > 0 || auth_len > 0) {
     	uint32_t *pdata32[3];
 
 	pdata32[0] = data32;
 	mydata[0].data32[0] = *data32;
 	IOV_CONSUME(iov, data32, data_i, data_l);
 
 	pdata32[1] = data32;
 	mydata[0].data32[1] = *data32;
 	IOV_CONSUME(iov, data32, data_i, data_l);
 
 	pdata32[2] = data32;
 	mydata[1].data32[0] = *data32;
 	IOV_CONSUME(iov, data32, data_i, data_l);
 
 	mydata[1].data32[1] = *data32;
 
 
     	if (crypt_off <= 0) {
 	    if (crypt_len > 0) {
 		CVMX_MT_AES_ENC_CBC0(*pdata);
 		CVMX_MT_AES_ENC_CBC1(*data);
 		CVMX_MF_AES_RESULT(*pdata, 0);
 		CVMX_MF_AES_RESULT(*data, 1);
 		crypt_len -= 16;
 	    }
 	} else
 	    crypt_off -= 16;
 
     	if (auth_off <= 0) {
 	    if (auth_len > 0) {
 		CVM_LOAD_MD5_UNIT(*pdata, next);
 		CVM_LOAD_MD5_UNIT(*data, next);
 		auth_len -= 16;
 	    }
 	} else
 	    auth_off -= 16;
 
 	*pdata32[0] = mydata[0].data32[0];
 	*pdata32[1] = mydata[0].data32[1];
 	*pdata32[2] = mydata[1].data32[0];
 	*data32     = mydata[1].data32[1];
 
 	IOV_CONSUME(iov, data32, data_i, data_l);
     }
 
     /* finish the hash */
     CVMX_PREFETCH0(od->octo_hmouter);
 #if 0
     if (__predict_false(inplen)) {
 	uint64_t tmp = 0;
 	uint8_t *p = (uint8_t *) & tmp;
 	p[inplen] = 0x80;
 	do {
 	    inplen--;
 	    p[inplen] = ((uint8_t *) data)[inplen];
 	} while (inplen);
 	CVM_LOAD_MD5_UNIT(tmp, next);
     } else {
 	CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
     }
 #else
     CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
 #endif
 
     /* Finish Inner hash */
     while (next != 7) {
 	CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next);
     }
     CVMX_ES64(tmp1, ((alen + 64) << 3));
     CVM_LOAD_MD5_UNIT(tmp1, next);
 
     /* Get the inner hash of HMAC */
     CVMX_MF_HSH_IV(tmp1, 0);
     CVMX_MF_HSH_IV(tmp2, 1);
 
     /* Initialize hash unit */
     CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
     CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
 
     CVMX_MT_HSH_DAT(tmp1, 0);
     CVMX_MT_HSH_DAT(tmp2, 1);
     CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2);
     CVMX_MT_HSH_DATZ(3);
     CVMX_MT_HSH_DATZ(4);
     CVMX_MT_HSH_DATZ(5);
     CVMX_MT_HSH_DATZ(6);
     CVMX_ES64(tmp1, ((64 + 16) << 3));
     CVMX_MT_HSH_STARTMD5(tmp1);
 
     /* save the HMAC */
-    IOV_INIT(iov, data32, data_i, data_l);
-    while (icv_off > 0) {
-	IOV_CONSUME(iov, data32, data_i, data_l);
-	icv_off -= 4;
-    }
+    data32 = (uint32_t *)icv;
     CVMX_MF_HSH_IV(tmp1, 0);
     *data32 = (uint32_t) (tmp1 >> 32);
-    IOV_CONSUME(iov, data32, data_i, data_l);
+    data32++;
     *data32 = (uint32_t) tmp1;
-    IOV_CONSUME(iov, data32, data_i, data_l);
+    data32++;
     CVMX_MF_HSH_IV(tmp1, 1);
     *data32 = (uint32_t) (tmp1 >> 32);
 
     return 0;
 }
 
 int
 octo_aes_cbc_md5_decrypt(
     struct octo_sess *od,
     struct iovec *iov, size_t iovcnt, size_t iovlen,
     int auth_off, int auth_len,
     int crypt_off, int crypt_len,
-    int icv_off, uint8_t *ivp)
+    uint8_t *icv, uint8_t *ivp)
 {
     int next = 0;
     union {
 	uint32_t data32[2];
 	uint64_t data64[1];
     } mydata[2];
     uint64_t *pdata = &mydata[0].data64[0];
     uint64_t *data =  &mydata[1].data64[0];
     uint32_t *data32;
     uint64_t tmp1, tmp2;
     int data_i, data_l, alen = auth_len;
 
     dprintf("%s()\n", __func__);
 
     if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
 	    (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
 	    (crypt_len  & 0x7) ||
 	    (auth_len  & 0x7) ||
 	    (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
 	dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
 		"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
-		"icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
-		auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+		"icv=%p ivp=%p\n", __func__, od, iov, iovlen,
+		auth_off, auth_len, crypt_off, crypt_len, icv, ivp);
 	return -EINVAL;
     }
 
     IOV_INIT(iov, data32, data_i, data_l);
 
     CVMX_PREFETCH0(ivp);
     CVMX_PREFETCH0(od->octo_enckey);
 
     /* load AES Key */
     CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
     CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
 
     if (od->octo_encklen == 16) {
 	CVMX_MT_AES_KEY(0x0, 2);
 	CVMX_MT_AES_KEY(0x0, 3);
     } else if (od->octo_encklen == 24) {
 	CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
 	CVMX_MT_AES_KEY(0x0, 3);
     } else if (od->octo_encklen == 32) {
 	CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
 	CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
     } else {
 	dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
 	return -EINVAL;
     }
     CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
 
     CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
     CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
 
     /* Load MD5 IV */
     CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
     CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
 
     while (crypt_off > 0 && auth_off > 0) {
 	IOV_CONSUME(iov, data32, data_i, data_l);
 	crypt_off -= 4;
 	auth_off -= 4;
     }
 
     while (crypt_len > 0 || auth_len > 0) {
     	uint32_t *pdata32[3];
 
 	pdata32[0] = data32;
 	mydata[0].data32[0] = *data32;
 	IOV_CONSUME(iov, data32, data_i, data_l);
 	pdata32[1] = data32;
 	mydata[0].data32[1] = *data32;
 	IOV_CONSUME(iov, data32, data_i, data_l);
 	pdata32[2] = data32;
 	mydata[1].data32[0] = *data32;
 	IOV_CONSUME(iov, data32, data_i, data_l);
 	mydata[1].data32[1] = *data32;
 
     	if (auth_off <= 0) {
 	    if (auth_len > 0) {
 		CVM_LOAD_MD5_UNIT(*pdata, next);
 		CVM_LOAD_MD5_UNIT(*data, next);
 		auth_len -= 16;
 	    }
 	} else
 	    auth_off -= 16;
 
     	if (crypt_off <= 0) {
 	    if (crypt_len > 0) {
 		CVMX_MT_AES_DEC_CBC0(*pdata);
 		CVMX_MT_AES_DEC_CBC1(*data);
 		CVMX_MF_AES_RESULT(*pdata, 0);
 		CVMX_MF_AES_RESULT(*data, 1);
 		crypt_len -= 16;
 	    }
 	} else
 	    crypt_off -= 16;
 
 	*pdata32[0] = mydata[0].data32[0];
 	*pdata32[1] = mydata[0].data32[1];
 	*pdata32[2] = mydata[1].data32[0];
 	*data32     = mydata[1].data32[1];
 
 	IOV_CONSUME(iov, data32, data_i, data_l);
     }
 
     /* finish the hash */
     CVMX_PREFETCH0(od->octo_hmouter);
 #if 0
     if (__predict_false(inplen)) {
 	uint64_t tmp = 0;
 	uint8_t *p = (uint8_t *) & tmp;
 	p[inplen] = 0x80;
 	do {
 	    inplen--;
 	    p[inplen] = ((uint8_t *) data)[inplen];
 	} while (inplen);
 	CVM_LOAD_MD5_UNIT(tmp, next);
     } else {
 	CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
     }
 #else
     CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
 #endif
 
     /* Finish Inner hash */
     while (next != 7) {
 	CVM_LOAD_MD5_UNIT(((uint64_t) 0x0ULL), next);
     }
     CVMX_ES64(tmp1, ((alen + 64) << 3));
     CVM_LOAD_MD5_UNIT(tmp1, next);
 
     /* Get the inner hash of HMAC */
     CVMX_MF_HSH_IV(tmp1, 0);
     CVMX_MF_HSH_IV(tmp2, 1);
 
     /* Initialize hash unit */
     CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
     CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
 
     CVMX_MT_HSH_DAT(tmp1, 0);
     CVMX_MT_HSH_DAT(tmp2, 1);
     CVMX_MT_HSH_DAT(0x8000000000000000ULL, 2);
     CVMX_MT_HSH_DATZ(3);
     CVMX_MT_HSH_DATZ(4);
     CVMX_MT_HSH_DATZ(5);
     CVMX_MT_HSH_DATZ(6);
     CVMX_ES64(tmp1, ((64 + 16) << 3));
     CVMX_MT_HSH_STARTMD5(tmp1);
 
     /* save the HMAC */
-    IOV_INIT(iov, data32, data_i, data_l);
-    while (icv_off > 0) {
-	IOV_CONSUME(iov, data32, data_i, data_l);
-	icv_off -= 4;
-    }
+    data32 = (uint32_t *)icv;
     CVMX_MF_HSH_IV(tmp1, 0);
     *data32 = (uint32_t) (tmp1 >> 32);
-    IOV_CONSUME(iov, data32, data_i, data_l);
+    data32++;
     *data32 = (uint32_t) tmp1;
-    IOV_CONSUME(iov, data32, data_i, data_l);
+    data32++;
     CVMX_MF_HSH_IV(tmp1, 1);
     *data32 = (uint32_t) (tmp1 >> 32);
 
     return 0;
 }
 
 /****************************************************************************/
 /* AES SHA1 */
 
 int
 octo_aes_cbc_sha1_encrypt(
     struct octo_sess *od,
     struct iovec *iov, size_t iovcnt, size_t iovlen,
     int auth_off, int auth_len,
     int crypt_off, int crypt_len,
-    int icv_off, uint8_t *ivp)
+    uint8_t *icv, uint8_t *ivp)
 {
     int next = 0;
     union {
 	uint32_t data32[2];
 	uint64_t data64[1];
     } mydata[2];
     uint64_t *pdata = &mydata[0].data64[0];
     uint64_t *data =  &mydata[1].data64[0];
     uint32_t *data32;
     uint64_t tmp1, tmp2, tmp3;
     int data_i, data_l, alen = auth_len;
 
     dprintf("%s()\n", __func__);
 
     if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
 	    (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
 	    (crypt_len  & 0x7) ||
 	    (auth_len  & 0x7) ||
 	    (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
 	dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
 		"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
-		"icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
-		auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+		"icv=%p ivp=%p\n", __func__, od, iov, iovlen,
+		auth_off, auth_len, crypt_off, crypt_len, icv, ivp);
 	return -EINVAL;
     }
 
     IOV_INIT(iov, data32, data_i, data_l);
 
     CVMX_PREFETCH0(ivp);
     CVMX_PREFETCH0(od->octo_enckey);
 
     /* load AES Key */
     CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
     CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
 
     if (od->octo_encklen == 16) {
 	CVMX_MT_AES_KEY(0x0, 2);
 	CVMX_MT_AES_KEY(0x0, 3);
     } else if (od->octo_encklen == 24) {
 	CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
 	CVMX_MT_AES_KEY(0x0, 3);
     } else if (od->octo_encklen == 32) {
 	CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
 	CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
     } else {
 	dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
 	return -EINVAL;
     }
     CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
 
     CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
     CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
 
     /* Load SHA IV */
     CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
     CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
     CVMX_MT_HSH_IV(od->octo_hminner[2], 2);
 
     while (crypt_off > 0 && auth_off > 0) {
 	IOV_CONSUME(iov, data32, data_i, data_l);
 	crypt_off -= 4;
 	auth_off -= 4;
     }
 
     while (crypt_len > 0 || auth_len > 0) {
     	uint32_t *pdata32[3];
 
 	pdata32[0] = data32;
 	mydata[0].data32[0] = *data32;
 	IOV_CONSUME(iov, data32, data_i, data_l);
 	pdata32[1] = data32;
 	mydata[0].data32[1] = *data32;
 	IOV_CONSUME(iov, data32, data_i, data_l);
 	pdata32[2] = data32;
 	mydata[1].data32[0] = *data32;
 	IOV_CONSUME(iov, data32, data_i, data_l);
 	mydata[1].data32[1] = *data32;
 
 
     	if (crypt_off <= 0) {
 	    if (crypt_len > 0) {
 		CVMX_MT_AES_ENC_CBC0(*pdata);
 		CVMX_MT_AES_ENC_CBC1(*data);
 		CVMX_MF_AES_RESULT(*pdata, 0);
 		CVMX_MF_AES_RESULT(*data, 1);
 		crypt_len -= 16;
 	    }
 	} else
 	    crypt_off -= 16;
 
     	if (auth_off <= 0) {
 	    if (auth_len > 0) {
 		CVM_LOAD_SHA_UNIT(*pdata, next);
 		CVM_LOAD_SHA_UNIT(*data, next);
 		auth_len -= 16;
 	    }
 	} else
 	    auth_off -= 16;
 
 	*pdata32[0] = mydata[0].data32[0];
 	*pdata32[1] = mydata[0].data32[1];
 	*pdata32[2] = mydata[1].data32[0];
 	*data32     = mydata[1].data32[1];
 
 	IOV_CONSUME(iov, data32, data_i, data_l);
     }
 
     /* finish the hash */
     CVMX_PREFETCH0(od->octo_hmouter);
 #if 0
     if (__predict_false(inplen)) {
 	uint64_t tmp = 0;
 	uint8_t *p = (uint8_t *) & tmp;
 	p[inplen] = 0x80;
 	do {
 	    inplen--;
 	    p[inplen] = ((uint8_t *) data)[inplen];
 	} while (inplen);
 	CVM_LOAD_SHA_UNIT(tmp, next);
     } else {
 	CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
     }
 #else
     CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
 #endif
 
     /* Finish Inner hash */
     while (next != 7) {
 	CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next);
     }
 	CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next);
 
     /* Get the inner hash of HMAC */
     CVMX_MF_HSH_IV(tmp1, 0);
     CVMX_MF_HSH_IV(tmp2, 1);
     tmp3 = 0;
     CVMX_MF_HSH_IV(tmp3, 2);
 
     /* Initialize hash unit */
     CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
     CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
     CVMX_MT_HSH_IV(od->octo_hmouter[2], 2);
 
     CVMX_MT_HSH_DAT(tmp1, 0);
     CVMX_MT_HSH_DAT(tmp2, 1);
     tmp3 |= 0x0000000080000000;
     CVMX_MT_HSH_DAT(tmp3, 2);
     CVMX_MT_HSH_DATZ(3);
     CVMX_MT_HSH_DATZ(4);
     CVMX_MT_HSH_DATZ(5);
     CVMX_MT_HSH_DATZ(6);
     CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3));
 
     /* finish the hash */
     CVMX_PREFETCH0(od->octo_hmouter);
 #if 0
     if (__predict_false(inplen)) {
 	uint64_t tmp = 0;
 	uint8_t *p = (uint8_t *) & tmp;
 	p[inplen] = 0x80;
 	do {
 	    inplen--;
 	    p[inplen] = ((uint8_t *) data)[inplen];
 	} while (inplen);
 	CVM_LOAD_MD5_UNIT(tmp, next);
     } else {
 	CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
     }
 #else
     CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
 #endif
 
     /* save the HMAC */
-    IOV_INIT(iov, data32, data_i, data_l);
-    while (icv_off > 0) {
-	IOV_CONSUME(iov, data32, data_i, data_l);
-	icv_off -= 4;
-    }
+    data32 = (uint32_t *)icv;
     CVMX_MF_HSH_IV(tmp1, 0);
     *data32 = (uint32_t) (tmp1 >> 32);
-    IOV_CONSUME(iov, data32, data_i, data_l);
+    data32++;
     *data32 = (uint32_t) tmp1;
-    IOV_CONSUME(iov, data32, data_i, data_l);
+    data32++;
     CVMX_MF_HSH_IV(tmp1, 1);
     *data32 = (uint32_t) (tmp1 >> 32);
 
     return 0;
 }
 
 int
 octo_aes_cbc_sha1_decrypt(
     struct octo_sess *od,
     struct iovec *iov, size_t iovcnt, size_t iovlen,
     int auth_off, int auth_len,
     int crypt_off, int crypt_len,
-    int icv_off, uint8_t *ivp)
+    uint8_t *icv, uint8_t *ivp)
 {
     int next = 0;
     union {
 	uint32_t data32[2];
 	uint64_t data64[1];
     } mydata[2];
     uint64_t *pdata = &mydata[0].data64[0];
     uint64_t *data =  &mydata[1].data64[0];
     uint32_t *data32;
     uint64_t tmp1, tmp2, tmp3;
     int data_i, data_l, alen = auth_len;
 
     dprintf("%s()\n", __func__);
 
     if (__predict_false(od == NULL || iov==NULL || iovlen==0 || ivp==NULL ||
 	    (crypt_off & 0x3) || (crypt_off + crypt_len > iovlen) ||
 	    (crypt_len  & 0x7) ||
 	    (auth_len  & 0x7) ||
 	    (auth_off & 0x3) || (auth_off + auth_len > iovlen))) {
 	dprintf("%s: Bad parameters od=%p iov=%p iovlen=%jd "
 		"auth_off=%d auth_len=%d crypt_off=%d crypt_len=%d "
-		"icv_off=%d ivp=%p\n", __func__, od, iov, iovlen,
-		auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+		"icv=%p ivp=%p\n", __func__, od, iov, iovlen,
+		auth_off, auth_len, crypt_off, crypt_len, icv, ivp);
 	return -EINVAL;
     }
 
     IOV_INIT(iov, data32, data_i, data_l);
 
     CVMX_PREFETCH0(ivp);
     CVMX_PREFETCH0(od->octo_enckey);
 
     /* load AES Key */
     CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[0], 0);
     CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[1], 1);
 
     if (od->octo_encklen == 16) {
 	CVMX_MT_AES_KEY(0x0, 2);
 	CVMX_MT_AES_KEY(0x0, 3);
     } else if (od->octo_encklen == 24) {
 	CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
 	CVMX_MT_AES_KEY(0x0, 3);
     } else if (od->octo_encklen == 32) {
 	CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[2], 2);
 	CVMX_MT_AES_KEY(((uint64_t *) od->octo_enckey)[3], 3);
     } else {
 	dprintf("%s: Bad key length %d\n", __func__, od->octo_encklen);
 	return -EINVAL;
     }
     CVMX_MT_AES_KEYLENGTH(od->octo_encklen / 8 - 1);
 
     CVMX_MT_AES_IV(((uint64_t *) ivp)[0], 0);
     CVMX_MT_AES_IV(((uint64_t *) ivp)[1], 1);
 
     /* Load MD5 IV */
     CVMX_MT_HSH_IV(od->octo_hminner[0], 0);
     CVMX_MT_HSH_IV(od->octo_hminner[1], 1);
     CVMX_MT_HSH_IV(od->octo_hminner[2], 2);
 
     while (crypt_off > 0 && auth_off > 0) {
 	IOV_CONSUME(iov, data32, data_i, data_l);
 	crypt_off -= 4;
 	auth_off -= 4;
     }
 
     while (crypt_len > 0 || auth_len > 0) {
     	uint32_t *pdata32[3];
 
 	pdata32[0] = data32;
 	mydata[0].data32[0] = *data32;
 	IOV_CONSUME(iov, data32, data_i, data_l);
 	pdata32[1] = data32;
 	mydata[0].data32[1] = *data32;
 	IOV_CONSUME(iov, data32, data_i, data_l);
 	pdata32[2] = data32;
 	mydata[1].data32[0] = *data32;
 	IOV_CONSUME(iov, data32, data_i, data_l);
 	mydata[1].data32[1] = *data32;
 
     	if (auth_off <= 0) {
 	    if (auth_len > 0) {
 		CVM_LOAD_SHA_UNIT(*pdata, next);
 		CVM_LOAD_SHA_UNIT(*data, next);
 		auth_len -= 16;
 	    }
 	} else
 	    auth_off -= 16;
 
     	if (crypt_off <= 0) {
 	    if (crypt_len > 0) {
 		CVMX_MT_AES_DEC_CBC0(*pdata);
 		CVMX_MT_AES_DEC_CBC1(*data);
 		CVMX_MF_AES_RESULT(*pdata, 0);
 		CVMX_MF_AES_RESULT(*data, 1);
 		crypt_len -= 16;
 	    }
 	} else
 	    crypt_off -= 16;
 
 	*pdata32[0] = mydata[0].data32[0];
 	*pdata32[1] = mydata[0].data32[1];
 	*pdata32[2] = mydata[1].data32[0];
 	*data32     = mydata[1].data32[1];
 
 	IOV_CONSUME(iov, data32, data_i, data_l);
     }
 
     /* finish the hash */
     CVMX_PREFETCH0(od->octo_hmouter);
 #if 0
     if (__predict_false(inplen)) {
 	uint64_t tmp = 0;
 	uint8_t *p = (uint8_t *) & tmp;
 	p[inplen] = 0x80;
 	do {
 	    inplen--;
 	    p[inplen] = ((uint8_t *) data)[inplen];
 	} while (inplen);
 	CVM_LOAD_SHA_UNIT(tmp, next);
     } else {
 	CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
     }
 #else
     CVM_LOAD_SHA_UNIT(0x8000000000000000ULL, next);
 #endif
 
     /* Finish Inner hash */
     while (next != 7) {
 	CVM_LOAD_SHA_UNIT(((uint64_t) 0x0ULL), next);
     }
 	CVM_LOAD_SHA_UNIT((uint64_t) ((alen + 64) << 3), next);
 
     /* Get the inner hash of HMAC */
     CVMX_MF_HSH_IV(tmp1, 0);
     CVMX_MF_HSH_IV(tmp2, 1);
     tmp3 = 0;
     CVMX_MF_HSH_IV(tmp3, 2);
 
     /* Initialize hash unit */
     CVMX_MT_HSH_IV(od->octo_hmouter[0], 0);
     CVMX_MT_HSH_IV(od->octo_hmouter[1], 1);
     CVMX_MT_HSH_IV(od->octo_hmouter[2], 2);
 
     CVMX_MT_HSH_DAT(tmp1, 0);
     CVMX_MT_HSH_DAT(tmp2, 1);
     tmp3 |= 0x0000000080000000;
     CVMX_MT_HSH_DAT(tmp3, 2);
     CVMX_MT_HSH_DATZ(3);
     CVMX_MT_HSH_DATZ(4);
     CVMX_MT_HSH_DATZ(5);
     CVMX_MT_HSH_DATZ(6);
     CVMX_MT_HSH_STARTSHA((uint64_t) ((64 + 20) << 3));
 
     /* finish the hash */
     CVMX_PREFETCH0(od->octo_hmouter);
 #if 0
     if (__predict_false(inplen)) {
 	uint64_t tmp = 0;
 	uint8_t *p = (uint8_t *) & tmp;
 	p[inplen] = 0x80;
 	do {
 	    inplen--;
 	    p[inplen] = ((uint8_t *) data)[inplen];
 	} while (inplen);
 	CVM_LOAD_MD5_UNIT(tmp, next);
     } else {
 	CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
     }
 #else
     CVM_LOAD_MD5_UNIT(0x8000000000000000ULL, next);
 #endif
 
     /* save the HMAC */
-    IOV_INIT(iov, data32, data_i, data_l);
-    while (icv_off > 0) {
-	IOV_CONSUME(iov, data32, data_i, data_l);
-	icv_off -= 4;
-    }
+    data32 = (uint32_t *)icv;
     CVMX_MF_HSH_IV(tmp1, 0);
     *data32 = (uint32_t) (tmp1 >> 32);
-    IOV_CONSUME(iov, data32, data_i, data_l);
+    data32++;
     *data32 = (uint32_t) tmp1;
-    IOV_CONSUME(iov, data32, data_i, data_l);
+    data32++;
     CVMX_MF_HSH_IV(tmp1, 1);
     *data32 = (uint32_t) (tmp1 >> 32);
 
     return 0;
 }
 
 /****************************************************************************/
Index: head/sys/mips/cavium/cryptocteon/cryptocteon.c
===================================================================
--- head/sys/mips/cavium/cryptocteon/cryptocteon.c	(revision 359373)
+++ head/sys/mips/cavium/cryptocteon/cryptocteon.c	(revision 359374)
@@ -1,444 +1,469 @@
 /*
  * Octeon Crypto for OCF
  *
  * Written by David McCullough <david_mccullough@securecomputing.com>
  * Copyright (C) 2009 David McCullough
  *
  * LICENSE TERMS
  *
  * The free distribution and use of this software in both source and binary
  * form is allowed (with or without changes) provided that:
  *
  *   1. distributions of this source code include the above copyright
  *      notice, this list of conditions and the following disclaimer;
  *
  *   2. distributions in binary form include the above copyright
  *      notice, this list of conditions and the following disclaimer
  *      in the documentation and/or other associated materials;
  *
  *   3. the copyright holder's name is not used to endorse products
  *      built using this software without specific written permission.
  *
  * DISCLAIMER
  *
  * This software is provided 'as is' with no explicit or implied warranties
  * in respect of its properties, including, but not limited to, correctness
  * and/or fitness for purpose.
  * ---------------------------------------------------------------------------
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/bus.h>
 #include <sys/kernel.h>
 #include <sys/module.h>
 #include <sys/malloc.h>
 #include <sys/mbuf.h>
 #include <sys/uio.h>
 
 #include <opencrypto/cryptodev.h>
 
 #include <contrib/octeon-sdk/cvmx.h>
 
 #include <mips/cavium/cryptocteon/cryptocteonvar.h>
 
 #include "cryptodev_if.h"
 
 struct cryptocteon_softc {
 	int32_t			sc_cid;		/* opencrypto id */
 };
 
 int cryptocteon_debug = 0;
 TUNABLE_INT("hw.cryptocteon.debug", &cryptocteon_debug);
 
 static void cryptocteon_identify(driver_t *, device_t);
 static int cryptocteon_probe(device_t);
 static int cryptocteon_attach(device_t);
 
 static int cryptocteon_process(device_t, struct cryptop *, int);
-static int cryptocteon_newsession(device_t, crypto_session_t, struct cryptoini *);
+static int cryptocteon_probesession(device_t,
+    const struct crypto_session_params *);
+static int cryptocteon_newsession(device_t, crypto_session_t,
+    const struct crypto_session_params *);
 
 static void
 cryptocteon_identify(driver_t *drv, device_t parent)
 {
 	if (octeon_has_feature(OCTEON_FEATURE_CRYPTO))
 		BUS_ADD_CHILD(parent, 0, "cryptocteon", 0);
 }
 
 static int
 cryptocteon_probe(device_t dev)
 {
 	device_set_desc(dev, "Octeon Secure Coprocessor");
 	return (0);
 }
 
 static int
 cryptocteon_attach(device_t dev)
 {
 	struct cryptocteon_softc *sc;
 
 	sc = device_get_softc(dev);
 
 	sc->sc_cid = crypto_get_driverid(dev, sizeof(struct octo_sess),
 	    CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SYNC);
 	if (sc->sc_cid < 0) {
 		device_printf(dev, "crypto_get_driverid ret %d\n", sc->sc_cid);
 		return (ENXIO);
 	}
 
-	crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
-	crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
-	crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
-	crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
-	crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
-
 	return (0);
 }
 
-/*
- * Generate a new octo session.  We artifically limit it to a single
- * hash/cipher or hash-cipher combo just to make it easier, most callers
- * do not expect more than this anyway.
- */
-static int
-cryptocteon_newsession(device_t dev, crypto_session_t cses,
-    struct cryptoini *cri)
+static bool
+cryptocteon_auth_supported(const struct crypto_session_params *csp)
 {
-	struct cryptoini *c, *encini = NULL, *macini = NULL;
-	struct cryptocteon_softc *sc;
-	struct octo_sess *ocd;
-	int i;
+	u_int hash_len;
 
-	sc = device_get_softc(dev);
+	switch (csp->csp_auth_alg) {
+	case CRYPTO_MD5_HMAC:
+		hash_len = MD5_HASH_LEN;
+		break;
+	case CRYPTO_SHA1_HMAC:
+		hash_len = SHA1_HASH_LEN;
+		break;
+	default:
+		return (false);
+	}
 
-	if (cri == NULL || sc == NULL)
-		return (EINVAL);
+	if (csp->csp_auth_klen > hash_len)
+		return (false);
+	return (true);
+}
 
-	/*
-	 * To keep it simple, we only handle hash, cipher or hash/cipher in a
-	 * session,  you cannot currently do multiple ciphers/hashes in one
-	 * session even though it would be possibel to code this driver to
-	 * handle it.
-	 */
-	for (i = 0, c = cri; c && i < 2; i++) {
-		if (c->cri_alg == CRYPTO_MD5_HMAC ||
-		    c->cri_alg == CRYPTO_SHA1_HMAC ||
-		    c->cri_alg == CRYPTO_NULL_HMAC) {
-			if (macini) {
-				break;
-			}
-			macini = c;
-		}
-		if (c->cri_alg == CRYPTO_DES_CBC ||
-		    c->cri_alg == CRYPTO_3DES_CBC ||
-		    c->cri_alg == CRYPTO_AES_CBC ||
-		    c->cri_alg == CRYPTO_NULL_CBC) {
-			if (encini) {
-				break;
-			}
-			encini = c;
-		}
-		c = c->cri_next;
+static bool
+cryptocteon_cipher_supported(const struct crypto_session_params *csp)
+{
+
+	switch (csp->csp_cipher_alg) {
+	case CRYPTO_DES_CBC:
+	case CRYPTO_3DES_CBC:
+		if (csp->csp_ivlen != 8)
+			return (false);
+		if (csp->csp_cipher_klen != 8 &&
+		    csp->csp_cipher_klen != 24)
+			return (false);
+		break;
+	case CRYPTO_AES_CBC:
+		if (csp->csp_ivlen != 16)
+			return (false);
+		if (csp->csp_cipher_klen != 16 &&
+		    csp->csp_cipher_klen != 24 &&
+		    csp->csp_cipher_klen != 32)
+			return (false);
+		break;
+	default:
+		return (false);
 	}
-	if (!macini && !encini) {
-		dprintf("%s,%d - EINVAL bad cipher/hash or combination\n",
-				__FILE__, __LINE__);
-		return EINVAL;
-	}
-	if (c) {
-		dprintf("%s,%d - EINVAL cannot handle chained cipher/hash combos\n",
-				__FILE__, __LINE__);
-		return EINVAL;
-	}
 
-	/*
-	 * So we have something we can do, lets setup the session
-	 */
-	ocd = crypto_get_driver_session(cses);
+	return (true);
+}
 
-	if (encini && encini->cri_key) {
-		ocd->octo_encklen = (encini->cri_klen + 7) / 8;
-		memcpy(ocd->octo_enckey, encini->cri_key, ocd->octo_encklen);
-	}
+static int
+cryptocteon_probesession(device_t dev, const struct crypto_session_params *csp)
+{
 
-	if (macini && macini->cri_key) {
-		ocd->octo_macklen = (macini->cri_klen + 7) / 8;
-		memcpy(ocd->octo_mackey, macini->cri_key, ocd->octo_macklen);
+	if (csp->csp_flags != 0)
+		return (EINVAL);
+	switch (csp->csp_mode) {
+	case CSP_MODE_DIGEST:
+		if (!cryptocteon_auth_supported(csp))
+			return (EINVAL);
+		break;
+	case CSP_MODE_CIPHER:
+		if (!cryptocteon_cipher_supported(csp))
+			return (EINVAL);
+		break;
+	case CSP_MODE_ETA:
+		if (!cryptocteon_auth_supported(csp) ||
+		    !cryptocteon_cipher_supported(csp))
+			return (EINVAL);
+		break;
+	default:
+		return (EINVAL);
 	}
+	return (CRYPTODEV_PROBE_ACCEL_SOFTWARE);
+}
 
-	ocd->octo_mlen = 0;
-	if (encini && encini->cri_mlen)
-		ocd->octo_mlen = encini->cri_mlen;
-	else if (macini && macini->cri_mlen)
-		ocd->octo_mlen = macini->cri_mlen;
-	else
-		ocd->octo_mlen = 12;
+static void
+cryptocteon_calc_hash(const struct crypto_session_params *csp, const char *key,
+    struct octo_sess *ocd)
+{
+	char hash_key[SHA1_HASH_LEN];
 
-	/*
-	 * point c at the enc if it exists, otherwise the mac
-	 */
-	c = encini ? encini : macini;
+	memset(hash_key, 0, sizeof(hash_key));
+	memcpy(hash_key, key, csp->csp_auth_klen);
+	octo_calc_hash(csp->csp_auth_alg == CRYPTO_SHA1_HMAC, hash_key,
+	    ocd->octo_hminner, ocd->octo_hmouter);
+}
 
-	switch (c->cri_alg) {
-	case CRYPTO_DES_CBC:
-	case CRYPTO_3DES_CBC:
-		ocd->octo_ivsize  = 8;
-		switch (macini ? macini->cri_alg : -1) {
+/* Generate a new octo session. */
+static int
+cryptocteon_newsession(device_t dev, crypto_session_t cses,
+    const struct crypto_session_params *csp)
+{
+	struct cryptocteon_softc *sc;
+	struct octo_sess *ocd;
+
+	sc = device_get_softc(dev);
+
+	ocd = crypto_get_driver_session(cses);
+
+	ocd->octo_encklen = csp->csp_cipher_klen;
+	if (csp->csp_cipher_key != NULL)
+		memcpy(ocd->octo_enckey, csp->csp_cipher_key,
+		    ocd->octo_encklen);
+
+	if (csp->csp_auth_key != NULL)
+		cryptocteon_calc_hash(csp, csp->csp_auth_key, ocd);
+
+	ocd->octo_mlen = csp->csp_auth_mlen;
+	if (csp->csp_auth_mlen == 0) {
+		switch (csp->csp_auth_alg) {
 		case CRYPTO_MD5_HMAC:
-			ocd->octo_encrypt = octo_des_cbc_md5_encrypt;
-			ocd->octo_decrypt = octo_des_cbc_md5_decrypt;
-			octo_calc_hash(0, macini->cri_key, ocd->octo_hminner,
-					ocd->octo_hmouter);
+			ocd->octo_mlen = MD5_HASH_LEN;
 			break;
 		case CRYPTO_SHA1_HMAC:
-			ocd->octo_encrypt = octo_des_cbc_sha1_encrypt;
-			ocd->octo_decrypt = octo_des_cbc_sha1_encrypt;
-			octo_calc_hash(1, macini->cri_key, ocd->octo_hminner,
-					ocd->octo_hmouter);
+			ocd->octo_mlen = SHA1_HASH_LEN;
 			break;
-		case -1:
-			ocd->octo_encrypt = octo_des_cbc_encrypt;
-			ocd->octo_decrypt = octo_des_cbc_decrypt;
-			break;
-		default:
-			dprintf("%s,%d: EINVALn", __FILE__, __LINE__);
-			return EINVAL;
 		}
-		break;
-	case CRYPTO_AES_CBC:
-		ocd->octo_ivsize  = 16;
-		switch (macini ? macini->cri_alg : -1) {
+	}
+
+	switch (csp->csp_mode) {
+	case CSP_MODE_DIGEST:
+		switch (csp->csp_auth_alg) {
 		case CRYPTO_MD5_HMAC:
-			ocd->octo_encrypt = octo_aes_cbc_md5_encrypt;
-			ocd->octo_decrypt = octo_aes_cbc_md5_decrypt;
-			octo_calc_hash(0, macini->cri_key, ocd->octo_hminner,
-					ocd->octo_hmouter);
+			ocd->octo_encrypt = octo_null_md5_encrypt;
+			ocd->octo_decrypt = octo_null_md5_encrypt;
 			break;
 		case CRYPTO_SHA1_HMAC:
-			ocd->octo_encrypt = octo_aes_cbc_sha1_encrypt;
-			ocd->octo_decrypt = octo_aes_cbc_sha1_decrypt;
-			octo_calc_hash(1, macini->cri_key, ocd->octo_hminner,
-					ocd->octo_hmouter);
+			ocd->octo_encrypt = octo_null_sha1_encrypt;
+			ocd->octo_decrypt = octo_null_sha1_encrypt;
 			break;
-		case -1:
+		}
+		break;
+	case CSP_MODE_CIPHER:
+		switch (csp->csp_cipher_alg) {
+		case CRYPTO_DES_CBC:
+		case CRYPTO_3DES_CBC:
+			ocd->octo_encrypt = octo_des_cbc_encrypt;
+			ocd->octo_decrypt = octo_des_cbc_decrypt;
+			break;
+		case CRYPTO_AES_CBC:
 			ocd->octo_encrypt = octo_aes_cbc_encrypt;
 			ocd->octo_decrypt = octo_aes_cbc_decrypt;
 			break;
-		default:
-			dprintf("%s,%d: EINVALn", __FILE__, __LINE__);
-			return EINVAL;
 		}
 		break;
-	case CRYPTO_MD5_HMAC:
-		ocd->octo_encrypt = octo_null_md5_encrypt;
-		ocd->octo_decrypt = octo_null_md5_encrypt;
-		octo_calc_hash(0, macini->cri_key, ocd->octo_hminner,
-				ocd->octo_hmouter);
+	case CSP_MODE_ETA:
+		switch (csp->csp_cipher_alg) {
+		case CRYPTO_DES_CBC:
+		case CRYPTO_3DES_CBC:
+			switch (csp->csp_auth_alg) {
+			case CRYPTO_MD5_HMAC:
+				ocd->octo_encrypt = octo_des_cbc_md5_encrypt;
+				ocd->octo_decrypt = octo_des_cbc_md5_decrypt;
+				break;
+			case CRYPTO_SHA1_HMAC:
+				ocd->octo_encrypt = octo_des_cbc_sha1_encrypt;
+				ocd->octo_decrypt = octo_des_cbc_sha1_encrypt;
+				break;
+			}
+			break;
+		case CRYPTO_AES_CBC:
+			switch (csp->csp_auth_alg) {
+			case CRYPTO_MD5_HMAC:
+				ocd->octo_encrypt = octo_aes_cbc_md5_encrypt;
+				ocd->octo_decrypt = octo_aes_cbc_md5_decrypt;
+				break;
+			case CRYPTO_SHA1_HMAC:
+				ocd->octo_encrypt = octo_aes_cbc_sha1_encrypt;
+				ocd->octo_decrypt = octo_aes_cbc_sha1_decrypt;
+				break;
+			}
+			break;
+		}
 		break;
-	case CRYPTO_SHA1_HMAC:
-		ocd->octo_encrypt = octo_null_sha1_encrypt;
-		ocd->octo_decrypt = octo_null_sha1_encrypt;
-		octo_calc_hash(1, macini->cri_key, ocd->octo_hminner,
-				ocd->octo_hmouter);
-		break;
-	default:
-		dprintf("%s,%d: EINVALn", __FILE__, __LINE__);
-		return EINVAL;
 	}
 
-	ocd->octo_encalg = encini ? encini->cri_alg : -1;
-	ocd->octo_macalg = macini ? macini->cri_alg : -1;
+	KASSERT(ocd->octo_encrypt != NULL && ocd->octo_decrypt != NULL,
+	    ("%s: missing function pointers", __func__));
 
 	return (0);
 }
 
 /*
  * Process a request.
  */
 static int
 cryptocteon_process(device_t dev, struct cryptop *crp, int hint)
 {
-	struct cryptodesc *crd;
+	const struct crypto_session_params *csp;
 	struct octo_sess *od;
 	size_t iovcnt, iovlen;
 	struct mbuf *m = NULL;
 	struct uio *uiop = NULL;
-	struct cryptodesc *enccrd = NULL, *maccrd = NULL;
 	unsigned char *ivp = NULL;
-	unsigned char iv_data[HASH_MAX_LEN];
-	int auth_off = 0, auth_len = 0, crypt_off = 0, crypt_len = 0, icv_off = 0;
+	unsigned char iv_data[16];
+	unsigned char icv[SHA1_HASH_LEN], icv2[SHA1_HASH_LEN];
+	int auth_off, auth_len, crypt_off, crypt_len;
 	struct cryptocteon_softc *sc;
 
 	sc = device_get_softc(dev);
 
-	if (sc == NULL || crp == NULL)
-		return EINVAL;
-
 	crp->crp_etype = 0;
 
-	if (crp->crp_desc == NULL || crp->crp_buf == NULL) {
-		dprintf("%s,%d: EINVAL\n", __FILE__, __LINE__);
-		crp->crp_etype = EINVAL;
+	od = crypto_get_driver_session(crp->crp_session);
+	csp = crypto_get_params(crp->crp_session);
+
+	/*
+	 * The crypto routines assume that the regions to auth and
+	 * cipher are exactly 8 byte multiples and aligned on 8
+	 * byte logical boundaries within the iovecs.
+	 */
+	if (crp->crp_aad_length % 8 != 0 || crp->crp_payload_length % 8 != 0) {
+		crp->crp_etype = EFBIG;
 		goto done;
 	}
 
-	od = crypto_get_driver_session(crp->crp_session);
+	/*
+	 * As currently written, the crypto routines assume the AAD and
+	 * payload are adjacent.
+	 */
+	if (crp->crp_aad_length != 0 && crp->crp_payload_start !=
+	    crp->crp_aad_start + crp->crp_aad_length) {
+		crp->crp_etype = EFBIG;
+		goto done;
+	}
 
+	crypt_off = crp->crp_payload_start;
+	crypt_len = crp->crp_payload_length;
+	if (crp->crp_aad_length != 0) {
+		auth_off = crp->crp_aad_start;
+		auth_len = crp->crp_aad_length + crp->crp_payload_length;
+	} else {
+		auth_off = crypt_off;
+		auth_len = crypt_len;
+	}
+
 	/*
 	 * do some error checking outside of the loop for m and IOV processing
 	 * this leaves us with valid m or uiop pointers for later
 	 */
-	if (crp->crp_flags & CRYPTO_F_IMBUF) {
+	switch (crp->crp_buf_type) {
+	case CRYPTO_BUF_MBUF:
+	{
 		unsigned frags;
 
-		m = (struct mbuf *) crp->crp_buf;
+		m = crp->crp_mbuf;
 		for (frags = 0; m != NULL; frags++)
 			m = m->m_next;
 
 		if (frags >= UIO_MAXIOV) {
 			printf("%s,%d: %d frags > UIO_MAXIOV", __FILE__, __LINE__, frags);
+			crp->crp_etype = EFBIG;
 			goto done;
 		}
 
-		m = (struct mbuf *) crp->crp_buf;
-	} else if (crp->crp_flags & CRYPTO_F_IOV) {
-		uiop = (struct uio *) crp->crp_buf;
+		m = crp->crp_mbuf;
+		break;
+	}
+	case CRYPTO_BUF_UIO:
+		uiop = crp->crp_uio;
 		if (uiop->uio_iovcnt > UIO_MAXIOV) {
 			printf("%s,%d: %d uio_iovcnt > UIO_MAXIOV", __FILE__, __LINE__,
 			       uiop->uio_iovcnt);
+			crp->crp_etype = EFBIG;
 			goto done;
 		}
+		break;
 	}
 
-	/* point our enccrd and maccrd appropriately */
-	crd = crp->crp_desc;
-	if (crd->crd_alg == od->octo_encalg)
-		enccrd = crd;
-	if (crd->crd_alg == od->octo_macalg)
-		maccrd = crd;
-	crd = crd->crd_next;
-	if (crd) {
-		if (crd->crd_alg == od->octo_encalg)
-			enccrd = crd;
-		if (crd->crd_alg == od->octo_macalg)
-			maccrd = crd;
-		crd = crd->crd_next;
-	}
-	if (crd) {
-		crp->crp_etype = EINVAL;
-		dprintf("%s,%d: ENOENT - descriptors do not match session\n",
-				__FILE__, __LINE__);
-		goto done;
-	}
-
-	if (enccrd) {
-		if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) {
-			ivp = enccrd->crd_iv;
-		} else {
+	if (csp->csp_cipher_alg != 0) {
+		if (crp->crp_flags & CRYPTO_F_IV_GENERATE) {
+			arc4rand(iv_data, csp->csp_ivlen, 0);
+			crypto_copyback(crp, crp->crp_iv_start, csp->csp_ivlen,
+			    iv_data);
 			ivp = iv_data;
-			crypto_copydata(crp->crp_flags, crp->crp_buf,
-					enccrd->crd_inject, od->octo_ivsize, (caddr_t) ivp);
+		} else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+			ivp = crp->crp_iv;
+		else {
+			crypto_copydata(crp, crp->crp_iv_start, csp->csp_ivlen,
+			    iv_data);
+			ivp = iv_data;
 		}
-
-		if (maccrd) {
-			auth_off = maccrd->crd_skip;
-			auth_len = maccrd->crd_len;
-			icv_off  = maccrd->crd_inject;
-		}
-
-		crypt_off = enccrd->crd_skip;
-		crypt_len = enccrd->crd_len;
-	} else { /* if (maccrd) */
-		auth_off = maccrd->crd_skip;
-		auth_len = maccrd->crd_len;
-		icv_off  = maccrd->crd_inject;
 	}
 
 	/*
 	 * setup the I/O vector to cover the buffer
 	 */
-	if (crp->crp_flags & CRYPTO_F_IMBUF) {
+	switch (crp->crp_buf_type) {
+	case CRYPTO_BUF_MBUF:
 		iovcnt = 0;
 		iovlen = 0;
 
 		while (m != NULL) {
 			od->octo_iov[iovcnt].iov_base = mtod(m, void *);
 			od->octo_iov[iovcnt].iov_len = m->m_len;
 
 			m = m->m_next;
 			iovlen += od->octo_iov[iovcnt++].iov_len;
 		}
-	} else if (crp->crp_flags & CRYPTO_F_IOV) {
+		break;
+	case CRYPTO_BUF_UIO:
 		iovlen = 0;
 		for (iovcnt = 0; iovcnt < uiop->uio_iovcnt; iovcnt++) {
 			od->octo_iov[iovcnt].iov_base = uiop->uio_iov[iovcnt].iov_base;
 			od->octo_iov[iovcnt].iov_len = uiop->uio_iov[iovcnt].iov_len;
 
 			iovlen += od->octo_iov[iovcnt].iov_len;
 		}
-	} else {
+		break;
+	case CRYPTO_BUF_CONTIG:
 		iovlen = crp->crp_ilen;
 		od->octo_iov[0].iov_base = crp->crp_buf;
 		od->octo_iov[0].iov_len = crp->crp_ilen;
 		iovcnt = 1;
+		break;
+	default:
+		panic("can't happen");
 	}
 
 
 	/*
 	 * setup a new explicit key
 	 */
-	if (enccrd) {
-		if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT) {
-			od->octo_encklen = (enccrd->crd_klen + 7) / 8;
-			memcpy(od->octo_enckey, enccrd->crd_key, od->octo_encklen);
-		}
-	}
-	if (maccrd) {
-		if (maccrd->crd_flags & CRD_F_KEY_EXPLICIT) {
-			od->octo_macklen = (maccrd->crd_klen + 7) / 8;
-			memcpy(od->octo_mackey, maccrd->crd_key, od->octo_macklen);
-			od->octo_mackey_set = 0;
-		}
-		if (!od->octo_mackey_set) {
-			octo_calc_hash(maccrd->crd_alg == CRYPTO_MD5_HMAC ? 0 : 1,
-				maccrd->crd_key, od->octo_hminner, od->octo_hmouter);
-			od->octo_mackey_set = 1;
-		}
-	}
+	if (crp->crp_cipher_key != NULL)
+		memcpy(od->octo_enckey, crp->crp_cipher_key, od->octo_encklen);
+	if (crp->crp_auth_key != NULL)
+		cryptocteon_calc_hash(csp, crp->crp_auth_key, od);
 
 
-	if (!enccrd || (enccrd->crd_flags & CRD_F_ENCRYPT))
+	if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
 		(*od->octo_encrypt)(od, od->octo_iov, iovcnt, iovlen,
-				auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+		    auth_off, auth_len, crypt_off, crypt_len, icv, ivp);
 	else
 		(*od->octo_decrypt)(od, od->octo_iov, iovcnt, iovlen,
-				auth_off, auth_len, crypt_off, crypt_len, icv_off, ivp);
+		    auth_off, auth_len, crypt_off, crypt_len, icv, ivp);
 
+	if (csp->csp_auth_alg != 0) {
+		if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
+			crypto_copydata(crp, crp->crp_digest_start,
+			    od->octo_mlen, icv2);
+			if (timingsafe_bcmp(icv, icv2, od->octo_mlen) != 0)
+				crp->crp_etype = EBADMSG;
+		} else
+			crypto_copyback(crp, crp->crp_digest_start,
+			    od->octo_mlen, icv);
+	}
 done:
 	crypto_done(crp);
 	return (0);
 }
 
 static device_method_t cryptocteon_methods[] = {
 	/* device methods */
 	DEVMETHOD(device_identify,	cryptocteon_identify),
 	DEVMETHOD(device_probe,		cryptocteon_probe),
 	DEVMETHOD(device_attach,	cryptocteon_attach),
 
 	/* crypto device methods */
+	DEVMETHOD(cryptodev_probesession, cryptocteon_probesession),
 	DEVMETHOD(cryptodev_newsession,	cryptocteon_newsession),
 	DEVMETHOD(cryptodev_process,	cryptocteon_process),
 
 	{ 0, 0 }
 };
 
 static driver_t cryptocteon_driver = {
 	"cryptocteon",
 	cryptocteon_methods,
 	sizeof (struct cryptocteon_softc),
 };
 static devclass_t cryptocteon_devclass;
 DRIVER_MODULE(cryptocteon, nexus, cryptocteon_driver, cryptocteon_devclass, 0, 0);
Index: head/sys/mips/cavium/cryptocteon/cryptocteonvar.h
===================================================================
--- head/sys/mips/cavium/cryptocteon/cryptocteonvar.h	(revision 359373)
+++ head/sys/mips/cavium/cryptocteon/cryptocteonvar.h	(revision 359374)
@@ -1,94 +1,86 @@
 /*
  * Octeon Crypto for OCF
  *
  * Written by David McCullough <david_mccullough@securecomputing.com>
  * Copyright (C) 2009 David McCullough
  *
  * LICENSE TERMS
  *
  * The free distribution and use of this software in both source and binary
  * form is allowed (with or without changes) provided that:
  *
  *   1. distributions of this source code include the above copyright
  *      notice, this list of conditions and the following disclaimer;
  *
  *   2. distributions in binary form include the above copyright
  *      notice, this list of conditions and the following disclaimer
  *      in the documentation and/or other associated materials;
  *
  *   3. the copyright holder's name is not used to endorse products
  *      built using this software without specific written permission.
  *
  * DISCLAIMER
  *
  * This software is provided 'as is' with no explicit or implied warranties
  * in respect of its properties, including, but not limited to, correctness
  * and/or fitness for purpose.
  * ---------------------------------------------------------------------------
  *
  * $FreeBSD$
  */
 
 #ifndef	_MIPS_CAVIUM_CRYPTOCTEON_CRYPTOCTEONVAR_H_
 #define	_MIPS_CAVIUM_CRYPTOCTEON_CRYPTOCTEONVAR_H_
 
 struct octo_sess;
 
-typedef	int octo_encrypt_t(struct octo_sess *od, struct iovec *iov, size_t iovcnt, size_t iovlen, int auth_off, int auth_len, int crypt_off, int crypt_len, int icv_off, uint8_t *ivp);
-typedef	int octo_decrypt_t(struct octo_sess *od, struct iovec *iov, size_t iovcnt, size_t iovlen, int auth_off, int auth_len, int crypt_off, int crypt_len, int icv_off, uint8_t *ivp);
+typedef	int octo_encrypt_t(struct octo_sess *od, struct iovec *iov, size_t iovcnt, size_t iovlen, int auth_off, int auth_len, int crypt_off, int crypt_len, uint8_t *icv, uint8_t *ivp);
+typedef	int octo_decrypt_t(struct octo_sess *od, struct iovec *iov, size_t iovcnt, size_t iovlen, int auth_off, int auth_len, int crypt_off, int crypt_len, uint8_t *icv, uint8_t *ivp);
 
 struct octo_sess {
-	int					 octo_encalg;
 	#define MAX_CIPHER_KEYLEN	64
 	char				 octo_enckey[MAX_CIPHER_KEYLEN];
 	int					 octo_encklen;
 
-	int					 octo_macalg;
-	#define MAX_HASH_KEYLEN	64
-	char				 octo_mackey[MAX_HASH_KEYLEN];
-	int					 octo_macklen;
-	int					 octo_mackey_set;
-
 	int					 octo_mlen;
-	int					 octo_ivsize;
 
 	octo_encrypt_t				*octo_encrypt;
 	octo_decrypt_t				*octo_decrypt;
 
 	uint64_t			 octo_hminner[3];
 	uint64_t			 octo_hmouter[3];
 
 	struct iovec				octo_iov[UIO_MAXIOV];
 };
 
 #define	dprintf(fmt, ...)						\
 	do {								\
 		if (cryptocteon_debug)					\
 			printf("%s: " fmt, __func__, ## __VA_ARGS__);	\
 	} while (0)
 
 extern int cryptocteon_debug;
 
 void octo_calc_hash(uint8_t, unsigned char *, uint64_t *, uint64_t *);
 
 /* XXX Actually just hashing functions, not encryption.  */
 octo_encrypt_t octo_null_md5_encrypt;
 octo_encrypt_t octo_null_sha1_encrypt;
 
 octo_encrypt_t octo_des_cbc_encrypt;
 octo_encrypt_t octo_des_cbc_md5_encrypt;
 octo_encrypt_t octo_des_cbc_sha1_encrypt;
 
 octo_decrypt_t octo_des_cbc_decrypt;
 octo_decrypt_t octo_des_cbc_md5_decrypt;
 octo_decrypt_t octo_des_cbc_sha1_decrypt;
 
 octo_encrypt_t octo_aes_cbc_encrypt;
 octo_encrypt_t octo_aes_cbc_md5_encrypt;
 octo_encrypt_t octo_aes_cbc_sha1_encrypt;
 
 octo_decrypt_t octo_aes_cbc_decrypt;
 octo_decrypt_t octo_aes_cbc_md5_decrypt;
 octo_decrypt_t octo_aes_cbc_sha1_decrypt;
 
 #endif /* !_MIPS_CAVIUM_CRYPTOCTEON_CRYPTOCTEONVAR_H_ */
Index: head/sys/mips/nlm/dev/sec/nlmrsa.c
===================================================================
--- head/sys/mips/nlm/dev/sec/nlmrsa.c	(revision 359373)
+++ head/sys/mips/nlm/dev/sec/nlmrsa.c	(revision 359374)
@@ -1,500 +1,484 @@
 /*-
  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
  *
  * Copyright (c) 2003-2012 Broadcom Corporation
  * All Rights Reserved
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  *
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in
  *    the documentation and/or other materials provided with the
  *    distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY BROADCOM ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED. IN NO EVENT SHALL BROADCOM OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
  * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
  * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
  * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include <sys/cdefs.h>
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/proc.h>
 #include <sys/errno.h>
 #include <sys/endian.h>
 #include <sys/malloc.h>
 #include <sys/kernel.h>
 #include <sys/module.h>
 #include <sys/mbuf.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <sys/sysctl.h>
 #include <sys/bus.h>
 #include <sys/random.h>
 #include <sys/rman.h>
 #include <sys/uio.h>
 #include <sys/kobj.h>
 
 #include <dev/pci/pcivar.h>
 
 #include <opencrypto/cryptodev.h>
 
 #include "cryptodev_if.h"
 
 #include <vm/vm.h>
 #include <vm/pmap.h>
 
 #include <mips/nlm/hal/haldefs.h>
 #include <mips/nlm/hal/iomap.h>
 #include <mips/nlm/xlp.h>
 #include <mips/nlm/hal/sys.h>
 #include <mips/nlm/hal/fmn.h>
 #include <mips/nlm/hal/nlmsaelib.h>
 #include <mips/nlm/dev/sec/rsa_ucode.h>
 #include <mips/nlm/hal/cop2.h>
 #include <mips/nlm/hal/mips-extns.h>
 #include <mips/nlm/msgring.h>
 #include <mips/nlm/dev/sec/nlmrsalib.h>
 
 #ifdef NLM_RSA_DEBUG
 static	void print_krp_params(struct cryptkop *krp);
 #endif
 
 static	int xlp_rsa_init(struct xlp_rsa_softc *sc, int node);
-static	int xlp_rsa_newsession(device_t , crypto_session_t, struct cryptoini *);
 static	int xlp_rsa_kprocess(device_t , struct cryptkop *, int);
 static	int xlp_get_rsa_opsize(struct xlp_rsa_command *cmd, unsigned int bits);
 static	void xlp_free_cmd_params(struct xlp_rsa_command *cmd);
 static	int xlp_rsa_inp2hwformat(uint8_t *src, uint8_t *dst,
     uint32_t paramsize, uint8_t result);
 
 static	int xlp_rsa_probe(device_t);
 static	int xlp_rsa_attach(device_t);
 static	int xlp_rsa_detach(device_t);
 
 static device_method_t xlp_rsa_methods[] = {
 	/* device interface */
 	DEVMETHOD(device_probe, xlp_rsa_probe),
 	DEVMETHOD(device_attach, xlp_rsa_attach),
 	DEVMETHOD(device_detach, xlp_rsa_detach),
 
 	/* bus interface */
 	DEVMETHOD(bus_print_child, bus_generic_print_child),
 	DEVMETHOD(bus_driver_added, bus_generic_driver_added),
 
 	/* crypto device methods */
-	DEVMETHOD(cryptodev_newsession, xlp_rsa_newsession),
 	DEVMETHOD(cryptodev_kprocess,   xlp_rsa_kprocess),
 
 	DEVMETHOD_END
 };
 
 static driver_t xlp_rsa_driver = {
 	"nlmrsa",
 	xlp_rsa_methods,
 	sizeof(struct xlp_rsa_softc)
 };
 static devclass_t xlp_rsa_devclass;
 
 DRIVER_MODULE(nlmrsa, pci, xlp_rsa_driver, xlp_rsa_devclass, 0, 0);
 MODULE_DEPEND(nlmrsa, crypto, 1, 1, 1);
 
 #ifdef NLM_RSA_DEBUG
 static void
 print_krp_params(struct cryptkop *krp)
 {
 	int i;
 
 	printf("krp->krp_op	:%d\n", krp->krp_op);
 	printf("krp->krp_status	:%d\n", krp->krp_status);
 	printf("krp->krp_iparams:%d\n", krp->krp_iparams);
 	printf("krp->krp_oparams:%d\n", krp->krp_oparams);
 	for (i = 0; i < krp->krp_iparams + krp->krp_oparams; i++) {
 		printf("krp->krp_param[%d].crp_p	:0x%llx\n", i,
 		    (unsigned long long)krp->krp_param[i].crp_p);
 		printf("krp->krp_param[%d].crp_nbits	:%d\n", i,
 		    krp->krp_param[i].crp_nbits);
 		printf("krp->krp_param[%d].crp_nbytes	:%d\n", i,
 		    howmany(krp->krp_param[i].crp_nbits, 8));
 	}
 }
 #endif
 
 static int
 xlp_rsa_init(struct xlp_rsa_softc *sc, int node)
 {
 	struct xlp_rsa_command *cmd = NULL;
 	uint32_t fbvc, dstvc, endsel, regval;
 	struct nlm_fmn_msg m;
 	int err, ret, i;
 	uint64_t base;
 
 	/* Register interrupt handler for the RSA/ECC CMS messages */
 	if (register_msgring_handler(sc->rsaecc_vc_start,
 	    sc->rsaecc_vc_end, nlm_xlprsaecc_msgring_handler, sc) != 0) {
 		err = -1;
 		printf("Couldn't register rsa/ecc msgring handler\n");
 		goto errout;
 	}
 	fbvc = nlm_cpuid() * 4 + XLPGE_FB_VC;
 	/* Do the CMS credit initialization */
 	/* Currently it is configured by default to 50 when kernel comes up */
 
 #if BYTE_ORDER == LITTLE_ENDIAN
 	for (i = 0; i < nitems(nlm_rsa_ucode_data); i++)
 		nlm_rsa_ucode_data[i] = htobe64(nlm_rsa_ucode_data[i]);
 #endif
 	for (dstvc = sc->rsaecc_vc_start; dstvc <= sc->rsaecc_vc_end; dstvc++) {
 		cmd = malloc(sizeof(struct xlp_rsa_command), M_DEVBUF,
 		    M_NOWAIT | M_ZERO);
 		KASSERT(cmd != NULL, ("%s:cmd is NULL\n", __func__));
 		cmd->rsasrc = contigmalloc(sizeof(nlm_rsa_ucode_data),
 		    M_DEVBUF,
 		    (M_WAITOK | M_ZERO),
 		    0UL /* low address */, -1UL /* high address */,
 		    XLP_L2L3_CACHELINE_SIZE /* alignment */,
 		    0UL /* boundary */);
 		KASSERT(cmd->rsasrc != NULL,
 		    ("%s:cmd->rsasrc is NULL\n", __func__));
 		memcpy(cmd->rsasrc, nlm_rsa_ucode_data,
 		    sizeof(nlm_rsa_ucode_data));
 		m.msg[0] = nlm_crypto_form_rsa_ecc_fmn_entry0(1, 0x70, 0,
 		    vtophys(cmd->rsasrc));
 		m.msg[1] = nlm_crypto_form_rsa_ecc_fmn_entry1(0, 1, fbvc,
 		    vtophys(cmd->rsasrc));
 		/* Software scratch pad */
 		m.msg[2] = (uintptr_t)cmd;
 		m.msg[3] = 0;
 
 		ret = nlm_fmn_msgsend(dstvc, 3, FMN_SWCODE_RSA, &m);
 		if (ret != 0) {
 			err = -1;
 			printf("%s: msgsnd failed (%x)\n", __func__, ret);
 			goto errout;
 		}
 	}
 	/* Configure so that all VCs send request to all RSA pipes */
 	base = nlm_get_rsa_regbase(node);
 	if (nlm_is_xlp3xx()) {
 		endsel = 1;
 		regval = 0xFFFF;
 	} else {
 		endsel = 3;
 		regval = 0x07FFFFFF;
 	}
 	for (i = 0; i < endsel; i++)
 		nlm_write_rsa_reg(base, RSA_ENG_SEL_0 + i, regval);
 	return (0);
 errout:
 	xlp_free_cmd_params(cmd);
 	return (err);
 }
 
 /* This function is called from an interrupt handler */
 void
 nlm_xlprsaecc_msgring_handler(int vc, int size, int code, int src_id,
     struct nlm_fmn_msg *msg, void *data)
 {
 	struct xlp_rsa_command *cmd;
 	struct xlp_rsa_softc *sc;
 	struct crparam *outparam;
 	int ostart;
 
 	KASSERT(code == FMN_SWCODE_RSA,
 	    ("%s: bad code = %d, expected code = %d\n", __func__, code,
 	    FMN_SWCODE_RSA));
 
 	sc = data;
 	KASSERT(src_id >= sc->rsaecc_vc_start && src_id <= sc->rsaecc_vc_end,
 	    ("%s: bad src_id = %d, expect %d - %d\n", __func__,
 	    src_id, sc->rsaecc_vc_start, sc->rsaecc_vc_end));
 
 	cmd = (struct xlp_rsa_command *)(uintptr_t)msg->msg[1];
 	KASSERT(cmd != NULL, ("%s:cmd not received properly\n", __func__));
 
 	if (RSA_ERROR(msg->msg[0]) != 0) {
 		printf("%s: Message rcv msg0 %llx msg1 %llx err %x \n",
 		    __func__, (unsigned long long)msg->msg[0],
 		    (unsigned long long)msg->msg[1],
 		    (int)RSA_ERROR(msg->msg[0]));
 		cmd->krp->krp_status = EBADMSG;
 	}
 
 	if (cmd->krp != NULL) {
 		ostart = cmd->krp->krp_iparams;
 		outparam = &cmd->krp->krp_param[ostart];
 		xlp_rsa_inp2hwformat(cmd->rsasrc + cmd->rsaopsize * ostart,
 		    outparam->crp_p,
 		    howmany(outparam->crp_nbits, 8),
 		    1);
 		crypto_kdone(cmd->krp);
 	}
 
 	xlp_free_cmd_params(cmd);
 }
 
 static int
 xlp_rsa_probe(device_t dev)
 {
 	struct xlp_rsa_softc *sc;
 
 	if (pci_get_vendor(dev) == PCI_VENDOR_NETLOGIC &&
 	    pci_get_device(dev) == PCI_DEVICE_ID_NLM_RSA) {
 		sc = device_get_softc(dev);
 		return (BUS_PROBE_DEFAULT);
 	}
 	return (ENXIO);
 }
 
 /*
  * Attach an interface that successfully probed.
  */
 static int
 xlp_rsa_attach(device_t dev)
 {
 	struct xlp_rsa_softc *sc = device_get_softc(dev);
 	uint64_t base;
 	int qstart, qnum;
 	int freq, node;
 
 	sc->sc_dev = dev;
 
 	node = nlm_get_device_node(pci_get_slot(dev));
 	freq = nlm_set_device_frequency(node, DFS_DEVICE_RSA, 250);
 	if (bootverbose)
 		device_printf(dev, "RSA Freq: %dMHz\n", freq);
 	if (pci_get_device(dev) == PCI_DEVICE_ID_NLM_RSA) {
 		device_set_desc(dev, "XLP RSA/ECC Accelerator");
 		sc->sc_cid = crypto_get_driverid(dev,
 		    sizeof(struct xlp_rsa_session), CRYPTOCAP_F_HARDWARE);
 		if (sc->sc_cid < 0) {
 			printf("xlp_rsaecc-err:couldn't get the driver id\n");
 			goto error_exit;
 		}
 		if (crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0) != 0)
 			goto error_exit;
 
 		base = nlm_get_rsa_pcibase(node);
 		qstart = nlm_qidstart(base);
 		qnum = nlm_qnum(base);
 		sc->rsaecc_vc_start = qstart;
 		sc->rsaecc_vc_end = qstart + qnum - 1;
 	}
 	if (xlp_rsa_init(sc, node) != 0)
 		goto error_exit;
 	device_printf(dev, "RSA Initialization complete!\n");
 	return (0);
 
 error_exit:
 	return (ENXIO);
 }
 
 /*
  * Detach an interface that successfully probed.
  */
 static int
 xlp_rsa_detach(device_t dev)
 {
-	return (0);
-}
-
-/*
- * Allocate a new 'session' (unused).
- */
-static int
-xlp_rsa_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
-{
-	struct xlp_rsa_softc *sc = device_get_softc(dev);
-
-	if (cri == NULL || sc == NULL)
-		return (EINVAL);
-
 	return (0);
 }
 
 /*
  * XXX freesession should run a zero'd mac/encrypt key into context ram.
  * XXX to blow away any keys already stored there.
  */
 
 static void
 xlp_free_cmd_params(struct xlp_rsa_command *cmd)
 {
 
 	if (cmd == NULL)
 		return;
 	if (cmd->rsasrc != NULL) {
 		if (cmd->krp == NULL) /* Micro code load */
 			contigfree(cmd->rsasrc, sizeof(nlm_rsa_ucode_data),
 			    M_DEVBUF);
 		else
 			free(cmd->rsasrc, M_DEVBUF);
 	}
 	free(cmd, M_DEVBUF);
 }
 
 static int
 xlp_get_rsa_opsize(struct xlp_rsa_command *cmd, unsigned int bits)
 {
 
 	if (bits == 0 || bits > 8192)
 		return (-1);
 	/* XLP hardware expects always a fixed size with unused bytes
 	 * zeroed out in the input data */
 	if (bits <= 512) {
 		cmd->rsatype = 0x40;
 		cmd->rsaopsize = 64;
 	} else if (bits <= 1024) {
 		cmd->rsatype = 0x41;
 		cmd->rsaopsize = 128;
 	} else if (bits <= 2048) {
 		cmd->rsatype = 0x42;
 		cmd->rsaopsize = 256;
 	} else if (bits <= 4096) {
 		cmd->rsatype = 0x43;
 		cmd->rsaopsize = 512;
 	} else if (bits <= 8192) {
 		cmd->rsatype = 0x44;
 		cmd->rsaopsize = 1024;
 	}
 	return (0);
 }
 
 static int
 xlp_rsa_inp2hwformat(uint8_t *src, uint8_t *dst, uint32_t paramsize,
     uint8_t result)
 {
 	uint32_t pdwords, pbytes;
 	int i, j, k;
 
 	pdwords = paramsize / 8;
 	pbytes = paramsize % 8;
 
 	for (i = 0, k = 0; i < pdwords; i++) {
 		/* copy dwords of inp/hw to hw/out format */
 		for (j = 7; j >= 0; j--, k++)
 			dst[i * 8 + j] = src[k];
 	}
 	if (pbytes) {
 		if (result == 0) {
 			/* copy rem bytes of input data to hw format */
 			for (j = 7; k < paramsize; j--, k++)
 				dst[i * 8 + j] = src[k];
 		} else {
 			/* copy rem bytes of hw data to exp output format */
 			for (j = 7; k < paramsize; j--, k++)
 				dst[k] = src[i * 8 + j];
 		}
 	}
 
 	return (0);
 }
 
 static int
 nlm_crypto_complete_rsa_request(struct xlp_rsa_softc *sc,
     struct xlp_rsa_command *cmd)
 {
 	unsigned int fbvc;
 	struct nlm_fmn_msg m;
 	int ret;
 
 	fbvc = nlm_cpuid() * 4 + XLPGE_FB_VC;
 
 	m.msg[0] = nlm_crypto_form_rsa_ecc_fmn_entry0(1, cmd->rsatype,
 	    cmd->rsafn, vtophys(cmd->rsasrc));
 	m.msg[1] = nlm_crypto_form_rsa_ecc_fmn_entry1(0, 1, fbvc,
 	    vtophys(cmd->rsasrc + cmd->rsaopsize * cmd->krp->krp_iparams));
 	/* Software scratch pad */
 	m.msg[2] = (uintptr_t)cmd;
 	m.msg[3] = 0;
 
 	/* Send the message to rsa engine vc */
 	ret = nlm_fmn_msgsend(sc->rsaecc_vc_start, 3, FMN_SWCODE_RSA, &m);
         if (ret != 0) {
 #ifdef NLM_SEC_DEBUG
                 printf("%s: msgsnd failed (%x)\n", __func__, ret);
 #endif
 		return (ERESTART);
         }
 	return (0);
 }
 
 static int
 xlp_rsa_kprocess(device_t dev, struct cryptkop *krp, int hint)
 {
 	struct xlp_rsa_softc *sc = device_get_softc(dev);
 	struct xlp_rsa_command *cmd;
 	struct crparam *kp;
 	int err, i;
 
 	if (krp == NULL || krp->krp_callback == NULL)
 		return (EINVAL);
 
 	cmd = malloc(sizeof(struct xlp_rsa_command), M_DEVBUF,
 	    M_NOWAIT | M_ZERO);
 	KASSERT(cmd != NULL, ("%s:cmd is NULL\n", __func__));
 	cmd->krp = krp;
 
 #ifdef NLM_RSA_DEBUG
 	print_krp_params(krp);
 #endif
 	err = EOPNOTSUPP;
 	switch (krp->krp_op) {
 	case CRK_MOD_EXP:
 		if (krp->krp_iparams == 3 && krp->krp_oparams == 1)
 			break;
 		goto errout;
 	default:
 		device_printf(dev, "Op:%d not yet supported\n", krp->krp_op);
 		goto errout;
 	}
 
 	err = xlp_get_rsa_opsize(cmd,
 	    krp->krp_param[krp->krp_iparams - 1].crp_nbits);
 	if (err != 0) {
 		err = EINVAL;
 		goto errout;
 	}
 	cmd->rsafn = 0; /* Mod Exp */
 	cmd->rsasrc = malloc(
 	    cmd->rsaopsize * (krp->krp_iparams + krp->krp_oparams),
 	    M_DEVBUF,
 	    M_NOWAIT | M_ZERO);
 	if (cmd->rsasrc == NULL) {
 		err = ENOMEM;
 		goto errout;
 	}
 
 	for (i = 0, kp = krp->krp_param; i < krp->krp_iparams; i++, kp++) {
 		KASSERT(kp->crp_nbits != 0,
 		    ("%s: parameter[%d]'s length is zero\n", __func__, i));
 		xlp_rsa_inp2hwformat(kp->crp_p,
 		    cmd->rsasrc + i * cmd->rsaopsize,
 		    howmany(kp->crp_nbits, 8), 0);
 	}
 	err = nlm_crypto_complete_rsa_request(sc, cmd);
 	if (err != 0)
 		goto errout;
 
 	return (0);
 errout:
 	xlp_free_cmd_params(cmd);
 	krp->krp_status = err;
 	crypto_kdone(krp);
 	return (err);
 }
Index: head/sys/mips/nlm/dev/sec/nlmsec.c
===================================================================
--- head/sys/mips/nlm/dev/sec/nlmsec.c	(revision 359373)
+++ head/sys/mips/nlm/dev/sec/nlmsec.c	(revision 359374)
@@ -1,792 +1,706 @@
 /*-
  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
  *
  * Copyright (c) 2003-2012 Broadcom Corporation
  * All Rights Reserved
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  *
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in
  *    the documentation and/or other materials provided with the
  *    distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY BROADCOM ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED. IN NO EVENT SHALL BROADCOM OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
  * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
  * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
  * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include <sys/cdefs.h>
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/proc.h>
 #include <sys/errno.h>
 #include <sys/malloc.h>
 #include <sys/kernel.h>
 #include <sys/module.h>
 #include <sys/mbuf.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <sys/sysctl.h>
 #include <sys/bus.h>
 #include <sys/random.h>
 #include <sys/rman.h>
 #include <sys/uio.h>
 #include <sys/kobj.h>
 
 #include <dev/pci/pcivar.h>
 
 #include <opencrypto/cryptodev.h>
+#include <opencrypto/xform_auth.h>
 
 #include "cryptodev_if.h"
 
 #include <vm/vm.h>
 #include <vm/pmap.h>
 
 #include <mips/nlm/hal/haldefs.h>
 #include <mips/nlm/hal/iomap.h>
 #include <mips/nlm/xlp.h>
 #include <mips/nlm/hal/sys.h>
 #include <mips/nlm/hal/fmn.h>
 #include <mips/nlm/hal/nlmsaelib.h>
 #include <mips/nlm/dev/sec/nlmseclib.h>
 #include <mips/nlm/hal/cop2.h>
 #include <mips/nlm/hal/mips-extns.h>
 #include <mips/nlm/msgring.h>
 
 unsigned int creditleft;
 
-void xlp_sec_print_data(struct cryptop *crp);
-
 static	int xlp_sec_init(struct xlp_sec_softc *sc);
-static	int xlp_sec_newsession(device_t , crypto_session_t, struct cryptoini *);
+static	int xlp_sec_probesession(device_t,
+    const struct crypto_session_params *);
+static	int xlp_sec_newsession(device_t , crypto_session_t,
+    const struct crypto_session_params *);
 static	int xlp_sec_process(device_t , struct cryptop *, int);
-static  int xlp_copyiv(struct xlp_sec_softc *, struct xlp_sec_command *,
-    struct cryptodesc *enccrd);
+static void xlp_copyiv(struct xlp_sec_softc *, struct xlp_sec_command *,
+    const struct crypto_session_params *);
 static int xlp_get_nsegs(struct cryptop *, unsigned int *);
 static int xlp_alloc_cmd_params(struct xlp_sec_command *, unsigned int);
 static void  xlp_free_cmd_params(struct xlp_sec_command *);
 
 static	int xlp_sec_probe(device_t);
 static	int xlp_sec_attach(device_t);
 static	int xlp_sec_detach(device_t);
 
 static device_method_t xlp_sec_methods[] = {
 	/* device interface */
 	DEVMETHOD(device_probe, xlp_sec_probe),
 	DEVMETHOD(device_attach, xlp_sec_attach),
 	DEVMETHOD(device_detach, xlp_sec_detach),
 
 	/* bus interface */
 	DEVMETHOD(bus_print_child, bus_generic_print_child),
 	DEVMETHOD(bus_driver_added, bus_generic_driver_added),
 
 	/* crypto device methods */
+	DEVMETHOD(cryptodev_probesession, xlp_sec_probesession),
 	DEVMETHOD(cryptodev_newsession, xlp_sec_newsession),
 	DEVMETHOD(cryptodev_process,    xlp_sec_process),
 
 	DEVMETHOD_END
 };
 
 static driver_t xlp_sec_driver = {
 	"nlmsec",
 	xlp_sec_methods,
 	sizeof(struct xlp_sec_softc)
 };
 static devclass_t xlp_sec_devclass;
 
 DRIVER_MODULE(nlmsec, pci, xlp_sec_driver, xlp_sec_devclass, 0, 0);
 MODULE_DEPEND(nlmsec, crypto, 1, 1, 1);
 
 void
 nlm_xlpsec_msgring_handler(int vc, int size, int code, int src_id,
     struct nlm_fmn_msg *msg, void *data);
 
 #ifdef NLM_SEC_DEBUG
 
 #define extract_bits(x, bitshift, bitcnt)				\
     (((unsigned long long)x >> bitshift) & ((1ULL << bitcnt) - 1))
 
 void
 print_crypto_params(struct xlp_sec_command *cmd, struct nlm_fmn_msg m)
 {
 	unsigned long long msg0,msg1,msg2,msg3,msg4,msg5,msg6,msg7,msg8;
 
 	msg0 = cmd->ctrlp->desc0;
 	msg1 = cmd->paramp->desc0;
 	msg2 = cmd->paramp->desc1;
 	msg3 = cmd->paramp->desc2;
 	msg4 = cmd->paramp->desc3;
 	msg5 = cmd->paramp->segment[0][0];
 	msg6 = cmd->paramp->segment[0][1];
 	msg7 = m.msg[0];
 	msg8 = m.msg[1];
 
 	printf("msg0 %llx msg1 %llx msg2 %llx msg3 %llx msg4 %llx msg5 %llx"
 	    "msg6 %llx msg7 %llx msg8 %llx\n", msg0, msg1, msg2, msg3, msg4,
 	    msg5, msg6, msg7, msg8);
 
 	printf("c0: hmac %d htype %d hmode %d ctype %d cmode %d arc4 %x\n",
 	    (unsigned int)extract_bits(msg0, 61, 1),
 	    (unsigned int)extract_bits(msg0, 52, 8),
 	    (unsigned int)extract_bits(msg0, 43, 8),
 	    (unsigned int)extract_bits(msg0, 34, 8),
 	    (unsigned int)extract_bits(msg0, 25, 8),
 	    (unsigned int)extract_bits(msg0, 0, 23));
 
 	printf("p0: tls %d hsrc %d hl3 %d enc %d ivl %d hd %llx\n",
 	    (unsigned int)extract_bits(msg1, 63, 1),
 	    (unsigned int)extract_bits(msg1,62,1),
 	    (unsigned int)extract_bits(msg1,60,1),
 	    (unsigned int)extract_bits(msg1,59,1),
 	    (unsigned int)extract_bits(msg1,41,16), extract_bits(msg1,0,40));
 
 	printf("p1: clen %u hl %u\n",  (unsigned int)extract_bits(msg2, 32, 32),
 	    (unsigned int)extract_bits(msg2,0,32));
 
 	printf("p2: ivoff %d cbit %d coff %d hbit %d hclb %d hoff %d\n",
 	    (unsigned int)extract_bits(msg3, 45, 17),
 	    (unsigned int)extract_bits(msg3, 42,3),
 	    (unsigned int)extract_bits(msg3, 22,16),
 	    (unsigned int)extract_bits(msg3, 19,3),
 	    (unsigned int)extract_bits(msg3, 18,1),
 	    (unsigned int)extract_bits(msg3, 0, 16));
 
 	printf("p3: desfbid %d tlen %d arc4 %x hmacpad %d\n",
 	    (unsigned int)extract_bits(msg4, 48,16),
 	    (unsigned int)extract_bits(msg4,11,16),
 	    (unsigned int)extract_bits(msg4,6,3),
 	    (unsigned int)extract_bits(msg4,5,1));
 
 	printf("p4: sflen %d sddr %llx \n",
 	    (unsigned int)extract_bits(msg5, 48, 16),extract_bits(msg5, 0, 40));
 
 	printf("p5: dflen %d cl3 %d cclob %d cdest %llx \n",
 	    (unsigned int)extract_bits(msg6, 48, 16),
 	    (unsigned int)extract_bits(msg6, 46, 1),
 	    (unsigned int)extract_bits(msg6, 41, 1), extract_bits(msg6, 0, 40));
 
 	printf("fmn0: fbid %d dfrlen %d dfrv %d cklen %d cdescaddr %llx\n",
 	    (unsigned int)extract_bits(msg7, 48, 16),
 	    (unsigned int)extract_bits(msg7,46,2),
 	    (unsigned int)extract_bits(msg7,45,1),
 	    (unsigned int)extract_bits(msg7,40,5),
 	    (extract_bits(msg7,0,34)<< 6));
 
 	printf("fmn1: arc4 %d hklen %d pdesclen %d pktdescad %llx\n",
 	    (unsigned int)extract_bits(msg8, 63, 1),
 	    (unsigned int)extract_bits(msg8,56,5),
 	    (unsigned int)extract_bits(msg8,43,12),
 	    (extract_bits(msg8,0,34) << 6));
 
 	return;
 }
 
 void
-xlp_sec_print_data(struct cryptop *crp)
-{
-	int i, key_len;
-	struct cryptodesc *crp_desc;
-
-	printf("session = %p, crp_ilen = %d, crp_olen=%d \n", crp->crp_session,
-	    crp->crp_ilen, crp->crp_olen);
-
-	printf("crp_flags = 0x%x\n", crp->crp_flags);
-
-	printf("crp buf:\n");
-	for (i = 0; i < crp->crp_ilen; i++) {
-		printf("%c  ", crp->crp_buf[i]);
-		if (i % 10 == 0)
-			printf("\n");
-	}
-
-	printf("\n");
-	printf("****************** desc ****************\n");
-	crp_desc = crp->crp_desc;
-	printf("crd_skip=%d, crd_len=%d, crd_flags=0x%x, crd_alg=%d\n",
-	    crp_desc->crd_skip, crp_desc->crd_len, crp_desc->crd_flags,
-	    crp_desc->crd_alg);
-
-	key_len = crp_desc->crd_klen / 8;
-	printf("key(%d) :\n", key_len);
-	for (i = 0; i < key_len; i++)
-		printf("%d", crp_desc->crd_key[i]);
-	printf("\n");
-
-	printf(" IV : \n");
-	for (i = 0; i < EALG_MAX_BLOCK_LEN; i++)
-		printf("%d", crp_desc->crd_iv[i]);
-	printf("\n");
-
-	printf("crd_next=%p\n", crp_desc->crd_next);
-	return;
-}
-
-void
 print_cmd(struct xlp_sec_command *cmd)
 {
 	printf("session_num		:%d\n",cmd->session_num);
 	printf("crp			:0x%x\n",(uint32_t)cmd->crp);
 	printf("enccrd			:0x%x\n",(uint32_t)cmd->enccrd);
 	printf("maccrd			:0x%x\n",(uint32_t)cmd->maccrd);
 	printf("ses			:%d\n",(uint32_t)cmd->ses);
 	printf("ctrlp			:0x%x\n",(uint32_t)cmd->ctrlp);
 	printf("paramp			:0x%x\n",(uint32_t)cmd->paramp);
 	printf("hashdest		:0x%x\n",(uint32_t)cmd->hashdest);
 	printf("hashsrc			:%d\n",cmd->hashsrc);
 	printf("hmacpad			:%d\n",cmd->hmacpad);
 	printf("hashoff			:%d\n",cmd->hashoff);
 	printf("hashlen			:%d\n",cmd->hashlen);
 	printf("cipheroff		:%d\n",cmd->cipheroff);
 	printf("cipherlen		:%d\n",cmd->cipherlen);
 	printf("ivoff			:%d\n",cmd->ivoff);
 	printf("ivlen			:%d\n",cmd->ivlen);
 	printf("hashalg			:%d\n",cmd->hashalg);
 	printf("hashmode		:%d\n",cmd->hashmode);
 	printf("cipheralg		:%d\n",cmd->cipheralg);
 	printf("ciphermode		:%d\n",cmd->ciphermode);
 	printf("nsegs     		:%d\n",cmd->nsegs);
 	printf("hash_dst_len		:%d\n",cmd->hash_dst_len);
 }
 #endif /* NLM_SEC_DEBUG */
 
 static int
 xlp_sec_init(struct xlp_sec_softc *sc)
 {
 
 	/* Register interrupt handler for the SEC CMS messages */
 	if (register_msgring_handler(sc->sec_vc_start,
 	    sc->sec_vc_end, nlm_xlpsec_msgring_handler, sc) != 0) {
 		printf("Couldn't register sec msgring handler\n");
 		return (-1);
 	}
 
 	/* Do the CMS credit initialization */
 	/* Currently it is configured by default to 50 when kernel comes up */
 
 	return (0);
 }
 
 /* This function is called from an interrupt handler */
 void
 nlm_xlpsec_msgring_handler(int vc, int size, int code, int src_id,
     struct nlm_fmn_msg *msg, void *data)
 {
 	struct xlp_sec_command *cmd = NULL;
 	struct xlp_sec_softc *sc = NULL;
-	struct cryptodesc *crd = NULL;
-	unsigned int ivlen = 0;
+	uint8_t hash[HASH_MAX_LEN];
 
 	KASSERT(code == FMN_SWCODE_CRYPTO,
 	    ("%s: bad code = %d, expected code = %d\n", __FUNCTION__,
 	    code, FMN_SWCODE_CRYPTO));
 
 	sc = (struct xlp_sec_softc *)data;
 	KASSERT(src_id >= sc->sec_vc_start && src_id <= sc->sec_vc_end,
 	    ("%s: bad src_id = %d, expect %d - %d\n", __FUNCTION__,
 	    src_id, sc->sec_vc_start, sc->sec_vc_end));
 
 	cmd = (struct xlp_sec_command *)(uintptr_t)msg->msg[0];
 	KASSERT(cmd != NULL && cmd->crp != NULL,
 		("%s :cmd not received properly\n",__FUNCTION__));
 
 	KASSERT(CRYPTO_ERROR(msg->msg[1]) == 0,
 	    ("%s: Message rcv msg0 %llx msg1 %llx err %x \n", __FUNCTION__,
 	    (unsigned long long)msg->msg[0], (unsigned long long)msg->msg[1],
 	    (int)CRYPTO_ERROR(msg->msg[1])));
 
-	crd = cmd->enccrd;
-	/* Copy the last 8 or 16 bytes to the session iv, so that in few
-	 * cases this will be used as IV for the next request
-	 */
-	if (crd != NULL) {
-		if ((crd->crd_alg == CRYPTO_DES_CBC ||
-		    crd->crd_alg == CRYPTO_3DES_CBC ||
-		    crd->crd_alg == CRYPTO_AES_CBC) &&
-		    (crd->crd_flags & CRD_F_ENCRYPT)) {
-			ivlen = ((crd->crd_alg == CRYPTO_AES_CBC) ?
-			    XLP_SEC_AES_IV_LENGTH : XLP_SEC_DES_IV_LENGTH);
-			crypto_copydata(cmd->crp->crp_flags, cmd->crp->crp_buf,
-			    crd->crd_skip + crd->crd_len - ivlen, ivlen,
-			    cmd->ses->ses_iv);
-		}
-	}
-
 	/* If there are not enough credits to send, then send request
 	 * will fail with ERESTART and the driver will be blocked until it is
 	 * unblocked here after knowing that there are sufficient credits to
 	 * send the request again.
 	 */
 	if (sc->sc_needwakeup) {
 		atomic_add_int(&creditleft, sc->sec_msgsz);
 		if (creditleft >= (NLM_CRYPTO_LEFT_REQS)) {
 			crypto_unblock(sc->sc_cid, sc->sc_needwakeup);
 			sc->sc_needwakeup &= (~(CRYPTO_SYMQ | CRYPTO_ASYMQ));
 		}
 	}
-	if(cmd->maccrd) {
-		crypto_copyback(cmd->crp->crp_flags,
-		    cmd->crp->crp_buf, cmd->maccrd->crd_inject,
-		    cmd->hash_dst_len, cmd->hashdest);
+	if (cmd->hash_dst_len != 0) {
+		if (cmd->crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
+			crypto_copydata(cmd->crp, cmd->crp->crp_digest_start,
+			    cmd->hash_dst_len, hash);
+			if (timingsafe_bcmp(cmd->hashdest, hash,
+			    cmd->hash_dst_len) != 0)
+				cmd->crp->crp_etype = EBADMSG;
+		} else
+			crypto_copyback(cmd->crp, cmd->crp->crp_digest_start,
+			    cmd->hash_dst_len, cmd->hashdest);
 	}
 
 	/* This indicates completion of the crypto operation */
 	crypto_done(cmd->crp);
 
 	xlp_free_cmd_params(cmd);
 
 	return;
 }
 
 static int
 xlp_sec_probe(device_t dev)
 {
 	struct xlp_sec_softc *sc;
 
 	if (pci_get_vendor(dev) == PCI_VENDOR_NETLOGIC &&
 	    pci_get_device(dev) == PCI_DEVICE_ID_NLM_SAE) {
 		sc = device_get_softc(dev);
 		return (BUS_PROBE_DEFAULT);
 	}
 	return (ENXIO);
 }
 
 /*
  * Attach an interface that successfully probed.
  */
 static int
 xlp_sec_attach(device_t dev)
 {
 	struct xlp_sec_softc *sc = device_get_softc(dev);
 	uint64_t base;
 	int qstart, qnum;
 	int freq, node;
 
 	sc->sc_dev = dev;
 
 	node = nlm_get_device_node(pci_get_slot(dev));
 	freq = nlm_set_device_frequency(node, DFS_DEVICE_SAE, 250);
 	if (bootverbose)
 		device_printf(dev, "SAE Freq: %dMHz\n", freq);
 	if(pci_get_device(dev) == PCI_DEVICE_ID_NLM_SAE) {
 		device_set_desc(dev, "XLP Security Accelerator");
 		sc->sc_cid = crypto_get_driverid(dev,
 		    sizeof(struct xlp_sec_session), CRYPTOCAP_F_HARDWARE);
 		if (sc->sc_cid < 0) {
 			printf("xlp_sec - error : could not get the driver"
 			    " id\n");
 			goto error_exit;
 		}
-		if (crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0) != 0)
-			printf("register failed for CRYPTO_DES_CBC\n");
 
-		if (crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0) != 0)
-			printf("register failed for CRYPTO_3DES_CBC\n");
-
-		if (crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0) != 0)
-			printf("register failed for CRYPTO_AES_CBC\n");
-
-		if (crypto_register(sc->sc_cid, CRYPTO_ARC4, 0, 0) != 0)
-			printf("register failed for CRYPTO_ARC4\n");
-
-		if (crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0) != 0)
-			printf("register failed for CRYPTO_MD5\n");
-
-		if (crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0) != 0)
-			printf("register failed for CRYPTO_SHA1\n");
-
-		if (crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0) != 0)
-			printf("register failed for CRYPTO_MD5_HMAC\n");
-
-		if (crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0) != 0)
-			printf("register failed for CRYPTO_SHA1_HMAC\n");
-
 		base = nlm_get_sec_pcibase(node);
 		qstart = nlm_qidstart(base);
 		qnum = nlm_qnum(base);
 		sc->sec_vc_start = qstart;
 		sc->sec_vc_end = qstart + qnum - 1;
 	}
 
 	if (xlp_sec_init(sc) != 0)
 		goto error_exit;
 	if (bootverbose)
 		device_printf(dev, "SEC Initialization complete!\n");
 	return (0);
 
 error_exit:
 	return (ENXIO);
 
 }
 
 /*
  * Detach an interface that successfully probed.
  */
 static int
 xlp_sec_detach(device_t dev)
 {
 	return (0);
 }
 
+static bool
+xlp_sec_auth_supported(const struct crypto_session_params *csp)
+{
+
+	switch (csp->csp_auth_alg) {
+	case CRYPTO_MD5:
+	case CRYPTO_SHA1:
+	case CRYPTO_MD5_HMAC:
+	case CRYPTO_SHA1_HMAC:
+		break;
+	default:
+		return (false);
+	}
+	return (true);
+}
+
+static bool
+xlp_sec_cipher_supported(const struct crypto_session_params *csp)
+{
+
+	switch (csp->csp_cipher_alg) {
+	case CRYPTO_DES_CBC:
+	case CRYPTO_3DES_CBC:
+		if (csp->csp_ivlen != XLP_SEC_DES_IV_LENGTH)
+			return (false);
+		break;
+	case CRYPTO_AES_CBC:
+		if (csp->csp_ivlen != XLP_SEC_AES_IV_LENGTH)
+			return (false);
+		break;
+	case CRYPTO_ARC4:
+		if (csp->csp_ivlen != XLP_SEC_ARC4_IV_LENGTH)
+			return (false);
+		break;
+	default:
+		return (false);
+	}
+
+	return (true);
+}
+
 static int
-xlp_sec_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
+xlp_sec_probesession(device_t dev, const struct crypto_session_params *csp)
 {
-	struct cryptoini *c;
-	struct xlp_sec_softc *sc = device_get_softc(dev);
-	int mac = 0, cry = 0;
-	struct xlp_sec_session *ses;
-	struct xlp_sec_command *cmd = NULL;
 
-	if (cri == NULL || sc == NULL)
+	if (csp->csp_flags != 0)
 		return (EINVAL);
+	switch (csp->csp_mode) {
+	case CSP_MODE_DIGEST:
+		if (!xlp_sec_auth_supported(csp))
+			return (EINVAL);
+		break;
+	case CSP_MODE_CIPHER:
+		if (!xlp_sec_cipher_supported(csp))
+			return (EINVAL);
+		break;
+	case CSP_MODE_ETA:
+		if (!xlp_sec_auth_supported(csp) ||
+		    !xlp_sec_cipher_supported(csp))
+			return (EINVAL);
+		break;
+	default:
+		return (EINVAL);
+	}
+	return (CRYPTODEV_PROBE_HARDWARE);
+}
 
+static int
+xlp_sec_newsession(device_t dev, crypto_session_t cses,
+    const struct crypto_session_params *csp)
+{
+	struct xlp_sec_session *ses;
+
 	ses = crypto_get_driver_session(cses);
-	cmd = &ses->cmd;
 
-	for (c = cri; c != NULL; c = c->cri_next) {
-		switch (c->cri_alg) {
-		case CRYPTO_MD5:
-		case CRYPTO_SHA1:
-		case CRYPTO_MD5_HMAC:
-		case CRYPTO_SHA1_HMAC:
-			if (mac)
-				return (EINVAL);
-			mac = 1;
-			ses->hs_mlen = c->cri_mlen;
-			if (ses->hs_mlen == 0) {
-				switch (c->cri_alg) {
-				case CRYPTO_MD5:
-				case CRYPTO_MD5_HMAC:
-					ses->hs_mlen = 16;
-					break;
-				case CRYPTO_SHA1:
-				case CRYPTO_SHA1_HMAC:
-					ses->hs_mlen = 20;
-					break;
-				}
-			}
-			break;
-		case CRYPTO_DES_CBC:
-		case CRYPTO_3DES_CBC:
-		case CRYPTO_AES_CBC:
-			/* XXX this may read fewer, does it matter? */
-			read_random(ses->ses_iv, c->cri_alg ==
-			    CRYPTO_AES_CBC ? XLP_SEC_AES_IV_LENGTH :
-			    XLP_SEC_DES_IV_LENGTH);
-			/* FALLTHROUGH */
-		case CRYPTO_ARC4:
-			if (cry)
-				return (EINVAL);
-			cry = 1;
-			break;
-		default:
-			return (EINVAL);
-		}
+	if (csp->csp_auth_alg != 0) {
+		if (csp->csp_auth_mlen == 0)
+			ses->hs_mlen = crypto_auth_hash(csp)->hashsize;
+		else
+			ses->hs_mlen = csp->csp_auth_mlen;
 	}
-	if (mac == 0 && cry == 0)
-		return (EINVAL);
 
-	cmd->hash_dst_len = ses->hs_mlen;
 	return (0);
 }
 
 /*
  * XXX freesession routine should run a zero'd mac/encrypt key into context
  * ram.  to blow away any keys already stored there.
  */
 
-static int
+static void
 xlp_copyiv(struct xlp_sec_softc *sc, struct xlp_sec_command *cmd,
-    struct cryptodesc *enccrd)
+    const struct crypto_session_params *csp)
 {
-	unsigned int ivlen = 0;
 	struct cryptop *crp = NULL;
 
 	crp = cmd->crp;
 
-	if (enccrd->crd_alg != CRYPTO_ARC4) {
-		ivlen = ((enccrd->crd_alg == CRYPTO_AES_CBC) ?
-		    XLP_SEC_AES_IV_LENGTH : XLP_SEC_DES_IV_LENGTH);
-		if (enccrd->crd_flags & CRD_F_ENCRYPT) {
-			if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) {
-				bcopy(enccrd->crd_iv, cmd->iv, ivlen);
-			} else {
-				bcopy(cmd->ses->ses_iv, cmd->iv, ivlen);
-			}
-			if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
-				crypto_copyback(crp->crp_flags,
-				    crp->crp_buf, enccrd->crd_inject,
-				    ivlen, cmd->iv);
-			}
-		} else {
-			if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) {
-				bcopy(enccrd->crd_iv, cmd->iv, ivlen);
-			} else {
-				crypto_copydata(crp->crp_flags, crp->crp_buf,
-				    enccrd->crd_inject, ivlen, cmd->iv);
-			}
-		}
+	if (csp->csp_cipher_alg != CRYPTO_ARC4) {
+		if (crp->crp_flags & CRYPTO_F_IV_GENERATE) {
+			arc4rand(cmd->iv, csp->csp_ivlen, 0);
+			crypto_copyback(crp, crp->crp_iv_start, csp->csp_ivlen,
+			    cmd->iv);
+		} else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+			memcpy(cmd->iv, crp->crp_iv, csp->csp_ivlen);
 	}
-	return (0);
 }
 
 static int
 xlp_get_nsegs(struct cryptop *crp, unsigned int *nsegs)
 {
 
-	if (crp->crp_flags & CRYPTO_F_IMBUF) {
+	switch (crp->crp_buf_type) {
+	case CRYPTO_BUF_MBUF:
+	{
 		struct mbuf *m = NULL;
 
-		m = (struct mbuf *)crp->crp_buf;
+		m = crp->crp_mbuf;
 		while (m != NULL) {
 			*nsegs += NLM_CRYPTO_NUM_SEGS_REQD(m->m_len);
 			m = m->m_next;
 		}
-	} else if (crp->crp_flags & CRYPTO_F_IOV) {
+		break;
+	}
+	case CRYPTO_BUF_UIO:
+	{
 		struct uio *uio = NULL;
 		struct iovec *iov = NULL;
 		int iol = 0;
 
 		uio = (struct uio *)crp->crp_buf;
 		iov = (struct iovec *)uio->uio_iov;
 		iol = uio->uio_iovcnt;
 		while (iol > 0) {
 			*nsegs += NLM_CRYPTO_NUM_SEGS_REQD(iov->iov_len);
 			iol--;
 			iov++;
 		}
-	} else {
+		break;
+	}
+	case CRYPTO_BUF_CONTIG:
 		*nsegs = NLM_CRYPTO_NUM_SEGS_REQD(crp->crp_ilen);
+		break;
+	default:
+		return (EINVAL);
 	}
 	return (0);
 }
 
 static int
 xlp_alloc_cmd_params(struct xlp_sec_command *cmd, unsigned int nsegs)
 {
 	int err = 0;
 
 	if(cmd == NULL) {
 		err = EINVAL;
 		goto error;
 	}
 	if ((cmd->ctrlp = malloc(sizeof(struct nlm_crypto_pkt_ctrl), M_DEVBUF,
 	    M_NOWAIT | M_ZERO)) == NULL) {
 		err = ENOMEM;
 		goto error;
 	}
 	if (((uintptr_t)cmd->ctrlp & (XLP_L2L3_CACHELINE_SIZE - 1))) {
 		err = EINVAL;
 		goto error;
 	}
 	/* (nsegs - 1) because one seg is part of the structure already */
 	if ((cmd->paramp = malloc(sizeof(struct nlm_crypto_pkt_param) +
 	    (16 * (nsegs - 1)), M_DEVBUF, M_NOWAIT | M_ZERO)) == NULL) {
 		err = ENOMEM;
 		goto error;
 	}
 	if (((uintptr_t)cmd->paramp & (XLP_L2L3_CACHELINE_SIZE - 1))) {
 		err = EINVAL;
 		goto error;
 	}
 	if ((cmd->iv = malloc(EALG_MAX_BLOCK_LEN, M_DEVBUF,
 	    M_NOWAIT | M_ZERO)) == NULL) {
 		err = ENOMEM;
 		goto error;
 	}
 	if ((cmd->hashdest = malloc(HASH_MAX_LEN, M_DEVBUF,
 	    M_NOWAIT | M_ZERO)) == NULL) {
 		err = ENOMEM;
 		goto error;
 	}
 error:
 	return (err);
 }
 
 static void
 xlp_free_cmd_params(struct xlp_sec_command *cmd)
 {
 	if (cmd->ctrlp != NULL)
 		free(cmd->ctrlp, M_DEVBUF);
 	if (cmd->paramp != NULL)
 		free(cmd->paramp, M_DEVBUF);
 	if (cmd->iv != NULL)
 		free(cmd->iv, M_DEVBUF);
 	if (cmd->hashdest != NULL)
 		free(cmd->hashdest, M_DEVBUF);
 	if (cmd != NULL)
 		free(cmd, M_DEVBUF);
 	return;
 }
 
 static int
 xlp_sec_process(device_t dev, struct cryptop *crp, int hint)
 {
 	struct xlp_sec_softc *sc = device_get_softc(dev);
+	const struct crypto_session_params *csp;
 	struct xlp_sec_command *cmd = NULL;
 	int err = -1, ret = 0;
-	struct cryptodesc *crd1, *crd2;
 	struct xlp_sec_session *ses;
 	unsigned int nsegs = 0;
 
-	if (crp == NULL || crp->crp_callback == NULL) {
-		return (EINVAL);
-	}
-	if (sc == NULL) {
-		err = EINVAL;
+	ses = crypto_get_driver_session(crp->crp_session);
+	csp = crypto_get_params(crp->crp_session);
+
+	/*
+	 * This device only support AAD requests where the AAD is
+	 * adjacent to the payload.
+	 */
+	if (crp->crp_aad_length != 0 && crp->crp_payload_start !=
+	    crp->crp_aad_start + crp->crp_aad_length) {
+		err = EFBIG;
 		goto errout;
 	}
-	ses = crypto_get_driver_session(crp->crp_session);
 
 	if ((cmd = malloc(sizeof(struct xlp_sec_command), M_DEVBUF,
 	    M_NOWAIT | M_ZERO)) == NULL) {
 		err = ENOMEM;
 		goto errout;
 	}
 
 	cmd->crp = crp;
 	cmd->ses = ses;
 	cmd->hash_dst_len = ses->hs_mlen;
 
-	if ((crd1 = crp->crp_desc) == NULL) {
-		err = EINVAL;
-		goto errout;
-	}
-	crd2 = crd1->crd_next;
-
 	if ((ret = xlp_get_nsegs(crp, &nsegs)) != 0) {
 		err = EINVAL;
 		goto errout;
 	}
-	if (((crd1 != NULL) && (crd1->crd_flags & CRD_F_IV_EXPLICIT)) ||
-	    ((crd2 != NULL) && (crd2->crd_flags & CRD_F_IV_EXPLICIT))) {
+
+	if (crp->crp_flags & CRYPTO_F_IV_SEPARATE) {
 		/* Since IV is given as separate segment to avoid copy */
 		nsegs += 1;
 	}
 	cmd->nsegs = nsegs;
 
 	if ((err = xlp_alloc_cmd_params(cmd, nsegs)) != 0)
 		goto errout;
 
-	if ((crd1 != NULL) && (crd2 == NULL)) {
-		if (crd1->crd_alg == CRYPTO_DES_CBC ||
-		    crd1->crd_alg == CRYPTO_3DES_CBC ||
-		    crd1->crd_alg == CRYPTO_AES_CBC ||
-		    crd1->crd_alg == CRYPTO_ARC4) {
-			cmd->enccrd = crd1;
-			cmd->maccrd = NULL;
-			if ((ret = nlm_get_cipher_param(cmd)) != 0) {
-				err = EINVAL;
-				goto errout;
-			}
-			if (crd1->crd_flags & CRD_F_IV_EXPLICIT)
-				cmd->cipheroff = cmd->ivlen;
-			else
-				cmd->cipheroff = cmd->enccrd->crd_skip;
-			cmd->cipherlen = cmd->enccrd->crd_len;
-			if (crd1->crd_flags & CRD_F_IV_PRESENT)
-				cmd->ivoff  = 0;
-			else
-				cmd->ivoff  = cmd->enccrd->crd_inject;
-			if ((err = xlp_copyiv(sc, cmd, cmd->enccrd)) != 0)
-				goto errout;
-			if ((err = nlm_crypto_do_cipher(sc, cmd)) != 0)
-				goto errout;
-		} else if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
-		    crd1->crd_alg == CRYPTO_SHA1_HMAC ||
-		    crd1->crd_alg == CRYPTO_SHA1 ||
-		    crd1->crd_alg == CRYPTO_MD5) {
-			cmd->enccrd = NULL;
-			cmd->maccrd = crd1;
-			if ((ret = nlm_get_digest_param(cmd)) != 0) {
-				err = EINVAL;
-				goto errout;
-			}
-			cmd->hashoff = cmd->maccrd->crd_skip;
-			cmd->hashlen = cmd->maccrd->crd_len;
-			cmd->hmacpad = 0;
-			cmd->hashsrc = 0;
-			if ((err = nlm_crypto_do_digest(sc, cmd)) != 0)
-				goto errout;
-		} else {
+	switch (csp->csp_mode) {
+	case CSP_MODE_CIPHER:
+		if ((ret = nlm_get_cipher_param(cmd, csp)) != 0) {
 			err = EINVAL;
 			goto errout;
 		}
-	} else if( (crd1 != NULL) && (crd2 != NULL) ) {
-		if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
-		    crd1->crd_alg == CRYPTO_SHA1_HMAC ||
-		    crd1->crd_alg == CRYPTO_MD5 ||
-		    crd1->crd_alg == CRYPTO_SHA1) &&
-		    (crd2->crd_alg == CRYPTO_DES_CBC ||
-		    crd2->crd_alg == CRYPTO_3DES_CBC ||
-		    crd2->crd_alg == CRYPTO_AES_CBC ||
-		    crd2->crd_alg == CRYPTO_ARC4)) {
-			cmd->maccrd = crd1;
-			cmd->enccrd = crd2;
-		} else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
-		    crd1->crd_alg == CRYPTO_ARC4 ||
-		    crd1->crd_alg == CRYPTO_3DES_CBC ||
-		    crd1->crd_alg == CRYPTO_AES_CBC) &&
-		    (crd2->crd_alg == CRYPTO_MD5_HMAC ||
-		    crd2->crd_alg == CRYPTO_SHA1_HMAC ||
-		    crd2->crd_alg == CRYPTO_MD5 ||
-		    crd2->crd_alg == CRYPTO_SHA1)) {
-			cmd->enccrd = crd1;
-			cmd->maccrd = crd2;
-		} else {
+		cmd->cipheroff = crp->crp_payload_start;
+		cmd->cipherlen = crp->crp_payload_length;
+		if (crp->crp_flags & CRYPTO_F_IV_SEPARATE) {
+			cmd->cipheroff += cmd->ivlen;
+			cmd->ivoff = 0;
+		} else
+			cmd->ivoff = crp->crp_iv_start;
+		xlp_copyiv(sc, cmd, csp);
+		if ((err = nlm_crypto_do_cipher(sc, cmd, csp)) != 0)
+			goto errout;
+		break;
+	case CSP_MODE_DIGEST:
+		if ((ret = nlm_get_digest_param(cmd, csp)) != 0) {
 			err = EINVAL;
 			goto errout;
 		}
-		if ((ret = nlm_get_cipher_param(cmd)) != 0) {
+		cmd->hashoff = crp->crp_payload_start;
+		cmd->hashlen = crp->crp_payload_length;
+		cmd->hmacpad = 0;
+		cmd->hashsrc = 0;
+		if ((err = nlm_crypto_do_digest(sc, cmd, csp)) != 0)
+			goto errout;
+		break;
+	case CSP_MODE_ETA:
+		if ((ret = nlm_get_cipher_param(cmd, csp)) != 0) {
 			err = EINVAL;
 			goto errout;
 		}
-		if ((ret = nlm_get_digest_param(cmd)) != 0) {
+		if ((ret = nlm_get_digest_param(cmd, csp)) != 0) {
 			err = EINVAL;
 			goto errout;
 		}
-		cmd->ivoff  = cmd->enccrd->crd_inject;
-		cmd->hashoff = cmd->maccrd->crd_skip;
-		cmd->hashlen = cmd->maccrd->crd_len;
+		if (crp->crp_aad_length != 0) {
+			cmd->hashoff = crp->crp_aad_start;
+			cmd->hashlen = crp->crp_aad_length +
+			    crp->crp_payload_length;
+		} else {
+			cmd->hashoff = crp->crp_payload_start;
+			cmd->hashlen = crp->crp_payload_length;
+		}
 		cmd->hmacpad = 0;
-		if (cmd->enccrd->crd_flags & CRD_F_ENCRYPT)
+		if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op))
 			cmd->hashsrc = 1;
 		else
 			cmd->hashsrc = 0;
-		cmd->cipheroff = cmd->enccrd->crd_skip;
-		cmd->cipherlen = cmd->enccrd->crd_len;
-		if ((err = xlp_copyiv(sc, cmd, cmd->enccrd)) != 0)
+		cmd->cipheroff = crp->crp_payload_start;
+		cmd->cipherlen = crp->crp_payload_length;
+		if (crp->crp_flags & CRYPTO_F_IV_SEPARATE) {
+			cmd->hashoff += cmd->ivlen;
+			cmd->cipheroff += cmd->ivlen;
+			cmd->ivoff = 0;
+		} else
+			cmd->ivoff = crp->crp_iv_start;
+		xlp_copyiv(sc, cmd, csp);
+		if ((err = nlm_crypto_do_cipher_digest(sc, cmd, csp)) != 0)
 			goto errout;
-		if ((err = nlm_crypto_do_cipher_digest(sc, cmd)) != 0)
-			goto errout;
-	} else {
+		break;
+	default:
 		err = EINVAL;
 		goto errout;
 	}
 	return (0);
 errout:
 	xlp_free_cmd_params(cmd);
 	if (err == ERESTART) {
 		sc->sc_needwakeup |= CRYPTO_SYMQ;
 		creditleft = 0;
 		return (err);
 	}
 	crp->crp_etype = err;
 	crypto_done(crp);
 	return (err);
 }
Index: head/sys/mips/nlm/dev/sec/nlmseclib.c
===================================================================
--- head/sys/mips/nlm/dev/sec/nlmseclib.c	(revision 359373)
+++ head/sys/mips/nlm/dev/sec/nlmseclib.c	(revision 359374)
@@ -1,309 +1,328 @@
 /*-
  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
  *
  * Copyright (c) 2003-2012 Broadcom Corporation
  * All Rights Reserved
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  *
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in
  *    the documentation and/or other materials provided with the
  *    distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY BROADCOM ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED. IN NO EVENT SHALL BROADCOM OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
  * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
  * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
  * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/types.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/module.h>
 #include <sys/malloc.h>
 #include <sys/mbuf.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <sys/bus.h>
 #include <sys/uio.h>
 #include <machine/bus.h>
 #include <machine/md_var.h>
 #include <machine/cpuregs.h>
 
 #include <vm/vm.h>
 #include <vm/pmap.h>
 
 #include <opencrypto/cryptodev.h>
 
 #include <mips/nlm/hal/haldefs.h>
 #include <mips/nlm/hal/cop2.h>
 #include <mips/nlm/hal/fmn.h>
 #include <mips/nlm/hal/mips-extns.h>
 #include <mips/nlm/hal/nlmsaelib.h>
 #include <mips/nlm/dev/sec/nlmseclib.h>
 
 static int
 nlm_crypto_complete_sec_request(struct xlp_sec_softc *sc,
     struct xlp_sec_command *cmd)
 {
 	unsigned int fbvc;
 	struct nlm_fmn_msg m;
 	int ret;
 
 	fbvc = nlm_cpuid() / CMS_MAX_VCPU_VC;
 	m.msg[0] = m.msg[1] = m.msg[2] = m.msg[3] = 0;
 
 	m.msg[0] = nlm_crypto_form_pkt_fmn_entry0(fbvc, 0, 0,
 	    cmd->ctrlp->cipherkeylen, vtophys(cmd->ctrlp));
 
 	m.msg[1] = nlm_crypto_form_pkt_fmn_entry1(0, cmd->ctrlp->hashkeylen,
 	    NLM_CRYPTO_PKT_DESC_SIZE(cmd->nsegs), vtophys(cmd->paramp));
 
 	/* Software scratch pad */
 	m.msg[2] = (uintptr_t)cmd;
 	sc->sec_msgsz = 3;
 
 	/* Send the message to sec/rsa engine vc */
 	ret = nlm_fmn_msgsend(sc->sec_vc_start, sc->sec_msgsz,
 	    FMN_SWCODE_CRYPTO, &m);
 	if (ret != 0) {
 #ifdef NLM_SEC_DEBUG
 		printf("%s: msgsnd failed (%x)\n", __func__, ret);
 #endif
 		return (ERESTART);
 	}
 	return (0);
 }
 
 int
-nlm_crypto_form_srcdst_segs(struct xlp_sec_command *cmd)
+nlm_crypto_form_srcdst_segs(struct xlp_sec_command *cmd,
+    const struct crypto_session_params *csp)
 {
 	unsigned int srcseg = 0, dstseg = 0;
-	struct cryptodesc *cipdesc = NULL;
 	struct cryptop *crp = NULL;
 
 	crp = cmd->crp;
-	cipdesc = cmd->enccrd;
 
-	if (cipdesc != NULL) {
+	if (csp->csp_mode != CSP_MODE_DIGEST) {
 		/* IV is given as ONE segment to avoid copy */
-		if (cipdesc->crd_flags & CRD_F_IV_EXPLICIT) {
+		if (crp->crp_flags & CRYPTO_F_IV_SEPARATE) {
 			srcseg = nlm_crypto_fill_src_seg(cmd->paramp, srcseg,
 			    cmd->iv, cmd->ivlen);
 			dstseg = nlm_crypto_fill_dst_seg(cmd->paramp, dstseg,
 			    cmd->iv, cmd->ivlen);
 		}
 	}
 
-	if (crp->crp_flags & CRYPTO_F_IMBUF) {
+	switch (crp->crp_buf_type) {
+	case CRYPTO_BUF_MBUF:
+	{
 		struct mbuf *m = NULL;
 
-		m  = (struct mbuf *)crp->crp_buf;
+		m = crp->crp_mbuf;
 		while (m != NULL) {
 			srcseg = nlm_crypto_fill_src_seg(cmd->paramp, srcseg,
 			    mtod(m,caddr_t), m->m_len);
-			if (cipdesc != NULL) {
+			if (csp->csp_mode != CSP_MODE_DIGEST) {
 				dstseg = nlm_crypto_fill_dst_seg(cmd->paramp,
 				    dstseg, mtod(m,caddr_t), m->m_len);
 			}
 			m = m->m_next;
 		}
-	} else if (crp->crp_flags & CRYPTO_F_IOV) {
+		break;
+	}
+	case CRYPTO_BUF_UIO:
+	{
 		struct uio *uio = NULL;
 		struct iovec *iov = NULL;
 	        int iol = 0;
 
-		uio = (struct uio *)crp->crp_buf;
-		iov = (struct iovec *)uio->uio_iov;
+		uio = crp->crp_uio;
+		iov = uio->uio_iov;
 		iol = uio->uio_iovcnt;
 
 		while (iol > 0) {
 			srcseg = nlm_crypto_fill_src_seg(cmd->paramp, srcseg,
 			    (caddr_t)iov->iov_base, iov->iov_len);
-			if (cipdesc != NULL) {
+			if (csp->csp_mode != CSP_MODE_DIGEST) {
 				dstseg = nlm_crypto_fill_dst_seg(cmd->paramp,
 				    dstseg, (caddr_t)iov->iov_base,
 				    iov->iov_len);
 			}
 			iov++;
 			iol--;
 		}
-	} else {
+	}
+	case CRYPTO_BUF_CONTIG:
 		srcseg = nlm_crypto_fill_src_seg(cmd->paramp, srcseg,
 		    ((caddr_t)crp->crp_buf), crp->crp_ilen);
-		if (cipdesc != NULL) {
+		if (csp->csp_mode != CSP_MODE_DIGEST) {
 			dstseg = nlm_crypto_fill_dst_seg(cmd->paramp, dstseg,
 			    ((caddr_t)crp->crp_buf), crp->crp_ilen);
 		}
+		break;
 	}
 	return (0);
 }
 
 int
-nlm_crypto_do_cipher(struct xlp_sec_softc *sc, struct xlp_sec_command *cmd)
+nlm_crypto_do_cipher(struct xlp_sec_softc *sc, struct xlp_sec_command *cmd,
+    const struct crypto_session_params *csp)
 {
-	struct cryptodesc *cipdesc = NULL;
-	unsigned char *cipkey = NULL;
+	const unsigned char *cipkey = NULL;
 	int ret = 0;
 
-	cipdesc = cmd->enccrd;
-	cipkey = (unsigned char *)cipdesc->crd_key;
+	if (cmd->crp->crp_cipher_key != NULL)
+		cipkey = cmd->crp->crp_cipher_key;
+	else
+		cipkey = csp->csp_cipher_key;
 	if (cmd->cipheralg == NLM_CIPHER_3DES) {
-		if (!(cipdesc->crd_flags & CRD_F_ENCRYPT)) {
-                        uint64_t *k, *tkey;
-			k = (uint64_t *)cipdesc->crd_key;
+		if (!CRYPTO_OP_IS_ENCRYPT(cmd->crp->crp_op)) {
+                        const uint64_t *k;
+                        uint64_t *tkey;
+			k = (const uint64_t *)cipkey;
 			tkey = (uint64_t *)cmd->des3key;
 			tkey[2] = k[0];
 			tkey[1] = k[1];
 			tkey[0] = k[2];
-			cipkey = (unsigned char *)tkey;
+			cipkey = (const unsigned char *)tkey;
 		}
 	}
 	nlm_crypto_fill_pkt_ctrl(cmd->ctrlp, 0, NLM_HASH_BYPASS, 0,
 	    cmd->cipheralg, cmd->ciphermode, cipkey,
-	    (cipdesc->crd_klen >> 3), NULL, 0);
+	    csp->csp_cipher_klen, NULL, 0);
 
 	nlm_crypto_fill_cipher_pkt_param(cmd->ctrlp, cmd->paramp,
-	    (cipdesc->crd_flags & CRD_F_ENCRYPT) ? 1 : 0, cmd->ivoff,
+	    CRYPTO_OP_IS_ENCRYPT(cmd->crp->crp_op) ? 1 : 0, cmd->ivoff,
 	    cmd->ivlen, cmd->cipheroff, cmd->cipherlen);
-	nlm_crypto_form_srcdst_segs(cmd);
+	nlm_crypto_form_srcdst_segs(cmd, csp);
 
 	ret = nlm_crypto_complete_sec_request(sc, cmd);
 	return (ret);
 }
 
 int
-nlm_crypto_do_digest(struct xlp_sec_softc *sc, struct xlp_sec_command *cmd)
+nlm_crypto_do_digest(struct xlp_sec_softc *sc, struct xlp_sec_command *cmd,
+    const struct crypto_session_params *csp)
 {
-	struct cryptodesc *digdesc = NULL;
+	const char *key;
 	int ret=0;
 
-	digdesc = cmd->maccrd;
-
-	nlm_crypto_fill_pkt_ctrl(cmd->ctrlp, (digdesc->crd_klen) ? 1 : 0,
+	if (cmd->crp->crp_auth_key != NULL)
+		key = cmd->crp->crp_auth_key;
+	else
+		key = csp->csp_auth_key;
+	nlm_crypto_fill_pkt_ctrl(cmd->ctrlp, csp->csp_auth_klen ? 1 : 0,
 	    cmd->hashalg, cmd->hashmode, NLM_CIPHER_BYPASS, 0,
-	    NULL, 0, digdesc->crd_key, digdesc->crd_klen >> 3);
+	    NULL, 0, key, csp->csp_auth_klen);
 
 	nlm_crypto_fill_auth_pkt_param(cmd->ctrlp, cmd->paramp,
 	    cmd->hashoff, cmd->hashlen, cmd->hmacpad,
 	    (unsigned char *)cmd->hashdest);
 
-	nlm_crypto_form_srcdst_segs(cmd);
+	nlm_crypto_form_srcdst_segs(cmd, csp);
 
 	ret = nlm_crypto_complete_sec_request(sc, cmd);
 
 	return (ret);
 }
 
 int
 nlm_crypto_do_cipher_digest(struct xlp_sec_softc *sc,
-    struct xlp_sec_command *cmd)
+    struct xlp_sec_command *cmd, const struct crypto_session_params *csp)
 {
-	struct cryptodesc *cipdesc=NULL, *digdesc=NULL;
-	unsigned char *cipkey = NULL;
+	const unsigned char *cipkey = NULL;
+	const char *authkey;
 	int ret=0;
 
-	cipdesc = cmd->enccrd;
-	digdesc = cmd->maccrd;
-
-	cipkey = (unsigned char *)cipdesc->crd_key;
+	if (cmd->crp->crp_cipher_key != NULL)
+		cipkey = cmd->crp->crp_cipher_key;
+	else
+		cipkey = csp->csp_cipher_key;
+	if (cmd->crp->crp_auth_key != NULL)
+		authkey = cmd->crp->crp_auth_key;
+	else
+		authkey = csp->csp_auth_key;
 	if (cmd->cipheralg == NLM_CIPHER_3DES) {
-		if (!(cipdesc->crd_flags & CRD_F_ENCRYPT)) {
-			uint64_t *k, *tkey;
-			k = (uint64_t *)cipdesc->crd_key;
+		if (!CRYPTO_OP_IS_ENCRYPT(cmd->crp->crp_op)) {
+			const uint64_t *k;
+			uint64_t *tkey;
+			k = (const uint64_t *)cipkey;
 			tkey = (uint64_t *)cmd->des3key;
 			tkey[2] = k[0];
 			tkey[1] = k[1];
 			tkey[0] = k[2];
-			cipkey = (unsigned char *)tkey;
+			cipkey = (const unsigned char *)tkey;
 		}
 	}
-	nlm_crypto_fill_pkt_ctrl(cmd->ctrlp, (digdesc->crd_klen) ? 1 : 0,
+	nlm_crypto_fill_pkt_ctrl(cmd->ctrlp, csp->csp_auth_klen ? 1 : 0,
 	    cmd->hashalg, cmd->hashmode, cmd->cipheralg, cmd->ciphermode,
-	    cipkey, (cipdesc->crd_klen >> 3),
-	    digdesc->crd_key, (digdesc->crd_klen >> 3));
+	    cipkey, csp->csp_cipher_klen,
+	    authkey, csp->csp_auth_klen);
 
 	nlm_crypto_fill_cipher_auth_pkt_param(cmd->ctrlp, cmd->paramp,
-	    (cipdesc->crd_flags & CRD_F_ENCRYPT) ? 1 : 0, cmd->hashsrc,
+	    CRYPTO_OP_IS_ENCRYPT(cmd->crp->crp_op) ? 1 : 0, cmd->hashsrc,
 	    cmd->ivoff, cmd->ivlen, cmd->hashoff, cmd->hashlen,
 	    cmd->hmacpad, cmd->cipheroff, cmd->cipherlen,
 	    (unsigned char *)cmd->hashdest);
 
-	nlm_crypto_form_srcdst_segs(cmd);
+	nlm_crypto_form_srcdst_segs(cmd, csp);
 
 	ret = nlm_crypto_complete_sec_request(sc, cmd);
 	return (ret);
 }
 
 int
-nlm_get_digest_param(struct xlp_sec_command *cmd)
+nlm_get_digest_param(struct xlp_sec_command *cmd,
+    const struct crypto_session_params *csp)
 {
-	switch(cmd->maccrd->crd_alg) {
+	switch(csp->csp_auth_alg) {
 	case CRYPTO_MD5:
 		cmd->hashalg  = NLM_HASH_MD5;
 		cmd->hashmode = NLM_HASH_MODE_SHA1;
 		break;
 	case CRYPTO_SHA1:
 		cmd->hashalg  = NLM_HASH_SHA;
 		cmd->hashmode = NLM_HASH_MODE_SHA1;
 		break;
 	case CRYPTO_MD5_HMAC:
 		cmd->hashalg  = NLM_HASH_MD5;
 		cmd->hashmode = NLM_HASH_MODE_SHA1;
 		break;
 	case CRYPTO_SHA1_HMAC:
 		cmd->hashalg  = NLM_HASH_SHA;
 		cmd->hashmode = NLM_HASH_MODE_SHA1;
 		break;
 	default:
 		/* Not supported */
 		return (-1);
 	}
 	return (0);
 }
 int
-nlm_get_cipher_param(struct xlp_sec_command *cmd)
+nlm_get_cipher_param(struct xlp_sec_command *cmd,
+    const struct crypto_session_params *csp)
 {
-	switch(cmd->enccrd->crd_alg) {
+	switch(csp->csp_cipher_alg) {
 	case CRYPTO_DES_CBC:
 		cmd->cipheralg  = NLM_CIPHER_DES;
 		cmd->ciphermode = NLM_CIPHER_MODE_CBC;
 		cmd->ivlen	= XLP_SEC_DES_IV_LENGTH;
 		break;
 	case CRYPTO_3DES_CBC:
 		cmd->cipheralg  = NLM_CIPHER_3DES;
 		cmd->ciphermode = NLM_CIPHER_MODE_CBC;
 		cmd->ivlen	= XLP_SEC_DES_IV_LENGTH;
 		break;
 	case CRYPTO_AES_CBC:
 		cmd->cipheralg  = NLM_CIPHER_AES128;
 		cmd->ciphermode = NLM_CIPHER_MODE_CBC;
 		cmd->ivlen	= XLP_SEC_AES_IV_LENGTH;
 		break;
 	case CRYPTO_ARC4:
 		cmd->cipheralg  = NLM_CIPHER_ARC4;
 		cmd->ciphermode = NLM_CIPHER_MODE_ECB;
 		cmd->ivlen	= XLP_SEC_ARC4_IV_LENGTH;
 		break;
 	default:
 		/* Not supported */
 		return (-1);
 	}
 	return (0);
 }
Index: head/sys/mips/nlm/dev/sec/nlmseclib.h
===================================================================
--- head/sys/mips/nlm/dev/sec/nlmseclib.h	(revision 359373)
+++ head/sys/mips/nlm/dev/sec/nlmseclib.h	(revision 359374)
@@ -1,151 +1,153 @@
 /*-
  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
  *
  * Copyright (c) 2003-2012 Broadcom Corporation
  * All Rights Reserved
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  *
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in
  *    the documentation and/or other materials provided with the
  *    distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY BROADCOM ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED. IN NO EVENT SHALL BROADCOM OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
  * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
  * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
  * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  * $FreeBSD$
  */
 
 #ifndef _NLMSECLIB_H_
 #define	_NLMSECLIB_H_
 
 /*
  * Cryptographic parameter definitions
  */
 #define	XLP_SEC_DES_KEY_LENGTH		8	/* Bytes */
 #define	XLP_SEC_3DES_KEY_LENGTH		24	/* Bytes */
 #define	XLP_SEC_AES128_KEY_LENGTH	16	/* Bytes */
 #define	XLP_SEC_AES192_KEY_LENGTH	24	/* Bytes */
 #define	XLP_SEC_AES256_KEY_LENGTH	32	/* Bytes */
 #define	XLP_SEC_AES128F8_KEY_LENGTH	32	/* Bytes */
 #define	XLP_SEC_AES192F8_KEY_LENGTH	48	/* Bytes */
 #define	XLP_SEC_AES256F8_KEY_LENGTH	64	/* Bytes */
 #define	XLP_SEC_KASUMI_F8_KEY_LENGTH	16	/* Bytes */
 #define	XLP_SEC_MAX_CRYPT_KEY_LENGTH	XLP_SEC_AES256F8_KEY_LENGTH
 
 
 #define	XLP_SEC_DES_IV_LENGTH		8	/* Bytes */
 #define	XLP_SEC_AES_IV_LENGTH		16	/* Bytes */
 #define	XLP_SEC_ARC4_IV_LENGTH		0	/* Bytes */
 #define	XLP_SEC_KASUMI_F8_IV_LENGTH	16	/* Bytes */
 #define	XLP_SEC_MAX_IV_LENGTH		16	/* Bytes */
 #define	XLP_SEC_IV_LENGTH_BYTES		8	/* Bytes */
 
 #define	XLP_SEC_AES_BLOCK_SIZE		16	/* Bytes */
 #define	XLP_SEC_DES_BLOCK_SIZE		8	/* Bytes */
 #define	XLP_SEC_3DES_BLOCK_SIZE		8	/* Bytes */
 
 #define	XLP_SEC_MD5_BLOCK_SIZE		64	/* Bytes */
 #define	XLP_SEC_SHA1_BLOCK_SIZE		64	/* Bytes */
 #define	XLP_SEC_SHA256_BLOCK_SIZE	64	/* Bytes */
 #define	XLP_SEC_SHA384_BLOCK_SIZE	128	/* Bytes */
 #define	XLP_SEC_SHA512_BLOCK_SIZE	128	/* Bytes */
 #define	XLP_SEC_GCM_BLOCK_SIZE		16	/* XXX: Bytes */
 #define	XLP_SEC_KASUMI_F9_BLOCK_SIZE	16	/* XXX: Bytes */
 #define	XLP_SEC_MAX_BLOCK_SIZE		64	/* Max of MD5/SHA */
 #define	XLP_SEC_MD5_LENGTH		16	/* Bytes */
 #define	XLP_SEC_SHA1_LENGTH		20	/* Bytes */
 #define	XLP_SEC_SHA256_LENGTH		32	/* Bytes */
 #define	XLP_SEC_SHA384_LENGTH		64	/* Bytes */
 #define	XLP_SEC_SHA512_LENGTH		64	/* Bytes */
 #define	XLP_SEC_GCM_LENGTH		16	/* Bytes */
 #define	XLP_SEC_KASUMI_F9_LENGTH	16	/* Bytes */
 #define	XLP_SEC_KASUMI_F9_RESULT_LENGTH	4	/* Bytes */
 #define	XLP_SEC_HMAC_LENGTH		64	/* Max of MD5/SHA/SHA256 */
 #define	XLP_SEC_MAX_AUTH_KEY_LENGTH	XLP_SEC_SHA512_BLOCK_SIZE
 #define	XLP_SEC_MAX_RC4_STATE_SIZE	264	/* char s[256], int i, int j */
 
 #define	CRYPTO_ERROR(msg1)	((unsigned int)msg1)
 
 #define	NLM_CRYPTO_LEFT_REQS (CMS_DEFAULT_CREDIT/2)
 #define	NLM_CRYPTO_NUM_SEGS_REQD(__bufsize)				\
 	((__bufsize + NLM_CRYPTO_MAX_SEG_LEN - 1) / NLM_CRYPTO_MAX_SEG_LEN)
 
 #define	NLM_CRYPTO_PKT_DESC_SIZE(nsegs) (32 + (nsegs * 16))
 
 extern unsigned int creditleft;
 
 struct xlp_sec_command {
 	struct cryptop *crp;
-	struct cryptodesc *enccrd, *maccrd;
 	struct xlp_sec_session *ses;
 	struct nlm_crypto_pkt_ctrl *ctrlp;
 	struct nlm_crypto_pkt_param *paramp;
 	void *iv;
 	uint8_t des3key[24];
 	uint8_t *hashdest;
 	uint8_t hashsrc;
 	uint8_t hmacpad;
 	uint32_t hashoff;
 	uint32_t hashlen;
 	uint32_t cipheroff;
 	uint32_t cipherlen;
 	uint32_t ivoff;
 	uint32_t ivlen;
 	uint32_t hashalg;
 	uint32_t hashmode;
 	uint32_t cipheralg;
 	uint32_t ciphermode;
 	uint32_t nsegs;
 	uint32_t hash_dst_len; /* used to store hash alg dst size */
 };
 
 struct xlp_sec_session {
 	int hs_mlen;
-	uint8_t ses_iv[EALG_MAX_BLOCK_LEN];
-	struct xlp_sec_command cmd;
 };
 
 /*
  * Holds data specific to nlm security accelerators
  */
 struct xlp_sec_softc {
 	device_t sc_dev;	/* device backpointer */
 	uint64_t sec_base;
 	int32_t sc_cid;
 	int sc_needwakeup;
 	uint32_t sec_vc_start;
 	uint32_t sec_vc_end;
 	uint32_t sec_msgsz;
 };
 
 #ifdef NLM_SEC_DEBUG
 void	print_crypto_params(struct xlp_sec_command *cmd, struct nlm_fmn_msg m);
-void	xlp_sec_print_data(struct cryptop *crp);
 void	print_cmd(struct xlp_sec_command *cmd);
 #endif
-int	nlm_crypto_form_srcdst_segs(struct xlp_sec_command *cmd);
+int	nlm_crypto_form_srcdst_segs(struct xlp_sec_command *cmd,
+	    const struct crypto_session_params *csp);
 int	nlm_crypto_do_cipher(struct xlp_sec_softc *sc,
-	    struct xlp_sec_command *cmd);
+	    struct xlp_sec_command *cmd,
+	    const struct crypto_session_params *csp);
 int	nlm_crypto_do_digest(struct xlp_sec_softc *sc,
-	    struct xlp_sec_command *cmd);
+	    struct xlp_sec_command *cmd,
+	    const struct crypto_session_params *csp);
 int	nlm_crypto_do_cipher_digest(struct xlp_sec_softc *sc,
-	    struct xlp_sec_command *cmd);
-int	nlm_get_digest_param(struct xlp_sec_command *cmd);
-int	nlm_get_cipher_param(struct xlp_sec_command *cmd);
+	    struct xlp_sec_command *cmd,
+	    const struct crypto_session_params *csp);
+int	nlm_get_digest_param(struct xlp_sec_command *cmd,
+	    const struct crypto_session_params *csp);
+int	nlm_get_cipher_param(struct xlp_sec_command *cmd,
+	    const struct crypto_session_params *csp);
 
 #endif /* _NLMSECLIB_H_ */
Index: head/sys/mips/nlm/hal/nlmsaelib.h
===================================================================
--- head/sys/mips/nlm/hal/nlmsaelib.h	(revision 359373)
+++ head/sys/mips/nlm/hal/nlmsaelib.h	(revision 359374)
@@ -1,607 +1,607 @@
 /*-
  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
  *
  * Copyright (c) 2003-2012 Broadcom Corporation
  * All Rights Reserved
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  *
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in
  *    the documentation and/or other materials provided with the
  *    distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY BROADCOM ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED. IN NO EVENT SHALL BROADCOM OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
  * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
  * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
  * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
  * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  * $FreeBSD$
  */
 
 #ifndef _NLM_HAL_CRYPTO_H_
 #define _NLM_HAL_CRYPTO_H_
 
 #define	SAE_CFG_REG		0x00
 #define SAE_ENG_SEL_0		0x01
 #define SAE_ENG_SEL_1		0x02
 #define SAE_ENG_SEL_2		0x03
 #define SAE_ENG_SEL_3		0x04
 #define SAE_ENG_SEL_4		0x05
 #define SAE_ENG_SEL_5		0x06
 #define SAE_ENG_SEL_6		0x07
 #define SAE_ENG_SEL_7		0x08
 
 #define	RSA_CFG_REG		0x00
 #define RSA_ENG_SEL_0		0x01
 #define RSA_ENG_SEL_1		0x02
 #define RSA_ENG_SEL_2		0x03
 
 #define nlm_read_sec_reg(b, r)		nlm_read_reg(b, r)
 #define nlm_write_sec_reg(b, r, v)	nlm_write_reg(b, r, v)
 #define nlm_get_sec_pcibase(node)	nlm_pcicfg_base(XLP_IO_SEC_OFFSET(node))
 #define nlm_get_sec_regbase(node)        \
                         (nlm_get_sec_pcibase(node) + XLP_IO_PCI_HDRSZ)
 
 #define nlm_read_rsa_reg(b, r)		nlm_read_reg(b, r)
 #define nlm_write_rsa_reg(b, r, v)	nlm_write_reg(b, r, v)
 #define nlm_get_rsa_pcibase(node)	nlm_pcicfg_base(XLP_IO_RSA_OFFSET(node))
 #define nlm_get_rsa_regbase(node)        \
                         (nlm_get_rsa_pcibase(node) + XLP_IO_PCI_HDRSZ)
 
 #define nlm_pcibase_sec(node)     nlm_pcicfg_base(XLP_IO_SEC_OFFSET(node))
 #define nlm_qidstart_sec(node)    nlm_qidstart_kseg(nlm_pcibase_sec(node))
 #define nlm_qnum_sec(node)        nlm_qnum_kseg(nlm_pcibase_sec(node))
 
 /*
  * Since buffer allocation for crypto at kernel is done as malloc, each
  * segment size is given as page size which is 4K by default
  */
 #define NLM_CRYPTO_MAX_SEG_LEN	PAGE_SIZE
 
 #define MAX_KEY_LEN_IN_DW		20
 
 #define left_shift64(x, bitshift, numofbits)			\
     ((uint64_t)(x) << (bitshift))
 
 #define left_shift64_mask(x, bitshift, numofbits)			\
     (((uint64_t)(x) & ((1ULL << (numofbits)) - 1)) << (bitshift))
 
 /**
 * @brief cipher algorithms
 * @ingroup crypto
 */
 enum nlm_cipher_algo {
 	NLM_CIPHER_BYPASS = 0,
 	NLM_CIPHER_DES = 1,
 	NLM_CIPHER_3DES = 2,
 	NLM_CIPHER_AES128 = 3,
 	NLM_CIPHER_AES192 = 4,
 	NLM_CIPHER_AES256 = 5,
 	NLM_CIPHER_ARC4 = 6,
 	NLM_CIPHER_KASUMI_F8 = 7,
 	NLM_CIPHER_SNOW3G_F8 = 8,
 	NLM_CIPHER_CAMELLIA128 = 9,
 	NLM_CIPHER_CAMELLIA192 = 0xA,
 	NLM_CIPHER_CAMELLIA256 = 0xB,
 	NLM_CIPHER_MAX = 0xC,
 };
 
 /**
 * @brief cipher modes
 * @ingroup crypto
 */
 enum nlm_cipher_mode {
 	NLM_CIPHER_MODE_ECB = 0,
 	NLM_CIPHER_MODE_CBC = 1,
 	NLM_CIPHER_MODE_CFB = 2,
 	NLM_CIPHER_MODE_OFB = 3,
 	NLM_CIPHER_MODE_CTR = 4,
 	NLM_CIPHER_MODE_AES_F8 = 5,
 	NLM_CIPHER_MODE_GCM = 6,
 	NLM_CIPHER_MODE_CCM = 7,
 	NLM_CIPHER_MODE_UNDEFINED1 = 8,
 	NLM_CIPHER_MODE_UNDEFINED2 = 9,
 	NLM_CIPHER_MODE_LRW = 0xA,
 	NLM_CIPHER_MODE_XTS = 0xB,
 	NLM_CIPHER_MODE_MAX = 0xC,
 };
 
 /**
 * @brief hash algorithms
 * @ingroup crypto
 */
 enum nlm_hash_algo {
 	NLM_HASH_BYPASS = 0,
 	NLM_HASH_MD5 = 1,
 	NLM_HASH_SHA = 2,
 	NLM_HASH_UNDEFINED = 3,
 	NLM_HASH_AES128 = 4,
 	NLM_HASH_AES192 = 5,
 	NLM_HASH_AES256 = 6,
 	NLM_HASH_KASUMI_F9 = 7,
 	NLM_HASH_SNOW3G_F9 = 8,
 	NLM_HASH_CAMELLIA128 = 9,
 	NLM_HASH_CAMELLIA192 = 0xA,
 	NLM_HASH_CAMELLIA256 = 0xB,
 	NLM_HASH_GHASH = 0xC,
 	NLM_HASH_MAX = 0xD
 };
 
 /**
 * @brief hash modes
 * @ingroup crypto
 */
 enum nlm_hash_mode {
 	NLM_HASH_MODE_SHA1 = 0,	/* Only SHA */
 	NLM_HASH_MODE_SHA224 = 1,	/* Only SHA */
 	NLM_HASH_MODE_SHA256 = 2,	/* Only SHA */
 	NLM_HASH_MODE_SHA384 = 3,	/* Only SHA */
 	NLM_HASH_MODE_SHA512 = 4,	/* Only SHA */
 	NLM_HASH_MODE_CMAC = 5,	/* AES and Camellia */
 	NLM_HASH_MODE_XCBC = 6,	/* AES and Camellia */
 	NLM_HASH_MODE_CBC_MAC = 7,	/* AES and Camellia */
 	NLM_HASH_MODE_CCM = 8,	/* AES */
 	NLM_HASH_MODE_GCM = 9,	/* AES */
 	NLM_HASH_MODE_MAX = 0xA,
 };
 
 /**
 * @brief crypto control descriptor, should be cache aligned
 * @ingroup crypto
 */
 struct nlm_crypto_pkt_ctrl {
 	uint64_t desc0;
 	/* combination of cipher and hash keys */
 	uint64_t key[MAX_KEY_LEN_IN_DW];
 	uint32_t cipherkeylen;
 	uint32_t hashkeylen;
 	uint32_t taglen;
 };
 
 /**
 * @brief crypto packet descriptor, should be cache aligned
 * @ingroup crypto
 */
 struct nlm_crypto_pkt_param {
 	uint64_t desc0;
 	uint64_t desc1;
 	uint64_t desc2;
 	uint64_t desc3;
 	uint64_t segment[1][2];
 };
 
 static __inline__ uint64_t
 nlm_crypto_form_rsa_ecc_fmn_entry0(unsigned int l3alloc, unsigned int type,
     unsigned int func, uint64_t srcaddr)
 {
 	return (left_shift64(l3alloc, 61, 1) |
 	    left_shift64(type, 46, 7) |
 	    left_shift64(func, 40, 6) |
 	    left_shift64(srcaddr, 0, 40));
 }
 
 static __inline__ uint64_t
 nlm_crypto_form_rsa_ecc_fmn_entry1(unsigned int dstclobber,
     unsigned int l3alloc, unsigned int fbvc, uint64_t dstaddr)
 {
 	return (left_shift64(dstclobber, 62, 1) |
 	    left_shift64(l3alloc, 61, 1) |
 	    left_shift64(fbvc, 40, 12) |
 	    left_shift64(dstaddr, 0, 40));
 }
 
 /**
 * @brief Generate cypto control descriptor
 * @ingroup crypto
 * hmac : 1 for hash with hmac
 * hashalg, see hash_alg enums
 * hashmode, see hash_mode enums
 * cipherhalg, see  cipher_alg enums
 * ciphermode, see  cipher_mode enums
 * arc4_cipherkeylen : length of arc4 cipher key, 0 is interpreted as 32
 * arc4_keyinit :
 * cfbmask : cipher text for feedback,
 *           0(1 bit), 1(2 bits), 2(4 bits), 3(8 bits), 4(16bits), 5(32 bits),
 *           6(64 bits), 7(128 bits)
 */
 static __inline__ uint64_t
 nlm_crypto_form_pkt_ctrl_desc(unsigned int hmac, unsigned int hashalg,
     unsigned int hashmode, unsigned int cipheralg, unsigned int ciphermode,
     unsigned int arc4_cipherkeylen, unsigned int arc4_keyinit,
     unsigned int cfbmask)
 {
 	return (left_shift64(hmac, 61, 1) |
 	    left_shift64(hashalg, 52, 8) |
 	    left_shift64(hashmode, 43, 8) |
 	    left_shift64(cipheralg, 34, 8) |
 	    left_shift64(ciphermode, 25, 8) |
 	    left_shift64(arc4_cipherkeylen, 18, 5) |
 	    left_shift64(arc4_keyinit, 17, 1) |
 	    left_shift64(cfbmask, 0, 3));
 }
 /**
 * @brief Generate cypto packet descriptor 0
 * @ingroup crypto
 * tls : 1 (tls enabled) 0(tls disabled)
 * hash_source : 1 (encrypted data is sent to the auth engine)
 *               0 (plain data is sent to the auth engine)
 * hashout_l3alloc : 1 (auth output is transited through l3 cache)
 * encrypt : 1 (for encrypt) 0 (for decrypt)
 * ivlen : iv length in bytes
 * hashdst_addr : hash out physical address, byte aligned
 */
 static __inline__ uint64_t
 nlm_crypto_form_pkt_desc0(unsigned int tls, unsigned int hash_source,
     unsigned int hashout_l3alloc, unsigned int encrypt, unsigned int ivlen,
     uint64_t hashdst_addr)
 {
 	return (left_shift64(tls, 63, 1) |
 	    left_shift64(hash_source, 62, 1) |
 	    left_shift64(hashout_l3alloc, 60, 1) |
 	    left_shift64(encrypt, 59, 1) |
 	    left_shift64_mask((ivlen - 1), 41, 16) |
 	    left_shift64(hashdst_addr, 0, 40));
 }
 
 /**
 * @brief Generate cypto packet descriptor 1
 * @ingroup crypto
 * cipherlen : cipher length in bytes
 * hashlen : hash length in bytes
 */
 static __inline__ uint64_t
 nlm_crypto_form_pkt_desc1(unsigned int cipherlen, unsigned int hashlen)
 {
 	return (left_shift64_mask((cipherlen - 1), 32, 32) |
 	    left_shift64_mask((hashlen - 1), 0, 32));
 }
 
 /**
 * @brief Generate cypto packet descriptor 2
 * @ingroup crypto
 * ivoff : iv offset, offset from start of src data addr
 * ciperbit_cnt : number of valid bits in the last input byte to the cipher,
 *                0 (8 bits), 1 (1 bit)..7 (7 bits)
 * cipheroff : cipher offset, offset from start of src data addr
 * hashbit_cnt : number of valid bits in the last input byte to the auth
 *              0 (8 bits), 1 (1 bit)..7 (7 bits)
 * hashclobber : 1 (hash output will be written as multiples of cachelines, no
 *              read modify write)
 * hashoff : hash offset, offset from start of src data addr
 */
 
 static __inline__ uint64_t
 nlm_crypto_form_pkt_desc2(unsigned int ivoff, unsigned int cipherbit_cnt,
     unsigned int cipheroff, unsigned int hashbit_cnt, unsigned int hashclobber,
     unsigned int hashoff)
 {
 	return (left_shift64(ivoff , 45, 16) |
 	    left_shift64(cipherbit_cnt, 42, 3) |
 	    left_shift64(cipheroff, 22, 16) |
 	    left_shift64(hashbit_cnt, 19, 3) |
 	    left_shift64(hashclobber, 18, 1) |
 	    left_shift64(hashoff, 0, 16));
 }
 
 /**
 * @brief Generate cypto packet descriptor 3
 * @ingroup crypto
 * designer_vc : designer freeback fmn destination id
 * taglen : length in bits of the tag generated by the auth engine
 *          md5 (128 bits), sha1 (160), sha224 (224), sha384 (384),
 *          sha512 (512), Kasumi (32), snow3g (32), gcm (128)
 * hmacpad : 1 if hmac padding is already done
 */
 static  __inline__ uint64_t
 nlm_crypto_form_pkt_desc3(unsigned int designer_vc, unsigned int taglen,
     unsigned int arc4_state_save_l3, unsigned int arc4_save_state,
     unsigned int hmacpad)
 {
 	return (left_shift64(designer_vc, 48, 16) |
 	    left_shift64(taglen, 11, 16) |
 	    left_shift64(arc4_state_save_l3, 8, 1) |
 	    left_shift64(arc4_save_state, 6, 1) |
 	    left_shift64(hmacpad, 5, 1));
 }
 
 /**
 * @brief Generate cypto packet descriptor 4
 * @ingroup crypto
 * srcfraglen : length of the source fragment(header + data + tail) in bytes
 * srcfragaddr : physical address of the srouce fragment
 */
 static __inline__ uint64_t
 nlm_crypto_form_pkt_desc4(uint64_t srcfraglen,
     unsigned int srcfragaddr )
 {
 	return (left_shift64_mask((srcfraglen - 1), 48, 16) |
 	    left_shift64(srcfragaddr, 0, 40));
 }
 
 /**
 * @brief Generate cypto packet descriptor 5
 * @ingroup crypto
 * dstfraglen : length of the dst fragment(header + data + tail) in bytes
 * chipherout_l3alloc : 1(cipher output is transited through l3 cache)
 * cipherclobber : 1 (cipher output will be written as multiples of cachelines,
 *                 no read modify write)
 * chiperdst_addr : physical address of the cipher destination address
 */
 static __inline__ uint64_t
 nlm_crypto_form_pkt_desc5(unsigned int dstfraglen,
     unsigned int cipherout_l3alloc, unsigned int cipherclobber,
     uint64_t cipherdst_addr)
 
 {
 	return (left_shift64_mask((dstfraglen - 1), 48, 16) |
 	    left_shift64(cipherout_l3alloc, 46, 1) |
 	    left_shift64(cipherclobber, 41, 1) |
 	    left_shift64(cipherdst_addr, 0, 40));
 }
 
 /**
   * @brief Generate crypto packet fmn message entry 0
   * @ingroup crypto
   * freeback_vc: freeback response destination address
   * designer_fblen : Designer freeback length, 1 - 4
   * designerdesc_valid : designer desc valid or not
   * cipher_keylen : cipher key length in bytes
   * ctrldesc_addr : physicall address of the control descriptor
   */
 static __inline__ uint64_t
 nlm_crypto_form_pkt_fmn_entry0(unsigned int freeback_vc,
     unsigned int designer_fblen, unsigned int designerdesc_valid,
     unsigned int cipher_keylen, uint64_t cntldesc_addr)
 {
 	return (left_shift64(freeback_vc, 48, 16) |
 	    left_shift64_mask(designer_fblen - 1, 46, 2) |
 	    left_shift64(designerdesc_valid, 45, 1) |
 	    left_shift64_mask(((cipher_keylen + 7) >> 3), 40, 5) |
 	    left_shift64(cntldesc_addr >> 6, 0, 34));
 }
 
 /**
   * @brief Generate crypto packet fmn message entry 1
   * @ingroup crypto
   * arc4load_state : 1 if load state required 0 otherwise
   * hash_keylen : hash key length in bytes
   * pktdesc_size : packet descriptor size in bytes
   * pktdesc_addr : physicall address of the packet descriptor
   */
 static __inline__ uint64_t
 nlm_crypto_form_pkt_fmn_entry1(unsigned int arc4load_state,
     unsigned int hash_keylen, unsigned int pktdesc_size,
     uint64_t pktdesc_addr)
 {
 	return (left_shift64(arc4load_state, 63, 1) |
 	    left_shift64_mask(((hash_keylen + 7) >> 3), 56, 5) |
 	    left_shift64_mask(((pktdesc_size >> 4) - 1), 43, 12) |
 	    left_shift64(pktdesc_addr >> 6, 0, 34));
 }
 
 static __inline__ int
 nlm_crypto_get_hklen_taglen(enum nlm_hash_algo hashalg,
     enum nlm_hash_mode hashmode, unsigned int *taglen, unsigned int *hklen)
 {
 	if (hashalg == NLM_HASH_MD5) {
 		*taglen = 128;
 		*hklen  = 64;
 	} else if (hashalg == NLM_HASH_SHA) {
 		switch (hashmode) {
 		case NLM_HASH_MODE_SHA1:
 			*taglen = 160;
 			*hklen  = 64;
 			break;
 		case NLM_HASH_MODE_SHA224:
 			*taglen = 224;
 			*hklen  = 64;
 			break;
 		case NLM_HASH_MODE_SHA256:
 			*taglen = 256;
 			*hklen  = 64;
 			break;
 		case NLM_HASH_MODE_SHA384:
 			*taglen = 384;
 			*hklen  = 128;
 			break;
 		case NLM_HASH_MODE_SHA512:
 			*taglen = 512;
 			*hklen  = 128;
 			break;
 		default:
 			printf("Error : invalid shaid (%s)\n", __func__);
 			return (-1);
 		}
 	} else if (hashalg == NLM_HASH_KASUMI_F9) {
 		*taglen = 32;
 		*hklen  = 0;
 	} else if (hashalg == NLM_HASH_SNOW3G_F9) {
 		*taglen = 32;
 		*hklen  = 0;
 	} else if (hashmode == NLM_HASH_MODE_XCBC) {
 		*taglen = 128;
 		*hklen  = 0;
 	} else if (hashmode == NLM_HASH_MODE_GCM) {
 		*taglen = 128;
 		*hklen  = 0;
 	} else if (hashalg == NLM_HASH_BYPASS) {
 		*taglen = 0;
 		*hklen  = 0;
 	} else {
 		printf("Error:Hash alg/mode not found\n");
 		return (-1);
 	}
 
 	/* TODO : Add remaining cases */
 	return (0);
 }
 
 /**
 * @brief Generate fill cryto control info structure
 * @ingroup crypto
 * hmac : 1 for hash with hmac
 * hashalg: see above,  hash_alg enums
 * hashmode: see above, hash_mode enums
 * cipherhalg: see above,  cipher_alg enums
 * ciphermode: see above, cipher_mode enums
 *
 */
 static __inline__ int
 nlm_crypto_fill_pkt_ctrl(struct nlm_crypto_pkt_ctrl *ctrl, unsigned int hmac,
     enum nlm_hash_algo hashalg, enum nlm_hash_mode hashmode,
     enum nlm_cipher_algo cipheralg, enum nlm_cipher_mode ciphermode,
-    unsigned char *cipherkey, unsigned int cipherkeylen,
-    unsigned char *hashkey, unsigned int hashkeylen)
+    const unsigned char *cipherkey, unsigned int cipherkeylen,
+    const unsigned char *hashkey, unsigned int hashkeylen)
 {
 	unsigned int taglen = 0, hklen = 0;
 
 	ctrl->desc0 = nlm_crypto_form_pkt_ctrl_desc(hmac, hashalg, hashmode,
 	    cipheralg, ciphermode, 0, 0, 0);
 	memset(ctrl->key, 0, sizeof(ctrl->key));
 	if (cipherkey)
 		memcpy(ctrl->key, cipherkey, cipherkeylen);
 	if (hashkey)
 		memcpy((unsigned char *)&ctrl->key[(cipherkeylen + 7) / 8],
 			    hashkey, hashkeylen);
 	if (nlm_crypto_get_hklen_taglen(hashalg, hashmode, &taglen, &hklen)
 	    < 0)
 		return (-1);
 
 	ctrl->cipherkeylen = cipherkeylen;
 	ctrl->hashkeylen = hklen;
 	ctrl->taglen = taglen;
 
 	/* TODO : add the invalid checks and return error */
 	return (0);
 }
 
 /**
 * @brief Top level function for generation pkt desc 0 to 3 for cipher auth
 * @ingroup crypto
 * ctrl : pointer to control structure
 * param : pointer to the param structure
 * encrypt : 1(for encrypt) 0(for decrypt)
 * hash_source : 1(encrypted data is sent to the auth engine) 0(plain data is
 *		sent to the auth engine)
 * ivoff : iv offset from start of data
 * ivlen : iv length in bytes
 * hashoff : hash offset from start of data
 * hashlen : hash length in bytes
 * hmacpad : hmac padding required or not, 1 if already padded
 * cipheroff : cipher offset from start of data
 * cipherlen : cipher length in bytes
 * hashdst_addr : hash destination physical address
 */
 static __inline__ void
 nlm_crypto_fill_cipher_auth_pkt_param(struct nlm_crypto_pkt_ctrl *ctrl,
     struct nlm_crypto_pkt_param *param, unsigned int encrypt,
     unsigned int hash_source, unsigned int ivoff, unsigned int ivlen,
     unsigned int hashoff, unsigned int hashlen, unsigned int hmacpad,
     unsigned int cipheroff, unsigned int cipherlen, unsigned char *hashdst_addr)
 {
 	param->desc0 = nlm_crypto_form_pkt_desc0(0, hash_source, 1, encrypt,
 			   ivlen, vtophys(hashdst_addr));
 	param->desc1 = nlm_crypto_form_pkt_desc1(cipherlen, hashlen);
 	param->desc2 = nlm_crypto_form_pkt_desc2(ivoff, 0, cipheroff, 0, 0,
 			   hashoff);
 	param->desc3 = nlm_crypto_form_pkt_desc3(0, ctrl->taglen, 0, 0,
 			   hmacpad);
 }
 
 /**
 * @brief Top level function for generation pkt desc 0 to 3 for cipher operation
 * @ingroup crypto
 * ctrl : pointer to control structure
 * param : pointer to the param structure
 * encrypt : 1(for encrypt) 0(for decrypt)
 * ivoff : iv offset from start of data
 * ivlen : iv length in bytes
 * cipheroff : cipher offset from start of data
 * cipherlen : cipher length in bytes
 */
 static __inline__ void
 nlm_crypto_fill_cipher_pkt_param(struct nlm_crypto_pkt_ctrl *ctrl,
     struct nlm_crypto_pkt_param *param, unsigned int encrypt,
     unsigned int ivoff, unsigned int ivlen, unsigned int cipheroff,
     unsigned int cipherlen)
 {
 	param->desc0 = nlm_crypto_form_pkt_desc0(0, 0, 0, encrypt, ivlen, 0ULL);
 	param->desc1 = nlm_crypto_form_pkt_desc1(cipherlen, 1);
 	param->desc2 = nlm_crypto_form_pkt_desc2(ivoff, 0, cipheroff, 0, 0, 0);
 	param->desc3 = nlm_crypto_form_pkt_desc3(0, ctrl->taglen, 0, 0, 0);
 }
 
 /**
 * @brief Top level function for generation pkt desc 0 to 3 for auth operation
 * @ingroup crypto
 * ctrl : pointer to control structure
 * param : pointer to the param structure
 * hashoff : hash offset from start of data
 * hashlen : hash length in bytes
 * hmacpad : hmac padding required or not, 1 if already padded
 * hashdst_addr : hash destination physical address
 */
 static __inline__ void
 nlm_crypto_fill_auth_pkt_param(struct nlm_crypto_pkt_ctrl *ctrl,
     struct nlm_crypto_pkt_param *param, unsigned int hashoff,
     unsigned int hashlen, unsigned int hmacpad, unsigned char *hashdst_addr)
 {
 	param->desc0 = nlm_crypto_form_pkt_desc0(0, 0, 1, 0, 1,
 			   vtophys(hashdst_addr));
 	param->desc1 = nlm_crypto_form_pkt_desc1(1, hashlen);
 	param->desc2 = nlm_crypto_form_pkt_desc2(0, 0, 0, 0, 0, hashoff);
 	param->desc3 = nlm_crypto_form_pkt_desc3(0, ctrl->taglen, 0, 0,
 			   hmacpad);
 }
 
 static __inline__ unsigned int
 nlm_crypto_fill_src_seg(struct nlm_crypto_pkt_param *param, int seg,
     unsigned char *input, unsigned int inlen)
 {
 	unsigned off = 0, len = 0;
 	unsigned int remlen = inlen;
 
 	for (; remlen > 0;) {
 		len = remlen > NLM_CRYPTO_MAX_SEG_LEN ?
 		    NLM_CRYPTO_MAX_SEG_LEN : remlen;
 		param->segment[seg][0] = nlm_crypto_form_pkt_desc4(len,
 		    vtophys(input + off));
 		remlen -= len;
 		off += len;
 		seg++;
 	}
 	return (seg);
 }
 
 static __inline__ unsigned int
 nlm_crypto_fill_dst_seg(struct nlm_crypto_pkt_param *param,
 		int seg, unsigned char *output, unsigned int outlen)
 {
 	unsigned off = 0, len = 0;
 	unsigned int remlen = outlen;
 
 	for (; remlen > 0;) {
 		len = remlen > NLM_CRYPTO_MAX_SEG_LEN ?
 		    NLM_CRYPTO_MAX_SEG_LEN : remlen;
 		param->segment[seg][1] = nlm_crypto_form_pkt_desc5(len, 1, 0,
 		    vtophys(output + off));
 		remlen -= len;
 		off += len;
 		seg++;
 	}
 	return (seg);
 }
 
 #endif
Index: head/sys/netipsec/xform.h
===================================================================
--- head/sys/netipsec/xform.h	(revision 359373)
+++ head/sys/netipsec/xform.h	(revision 359374)
@@ -1,121 +1,122 @@
 /*	$FreeBSD$	*/
 /*	$OpenBSD: ip_ipsp.h,v 1.119 2002/03/14 01:27:11 millert Exp $	*/
 /*-
  * The authors of this code are John Ioannidis (ji@tla.org),
  * Angelos D. Keromytis (kermit@csd.uch.gr),
  * Niels Provos (provos@physnet.uni-hamburg.de) and
  * Niklas Hallqvist (niklas@appli.se).
  *
  * The original version of this code was written by John Ioannidis
  * for BSD/OS in Athens, Greece, in November 1995.
  *
  * Ported to OpenBSD and NetBSD, with additional transforms, in December 1996,
  * by Angelos D. Keromytis.
  *
  * Additional transforms and features in 1997 and 1998 by Angelos D. Keromytis
  * and Niels Provos.
  *
  * Additional features in 1999 by Angelos D. Keromytis and Niklas Hallqvist.
  *
  * Copyright (c) 1995, 1996, 1997, 1998, 1999 by John Ioannidis,
  * Angelos D. Keromytis and Niels Provos.
  * Copyright (c) 1999 Niklas Hallqvist.
  * Copyright (c) 2001, Angelos D. Keromytis.
  *
  * Permission to use, copy, and modify this software with or without fee
  * is hereby granted, provided that this entire notice is included in
  * all copies of any software which is or includes a copy or
  * modification of this software.
  * You may use this code under the GNU public license if you so wish. Please
  * contribute changes back to the authors under this freer than GPL license
  * so that we may further the use of strong encryption without limitations to
  * all.
  *
  * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR
  * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY
  * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE
  * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR
  * PURPOSE.
  */
 
 #ifndef _NETIPSEC_XFORM_H_
 #define _NETIPSEC_XFORM_H_
 
 #include <sys/types.h>
 #include <sys/queue.h>
 #include <netinet/in.h>
 #include <opencrypto/xform.h>
 
 #define	AH_HMAC_HASHLEN		12	/* 96 bits of authenticator */
 #define	AH_HMAC_MAXHASHLEN	(SHA2_512_HASH_LEN/2)	/* Keep this updated */
 #define	AH_HMAC_INITIAL_RPL	1	/* replay counter initial value */
 
 #ifdef _KERNEL
 struct secpolicy;
 struct secasvar;
 
 /*
  * Packet tag assigned on completion of IPsec processing; used
  * to speedup security policy checking for INBOUND packets.
  */
 struct xform_history {
 	union sockaddr_union	dst;		/* destination address */
 	uint32_t		spi;		/* Security Parameters Index */
 	uint8_t			proto;		/* IPPROTO_ESP or IPPROTO_AH */
 	uint8_t			mode;		/* transport or tunnel */
 };
 
 /*
  * Opaque data structure hung off a crypto operation descriptor.
  */
 struct xform_data {
 	struct secpolicy	*sp;		/* security policy */
 	struct secasvar		*sav;		/* related SA */
 	crypto_session_t	cryptoid;	/* used crypto session */
 	u_int			idx;		/* IPsec request index */
 	int			protoff;	/* current protocol offset */
 	int			skip;		/* data offset */
 	uint8_t			nxt;		/* next protocol, e.g. IPV4 */
 	struct vnet		*vnet;
 };
 
 #define	XF_IP4		1	/* unused */
 #define	XF_AH		2	/* AH */
 #define	XF_ESP		3	/* ESP */
 #define	XF_TCPSIGNATURE	5	/* TCP MD5 Signature option, RFC 2358 */
 #define	XF_IPCOMP	6	/* IPCOMP */
 
 struct xformsw {
 	u_short			xf_type;	/* xform ID */
 	const char		*xf_name;	/* human-readable name */
 	int	(*xf_init)(struct secasvar*, struct xformsw*);	/* setup */
 	int	(*xf_zeroize)(struct secasvar*);		/* cleanup */
 	int	(*xf_input)(struct mbuf*, struct secasvar*,	/* input */
 			int, int);
 	int	(*xf_output)(struct mbuf*,			/* output */
 	    struct secpolicy *, struct secasvar *, u_int, int, int);
 
 	volatile u_int		xf_cntr;
 	LIST_ENTRY(xformsw)	chain;
 };
 
 const struct enc_xform * enc_algorithm_lookup(int);
 const struct auth_hash * auth_algorithm_lookup(int);
 const struct comp_algo * comp_algorithm_lookup(int);
 
 void xform_attach(void *);
 void xform_detach(void *);
 int xform_init(struct secasvar *, u_short);
 
-struct cryptoini;
+struct crypto_session_params;
 /* XF_AH */
 int xform_ah_authsize(const struct auth_hash *);
-extern int ah_init0(struct secasvar *, struct xformsw *, struct cryptoini *);
+int ah_init0(struct secasvar *, struct xformsw *,
+    struct crypto_session_params *);
 extern int ah_zeroize(struct secasvar *sav);
 extern size_t ah_hdrsiz(struct secasvar *);
 
 /* XF_ESP */
 extern size_t esp_hdrsiz(struct secasvar *sav);
 
 #endif /* _KERNEL */
 #endif /* _NETIPSEC_XFORM_H_ */
Index: head/sys/netipsec/xform_ah.c
===================================================================
--- head/sys/netipsec/xform_ah.c	(revision 359373)
+++ head/sys/netipsec/xform_ah.c	(revision 359374)
@@ -1,1194 +1,1180 @@
 /*	$FreeBSD$	*/
 /*	$OpenBSD: ip_ah.c,v 1.63 2001/06/26 06:18:58 angelos Exp $ */
 /*-
  * The authors of this code are John Ioannidis (ji@tla.org),
  * Angelos D. Keromytis (kermit@csd.uch.gr) and
  * Niels Provos (provos@physnet.uni-hamburg.de).
  *
  * The original version of this code was written by John Ioannidis
  * for BSD/OS in Athens, Greece, in November 1995.
  *
  * Ported to OpenBSD and NetBSD, with additional transforms, in December 1996,
  * by Angelos D. Keromytis.
  *
  * Additional transforms and features in 1997 and 1998 by Angelos D. Keromytis
  * and Niels Provos.
  *
  * Additional features in 1999 by Angelos D. Keromytis and Niklas Hallqvist.
  *
  * Copyright (c) 1995, 1996, 1997, 1998, 1999 by John Ioannidis,
  * Angelos D. Keromytis and Niels Provos.
  * Copyright (c) 1999 Niklas Hallqvist.
  * Copyright (c) 2001 Angelos D. Keromytis.
  *
  * Permission to use, copy, and modify this software with or without fee
  * is hereby granted, provided that this entire notice is included in
  * all copies of any software which is or includes a copy or
  * modification of this software.
  * You may use this code under the GNU public license if you so wish. Please
  * contribute changes back to the authors under this freer than GPL license
  * so that we may further the use of strong encryption without limitations to
  * all.
  *
  * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR
  * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY
  * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE
  * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR
  * PURPOSE.
  */
 #include "opt_inet.h"
 #include "opt_inet6.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/mbuf.h>
 #include <sys/socket.h>
 #include <sys/syslog.h>
 #include <sys/kernel.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <sys/sysctl.h>
 
 #include <net/if.h>
 #include <net/vnet.h>
 
 #include <netinet/in.h>
 #include <netinet/in_systm.h>
 #include <netinet/ip.h>
 #include <netinet/ip_ecn.h>
 #include <netinet/ip6.h>
 
 #include <netipsec/ipsec.h>
 #include <netipsec/ah.h>
 #include <netipsec/ah_var.h>
 #include <netipsec/xform.h>
 
 #ifdef INET6
 #include <netinet6/ip6_var.h>
 #include <netipsec/ipsec6.h>
 #include <netinet6/ip6_ecn.h>
 #endif
 
 #include <netipsec/key.h>
 #include <netipsec/key_debug.h>
 
 #include <opencrypto/cryptodev.h>
 
 /*
  * Return header size in bytes.  The old protocol did not support
  * the replay counter; the new protocol always includes the counter.
  */
 #define HDRSIZE(sav) \
 	(((sav)->flags & SADB_X_EXT_OLD) ? \
 		sizeof (struct ah) : sizeof (struct ah) + sizeof (u_int32_t))
 /* 
  * Return authenticator size in bytes, based on a field in the
  * algorithm descriptor.
  */
 #define	AUTHSIZE(sav)	((sav->flags & SADB_X_EXT_OLD) ? 16 :	\
 			 xform_ah_authsize((sav)->tdb_authalgxform))
 
 VNET_DEFINE(int, ah_enable) = 1;	/* control flow of packets with AH */
 VNET_DEFINE(int, ah_cleartos) = 1;	/* clear ip_tos when doing AH calc */
 VNET_PCPUSTAT_DEFINE(struct ahstat, ahstat);
 VNET_PCPUSTAT_SYSINIT(ahstat);
 
 #ifdef VIMAGE
 VNET_PCPUSTAT_SYSUNINIT(ahstat);
 #endif /* VIMAGE */
 
 #ifdef INET
 SYSCTL_DECL(_net_inet_ah);
 SYSCTL_INT(_net_inet_ah, OID_AUTO, ah_enable,
 	CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(ah_enable), 0, "");
 SYSCTL_INT(_net_inet_ah, OID_AUTO, ah_cleartos,
 	CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(ah_cleartos), 0, "");
 SYSCTL_VNET_PCPUSTAT(_net_inet_ah, IPSECCTL_STATS, stats, struct ahstat,
     ahstat, "AH statistics (struct ahstat, netipsec/ah_var.h)");
 #endif
 
 static unsigned char ipseczeroes[256];	/* larger than an ip6 extension hdr */
 static struct timeval md5warn, ripewarn, kpdkmd5warn, kpdksha1warn;
 
 static int ah_input_cb(struct cryptop*);
 static int ah_output_cb(struct cryptop*);
 
 int
 xform_ah_authsize(const struct auth_hash *esph)
 {
 	int alen;
 
 	if (esph == NULL)
 		return 0;
 
 	switch (esph->type) {
 	case CRYPTO_SHA2_256_HMAC:
 	case CRYPTO_SHA2_384_HMAC:
 	case CRYPTO_SHA2_512_HMAC:
 		alen = esph->hashsize / 2;	/* RFC4868 2.3 */
 		break;
 
-	case CRYPTO_AES_128_NIST_GMAC:
-	case CRYPTO_AES_192_NIST_GMAC:
-	case CRYPTO_AES_256_NIST_GMAC:
+	case CRYPTO_AES_NIST_GMAC:
 		alen = esph->hashsize;
 		break;
 
 	default:
 		alen = AH_HMAC_HASHLEN;
 		break;
 	}
 
 	return alen;
 }
 
 size_t
 ah_hdrsiz(struct secasvar *sav)
 {
 	size_t size;
 
 	if (sav != NULL) {
 		int authsize, rplen, align;
 
 		IPSEC_ASSERT(sav->tdb_authalgxform != NULL, ("null xform"));
 		/*XXX not right for null algorithm--does it matter??*/
 
 		/* RFC4302: use the correct alignment. */
 		align = sizeof(uint32_t);
 #ifdef INET6
 		if (sav->sah->saidx.dst.sa.sa_family == AF_INET6) {
 			align = sizeof(uint64_t);
 		}
 #endif
 		rplen = HDRSIZE(sav);
 		authsize = AUTHSIZE(sav);
 		size = roundup(rplen + authsize, align);
 	} else {
 		/* default guess */
 		size = sizeof (struct ah) + sizeof (u_int32_t) + 16;
 	}
 	return size;
 }
 
 /*
  * NB: public for use by esp_init.
  */
 int
-ah_init0(struct secasvar *sav, struct xformsw *xsp, struct cryptoini *cria)
+ah_init0(struct secasvar *sav, struct xformsw *xsp,
+    struct crypto_session_params *csp)
 {
 	const struct auth_hash *thash;
 	int keylen;
 
 	thash = auth_algorithm_lookup(sav->alg_auth);
 	if (thash == NULL) {
 		DPRINTF(("%s: unsupported authentication algorithm %u\n",
 			__func__, sav->alg_auth));
 		return EINVAL;
 	}
 
 	switch (sav->alg_auth) {
 	case SADB_AALG_MD5HMAC:
 		if (ratecheck(&md5warn, &ipsec_warn_interval))
 			gone_in(13, "MD5-HMAC authenticator for IPsec");
 		break;
 	case SADB_X_AALG_RIPEMD160HMAC:
 		if (ratecheck(&ripewarn, &ipsec_warn_interval))
 			gone_in(13, "RIPEMD160-HMAC authenticator for IPsec");
 		break;
 	case SADB_X_AALG_MD5:
 		if (ratecheck(&kpdkmd5warn, &ipsec_warn_interval))
 			gone_in(13, "Keyed-MD5 authenticator for IPsec");
 		break;
 	case SADB_X_AALG_SHA:
 		if (ratecheck(&kpdksha1warn, &ipsec_warn_interval))
 			gone_in(13, "Keyed-SHA1 authenticator for IPsec");
 		break;
 	}
 
 	/*
 	 * Verify the replay state block allocation is consistent with
 	 * the protocol type.  We check here so we can make assumptions
 	 * later during protocol processing.
 	 */
 	/* NB: replay state is setup elsewhere (sigh) */
 	if (((sav->flags&SADB_X_EXT_OLD) == 0) ^ (sav->replay != NULL)) {
 		DPRINTF(("%s: replay state block inconsistency, "
 			"%s algorithm %s replay state\n", __func__,
 			(sav->flags & SADB_X_EXT_OLD) ? "old" : "new",
 			sav->replay == NULL ? "without" : "with"));
 		return EINVAL;
 	}
 	if (sav->key_auth == NULL) {
 		DPRINTF(("%s: no authentication key for %s algorithm\n",
 			__func__, thash->name));
 		return EINVAL;
 	}
 	keylen = _KEYLEN(sav->key_auth);
 	if (keylen > thash->keysize && thash->keysize != 0) {
 		DPRINTF(("%s: invalid keylength %d, algorithm %s requires "
 			"keysize less than %d\n", __func__,
 			 keylen, thash->name, thash->keysize));
 		return EINVAL;
 	}
 
 	sav->tdb_xform = xsp;
 	sav->tdb_authalgxform = thash;
 
 	/* Initialize crypto session. */
-	bzero(cria, sizeof (*cria));
-	cria->cri_alg = sav->tdb_authalgxform->type;
-	cria->cri_klen = _KEYBITS(sav->key_auth);
-	cria->cri_key = sav->key_auth->key_data;
-	cria->cri_mlen = AUTHSIZE(sav);
+	csp->csp_auth_alg = sav->tdb_authalgxform->type;
+	csp->csp_auth_klen = _KEYBITS(sav->key_auth) / 8;
+	csp->csp_auth_key = sav->key_auth->key_data;
+	csp->csp_auth_mlen = AUTHSIZE(sav);
 
 	return 0;
 }
 
 /*
  * ah_init() is called when an SPI is being set up.
  */
 static int
 ah_init(struct secasvar *sav, struct xformsw *xsp)
 {
-	struct cryptoini cria;
+	struct crypto_session_params csp;
 	int error;
 
-	error = ah_init0(sav, xsp, &cria);
+	memset(&csp, 0, sizeof(csp));
+	csp.csp_mode = CSP_MODE_DIGEST;
+	error = ah_init0(sav, xsp, &csp);
 	return error ? error :
-		 crypto_newsession(&sav->tdb_cryptoid, &cria, V_crypto_support);
+		 crypto_newsession(&sav->tdb_cryptoid, &csp, V_crypto_support);
 }
 
 /*
  * Paranoia.
  *
  * NB: public for use by esp_zeroize (XXX).
  */
 int
 ah_zeroize(struct secasvar *sav)
 {
 
 	if (sav->key_auth)
 		bzero(sav->key_auth->key_data, _KEYLEN(sav->key_auth));
 
 	crypto_freesession(sav->tdb_cryptoid);
 	sav->tdb_cryptoid = NULL;
 	sav->tdb_authalgxform = NULL;
 	sav->tdb_xform = NULL;
 	return 0;
 }
 
 /*
  * Massage IPv4/IPv6 headers for AH processing.
  */
 static int
 ah_massage_headers(struct mbuf **m0, int proto, int skip, int alg, int out)
 {
 	struct mbuf *m = *m0;
 	unsigned char *ptr;
 	int off, count;
 
 #ifdef INET
 	struct ip *ip;
 #endif /* INET */
 
 #ifdef INET6
 	struct ip6_ext *ip6e;
 	struct ip6_hdr ip6;
 	int ad, alloc, nxt, noff;
 #endif /* INET6 */
 
 	switch (proto) {
 #ifdef INET
 	case AF_INET:
 		/*
 		 * This is the least painful way of dealing with IPv4 header
 		 * and option processing -- just make sure they're in
 		 * contiguous memory.
 		 */
 		*m0 = m = m_pullup(m, skip);
 		if (m == NULL) {
 			DPRINTF(("%s: m_pullup failed\n", __func__));
 			return ENOBUFS;
 		}
 
 		/* Fix the IP header */
 		ip = mtod(m, struct ip *);
 		if (V_ah_cleartos)
 			ip->ip_tos = 0;
 		ip->ip_ttl = 0;
 		ip->ip_sum = 0;
 
 		if (alg == CRYPTO_MD5_KPDK || alg == CRYPTO_SHA1_KPDK)
 			ip->ip_off &= htons(IP_DF);
 		else
 			ip->ip_off = htons(0);
 
 		ptr = mtod(m, unsigned char *);
 
 		/* IPv4 option processing */
 		for (off = sizeof(struct ip); off < skip;) {
 			if (ptr[off] == IPOPT_EOL || ptr[off] == IPOPT_NOP ||
 			    off + 1 < skip)
 				;
 			else {
 				DPRINTF(("%s: illegal IPv4 option length for "
 					"option %d\n", __func__, ptr[off]));
 
 				m_freem(m);
 				return EINVAL;
 			}
 
 			switch (ptr[off]) {
 			case IPOPT_EOL:
 				off = skip;  /* End the loop. */
 				break;
 
 			case IPOPT_NOP:
 				off++;
 				break;
 
 			case IPOPT_SECURITY:	/* 0x82 */
 			case 0x85:	/* Extended security. */
 			case 0x86:	/* Commercial security. */
 			case 0x94:	/* Router alert */
 			case 0x95:	/* RFC1770 */
 				/* Sanity check for option length. */
 				if (ptr[off + 1] < 2) {
 					DPRINTF(("%s: illegal IPv4 option "
 						"length for option %d\n",
 						__func__, ptr[off]));
 
 					m_freem(m);
 					return EINVAL;
 				}
 
 				off += ptr[off + 1];
 				break;
 
 			case IPOPT_LSRR:
 			case IPOPT_SSRR:
 				/* Sanity check for option length. */
 				if (ptr[off + 1] < 2) {
 					DPRINTF(("%s: illegal IPv4 option "
 						"length for option %d\n",
 						__func__, ptr[off]));
 
 					m_freem(m);
 					return EINVAL;
 				}
 
 				/*
 				 * On output, if we have either of the
 				 * source routing options, we should
 				 * swap the destination address of the
 				 * IP header with the last address
 				 * specified in the option, as that is
 				 * what the destination's IP header
 				 * will look like.
 				 */
 				if (out)
 					bcopy(ptr + off + ptr[off + 1] -
 					    sizeof(struct in_addr),
 					    &(ip->ip_dst), sizeof(struct in_addr));
 
 				/* Fall through */
 			default:
 				/* Sanity check for option length. */
 				if (ptr[off + 1] < 2) {
 					DPRINTF(("%s: illegal IPv4 option "
 						"length for option %d\n",
 						__func__, ptr[off]));
 					m_freem(m);
 					return EINVAL;
 				}
 
 				/* Zeroize all other options. */
 				count = ptr[off + 1];
 				bcopy(ipseczeroes, ptr + off, count);
 				off += count;
 				break;
 			}
 
 			/* Sanity check. */
 			if (off > skip)	{
 				DPRINTF(("%s: malformed IPv4 options header\n",
 					__func__));
 
 				m_freem(m);
 				return EINVAL;
 			}
 		}
 
 		break;
 #endif /* INET */
 
 #ifdef INET6
 	case AF_INET6:  /* Ugly... */
 		/* Copy and "cook" the IPv6 header. */
 		m_copydata(m, 0, sizeof(ip6), (caddr_t) &ip6);
 
 		/* We don't do IPv6 Jumbograms. */
 		if (ip6.ip6_plen == 0) {
 			DPRINTF(("%s: unsupported IPv6 jumbogram\n", __func__));
 			m_freem(m);
 			return EMSGSIZE;
 		}
 
 		ip6.ip6_flow = 0;
 		ip6.ip6_hlim = 0;
 		ip6.ip6_vfc &= ~IPV6_VERSION_MASK;
 		ip6.ip6_vfc |= IPV6_VERSION;
 
 		/* Scoped address handling. */
 		if (IN6_IS_SCOPE_LINKLOCAL(&ip6.ip6_src))
 			ip6.ip6_src.s6_addr16[1] = 0;
 		if (IN6_IS_SCOPE_LINKLOCAL(&ip6.ip6_dst))
 			ip6.ip6_dst.s6_addr16[1] = 0;
 
 		/* Done with IPv6 header. */
 		m_copyback(m, 0, sizeof(struct ip6_hdr), (caddr_t) &ip6);
 
 		/* Let's deal with the remaining headers (if any). */
 		if (skip - sizeof(struct ip6_hdr) > 0) {
 			if (m->m_len <= skip) {
 				ptr = (unsigned char *) malloc(
 				    skip - sizeof(struct ip6_hdr),
 				    M_XDATA, M_NOWAIT);
 				if (ptr == NULL) {
 					DPRINTF(("%s: failed to allocate memory"
 						"for IPv6 headers\n",__func__));
 					m_freem(m);
 					return ENOBUFS;
 				}
 
 				/*
 				 * Copy all the protocol headers after
 				 * the IPv6 header.
 				 */
 				m_copydata(m, sizeof(struct ip6_hdr),
 				    skip - sizeof(struct ip6_hdr), ptr);
 				alloc = 1;
 			} else {
 				/* No need to allocate memory. */
 				ptr = mtod(m, unsigned char *) +
 				    sizeof(struct ip6_hdr);
 				alloc = 0;
 			}
 		} else
 			break;
 
 		nxt = ip6.ip6_nxt & 0xff; /* Next header type. */
 
 		for (off = 0; off < skip - sizeof(struct ip6_hdr);)
 			switch (nxt) {
 			case IPPROTO_HOPOPTS:
 			case IPPROTO_DSTOPTS:
 				ip6e = (struct ip6_ext *)(ptr + off);
 				noff = off + ((ip6e->ip6e_len + 1) << 3);
 
 				/* Sanity check. */
 				if (noff > skip - sizeof(struct ip6_hdr))
 					goto error6;
 
 				/*
 				 * Zero out mutable options.
 				 */
 				for (count = off + sizeof(struct ip6_ext);
 				     count < noff;) {
 					if (ptr[count] == IP6OPT_PAD1) {
 						count++;
 						continue; /* Skip padding. */
 					}
 
 					ad = ptr[count + 1] + 2;
 					if (count + ad > noff)
 						goto error6;
 
 					if (ptr[count] & IP6OPT_MUTABLE)
 						memset(ptr + count, 0, ad);
 					count += ad;
 				}
 
 				if (count != noff)
 					goto error6;
 
 				/* Advance. */
 				off += ((ip6e->ip6e_len + 1) << 3);
 				nxt = ip6e->ip6e_nxt;
 				break;
 
 			case IPPROTO_ROUTING:
 				/*
 				 * Always include routing headers in
 				 * computation.
 				 */
 				ip6e = (struct ip6_ext *) (ptr + off);
 				off += ((ip6e->ip6e_len + 1) << 3);
 				nxt = ip6e->ip6e_nxt;
 				break;
 
 			default:
 				DPRINTF(("%s: unexpected IPv6 header type %d",
 					__func__, off));
 error6:
 				if (alloc)
 					free(ptr, M_XDATA);
 				m_freem(m);
 				return EINVAL;
 			}
 
 		/* Copyback and free, if we allocated. */
 		if (alloc) {
 			m_copyback(m, sizeof(struct ip6_hdr),
 			    skip - sizeof(struct ip6_hdr), ptr);
 			free(ptr, M_XDATA);
 		}
 
 		break;
 #endif /* INET6 */
 	}
 
 	return 0;
 }
 
 /*
  * ah_input() gets called to verify that an input packet
  * passes authentication.
  */
 static int
 ah_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
 {
 	IPSEC_DEBUG_DECLARE(char buf[128]);
 	const struct auth_hash *ahx;
-	struct cryptodesc *crda;
 	struct cryptop *crp;
 	struct xform_data *xd;
 	struct newah *ah;
 	crypto_session_t cryptoid;
 	int hl, rplen, authsize, ahsize, error;
 
 	IPSEC_ASSERT(sav != NULL, ("null SA"));
 	IPSEC_ASSERT(sav->key_auth != NULL, ("null authentication key"));
 	IPSEC_ASSERT(sav->tdb_authalgxform != NULL,
 		("null authentication xform"));
 
 	/* Figure out header size. */
 	rplen = HDRSIZE(sav);
 
 	if (m->m_len < skip + rplen) {
 		m = m_pullup(m, skip + rplen);
 		if (m == NULL) {
 			DPRINTF(("ah_input: cannot pullup header\n"));
 			AHSTAT_INC(ahs_hdrops);		/*XXX*/
 			error = ENOBUFS;
 			goto bad;
 		}
 	}
 	ah = (struct newah *)(mtod(m, caddr_t) + skip);
 
 	/* Check replay window, if applicable. */
 	SECASVAR_LOCK(sav);
 	if (sav->replay != NULL && sav->replay->wsize != 0 &&
 	    ipsec_chkreplay(ntohl(ah->ah_seq), sav) == 0) {
 		SECASVAR_UNLOCK(sav);
 		AHSTAT_INC(ahs_replay);
 		DPRINTF(("%s: packet replay failure: %s\n", __func__,
 		    ipsec_sa2str(sav, buf, sizeof(buf))));
 		error = EACCES;
 		goto bad;
 	}
 	cryptoid = sav->tdb_cryptoid;
 	SECASVAR_UNLOCK(sav);
 
 	/* Verify AH header length. */
 	hl = sizeof(struct ah) + (ah->ah_len * sizeof (u_int32_t));
 	ahx = sav->tdb_authalgxform;
 	authsize = AUTHSIZE(sav);
 	ahsize = ah_hdrsiz(sav);
 	if (hl != ahsize) {
 		DPRINTF(("%s: bad authenticator length %u (expecting %lu)"
 		    " for packet in SA %s/%08lx\n", __func__, hl,
 		    (u_long)ahsize,
 		    ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)),
 		    (u_long) ntohl(sav->spi)));
 		AHSTAT_INC(ahs_badauthl);
 		error = EACCES;
 		goto bad;
 	}
 	if (skip + ahsize > m->m_pkthdr.len) {
 		DPRINTF(("%s: bad mbuf length %u (expecting %lu)"
 		    " for packet in SA %s/%08lx\n", __func__,
 		    m->m_pkthdr.len, (u_long)(skip + ahsize),
 		    ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)),
 		    (u_long) ntohl(sav->spi)));
 		AHSTAT_INC(ahs_badauthl);
 		error = EACCES;
 		goto bad;
 	}
 	AHSTAT_ADD(ahs_ibytes, m->m_pkthdr.len - skip - hl);
 
 	/* Get crypto descriptors. */
-	crp = crypto_getreq(1);
+	crp = crypto_getreq(cryptoid, M_NOWAIT);
 	if (crp == NULL) {
 		DPRINTF(("%s: failed to acquire crypto descriptor\n",
 		    __func__));
 		AHSTAT_INC(ahs_crypto);
 		error = ENOBUFS;
 		goto bad;
 	}
 
-	crda = crp->crp_desc;
-	IPSEC_ASSERT(crda != NULL, ("null crypto descriptor"));
+	crp->crp_payload_start = 0;
+	crp->crp_payload_length = m->m_pkthdr.len;
+	crp->crp_digest_start = skip + rplen;
 
-	crda->crd_skip = 0;
-	crda->crd_len = m->m_pkthdr.len;
-	crda->crd_inject = skip + rplen;
-
-	/* Authentication operation. */
-	crda->crd_alg = ahx->type;
-	crda->crd_klen = _KEYBITS(sav->key_auth);
-	crda->crd_key = sav->key_auth->key_data;
-
 	/* Allocate IPsec-specific opaque crypto info. */
 	xd = malloc(sizeof(*xd) + skip + rplen + authsize, M_XDATA,
 	    M_NOWAIT | M_ZERO);
 	if (xd == NULL) {
 		DPRINTF(("%s: failed to allocate xform_data\n", __func__));
 		AHSTAT_INC(ahs_crypto);
 		crypto_freereq(crp);
 		error = ENOBUFS;
 		goto bad;
 	}
 
 	/*
 	 * Save the authenticator, the skipped portion of the packet,
 	 * and the AH header.
 	 */
 	m_copydata(m, 0, skip + rplen + authsize, (caddr_t)(xd + 1));
 
 	/* Zeroize the authenticator on the packet. */
 	m_copyback(m, skip + rplen, authsize, ipseczeroes);
 
 	/* Save ah_nxt, since ah pointer can become invalid after "massage" */
 	hl = ah->ah_nxt;
 
 	/* "Massage" the packet headers for crypto processing. */
 	error = ah_massage_headers(&m, sav->sah->saidx.dst.sa.sa_family,
 	    skip, ahx->type, 0);
 	if (error != 0) {
 		/* NB: mbuf is free'd by ah_massage_headers */
 		AHSTAT_INC(ahs_hdrops);
 		free(xd, M_XDATA);
 		crypto_freereq(crp);
 		key_freesav(&sav);
 		return (error);
 	}
 
 	/* Crypto operation descriptor. */
 	crp->crp_ilen = m->m_pkthdr.len; /* Total input length. */
-	crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_CBIFSYNC;
+	crp->crp_op = CRYPTO_OP_COMPUTE_DIGEST;
+	crp->crp_flags = CRYPTO_F_CBIFSYNC;
 	if (V_async_crypto)
 		crp->crp_flags |= CRYPTO_F_ASYNC | CRYPTO_F_ASYNC_KEEPORDER;
-	crp->crp_buf = (caddr_t) m;
+	crp->crp_mbuf = m;
+	crp->crp_buf_type = CRYPTO_BUF_MBUF;
 	crp->crp_callback = ah_input_cb;
-	crp->crp_session = cryptoid;
-	crp->crp_opaque = (caddr_t) xd;
+	crp->crp_opaque = xd;
 
 	/* These are passed as-is to the callback. */
 	xd->sav = sav;
 	xd->nxt = hl;
 	xd->protoff = protoff;
 	xd->skip = skip;
 	xd->cryptoid = cryptoid;
 	xd->vnet = curvnet;
 	return (crypto_dispatch(crp));
 bad:
 	m_freem(m);
 	key_freesav(&sav);
 	return (error);
 }
 
 /*
  * AH input callback from the crypto driver.
  */
 static int
 ah_input_cb(struct cryptop *crp)
 {
 	IPSEC_DEBUG_DECLARE(char buf[IPSEC_ADDRSTRLEN]);
 	unsigned char calc[AH_ALEN_MAX];
 	struct mbuf *m;
 	struct xform_data *xd;
 	struct secasvar *sav;
 	struct secasindex *saidx;
 	caddr_t ptr;
 	crypto_session_t cryptoid;
 	int authsize, rplen, ahsize, error, skip, protoff;
 	uint8_t nxt;
 
-	m = (struct mbuf *) crp->crp_buf;
-	xd = (struct xform_data *) crp->crp_opaque;
+	m = crp->crp_mbuf;
+	xd = crp->crp_opaque;
 	CURVNET_SET(xd->vnet);
 	sav = xd->sav;
 	skip = xd->skip;
 	nxt = xd->nxt;
 	protoff = xd->protoff;
 	cryptoid = xd->cryptoid;
 	saidx = &sav->sah->saidx;
 	IPSEC_ASSERT(saidx->dst.sa.sa_family == AF_INET ||
 		saidx->dst.sa.sa_family == AF_INET6,
 		("unexpected protocol family %u", saidx->dst.sa.sa_family));
 
 	/* Check for crypto errors. */
 	if (crp->crp_etype) {
 		if (crp->crp_etype == EAGAIN) {
 			/* Reset the session ID */
 			if (ipsec_updateid(sav, &crp->crp_session, &cryptoid) != 0)
 				crypto_freesession(cryptoid);
 			xd->cryptoid = crp->crp_session;
 			CURVNET_RESTORE();
 			return (crypto_dispatch(crp));
 		}
 		AHSTAT_INC(ahs_noxform);
 		DPRINTF(("%s: crypto error %d\n", __func__, crp->crp_etype));
 		error = crp->crp_etype;
 		goto bad;
 	} else {
 		AHSTAT_INC(ahs_hist[sav->alg_auth]);
 		crypto_freereq(crp);		/* No longer needed. */
 		crp = NULL;
 	}
 
 	/* Shouldn't happen... */
 	if (m == NULL) {
 		AHSTAT_INC(ahs_crypto);
 		DPRINTF(("%s: bogus returned buffer from crypto\n", __func__));
 		error = EINVAL;
 		goto bad;
 	}
 
 	/* Figure out header size. */
 	rplen = HDRSIZE(sav);
 	authsize = AUTHSIZE(sav);
 	ahsize = ah_hdrsiz(sav);
 
 	/* Copy authenticator off the packet. */
 	m_copydata(m, skip + rplen, authsize, calc);
 
 	/* Verify authenticator. */
 	ptr = (caddr_t) (xd + 1);
 	if (timingsafe_bcmp(ptr + skip + rplen, calc, authsize)) {
 		DPRINTF(("%s: authentication hash mismatch for packet "
 		    "in SA %s/%08lx\n", __func__,
 		    ipsec_address(&saidx->dst, buf, sizeof(buf)),
 		    (u_long) ntohl(sav->spi)));
 		AHSTAT_INC(ahs_badauth);
 		error = EACCES;
 		goto bad;
 	}
 	/* Fix the Next Protocol field. */
 	((uint8_t *) ptr)[protoff] = nxt;
 
 	/* Copyback the saved (uncooked) network headers. */
 	m_copyback(m, 0, skip, ptr);
 	free(xd, M_XDATA), xd = NULL;			/* No longer needed */
 
 	/*
 	 * Header is now authenticated.
 	 */
 	m->m_flags |= M_AUTHIPHDR|M_AUTHIPDGM;
 
 	/*
 	 * Update replay sequence number, if appropriate.
 	 */
 	if (sav->replay) {
 		u_int32_t seq;
 
 		m_copydata(m, skip + offsetof(struct newah, ah_seq),
 			   sizeof (seq), (caddr_t) &seq);
 		SECASVAR_LOCK(sav);
 		if (ipsec_updatereplay(ntohl(seq), sav)) {
 			SECASVAR_UNLOCK(sav);
 			AHSTAT_INC(ahs_replay);
 			error = EACCES;
 			goto bad;
 		}
 		SECASVAR_UNLOCK(sav);
 	}
 
 	/*
 	 * Remove the AH header and authenticator from the mbuf.
 	 */
 	error = m_striphdr(m, skip, ahsize);
 	if (error) {
 		DPRINTF(("%s: mangled mbuf chain for SA %s/%08lx\n", __func__,
 		    ipsec_address(&saidx->dst, buf, sizeof(buf)),
 		    (u_long) ntohl(sav->spi)));
 		AHSTAT_INC(ahs_hdrops);
 		goto bad;
 	}
 
 	switch (saidx->dst.sa.sa_family) {
 #ifdef INET6
 	case AF_INET6:
 		error = ipsec6_common_input_cb(m, sav, skip, protoff);
 		break;
 #endif
 #ifdef INET
 	case AF_INET:
 		error = ipsec4_common_input_cb(m, sav, skip, protoff);
 		break;
 #endif
 	default:
 		panic("%s: Unexpected address family: %d saidx=%p", __func__,
 		    saidx->dst.sa.sa_family, saidx);
 	}
 	CURVNET_RESTORE();
 	return error;
 bad:
 	CURVNET_RESTORE();
 	if (sav)
 		key_freesav(&sav);
 	if (m != NULL)
 		m_freem(m);
 	if (xd != NULL)
 		free(xd, M_XDATA);
 	if (crp != NULL)
 		crypto_freereq(crp);
 	return error;
 }
 
 /*
  * AH output routine, called by ipsec[46]_perform_request().
  */
 static int
 ah_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
     u_int idx, int skip, int protoff)
 {
 	IPSEC_DEBUG_DECLARE(char buf[IPSEC_ADDRSTRLEN]);
 	const struct auth_hash *ahx;
-	struct cryptodesc *crda;
 	struct xform_data *xd;
 	struct mbuf *mi;
 	struct cryptop *crp;
 	struct newah *ah;
 	crypto_session_t cryptoid;
 	uint16_t iplen;
 	int error, rplen, authsize, ahsize, maxpacketsize, roff;
 	uint8_t prot;
 
 	IPSEC_ASSERT(sav != NULL, ("null SA"));
 	ahx = sav->tdb_authalgxform;
 	IPSEC_ASSERT(ahx != NULL, ("null authentication xform"));
 
 	AHSTAT_INC(ahs_output);
 
 	/* Figure out header size. */
 	rplen = HDRSIZE(sav);
 	authsize = AUTHSIZE(sav);
 	ahsize = ah_hdrsiz(sav);
 
 	/* Check for maximum packet size violations. */
 	switch (sav->sah->saidx.dst.sa.sa_family) {
 #ifdef INET
 	case AF_INET:
 		maxpacketsize = IP_MAXPACKET;
 		break;
 #endif /* INET */
 #ifdef INET6
 	case AF_INET6:
 		maxpacketsize = IPV6_MAXPACKET;
 		break;
 #endif /* INET6 */
 	default:
 		DPRINTF(("%s: unknown/unsupported protocol family %u, "
 		    "SA %s/%08lx\n", __func__,
 		    sav->sah->saidx.dst.sa.sa_family,
 		    ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)),
 		    (u_long) ntohl(sav->spi)));
 		AHSTAT_INC(ahs_nopf);
 		error = EPFNOSUPPORT;
 		goto bad;
 	}
 	if (ahsize + m->m_pkthdr.len > maxpacketsize) {
 		DPRINTF(("%s: packet in SA %s/%08lx got too big "
 		    "(len %u, max len %u)\n", __func__,
 		    ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)),
 		    (u_long) ntohl(sav->spi),
 		    ahsize + m->m_pkthdr.len, maxpacketsize));
 		AHSTAT_INC(ahs_toobig);
 		error = EMSGSIZE;
 		goto bad;
 	}
 
 	/* Update the counters. */
 	AHSTAT_ADD(ahs_obytes, m->m_pkthdr.len - skip);
 
 	m = m_unshare(m, M_NOWAIT);
 	if (m == NULL) {
 		DPRINTF(("%s: cannot clone mbuf chain, SA %s/%08lx\n", __func__,
 		    ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)),
 		    (u_long) ntohl(sav->spi)));
 		AHSTAT_INC(ahs_hdrops);
 		error = ENOBUFS;
 		goto bad;
 	}
 
 	/* Inject AH header. */
 	mi = m_makespace(m, skip, ahsize, &roff);
 	if (mi == NULL) {
 		DPRINTF(("%s: failed to inject %u byte AH header for SA "
 		    "%s/%08lx\n", __func__, ahsize,
 		    ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)),
 		    (u_long) ntohl(sav->spi)));
 		AHSTAT_INC(ahs_hdrops);		/*XXX differs from openbsd */
 		error = ENOBUFS;
 		goto bad;
 	}
 
 	/*
 	 * The AH header is guaranteed by m_makespace() to be in
 	 * contiguous memory, at roff bytes offset into the returned mbuf.
 	 */
 	ah = (struct newah *)(mtod(mi, caddr_t) + roff);
 
 	/* Initialize the AH header. */
 	m_copydata(m, protoff, sizeof(u_int8_t), (caddr_t) &ah->ah_nxt);
 	ah->ah_len = (ahsize - sizeof(struct ah)) / sizeof(u_int32_t);
 	ah->ah_reserve = 0;
 	ah->ah_spi = sav->spi;
 
 	/* Zeroize authenticator. */
 	m_copyback(m, skip + rplen, authsize, ipseczeroes);
 
 	/* Zeroize padding */
 	m_copyback(m, skip + rplen + authsize, ahsize - (rplen + authsize),
 	    ipseczeroes);
 
 	/* Insert packet replay counter, as requested.  */
 	SECASVAR_LOCK(sav);
 	if (sav->replay) {
 		if (sav->replay->count == ~0 &&
 		    (sav->flags & SADB_X_EXT_CYCSEQ) == 0) {
 			SECASVAR_UNLOCK(sav);
 			DPRINTF(("%s: replay counter wrapped for SA %s/%08lx\n",
 			    __func__, ipsec_address(&sav->sah->saidx.dst, buf,
 			    sizeof(buf)), (u_long) ntohl(sav->spi)));
 			AHSTAT_INC(ahs_wrap);
 			error = EACCES;
 			goto bad;
 		}
 #ifdef REGRESSION
 		/* Emulate replay attack when ipsec_replay is TRUE. */
 		if (!V_ipsec_replay)
 #endif
 			sav->replay->count++;
 		ah->ah_seq = htonl(sav->replay->count);
 	}
 	cryptoid = sav->tdb_cryptoid;
 	SECASVAR_UNLOCK(sav);
 
 	/* Get crypto descriptors. */
-	crp = crypto_getreq(1);
+	crp = crypto_getreq(cryptoid, M_NOWAIT);
 	if (crp == NULL) {
 		DPRINTF(("%s: failed to acquire crypto descriptors\n",
 			__func__));
 		AHSTAT_INC(ahs_crypto);
 		error = ENOBUFS;
 		goto bad;
 	}
 
-	crda = crp->crp_desc;
-	crda->crd_skip = 0;
-	crda->crd_inject = skip + rplen;
-	crda->crd_len = m->m_pkthdr.len;
+	crp->crp_payload_start = 0;
+	crp->crp_payload_length = m->m_pkthdr.len;
+	crp->crp_digest_start = skip + rplen;
 
-	/* Authentication operation. */
-	crda->crd_alg = ahx->type;
-	crda->crd_key = sav->key_auth->key_data;
-	crda->crd_klen = _KEYBITS(sav->key_auth);
-
 	/* Allocate IPsec-specific opaque crypto info. */
 	xd =  malloc(sizeof(struct xform_data) + skip, M_XDATA,
 	    M_NOWAIT | M_ZERO);
 	if (xd == NULL) {
 		crypto_freereq(crp);
 		DPRINTF(("%s: failed to allocate xform_data\n", __func__));
 		AHSTAT_INC(ahs_crypto);
 		error = ENOBUFS;
 		goto bad;
 	}
 
 	/* Save the skipped portion of the packet. */
 	m_copydata(m, 0, skip, (caddr_t) (xd + 1));
 
 	/*
 	 * Fix IP header length on the header used for
 	 * authentication. We don't need to fix the original
 	 * header length as it will be fixed by our caller.
 	 */
 	switch (sav->sah->saidx.dst.sa.sa_family) {
 #ifdef INET
 	case AF_INET:
 		bcopy(((caddr_t)(xd + 1)) +
 		    offsetof(struct ip, ip_len),
 		    (caddr_t) &iplen, sizeof(u_int16_t));
 		iplen = htons(ntohs(iplen) + ahsize);
 		m_copyback(m, offsetof(struct ip, ip_len),
 		    sizeof(u_int16_t), (caddr_t) &iplen);
 		break;
 #endif /* INET */
 
 #ifdef INET6
 	case AF_INET6:
 		bcopy(((caddr_t)(xd + 1)) +
 		    offsetof(struct ip6_hdr, ip6_plen),
 		    (caddr_t) &iplen, sizeof(uint16_t));
 		iplen = htons(ntohs(iplen) + ahsize);
 		m_copyback(m, offsetof(struct ip6_hdr, ip6_plen),
 		    sizeof(uint16_t), (caddr_t) &iplen);
 		break;
 #endif /* INET6 */
 	}
 
 	/* Fix the Next Header field in saved header. */
 	((uint8_t *) (xd + 1))[protoff] = IPPROTO_AH;
 
 	/* Update the Next Protocol field in the IP header. */
 	prot = IPPROTO_AH;
 	m_copyback(m, protoff, sizeof(uint8_t), (caddr_t) &prot);
 
 	/* "Massage" the packet headers for crypto processing. */
 	error = ah_massage_headers(&m, sav->sah->saidx.dst.sa.sa_family,
 			skip, ahx->type, 1);
 	if (error != 0) {
 		m = NULL;	/* mbuf was free'd by ah_massage_headers. */
 		free(xd, M_XDATA);
 		crypto_freereq(crp);
 		goto bad;
 	}
 
 	/* Crypto operation descriptor. */
 	crp->crp_ilen = m->m_pkthdr.len; /* Total input length. */
-	crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_CBIFSYNC;
+	crp->crp_op = CRYPTO_OP_COMPUTE_DIGEST;
+	crp->crp_flags = CRYPTO_F_CBIFSYNC;
 	if (V_async_crypto)
 		crp->crp_flags |= CRYPTO_F_ASYNC | CRYPTO_F_ASYNC_KEEPORDER;
-	crp->crp_buf = (caddr_t) m;
+	crp->crp_mbuf = m;
+	crp->crp_buf_type = CRYPTO_BUF_MBUF;
 	crp->crp_callback = ah_output_cb;
-	crp->crp_session = cryptoid;
-	crp->crp_opaque = (caddr_t) xd;
+	crp->crp_opaque = xd;
 
 	/* These are passed as-is to the callback. */
 	xd->sp = sp;
 	xd->sav = sav;
 	xd->skip = skip;
 	xd->idx = idx;
 	xd->cryptoid = cryptoid;
 	xd->vnet = curvnet;
 
 	return crypto_dispatch(crp);
 bad:
 	if (m)
 		m_freem(m);
 	key_freesav(&sav);
 	key_freesp(&sp);
 	return (error);
 }
 
 /*
  * AH output callback from the crypto driver.
  */
 static int
 ah_output_cb(struct cryptop *crp)
 {
 	struct xform_data *xd;
 	struct secpolicy *sp;
 	struct secasvar *sav;
 	struct mbuf *m;
 	crypto_session_t cryptoid;
 	caddr_t ptr;
 	u_int idx;
 	int skip, error;
 
 	m = (struct mbuf *) crp->crp_buf;
 	xd = (struct xform_data *) crp->crp_opaque;
 	CURVNET_SET(xd->vnet);
 	sp = xd->sp;
 	sav = xd->sav;
 	skip = xd->skip;
 	idx = xd->idx;
 	cryptoid = xd->cryptoid;
 	ptr = (caddr_t) (xd + 1);
 
 	/* Check for crypto errors. */
 	if (crp->crp_etype) {
 		if (crp->crp_etype == EAGAIN) {
 			/* Reset the session ID */
 			if (ipsec_updateid(sav, &crp->crp_session, &cryptoid) != 0)
 				crypto_freesession(cryptoid);
 			xd->cryptoid = crp->crp_session;
 			CURVNET_RESTORE();
 			return (crypto_dispatch(crp));
 		}
 		AHSTAT_INC(ahs_noxform);
 		DPRINTF(("%s: crypto error %d\n", __func__, crp->crp_etype));
 		error = crp->crp_etype;
 		m_freem(m);
 		goto bad;
 	}
 
 	/* Shouldn't happen... */
 	if (m == NULL) {
 		AHSTAT_INC(ahs_crypto);
 		DPRINTF(("%s: bogus returned buffer from crypto\n", __func__));
 		error = EINVAL;
 		goto bad;
 	}
 	/*
 	 * Copy original headers (with the new protocol number) back
 	 * in place.
 	 */
 	m_copyback(m, 0, skip, ptr);
 
 	free(xd, M_XDATA);
 	crypto_freereq(crp);
 	AHSTAT_INC(ahs_hist[sav->alg_auth]);
 #ifdef REGRESSION
 	/* Emulate man-in-the-middle attack when ipsec_integrity is TRUE. */
 	if (V_ipsec_integrity) {
 		int alen;
 
 		/*
 		 * Corrupt HMAC if we want to test integrity verification of
 		 * the other side.
 		 */
 		alen = AUTHSIZE(sav);
 		m_copyback(m, m->m_pkthdr.len - alen, alen, ipseczeroes);
 	}
 #endif
 
 	/* NB: m is reclaimed by ipsec_process_done. */
 	error = ipsec_process_done(m, sp, sav, idx);
 	CURVNET_RESTORE();
 	return (error);
 bad:
 	CURVNET_RESTORE();
 	free(xd, M_XDATA);
 	crypto_freereq(crp);
 	key_freesav(&sav);
 	key_freesp(&sp);
 	return (error);
 }
 
 static struct xformsw ah_xformsw = {
 	.xf_type =	XF_AH,
 	.xf_name =	"IPsec AH",
 	.xf_init =	ah_init,
 	.xf_zeroize =	ah_zeroize,
 	.xf_input =	ah_input,
 	.xf_output =	ah_output,
 };
 
 SYSINIT(ah_xform_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_MIDDLE,
     xform_attach, &ah_xformsw);
 SYSUNINIT(ah_xform_uninit, SI_SUB_PROTO_DOMAIN, SI_ORDER_MIDDLE,
     xform_detach, &ah_xformsw);
Index: head/sys/netipsec/xform_esp.c
===================================================================
--- head/sys/netipsec/xform_esp.c	(revision 359373)
+++ head/sys/netipsec/xform_esp.c	(revision 359374)
@@ -1,1013 +1,974 @@
 /*	$FreeBSD$	*/
 /*	$OpenBSD: ip_esp.c,v 1.69 2001/06/26 06:18:59 angelos Exp $ */
 /*-
  * The authors of this code are John Ioannidis (ji@tla.org),
  * Angelos D. Keromytis (kermit@csd.uch.gr) and
  * Niels Provos (provos@physnet.uni-hamburg.de).
  *
  * The original version of this code was written by John Ioannidis
  * for BSD/OS in Athens, Greece, in November 1995.
  *
  * Ported to OpenBSD and NetBSD, with additional transforms, in December 1996,
  * by Angelos D. Keromytis.
  *
  * Additional transforms and features in 1997 and 1998 by Angelos D. Keromytis
  * and Niels Provos.
  *
  * Additional features in 1999 by Angelos D. Keromytis.
  *
  * Copyright (C) 1995, 1996, 1997, 1998, 1999 by John Ioannidis,
  * Angelos D. Keromytis and Niels Provos.
  * Copyright (c) 2001 Angelos D. Keromytis.
  *
  * Permission to use, copy, and modify this software with or without fee
  * is hereby granted, provided that this entire notice is included in
  * all copies of any software which is or includes a copy or
  * modification of this software.
  * You may use this code under the GNU public license if you so wish. Please
  * contribute changes back to the authors under this freer than GPL license
  * so that we may further the use of strong encryption without limitations to
  * all.
  *
  * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR
  * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY
  * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE
  * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR
  * PURPOSE.
  */
 #include "opt_inet.h"
 #include "opt_inet6.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/mbuf.h>
 #include <sys/socket.h>
 #include <sys/syslog.h>
 #include <sys/kernel.h>
 #include <sys/lock.h>
 #include <sys/random.h>
 #include <sys/mutex.h>
 #include <sys/sysctl.h>
 #include <sys/mutex.h>
 #include <machine/atomic.h>
 
 #include <net/if.h>
 #include <net/vnet.h>
 
 #include <netinet/in.h>
 #include <netinet/in_systm.h>
 #include <netinet/ip.h>
 #include <netinet/ip_ecn.h>
 #include <netinet/ip6.h>
 
 #include <netipsec/ipsec.h>
 #include <netipsec/ah.h>
 #include <netipsec/ah_var.h>
 #include <netipsec/esp.h>
 #include <netipsec/esp_var.h>
 #include <netipsec/xform.h>
 
 #ifdef INET6
 #include <netinet6/ip6_var.h>
 #include <netipsec/ipsec6.h>
 #include <netinet6/ip6_ecn.h>
 #endif
 
 #include <netipsec/key.h>
 #include <netipsec/key_debug.h>
 
 #include <opencrypto/cryptodev.h>
 #include <opencrypto/xform.h>
 
 VNET_DEFINE(int, esp_enable) = 1;
 VNET_PCPUSTAT_DEFINE(struct espstat, espstat);
 VNET_PCPUSTAT_SYSINIT(espstat);
 
 #ifdef VIMAGE
 VNET_PCPUSTAT_SYSUNINIT(espstat);
 #endif /* VIMAGE */
 
 SYSCTL_DECL(_net_inet_esp);
 SYSCTL_INT(_net_inet_esp, OID_AUTO, esp_enable,
 	CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(esp_enable), 0, "");
 SYSCTL_VNET_PCPUSTAT(_net_inet_esp, IPSECCTL_STATS, stats,
     struct espstat, espstat,
     "ESP statistics (struct espstat, netipsec/esp_var.h");
 
 static struct timeval deswarn, blfwarn, castwarn, camelliawarn;
 
 static int esp_input_cb(struct cryptop *op);
 static int esp_output_cb(struct cryptop *crp);
 
 size_t
 esp_hdrsiz(struct secasvar *sav)
 {
 	size_t size;
 
 	if (sav != NULL) {
 		/*XXX not right for null algorithm--does it matter??*/
 		IPSEC_ASSERT(sav->tdb_encalgxform != NULL,
 			("SA with null xform"));
 		if (sav->flags & SADB_X_EXT_OLD)
 			size = sizeof (struct esp);
 		else
 			size = sizeof (struct newesp);
 		size += sav->tdb_encalgxform->blocksize + 9;
 		/*XXX need alg check???*/
 		if (sav->tdb_authalgxform != NULL && sav->replay)
 			size += ah_hdrsiz(sav);
 	} else {
 		/*
 		 *   base header size
 		 * + max iv length for CBC mode
 		 * + max pad length
 		 * + sizeof (pad length field)
 		 * + sizeof (next header field)
 		 * + max icv supported.
 		 */
 		size = sizeof (struct newesp) + EALG_MAX_BLOCK_LEN + 9 + 16;
 	}
 	return size;
 }
 
 /*
  * esp_init() is called when an SPI is being set up.
  */
 static int
 esp_init(struct secasvar *sav, struct xformsw *xsp)
 {
 	const struct enc_xform *txform;
-	struct cryptoini cria, crie;
+	struct crypto_session_params csp;
 	int keylen;
 	int error;
 
 	txform = enc_algorithm_lookup(sav->alg_enc);
 	if (txform == NULL) {
 		DPRINTF(("%s: unsupported encryption algorithm %d\n",
 			__func__, sav->alg_enc));
 		return EINVAL;
 	}
 	if (sav->key_enc == NULL) {
 		DPRINTF(("%s: no encoding key for %s algorithm\n",
 			 __func__, txform->name));
 		return EINVAL;
 	}
 	if ((sav->flags & (SADB_X_EXT_OLD | SADB_X_EXT_IV4B)) ==
 	    SADB_X_EXT_IV4B) {
 		DPRINTF(("%s: 4-byte IV not supported with protocol\n",
 			__func__));
 		return EINVAL;
 	}
 
 	switch (sav->alg_enc) {
 	case SADB_EALG_DESCBC:
 		if (ratecheck(&deswarn, &ipsec_warn_interval))
 			gone_in(13, "DES cipher for IPsec");
 		break;
 	case SADB_X_EALG_BLOWFISHCBC:
 		if (ratecheck(&blfwarn, &ipsec_warn_interval))
 			gone_in(13, "Blowfish cipher for IPsec");
 		break;
 	case SADB_X_EALG_CAST128CBC:
 		if (ratecheck(&castwarn, &ipsec_warn_interval))
 			gone_in(13, "CAST cipher for IPsec");
 		break;
 	case SADB_X_EALG_CAMELLIACBC:
 		if (ratecheck(&camelliawarn, &ipsec_warn_interval))
 			gone_in(13, "Camellia cipher for IPsec");
 		break;
 	}
 
 	/* subtract off the salt, RFC4106, 8.1 and RFC3686, 5.1 */
 	keylen = _KEYLEN(sav->key_enc) - SAV_ISCTRORGCM(sav) * 4;
 	if (txform->minkey > keylen || keylen > txform->maxkey) {
 		DPRINTF(("%s: invalid key length %u, must be in the range "
 			"[%u..%u] for algorithm %s\n", __func__,
 			keylen, txform->minkey, txform->maxkey,
 			txform->name));
 		return EINVAL;
 	}
 
 	if (SAV_ISCTRORGCM(sav))
 		sav->ivlen = 8;	/* RFC4106 3.1 and RFC3686 3.1 */
 	else
 		sav->ivlen = txform->ivsize;
 
+	memset(&csp, 0, sizeof(csp));
+
 	/*
 	 * Setup AH-related state.
 	 */
 	if (sav->alg_auth != 0) {
-		error = ah_init0(sav, xsp, &cria);
+		error = ah_init0(sav, xsp, &csp);
 		if (error)
 			return error;
 	}
 
 	/* NB: override anything set in ah_init0 */
 	sav->tdb_xform = xsp;
 	sav->tdb_encalgxform = txform;
 
 	/*
 	 * Whenever AES-GCM is used for encryption, one
 	 * of the AES authentication algorithms is chosen
 	 * as well, based on the key size.
 	 */
 	if (sav->alg_enc == SADB_X_EALG_AESGCM16) {
 		switch (keylen) {
 		case AES_128_GMAC_KEY_LEN:
 			sav->alg_auth = SADB_X_AALG_AES128GMAC;
 			sav->tdb_authalgxform = &auth_hash_nist_gmac_aes_128;
 			break;
 		case AES_192_GMAC_KEY_LEN:
 			sav->alg_auth = SADB_X_AALG_AES192GMAC;
 			sav->tdb_authalgxform = &auth_hash_nist_gmac_aes_192;
 			break;
 		case AES_256_GMAC_KEY_LEN:
 			sav->alg_auth = SADB_X_AALG_AES256GMAC;
 			sav->tdb_authalgxform = &auth_hash_nist_gmac_aes_256;
 			break;
 		default:
 			DPRINTF(("%s: invalid key length %u"
 				 "for algorithm %s\n", __func__,
 				 keylen, txform->name));
 			return EINVAL;
 		}
-		bzero(&cria, sizeof(cria));
-		cria.cri_alg = sav->tdb_authalgxform->type;
-		cria.cri_key = sav->key_enc->key_data;
-		cria.cri_klen = _KEYBITS(sav->key_enc) - SAV_ISGCM(sav) * 32;
-	}
+		csp.csp_mode = CSP_MODE_AEAD;
+	} else if (sav->alg_auth != 0)
+		csp.csp_mode = CSP_MODE_ETA;
+	else
+		csp.csp_mode = CSP_MODE_CIPHER;
 
 	/* Initialize crypto session. */
-	bzero(&crie, sizeof(crie));
-	crie.cri_alg = sav->tdb_encalgxform->type;
-	crie.cri_key = sav->key_enc->key_data;
-	crie.cri_klen = _KEYBITS(sav->key_enc) - SAV_ISCTRORGCM(sav) * 32;
+	csp.csp_cipher_alg = sav->tdb_encalgxform->type;
+	csp.csp_cipher_key = sav->key_enc->key_data;
+	csp.csp_cipher_klen = _KEYBITS(sav->key_enc) / 8 -
+	    SAV_ISCTRORGCM(sav) * 4;
+	csp.csp_ivlen = txform->ivsize;
 
-	if (sav->tdb_authalgxform && sav->tdb_encalgxform) {
-		/* init both auth & enc */
-		crie.cri_next = &cria;
-		error = crypto_newsession(&sav->tdb_cryptoid,
-					  &crie, V_crypto_support);
-	} else if (sav->tdb_encalgxform) {
-		error = crypto_newsession(&sav->tdb_cryptoid,
-					  &crie, V_crypto_support);
-	} else if (sav->tdb_authalgxform) {
-		error = crypto_newsession(&sav->tdb_cryptoid,
-					  &cria, V_crypto_support);
-	} else {
-		/* XXX cannot happen? */
-		DPRINTF(("%s: no encoding OR authentication xform!\n",
-			__func__));
-		error = EINVAL;
-	}
+	error = crypto_newsession(&sav->tdb_cryptoid, &csp, V_crypto_support);
 	return error;
 }
 
 /*
  * Paranoia.
  */
 static int
 esp_zeroize(struct secasvar *sav)
 {
 	/* NB: ah_zerorize free's the crypto session state */
 	int error = ah_zeroize(sav);
 
 	if (sav->key_enc)
 		bzero(sav->key_enc->key_data, _KEYLEN(sav->key_enc));
 	sav->tdb_encalgxform = NULL;
 	sav->tdb_xform = NULL;
 	return error;
 }
 
 /*
  * ESP input processing, called (eventually) through the protocol switch.
  */
 static int
 esp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
 {
 	IPSEC_DEBUG_DECLARE(char buf[128]);
 	const struct auth_hash *esph;
 	const struct enc_xform *espx;
 	struct xform_data *xd;
-	struct cryptodesc *crde;
 	struct cryptop *crp;
 	struct newesp *esp;
 	uint8_t *ivp;
 	crypto_session_t cryptoid;
 	int alen, error, hlen, plen;
 
 	IPSEC_ASSERT(sav != NULL, ("null SA"));
 	IPSEC_ASSERT(sav->tdb_encalgxform != NULL, ("null encoding xform"));
 
 	error = EINVAL;
 	/* Valid IP Packet length ? */
 	if ( (skip&3) || (m->m_pkthdr.len&3) ){
 		DPRINTF(("%s: misaligned packet, skip %u pkt len %u",
 				__func__, skip, m->m_pkthdr.len));
 		ESPSTAT_INC(esps_badilen);
 		goto bad;
 	}
 
 	if (m->m_len < skip + sizeof(*esp)) {
 		m = m_pullup(m, skip + sizeof(*esp));
 		if (m == NULL) {
 			DPRINTF(("%s: cannot pullup header\n", __func__));
 			ESPSTAT_INC(esps_hdrops);	/*XXX*/
 			error = ENOBUFS;
 			goto bad;
 		}
 	}
 	esp = (struct newesp *)(mtod(m, caddr_t) + skip);
 
 	esph = sav->tdb_authalgxform;
 	espx = sav->tdb_encalgxform;
 
 	/* Determine the ESP header and auth length */
 	if (sav->flags & SADB_X_EXT_OLD)
 		hlen = sizeof (struct esp) + sav->ivlen;
 	else
 		hlen = sizeof (struct newesp) + sav->ivlen;
 
 	alen = xform_ah_authsize(esph);
 
 	/*
 	 * Verify payload length is multiple of encryption algorithm
 	 * block size.
 	 *
 	 * NB: This works for the null algorithm because the blocksize
 	 *     is 4 and all packets must be 4-byte aligned regardless
 	 *     of the algorithm.
 	 */
 	plen = m->m_pkthdr.len - (skip + hlen + alen);
 	if ((plen & (espx->blocksize - 1)) || (plen <= 0)) {
 		DPRINTF(("%s: payload of %d octets not a multiple of %d octets,"
 		    "  SA %s/%08lx\n", __func__, plen, espx->blocksize,
 		    ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)),
 		    (u_long)ntohl(sav->spi)));
 		ESPSTAT_INC(esps_badilen);
 		goto bad;
 	}
 
 	/*
 	 * Check sequence number.
 	 */
 	SECASVAR_LOCK(sav);
 	if (esph != NULL && sav->replay != NULL && sav->replay->wsize != 0) {
 		if (ipsec_chkreplay(ntohl(esp->esp_seq), sav) == 0) {
 			SECASVAR_UNLOCK(sav);
 			DPRINTF(("%s: packet replay check for %s\n", __func__,
 			    ipsec_sa2str(sav, buf, sizeof(buf))));
 			ESPSTAT_INC(esps_replay);
 			error = EACCES;
 			goto bad;
 		}
 	}
 	cryptoid = sav->tdb_cryptoid;
 	SECASVAR_UNLOCK(sav);
 
 	/* Update the counters */
 	ESPSTAT_ADD(esps_ibytes, m->m_pkthdr.len - (skip + hlen + alen));
 
 	/* Get crypto descriptors */
-	crp = crypto_getreq(esph && espx ? 2 : 1);
+	crp = crypto_getreq(cryptoid, M_NOWAIT);
 	if (crp == NULL) {
 		DPRINTF(("%s: failed to acquire crypto descriptors\n",
 			__func__));
 		ESPSTAT_INC(esps_crypto);
 		error = ENOBUFS;
 		goto bad;
 	}
 
 	/* Get IPsec-specific opaque pointer */
-	xd = malloc(sizeof(*xd) + alen, M_XDATA, M_NOWAIT | M_ZERO);
+	xd = malloc(sizeof(*xd), M_XDATA, M_NOWAIT | M_ZERO);
 	if (xd == NULL) {
 		DPRINTF(("%s: failed to allocate xform_data\n", __func__));
 		ESPSTAT_INC(esps_crypto);
 		crypto_freereq(crp);
 		error = ENOBUFS;
 		goto bad;
 	}
 
 	if (esph != NULL) {
-		struct cryptodesc *crda = crp->crp_desc;
-
-		IPSEC_ASSERT(crda != NULL, ("null ah crypto descriptor"));
-
-		/* Authentication descriptor */
-		crda->crd_skip = skip;
+		crp->crp_op = CRYPTO_OP_VERIFY_DIGEST;
+		crp->crp_aad_start = skip;
 		if (SAV_ISGCM(sav))
-			crda->crd_len = 8;	/* RFC4106 5, SPI + SN */
+			crp->crp_aad_length = 8; /* RFC4106 5, SPI + SN */
 		else
-			crda->crd_len = m->m_pkthdr.len - (skip + alen);
-		crda->crd_inject = m->m_pkthdr.len - alen;
-
-		crda->crd_alg = esph->type;
-
-		/* Copy the authenticator */
-		m_copydata(m, m->m_pkthdr.len - alen, alen,
-		    (caddr_t) (xd + 1));
-
-		/* Chain authentication request */
-		crde = crda->crd_next;
-	} else {
-		crde = crp->crp_desc;
+			crp->crp_aad_length = hlen;
+		crp->crp_digest_start = m->m_pkthdr.len - alen;
 	}
 
 	/* Crypto operation descriptor */
 	crp->crp_ilen = m->m_pkthdr.len; /* Total input length */
-	crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_CBIFSYNC;
+	crp->crp_flags = CRYPTO_F_CBIFSYNC;
 	if (V_async_crypto)
 		crp->crp_flags |= CRYPTO_F_ASYNC | CRYPTO_F_ASYNC_KEEPORDER;
-	crp->crp_buf = (caddr_t) m;
+	crp->crp_mbuf = m;
+	crp->crp_buf_type = CRYPTO_BUF_MBUF;
 	crp->crp_callback = esp_input_cb;
-	crp->crp_session = cryptoid;
-	crp->crp_opaque = (caddr_t) xd;
+	crp->crp_opaque = xd;
 
 	/* These are passed as-is to the callback */
 	xd->sav = sav;
 	xd->protoff = protoff;
 	xd->skip = skip;
 	xd->cryptoid = cryptoid;
 	xd->vnet = curvnet;
 
 	/* Decryption descriptor */
-	IPSEC_ASSERT(crde != NULL, ("null esp crypto descriptor"));
-	crde->crd_skip = skip + hlen;
-	crde->crd_len = m->m_pkthdr.len - (skip + hlen + alen);
-	crde->crd_inject = skip + hlen - sav->ivlen;
+	crp->crp_op |= CRYPTO_OP_DECRYPT;
+	crp->crp_payload_start = skip + hlen;
+	crp->crp_payload_length = m->m_pkthdr.len - (skip + hlen + alen);
 
 	if (SAV_ISCTRORGCM(sav)) {
-		ivp = &crde->crd_iv[0];
+		ivp = &crp->crp_iv[0];
 
 		/* GCM IV Format: RFC4106 4 */
 		/* CTR IV Format: RFC3686 4 */
 		/* Salt is last four bytes of key, RFC4106 8.1 */
 		/* Nonce is last four bytes of key, RFC3686 5.1 */
 		memcpy(ivp, sav->key_enc->key_data +
 		    _KEYLEN(sav->key_enc) - 4, 4);
 
 		if (SAV_ISCTR(sav)) {
 			/* Initial block counter is 1, RFC3686 4 */
 			be32enc(&ivp[sav->ivlen + 4], 1);
 		}
 
 		m_copydata(m, skip + hlen - sav->ivlen, sav->ivlen, &ivp[4]);
-		crde->crd_flags |= CRD_F_IV_EXPLICIT;
-	}
+		crp->crp_flags |= CRYPTO_F_IV_SEPARATE;
+	} else if (sav->ivlen != 0)
+		crp->crp_iv_start = skip + hlen - sav->ivlen;
 
-	crde->crd_alg = espx->type;
-
 	return (crypto_dispatch(crp));
 bad:
 	m_freem(m);
 	key_freesav(&sav);
 	return (error);
 }
 
 /*
  * ESP input callback from the crypto driver.
  */
 static int
 esp_input_cb(struct cryptop *crp)
 {
 	IPSEC_DEBUG_DECLARE(char buf[128]);
-	u_int8_t lastthree[3], aalg[AH_HMAC_MAXHASHLEN];
+	uint8_t lastthree[3];
 	const struct auth_hash *esph;
 	struct mbuf *m;
-	struct cryptodesc *crd;
 	struct xform_data *xd;
 	struct secasvar *sav;
 	struct secasindex *saidx;
-	caddr_t ptr;
 	crypto_session_t cryptoid;
 	int hlen, skip, protoff, error, alen;
 
-	crd = crp->crp_desc;
-	IPSEC_ASSERT(crd != NULL, ("null crypto descriptor!"));
-
-	m = (struct mbuf *) crp->crp_buf;
-	xd = (struct xform_data *) crp->crp_opaque;
+	m = crp->crp_mbuf;
+	xd = crp->crp_opaque;
 	CURVNET_SET(xd->vnet);
 	sav = xd->sav;
 	skip = xd->skip;
 	protoff = xd->protoff;
 	cryptoid = xd->cryptoid;
 	saidx = &sav->sah->saidx;
 	esph = sav->tdb_authalgxform;
 
 	/* Check for crypto errors */
 	if (crp->crp_etype) {
 		if (crp->crp_etype == EAGAIN) {
 			/* Reset the session ID */
 			if (ipsec_updateid(sav, &crp->crp_session, &cryptoid) != 0)
 				crypto_freesession(cryptoid);
 			xd->cryptoid = crp->crp_session;
 			CURVNET_RESTORE();
 			return (crypto_dispatch(crp));
 		}
-		ESPSTAT_INC(esps_noxform);
-		DPRINTF(("%s: crypto error %d\n", __func__, crp->crp_etype));
-		error = crp->crp_etype;
-		goto bad;
+
+		/* EBADMSG indicates authentication failure. */
+		if (!(crp->crp_etype == EBADMSG && esph != NULL)) {
+			ESPSTAT_INC(esps_noxform);
+			DPRINTF(("%s: crypto error %d\n", __func__,
+				crp->crp_etype));
+			error = crp->crp_etype;
+			goto bad;
+		}
 	}
 
 	/* Shouldn't happen... */
 	if (m == NULL) {
 		ESPSTAT_INC(esps_crypto);
 		DPRINTF(("%s: bogus returned buffer from crypto\n", __func__));
 		error = EINVAL;
 		goto bad;
 	}
 	ESPSTAT_INC(esps_hist[sav->alg_enc]);
 
 	/* If authentication was performed, check now. */
 	if (esph != NULL) {
 		alen = xform_ah_authsize(esph);
 		AHSTAT_INC(ahs_hist[sav->alg_auth]);
-		/* Copy the authenticator from the packet */
-		m_copydata(m, m->m_pkthdr.len - alen, alen, aalg);
-		ptr = (caddr_t) (xd + 1);
-
-		/* Verify authenticator */
-		if (timingsafe_bcmp(ptr, aalg, alen) != 0) {
+		if (crp->crp_etype == EBADMSG) {
 			DPRINTF(("%s: authentication hash mismatch for "
 			    "packet in SA %s/%08lx\n", __func__,
 			    ipsec_address(&saidx->dst, buf, sizeof(buf)),
 			    (u_long) ntohl(sav->spi)));
 			ESPSTAT_INC(esps_badauth);
 			error = EACCES;
 			goto bad;
 		}
 		m->m_flags |= M_AUTHIPDGM;
 		/* Remove trailing authenticator */
 		m_adj(m, -alen);
 	}
 
 	/* Release the crypto descriptors */
 	free(xd, M_XDATA), xd = NULL;
 	crypto_freereq(crp), crp = NULL;
 
 	/*
 	 * Packet is now decrypted.
 	 */
 	m->m_flags |= M_DECRYPTED;
 
 	/*
 	 * Update replay sequence number, if appropriate.
 	 */
 	if (sav->replay) {
 		u_int32_t seq;
 
 		m_copydata(m, skip + offsetof(struct newesp, esp_seq),
 			   sizeof (seq), (caddr_t) &seq);
 		SECASVAR_LOCK(sav);
 		if (ipsec_updatereplay(ntohl(seq), sav)) {
 			SECASVAR_UNLOCK(sav);
 			DPRINTF(("%s: packet replay check for %s\n", __func__,
 			    ipsec_sa2str(sav, buf, sizeof(buf))));
 			ESPSTAT_INC(esps_replay);
 			error = EACCES;
 			goto bad;
 		}
 		SECASVAR_UNLOCK(sav);
 	}
 
 	/* Determine the ESP header length */
 	if (sav->flags & SADB_X_EXT_OLD)
 		hlen = sizeof (struct esp) + sav->ivlen;
 	else
 		hlen = sizeof (struct newesp) + sav->ivlen;
 
 	/* Remove the ESP header and IV from the mbuf. */
 	error = m_striphdr(m, skip, hlen);
 	if (error) {
 		ESPSTAT_INC(esps_hdrops);
 		DPRINTF(("%s: bad mbuf chain, SA %s/%08lx\n", __func__,
 		    ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)),
 		    (u_long) ntohl(sav->spi)));
 		goto bad;
 	}
 
 	/* Save the last three bytes of decrypted data */
 	m_copydata(m, m->m_pkthdr.len - 3, 3, lastthree);
 
 	/* Verify pad length */
 	if (lastthree[1] + 2 > m->m_pkthdr.len - skip) {
 		ESPSTAT_INC(esps_badilen);
 		DPRINTF(("%s: invalid padding length %d for %u byte packet "
 		    "in SA %s/%08lx\n", __func__, lastthree[1],
 		    m->m_pkthdr.len - skip,
 		    ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)),
 		    (u_long) ntohl(sav->spi)));
 		error = EINVAL;
 		goto bad;
 	}
 
 	/* Verify correct decryption by checking the last padding bytes */
 	if ((sav->flags & SADB_X_EXT_PMASK) != SADB_X_EXT_PRAND) {
 		if (lastthree[1] != lastthree[0] && lastthree[1] != 0) {
 			ESPSTAT_INC(esps_badenc);
 			DPRINTF(("%s: decryption failed for packet in "
 			    "SA %s/%08lx\n", __func__, ipsec_address(
 			    &sav->sah->saidx.dst, buf, sizeof(buf)),
 			    (u_long) ntohl(sav->spi)));
 			error = EINVAL;
 			goto bad;
 		}
 	}
 
 	/*
 	 * RFC4303 2.6:
 	 * Silently drop packet if next header field is IPPROTO_NONE.
 	 */
 	if (lastthree[2] == IPPROTO_NONE)
 		goto bad;
 
 	/* Trim the mbuf chain to remove trailing authenticator and padding */
 	m_adj(m, -(lastthree[1] + 2));
 
 	/* Restore the Next Protocol field */
 	m_copyback(m, protoff, sizeof (u_int8_t), lastthree + 2);
 
 	switch (saidx->dst.sa.sa_family) {
 #ifdef INET6
 	case AF_INET6:
 		error = ipsec6_common_input_cb(m, sav, skip, protoff);
 		break;
 #endif
 #ifdef INET
 	case AF_INET:
 		error = ipsec4_common_input_cb(m, sav, skip, protoff);
 		break;
 #endif
 	default:
 		panic("%s: Unexpected address family: %d saidx=%p", __func__,
 		    saidx->dst.sa.sa_family, saidx);
 	}
 	CURVNET_RESTORE();
 	return error;
 bad:
 	CURVNET_RESTORE();
 	if (sav != NULL)
 		key_freesav(&sav);
 	if (m != NULL)
 		m_freem(m);
 	if (xd != NULL)
 		free(xd, M_XDATA);
 	if (crp != NULL)
 		crypto_freereq(crp);
 	return error;
 }
 /*
  * ESP output routine, called by ipsec[46]_perform_request().
  */
 static int
 esp_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
     u_int idx, int skip, int protoff)
 {
 	IPSEC_DEBUG_DECLARE(char buf[IPSEC_ADDRSTRLEN]);
-	struct cryptodesc *crde = NULL, *crda = NULL;
 	struct cryptop *crp;
 	const struct auth_hash *esph;
 	const struct enc_xform *espx;
 	struct mbuf *mo = NULL;
 	struct xform_data *xd;
 	struct secasindex *saidx;
 	unsigned char *pad;
 	uint8_t *ivp;
 	uint64_t cntr;
 	crypto_session_t cryptoid;
 	int hlen, rlen, padding, blks, alen, i, roff;
 	int error, maxpacketsize;
 	uint8_t prot;
 
 	IPSEC_ASSERT(sav != NULL, ("null SA"));
 	esph = sav->tdb_authalgxform;
 	espx = sav->tdb_encalgxform;
 	IPSEC_ASSERT(espx != NULL, ("null encoding xform"));
 
 	if (sav->flags & SADB_X_EXT_OLD)
 		hlen = sizeof (struct esp) + sav->ivlen;
 	else
 		hlen = sizeof (struct newesp) + sav->ivlen;
 
 	rlen = m->m_pkthdr.len - skip;	/* Raw payload length. */
 	/*
 	 * RFC4303 2.4 Requires 4 byte alignment.
 	 */
 	blks = MAX(4, espx->blocksize);		/* Cipher blocksize */
 
 	/* XXX clamp padding length a la KAME??? */
 	padding = ((blks - ((rlen + 2) % blks)) % blks) + 2;
 
 	alen = xform_ah_authsize(esph);
 
 	ESPSTAT_INC(esps_output);
 
 	saidx = &sav->sah->saidx;
 	/* Check for maximum packet size violations. */
 	switch (saidx->dst.sa.sa_family) {
 #ifdef INET
 	case AF_INET:
 		maxpacketsize = IP_MAXPACKET;
 		break;
 #endif /* INET */
 #ifdef INET6
 	case AF_INET6:
 		maxpacketsize = IPV6_MAXPACKET;
 		break;
 #endif /* INET6 */
 	default:
 		DPRINTF(("%s: unknown/unsupported protocol "
 		    "family %d, SA %s/%08lx\n", __func__,
 		    saidx->dst.sa.sa_family, ipsec_address(&saidx->dst,
 			buf, sizeof(buf)), (u_long) ntohl(sav->spi)));
 		ESPSTAT_INC(esps_nopf);
 		error = EPFNOSUPPORT;
 		goto bad;
 	}
 	/*
 	DPRINTF(("%s: skip %d hlen %d rlen %d padding %d alen %d blksd %d\n",
 		__func__, skip, hlen, rlen, padding, alen, blks)); */
 	if (skip + hlen + rlen + padding + alen > maxpacketsize) {
 		DPRINTF(("%s: packet in SA %s/%08lx got too big "
 		    "(len %u, max len %u)\n", __func__,
 		    ipsec_address(&saidx->dst, buf, sizeof(buf)),
 		    (u_long) ntohl(sav->spi),
 		    skip + hlen + rlen + padding + alen, maxpacketsize));
 		ESPSTAT_INC(esps_toobig);
 		error = EMSGSIZE;
 		goto bad;
 	}
 
 	/* Update the counters. */
 	ESPSTAT_ADD(esps_obytes, m->m_pkthdr.len - skip);
 
 	m = m_unshare(m, M_NOWAIT);
 	if (m == NULL) {
 		DPRINTF(("%s: cannot clone mbuf chain, SA %s/%08lx\n", __func__,
 		    ipsec_address(&saidx->dst, buf, sizeof(buf)),
 		    (u_long) ntohl(sav->spi)));
 		ESPSTAT_INC(esps_hdrops);
 		error = ENOBUFS;
 		goto bad;
 	}
 
 	/* Inject ESP header. */
 	mo = m_makespace(m, skip, hlen, &roff);
 	if (mo == NULL) {
 		DPRINTF(("%s: %u byte ESP hdr inject failed for SA %s/%08lx\n",
 		    __func__, hlen, ipsec_address(&saidx->dst, buf,
 		    sizeof(buf)), (u_long) ntohl(sav->spi)));
 		ESPSTAT_INC(esps_hdrops);	/* XXX diffs from openbsd */
 		error = ENOBUFS;
 		goto bad;
 	}
 
 	/* Initialize ESP header. */
 	bcopy((caddr_t) &sav->spi, mtod(mo, caddr_t) + roff,
 	    sizeof(uint32_t));
 	SECASVAR_LOCK(sav);
 	if (sav->replay) {
 		uint32_t replay;
 
 #ifdef REGRESSION
 		/* Emulate replay attack when ipsec_replay is TRUE. */
 		if (!V_ipsec_replay)
 #endif
 			sav->replay->count++;
 		replay = htonl(sav->replay->count);
 
 		bcopy((caddr_t) &replay, mtod(mo, caddr_t) + roff +
 		    sizeof(uint32_t), sizeof(uint32_t));
 	}
 	cryptoid = sav->tdb_cryptoid;
 	if (SAV_ISCTRORGCM(sav))
 		cntr = sav->cntr++;
 	SECASVAR_UNLOCK(sav);
 
 	/*
 	 * Add padding -- better to do it ourselves than use the crypto engine,
 	 * although if/when we support compression, we'd have to do that.
 	 */
 	pad = (u_char *) m_pad(m, padding + alen);
 	if (pad == NULL) {
 		DPRINTF(("%s: m_pad failed for SA %s/%08lx\n", __func__,
 		    ipsec_address(&saidx->dst, buf, sizeof(buf)),
 		    (u_long) ntohl(sav->spi)));
 		m = NULL;		/* NB: free'd by m_pad */
 		error = ENOBUFS;
 		goto bad;
 	}
 
 	/*
 	 * Add padding: random, zero, or self-describing.
 	 * XXX catch unexpected setting
 	 */
 	switch (sav->flags & SADB_X_EXT_PMASK) {
 	case SADB_X_EXT_PRAND:
 		arc4random_buf(pad, padding - 2);
 		break;
 	case SADB_X_EXT_PZERO:
 		bzero(pad, padding - 2);
 		break;
 	case SADB_X_EXT_PSEQ:
 		for (i = 0; i < padding - 2; i++)
 			pad[i] = i+1;
 		break;
 	}
 
 	/* Fix padding length and Next Protocol in padding itself. */
 	pad[padding - 2] = padding - 2;
 	m_copydata(m, protoff, sizeof(u_int8_t), pad + padding - 1);
 
 	/* Fix Next Protocol in IPv4/IPv6 header. */
 	prot = IPPROTO_ESP;
 	m_copyback(m, protoff, sizeof(u_int8_t), (u_char *) &prot);
 
-	/* Get crypto descriptors. */
-	crp = crypto_getreq(esph != NULL ? 2 : 1);
+	/* Get crypto descriptor. */
+	crp = crypto_getreq(cryptoid, M_NOWAIT);
 	if (crp == NULL) {
-		DPRINTF(("%s: failed to acquire crypto descriptors\n",
+		DPRINTF(("%s: failed to acquire crypto descriptor\n",
 			__func__));
 		ESPSTAT_INC(esps_crypto);
 		error = ENOBUFS;
 		goto bad;
 	}
 
 	/* IPsec-specific opaque crypto info. */
 	xd =  malloc(sizeof(struct xform_data), M_XDATA, M_NOWAIT | M_ZERO);
 	if (xd == NULL) {
 		crypto_freereq(crp);
 		DPRINTF(("%s: failed to allocate xform_data\n", __func__));
 		ESPSTAT_INC(esps_crypto);
 		error = ENOBUFS;
 		goto bad;
 	}
 
-	crde = crp->crp_desc;
-	crda = crde->crd_next;
-
 	/* Encryption descriptor. */
-	crde->crd_skip = skip + hlen;
-	crde->crd_len = m->m_pkthdr.len - (skip + hlen + alen);
-	crde->crd_flags = CRD_F_ENCRYPT;
-	crde->crd_inject = skip + hlen - sav->ivlen;
+	crp->crp_payload_start = skip + hlen;
+	crp->crp_payload_length = m->m_pkthdr.len - (skip + hlen + alen);
+	crp->crp_op = CRYPTO_OP_ENCRYPT;
 
 	/* Encryption operation. */
-	crde->crd_alg = espx->type;
 	if (SAV_ISCTRORGCM(sav)) {
-		ivp = &crde->crd_iv[0];
+		ivp = &crp->crp_iv[0];
 
 		/* GCM IV Format: RFC4106 4 */
 		/* CTR IV Format: RFC3686 4 */
 		/* Salt is last four bytes of key, RFC4106 8.1 */
 		/* Nonce is last four bytes of key, RFC3686 5.1 */
 		memcpy(ivp, sav->key_enc->key_data +
 		    _KEYLEN(sav->key_enc) - 4, 4);
 		be64enc(&ivp[4], cntr);
 		if (SAV_ISCTR(sav)) {
 			/* Initial block counter is 1, RFC3686 4 */
 			/* XXXAE: should we use this only for first packet? */
 			be32enc(&ivp[sav->ivlen + 4], 1);
 		}
 
 		m_copyback(m, skip + hlen - sav->ivlen, sav->ivlen, &ivp[4]);
-		crde->crd_flags |= CRD_F_IV_EXPLICIT|CRD_F_IV_PRESENT;
+		crp->crp_flags |= CRYPTO_F_IV_SEPARATE;
+	} else if (sav->ivlen != 0) {
+		crp->crp_iv_start = skip + hlen - sav->ivlen;
+		crp->crp_flags |= CRYPTO_F_IV_GENERATE;
 	}
 
 	/* Callback parameters */
 	xd->sp = sp;
 	xd->sav = sav;
 	xd->idx = idx;
 	xd->cryptoid = cryptoid;
 	xd->vnet = curvnet;
 
 	/* Crypto operation descriptor. */
 	crp->crp_ilen = m->m_pkthdr.len; /* Total input length. */
-	crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_CBIFSYNC;
+	crp->crp_flags |= CRYPTO_F_CBIFSYNC;
 	if (V_async_crypto)
 		crp->crp_flags |= CRYPTO_F_ASYNC | CRYPTO_F_ASYNC_KEEPORDER;
-	crp->crp_buf = (caddr_t) m;
+	crp->crp_mbuf = m;
+	crp->crp_buf_type = CRYPTO_BUF_MBUF;
 	crp->crp_callback = esp_output_cb;
-	crp->crp_opaque = (caddr_t) xd;
-	crp->crp_session = cryptoid;
+	crp->crp_opaque = xd;
 
 	if (esph) {
 		/* Authentication descriptor. */
-		crda->crd_alg = esph->type;
-		crda->crd_skip = skip;
+		crp->crp_op |= CRYPTO_OP_COMPUTE_DIGEST;
+		crp->crp_aad_start = skip;
 		if (SAV_ISGCM(sav))
-			crda->crd_len = 8;	/* RFC4106 5, SPI + SN */
+			crp->crp_aad_length = 8; /* RFC4106 5, SPI + SN */
 		else
-			crda->crd_len = m->m_pkthdr.len - (skip + alen);
-		crda->crd_inject = m->m_pkthdr.len - alen;
+			crp->crp_aad_length = hlen;
+		crp->crp_digest_start = m->m_pkthdr.len - alen;
 	}
 
 	return crypto_dispatch(crp);
 bad:
 	if (m)
 		m_freem(m);
 	key_freesav(&sav);
 	key_freesp(&sp);
 	return (error);
 }
 /*
  * ESP output callback from the crypto driver.
  */
 static int
 esp_output_cb(struct cryptop *crp)
 {
 	struct xform_data *xd;
 	struct secpolicy *sp;
 	struct secasvar *sav;
 	struct mbuf *m;
 	crypto_session_t cryptoid;
 	u_int idx;
 	int error;
 
 	xd = (struct xform_data *) crp->crp_opaque;
 	CURVNET_SET(xd->vnet);
 	m = (struct mbuf *) crp->crp_buf;
 	sp = xd->sp;
 	sav = xd->sav;
 	idx = xd->idx;
 	cryptoid = xd->cryptoid;
 
 	/* Check for crypto errors. */
 	if (crp->crp_etype) {
 		if (crp->crp_etype == EAGAIN) {
 			/* Reset the session ID */
 			if (ipsec_updateid(sav, &crp->crp_session, &cryptoid) != 0)
 				crypto_freesession(cryptoid);
 			xd->cryptoid = crp->crp_session;
 			CURVNET_RESTORE();
 			return (crypto_dispatch(crp));
 		}
 		ESPSTAT_INC(esps_noxform);
 		DPRINTF(("%s: crypto error %d\n", __func__, crp->crp_etype));
 		error = crp->crp_etype;
 		m_freem(m);
 		goto bad;
 	}
 
 	/* Shouldn't happen... */
 	if (m == NULL) {
 		ESPSTAT_INC(esps_crypto);
 		DPRINTF(("%s: bogus returned buffer from crypto\n", __func__));
 		error = EINVAL;
 		goto bad;
 	}
 	free(xd, M_XDATA);
 	crypto_freereq(crp);
 	ESPSTAT_INC(esps_hist[sav->alg_enc]);
 	if (sav->tdb_authalgxform != NULL)
 		AHSTAT_INC(ahs_hist[sav->alg_auth]);
 
 #ifdef REGRESSION
 	/* Emulate man-in-the-middle attack when ipsec_integrity is TRUE. */
 	if (V_ipsec_integrity) {
 		static unsigned char ipseczeroes[AH_HMAC_MAXHASHLEN];
 		const struct auth_hash *esph;
 
 		/*
 		 * Corrupt HMAC if we want to test integrity verification of
 		 * the other side.
 		 */
 		esph = sav->tdb_authalgxform;
 		if (esph !=  NULL) {
 			int alen;
 
 			alen = xform_ah_authsize(esph);
 			m_copyback(m, m->m_pkthdr.len - alen,
 			    alen, ipseczeroes);
 		}
 	}
 #endif
 
 	/* NB: m is reclaimed by ipsec_process_done. */
 	error = ipsec_process_done(m, sp, sav, idx);
 	CURVNET_RESTORE();
 	return (error);
 bad:
 	CURVNET_RESTORE();
 	free(xd, M_XDATA);
 	crypto_freereq(crp);
 	key_freesav(&sav);
 	key_freesp(&sp);
 	return (error);
 }
 
 static struct xformsw esp_xformsw = {
 	.xf_type =	XF_ESP,
 	.xf_name =	"IPsec ESP",
 	.xf_init =	esp_init,
 	.xf_zeroize =	esp_zeroize,
 	.xf_input =	esp_input,
 	.xf_output =	esp_output,
 };
 
 SYSINIT(esp_xform_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_MIDDLE,
     xform_attach, &esp_xformsw);
 SYSUNINIT(esp_xform_uninit, SI_SUB_PROTO_DOMAIN, SI_ORDER_MIDDLE,
     xform_detach, &esp_xformsw);
Index: head/sys/netipsec/xform_ipcomp.c
===================================================================
--- head/sys/netipsec/xform_ipcomp.c	(revision 359373)
+++ head/sys/netipsec/xform_ipcomp.c	(revision 359374)
@@ -1,780 +1,779 @@
 /*	$FreeBSD$	*/
 /* $OpenBSD: ip_ipcomp.c,v 1.1 2001/07/05 12:08:52 jjbg Exp $ */
 
 /*-
  * SPDX-License-Identifier: BSD-3-Clause
  *
  * Copyright (c) 2001 Jean-Jacques Bernard-Gundol (jj@wabbitt.org)
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  *
  * 1. Redistributions of source code must retain the above copyright
  *   notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *   notice, this list of conditions and the following disclaimer in the
  *   documentation and/or other materials provided with the distribution.
  * 3. The name of the author may not be used to endorse or promote products
  *   derived from this software without specific prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
 /* IP payload compression protocol (IPComp), see RFC 2393 */
 #include "opt_inet.h"
 #include "opt_inet6.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/mbuf.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <sys/socket.h>
 #include <sys/kernel.h>
 #include <sys/protosw.h>
 #include <sys/sysctl.h>
 
 #include <netinet/in.h>
 #include <netinet/in_systm.h>
 #include <netinet/ip.h>
 #include <netinet/ip_var.h>
 #include <netinet/ip_encap.h>
 
 #include <net/netisr.h>
 #include <net/vnet.h>
 #include <net/if.h>	/* XXXGL: net_epoch should move out there */
 #include <net/if_var.h>	/* XXXGL: net_epoch should move out there */
 
 #include <netipsec/ipsec.h>
 #include <netipsec/xform.h>
 
 #ifdef INET6
 #include <netinet/ip6.h>
 #include <netinet6/ip6_var.h>
 #include <netipsec/ipsec6.h>
 #endif
 
 #include <netipsec/ipcomp.h>
 #include <netipsec/ipcomp_var.h>
 
 #include <netipsec/key.h>
 #include <netipsec/key_debug.h>
 
 #include <opencrypto/cryptodev.h>
 #include <opencrypto/deflate.h>
 #include <opencrypto/xform.h>
 
 VNET_DEFINE(int, ipcomp_enable) = 1;
 VNET_PCPUSTAT_DEFINE(struct ipcompstat, ipcompstat);
 VNET_PCPUSTAT_SYSINIT(ipcompstat);
 
 #ifdef VIMAGE
 VNET_PCPUSTAT_SYSUNINIT(ipcompstat);
 #endif /* VIMAGE */
 
 SYSCTL_DECL(_net_inet_ipcomp);
 SYSCTL_INT(_net_inet_ipcomp, OID_AUTO, ipcomp_enable,
 	CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(ipcomp_enable), 0, "");
 SYSCTL_VNET_PCPUSTAT(_net_inet_ipcomp, IPSECCTL_STATS, stats,
     struct ipcompstat, ipcompstat,
     "IPCOMP statistics (struct ipcompstat, netipsec/ipcomp_var.h");
 
 static int ipcomp_input_cb(struct cryptop *crp);
 static int ipcomp_output_cb(struct cryptop *crp);
 
 /*
  * RFC 3173 p 2.2. Non-Expansion Policy:
  * If the total size of a compressed payload and the IPComp header, as
  * defined in section 3, is not smaller than the size of the original
  * payload, the IP datagram MUST be sent in the original non-compressed
  * form.
  *
  * When we use IPComp in tunnel mode, for small packets we will receive
  * encapsulated IP-IP datagrams without any compression and without IPComp
  * header.
  */
 static int
 ipcomp_encapcheck(union sockaddr_union *src, union sockaddr_union *dst)
 {
 	struct secasvar *sav;
 
 	sav = key_allocsa_tunnel(src, dst, IPPROTO_IPCOMP);
 	if (sav == NULL)
 		return (0);
 	key_freesav(&sav);
 
 	if (src->sa.sa_family == AF_INET)
 		return (sizeof(struct in_addr) << 4);
 	else
 		return (sizeof(struct in6_addr) << 4);
 }
 
 static int
 ipcomp_nonexp_input(struct mbuf *m, int off, int proto, void *arg __unused)
 {
 	int isr;
 
 	NET_EPOCH_ASSERT();
 
 	switch (proto) {
 #ifdef INET
 	case IPPROTO_IPV4:
 		isr = NETISR_IP;
 		break;
 #endif
 #ifdef INET6
 	case IPPROTO_IPV6:
 		isr = NETISR_IPV6;
 		break;
 #endif
 	default:
 		IPCOMPSTAT_INC(ipcomps_nopf);
 		m_freem(m);
 		return (IPPROTO_DONE);
 	}
 	m_adj(m, off);
 	IPCOMPSTAT_ADD(ipcomps_ibytes, m->m_pkthdr.len);
 	IPCOMPSTAT_INC(ipcomps_input);
 	netisr_dispatch(isr, m);
 	return (IPPROTO_DONE);
 }
 
 /*
  * ipcomp_init() is called when an CPI is being set up.
  */
 static int
 ipcomp_init(struct secasvar *sav, struct xformsw *xsp)
 {
 	const struct comp_algo *tcomp;
-	struct cryptoini cric;
+	struct crypto_session_params csp;
 
 	/* NB: algorithm really comes in alg_enc and not alg_comp! */
 	tcomp = comp_algorithm_lookup(sav->alg_enc);
 	if (tcomp == NULL) {
 		DPRINTF(("%s: unsupported compression algorithm %d\n", __func__,
 			 sav->alg_comp));
 		return EINVAL;
 	}
 	sav->alg_comp = sav->alg_enc;		/* set for doing histogram */
 	sav->tdb_xform = xsp;
 	sav->tdb_compalgxform = tcomp;
 
 	/* Initialize crypto session */
-	bzero(&cric, sizeof (cric));
-	cric.cri_alg = sav->tdb_compalgxform->type;
+	memset(&csp, 0, sizeof(csp));
+	csp.csp_mode = CSP_MODE_COMPRESS;
+	csp.csp_cipher_alg = sav->tdb_compalgxform->type;
 
-	return crypto_newsession(&sav->tdb_cryptoid, &cric, V_crypto_support);
+	return crypto_newsession(&sav->tdb_cryptoid, &csp, V_crypto_support);
 }
 
 /*
  * ipcomp_zeroize() used when IPCA is deleted
  */
 static int
 ipcomp_zeroize(struct secasvar *sav)
 {
 
 	crypto_freesession(sav->tdb_cryptoid);
 	sav->tdb_cryptoid = NULL;
 	return 0;
 }
 
 /*
  * ipcomp_input() gets called to uncompress an input packet
  */
 static int
 ipcomp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
 {
 	struct xform_data *xd;
-	struct cryptodesc *crdc;
 	struct cryptop *crp;
 	struct ipcomp *ipcomp;
+	crypto_session_t cryptoid;
 	caddr_t addr;
 	int error, hlen = IPCOMP_HLENGTH;
 
 	/*
 	 * Check that the next header of the IPComp is not IPComp again, before
 	 * doing any real work.  Given it is not possible to do double
 	 * compression it means someone is playing tricks on us.
 	 */
 	error = ENOBUFS;
 	if (m->m_len < skip + hlen && (m = m_pullup(m, skip + hlen)) == NULL) {
 		IPCOMPSTAT_INC(ipcomps_hdrops);		/*XXX*/
 		DPRINTF(("%s: m_pullup failed\n", __func__));
 		key_freesav(&sav);
 		return (error);
 	}
 	addr = (caddr_t) mtod(m, struct ip *) + skip;
 	ipcomp = (struct ipcomp *)addr;
 	if (ipcomp->comp_nxt == IPPROTO_IPCOMP) {
 		IPCOMPSTAT_INC(ipcomps_pdrops);	/* XXX have our own stats? */
 		DPRINTF(("%s: recursive compression detected\n", __func__));
 		error = EINVAL;
 		goto bad;
 	}
 
+	SECASVAR_LOCK(sav);
+	cryptoid = sav->tdb_cryptoid;
+	SECASVAR_UNLOCK(sav);
+
 	/* Get crypto descriptors */
-	crp = crypto_getreq(1);
+	crp = crypto_getreq(cryptoid, M_NOWAIT);
 	if (crp == NULL) {
 		DPRINTF(("%s: no crypto descriptors\n", __func__));
 		IPCOMPSTAT_INC(ipcomps_crypto);
 		goto bad;
 	}
 	/* Get IPsec-specific opaque pointer */
 	xd = malloc(sizeof(*xd), M_XDATA, M_NOWAIT | M_ZERO);
 	if (xd == NULL) {
 		DPRINTF(("%s: cannot allocate xform_data\n", __func__));
 		IPCOMPSTAT_INC(ipcomps_crypto);
 		crypto_freereq(crp);
 		goto bad;
 	}
-	crdc = crp->crp_desc;
 
-	crdc->crd_skip = skip + hlen;
-	crdc->crd_len = m->m_pkthdr.len - (skip + hlen);
-	crdc->crd_inject = skip;
-
 	/* Decompression operation */
-	crdc->crd_alg = sav->tdb_compalgxform->type;
+	crp->crp_op = CRYPTO_OP_DECOMPRESS;
+	crp->crp_payload_start = skip + hlen;
+	crp->crp_payload_length = m->m_pkthdr.len - (skip + hlen);
 
-
 	/* Crypto operation descriptor */
 	crp->crp_ilen = m->m_pkthdr.len - (skip + hlen);
-	crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_CBIFSYNC;
-	crp->crp_buf = (caddr_t) m;
+	crp->crp_flags = CRYPTO_F_CBIFSYNC;
+	crp->crp_mbuf = m;
+	crp->crp_buf_type = CRYPTO_BUF_MBUF;
 	crp->crp_callback = ipcomp_input_cb;
-	crp->crp_opaque = (caddr_t) xd;
+	crp->crp_opaque = xd;
 
 	/* These are passed as-is to the callback */
 	xd->sav = sav;
 	xd->protoff = protoff;
 	xd->skip = skip;
 	xd->vnet = curvnet;
+	xd->cryptoid = cryptoid;
 
 	SECASVAR_LOCK(sav);
 	crp->crp_session = xd->cryptoid = sav->tdb_cryptoid;
 	SECASVAR_UNLOCK(sav);
 
 	return crypto_dispatch(crp);
 bad:
 	m_freem(m);
 	key_freesav(&sav);
 	return (error);
 }
 
 /*
  * IPComp input callback from the crypto driver.
  */
 static int
 ipcomp_input_cb(struct cryptop *crp)
 {
 	IPSEC_DEBUG_DECLARE(char buf[IPSEC_ADDRSTRLEN]);
 	struct xform_data *xd;
 	struct mbuf *m;
 	struct secasvar *sav;
 	struct secasindex *saidx;
 	caddr_t addr;
 	crypto_session_t cryptoid;
 	int hlen = IPCOMP_HLENGTH, error, clen;
 	int skip, protoff;
 	uint8_t nproto;
 
-	m = (struct mbuf *) crp->crp_buf;
-	xd = (struct xform_data *) crp->crp_opaque;
+	m = crp->crp_mbuf;
+	xd = crp->crp_opaque;
 	CURVNET_SET(xd->vnet);
 	sav = xd->sav;
 	skip = xd->skip;
 	protoff = xd->protoff;
 	cryptoid = xd->cryptoid;
 	saidx = &sav->sah->saidx;
 	IPSEC_ASSERT(saidx->dst.sa.sa_family == AF_INET ||
 		saidx->dst.sa.sa_family == AF_INET6,
 		("unexpected protocol family %u", saidx->dst.sa.sa_family));
 
 	/* Check for crypto errors */
 	if (crp->crp_etype) {
 		if (crp->crp_etype == EAGAIN) {
 			/* Reset the session ID */
 			if (ipsec_updateid(sav, &crp->crp_session, &cryptoid) != 0)
 				crypto_freesession(cryptoid);
 			xd->cryptoid = crp->crp_session;
 			CURVNET_RESTORE();
 			return (crypto_dispatch(crp));
 		}
 		IPCOMPSTAT_INC(ipcomps_noxform);
 		DPRINTF(("%s: crypto error %d\n", __func__, crp->crp_etype));
 		error = crp->crp_etype;
 		goto bad;
 	}
 	/* Shouldn't happen... */
 	if (m == NULL) {
 		IPCOMPSTAT_INC(ipcomps_crypto);
 		DPRINTF(("%s: null mbuf returned from crypto\n", __func__));
 		error = EINVAL;
 		goto bad;
 	}
 	IPCOMPSTAT_INC(ipcomps_hist[sav->alg_comp]);
 
 	clen = crp->crp_olen;		/* Length of data after processing */
 
 	/* Release the crypto descriptors */
 	free(xd, M_XDATA), xd = NULL;
 	crypto_freereq(crp), crp = NULL;
 
 	/* In case it's not done already, adjust the size of the mbuf chain */
 	m->m_pkthdr.len = clen + hlen + skip;
 
 	if (m->m_len < skip + hlen && (m = m_pullup(m, skip + hlen)) == NULL) {
 		IPCOMPSTAT_INC(ipcomps_hdrops);		/*XXX*/
 		DPRINTF(("%s: m_pullup failed\n", __func__));
 		error = EINVAL;				/*XXX*/
 		goto bad;
 	}
 
 	/* Keep the next protocol field */
 	addr = (caddr_t) mtod(m, struct ip *) + skip;
 	nproto = ((struct ipcomp *) addr)->comp_nxt;
 
 	/* Remove the IPCOMP header */
 	error = m_striphdr(m, skip, hlen);
 	if (error) {
 		IPCOMPSTAT_INC(ipcomps_hdrops);
 		DPRINTF(("%s: bad mbuf chain, IPCA %s/%08lx\n", __func__,
 		    ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)),
 		    (u_long) ntohl(sav->spi)));
 		goto bad;
 	}
 
 	/* Restore the Next Protocol field */
 	m_copyback(m, protoff, sizeof (u_int8_t), (u_int8_t *) &nproto);
 
 	switch (saidx->dst.sa.sa_family) {
 #ifdef INET6
 	case AF_INET6:
 		error = ipsec6_common_input_cb(m, sav, skip, protoff);
 		break;
 #endif
 #ifdef INET
 	case AF_INET:
 		error = ipsec4_common_input_cb(m, sav, skip, protoff);
 		break;
 #endif
 	default:
 		panic("%s: Unexpected address family: %d saidx=%p", __func__,
 		    saidx->dst.sa.sa_family, saidx);
 	}
 	CURVNET_RESTORE();
 	return error;
 bad:
 	CURVNET_RESTORE();
 	if (sav != NULL)
 		key_freesav(&sav);
 	if (m != NULL)
 		m_freem(m);
 	if (xd != NULL)
 		free(xd, M_XDATA);
 	if (crp != NULL)
 		crypto_freereq(crp);
 	return error;
 }
 
 /*
  * IPComp output routine, called by ipsec[46]_perform_request()
  */
 static int
 ipcomp_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
     u_int idx, int skip, int protoff)
 {
 	IPSEC_DEBUG_DECLARE(char buf[IPSEC_ADDRSTRLEN]);
 	const struct comp_algo *ipcompx;
-	struct cryptodesc *crdc;
 	struct cryptop *crp;
 	struct xform_data *xd;
+	crypto_session_t cryptoid;
 	int error, ralen, maxpacketsize;
 
 	IPSEC_ASSERT(sav != NULL, ("null SA"));
 	ipcompx = sav->tdb_compalgxform;
 	IPSEC_ASSERT(ipcompx != NULL, ("null compression xform"));
 
 	/*
 	 * Do not touch the packet in case our payload to compress
 	 * is lower than the minimal threshold of the compression
 	 * alogrithm.  We will just send out the data uncompressed.
 	 * See RFC 3173, 2.2. Non-Expansion Policy.
 	 */
 	if (m->m_pkthdr.len <= ipcompx->minlen) {
 		IPCOMPSTAT_INC(ipcomps_threshold);
 		return ipsec_process_done(m, sp, sav, idx);
 	}
 
 	ralen = m->m_pkthdr.len - skip;	/* Raw payload length before comp. */
 	IPCOMPSTAT_INC(ipcomps_output);
 
 	/* Check for maximum packet size violations. */
 	switch (sav->sah->saidx.dst.sa.sa_family) {
 #ifdef INET
 	case AF_INET:
 		maxpacketsize = IP_MAXPACKET;
 		break;
 #endif /* INET */
 #ifdef INET6
 	case AF_INET6:
 		maxpacketsize = IPV6_MAXPACKET;
 		break;
 #endif /* INET6 */
 	default:
 		IPCOMPSTAT_INC(ipcomps_nopf);
 		DPRINTF(("%s: unknown/unsupported protocol family %d, "
 		    "IPCA %s/%08lx\n", __func__,
 		    sav->sah->saidx.dst.sa.sa_family,
 		    ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)),
 		    (u_long) ntohl(sav->spi)));
 		error = EPFNOSUPPORT;
 		goto bad;
 	}
 	if (ralen + skip + IPCOMP_HLENGTH > maxpacketsize) {
 		IPCOMPSTAT_INC(ipcomps_toobig);
 		DPRINTF(("%s: packet in IPCA %s/%08lx got too big "
 		    "(len %u, max len %u)\n", __func__,
 		    ipsec_address(&sav->sah->saidx.dst, buf, sizeof(buf)),
 		    (u_long) ntohl(sav->spi),
 		    ralen + skip + IPCOMP_HLENGTH, maxpacketsize));
 		error = EMSGSIZE;
 		goto bad;
 	}
 
 	/* Update the counters */
 	IPCOMPSTAT_ADD(ipcomps_obytes, m->m_pkthdr.len - skip);
 
 	m = m_unshare(m, M_NOWAIT);
 	if (m == NULL) {
 		IPCOMPSTAT_INC(ipcomps_hdrops);
 		DPRINTF(("%s: cannot clone mbuf chain, IPCA %s/%08lx\n",
 		    __func__, ipsec_address(&sav->sah->saidx.dst, buf,
 		    sizeof(buf)), (u_long) ntohl(sav->spi)));
 		error = ENOBUFS;
 		goto bad;
 	}
 
 	/* Ok now, we can pass to the crypto processing. */
+	SECASVAR_LOCK(sav);
+	cryptoid = sav->tdb_cryptoid;
+	SECASVAR_UNLOCK(sav);
 
 	/* Get crypto descriptors */
-	crp = crypto_getreq(1);
+	crp = crypto_getreq(cryptoid, M_NOWAIT);
 	if (crp == NULL) {
 		IPCOMPSTAT_INC(ipcomps_crypto);
 		DPRINTF(("%s: failed to acquire crypto descriptor\n",__func__));
 		error = ENOBUFS;
 		goto bad;
 	}
-	crdc = crp->crp_desc;
 
 	/* Compression descriptor */
-	crdc->crd_skip = skip;
-	crdc->crd_len = ralen;
-	crdc->crd_flags = CRD_F_COMP;
-	crdc->crd_inject = skip;
+	crp->crp_op = CRYPTO_OP_COMPRESS;
+	crp->crp_payload_start = skip;
+	crp->crp_payload_length = ralen;
 
-	/* Compression operation */
-	crdc->crd_alg = ipcompx->type;
-
 	/* IPsec-specific opaque crypto info */
 	xd =  malloc(sizeof(struct xform_data), M_XDATA, M_NOWAIT | M_ZERO);
 	if (xd == NULL) {
 		IPCOMPSTAT_INC(ipcomps_crypto);
 		DPRINTF(("%s: failed to allocate xform_data\n", __func__));
 		crypto_freereq(crp);
 		error = ENOBUFS;
 		goto bad;
 	}
 
 	xd->sp = sp;
 	xd->sav = sav;
 	xd->idx = idx;
 	xd->skip = skip;
 	xd->protoff = protoff;
 	xd->vnet = curvnet;
+	xd->cryptoid = cryptoid;
 
 	/* Crypto operation descriptor */
 	crp->crp_ilen = m->m_pkthdr.len;	/* Total input length */
-	crp->crp_flags = CRYPTO_F_IMBUF | CRYPTO_F_CBIFSYNC;
-	crp->crp_buf = (caddr_t) m;
+	crp->crp_flags = CRYPTO_F_CBIFSYNC;
+	crp->crp_mbuf = m;
+	crp->crp_buf_type = CRYPTO_BUF_MBUF;
 	crp->crp_callback = ipcomp_output_cb;
-	crp->crp_opaque = (caddr_t) xd;
+	crp->crp_opaque = xd;
 
-	SECASVAR_LOCK(sav);
-	crp->crp_session = xd->cryptoid = sav->tdb_cryptoid;
-	SECASVAR_UNLOCK(sav);
-
 	return crypto_dispatch(crp);
 bad:
 	if (m)
 		m_freem(m);
 	key_freesav(&sav);
 	key_freesp(&sp);
 	return (error);
 }
 
 /*
  * IPComp output callback from the crypto driver.
  */
 static int
 ipcomp_output_cb(struct cryptop *crp)
 {
 	IPSEC_DEBUG_DECLARE(char buf[IPSEC_ADDRSTRLEN]);
 	struct xform_data *xd;
 	struct secpolicy *sp;
 	struct secasvar *sav;
 	struct mbuf *m;
 	crypto_session_t cryptoid;
 	u_int idx;
 	int error, skip, protoff;
 
-	m = (struct mbuf *) crp->crp_buf;
-	xd = (struct xform_data *) crp->crp_opaque;
+	m = crp->crp_mbuf;
+	xd = crp->crp_opaque;
 	CURVNET_SET(xd->vnet);
 	idx = xd->idx;
 	sp = xd->sp;
 	sav = xd->sav;
 	skip = xd->skip;
 	protoff = xd->protoff;
 	cryptoid = xd->cryptoid;
 
 	/* Check for crypto errors */
 	if (crp->crp_etype) {
 		if (crp->crp_etype == EAGAIN) {
 			/* Reset the session ID */
 			if (ipsec_updateid(sav, &crp->crp_session, &cryptoid) != 0)
 				crypto_freesession(cryptoid);
 			xd->cryptoid = crp->crp_session;
 			CURVNET_RESTORE();
 			return (crypto_dispatch(crp));
 		}
 		IPCOMPSTAT_INC(ipcomps_noxform);
 		DPRINTF(("%s: crypto error %d\n", __func__, crp->crp_etype));
 		error = crp->crp_etype;
 		goto bad;
 	}
 	/* Shouldn't happen... */
 	if (m == NULL) {
 		IPCOMPSTAT_INC(ipcomps_crypto);
 		DPRINTF(("%s: bogus return buffer from crypto\n", __func__));
 		error = EINVAL;
 		goto bad;
 	}
 	IPCOMPSTAT_INC(ipcomps_hist[sav->alg_comp]);
 
-	if (crp->crp_ilen - skip > crp->crp_olen) {
+	if (crp->crp_payload_length > crp->crp_olen) {
 		struct mbuf *mo;
 		struct ipcomp *ipcomp;
 		int roff;
 		uint8_t prot;
 
 		/* Compression helped, inject IPCOMP header. */
 		mo = m_makespace(m, skip, IPCOMP_HLENGTH, &roff);
 		if (mo == NULL) {
 			IPCOMPSTAT_INC(ipcomps_wrap);
 			DPRINTF(("%s: IPCOMP header inject failed "
 			    "for IPCA %s/%08lx\n",
 			    __func__, ipsec_address(&sav->sah->saidx.dst, buf,
 			    sizeof(buf)), (u_long) ntohl(sav->spi)));
 			error = ENOBUFS;
 			goto bad;
 		}
 		ipcomp = (struct ipcomp *)(mtod(mo, caddr_t) + roff);
 
 		/* Initialize the IPCOMP header */
 		/* XXX alignment always correct? */
 		switch (sav->sah->saidx.dst.sa.sa_family) {
 #ifdef INET
 		case AF_INET:
 			ipcomp->comp_nxt = mtod(m, struct ip *)->ip_p;
 			break;
 #endif /* INET */
 #ifdef INET6
 		case AF_INET6:
 			ipcomp->comp_nxt = mtod(m, struct ip6_hdr *)->ip6_nxt;
 			break;
 #endif
 		}
 		ipcomp->comp_flags = 0;
 		ipcomp->comp_cpi = htons((u_int16_t) ntohl(sav->spi));
 
 		/* Fix Next Protocol in IPv4/IPv6 header */
 		prot = IPPROTO_IPCOMP;
 		m_copyback(m, protoff, sizeof(u_int8_t),
 		    (u_char *)&prot);
 
 		/* Adjust the length in the IP header */
 		switch (sav->sah->saidx.dst.sa.sa_family) {
 #ifdef INET
 		case AF_INET:
 			mtod(m, struct ip *)->ip_len = htons(m->m_pkthdr.len);
 			break;
 #endif /* INET */
 #ifdef INET6
 		case AF_INET6:
 			mtod(m, struct ip6_hdr *)->ip6_plen =
 				htons(m->m_pkthdr.len) - sizeof(struct ip6_hdr);
 			break;
 #endif /* INET6 */
 		default:
 			IPCOMPSTAT_INC(ipcomps_nopf);
 			DPRINTF(("%s: unknown/unsupported protocol "
 			    "family %d, IPCA %s/%08lx\n", __func__,
 			    sav->sah->saidx.dst.sa.sa_family,
 			    ipsec_address(&sav->sah->saidx.dst, buf,
 				sizeof(buf)), (u_long) ntohl(sav->spi)));
 			error = EPFNOSUPPORT;
 			goto bad;
 		}
 	} else {
 		/* Compression was useless, we have lost time. */
 		IPCOMPSTAT_INC(ipcomps_uncompr);
-		DPRINTF(("%s: compressions was useless %d - %d <= %d\n",
-		    __func__, crp->crp_ilen, skip, crp->crp_olen));
+		DPRINTF(("%s: compressions was useless %d <= %d\n",
+		    __func__, crp->crp_payload_length, crp->crp_olen));
 		/* XXX remember state to not compress the next couple
 		 *     of packets, RFC 3173, 2.2. Non-Expansion Policy */
 	}
 
 	/* Release the crypto descriptor */
 	free(xd, M_XDATA);
 	crypto_freereq(crp);
 
 	/* NB: m is reclaimed by ipsec_process_done. */
 	error = ipsec_process_done(m, sp, sav, idx);
 	CURVNET_RESTORE();
 	return (error);
 bad:
 	if (m)
 		m_freem(m);
 	CURVNET_RESTORE();
 	free(xd, M_XDATA);
 	crypto_freereq(crp);
 	key_freesav(&sav);
 	key_freesp(&sp);
 	return (error);
 }
 
 #ifdef INET
 static int
 ipcomp4_nonexp_encapcheck(const struct mbuf *m, int off, int proto,
     void *arg __unused)
 {
 	union sockaddr_union src, dst;
 	const struct ip *ip;
 
 	if (V_ipcomp_enable == 0)
 		return (0);
 	if (proto != IPPROTO_IPV4 && proto != IPPROTO_IPV6)
 		return (0);
 	bzero(&src, sizeof(src));
 	bzero(&dst, sizeof(dst));
 	src.sa.sa_family = dst.sa.sa_family = AF_INET;
 	src.sin.sin_len = dst.sin.sin_len = sizeof(struct sockaddr_in);
 	ip = mtod(m, const struct ip *);
 	src.sin.sin_addr = ip->ip_src;
 	dst.sin.sin_addr = ip->ip_dst;
 	return (ipcomp_encapcheck(&src, &dst));
 }
 
 static const struct encaptab *ipe4_cookie = NULL;
 static const struct encap_config ipv4_encap_cfg = {
 	.proto = -1,
 	.min_length = sizeof(struct ip),
 	.exact_match = sizeof(in_addr_t) << 4,
 	.check = ipcomp4_nonexp_encapcheck,
 	.input = ipcomp_nonexp_input
 };
 #endif
 #ifdef INET6
 static int
 ipcomp6_nonexp_encapcheck(const struct mbuf *m, int off, int proto,
     void *arg __unused)
 {
 	union sockaddr_union src, dst;
 	const struct ip6_hdr *ip6;
 
 	if (V_ipcomp_enable == 0)
 		return (0);
 	if (proto != IPPROTO_IPV4 && proto != IPPROTO_IPV6)
 		return (0);
 	bzero(&src, sizeof(src));
 	bzero(&dst, sizeof(dst));
 	src.sa.sa_family = dst.sa.sa_family = AF_INET;
 	src.sin6.sin6_len = dst.sin6.sin6_len = sizeof(struct sockaddr_in6);
 	ip6 = mtod(m, const struct ip6_hdr *);
 	src.sin6.sin6_addr = ip6->ip6_src;
 	dst.sin6.sin6_addr = ip6->ip6_dst;
 	if (IN6_IS_SCOPE_LINKLOCAL(&src.sin6.sin6_addr)) {
 		/* XXX: sa6_recoverscope() */
 		src.sin6.sin6_scope_id =
 		    ntohs(src.sin6.sin6_addr.s6_addr16[1]);
 		src.sin6.sin6_addr.s6_addr16[1] = 0;
 	}
 	if (IN6_IS_SCOPE_LINKLOCAL(&dst.sin6.sin6_addr)) {
 		/* XXX: sa6_recoverscope() */
 		dst.sin6.sin6_scope_id =
 		    ntohs(dst.sin6.sin6_addr.s6_addr16[1]);
 		dst.sin6.sin6_addr.s6_addr16[1] = 0;
 	}
 	return (ipcomp_encapcheck(&src, &dst));
 }
 
 static const struct encaptab *ipe6_cookie = NULL;
 static const struct encap_config ipv6_encap_cfg = {
 	.proto = -1,
 	.min_length = sizeof(struct ip6_hdr),
 	.exact_match = sizeof(struct in6_addr) << 4,
 	.check = ipcomp6_nonexp_encapcheck,
 	.input = ipcomp_nonexp_input
 };
 #endif
 
 static struct xformsw ipcomp_xformsw = {
 	.xf_type =	XF_IPCOMP,
 	.xf_name =	"IPcomp",
 	.xf_init =	ipcomp_init,
 	.xf_zeroize =	ipcomp_zeroize,
 	.xf_input =	ipcomp_input,
 	.xf_output =	ipcomp_output,
 };
 
 static void
 ipcomp_attach(void)
 {
 
 #ifdef INET
 	ipe4_cookie = ip_encap_attach(&ipv4_encap_cfg, NULL, M_WAITOK);
 #endif
 #ifdef INET6
 	ipe6_cookie = ip6_encap_attach(&ipv6_encap_cfg, NULL, M_WAITOK);
 #endif
 	xform_attach(&ipcomp_xformsw);
 }
 
 static void
 ipcomp_detach(void)
 {
 
 #ifdef INET
 	ip_encap_detach(ipe4_cookie);
 #endif
 #ifdef INET6
 	ip6_encap_detach(ipe6_cookie);
 #endif
 	xform_detach(&ipcomp_xformsw);
 }
 
 SYSINIT(ipcomp_xform_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_MIDDLE,
     ipcomp_attach, NULL);
 SYSUNINIT(ipcomp_xform_uninit, SI_SUB_PROTO_DOMAIN, SI_ORDER_MIDDLE,
     ipcomp_detach, NULL);
Index: head/sys/opencrypto/cryptosoft.h
===================================================================
--- head/sys/opencrypto/cryptosoft.h	(revision 359373)
+++ head/sys/opencrypto/cryptosoft.h	(nonexistent)
@@ -1,71 +0,0 @@
-/*	$FreeBSD$	*/
-/*	$OpenBSD: cryptosoft.h,v 1.10 2002/04/22 23:10:09 deraadt Exp $	*/
-
-/*-
- * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
- *
- * This code was written by Angelos D. Keromytis in Athens, Greece, in
- * February 2000. Network Security Technologies Inc. (NSTI) kindly
- * supported the development of this code.
- *
- * Copyright (c) 2000 Angelos D. Keromytis
- *
- * Permission to use, copy, and modify this software with or without fee
- * is hereby granted, provided that this entire notice is included in
- * all source code copies of any software which is or includes a copy or
- * modification of this software.
- *
- * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR
- * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY
- * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE
- * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR
- * PURPOSE.
- */
-
-#ifndef _CRYPTO_CRYPTOSOFT_H_
-#define _CRYPTO_CRYPTOSOFT_H_
-
-/* Software session entry */
-struct swcr_data {
-	int		sw_alg;		/* Algorithm */
-	union {
-		struct {
-			u_int8_t	 *SW_ictx;
-			u_int8_t	 *SW_octx;
-			u_int16_t	 SW_klen;
-			u_int16_t	 SW_mlen;
-			struct auth_hash *SW_axf;
-		} SWCR_AUTH;
-		struct {
-			u_int8_t	 *SW_kschedule;
-			struct enc_xform *SW_exf;
-		} SWCR_ENC;
-		struct {
-			u_int32_t	 SW_size;
-			struct comp_algo *SW_cxf;
-		} SWCR_COMP;
-	} SWCR_UN;
-
-#define sw_ictx		SWCR_UN.SWCR_AUTH.SW_ictx
-#define sw_octx		SWCR_UN.SWCR_AUTH.SW_octx
-#define sw_klen		SWCR_UN.SWCR_AUTH.SW_klen
-#define sw_mlen		SWCR_UN.SWCR_AUTH.SW_mlen
-#define sw_axf		SWCR_UN.SWCR_AUTH.SW_axf
-#define sw_kschedule	SWCR_UN.SWCR_ENC.SW_kschedule
-#define sw_exf		SWCR_UN.SWCR_ENC.SW_exf
-#define sw_size		SWCR_UN.SWCR_COMP.SW_size
-#define sw_cxf		SWCR_UN.SWCR_COMP.SW_cxf
-};
-
-struct swcr_session {
-	struct mtx	swcr_lock;
-	struct swcr_data swcr_algorithms[2];
-	unsigned swcr_nalgs;
-};
-
-#ifdef _KERNEL
-extern u_int8_t hmac_ipad_buffer[];
-extern u_int8_t hmac_opad_buffer[];
-#endif /* _KERNEL */
-
-#endif /* _CRYPTO_CRYPTO_H_ */

Property changes on: head/sys/opencrypto/cryptosoft.h
___________________________________________________________________
Deleted: svn:keywords
## -1 +0,0 ##
-FreeBSD=%H
\ No newline at end of property
Index: head/sys/opencrypto/criov.c
===================================================================
--- head/sys/opencrypto/criov.c	(revision 359373)
+++ head/sys/opencrypto/criov.c	(revision 359374)
@@ -1,295 +1,316 @@
 /*      $OpenBSD: criov.c,v 1.9 2002/01/29 15:48:29 jason Exp $	*/
 
 /*-
  * Copyright (c) 1999 Theo de Raadt
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  *
  * 1. Redistributions of source code must retain the above copyright
  *   notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *   notice, this list of conditions and the following disclaimer in the
  *   documentation and/or other materials provided with the distribution.
  * 3. The name of the author may not be used to endorse or promote products
  *   derived from this software without specific prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/proc.h>
 #include <sys/errno.h>
 #include <sys/malloc.h>
 #include <sys/kernel.h>
 #include <sys/mbuf.h>
 #include <sys/uio.h>
 #include <sys/limits.h>
 #include <sys/lock.h>
 
 #include <opencrypto/cryptodev.h>
 
 /*
  * This macro is only for avoiding code duplication, as we need to skip
  * given number of bytes in the same way in three functions below.
  */
 #define	CUIO_SKIP()	do {						\
 	KASSERT(off >= 0, ("%s: off %d < 0", __func__, off));		\
 	KASSERT(len >= 0, ("%s: len %d < 0", __func__, len));		\
 	while (off > 0) {						\
 		KASSERT(iol >= 0, ("%s: empty in skip", __func__));	\
 		if (off < iov->iov_len)					\
 			break;						\
 		off -= iov->iov_len;					\
 		iol--;							\
 		iov++;							\
 	}								\
 } while (0)
 
 void
 cuio_copydata(struct uio* uio, int off, int len, caddr_t cp)
 {
 	struct iovec *iov = uio->uio_iov;
 	int iol = uio->uio_iovcnt;
 	unsigned count;
 
 	CUIO_SKIP();
 	while (len > 0) {
 		KASSERT(iol >= 0, ("%s: empty", __func__));
 		count = min(iov->iov_len - off, len);
 		bcopy(((caddr_t)iov->iov_base) + off, cp, count);
 		len -= count;
 		cp += count;
 		off = 0;
 		iol--;
 		iov++;
 	}
 }
 
 void
 cuio_copyback(struct uio* uio, int off, int len, c_caddr_t cp)
 {
 	struct iovec *iov = uio->uio_iov;
 	int iol = uio->uio_iovcnt;
 	unsigned count;
 
 	CUIO_SKIP();
 	while (len > 0) {
 		KASSERT(iol >= 0, ("%s: empty", __func__));
 		count = min(iov->iov_len - off, len);
 		bcopy(cp, ((caddr_t)iov->iov_base) + off, count);
 		len -= count;
 		cp += count;
 		off = 0;
 		iol--;
 		iov++;
 	}
 }
 
 /*
  * Return the index and offset of location in iovec list.
  */
 int
 cuio_getptr(struct uio *uio, int loc, int *off)
 {
 	int ind, len;
 
 	ind = 0;
 	while (loc >= 0 && ind < uio->uio_iovcnt) {
 		len = uio->uio_iov[ind].iov_len;
 		if (len > loc) {
 	    		*off = loc;
 	    		return (ind);
 		}
 		loc -= len;
 		ind++;
 	}
 
 	if (ind > 0 && loc == 0) {
 		ind--;
 		*off = uio->uio_iov[ind].iov_len;
 		return (ind);
 	}
 
 	return (-1);
 }
 
 /*
  * Apply function f to the data in an iovec list starting "off" bytes from
  * the beginning, continuing for "len" bytes.
  */
 int
 cuio_apply(struct uio *uio, int off, int len, int (*f)(void *, void *, u_int),
     void *arg)
 {
 	struct iovec *iov = uio->uio_iov;
 	int iol = uio->uio_iovcnt;
 	unsigned count;
 	int rval;
 
 	CUIO_SKIP();
 	while (len > 0) {
 		KASSERT(iol >= 0, ("%s: empty", __func__));
 		count = min(iov->iov_len - off, len);
 		rval = (*f)(arg, ((caddr_t)iov->iov_base) + off, count);
 		if (rval)
 			return (rval);
 		len -= count;
 		off = 0;
 		iol--;
 		iov++;
 	}
 	return (0);
 }
 
 void
-crypto_copyback(int flags, caddr_t buf, int off, int size, c_caddr_t in)
+crypto_copyback(struct cryptop *crp, int off, int size, const void *src)
 {
 
-	if ((flags & CRYPTO_F_IMBUF) != 0)
-		m_copyback((struct mbuf *)buf, off, size, in);
-	else if ((flags & CRYPTO_F_IOV) != 0)
-		cuio_copyback((struct uio *)buf, off, size, in);
-	else
-		bcopy(in, buf + off, size);
+	switch (crp->crp_buf_type) {
+	case CRYPTO_BUF_MBUF:
+		m_copyback(crp->crp_mbuf, off, size, src);
+		break;
+	case CRYPTO_BUF_UIO:
+		cuio_copyback(crp->crp_uio, off, size, src);
+		break;
+	case CRYPTO_BUF_CONTIG:
+		bcopy(src, crp->crp_buf + off, size);
+		break;
+	default:
+		panic("invalid crp buf type %d", crp->crp_buf_type);
+	}
 }
 
 void
-crypto_copydata(int flags, caddr_t buf, int off, int size, caddr_t out)
+crypto_copydata(struct cryptop *crp, int off, int size, void *dst)
 {
 
-	if ((flags & CRYPTO_F_IMBUF) != 0)
-		m_copydata((struct mbuf *)buf, off, size, out);
-	else if ((flags & CRYPTO_F_IOV) != 0)
-		cuio_copydata((struct uio *)buf, off, size, out);
-	else
-		bcopy(buf + off, out, size);
+	switch (crp->crp_buf_type) {
+	case CRYPTO_BUF_MBUF:
+		m_copydata(crp->crp_mbuf, off, size, dst);
+		break;
+	case CRYPTO_BUF_UIO:
+		cuio_copydata(crp->crp_uio, off, size, dst);
+		break;
+	case CRYPTO_BUF_CONTIG:
+		bcopy(crp->crp_buf + off, dst, size);
+		break;
+	default:
+		panic("invalid crp buf type %d", crp->crp_buf_type);
+	}
 }
 
 int
-crypto_apply(int flags, caddr_t buf, int off, int len,
+crypto_apply(struct cryptop *crp, int off, int len,
     int (*f)(void *, void *, u_int), void *arg)
 {
 	int error;
 
-	if ((flags & CRYPTO_F_IMBUF) != 0)
-		error = m_apply((struct mbuf *)buf, off, len, f, arg);
-	else if ((flags & CRYPTO_F_IOV) != 0)
-		error = cuio_apply((struct uio *)buf, off, len, f, arg);
-	else
-		error = (*f)(arg, buf + off, len);
+	switch (crp->crp_buf_type) {
+	case CRYPTO_BUF_MBUF:
+		error = m_apply(crp->crp_mbuf, off, len, f, arg);
+		break;
+	case CRYPTO_BUF_UIO:
+		error = cuio_apply(crp->crp_uio, off, len, f, arg);
+		break;
+	case CRYPTO_BUF_CONTIG:
+		error = (*f)(arg, crp->crp_buf + off, len);
+		break;
+	default:
+		panic("invalid crp buf type %d", crp->crp_buf_type);
+	}
 	return (error);
 }
 
 int
 crypto_mbuftoiov(struct mbuf *mbuf, struct iovec **iovptr, int *cnt,
     int *allocated)
 {
 	struct iovec *iov;
 	struct mbuf *m, *mtmp;
 	int i, j;
 
 	*allocated = 0;
 	iov = *iovptr;
 	if (iov == NULL)
 		*cnt = 0;
 
 	m = mbuf;
 	i = 0;
 	while (m != NULL) {
 		if (i == *cnt) {
 			/* we need to allocate a larger array */
 			j = 1;
 			mtmp = m;
 			while ((mtmp = mtmp->m_next) != NULL)
 				j++;
 			iov = malloc(sizeof *iov * (i + j), M_CRYPTO_DATA,
 			    M_NOWAIT);
 			if (iov == NULL)
 				return ENOMEM;
 			*allocated = 1;
 			*cnt = i + j;
 			memcpy(iov, *iovptr, sizeof *iov * i);
 		}
 
 		iov[i].iov_base = m->m_data;
 		iov[i].iov_len = m->m_len;
 
 		i++;
 		m = m->m_next;
 	}
 
 	if (*allocated)
 		KASSERT(*cnt == i, ("did not allocate correct amount: %d != %d",
 		    *cnt, i));
 
 	*iovptr = iov;
 	*cnt = i;
 	return 0;
 }
 
 static inline void *
 m_contiguous_subsegment(struct mbuf *m, size_t skip, size_t len)
 {
 	int rel_off;
 
 	MPASS(skip <= INT_MAX);
 
 	m = m_getptr(m, (int)skip, &rel_off);
 	if (m == NULL)
 		return (NULL);
 
 	MPASS(rel_off >= 0);
 	skip = rel_off;
 	if (skip + len > m->m_len)
 		return (NULL);
 
 	return (mtod(m, char*) + skip);
 }
 
 static inline void *
 cuio_contiguous_segment(struct uio *uio, size_t skip, size_t len)
 {
 	int rel_off, idx;
 
 	MPASS(skip <= INT_MAX);
 	idx = cuio_getptr(uio, (int)skip, &rel_off);
 	if (idx < 0)
 		return (NULL);
 
 	MPASS(rel_off >= 0);
 	skip = rel_off;
 	if (skip + len > uio->uio_iov[idx].iov_len)
 		return (NULL);
 	return ((char *)uio->uio_iov[idx].iov_base + skip);
 }
 
 void *
-crypto_contiguous_subsegment(int crp_flags, void *crpbuf,
-    size_t skip, size_t len)
+crypto_contiguous_subsegment(struct cryptop *crp, size_t skip, size_t len)
 {
-	if ((crp_flags & CRYPTO_F_IMBUF) != 0)
-		return (m_contiguous_subsegment(crpbuf, skip, len));
-	else if ((crp_flags & CRYPTO_F_IOV) != 0)
-		return (cuio_contiguous_segment(crpbuf, skip, len));
-	else {
-		MPASS((crp_flags & (CRYPTO_F_IMBUF | CRYPTO_F_IOV)) !=
-		    (CRYPTO_F_IMBUF | CRYPTO_F_IOV));
-		return ((char*)crpbuf + skip);
+
+	switch (crp->crp_buf_type) {
+	case CRYPTO_BUF_MBUF:
+		return (m_contiguous_subsegment(crp->crp_mbuf, skip, len));
+	case CRYPTO_BUF_UIO:
+		return (cuio_contiguous_segment(crp->crp_uio, skip, len));
+	case CRYPTO_BUF_CONTIG:
+		return (crp->crp_buf + skip);
+	default:
+		panic("invalid crp buf type %d", crp->crp_buf_type);
 	}
 }
-
Index: head/sys/opencrypto/crypto.c
===================================================================
--- head/sys/opencrypto/crypto.c	(revision 359373)
+++ head/sys/opencrypto/crypto.c	(revision 359374)
@@ -1,1853 +1,2274 @@
 /*-
  * Copyright (c) 2002-2006 Sam Leffler.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 /*
  * Cryptographic Subsystem.
  *
  * This code is derived from the Openbsd Cryptographic Framework (OCF)
  * that has the copyright shown below.  Very little of the original
  * code remains.
  */
 
 /*-
  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
  *
  * This code was written by Angelos D. Keromytis in Athens, Greece, in
  * February 2000. Network Security Technologies Inc. (NSTI) kindly
  * supported the development of this code.
  *
  * Copyright (c) 2000, 2001 Angelos D. Keromytis
  *
  * Permission to use, copy, and modify this software with or without fee
  * is hereby granted, provided that this entire notice is included in
  * all source code copies of any software which is or includes a copy or
  * modification of this software.
  *
  * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR
  * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY
  * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE
  * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR
  * PURPOSE.
  */
 
 #define	CRYPTO_TIMING				/* enable timing support */
 
+#include "opt_compat.h"
 #include "opt_ddb.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/eventhandler.h>
 #include <sys/kernel.h>
 #include <sys/kthread.h>
 #include <sys/linker.h>
 #include <sys/lock.h>
 #include <sys/module.h>
 #include <sys/mutex.h>
 #include <sys/malloc.h>
 #include <sys/proc.h>
+#include <sys/refcount.h>
 #include <sys/sdt.h>
 #include <sys/smp.h>
 #include <sys/sysctl.h>
 #include <sys/taskqueue.h>
 
 #include <ddb/ddb.h>
 
 #include <vm/uma.h>
 #include <crypto/intake.h>
 #include <opencrypto/cryptodev.h>
-#include <opencrypto/xform.h>			/* XXX for M_XDATA */
+#include <opencrypto/xform_auth.h>
+#include <opencrypto/xform_enc.h>
 
 #include <sys/kobj.h>
 #include <sys/bus.h>
 #include "cryptodev_if.h"
 
 #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__)
 #include <machine/pcb.h>
 #endif
 
-struct crypto_session {
-	device_t parent;
-	void *softc;
-	uint32_t hid;
-	uint32_t capabilities;
-};
-
 SDT_PROVIDER_DEFINE(opencrypto);
 
 /*
  * Crypto drivers register themselves by allocating a slot in the
  * crypto_drivers table with crypto_get_driverid() and then registering
- * each algorithm they support with crypto_register() and crypto_kregister().
+ * each asym algorithm they support with crypto_kregister().
  */
 static	struct mtx crypto_drivers_mtx;		/* lock on driver table */
 #define	CRYPTO_DRIVER_LOCK()	mtx_lock(&crypto_drivers_mtx)
 #define	CRYPTO_DRIVER_UNLOCK()	mtx_unlock(&crypto_drivers_mtx)
 #define	CRYPTO_DRIVER_ASSERT()	mtx_assert(&crypto_drivers_mtx, MA_OWNED)
 
 /*
  * Crypto device/driver capabilities structure.
  *
  * Synchronization:
  * (d) - protected by CRYPTO_DRIVER_LOCK()
  * (q) - protected by CRYPTO_Q_LOCK()
  * Not tagged fields are read-only.
  */
 struct cryptocap {
-	device_t	cc_dev;			/* (d) device/driver */
+	device_t	cc_dev;
+	uint32_t	cc_hid;
 	u_int32_t	cc_sessions;		/* (d) # of sessions */
 	u_int32_t	cc_koperations;		/* (d) # os asym operations */
-	/*
-	 * Largest possible operator length (in bits) for each type of
-	 * encryption algorithm. XXX not used
-	 */
-	u_int16_t	cc_max_op_len[CRYPTO_ALGORITHM_MAX + 1];
-	u_int8_t	cc_alg[CRYPTO_ALGORITHM_MAX + 1];
 	u_int8_t	cc_kalg[CRK_ALGORITHM_MAX + 1];
 
 	int		cc_flags;		/* (d) flags */
 #define CRYPTOCAP_F_CLEANUP	0x80000000	/* needs resource cleanup */
 	int		cc_qblocked;		/* (q) symmetric q blocked */
 	int		cc_kqblocked;		/* (q) asymmetric q blocked */
 	size_t		cc_session_size;
+	volatile int	cc_refs;
 };
-static	struct cryptocap *crypto_drivers = NULL;
-static	int crypto_drivers_num = 0;
 
+static	struct cryptocap **crypto_drivers = NULL;
+static	int crypto_drivers_size = 0;
+
+struct crypto_session {
+	struct cryptocap *cap;
+	void *softc;
+	struct crypto_session_params csp;
+};
+
 /*
  * There are two queues for crypto requests; one for symmetric (e.g.
  * cipher) operations and one for asymmetric (e.g. MOD)operations.
  * A single mutex is used to lock access to both queues.  We could
  * have one per-queue but having one simplifies handling of block/unblock
  * operations.
  */
 static	int crp_sleep = 0;
 static	TAILQ_HEAD(cryptop_q ,cryptop) crp_q;		/* request queues */
 static	TAILQ_HEAD(,cryptkop) crp_kq;
 static	struct mtx crypto_q_mtx;
 #define	CRYPTO_Q_LOCK()		mtx_lock(&crypto_q_mtx)
 #define	CRYPTO_Q_UNLOCK()	mtx_unlock(&crypto_q_mtx)
 
+static SYSCTL_NODE(_kern, OID_AUTO, crypto, CTLFLAG_RW, 0,
+    "In-kernel cryptography");
+
 /*
  * Taskqueue used to dispatch the crypto requests
  * that have the CRYPTO_F_ASYNC flag
  */
 static struct taskqueue *crypto_tq;
 
 /*
  * Crypto seq numbers are operated on with modular arithmetic
  */
 #define	CRYPTO_SEQ_GT(a,b)	((int)((a)-(b)) > 0)
 
 struct crypto_ret_worker {
 	struct mtx crypto_ret_mtx;
 
 	TAILQ_HEAD(,cryptop) crp_ordered_ret_q;	/* ordered callback queue for symetric jobs */
 	TAILQ_HEAD(,cryptop) crp_ret_q;		/* callback queue for symetric jobs */
 	TAILQ_HEAD(,cryptkop) crp_ret_kq;	/* callback queue for asym jobs */
 
 	u_int32_t reorder_ops;		/* total ordered sym jobs received */
 	u_int32_t reorder_cur_seq;	/* current sym job dispatched */
 
 	struct proc *cryptoretproc;
 };
 static struct crypto_ret_worker *crypto_ret_workers = NULL;
 
 #define CRYPTO_RETW(i)		(&crypto_ret_workers[i])
 #define CRYPTO_RETW_ID(w)	((w) - crypto_ret_workers)
 #define FOREACH_CRYPTO_RETW(w) \
 	for (w = crypto_ret_workers; w < crypto_ret_workers + crypto_workers_num; ++w)
 
 #define	CRYPTO_RETW_LOCK(w)	mtx_lock(&w->crypto_ret_mtx)
 #define	CRYPTO_RETW_UNLOCK(w)	mtx_unlock(&w->crypto_ret_mtx)
 #define	CRYPTO_RETW_EMPTY(w) \
 	(TAILQ_EMPTY(&w->crp_ret_q) && TAILQ_EMPTY(&w->crp_ret_kq) && TAILQ_EMPTY(&w->crp_ordered_ret_q))
 
 static int crypto_workers_num = 0;
+SYSCTL_INT(_kern_crypto, OID_AUTO, num_workers, CTLFLAG_RDTUN,
+	   &crypto_workers_num, 0,
+	   "Number of crypto workers used to dispatch crypto jobs");
+#ifdef COMPAT_FREEBSD12
 SYSCTL_INT(_kern, OID_AUTO, crypto_workers_num, CTLFLAG_RDTUN,
 	   &crypto_workers_num, 0,
 	   "Number of crypto workers used to dispatch crypto jobs");
+#endif
 
 static	uma_zone_t cryptop_zone;
-static	uma_zone_t cryptodesc_zone;
 static	uma_zone_t cryptoses_zone;
 
-int	crypto_userasymcrypto = 1;	/* userland may do asym crypto reqs */
+int	crypto_userasymcrypto = 1;
+SYSCTL_INT(_kern_crypto, OID_AUTO, asym_enable, CTLFLAG_RW,
+	   &crypto_userasymcrypto, 0,
+	   "Enable user-mode access to asymmetric crypto support");
+#ifdef COMPAT_FREEBSD12
 SYSCTL_INT(_kern, OID_AUTO, userasymcrypto, CTLFLAG_RW,
 	   &crypto_userasymcrypto, 0,
 	   "Enable/disable user-mode access to asymmetric crypto support");
-int	crypto_devallowsoft = 0;	/* only use hardware crypto */
+#endif
+
+int	crypto_devallowsoft = 0;
+SYSCTL_INT(_kern_crypto, OID_AUTO, allow_soft, CTLFLAG_RW,
+	   &crypto_devallowsoft, 0,
+	   "Enable use of software crypto by /dev/crypto");
+#ifdef COMPAT_FREEBSD12
 SYSCTL_INT(_kern, OID_AUTO, cryptodevallowsoft, CTLFLAG_RW,
 	   &crypto_devallowsoft, 0,
 	   "Enable/disable use of software crypto by /dev/crypto");
+#endif
 
 MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records");
 
 static	void crypto_proc(void);
 static	struct proc *cryptoproc;
 static	void crypto_ret_proc(struct crypto_ret_worker *ret_worker);
 static	void crypto_destroy(void);
 static	int crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint);
-static	int crypto_kinvoke(struct cryptkop *krp, int flags);
-static	void crypto_remove(struct cryptocap *cap);
+static	int crypto_kinvoke(struct cryptkop *krp);
 static	void crypto_task_invoke(void *ctx, int pending);
 static void crypto_batch_enqueue(struct cryptop *crp);
 
 static	struct cryptostats cryptostats;
-SYSCTL_STRUCT(_kern, OID_AUTO, crypto_stats, CTLFLAG_RW, &cryptostats,
+SYSCTL_STRUCT(_kern_crypto, OID_AUTO, stats, CTLFLAG_RW, &cryptostats,
 	    cryptostats, "Crypto system statistics");
 
 #ifdef CRYPTO_TIMING
 static	int crypto_timing = 0;
 SYSCTL_INT(_debug, OID_AUTO, crypto_timing, CTLFLAG_RW,
 	   &crypto_timing, 0, "Enable/disable crypto timing support");
 #endif
 
 /* Try to avoid directly exposing the key buffer as a symbol */
 static struct keybuf *keybuf;
 
 static struct keybuf empty_keybuf = {
         .kb_nents = 0
 };
 
 /* Obtain the key buffer from boot metadata */
 static void
 keybuf_init(void)
 {
 	caddr_t kmdp;
 
 	kmdp = preload_search_by_type("elf kernel");
 
 	if (kmdp == NULL)
 		kmdp = preload_search_by_type("elf64 kernel");
 
 	keybuf = (struct keybuf *)preload_search_info(kmdp,
 	    MODINFO_METADATA | MODINFOMD_KEYBUF);
 
         if (keybuf == NULL)
                 keybuf = &empty_keybuf;
 }
 
 /* It'd be nice if we could store these in some kind of secure memory... */
 struct keybuf * get_keybuf(void) {
 
         return (keybuf);
 }
 
+static struct cryptocap *
+cap_ref(struct cryptocap *cap)
+{
+
+	refcount_acquire(&cap->cc_refs);
+	return (cap);
+}
+
+static void
+cap_rele(struct cryptocap *cap)
+{
+
+	if (refcount_release(&cap->cc_refs) == 0)
+		return;
+
+	KASSERT(cap->cc_sessions == 0,
+	    ("freeing crypto driver with active sessions"));
+	KASSERT(cap->cc_koperations == 0,
+	    ("freeing crypto driver with active key operations"));
+
+	free(cap, M_CRYPTO_DATA);
+}
+
 static int
 crypto_init(void)
 {
 	struct crypto_ret_worker *ret_worker;
 	int error;
 
 	mtx_init(&crypto_drivers_mtx, "crypto", "crypto driver table",
 		MTX_DEF|MTX_QUIET);
 
 	TAILQ_INIT(&crp_q);
 	TAILQ_INIT(&crp_kq);
 	mtx_init(&crypto_q_mtx, "crypto", "crypto op queues", MTX_DEF);
 
 	cryptop_zone = uma_zcreate("cryptop", sizeof (struct cryptop),
 				    0, 0, 0, 0,
 				    UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
-	cryptodesc_zone = uma_zcreate("cryptodesc", sizeof (struct cryptodesc),
-				    0, 0, 0, 0,
-				    UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
 	cryptoses_zone = uma_zcreate("crypto_session",
 	    sizeof(struct crypto_session), NULL, NULL, NULL, NULL,
 	    UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
 
-	if (cryptodesc_zone == NULL || cryptop_zone == NULL ||
-	    cryptoses_zone == NULL) {
+	if (cryptop_zone == NULL || cryptoses_zone == NULL) {
 		printf("crypto_init: cannot setup crypto zones\n");
 		error = ENOMEM;
 		goto bad;
 	}
 
-	crypto_drivers_num = CRYPTO_DRIVERS_INITIAL;
-	crypto_drivers = malloc(crypto_drivers_num *
+	crypto_drivers_size = CRYPTO_DRIVERS_INITIAL;
+	crypto_drivers = malloc(crypto_drivers_size *
 	    sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT | M_ZERO);
 	if (crypto_drivers == NULL) {
 		printf("crypto_init: cannot setup crypto drivers\n");
 		error = ENOMEM;
 		goto bad;
 	}
 
 	if (crypto_workers_num < 1 || crypto_workers_num > mp_ncpus)
 		crypto_workers_num = mp_ncpus;
 
 	crypto_tq = taskqueue_create("crypto", M_WAITOK|M_ZERO,
 				taskqueue_thread_enqueue, &crypto_tq);
 	if (crypto_tq == NULL) {
 		printf("crypto init: cannot setup crypto taskqueue\n");
 		error = ENOMEM;
 		goto bad;
 	}
 
 	taskqueue_start_threads(&crypto_tq, crypto_workers_num, PRI_MIN_KERN,
 		"crypto");
 
 	error = kproc_create((void (*)(void *)) crypto_proc, NULL,
 		    &cryptoproc, 0, 0, "crypto");
 	if (error) {
 		printf("crypto_init: cannot start crypto thread; error %d",
 			error);
 		goto bad;
 	}
 
 	crypto_ret_workers = malloc(crypto_workers_num * sizeof(struct crypto_ret_worker),
 			M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
 	if (crypto_ret_workers == NULL) {
 		error = ENOMEM;
 		printf("crypto_init: cannot allocate ret workers\n");
 		goto bad;
 	}
 
 
 	FOREACH_CRYPTO_RETW(ret_worker) {
 		TAILQ_INIT(&ret_worker->crp_ordered_ret_q);
 		TAILQ_INIT(&ret_worker->crp_ret_q);
 		TAILQ_INIT(&ret_worker->crp_ret_kq);
 
 		ret_worker->reorder_ops = 0;
 		ret_worker->reorder_cur_seq = 0;
 
 		mtx_init(&ret_worker->crypto_ret_mtx, "crypto", "crypto return queues", MTX_DEF);
 
 		error = kproc_create((void (*)(void *)) crypto_ret_proc, ret_worker,
 				&ret_worker->cryptoretproc, 0, 0, "crypto returns %td", CRYPTO_RETW_ID(ret_worker));
 		if (error) {
 			printf("crypto_init: cannot start cryptoret thread; error %d",
 				error);
 			goto bad;
 		}
 	}
 
 	keybuf_init();
 
 	return 0;
 bad:
 	crypto_destroy();
 	return error;
 }
 
 /*
  * Signal a crypto thread to terminate.  We use the driver
  * table lock to synchronize the sleep/wakeups so that we
  * are sure the threads have terminated before we release
  * the data structures they use.  See crypto_finis below
  * for the other half of this song-and-dance.
  */
 static void
 crypto_terminate(struct proc **pp, void *q)
 {
 	struct proc *p;
 
 	mtx_assert(&crypto_drivers_mtx, MA_OWNED);
 	p = *pp;
 	*pp = NULL;
 	if (p) {
 		wakeup_one(q);
 		PROC_LOCK(p);		/* NB: insure we don't miss wakeup */
 		CRYPTO_DRIVER_UNLOCK();	/* let crypto_finis progress */
 		msleep(p, &p->p_mtx, PWAIT, "crypto_destroy", 0);
 		PROC_UNLOCK(p);
 		CRYPTO_DRIVER_LOCK();
 	}
 }
 
 static void
+hmac_init_pad(struct auth_hash *axf, const char *key, int klen, void *auth_ctx,
+    uint8_t padval)
+{
+	uint8_t hmac_key[HMAC_MAX_BLOCK_LEN];
+	u_int i;
+
+	KASSERT(axf->blocksize <= sizeof(hmac_key),
+	    ("Invalid HMAC block size %d", axf->blocksize));
+
+	/*
+	 * If the key is larger than the block size, use the digest of
+	 * the key as the key instead.
+	 */
+	memset(hmac_key, 0, sizeof(hmac_key));
+	if (klen > axf->blocksize) {
+		axf->Init(auth_ctx);
+		axf->Update(auth_ctx, key, klen);
+		axf->Final(hmac_key, auth_ctx);
+		klen = axf->hashsize;
+	} else
+		memcpy(hmac_key, key, klen);
+
+	for (i = 0; i < axf->blocksize; i++)
+		hmac_key[i] ^= padval;
+
+	axf->Init(auth_ctx);
+	axf->Update(auth_ctx, hmac_key, axf->blocksize);
+}
+
+void
+hmac_init_ipad(struct auth_hash *axf, const char *key, int klen,
+    void *auth_ctx)
+{
+
+	hmac_init_pad(axf, key, klen, auth_ctx, HMAC_IPAD_VAL);
+}
+
+void
+hmac_init_opad(struct auth_hash *axf, const char *key, int klen,
+    void *auth_ctx)
+{
+
+	hmac_init_pad(axf, key, klen, auth_ctx, HMAC_OPAD_VAL);
+}
+
+static void
 crypto_destroy(void)
 {
 	struct crypto_ret_worker *ret_worker;
+	int i;
 
 	/*
 	 * Terminate any crypto threads.
 	 */
 	if (crypto_tq != NULL)
 		taskqueue_drain_all(crypto_tq);
 	CRYPTO_DRIVER_LOCK();
 	crypto_terminate(&cryptoproc, &crp_q);
 	FOREACH_CRYPTO_RETW(ret_worker)
 		crypto_terminate(&ret_worker->cryptoretproc, &ret_worker->crp_ret_q);
 	CRYPTO_DRIVER_UNLOCK();
 
 	/* XXX flush queues??? */
 
 	/*
 	 * Reclaim dynamically allocated resources.
 	 */
-	if (crypto_drivers != NULL)
-		free(crypto_drivers, M_CRYPTO_DATA);
+	for (i = 0; i < crypto_drivers_size; i++) {
+		if (crypto_drivers[i] != NULL)
+			cap_rele(crypto_drivers[i]);
+	}
+	free(crypto_drivers, M_CRYPTO_DATA);
 
 	if (cryptoses_zone != NULL)
 		uma_zdestroy(cryptoses_zone);
-	if (cryptodesc_zone != NULL)
-		uma_zdestroy(cryptodesc_zone);
 	if (cryptop_zone != NULL)
 		uma_zdestroy(cryptop_zone);
 	mtx_destroy(&crypto_q_mtx);
 	FOREACH_CRYPTO_RETW(ret_worker)
 		mtx_destroy(&ret_worker->crypto_ret_mtx);
 	free(crypto_ret_workers, M_CRYPTO_DATA);
 	if (crypto_tq != NULL)
 		taskqueue_free(crypto_tq);
 	mtx_destroy(&crypto_drivers_mtx);
 }
 
 uint32_t
 crypto_ses2hid(crypto_session_t crypto_session)
 {
-	return (crypto_session->hid);
+	return (crypto_session->cap->cc_hid);
 }
 
 uint32_t
 crypto_ses2caps(crypto_session_t crypto_session)
 {
-	return (crypto_session->capabilities);
+	return (crypto_session->cap->cc_flags & 0xff000000);
 }
 
 void *
 crypto_get_driver_session(crypto_session_t crypto_session)
 {
 	return (crypto_session->softc);
 }
 
-static struct cryptocap *
-crypto_checkdriver(u_int32_t hid)
+const struct crypto_session_params *
+crypto_get_params(crypto_session_t crypto_session)
 {
-	if (crypto_drivers == NULL)
-		return NULL;
-	return (hid >= crypto_drivers_num ? NULL : &crypto_drivers[hid]);
+	return (&crypto_session->csp);
 }
 
-/*
- * Compare a driver's list of supported algorithms against another
- * list; return non-zero if all algorithms are supported.
- */
-static int
-driver_suitable(const struct cryptocap *cap, const struct cryptoini *cri)
+struct auth_hash *
+crypto_auth_hash(const struct crypto_session_params *csp)
 {
-	const struct cryptoini *cr;
 
-	/* See if all the algorithms are supported. */
-	for (cr = cri; cr; cr = cr->cri_next)
-		if (cap->cc_alg[cr->cri_alg] == 0)
-			return 0;
-	return 1;
+	switch (csp->csp_auth_alg) {
+	case CRYPTO_MD5_HMAC:
+		return (&auth_hash_hmac_md5);
+	case CRYPTO_SHA1_HMAC:
+		return (&auth_hash_hmac_sha1);
+	case CRYPTO_SHA2_224_HMAC:
+		return (&auth_hash_hmac_sha2_224);
+	case CRYPTO_SHA2_256_HMAC:
+		return (&auth_hash_hmac_sha2_256);
+	case CRYPTO_SHA2_384_HMAC:
+		return (&auth_hash_hmac_sha2_384);
+	case CRYPTO_SHA2_512_HMAC:
+		return (&auth_hash_hmac_sha2_512);
+	case CRYPTO_NULL_HMAC:
+		return (&auth_hash_null);
+	case CRYPTO_RIPEMD160_HMAC:
+		return (&auth_hash_hmac_ripemd_160);
+	case CRYPTO_MD5_KPDK:
+		return (&auth_hash_key_md5);
+	case CRYPTO_SHA1_KPDK:
+		return (&auth_hash_key_sha1);
+#ifdef notyet
+	case CRYPTO_MD5:
+		return (&auth_hash_md5);
+#endif
+	case CRYPTO_SHA1:
+		return (&auth_hash_sha1);
+	case CRYPTO_SHA2_224:
+		return (&auth_hash_sha2_224);
+	case CRYPTO_SHA2_256:
+		return (&auth_hash_sha2_256);
+	case CRYPTO_SHA2_384:
+		return (&auth_hash_sha2_384);
+	case CRYPTO_SHA2_512:
+		return (&auth_hash_sha2_512);
+	case CRYPTO_AES_NIST_GMAC:
+		switch (csp->csp_auth_klen) {
+		case 128 / 8:
+			return (&auth_hash_nist_gmac_aes_128);
+		case 192 / 8:
+			return (&auth_hash_nist_gmac_aes_192);
+		case 256 / 8:
+			return (&auth_hash_nist_gmac_aes_256);
+		default:
+			return (NULL);
+		}
+	case CRYPTO_BLAKE2B:
+		return (&auth_hash_blake2b);
+	case CRYPTO_BLAKE2S:
+		return (&auth_hash_blake2s);
+	case CRYPTO_POLY1305:
+		return (&auth_hash_poly1305);
+	case CRYPTO_AES_CCM_CBC_MAC:
+		switch (csp->csp_auth_klen) {
+		case 128 / 8:
+			return (&auth_hash_ccm_cbc_mac_128);
+		case 192 / 8:
+			return (&auth_hash_ccm_cbc_mac_192);
+		case 256 / 8:
+			return (&auth_hash_ccm_cbc_mac_256);
+		default:
+			return (NULL);
+		}
+	default:
+		return (NULL);
+	}
 }
 
+struct enc_xform *
+crypto_cipher(const struct crypto_session_params *csp)
+{
+
+	switch (csp->csp_cipher_alg) {
+	case CRYPTO_DES_CBC:
+		return (&enc_xform_des);
+	case CRYPTO_3DES_CBC:
+		return (&enc_xform_3des);
+	case CRYPTO_BLF_CBC:
+		return (&enc_xform_blf);
+	case CRYPTO_CAST_CBC:
+		return (&enc_xform_cast5);
+	case CRYPTO_SKIPJACK_CBC:
+		return (&enc_xform_skipjack);
+	case CRYPTO_RIJNDAEL128_CBC:
+		return (&enc_xform_rijndael128);
+	case CRYPTO_AES_XTS:
+		return (&enc_xform_aes_xts);
+	case CRYPTO_AES_ICM:
+		return (&enc_xform_aes_icm);
+	case CRYPTO_AES_NIST_GCM_16:
+		return (&enc_xform_aes_nist_gcm);
+	case CRYPTO_CAMELLIA_CBC:
+		return (&enc_xform_camellia);
+	case CRYPTO_NULL_CBC:
+		return (&enc_xform_null);
+	case CRYPTO_CHACHA20:
+		return (&enc_xform_chacha20);
+	case CRYPTO_AES_CCM_16:
+		return (&enc_xform_ccm);
+	default:
+		return (NULL);
+	}
+}
+
+static struct cryptocap *
+crypto_checkdriver(u_int32_t hid)
+{
+
+	return (hid >= crypto_drivers_size ? NULL : crypto_drivers[hid]);
+}
+
 /*
  * Select a driver for a new session that supports the specified
  * algorithms and, optionally, is constrained according to the flags.
- * The algorithm we use here is pretty stupid; just use the
- * first driver that supports all the algorithms we need. If there
- * are multiple drivers we choose the driver with the fewest active
- * sessions.  We prefer hardware-backed drivers to software ones.
- *
- * XXX We need more smarts here (in real life too, but that's
- * XXX another story altogether).
  */
 static struct cryptocap *
-crypto_select_driver(const struct cryptoini *cri, int flags)
+crypto_select_driver(const struct crypto_session_params *csp, int flags)
 {
 	struct cryptocap *cap, *best;
-	int match, hid;
+	int best_match, error, hid;
 
 	CRYPTO_DRIVER_ASSERT();
 
-	/*
-	 * Look first for hardware crypto devices if permitted.
-	 */
-	if (flags & CRYPTOCAP_F_HARDWARE)
-		match = CRYPTOCAP_F_HARDWARE;
-	else
-		match = CRYPTOCAP_F_SOFTWARE;
 	best = NULL;
-again:
-	for (hid = 0; hid < crypto_drivers_num; hid++) {
-		cap = &crypto_drivers[hid];
+	for (hid = 0; hid < crypto_drivers_size; hid++) {
 		/*
-		 * If it's not initialized, is in the process of
-		 * going away, or is not appropriate (hardware
-		 * or software based on match), then skip.
+		 * If there is no driver for this slot, or the driver
+		 * is not appropriate (hardware or software based on
+		 * match), then skip.
 		 */
-		if (cap->cc_dev == NULL ||
-		    (cap->cc_flags & CRYPTOCAP_F_CLEANUP) ||
-		    (cap->cc_flags & match) == 0)
+		cap = crypto_drivers[hid];
+		if (cap == NULL ||
+		    (cap->cc_flags & flags) == 0)
 			continue;
 
-		/* verify all the algorithms are supported. */
-		if (driver_suitable(cap, cri)) {
-			if (best == NULL ||
-			    cap->cc_sessions < best->cc_sessions)
-				best = cap;
+		error = CRYPTODEV_PROBESESSION(cap->cc_dev, csp);
+		if (error >= 0)
+			continue;
+
+		/*
+		 * Use the driver with the highest probe value.
+		 * Hardware drivers use a higher probe value than
+		 * software.  In case of a tie, prefer the driver with
+		 * the fewest active sessions.
+		 */
+		if (best == NULL || error > best_match ||
+		    (error == best_match &&
+		    cap->cc_sessions < best->cc_sessions)) {
+			best = cap;
+			best_match = error;
 		}
 	}
-	if (best == NULL && match == CRYPTOCAP_F_HARDWARE &&
-	    (flags & CRYPTOCAP_F_SOFTWARE)) {
-		/* sort of an Algol 68-style for loop */
-		match = CRYPTOCAP_F_SOFTWARE;
-		goto again;
-	}
 	return best;
 }
 
+static bool
+alg_is_compression(int alg)
+{
+
+	if (alg == CRYPTO_DEFLATE_COMP)
+		return (true);
+	return (false);
+}
+
+static bool
+alg_is_cipher(int alg)
+{
+
+	if (alg >= CRYPTO_DES_CBC && alg <= CRYPTO_SKIPJACK_CBC)
+		return (true);
+	if (alg >= CRYPTO_AES_CBC && alg <= CRYPTO_ARC4)
+		return (true);
+	if (alg == CRYPTO_NULL_CBC)
+		return (true);
+	if (alg >= CRYPTO_CAMELLIA_CBC && alg <= CRYPTO_AES_ICM)
+		return (true);
+	if (alg == CRYPTO_CHACHA20)
+		return (true);
+	return (false);
+}
+
+static bool
+alg_is_digest(int alg)
+{
+
+	if (alg >= CRYPTO_MD5_HMAC && alg <= CRYPTO_SHA1_KPDK)
+		return (true);
+	if (alg >= CRYPTO_MD5 && alg <= CRYPTO_SHA1)
+		return (true);
+	if (alg == CRYPTO_NULL_HMAC)
+		return (true);
+	if (alg >= CRYPTO_SHA2_256_HMAC && alg <= CRYPTO_SHA2_512_HMAC)
+		return (true);
+	if (alg >= CRYPTO_SHA2_256_HMAC && alg <= CRYPTO_SHA2_512_HMAC)
+		return (true);
+	if (alg == CRYPTO_AES_NIST_GMAC)
+		return (true);
+	if (alg >= CRYPTO_BLAKE2B && alg <= CRYPTO_BLAKE2S)
+		return (true);
+	if (alg >= CRYPTO_SHA2_224_HMAC && alg <= CRYPTO_POLY1305)
+		return (true);
+	if (alg == CRYPTO_AES_CCM_CBC_MAC)
+		return (true);
+	return (false);
+}
+
+static bool
+alg_is_keyed_digest(int alg)
+{
+
+	if (alg >= CRYPTO_MD5_HMAC && alg <= CRYPTO_SHA1_KPDK)
+		return (true);
+	if (alg >= CRYPTO_SHA2_256_HMAC && alg <= CRYPTO_SHA2_512_HMAC)
+		return (true);
+	if (alg == CRYPTO_AES_NIST_GMAC)
+		return (true);
+	if (alg >= CRYPTO_BLAKE2B && alg <= CRYPTO_BLAKE2S)
+		return (true);
+	if (alg == CRYPTO_SHA2_224_HMAC)
+		return (true);
+	if (alg == CRYPTO_POLY1305)
+		return (true);
+	if (alg == CRYPTO_AES_CCM_CBC_MAC)
+		return (true);
+	return (false);
+}
+
+static bool
+alg_is_aead(int alg)
+{
+
+	if (alg == CRYPTO_AES_NIST_GCM_16)
+		return (true);
+	if (alg == CRYPTO_AES_CCM_16)
+		return (true);
+	return (false);
+}
+
+/* Various sanity checks on crypto session parameters. */
+static bool
+check_csp(const struct crypto_session_params *csp)
+{
+	struct auth_hash *axf;
+
+	/* Mode-independent checks. */
+	if (csp->csp_flags != 0)
+		return (false);
+	if (csp->csp_ivlen < 0 || csp->csp_cipher_klen < 0 ||
+	    csp->csp_auth_klen < 0 || csp->csp_auth_mlen < 0)
+		return (false);
+	if (csp->csp_auth_key != NULL && csp->csp_auth_klen == 0)
+		return (false);
+	if (csp->csp_cipher_key != NULL && csp->csp_cipher_klen == 0)
+		return (false);
+
+	switch (csp->csp_mode) {
+	case CSP_MODE_COMPRESS:
+		if (!alg_is_compression(csp->csp_cipher_alg))
+			return (false);
+		if (csp->csp_flags != 0)
+			return (false);
+		if (csp->csp_cipher_klen != 0 || csp->csp_ivlen != 0 ||
+		    csp->csp_auth_alg != 0 || csp->csp_auth_klen != 0 ||
+		    csp->csp_auth_mlen != 0)
+			return (false);
+		break;
+	case CSP_MODE_CIPHER:
+		if (!alg_is_cipher(csp->csp_cipher_alg))
+			return (false);
+		if (csp->csp_cipher_alg != CRYPTO_NULL_CBC) {
+			if (csp->csp_cipher_klen == 0)
+				return (false);
+			if (csp->csp_cipher_alg != CRYPTO_ARC4) {
+				if (csp->csp_ivlen == 0)
+					return (false);
+			}
+		}
+		if (csp->csp_ivlen >= EALG_MAX_BLOCK_LEN)
+			return (false);
+		if (csp->csp_auth_alg != 0 || csp->csp_auth_klen != 0 ||
+		    csp->csp_auth_mlen != 0)
+			return (false);
+		break;
+	case CSP_MODE_DIGEST:
+		if (csp->csp_cipher_alg != 0 || csp->csp_cipher_klen != 0)
+			return (false);
+
+		/* IV is optional for digests (e.g. GMAC). */
+		if (csp->csp_ivlen >= EALG_MAX_BLOCK_LEN)
+			return (false);
+		if (!alg_is_digest(csp->csp_auth_alg))
+			return (false);
+
+		/* Key is optional for BLAKE2 digests. */
+		if (csp->csp_auth_alg == CRYPTO_BLAKE2B ||
+		    csp->csp_auth_alg == CRYPTO_BLAKE2S)
+			;
+		else if (alg_is_keyed_digest(csp->csp_auth_alg)) {
+			if (csp->csp_auth_klen == 0)
+				return (false);
+		} else {
+			if (csp->csp_auth_klen != 0)
+				return (false);
+		}
+		if (csp->csp_auth_mlen != 0) {
+			axf = crypto_auth_hash(csp);
+			if (axf == NULL || csp->csp_auth_mlen > axf->hashsize)
+				return (false);
+		}
+		break;
+	case CSP_MODE_AEAD:
+		if (!alg_is_aead(csp->csp_cipher_alg))
+			return (false);
+		if (csp->csp_cipher_klen == 0)
+			return (false);
+		if (csp->csp_ivlen == 0 ||
+		    csp->csp_ivlen >= EALG_MAX_BLOCK_LEN)
+			return (false);
+		if (csp->csp_auth_alg != 0 || csp->csp_auth_klen != 0)
+			return (false);
+
+		/*
+		 * XXX: Would be nice to have a better way to get this
+		 * value.
+		 */
+		switch (csp->csp_cipher_alg) {
+		case CRYPTO_AES_NIST_GCM_16:
+		case CRYPTO_AES_CCM_16:
+			if (csp->csp_auth_mlen > 16)
+				return (false);
+			break;
+		}
+		break;
+	case CSP_MODE_ETA:
+		if (!alg_is_cipher(csp->csp_cipher_alg))
+			return (false);
+		if (csp->csp_cipher_alg != CRYPTO_NULL_CBC) {
+			if (csp->csp_cipher_klen == 0)
+				return (false);
+			if (csp->csp_cipher_alg != CRYPTO_ARC4) {
+				if (csp->csp_ivlen == 0)
+					return (false);
+			}
+		}
+		if (csp->csp_ivlen >= EALG_MAX_BLOCK_LEN)
+			return (false);
+		if (!alg_is_digest(csp->csp_auth_alg))
+			return (false);
+
+		/* Key is optional for BLAKE2 digests. */
+		if (csp->csp_auth_alg == CRYPTO_BLAKE2B ||
+		    csp->csp_auth_alg == CRYPTO_BLAKE2S)
+			;
+		else if (alg_is_keyed_digest(csp->csp_auth_alg)) {
+			if (csp->csp_auth_klen == 0)
+				return (false);
+		} else {
+			if (csp->csp_auth_klen != 0)
+				return (false);
+		}
+		if (csp->csp_auth_mlen != 0) {
+			axf = crypto_auth_hash(csp);
+			if (axf == NULL || csp->csp_auth_mlen > axf->hashsize)
+				return (false);
+		}
+		break;
+	default:
+		return (false);
+	}
+
+	return (true);
+}
+
 /*
+ * Delete a session after it has been detached from its driver.
+ */
+static void
+crypto_deletesession(crypto_session_t cses)
+{
+	struct cryptocap *cap;
+
+	cap = cses->cap;
+
+	explicit_bzero(cses->softc, cap->cc_session_size);
+	free(cses->softc, M_CRYPTO_DATA);
+	uma_zfree(cryptoses_zone, cses);
+
+	CRYPTO_DRIVER_LOCK();
+	cap->cc_sessions--;
+	if (cap->cc_sessions == 0 && cap->cc_flags & CRYPTOCAP_F_CLEANUP)
+		wakeup(cap);
+	CRYPTO_DRIVER_UNLOCK();
+	cap_rele(cap);
+}
+
+/*
  * Create a new session.  The crid argument specifies a crypto
  * driver to use or constraints on a driver to select (hardware
  * only, software only, either).  Whatever driver is selected
  * must be capable of the requested crypto algorithms.
  */
 int
-crypto_newsession(crypto_session_t *cses, struct cryptoini *cri, int crid)
+crypto_newsession(crypto_session_t *cses,
+    const struct crypto_session_params *csp, int crid)
 {
 	crypto_session_t res;
-	void *softc_mem;
 	struct cryptocap *cap;
-	u_int32_t hid;
-	size_t softc_size;
 	int err;
 
-restart:
+	if (!check_csp(csp))
+		return (EINVAL);
+
 	res = NULL;
-	softc_mem = NULL;
 
 	CRYPTO_DRIVER_LOCK();
 	if ((crid & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) {
 		/*
 		 * Use specified driver; verify it is capable.
 		 */
 		cap = crypto_checkdriver(crid);
-		if (cap != NULL && !driver_suitable(cap, cri))
+		if (cap != NULL && CRYPTODEV_PROBESESSION(cap->cc_dev, csp) > 0)
 			cap = NULL;
 	} else {
 		/*
 		 * No requested driver; select based on crid flags.
 		 */
-		cap = crypto_select_driver(cri, crid);
-		/*
-		 * if NULL then can't do everything in one session.
-		 * XXX Fix this. We need to inject a "virtual" session
-		 * XXX layer right about here.
-		 */
+		cap = crypto_select_driver(csp, crid);
 	}
 	if (cap == NULL) {
+		CRYPTO_DRIVER_UNLOCK();
 		CRYPTDEB("no driver");
-		err = EOPNOTSUPP;
-		goto out;
+		return (EOPNOTSUPP);
 	}
+	cap_ref(cap);
 	cap->cc_sessions++;
-	softc_size = cap->cc_session_size;
-	hid = cap - crypto_drivers;
-	cap = NULL;
 	CRYPTO_DRIVER_UNLOCK();
 
-	softc_mem = malloc(softc_size, M_CRYPTO_DATA, M_WAITOK | M_ZERO);
 	res = uma_zalloc(cryptoses_zone, M_WAITOK | M_ZERO);
-	res->softc = softc_mem;
+	res->cap = cap;
+	res->softc = malloc(cap->cc_session_size, M_CRYPTO_DATA, M_WAITOK |
+	    M_ZERO);
+	res->csp = *csp;
 
-	CRYPTO_DRIVER_LOCK();
-	cap = crypto_checkdriver(hid);
-	if (cap != NULL && (cap->cc_flags & CRYPTOCAP_F_CLEANUP) != 0) {
-		cap->cc_sessions--;
-		crypto_remove(cap);
-		cap = NULL;
-	}
-	if (cap == NULL) {
-		free(softc_mem, M_CRYPTO_DATA);
-		uma_zfree(cryptoses_zone, res);
-		CRYPTO_DRIVER_UNLOCK();
-		goto restart;
-	}
-
 	/* Call the driver initialization routine. */
-	err = CRYPTODEV_NEWSESSION(cap->cc_dev, res, cri);
+	err = CRYPTODEV_NEWSESSION(cap->cc_dev, res, csp);
 	if (err != 0) {
 		CRYPTDEB("dev newsession failed: %d", err);
-		goto out;
+		crypto_deletesession(res);
+		return (err);
 	}
 
-	res->capabilities = cap->cc_flags & 0xff000000;
-	res->hid = hid;
 	*cses = res;
-
-out:
-	CRYPTO_DRIVER_UNLOCK();
-	if (err != 0) {
-		free(softc_mem, M_CRYPTO_DATA);
-		if (res != NULL)
-			uma_zfree(cryptoses_zone, res);
-	}
-	return err;
+	return (0);
 }
 
-static void
-crypto_remove(struct cryptocap *cap)
-{
-
-	mtx_assert(&crypto_drivers_mtx, MA_OWNED);
-	if (cap->cc_sessions == 0 && cap->cc_koperations == 0)
-		bzero(cap, sizeof(*cap));
-}
-
 /*
  * Delete an existing session (or a reserved session on an unregistered
  * driver).
  */
 void
 crypto_freesession(crypto_session_t cses)
 {
 	struct cryptocap *cap;
-	void *ses;
-	size_t ses_size;
-	u_int32_t hid;
 
 	if (cses == NULL)
 		return;
 
-	CRYPTO_DRIVER_LOCK();
+	cap = cses->cap;
 
-	hid = crypto_ses2hid(cses);
-	KASSERT(hid < crypto_drivers_num,
-	    ("bogus crypto_session %p hid %u", cses, hid));
-	cap = &crypto_drivers[hid];
-
-	ses = cses->softc;
-	ses_size = cap->cc_session_size;
-
-	if (cap->cc_sessions)
-		cap->cc_sessions--;
-
 	/* Call the driver cleanup routine, if available. */
 	CRYPTODEV_FREESESSION(cap->cc_dev, cses);
 
-	explicit_bzero(ses, ses_size);
-	free(ses, M_CRYPTO_DATA);
-	uma_zfree(cryptoses_zone, cses);
-
-	if (cap->cc_flags & CRYPTOCAP_F_CLEANUP)
-		crypto_remove(cap);
-
-	CRYPTO_DRIVER_UNLOCK();
+	crypto_deletesession(cses);
 }
 
 /*
- * Return an unused driver id.  Used by drivers prior to registering
- * support for the algorithms they handle.
+ * Return a new driver id.  Registers a driver with the system so that
+ * it can be probed by subsequent sessions.
  */
 int32_t
 crypto_get_driverid(device_t dev, size_t sessionsize, int flags)
 {
-	struct cryptocap *newdrv;
+	struct cryptocap *cap, **newdrv;
 	int i;
 
 	if ((flags & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) {
-		printf("%s: no flags specified when registering driver\n",
-		    device_get_nameunit(dev));
+		device_printf(dev,
+		    "no flags specified when registering driver\n");
 		return -1;
 	}
 
+	cap = malloc(sizeof(*cap), M_CRYPTO_DATA, M_WAITOK | M_ZERO);
+	cap->cc_dev = dev;
+	cap->cc_session_size = sessionsize;
+	cap->cc_flags = flags;
+	refcount_init(&cap->cc_refs, 1);
+
 	CRYPTO_DRIVER_LOCK();
+	for (;;) {
+		for (i = 0; i < crypto_drivers_size; i++) {
+			if (crypto_drivers[i] == NULL)
+				break;
+		}
 
-	for (i = 0; i < crypto_drivers_num; i++) {
-		if (crypto_drivers[i].cc_dev == NULL &&
-		    (crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP) == 0) {
+		if (i < crypto_drivers_size)
 			break;
-		}
-	}
 
-	/* Out of entries, allocate some more. */
-	if (i == crypto_drivers_num) {
-		/* Be careful about wrap-around. */
-		if (2 * crypto_drivers_num <= crypto_drivers_num) {
+		/* Out of entries, allocate some more. */
+
+		if (2 * crypto_drivers_size <= crypto_drivers_size) {
 			CRYPTO_DRIVER_UNLOCK();
 			printf("crypto: driver count wraparound!\n");
-			return -1;
+			cap_rele(cap);
+			return (-1);
 		}
+		CRYPTO_DRIVER_UNLOCK();
 
-		newdrv = malloc(2 * crypto_drivers_num *
-		    sizeof(struct cryptocap), M_CRYPTO_DATA, M_NOWAIT|M_ZERO);
-		if (newdrv == NULL) {
-			CRYPTO_DRIVER_UNLOCK();
-			printf("crypto: no space to expand driver table!\n");
-			return -1;
-		}
+		newdrv = malloc(2 * crypto_drivers_size *
+		    sizeof(*crypto_drivers), M_CRYPTO_DATA, M_WAITOK | M_ZERO);
 
-		bcopy(crypto_drivers, newdrv,
-		    crypto_drivers_num * sizeof(struct cryptocap));
+		CRYPTO_DRIVER_LOCK();
+		memcpy(newdrv, crypto_drivers,
+		    crypto_drivers_size * sizeof(*crypto_drivers));
 
-		crypto_drivers_num *= 2;
+		crypto_drivers_size *= 2;
 
 		free(crypto_drivers, M_CRYPTO_DATA);
 		crypto_drivers = newdrv;
 	}
 
-	/* NB: state is zero'd on free */
-	crypto_drivers[i].cc_sessions = 1;	/* Mark */
-	crypto_drivers[i].cc_dev = dev;
-	crypto_drivers[i].cc_flags = flags;
-	crypto_drivers[i].cc_session_size = sessionsize;
+	cap->cc_hid = i;
+	crypto_drivers[i] = cap;
+	CRYPTO_DRIVER_UNLOCK();
+
 	if (bootverbose)
 		printf("crypto: assign %s driver id %u, flags 0x%x\n",
 		    device_get_nameunit(dev), i, flags);
 
-	CRYPTO_DRIVER_UNLOCK();
-
 	return i;
 }
 
 /*
  * Lookup a driver by name.  We match against the full device
  * name and unit, and against just the name.  The latter gives
  * us a simple widlcarding by device name.  On success return the
  * driver/hardware identifier; otherwise return -1.
  */
 int
 crypto_find_driver(const char *match)
 {
+	struct cryptocap *cap;
 	int i, len = strlen(match);
 
 	CRYPTO_DRIVER_LOCK();
-	for (i = 0; i < crypto_drivers_num; i++) {
-		device_t dev = crypto_drivers[i].cc_dev;
-		if (dev == NULL ||
-		    (crypto_drivers[i].cc_flags & CRYPTOCAP_F_CLEANUP))
+	for (i = 0; i < crypto_drivers_size; i++) {
+		if (crypto_drivers[i] == NULL)
 			continue;
-		if (strncmp(match, device_get_nameunit(dev), len) == 0 ||
-		    strncmp(match, device_get_name(dev), len) == 0)
-			break;
+		cap = crypto_drivers[i];
+		if (strncmp(match, device_get_nameunit(cap->cc_dev), len) == 0 ||
+		    strncmp(match, device_get_name(cap->cc_dev), len) == 0) {
+			CRYPTO_DRIVER_UNLOCK();
+			return (i);
+		}
 	}
 	CRYPTO_DRIVER_UNLOCK();
-	return i < crypto_drivers_num ? i : -1;
+	return (-1);
 }
 
 /*
  * Return the device_t for the specified driver or NULL
  * if the driver identifier is invalid.
  */
 device_t
 crypto_find_device_byhid(int hid)
 {
-	struct cryptocap *cap = crypto_checkdriver(hid);
-	return cap != NULL ? cap->cc_dev : NULL;
+	struct cryptocap *cap;
+	device_t dev;
+
+	dev = NULL;
+	CRYPTO_DRIVER_LOCK();
+	cap = crypto_checkdriver(hid);
+	if (cap != NULL)
+		dev = cap->cc_dev;
+	CRYPTO_DRIVER_UNLOCK();
+	return (dev);
 }
 
 /*
  * Return the device/driver capabilities.
  */
 int
 crypto_getcaps(int hid)
 {
-	struct cryptocap *cap = crypto_checkdriver(hid);
-	return cap != NULL ? cap->cc_flags : 0;
+	struct cryptocap *cap;
+	int flags;
+
+	flags = 0;
+	CRYPTO_DRIVER_LOCK();
+	cap = crypto_checkdriver(hid);
+	if (cap != NULL)
+		flags = cap->cc_flags;
+	CRYPTO_DRIVER_UNLOCK();
+	return (flags);
 }
 
 /*
  * Register support for a key-related algorithm.  This routine
  * is called once for each algorithm supported a driver.
  */
 int
 crypto_kregister(u_int32_t driverid, int kalg, u_int32_t flags)
 {
 	struct cryptocap *cap;
 	int err;
 
 	CRYPTO_DRIVER_LOCK();
 
 	cap = crypto_checkdriver(driverid);
 	if (cap != NULL &&
 	    (CRK_ALGORITM_MIN <= kalg && kalg <= CRK_ALGORITHM_MAX)) {
 		/*
 		 * XXX Do some performance testing to determine placing.
 		 * XXX We probably need an auxiliary data structure that
 		 * XXX describes relative performances.
 		 */
 
 		cap->cc_kalg[kalg] = flags | CRYPTO_ALG_FLAG_SUPPORTED;
 		if (bootverbose)
 			printf("crypto: %s registers key alg %u flags %u\n"
 				, device_get_nameunit(cap->cc_dev)
 				, kalg
 				, flags
 			);
 		err = 0;
 	} else
 		err = EINVAL;
 
 	CRYPTO_DRIVER_UNLOCK();
 	return err;
 }
 
 /*
- * Register support for a non-key-related algorithm.  This routine
- * is called once for each such algorithm supported by a driver.
- */
-int
-crypto_register(u_int32_t driverid, int alg, u_int16_t maxoplen,
-    u_int32_t flags)
-{
-	struct cryptocap *cap;
-	int err;
-
-	CRYPTO_DRIVER_LOCK();
-
-	cap = crypto_checkdriver(driverid);
-	/* NB: algorithms are in the range [1..max] */
-	if (cap != NULL &&
-	    (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX)) {
-		/*
-		 * XXX Do some performance testing to determine placing.
-		 * XXX We probably need an auxiliary data structure that
-		 * XXX describes relative performances.
-		 */
-
-		cap->cc_alg[alg] = flags | CRYPTO_ALG_FLAG_SUPPORTED;
-		cap->cc_max_op_len[alg] = maxoplen;
-		if (bootverbose)
-			printf("crypto: %s registers alg %u flags %u maxoplen %u\n"
-				, device_get_nameunit(cap->cc_dev)
-				, alg
-				, flags
-				, maxoplen
-			);
-		cap->cc_sessions = 0;		/* Unmark */
-		err = 0;
-	} else
-		err = EINVAL;
-
-	CRYPTO_DRIVER_UNLOCK();
-	return err;
-}
-
-static void
-driver_finis(struct cryptocap *cap)
-{
-	u_int32_t ses, kops;
-
-	CRYPTO_DRIVER_ASSERT();
-
-	ses = cap->cc_sessions;
-	kops = cap->cc_koperations;
-	bzero(cap, sizeof(*cap));
-	if (ses != 0 || kops != 0) {
-		/*
-		 * If there are pending sessions,
-		 * just mark as invalid.
-		 */
-		cap->cc_flags |= CRYPTOCAP_F_CLEANUP;
-		cap->cc_sessions = ses;
-		cap->cc_koperations = kops;
-	}
-}
-
-/*
- * Unregister a crypto driver. If there are pending sessions using it,
- * leave enough information around so that subsequent calls using those
- * sessions will correctly detect the driver has been unregistered and
- * reroute requests.
- */
-int
-crypto_unregister(u_int32_t driverid, int alg)
-{
-	struct cryptocap *cap;
-	int i, err;
-
-	CRYPTO_DRIVER_LOCK();
-	cap = crypto_checkdriver(driverid);
-	if (cap != NULL &&
-	    (CRYPTO_ALGORITHM_MIN <= alg && alg <= CRYPTO_ALGORITHM_MAX) &&
-	    cap->cc_alg[alg] != 0) {
-		cap->cc_alg[alg] = 0;
-		cap->cc_max_op_len[alg] = 0;
-
-		/* Was this the last algorithm ? */
-		for (i = 1; i <= CRYPTO_ALGORITHM_MAX; i++)
-			if (cap->cc_alg[i] != 0)
-				break;
-
-		if (i == CRYPTO_ALGORITHM_MAX + 1)
-			driver_finis(cap);
-		err = 0;
-	} else
-		err = EINVAL;
-	CRYPTO_DRIVER_UNLOCK();
-
-	return err;
-}
-
-/*
  * Unregister all algorithms associated with a crypto driver.
  * If there are pending sessions using it, leave enough information
  * around so that subsequent calls using those sessions will
  * correctly detect the driver has been unregistered and reroute
  * requests.
  */
 int
 crypto_unregister_all(u_int32_t driverid)
 {
 	struct cryptocap *cap;
-	int err;
 
 	CRYPTO_DRIVER_LOCK();
 	cap = crypto_checkdriver(driverid);
-	if (cap != NULL) {
-		driver_finis(cap);
-		err = 0;
-	} else
-		err = EINVAL;
+	if (cap == NULL) {
+		CRYPTO_DRIVER_UNLOCK();
+		return (EINVAL);
+	}
+
+	cap->cc_flags |= CRYPTOCAP_F_CLEANUP;
+	crypto_drivers[driverid] = NULL;
+
+	/*
+	 * XXX: This doesn't do anything to kick sessions that
+	 * have no pending operations.
+	 */
+	while (cap->cc_sessions != 0 || cap->cc_koperations != 0)
+		mtx_sleep(cap, &crypto_drivers_mtx, 0, "cryunreg", 0);
 	CRYPTO_DRIVER_UNLOCK();
+	cap_rele(cap);
 
-	return err;
+	return (0);
 }
 
 /*
  * Clear blockage on a driver.  The what parameter indicates whether
  * the driver is now ready for cryptop's and/or cryptokop's.
  */
 int
 crypto_unblock(u_int32_t driverid, int what)
 {
 	struct cryptocap *cap;
 	int err;
 
 	CRYPTO_Q_LOCK();
 	cap = crypto_checkdriver(driverid);
 	if (cap != NULL) {
 		if (what & CRYPTO_SYMQ)
 			cap->cc_qblocked = 0;
 		if (what & CRYPTO_ASYMQ)
 			cap->cc_kqblocked = 0;
 		if (crp_sleep)
 			wakeup_one(&crp_q);
 		err = 0;
 	} else
 		err = EINVAL;
 	CRYPTO_Q_UNLOCK();
 
 	return err;
 }
 
+#ifdef INVARIANTS
+/* Various sanity checks on crypto requests. */
+static void
+crp_sanity(struct cryptop *crp)
+{
+	struct crypto_session_params *csp;
+
+	KASSERT(crp->crp_session != NULL, ("incoming crp without a session"));
+	KASSERT(crp->crp_ilen >= 0, ("incoming crp with -ve input length"));
+	KASSERT(crp->crp_etype == 0, ("incoming crp with error"));
+	KASSERT(!(crp->crp_flags & CRYPTO_F_DONE),
+	    ("incoming crp already done"));
+
+	csp = &crp->crp_session->csp;
+	switch (csp->csp_mode) {
+	case CSP_MODE_COMPRESS:
+		KASSERT(crp->crp_op == CRYPTO_OP_COMPRESS ||
+		    crp->crp_op == CRYPTO_OP_DECOMPRESS,
+		    ("invalid compression op %x", crp->crp_op));
+		break;
+	case CSP_MODE_CIPHER:
+		KASSERT(crp->crp_op == CRYPTO_OP_ENCRYPT ||
+		    crp->crp_op == CRYPTO_OP_DECRYPT,
+		    ("invalid cipher op %x", crp->crp_op));
+		break;
+	case CSP_MODE_DIGEST:
+		KASSERT(crp->crp_op == CRYPTO_OP_COMPUTE_DIGEST ||
+		    crp->crp_op == CRYPTO_OP_VERIFY_DIGEST,
+		    ("invalid digest op %x", crp->crp_op));
+		break;
+	case CSP_MODE_AEAD:
+		KASSERT(crp->crp_op ==
+		    (CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST) ||
+		    crp->crp_op ==
+		    (CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST),
+		    ("invalid AEAD op %x", crp->crp_op));
+		if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16)
+			KASSERT(crp->crp_flags & CRYPTO_F_IV_SEPARATE,
+			    ("GCM without a separate IV"));
+		if (csp->csp_cipher_alg == CRYPTO_AES_CCM_16)
+			KASSERT(crp->crp_flags & CRYPTO_F_IV_SEPARATE,
+			    ("CCM without a separate IV"));
+		break;
+	case CSP_MODE_ETA:
+		KASSERT(crp->crp_op ==
+		    (CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST) ||
+		    crp->crp_op ==
+		    (CRYPTO_OP_DECRYPT | CRYPTO_OP_VERIFY_DIGEST),
+		    ("invalid ETA op %x", crp->crp_op));
+		break;
+	}
+	KASSERT((crp->crp_flags & CRYPTO_F_IV_GENERATE) == 0 ||
+	    crp->crp_op == CRYPTO_OP_ENCRYPT ||
+	    crp->crp_op == (CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST),
+	    ("IV_GENERATE set for non-encryption operation %x", crp->crp_op));
+	KASSERT((crp->crp_flags &
+	    (CRYPTO_F_IV_SEPARATE | CRYPTO_F_IV_GENERATE)) !=
+	    (CRYPTO_F_IV_SEPARATE | CRYPTO_F_IV_GENERATE),
+	    ("crp with both IV_SEPARATE and IV_GENERATE set"));
+	KASSERT(crp->crp_buf_type >= CRYPTO_BUF_CONTIG &&
+	    crp->crp_buf_type <= CRYPTO_BUF_MBUF,
+	    ("invalid crp buffer type %d", crp->crp_buf_type));
+	if (csp->csp_mode == CSP_MODE_AEAD || csp->csp_mode == CSP_MODE_ETA) {
+		KASSERT(crp->crp_aad_start == 0 ||
+		    crp->crp_aad_start < crp->crp_ilen,
+		    ("invalid AAD start"));
+		KASSERT(crp->crp_aad_length != 0 || crp->crp_aad_start == 0,
+		    ("AAD with zero length and non-zero start"));
+		KASSERT(crp->crp_aad_length == 0 ||
+		    crp->crp_aad_start + crp->crp_aad_length <= crp->crp_ilen,
+		    ("AAD outside input length"));
+	} else {
+		KASSERT(crp->crp_aad_start == 0 && crp->crp_aad_length == 0,
+		    ("AAD region in request not supporting AAD"));
+	}
+	if (csp->csp_ivlen == 0) {
+		KASSERT((crp->crp_flags &
+		    (CRYPTO_F_IV_SEPARATE | CRYPTO_F_IV_GENERATE)) == 0,
+		    ("IV_GENERATE or IV_SEPARATE set when IV isn't used"));
+		KASSERT(crp->crp_iv_start == 0,
+		    ("crp_iv_start set when IV isn't used"));
+	} else if (crp->crp_flags & CRYPTO_F_IV_SEPARATE) {
+		KASSERT(crp->crp_iv_start == 0,
+		    ("IV_SEPARATE used with non-zero IV start"));
+	} else {
+		KASSERT(crp->crp_iv_start < crp->crp_ilen,
+		    ("invalid IV start"));
+		KASSERT(crp->crp_iv_start + csp->csp_ivlen <= crp->crp_ilen,
+		    ("IV outside input length"));
+	}
+	KASSERT(crp->crp_payload_start == 0 ||
+	    crp->crp_payload_start < crp->crp_ilen,
+	    ("invalid payload start"));
+	KASSERT(crp->crp_payload_start + crp->crp_payload_length <=
+	    crp->crp_ilen, ("payload outside input length"));
+	if (csp->csp_mode == CSP_MODE_DIGEST ||
+	    csp->csp_mode == CSP_MODE_AEAD || csp->csp_mode == CSP_MODE_ETA) {
+		KASSERT(crp->crp_digest_start == 0 ||
+		    crp->crp_digest_start < crp->crp_ilen,
+		    ("invalid digest start"));
+		/* XXX: For the mlen == 0 case this check isn't perfect. */
+		KASSERT(crp->crp_digest_start + csp->csp_auth_mlen <=
+		    crp->crp_ilen,
+		    ("digest outside input length"));
+	} else {
+		KASSERT(crp->crp_digest_start == 0,
+		    ("non-zero digest start for request without a digest"));
+	}
+	if (csp->csp_cipher_klen != 0)
+		KASSERT(csp->csp_cipher_key != NULL ||
+		    crp->crp_cipher_key != NULL,
+		    ("cipher request without a key"));
+	if (csp->csp_auth_klen != 0)
+		KASSERT(csp->csp_auth_key != NULL || crp->crp_auth_key != NULL,
+		    ("auth request without a key"));
+	KASSERT(crp->crp_callback != NULL, ("incoming crp without callback"));
+}
+#endif
+
 /*
  * Add a crypto request to a queue, to be processed by the kernel thread.
  */
 int
 crypto_dispatch(struct cryptop *crp)
 {
 	struct cryptocap *cap;
-	u_int32_t hid;
 	int result;
 
+#ifdef INVARIANTS
+	crp_sanity(crp);
+#endif
+
+	/* TODO: Handle CRYPTO_F_IV_GENERATE so drivers don't have to. */
+
 	cryptostats.cs_ops++;
 
 #ifdef CRYPTO_TIMING
 	if (crypto_timing)
 		binuptime(&crp->crp_tstamp);
 #endif
 
 	crp->crp_retw_id = ((uintptr_t)crp->crp_session) % crypto_workers_num;
 
 	if (CRYPTOP_ASYNC(crp)) {
 		if (crp->crp_flags & CRYPTO_F_ASYNC_KEEPORDER) {
 			struct crypto_ret_worker *ret_worker;
 
 			ret_worker = CRYPTO_RETW(crp->crp_retw_id);
 
 			CRYPTO_RETW_LOCK(ret_worker);
 			crp->crp_seq = ret_worker->reorder_ops++;
 			CRYPTO_RETW_UNLOCK(ret_worker);
 		}
 
 		TASK_INIT(&crp->crp_task, 0, crypto_task_invoke, crp);
 		taskqueue_enqueue(crypto_tq, &crp->crp_task);
 		return (0);
 	}
 
 	if ((crp->crp_flags & CRYPTO_F_BATCH) == 0) {
-		hid = crypto_ses2hid(crp->crp_session);
-
 		/*
 		 * Caller marked the request to be processed
 		 * immediately; dispatch it directly to the
 		 * driver unless the driver is currently blocked.
 		 */
-		cap = crypto_checkdriver(hid);
-		/* Driver cannot disappeared when there is an active session. */
-		KASSERT(cap != NULL, ("%s: Driver disappeared.", __func__));
+		cap = crp->crp_session->cap;
 		if (!cap->cc_qblocked) {
 			result = crypto_invoke(cap, crp, 0);
 			if (result != ERESTART)
 				return (result);
 			/*
 			 * The driver ran out of resources, put the request on
 			 * the queue.
 			 */
 		}
 	}
 	crypto_batch_enqueue(crp);
 	return 0;
 }
 
 void
 crypto_batch_enqueue(struct cryptop *crp)
 {
 
 	CRYPTO_Q_LOCK();
 	TAILQ_INSERT_TAIL(&crp_q, crp, crp_next);
 	if (crp_sleep)
 		wakeup_one(&crp_q);
 	CRYPTO_Q_UNLOCK();
 }
 
 /*
  * Add an asymetric crypto request to a queue,
  * to be processed by the kernel thread.
  */
 int
 crypto_kdispatch(struct cryptkop *krp)
 {
 	int error;
 
 	cryptostats.cs_kops++;
 
-	error = crypto_kinvoke(krp, krp->krp_crid);
+	krp->krp_cap = NULL;
+	error = crypto_kinvoke(krp);
 	if (error == ERESTART) {
 		CRYPTO_Q_LOCK();
 		TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next);
 		if (crp_sleep)
 			wakeup_one(&crp_q);
 		CRYPTO_Q_UNLOCK();
 		error = 0;
 	}
 	return error;
 }
 
 /*
  * Verify a driver is suitable for the specified operation.
  */
 static __inline int
 kdriver_suitable(const struct cryptocap *cap, const struct cryptkop *krp)
 {
 	return (cap->cc_kalg[krp->krp_op] & CRYPTO_ALG_FLAG_SUPPORTED) != 0;
 }
 
 /*
  * Select a driver for an asym operation.  The driver must
  * support the necessary algorithm.  The caller can constrain
  * which device is selected with the flags parameter.  The
  * algorithm we use here is pretty stupid; just use the first
  * driver that supports the algorithms we need. If there are
  * multiple suitable drivers we choose the driver with the
  * fewest active operations.  We prefer hardware-backed
  * drivers to software ones when either may be used.
  */
 static struct cryptocap *
 crypto_select_kdriver(const struct cryptkop *krp, int flags)
 {
 	struct cryptocap *cap, *best;
 	int match, hid;
 
 	CRYPTO_DRIVER_ASSERT();
 
 	/*
 	 * Look first for hardware crypto devices if permitted.
 	 */
 	if (flags & CRYPTOCAP_F_HARDWARE)
 		match = CRYPTOCAP_F_HARDWARE;
 	else
 		match = CRYPTOCAP_F_SOFTWARE;
 	best = NULL;
 again:
-	for (hid = 0; hid < crypto_drivers_num; hid++) {
-		cap = &crypto_drivers[hid];
+	for (hid = 0; hid < crypto_drivers_size; hid++) {
 		/*
-		 * If it's not initialized, is in the process of
-		 * going away, or is not appropriate (hardware
-		 * or software based on match), then skip.
+		 * If there is no driver for this slot, or the driver
+		 * is not appropriate (hardware or software based on
+		 * match), then skip.
 		 */
+		cap = crypto_drivers[hid];
 		if (cap->cc_dev == NULL ||
-		    (cap->cc_flags & CRYPTOCAP_F_CLEANUP) ||
 		    (cap->cc_flags & match) == 0)
 			continue;
 
 		/* verify all the algorithms are supported. */
 		if (kdriver_suitable(cap, krp)) {
 			if (best == NULL ||
 			    cap->cc_koperations < best->cc_koperations)
 				best = cap;
 		}
 	}
 	if (best != NULL)
 		return best;
 	if (match == CRYPTOCAP_F_HARDWARE && (flags & CRYPTOCAP_F_SOFTWARE)) {
 		/* sort of an Algol 68-style for loop */
 		match = CRYPTOCAP_F_SOFTWARE;
 		goto again;
 	}
 	return best;
 }
 
 /*
- * Dispatch an asymmetric crypto request.
+ * Choose a driver for an asymmetric crypto request.
  */
-static int
-crypto_kinvoke(struct cryptkop *krp, int crid)
+static struct cryptocap *
+crypto_lookup_kdriver(struct cryptkop *krp)
 {
-	struct cryptocap *cap = NULL;
-	int error;
+	struct cryptocap *cap;
+	uint32_t crid;
 
-	KASSERT(krp != NULL, ("%s: krp == NULL", __func__));
-	KASSERT(krp->krp_callback != NULL,
-	    ("%s: krp->crp_callback == NULL", __func__));
+	/* If this request is requeued, it might already have a driver. */
+	cap = krp->krp_cap;
+	if (cap != NULL)
+		return (cap);
 
-	CRYPTO_DRIVER_LOCK();
+	/* Use krp_crid to choose a driver. */
+	crid = krp->krp_crid;
 	if ((crid & (CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE)) == 0) {
 		cap = crypto_checkdriver(crid);
 		if (cap != NULL) {
 			/*
-			 * Driver present, it must support the necessary
-			 * algorithm and, if s/w drivers are excluded,
-			 * it must be registered as hardware-backed.
+			 * Driver present, it must support the
+			 * necessary algorithm and, if s/w drivers are
+			 * excluded, it must be registered as
+			 * hardware-backed.
 			 */
 			if (!kdriver_suitable(cap, krp) ||
 			    (!crypto_devallowsoft &&
-			     (cap->cc_flags & CRYPTOCAP_F_HARDWARE) == 0))
+			    (cap->cc_flags & CRYPTOCAP_F_HARDWARE) == 0))
 				cap = NULL;
 		}
 	} else {
 		/*
 		 * No requested driver; select based on crid flags.
 		 */
 		if (!crypto_devallowsoft)	/* NB: disallow s/w drivers */
 			crid &= ~CRYPTOCAP_F_SOFTWARE;
 		cap = crypto_select_kdriver(krp, crid);
 	}
-	if (cap != NULL && !cap->cc_kqblocked) {
-		krp->krp_hid = cap - crypto_drivers;
-		cap->cc_koperations++;
+
+	if (cap != NULL) {
+		krp->krp_cap = cap_ref(cap);
+		krp->krp_hid = cap->cc_hid;
+	}
+	return (cap);
+}
+
+/*
+ * Dispatch an asymmetric crypto request.
+ */
+static int
+crypto_kinvoke(struct cryptkop *krp)
+{
+	struct cryptocap *cap = NULL;
+	int error;
+
+	KASSERT(krp != NULL, ("%s: krp == NULL", __func__));
+	KASSERT(krp->krp_callback != NULL,
+	    ("%s: krp->crp_callback == NULL", __func__));
+
+	CRYPTO_DRIVER_LOCK();
+	cap = crypto_lookup_kdriver(krp);
+	if (cap == NULL) {
 		CRYPTO_DRIVER_UNLOCK();
-		error = CRYPTODEV_KPROCESS(cap->cc_dev, krp, 0);
-		CRYPTO_DRIVER_LOCK();
-		if (error == ERESTART) {
-			cap->cc_koperations--;
-			CRYPTO_DRIVER_UNLOCK();
-			return (error);
-		}
-	} else {
+		krp->krp_status = ENODEV;
+		crypto_kdone(krp);
+		return (0);
+	}
+
+	/*
+	 * If the device is blocked, return ERESTART to requeue it.
+	 */
+	if (cap->cc_kqblocked) {
 		/*
-		 * NB: cap is !NULL if device is blocked; in
-		 *     that case return ERESTART so the operation
-		 *     is resubmitted if possible.
+		 * XXX: Previously this set krp_status to ERESTART and
+		 * invoked crypto_kdone but the caller would still
+		 * requeue it.
 		 */
-		error = (cap == NULL) ? ENODEV : ERESTART;
+		CRYPTO_DRIVER_UNLOCK();
+		return (ERESTART);
 	}
-	CRYPTO_DRIVER_UNLOCK();
 
-	if (error) {
-		krp->krp_status = error;
-		crypto_kdone(krp);
+	cap->cc_koperations++;
+	CRYPTO_DRIVER_UNLOCK();
+	error = CRYPTODEV_KPROCESS(cap->cc_dev, krp, 0);
+	if (error == ERESTART) {
+		CRYPTO_DRIVER_LOCK();
+		cap->cc_koperations--;
+		CRYPTO_DRIVER_UNLOCK();
+		return (error);
 	}
-	return 0;
+
+	KASSERT(error == 0, ("error %d returned from crypto_kprocess", error));
+	return (0);
 }
 
 #ifdef CRYPTO_TIMING
 static void
 crypto_tstat(struct cryptotstat *ts, struct bintime *bt)
 {
 	struct bintime now, delta;
 	struct timespec t;
 	uint64_t u;
 
 	binuptime(&now);
 	u = now.frac;
 	delta.frac = now.frac - bt->frac;
 	delta.sec = now.sec - bt->sec;
 	if (u < delta.frac)
 		delta.sec--;
 	bintime2timespec(&delta, &t);
 	timespecadd(&ts->acc, &t, &ts->acc);
 	if (timespeccmp(&t, &ts->min, <))
 		ts->min = t;
 	if (timespeccmp(&t, &ts->max, >))
 		ts->max = t;
 	ts->count++;
 
 	*bt = now;
 }
 #endif
 
 static void
 crypto_task_invoke(void *ctx, int pending)
 {
 	struct cryptocap *cap;
 	struct cryptop *crp;
-	int hid, result;
+	int result;
 
 	crp = (struct cryptop *)ctx;
-
-	hid = crypto_ses2hid(crp->crp_session);
-	cap = crypto_checkdriver(hid);
-
+	cap = crp->crp_session->cap;
 	result = crypto_invoke(cap, crp, 0);
 	if (result == ERESTART)
 		crypto_batch_enqueue(crp);
 }
 
 /*
  * Dispatch a crypto request to the appropriate crypto devices.
  */
 static int
 crypto_invoke(struct cryptocap *cap, struct cryptop *crp, int hint)
 {
 
 	KASSERT(crp != NULL, ("%s: crp == NULL", __func__));
 	KASSERT(crp->crp_callback != NULL,
 	    ("%s: crp->crp_callback == NULL", __func__));
-	KASSERT(crp->crp_desc != NULL, ("%s: crp->crp_desc == NULL", __func__));
+	KASSERT(crp->crp_session != NULL,
+	    ("%s: crp->crp_session == NULL", __func__));
 
 #ifdef CRYPTO_TIMING
 	if (crypto_timing)
 		crypto_tstat(&cryptostats.cs_invoke, &crp->crp_tstamp);
 #endif
 	if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) {
-		struct cryptodesc *crd;
+		struct crypto_session_params csp;
 		crypto_session_t nses;
 
 		/*
 		 * Driver has unregistered; migrate the session and return
 		 * an error to the caller so they'll resubmit the op.
 		 *
 		 * XXX: What if there are more already queued requests for this
 		 *      session?
+		 *
+		 * XXX: Real solution is to make sessions refcounted
+		 * and force callers to hold a reference when
+		 * assigning to crp_session.  Could maybe change
+		 * crypto_getreq to accept a session pointer to make
+		 * that work.  Alternatively, we could abandon the
+		 * notion of rewriting crp_session in requests forcing
+		 * the caller to deal with allocating a new session.
+		 * Perhaps provide a method to allow a crp's session to
+		 * be swapped that callers could use.
 		 */
+		csp = crp->crp_session->csp;
 		crypto_freesession(crp->crp_session);
 
-		for (crd = crp->crp_desc; crd->crd_next; crd = crd->crd_next)
-			crd->CRD_INI.cri_next = &(crd->crd_next->CRD_INI);
-
-		/* XXX propagate flags from initial session? */
-		if (crypto_newsession(&nses, &(crp->crp_desc->CRD_INI),
+		/*
+		 * XXX: Key pointers may no longer be valid.  If we
+		 * really want to support this we need to define the
+		 * KPI such that 'csp' is required to be valid for the
+		 * duration of a session by the caller perhaps.
+		 *
+		 * XXX: If the keys have been changed this will reuse
+		 * the old keys.  This probably suggests making
+		 * rekeying more explicit and updating the key
+		 * pointers in 'csp' when the keys change.
+		 */
+		if (crypto_newsession(&nses, &csp,
 		    CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE) == 0)
 			crp->crp_session = nses;
 
 		crp->crp_etype = EAGAIN;
 		crypto_done(crp);
 		return 0;
 	} else {
 		/*
 		 * Invoke the driver to process the request.
 		 */
 		return CRYPTODEV_PROCESS(cap->cc_dev, crp, hint);
 	}
 }
 
-/*
- * Release a set of crypto descriptors.
- */
 void
 crypto_freereq(struct cryptop *crp)
 {
-	struct cryptodesc *crd;
 
 	if (crp == NULL)
 		return;
 
 #ifdef DIAGNOSTIC
 	{
 		struct cryptop *crp2;
 		struct crypto_ret_worker *ret_worker;
 
 		CRYPTO_Q_LOCK();
 		TAILQ_FOREACH(crp2, &crp_q, crp_next) {
 			KASSERT(crp2 != crp,
 			    ("Freeing cryptop from the crypto queue (%p).",
 			    crp));
 		}
 		CRYPTO_Q_UNLOCK();
 
 		FOREACH_CRYPTO_RETW(ret_worker) {
 			CRYPTO_RETW_LOCK(ret_worker);
 			TAILQ_FOREACH(crp2, &ret_worker->crp_ret_q, crp_next) {
 				KASSERT(crp2 != crp,
 				    ("Freeing cryptop from the return queue (%p).",
 				    crp));
 			}
 			CRYPTO_RETW_UNLOCK(ret_worker);
 		}
 	}
 #endif
 
-	while ((crd = crp->crp_desc) != NULL) {
-		crp->crp_desc = crd->crd_next;
-		uma_zfree(cryptodesc_zone, crd);
-	}
 	uma_zfree(cryptop_zone, crp);
 }
 
-/*
- * Acquire a set of crypto descriptors.
- */
 struct cryptop *
-crypto_getreq(int num)
+crypto_getreq(crypto_session_t cses, int how)
 {
-	struct cryptodesc *crd;
 	struct cryptop *crp;
 
-	crp = uma_zalloc(cryptop_zone, M_NOWAIT|M_ZERO);
-	if (crp != NULL) {
-		while (num--) {
-			crd = uma_zalloc(cryptodesc_zone, M_NOWAIT|M_ZERO);
-			if (crd == NULL) {
-				crypto_freereq(crp);
-				return NULL;
-			}
-
-			crd->crd_next = crp->crp_desc;
-			crp->crp_desc = crd;
-		}
-	}
-	return crp;
+	MPASS(how == M_WAITOK || how == M_NOWAIT);
+	crp = uma_zalloc(cryptop_zone, how | M_ZERO);
+	crp->crp_session = cses;
+	return (crp);
 }
 
 /*
  * Invoke the callback on behalf of the driver.
  */
 void
 crypto_done(struct cryptop *crp)
 {
 	KASSERT((crp->crp_flags & CRYPTO_F_DONE) == 0,
 		("crypto_done: op already done, flags 0x%x", crp->crp_flags));
 	crp->crp_flags |= CRYPTO_F_DONE;
 	if (crp->crp_etype != 0)
 		cryptostats.cs_errs++;
 #ifdef CRYPTO_TIMING
 	if (crypto_timing)
 		crypto_tstat(&cryptostats.cs_done, &crp->crp_tstamp);
 #endif
 	/*
 	 * CBIMM means unconditionally do the callback immediately;
 	 * CBIFSYNC means do the callback immediately only if the
 	 * operation was done synchronously.  Both are used to avoid
 	 * doing extraneous context switches; the latter is mostly
 	 * used with the software crypto driver.
 	 */
 	if (!CRYPTOP_ASYNC_KEEPORDER(crp) &&
 	    ((crp->crp_flags & CRYPTO_F_CBIMM) ||
 	    ((crp->crp_flags & CRYPTO_F_CBIFSYNC) &&
 	     (crypto_ses2caps(crp->crp_session) & CRYPTOCAP_F_SYNC)))) {
 		/*
 		 * Do the callback directly.  This is ok when the
 		 * callback routine does very little (e.g. the
 		 * /dev/crypto callback method just does a wakeup).
 		 */
 #ifdef CRYPTO_TIMING
 		if (crypto_timing) {
 			/*
 			 * NB: We must copy the timestamp before
 			 * doing the callback as the cryptop is
 			 * likely to be reclaimed.
 			 */
 			struct bintime t = crp->crp_tstamp;
 			crypto_tstat(&cryptostats.cs_cb, &t);
 			crp->crp_callback(crp);
 			crypto_tstat(&cryptostats.cs_finis, &t);
 		} else
 #endif
 			crp->crp_callback(crp);
 	} else {
 		struct crypto_ret_worker *ret_worker;
 		bool wake;
 
 		ret_worker = CRYPTO_RETW(crp->crp_retw_id);
 		wake = false;
 
 		/*
 		 * Normal case; queue the callback for the thread.
 		 */
 		CRYPTO_RETW_LOCK(ret_worker);
 		if (CRYPTOP_ASYNC_KEEPORDER(crp)) {
 			struct cryptop *tmp;
 
 			TAILQ_FOREACH_REVERSE(tmp, &ret_worker->crp_ordered_ret_q,
 					cryptop_q, crp_next) {
 				if (CRYPTO_SEQ_GT(crp->crp_seq, tmp->crp_seq)) {
 					TAILQ_INSERT_AFTER(&ret_worker->crp_ordered_ret_q,
 							tmp, crp, crp_next);
 					break;
 				}
 			}
 			if (tmp == NULL) {
 				TAILQ_INSERT_HEAD(&ret_worker->crp_ordered_ret_q,
 						crp, crp_next);
 			}
 
 			if (crp->crp_seq == ret_worker->reorder_cur_seq)
 				wake = true;
 		}
 		else {
 			if (CRYPTO_RETW_EMPTY(ret_worker))
 				wake = true;
 
 			TAILQ_INSERT_TAIL(&ret_worker->crp_ret_q, crp, crp_next);
 		}
 
 		if (wake)
 			wakeup_one(&ret_worker->crp_ret_q);	/* shared wait channel */
 		CRYPTO_RETW_UNLOCK(ret_worker);
 	}
 }
 
 /*
  * Invoke the callback on behalf of the driver.
  */
 void
 crypto_kdone(struct cryptkop *krp)
 {
 	struct crypto_ret_worker *ret_worker;
 	struct cryptocap *cap;
 
 	if (krp->krp_status != 0)
 		cryptostats.cs_kerrs++;
 	CRYPTO_DRIVER_LOCK();
-	/* XXX: What if driver is loaded in the meantime? */
-	if (krp->krp_hid < crypto_drivers_num) {
-		cap = &crypto_drivers[krp->krp_hid];
-		KASSERT(cap->cc_koperations > 0, ("cc_koperations == 0"));
-		cap->cc_koperations--;
-		if (cap->cc_flags & CRYPTOCAP_F_CLEANUP)
-			crypto_remove(cap);
-	}
+	cap = krp->krp_cap;
+	KASSERT(cap->cc_koperations > 0, ("cc_koperations == 0"));
+	cap->cc_koperations--;
+	if (cap->cc_koperations == 0 && cap->cc_flags & CRYPTOCAP_F_CLEANUP)
+		wakeup(cap);
 	CRYPTO_DRIVER_UNLOCK();
+	krp->krp_cap = NULL;
+	cap_rele(cap);
 
 	ret_worker = CRYPTO_RETW(0);
 
 	CRYPTO_RETW_LOCK(ret_worker);
 	if (CRYPTO_RETW_EMPTY(ret_worker))
 		wakeup_one(&ret_worker->crp_ret_q);		/* shared wait channel */
 	TAILQ_INSERT_TAIL(&ret_worker->crp_ret_kq, krp, krp_next);
 	CRYPTO_RETW_UNLOCK(ret_worker);
 }
 
 int
 crypto_getfeat(int *featp)
 {
 	int hid, kalg, feat = 0;
 
 	CRYPTO_DRIVER_LOCK();
-	for (hid = 0; hid < crypto_drivers_num; hid++) {
-		const struct cryptocap *cap = &crypto_drivers[hid];
+	for (hid = 0; hid < crypto_drivers_size; hid++) {
+		const struct cryptocap *cap = crypto_drivers[hid];
 
-		if ((cap->cc_flags & CRYPTOCAP_F_SOFTWARE) &&
-		    !crypto_devallowsoft) {
+		if (cap == NULL ||
+		    ((cap->cc_flags & CRYPTOCAP_F_SOFTWARE) &&
+		    !crypto_devallowsoft)) {
 			continue;
 		}
 		for (kalg = 0; kalg < CRK_ALGORITHM_MAX; kalg++)
 			if (cap->cc_kalg[kalg] & CRYPTO_ALG_FLAG_SUPPORTED)
 				feat |=  1 << kalg;
 	}
 	CRYPTO_DRIVER_UNLOCK();
 	*featp = feat;
 	return (0);
 }
 
 /*
  * Terminate a thread at module unload.  The process that
  * initiated this is waiting for us to signal that we're gone;
  * wake it up and exit.  We use the driver table lock to insure
  * we don't do the wakeup before they're waiting.  There is no
  * race here because the waiter sleeps on the proc lock for the
  * thread so it gets notified at the right time because of an
  * extra wakeup that's done in exit1().
  */
 static void
 crypto_finis(void *chan)
 {
 	CRYPTO_DRIVER_LOCK();
 	wakeup_one(chan);
 	CRYPTO_DRIVER_UNLOCK();
 	kproc_exit(0);
 }
 
 /*
  * Crypto thread, dispatches crypto requests.
  */
 static void
 crypto_proc(void)
 {
 	struct cryptop *crp, *submit;
 	struct cryptkop *krp;
 	struct cryptocap *cap;
-	u_int32_t hid;
 	int result, hint;
 
 #if defined(__i386__) || defined(__amd64__) || defined(__aarch64__)
 	fpu_kern_thread(FPU_KERN_NORMAL);
 #endif
 
 	CRYPTO_Q_LOCK();
 	for (;;) {
 		/*
 		 * Find the first element in the queue that can be
 		 * processed and look-ahead to see if multiple ops
 		 * are ready for the same driver.
 		 */
 		submit = NULL;
 		hint = 0;
 		TAILQ_FOREACH(crp, &crp_q, crp_next) {
-			hid = crypto_ses2hid(crp->crp_session);
-			cap = crypto_checkdriver(hid);
+			cap = crp->crp_session->cap;
 			/*
 			 * Driver cannot disappeared when there is an active
 			 * session.
 			 */
 			KASSERT(cap != NULL, ("%s:%u Driver disappeared.",
 			    __func__, __LINE__));
-			if (cap == NULL || cap->cc_dev == NULL) {
+			if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) {
 				/* Op needs to be migrated, process it. */
 				if (submit == NULL)
 					submit = crp;
 				break;
 			}
 			if (!cap->cc_qblocked) {
 				if (submit != NULL) {
 					/*
 					 * We stop on finding another op,
 					 * regardless whether its for the same
 					 * driver or not.  We could keep
 					 * searching the queue but it might be
 					 * better to just use a per-driver
 					 * queue instead.
 					 */
-					if (crypto_ses2hid(submit->crp_session) == hid)
+					if (submit->crp_session->cap == cap)
 						hint = CRYPTO_HINT_MORE;
 					break;
 				} else {
 					submit = crp;
 					if ((submit->crp_flags & CRYPTO_F_BATCH) == 0)
 						break;
 					/* keep scanning for more are q'd */
 				}
 			}
 		}
 		if (submit != NULL) {
 			TAILQ_REMOVE(&crp_q, submit, crp_next);
-			hid = crypto_ses2hid(submit->crp_session);
-			cap = crypto_checkdriver(hid);
+			cap = submit->crp_session->cap;
 			KASSERT(cap != NULL, ("%s:%u Driver disappeared.",
 			    __func__, __LINE__));
+			CRYPTO_Q_UNLOCK();
 			result = crypto_invoke(cap, submit, hint);
+			CRYPTO_Q_LOCK();
 			if (result == ERESTART) {
 				/*
 				 * The driver ran out of resources, mark the
 				 * driver ``blocked'' for cryptop's and put
 				 * the request back in the queue.  It would
 				 * best to put the request back where we got
 				 * it but that's hard so for now we put it
 				 * at the front.  This should be ok; putting
 				 * it at the end does not work.
 				 */
-				/* XXX validate sid again? */
-				crypto_drivers[crypto_ses2hid(submit->crp_session)].cc_qblocked = 1;
+				cap->cc_qblocked = 1;
 				TAILQ_INSERT_HEAD(&crp_q, submit, crp_next);
 				cryptostats.cs_blocks++;
 			}
 		}
 
 		/* As above, but for key ops */
 		TAILQ_FOREACH(krp, &crp_kq, krp_next) {
-			cap = crypto_checkdriver(krp->krp_hid);
-			if (cap == NULL || cap->cc_dev == NULL) {
+			cap = krp->krp_cap;
+			if (cap->cc_flags & CRYPTOCAP_F_CLEANUP) {
 				/*
-				 * Operation needs to be migrated, invalidate
-				 * the assigned device so it will reselect a
-				 * new one below.  Propagate the original
-				 * crid selection flags if supplied.
+				 * Operation needs to be migrated,
+				 * clear krp_cap so a new driver is
+				 * selected.
 				 */
-				krp->krp_hid = krp->krp_crid &
-				    (CRYPTOCAP_F_SOFTWARE|CRYPTOCAP_F_HARDWARE);
-				if (krp->krp_hid == 0)
-					krp->krp_hid =
-				    CRYPTOCAP_F_SOFTWARE|CRYPTOCAP_F_HARDWARE;
+				krp->krp_cap = NULL;
+				cap_rele(cap);
 				break;
 			}
 			if (!cap->cc_kqblocked)
 				break;
 		}
 		if (krp != NULL) {
 			TAILQ_REMOVE(&crp_kq, krp, krp_next);
-			result = crypto_kinvoke(krp, krp->krp_hid);
+			CRYPTO_Q_UNLOCK();
+			result = crypto_kinvoke(krp);
+			CRYPTO_Q_LOCK();
 			if (result == ERESTART) {
 				/*
 				 * The driver ran out of resources, mark the
 				 * driver ``blocked'' for cryptkop's and put
 				 * the request back in the queue.  It would
 				 * best to put the request back where we got
 				 * it but that's hard so for now we put it
 				 * at the front.  This should be ok; putting
 				 * it at the end does not work.
 				 */
-				/* XXX validate sid again? */
-				crypto_drivers[krp->krp_hid].cc_kqblocked = 1;
+				krp->krp_cap->cc_kqblocked = 1;
 				TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next);
 				cryptostats.cs_kblocks++;
 			}
 		}
 
 		if (submit == NULL && krp == NULL) {
 			/*
 			 * Nothing more to be processed.  Sleep until we're
 			 * woken because there are more ops to process.
 			 * This happens either by submission or by a driver
 			 * becoming unblocked and notifying us through
 			 * crypto_unblock.  Note that when we wakeup we
 			 * start processing each queue again from the
 			 * front. It's not clear that it's important to
 			 * preserve this ordering since ops may finish
 			 * out of order if dispatched to different devices
 			 * and some become blocked while others do not.
 			 */
 			crp_sleep = 1;
 			msleep(&crp_q, &crypto_q_mtx, PWAIT, "crypto_wait", 0);
 			crp_sleep = 0;
 			if (cryptoproc == NULL)
 				break;
 			cryptostats.cs_intrs++;
 		}
 	}
 	CRYPTO_Q_UNLOCK();
 
 	crypto_finis(&crp_q);
 }
 
 /*
  * Crypto returns thread, does callbacks for processed crypto requests.
  * Callbacks are done here, rather than in the crypto drivers, because
  * callbacks typically are expensive and would slow interrupt handling.
  */
 static void
 crypto_ret_proc(struct crypto_ret_worker *ret_worker)
 {
 	struct cryptop *crpt;
 	struct cryptkop *krpt;
 
 	CRYPTO_RETW_LOCK(ret_worker);
 	for (;;) {
 		/* Harvest return q's for completed ops */
 		crpt = TAILQ_FIRST(&ret_worker->crp_ordered_ret_q);
 		if (crpt != NULL) {
 			if (crpt->crp_seq == ret_worker->reorder_cur_seq) {
 				TAILQ_REMOVE(&ret_worker->crp_ordered_ret_q, crpt, crp_next);
 				ret_worker->reorder_cur_seq++;
 			} else {
 				crpt = NULL;
 			}
 		}
 
 		if (crpt == NULL) {
 			crpt = TAILQ_FIRST(&ret_worker->crp_ret_q);
 			if (crpt != NULL)
 				TAILQ_REMOVE(&ret_worker->crp_ret_q, crpt, crp_next);
 		}
 
 		krpt = TAILQ_FIRST(&ret_worker->crp_ret_kq);
 		if (krpt != NULL)
 			TAILQ_REMOVE(&ret_worker->crp_ret_kq, krpt, krp_next);
 
 		if (crpt != NULL || krpt != NULL) {
 			CRYPTO_RETW_UNLOCK(ret_worker);
 			/*
 			 * Run callbacks unlocked.
 			 */
 			if (crpt != NULL) {
 #ifdef CRYPTO_TIMING
 				if (crypto_timing) {
 					/*
 					 * NB: We must copy the timestamp before
 					 * doing the callback as the cryptop is
 					 * likely to be reclaimed.
 					 */
 					struct bintime t = crpt->crp_tstamp;
 					crypto_tstat(&cryptostats.cs_cb, &t);
 					crpt->crp_callback(crpt);
 					crypto_tstat(&cryptostats.cs_finis, &t);
 				} else
 #endif
 					crpt->crp_callback(crpt);
 			}
 			if (krpt != NULL)
 				krpt->krp_callback(krpt);
 			CRYPTO_RETW_LOCK(ret_worker);
 		} else {
 			/*
 			 * Nothing more to be processed.  Sleep until we're
 			 * woken because there are more returns to process.
 			 */
 			msleep(&ret_worker->crp_ret_q, &ret_worker->crypto_ret_mtx, PWAIT,
 				"crypto_ret_wait", 0);
 			if (ret_worker->cryptoretproc == NULL)
 				break;
 			cryptostats.cs_rets++;
 		}
 	}
 	CRYPTO_RETW_UNLOCK(ret_worker);
 
 	crypto_finis(&ret_worker->crp_ret_q);
 }
 
 #ifdef DDB
 static void
 db_show_drivers(void)
 {
 	int hid;
 
 	db_printf("%12s %4s %4s %8s %2s %2s\n"
 		, "Device"
 		, "Ses"
 		, "Kops"
 		, "Flags"
 		, "QB"
 		, "KB"
 	);
-	for (hid = 0; hid < crypto_drivers_num; hid++) {
-		const struct cryptocap *cap = &crypto_drivers[hid];
-		if (cap->cc_dev == NULL)
+	for (hid = 0; hid < crypto_drivers_size; hid++) {
+		const struct cryptocap *cap = crypto_drivers[hid];
+		if (cap == NULL)
 			continue;
 		db_printf("%-12s %4u %4u %08x %2u %2u\n"
 		    , device_get_nameunit(cap->cc_dev)
 		    , cap->cc_sessions
 		    , cap->cc_koperations
 		    , cap->cc_flags
 		    , cap->cc_qblocked
 		    , cap->cc_kqblocked
 		);
 	}
 }
 
 DB_SHOW_COMMAND(crypto, db_show_crypto)
 {
 	struct cryptop *crp;
 	struct crypto_ret_worker *ret_worker;
 
 	db_show_drivers();
 	db_printf("\n");
 
 	db_printf("%4s %8s %4s %4s %4s %4s %8s %8s\n",
 	    "HID", "Caps", "Ilen", "Olen", "Etype", "Flags",
-	    "Desc", "Callback");
+	    "Device", "Callback");
 	TAILQ_FOREACH(crp, &crp_q, crp_next) {
 		db_printf("%4u %08x %4u %4u %4u %04x %8p %8p\n"
-		    , (int) crypto_ses2hid(crp->crp_session)
+		    , crp->crp_session->cap->cc_hid
 		    , (int) crypto_ses2caps(crp->crp_session)
 		    , crp->crp_ilen, crp->crp_olen
 		    , crp->crp_etype
 		    , crp->crp_flags
-		    , crp->crp_desc
+		    , device_get_nameunit(crp->crp_session->cap->cc_dev)
 		    , crp->crp_callback
 		);
 	}
 	FOREACH_CRYPTO_RETW(ret_worker) {
 		db_printf("\n%8s %4s %4s %4s %8s\n",
 		    "ret_worker", "HID", "Etype", "Flags", "Callback");
 		if (!TAILQ_EMPTY(&ret_worker->crp_ret_q)) {
 			TAILQ_FOREACH(crp, &ret_worker->crp_ret_q, crp_next) {
 				db_printf("%8td %4u %4u %04x %8p\n"
 				    , CRYPTO_RETW_ID(ret_worker)
-				    , (int) crypto_ses2hid(crp->crp_session)
+				    , crp->crp_session->cap->cc_hid
 				    , crp->crp_etype
 				    , crp->crp_flags
 				    , crp->crp_callback
 				);
 			}
 		}
 	}
 }
 
 DB_SHOW_COMMAND(kcrypto, db_show_kcrypto)
 {
 	struct cryptkop *krp;
 	struct crypto_ret_worker *ret_worker;
 
 	db_show_drivers();
 	db_printf("\n");
 
 	db_printf("%4s %5s %4s %4s %8s %4s %8s\n",
 	    "Op", "Status", "#IP", "#OP", "CRID", "HID", "Callback");
 	TAILQ_FOREACH(krp, &crp_kq, krp_next) {
 		db_printf("%4u %5u %4u %4u %08x %4u %8p\n"
 		    , krp->krp_op
 		    , krp->krp_status
 		    , krp->krp_iparams, krp->krp_oparams
 		    , krp->krp_crid, krp->krp_hid
 		    , krp->krp_callback
 		);
 	}
 
 	ret_worker = CRYPTO_RETW(0);
 	if (!TAILQ_EMPTY(&ret_worker->crp_ret_q)) {
 		db_printf("%4s %5s %8s %4s %8s\n",
 		    "Op", "Status", "CRID", "HID", "Callback");
 		TAILQ_FOREACH(krp, &ret_worker->crp_ret_kq, krp_next) {
 			db_printf("%4u %5u %08x %4u %8p\n"
 			    , krp->krp_op
 			    , krp->krp_status
 			    , krp->krp_crid, krp->krp_hid
 			    , krp->krp_callback
 			);
 		}
 	}
 }
 #endif
 
 int crypto_modevent(module_t mod, int type, void *unused);
 
 /*
  * Initialization code, both for static and dynamic loading.
  * Note this is not invoked with the usual MODULE_DECLARE
  * mechanism but instead is listed as a dependency by the
  * cryptosoft driver.  This guarantees proper ordering of
  * calls on module load/unload.
  */
 int
 crypto_modevent(module_t mod, int type, void *unused)
 {
 	int error = EINVAL;
 
 	switch (type) {
 	case MOD_LOAD:
 		error = crypto_init();
 		if (error == 0 && bootverbose)
 			printf("crypto: <crypto core>\n");
 		break;
 	case MOD_UNLOAD:
 		/*XXX disallow if active sessions */
 		error = 0;
 		crypto_destroy();
 		return 0;
 	}
 	return error;
 }
 MODULE_VERSION(crypto, 1);
 MODULE_DEPEND(crypto, zlib, 1, 1, 1);
Index: head/sys/opencrypto/cryptodev.c
===================================================================
--- head/sys/opencrypto/cryptodev.c	(revision 359373)
+++ head/sys/opencrypto/cryptodev.c	(revision 359374)
@@ -1,1550 +1,1657 @@
 /*	$OpenBSD: cryptodev.c,v 1.52 2002/06/19 07:22:46 deraadt Exp $	*/
 
 /*-
  * Copyright (c) 2001 Theo de Raadt
  * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting
  * Copyright (c) 2014 The FreeBSD Foundation
  * All rights reserved.
  *
  * Portions of this software were developed by John-Mark Gurney
  * under sponsorship of the FreeBSD Foundation and
  * Rubicon Communications, LLC (Netgate).
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  *
  * 1. Redistributions of source code must retain the above copyright
  *   notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *   notice, this list of conditions and the following disclaimer in the
  *   documentation and/or other materials provided with the distribution.
  * 3. The name of the author may not be used to endorse or promote products
  *   derived from this software without specific prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  * Effort sponsored in part by the Defense Advanced Research Projects
  * Agency (DARPA) and Air Force Research Laboratory, Air Force
  * Materiel Command, USAF, under agreement number F30602-01-2-0537.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/malloc.h>
 #include <sys/mbuf.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <sys/sysctl.h>
 #include <sys/file.h>
 #include <sys/filedesc.h>
 #include <sys/errno.h>
-#include <sys/uio.h>
 #include <sys/random.h>
 #include <sys/conf.h>
 #include <sys/kernel.h>
 #include <sys/module.h>
 #include <sys/fcntl.h>
 #include <sys/bus.h>
 #include <sys/user.h>
 #include <sys/sdt.h>
 
 #include <opencrypto/cryptodev.h>
 #include <opencrypto/xform.h>
 
 SDT_PROVIDER_DECLARE(opencrypto);
 
 SDT_PROBE_DEFINE1(opencrypto, dev, ioctl, error, "int"/*line number*/);
 
 #ifdef COMPAT_FREEBSD32
 #include <sys/mount.h>
 #include <compat/freebsd32/freebsd32.h>
 
 struct session_op32 {
 	u_int32_t	cipher;
 	u_int32_t	mac;
 	u_int32_t	keylen;
 	u_int32_t	key;
 	int		mackeylen;
 	u_int32_t	mackey;
 	u_int32_t	ses;
 };
 
 struct session2_op32 {
 	u_int32_t	cipher;
 	u_int32_t	mac;
 	u_int32_t	keylen;
 	u_int32_t	key;
 	int		mackeylen;
 	u_int32_t	mackey;
 	u_int32_t	ses;
 	int		crid;
 	int		pad[4];
 };
 
 struct crypt_op32 {
 	u_int32_t	ses;
 	u_int16_t	op;
 	u_int16_t	flags;
 	u_int		len;
 	u_int32_t	src, dst;
 	u_int32_t	mac;
 	u_int32_t	iv;
 };
 
 struct crparam32 {
 	u_int32_t	crp_p;
 	u_int		crp_nbits;
 };
 
 struct crypt_kop32 {
 	u_int		crk_op;
 	u_int		crk_status;
 	u_short		crk_iparams;
 	u_short		crk_oparams;
 	u_int		crk_crid;
 	struct crparam32	crk_param[CRK_MAXPARAM];
 };
 
 struct cryptotstat32 {
 	struct timespec32	acc;
 	struct timespec32	min;
 	struct timespec32	max;
 	u_int32_t	count;
 };
 
 struct cryptostats32 {
 	u_int32_t	cs_ops;
 	u_int32_t	cs_errs;
 	u_int32_t	cs_kops;
 	u_int32_t	cs_kerrs;
 	u_int32_t	cs_intrs;
 	u_int32_t	cs_rets;
 	u_int32_t	cs_blocks;
 	u_int32_t	cs_kblocks;
 	struct cryptotstat32 cs_invoke;
 	struct cryptotstat32 cs_done;
 	struct cryptotstat32 cs_cb;
 	struct cryptotstat32 cs_finis;
 };
 
 #define	CIOCGSESSION32	_IOWR('c', 101, struct session_op32)
 #define	CIOCCRYPT32	_IOWR('c', 103, struct crypt_op32)
 #define	CIOCKEY32	_IOWR('c', 104, struct crypt_kop32)
 #define	CIOCGSESSION232	_IOWR('c', 106, struct session2_op32)
 #define	CIOCKEY232	_IOWR('c', 107, struct crypt_kop32)
 
 static void
 session_op_from_32(const struct session_op32 *from, struct session_op *to)
 {
 
 	CP(*from, *to, cipher);
 	CP(*from, *to, mac);
 	CP(*from, *to, keylen);
 	PTRIN_CP(*from, *to, key);
 	CP(*from, *to, mackeylen);
 	PTRIN_CP(*from, *to, mackey);
 	CP(*from, *to, ses);
 }
 
 static void
 session2_op_from_32(const struct session2_op32 *from, struct session2_op *to)
 {
 
 	session_op_from_32((const struct session_op32 *)from,
 	    (struct session_op *)to);
 	CP(*from, *to, crid);
 }
 
 static void
 session_op_to_32(const struct session_op *from, struct session_op32 *to)
 {
 
 	CP(*from, *to, cipher);
 	CP(*from, *to, mac);
 	CP(*from, *to, keylen);
 	PTROUT_CP(*from, *to, key);
 	CP(*from, *to, mackeylen);
 	PTROUT_CP(*from, *to, mackey);
 	CP(*from, *to, ses);
 }
 
 static void
 session2_op_to_32(const struct session2_op *from, struct session2_op32 *to)
 {
 
 	session_op_to_32((const struct session_op *)from,
 	    (struct session_op32 *)to);
 	CP(*from, *to, crid);
 }
 
 static void
 crypt_op_from_32(const struct crypt_op32 *from, struct crypt_op *to)
 {
 
 	CP(*from, *to, ses);
 	CP(*from, *to, op);
 	CP(*from, *to, flags);
 	CP(*from, *to, len);
 	PTRIN_CP(*from, *to, src);
 	PTRIN_CP(*from, *to, dst);
 	PTRIN_CP(*from, *to, mac);
 	PTRIN_CP(*from, *to, iv);
 }
 
 static void
 crypt_op_to_32(const struct crypt_op *from, struct crypt_op32 *to)
 {
 
 	CP(*from, *to, ses);
 	CP(*from, *to, op);
 	CP(*from, *to, flags);
 	CP(*from, *to, len);
 	PTROUT_CP(*from, *to, src);
 	PTROUT_CP(*from, *to, dst);
 	PTROUT_CP(*from, *to, mac);
 	PTROUT_CP(*from, *to, iv);
 }
 
 static void
 crparam_from_32(const struct crparam32 *from, struct crparam *to)
 {
 
 	PTRIN_CP(*from, *to, crp_p);
 	CP(*from, *to, crp_nbits);
 }
 
 static void
 crparam_to_32(const struct crparam *from, struct crparam32 *to)
 {
 
 	PTROUT_CP(*from, *to, crp_p);
 	CP(*from, *to, crp_nbits);
 }
 
 static void
 crypt_kop_from_32(const struct crypt_kop32 *from, struct crypt_kop *to)
 {
 	int i;
 
 	CP(*from, *to, crk_op);
 	CP(*from, *to, crk_status);
 	CP(*from, *to, crk_iparams);
 	CP(*from, *to, crk_oparams);
 	CP(*from, *to, crk_crid);
 	for (i = 0; i < CRK_MAXPARAM; i++)
 		crparam_from_32(&from->crk_param[i], &to->crk_param[i]);
 }
 
 static void
 crypt_kop_to_32(const struct crypt_kop *from, struct crypt_kop32 *to)
 {
 	int i;
 
 	CP(*from, *to, crk_op);
 	CP(*from, *to, crk_status);
 	CP(*from, *to, crk_iparams);
 	CP(*from, *to, crk_oparams);
 	CP(*from, *to, crk_crid);
 	for (i = 0; i < CRK_MAXPARAM; i++)
 		crparam_to_32(&from->crk_param[i], &to->crk_param[i]);
 }
 #endif
 
 struct csession {
 	TAILQ_ENTRY(csession) next;
 	crypto_session_t cses;
 	volatile u_int	refs;
 	u_int32_t	ses;
 	struct mtx	lock;		/* for op submission */
 
-	u_int32_t	cipher;
 	struct enc_xform *txform;
-	u_int32_t	mac;
-	struct auth_hash *thash;
+	int		hashsize;
+	int		ivsize;
+	int		mode;
 
-	caddr_t		key;
-	int		keylen;
-
-	caddr_t		mackey;
-	int		mackeylen;
+	void		*key;
+	void		*mackey;
 };
 
 struct cryptop_data {
 	struct csession *cse;
 
-	struct iovec	iovec[1];
-	struct uio	uio;
+	char		*buf;
 	bool		done;
 };
 
 struct fcrypt {
 	TAILQ_HEAD(csessionlist, csession) csessions;
 	int		sesn;
 	struct mtx	lock;
 };
 
 static struct timeval warninterval = { .tv_sec = 60, .tv_usec = 0 };
 SYSCTL_TIMEVAL_SEC(_kern, OID_AUTO, cryptodev_warn_interval, CTLFLAG_RW,
     &warninterval,
     "Delay in seconds between warnings of deprecated /dev/crypto algorithms");
 
 static	int cryptof_ioctl(struct file *, u_long, void *,
 		    struct ucred *, struct thread *);
 static	int cryptof_stat(struct file *, struct stat *,
 		    struct ucred *, struct thread *);
 static	int cryptof_close(struct file *, struct thread *);
 static	int cryptof_fill_kinfo(struct file *, struct kinfo_file *,
 		    struct filedesc *);
 
 static struct fileops cryptofops = {
     .fo_read = invfo_rdwr,
     .fo_write = invfo_rdwr,
     .fo_truncate = invfo_truncate,
     .fo_ioctl = cryptof_ioctl,
     .fo_poll = invfo_poll,
     .fo_kqfilter = invfo_kqfilter,
     .fo_stat = cryptof_stat,
     .fo_close = cryptof_close,
     .fo_chmod = invfo_chmod,
     .fo_chown = invfo_chown,
     .fo_sendfile = invfo_sendfile,
     .fo_fill_kinfo = cryptof_fill_kinfo,
 };
 
 static struct csession *csefind(struct fcrypt *, u_int);
 static bool csedelete(struct fcrypt *, u_int);
-static struct csession *csecreate(struct fcrypt *, crypto_session_t, caddr_t,
-    u_int64_t, caddr_t, u_int64_t, u_int32_t, u_int32_t, struct enc_xform *,
-    struct auth_hash *);
+static struct csession *csecreate(struct fcrypt *, crypto_session_t,
+    struct crypto_session_params *, struct enc_xform *, void *,
+    struct auth_hash *, void *);
 static void csefree(struct csession *);
 
 static	int cryptodev_op(struct csession *, struct crypt_op *,
 			struct ucred *, struct thread *td);
 static	int cryptodev_aead(struct csession *, struct crypt_aead *,
 			struct ucred *, struct thread *);
 static	int cryptodev_key(struct crypt_kop *);
 static	int cryptodev_find(struct crypt_find_op *);
 
 /*
  * Check a crypto identifier to see if it requested
  * a software device/driver.  This can be done either
  * by device name/class or through search constraints.
  */
 static int
 checkforsoftware(int *cridp)
 {
 	int crid;
 
 	crid = *cridp;
 
 	if (!crypto_devallowsoft) {
 		if (crid & CRYPTOCAP_F_SOFTWARE) {
 			if (crid & CRYPTOCAP_F_HARDWARE) {
 				*cridp = CRYPTOCAP_F_HARDWARE;
 				return 0;
 			}
 			return EINVAL;
 		}
 		if ((crid & CRYPTOCAP_F_HARDWARE) == 0 &&
 		    (crypto_getcaps(crid) & CRYPTOCAP_F_HARDWARE) == 0)
 			return EINVAL;
 	}
 	return 0;
 }
 
 /* ARGSUSED */
 static int
 cryptof_ioctl(
 	struct file *fp,
 	u_long cmd,
 	void *data,
 	struct ucred *active_cred,
 	struct thread *td)
 {
 #define	SES2(p)	((struct session2_op *)p)
-	struct cryptoini cria, crie;
+	struct crypto_session_params csp;
 	struct fcrypt *fcr = fp->f_data;
 	struct csession *cse;
 	struct session_op *sop;
 	struct crypt_op *cop;
 	struct crypt_aead *caead;
 	struct enc_xform *txform = NULL;
 	struct auth_hash *thash = NULL;
+	void *key = NULL;
+	void *mackey = NULL;
 	struct crypt_kop *kop;
 	crypto_session_t cses;
 	u_int32_t ses;
 	int error = 0, crid;
 #ifdef COMPAT_FREEBSD32
 	struct session2_op sopc;
 	struct crypt_op copc;
 	struct crypt_kop kopc;
 #endif
 
 	switch (cmd) {
 	case CIOCGSESSION:
 	case CIOCGSESSION2:
 #ifdef COMPAT_FREEBSD32
 	case CIOCGSESSION32:
 	case CIOCGSESSION232:
 		if (cmd == CIOCGSESSION32) {
 			session_op_from_32(data, (struct session_op *)&sopc);
 			sop = (struct session_op *)&sopc;
 		} else if (cmd == CIOCGSESSION232) {
 			session2_op_from_32(data, &sopc);
 			sop = (struct session_op *)&sopc;
 		} else
 #endif
 			sop = (struct session_op *)data;
 		switch (sop->cipher) {
 		case 0:
 			break;
 		case CRYPTO_DES_CBC:
 			txform = &enc_xform_des;
 			break;
 		case CRYPTO_3DES_CBC:
 			txform = &enc_xform_3des;
 			break;
 		case CRYPTO_BLF_CBC:
 			txform = &enc_xform_blf;
 			break;
 		case CRYPTO_CAST_CBC:
 			txform = &enc_xform_cast5;
 			break;
 		case CRYPTO_SKIPJACK_CBC:
 			txform = &enc_xform_skipjack;
 			break;
 		case CRYPTO_AES_CBC:
 			txform = &enc_xform_rijndael128;
 			break;
 		case CRYPTO_AES_XTS:
 			txform = &enc_xform_aes_xts;
 			break;
 		case CRYPTO_NULL_CBC:
 			txform = &enc_xform_null;
 			break;
 		case CRYPTO_ARC4:
 			txform = &enc_xform_arc4;
 			break;
  		case CRYPTO_CAMELLIA_CBC:
  			txform = &enc_xform_camellia;
  			break;
 		case CRYPTO_AES_ICM:
 			txform = &enc_xform_aes_icm;
  			break;
 		case CRYPTO_AES_NIST_GCM_16:
 			txform = &enc_xform_aes_nist_gcm;
  			break;
 		case CRYPTO_CHACHA20:
 			txform = &enc_xform_chacha20;
 			break;
 		case CRYPTO_AES_CCM_16:
 			txform = &enc_xform_ccm;
 			break;
 
 		default:
 			CRYPTDEB("invalid cipher");
 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 			return (EINVAL);
 		}
 
 		switch (sop->mac) {
 		case 0:
 			break;
 		case CRYPTO_MD5_HMAC:
 			thash = &auth_hash_hmac_md5;
 			break;
 		case CRYPTO_POLY1305:
 			thash = &auth_hash_poly1305;
 			break;
 		case CRYPTO_SHA1_HMAC:
 			thash = &auth_hash_hmac_sha1;
 			break;
 		case CRYPTO_SHA2_224_HMAC:
 			thash = &auth_hash_hmac_sha2_224;
 			break;
 		case CRYPTO_SHA2_256_HMAC:
 			thash = &auth_hash_hmac_sha2_256;
 			break;
 		case CRYPTO_SHA2_384_HMAC:
 			thash = &auth_hash_hmac_sha2_384;
 			break;
 		case CRYPTO_SHA2_512_HMAC:
 			thash = &auth_hash_hmac_sha2_512;
 			break;
 		case CRYPTO_RIPEMD160_HMAC:
 			thash = &auth_hash_hmac_ripemd_160;
 			break;
+#ifdef COMPAT_FREEBSD12
 		case CRYPTO_AES_128_NIST_GMAC:
-			thash = &auth_hash_nist_gmac_aes_128;
-			break;
 		case CRYPTO_AES_192_NIST_GMAC:
-			thash = &auth_hash_nist_gmac_aes_192;
-			break;
 		case CRYPTO_AES_256_NIST_GMAC:
-			thash = &auth_hash_nist_gmac_aes_256;
+			/* Should always be paired with GCM. */
+			if (sop->cipher != CRYPTO_AES_NIST_GCM_16) {
+				CRYPTDEB("GMAC without GCM");
+				return (EINVAL);
+			}
 			break;
-
+#endif
+		case CRYPTO_AES_NIST_GMAC:
+			switch (sop->mackeylen * 8) {
+			case 128:
+				thash = &auth_hash_nist_gmac_aes_128;
+				break;
+			case 192:
+				thash = &auth_hash_nist_gmac_aes_192;
+				break;
+			case 256:
+				thash = &auth_hash_nist_gmac_aes_256;
+				break;
+			default:
+				CRYPTDEB("invalid GMAC key length");
+				SDT_PROBE1(opencrypto, dev, ioctl, error,
+				    __LINE__);
+				return (EINVAL);
+			}
+			break;
 		case CRYPTO_AES_CCM_CBC_MAC:
-			switch (sop->keylen) {
+			switch (sop->mackeylen) {
 			case 16:
 				thash = &auth_hash_ccm_cbc_mac_128;
 				break;
 			case 24:
 				thash = &auth_hash_ccm_cbc_mac_192;
 				break;
 			case 32:
 				thash = &auth_hash_ccm_cbc_mac_256;
 				break;
 			default:
 				CRYPTDEB("Invalid CBC MAC key size %d",
 				    sop->keylen);
 				SDT_PROBE1(opencrypto, dev, ioctl,
 				    error, __LINE__);
 				return (EINVAL);
 			}
 			break;
 #ifdef notdef
 		case CRYPTO_MD5:
 			thash = &auth_hash_md5;
 			break;
 #endif
 		case CRYPTO_SHA1:
 			thash = &auth_hash_sha1;
 			break;
 		case CRYPTO_SHA2_224:
 			thash = &auth_hash_sha2_224;
 			break;
 		case CRYPTO_SHA2_256:
 			thash = &auth_hash_sha2_256;
 			break;
 		case CRYPTO_SHA2_384:
 			thash = &auth_hash_sha2_384;
 			break;
 		case CRYPTO_SHA2_512:
 			thash = &auth_hash_sha2_512;
 			break;
 
 		case CRYPTO_NULL_HMAC:
 			thash = &auth_hash_null;
 			break;
 
 		case CRYPTO_BLAKE2B:
 			thash = &auth_hash_blake2b;
 			break;
 		case CRYPTO_BLAKE2S:
 			thash = &auth_hash_blake2s;
 			break;
 
 		default:
 			CRYPTDEB("invalid mac");
 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 			return (EINVAL);
 		}
 
-		bzero(&crie, sizeof(crie));
-		bzero(&cria, sizeof(cria));
+		if (txform == NULL && thash == NULL)
+			return (EINVAL);
 
+		memset(&csp, 0, sizeof(csp));
+
+		if (sop->cipher == CRYPTO_AES_NIST_GCM_16) {
+			switch (sop->mac) {
+#ifdef COMPAT_FREEBSD12
+			case CRYPTO_AES_128_NIST_GMAC:
+			case CRYPTO_AES_192_NIST_GMAC:
+			case CRYPTO_AES_256_NIST_GMAC:
+				if (sop->keylen != sop->mackeylen)
+					return (EINVAL);
+				break;
+#endif
+			case 0:
+				break;
+			default:
+				return (EINVAL);
+			}
+			csp.csp_mode = CSP_MODE_AEAD;
+		} else if (sop->cipher == CRYPTO_AES_CCM_16) {
+			switch (sop->mac) {
+#ifdef COMPAT_FREEBSD12
+			case CRYPTO_AES_CCM_CBC_MAC:
+				if (sop->keylen != sop->mackeylen)
+					return (EINVAL);
+				thash = NULL;
+				break;
+#endif
+			case 0:
+				break;
+			default:
+				return (EINVAL);
+			}
+			csp.csp_mode = CSP_MODE_AEAD;
+		} else if (txform && thash)
+			csp.csp_mode = CSP_MODE_ETA;
+		else if (txform)
+			csp.csp_mode = CSP_MODE_CIPHER;
+		else
+			csp.csp_mode = CSP_MODE_DIGEST;
+
 		if (txform) {
-			crie.cri_alg = txform->type;
-			crie.cri_klen = sop->keylen * 8;
+			csp.csp_cipher_alg = txform->type;
+			csp.csp_cipher_klen = sop->keylen;
 			if (sop->keylen > txform->maxkey ||
 			    sop->keylen < txform->minkey) {
 				CRYPTDEB("invalid cipher parameters");
 				error = EINVAL;
 				SDT_PROBE1(opencrypto, dev, ioctl, error,
 				    __LINE__);
 				goto bail;
 			}
 
-			crie.cri_key = malloc(crie.cri_klen / 8,
-			    M_XDATA, M_WAITOK);
-			if ((error = copyin(sop->key, crie.cri_key,
-			    crie.cri_klen / 8))) {
+			key = malloc(csp.csp_cipher_klen, M_XDATA, M_WAITOK);
+			error = copyin(sop->key, key, csp.csp_cipher_klen);
+			if (error) {
 				CRYPTDEB("invalid key");
 				SDT_PROBE1(opencrypto, dev, ioctl, error,
 				    __LINE__);
 				goto bail;
 			}
-			if (thash)
-				crie.cri_next = &cria;
+			csp.csp_cipher_key = key;
+			csp.csp_ivlen = txform->ivsize;
 		}
 
 		if (thash) {
-			cria.cri_alg = thash->type;
-			cria.cri_klen = sop->mackeylen * 8;
+			csp.csp_auth_alg = thash->type;
+			csp.csp_auth_klen = sop->mackeylen;
 			if (sop->mackeylen > thash->keysize ||
 			    sop->mackeylen < 0) {
 				CRYPTDEB("invalid mac key length");
 				error = EINVAL;
 				SDT_PROBE1(opencrypto, dev, ioctl, error,
 				    __LINE__);
 				goto bail;
 			}
 
-			if (cria.cri_klen) {
-				cria.cri_key = malloc(cria.cri_klen / 8,
-				    M_XDATA, M_WAITOK);
-				if ((error = copyin(sop->mackey, cria.cri_key,
-				    cria.cri_klen / 8))) {
+			if (csp.csp_auth_klen) {
+				mackey = malloc(csp.csp_auth_klen, M_XDATA,
+				    M_WAITOK);
+				error = copyin(sop->mackey, mackey,
+				    csp.csp_auth_klen);
+				if (error) {
 					CRYPTDEB("invalid mac key");
 					SDT_PROBE1(opencrypto, dev, ioctl,
 					    error, __LINE__);
 					goto bail;
 				}
+				csp.csp_auth_key = mackey;
 			}
+
+			if (csp.csp_auth_alg == CRYPTO_AES_NIST_GMAC)
+				csp.csp_ivlen = AES_GCM_IV_LEN;
+			if (csp.csp_auth_alg == CRYPTO_AES_CCM_CBC_MAC)
+				csp.csp_ivlen = AES_CCM_IV_LEN;
 		}
 
 		/* NB: CIOCGSESSION2 has the crid */
 		if (cmd == CIOCGSESSION2
 #ifdef COMPAT_FREEBSD32
 		    || cmd == CIOCGSESSION232
 #endif
 			) {
 			crid = SES2(sop)->crid;
 			error = checkforsoftware(&crid);
 			if (error) {
 				CRYPTDEB("checkforsoftware");
 				SDT_PROBE1(opencrypto, dev, ioctl, error,
 				    __LINE__);
 				goto bail;
 			}
 		} else
 			crid = CRYPTOCAP_F_HARDWARE;
-		error = crypto_newsession(&cses, (txform ? &crie : &cria), crid);
+		error = crypto_newsession(&cses, &csp, crid);
 		if (error) {
 			CRYPTDEB("crypto_newsession");
 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 			goto bail;
 		}
 
-		cse = csecreate(fcr, cses, crie.cri_key, crie.cri_klen,
-		    cria.cri_key, cria.cri_klen, sop->cipher, sop->mac, txform,
-		    thash);
+		cse = csecreate(fcr, cses, &csp, txform, key, thash, mackey);
 
 		if (cse == NULL) {
 			crypto_freesession(cses);
 			error = EINVAL;
 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 			CRYPTDEB("csecreate");
 			goto bail;
 		}
 		sop->ses = cse->ses;
 		if (cmd == CIOCGSESSION2
 #ifdef COMPAT_FREEBSD32
 		    || cmd == CIOCGSESSION232
 #endif
 		    ) {
 			/* return hardware/driver id */
 			SES2(sop)->crid = crypto_ses2hid(cse->cses);
 		}
 bail:
 		if (error) {
-			if (crie.cri_key)
-				free(crie.cri_key, M_XDATA);
-			if (cria.cri_key)
-				free(cria.cri_key, M_XDATA);
+			free(key, M_XDATA);
+			free(mackey, M_XDATA);
 		}
 #ifdef COMPAT_FREEBSD32
 		else {
 			if (cmd == CIOCGSESSION32)
 				session_op_to_32(sop, data);
 			else if (cmd == CIOCGSESSION232)
 				session2_op_to_32((struct session2_op *)sop,
 				    data);
 		}
 #endif
 		break;
 	case CIOCFSESSION:
 		ses = *(u_int32_t *)data;
 		if (!csedelete(fcr, ses)) {
 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 			return (EINVAL);
 		}
 		break;
 	case CIOCCRYPT:
 #ifdef COMPAT_FREEBSD32
 	case CIOCCRYPT32:
 		if (cmd == CIOCCRYPT32) {
 			cop = &copc;
 			crypt_op_from_32(data, cop);
 		} else
 #endif
 			cop = (struct crypt_op *)data;
 		cse = csefind(fcr, cop->ses);
 		if (cse == NULL) {
 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 			return (EINVAL);
 		}
 		error = cryptodev_op(cse, cop, active_cred, td);
 		csefree(cse);
 #ifdef COMPAT_FREEBSD32
 		if (error == 0 && cmd == CIOCCRYPT32)
 			crypt_op_to_32(cop, data);
 #endif
 		break;
 	case CIOCKEY:
 	case CIOCKEY2:
 #ifdef COMPAT_FREEBSD32
 	case CIOCKEY32:
 	case CIOCKEY232:
 #endif
 		if (!crypto_userasymcrypto) {
 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 			return (EPERM);		/* XXX compat? */
 		}
 #ifdef COMPAT_FREEBSD32
 		if (cmd == CIOCKEY32 || cmd == CIOCKEY232) {
 			kop = &kopc;
 			crypt_kop_from_32(data, kop);
 		} else
 #endif
 			kop = (struct crypt_kop *)data;
 		if (cmd == CIOCKEY
 #ifdef COMPAT_FREEBSD32
 		    || cmd == CIOCKEY32
 #endif
 		    ) {
 			/* NB: crypto core enforces s/w driver use */
 			kop->crk_crid =
 			    CRYPTOCAP_F_HARDWARE | CRYPTOCAP_F_SOFTWARE;
 		}
 		mtx_lock(&Giant);
 		error = cryptodev_key(kop);
 		mtx_unlock(&Giant);
 #ifdef COMPAT_FREEBSD32
 		if (cmd == CIOCKEY32 || cmd == CIOCKEY232)
 			crypt_kop_to_32(kop, data);
 #endif
 		break;
 	case CIOCASYMFEAT:
 		if (!crypto_userasymcrypto) {
 			/*
 			 * NB: if user asym crypto operations are
 			 * not permitted return "no algorithms"
 			 * so well-behaved applications will just
 			 * fallback to doing them in software.
 			 */
 			*(int *)data = 0;
 		} else {
 			error = crypto_getfeat((int *)data);
 			if (error)
 				SDT_PROBE1(opencrypto, dev, ioctl, error,
 				    __LINE__);
 		}
 		break;
 	case CIOCFINDDEV:
 		error = cryptodev_find((struct crypt_find_op *)data);
 		break;
 	case CIOCCRYPTAEAD:
 		caead = (struct crypt_aead *)data;
 		cse = csefind(fcr, caead->ses);
 		if (cse == NULL) {
 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 			return (EINVAL);
 		}
 		error = cryptodev_aead(cse, caead, active_cred, td);
 		csefree(cse);
 		break;
 	default:
 		error = EINVAL;
 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		break;
 	}
 	return (error);
 #undef SES2
 }
 
 static int cryptodev_cb(struct cryptop *);
 
 static struct cryptop_data *
 cod_alloc(struct csession *cse, size_t len, struct thread *td)
 {
 	struct cryptop_data *cod;
-	struct uio *uio;
 
 	cod = malloc(sizeof(struct cryptop_data), M_XDATA, M_WAITOK | M_ZERO);
 
 	cod->cse = cse;
-	uio = &cod->uio;
-	uio->uio_iov = cod->iovec;
-	uio->uio_iovcnt = 1;
-	uio->uio_resid = len;
-	uio->uio_segflg = UIO_SYSSPACE;
-	uio->uio_rw = UIO_WRITE;
-	uio->uio_td = td;
-	uio->uio_iov[0].iov_len = len;
-	uio->uio_iov[0].iov_base = malloc(len, M_XDATA, M_WAITOK);
+	cod->buf = malloc(len, M_XDATA, M_WAITOK);
 	return (cod);
 }
 
 static void
 cod_free(struct cryptop_data *cod)
 {
 
-	free(cod->uio.uio_iov[0].iov_base, M_XDATA);
+	free(cod->buf, M_XDATA);
 	free(cod, M_XDATA);
 }
 
 static void
 cryptodev_warn(struct csession *cse)
 {
 	static struct timeval arc4warn, blfwarn, castwarn, deswarn, md5warn;
 	static struct timeval skipwarn, tdeswarn;
+	const struct crypto_session_params *csp;
 
-	switch (cse->cipher) {
+	csp = crypto_get_params(cse->cses);
+	switch (csp->csp_cipher_alg) {
 	case CRYPTO_DES_CBC:
 		if (ratecheck(&deswarn, &warninterval))
 			gone_in(13, "DES cipher via /dev/crypto");
 		break;
 	case CRYPTO_3DES_CBC:
 		if (ratecheck(&tdeswarn, &warninterval))
 			gone_in(13, "3DES cipher via /dev/crypto");
 		break;
 	case CRYPTO_BLF_CBC:
 		if (ratecheck(&blfwarn, &warninterval))
 			gone_in(13, "Blowfish cipher via /dev/crypto");
 		break;
 	case CRYPTO_CAST_CBC:
 		if (ratecheck(&castwarn, &warninterval))
 			gone_in(13, "CAST128 cipher via /dev/crypto");
 		break;
 	case CRYPTO_SKIPJACK_CBC:
 		if (ratecheck(&skipwarn, &warninterval))
 			gone_in(13, "Skipjack cipher via /dev/crypto");
 		break;
 	case CRYPTO_ARC4:
 		if (ratecheck(&arc4warn, &warninterval))
 			gone_in(13, "ARC4 cipher via /dev/crypto");
 		break;
 	}
 
-	switch (cse->mac) {
+	switch (csp->csp_auth_alg) {
 	case CRYPTO_MD5_HMAC:
 		if (ratecheck(&md5warn, &warninterval))
 			gone_in(13, "MD5-HMAC authenticator via /dev/crypto");
 		break;
 	}
 }
 
 static int
 cryptodev_op(
 	struct csession *cse,
 	struct crypt_op *cop,
 	struct ucred *active_cred,
 	struct thread *td)
 {
 	struct cryptop_data *cod = NULL;
 	struct cryptop *crp = NULL;
-	struct cryptodesc *crde = NULL, *crda = NULL;
 	int error;
 
 	if (cop->len > 256*1024-4) {
 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		return (E2BIG);
 	}
 
 	if (cse->txform) {
 		if (cop->len == 0 || (cop->len % cse->txform->blocksize) != 0) {
 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 			return (EINVAL);
 		}
 	}
 
-	if (cse->thash)
-		cod = cod_alloc(cse, cop->len + cse->thash->hashsize, td);
-	else
-		cod = cod_alloc(cse, cop->len, td);
-
-	crp = crypto_getreq((cse->txform != NULL) + (cse->thash != NULL));
-	if (crp == NULL) {
+	if (cop->mac && cse->hashsize == 0) {
 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
-		error = ENOMEM;
+		error = EINVAL;
 		goto bail;
 	}
 
-	if (cse->thash && cse->txform) {
-		if (cop->flags & COP_F_CIPHER_FIRST) {
-			crde = crp->crp_desc;
-			crda = crde->crd_next;
-		} else {
-			crda = crp->crp_desc;
-			crde = crda->crd_next;
+	/*
+	 * The COP_F_CIPHER_FIRST flag predates explicit session
+	 * modes, but the only way it was used was for EtA so allow it
+	 * as long as it is consistent with EtA.
+	 */
+	if (cop->flags & COP_F_CIPHER_FIRST) {
+		if (cop->op != COP_ENCRYPT) {
+			SDT_PROBE1(opencrypto, dev, ioctl, error,  __LINE__);
+			return (EINVAL);
 		}
-	} else if (cse->thash) {
-		crda = crp->crp_desc;
-	} else if (cse->txform) {
-		crde = crp->crp_desc;
-	} else {
+	}
+
+	cod = cod_alloc(cse, cop->len + cse->hashsize, td);
+
+	crp = crypto_getreq(cse->cses, M_WAITOK);
+
+	error = copyin(cop->src, cod->buf, cop->len);
+	if (error) {
 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
-		error = EINVAL;
 		goto bail;
 	}
+	crp->crp_payload_start = 0;
+	crp->crp_payload_length = cop->len;
+	if (cse->hashsize)
+		crp->crp_digest_start = cop->len;
 
-	if ((error = copyin(cop->src, cod->uio.uio_iov[0].iov_base,
-	    cop->len))) {
+	switch (cse->mode) {
+	case CSP_MODE_COMPRESS:
+		switch (cop->op) {
+		case COP_ENCRYPT:
+			crp->crp_op = CRYPTO_OP_COMPRESS;
+			break;
+		case COP_DECRYPT:
+			crp->crp_op = CRYPTO_OP_DECOMPRESS;
+			break;
+		default:
+			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
+			error = EINVAL;
+			goto bail;
+		}
+		break;
+	case CSP_MODE_CIPHER:
+		switch (cop->op) {
+		case COP_ENCRYPT:
+			crp->crp_op = CRYPTO_OP_ENCRYPT;
+			break;
+		case COP_DECRYPT:
+			crp->crp_op = CRYPTO_OP_DECRYPT;
+			break;
+		default:
+			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
+			error = EINVAL;
+			goto bail;
+		}
+		break;
+	case CSP_MODE_DIGEST:
+		switch (cop->op) {
+		case 0:
+		case COP_ENCRYPT:
+		case COP_DECRYPT:
+			crp->crp_op = CRYPTO_OP_COMPUTE_DIGEST;
+			break;
+		default:
+			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
+			error = EINVAL;
+			goto bail;
+		}
+		break;
+	case CSP_MODE_ETA:
+		switch (cop->op) {
+		case COP_ENCRYPT:
+			crp->crp_op = CRYPTO_OP_ENCRYPT |
+			    CRYPTO_OP_COMPUTE_DIGEST;
+			break;
+		case COP_DECRYPT:
+			crp->crp_op = CRYPTO_OP_DECRYPT |
+			    CRYPTO_OP_VERIFY_DIGEST;
+			break;
+		default:
+			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
+			error = EINVAL;
+			goto bail;
+		}
+		break;
+	default:
 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
+		error = EINVAL;
 		goto bail;
 	}
 
-	if (crda) {
-		crda->crd_skip = 0;
-		crda->crd_len = cop->len;
-		crda->crd_inject = cop->len;
-
-		crda->crd_alg = cse->mac;
-		crda->crd_key = cse->mackey;
-		crda->crd_klen = cse->mackeylen * 8;
-	}
-
-	if (crde) {
-		if (cop->op == COP_ENCRYPT)
-			crde->crd_flags |= CRD_F_ENCRYPT;
-		else
-			crde->crd_flags &= ~CRD_F_ENCRYPT;
-		crde->crd_len = cop->len;
-		crde->crd_inject = 0;
-
-		crde->crd_alg = cse->cipher;
-		crde->crd_key = cse->key;
-		crde->crd_klen = cse->keylen * 8;
-	}
-
-	crp->crp_ilen = cop->len;
-	crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIMM
-		       | (cop->flags & COP_F_BATCH);
-	crp->crp_uio = &cod->uio;
+	crp->crp_ilen = cop->len + cse->hashsize;
+	crp->crp_flags = CRYPTO_F_CBIMM | (cop->flags & COP_F_BATCH);
+	crp->crp_buf = cod->buf;
+	crp->crp_buf_type = CRYPTO_BUF_CONTIG;
 	crp->crp_callback = cryptodev_cb;
-	crp->crp_session = cse->cses;
 	crp->crp_opaque = cod;
 
 	if (cop->iv) {
-		if (crde == NULL) {
+		if (cse->ivsize == 0) {
 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 			error = EINVAL;
 			goto bail;
 		}
-		if (cse->cipher == CRYPTO_ARC4) { /* XXX use flag? */
+		error = copyin(cop->iv, crp->crp_iv, cse->ivsize);
+		if (error) {
 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
-			error = EINVAL;
 			goto bail;
 		}
-		if ((error = copyin(cop->iv, crde->crd_iv,
-		    cse->txform->ivsize))) {
+		crp->crp_flags |= CRYPTO_F_IV_SEPARATE;
+	} else if (cse->ivsize != 0) {
+		crp->crp_iv_start = 0;
+		crp->crp_payload_start += cse->ivsize;
+		crp->crp_payload_length -= cse->ivsize;
+	}
+
+	if (cop->mac != NULL) {
+		error = copyin(cop->mac, cod->buf + cop->len, cse->hashsize);
+		if (error) {
 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 			goto bail;
 		}
-		crde->crd_flags |= CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
-		crde->crd_skip = 0;
-	} else if (cse->cipher == CRYPTO_ARC4) { /* XXX use flag? */
-		crde->crd_skip = 0;
-	} else if (crde) {
-		crde->crd_flags |= CRD_F_IV_PRESENT;
-		crde->crd_skip = cse->txform->ivsize;
-		crde->crd_len -= cse->txform->ivsize;
 	}
-
-	if (cop->mac && crda == NULL) {
-		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
-		error = EINVAL;
-		goto bail;
-	}
 	cryptodev_warn(cse);
-
 again:
 	/*
 	 * Let the dispatch run unlocked, then, interlock against the
 	 * callback before checking if the operation completed and going
 	 * to sleep.  This insures drivers don't inherit our lock which
 	 * results in a lock order reversal between crypto_dispatch forced
 	 * entry and the crypto_done callback into us.
 	 */
 	error = crypto_dispatch(crp);
 	if (error != 0) {
 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		goto bail;
 	}
 
 	mtx_lock(&cse->lock);
 	while (!cod->done)
 		mtx_sleep(cod, &cse->lock, PWAIT, "crydev", 0);
 	mtx_unlock(&cse->lock);
 
 	if (crp->crp_etype == EAGAIN) {
 		crp->crp_etype = 0;
 		crp->crp_flags &= ~CRYPTO_F_DONE;
 		cod->done = false;
 		goto again;
 	}
 
 	if (crp->crp_etype != 0) {
 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		error = crp->crp_etype;
 		goto bail;
 	}
 
-	if (cop->dst &&
-	    (error = copyout(cod->uio.uio_iov[0].iov_base, cop->dst,
-	    cop->len))) {
-		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
-		goto bail;
+	if (cop->dst != NULL) {
+		error = copyout(cod->buf, cop->dst, cop->len);
+		if (error) {
+			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
+			goto bail;
+		}
 	}
 
-	if (cop->mac &&
-	    (error = copyout((caddr_t)cod->uio.uio_iov[0].iov_base + cop->len,
-	    cop->mac, cse->thash->hashsize))) {
-		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
-		goto bail;
+	if (cop->mac != NULL) {
+		error = copyout(cod->buf + cop->len, cop->mac, cse->hashsize);
+		if (error) {
+			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
+			goto bail;
+		}
 	}
 
 bail:
 	if (crp)
 		crypto_freereq(crp);
 	if (cod)
 		cod_free(cod);
 
 	return (error);
 }
 
 static int
 cryptodev_aead(
 	struct csession *cse,
 	struct crypt_aead *caead,
 	struct ucred *active_cred,
 	struct thread *td)
 {
 	struct cryptop_data *cod = NULL;
 	struct cryptop *crp = NULL;
-	struct cryptodesc *crde = NULL, *crda = NULL;
 	int error;
 
 	if (caead->len > 256*1024-4 || caead->aadlen > 256*1024-4) {
 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		return (E2BIG);
 	}
 
-	if (cse->txform == NULL || cse->thash == NULL || caead->tag == NULL ||
+	if (cse->txform == NULL || cse->hashsize == 0 || caead->tag == NULL ||
 	    (caead->len % cse->txform->blocksize) != 0) {
 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		return (EINVAL);
 	}
 
-	cod = cod_alloc(cse, caead->aadlen + caead->len + cse->thash->hashsize,
-	    td);
+	/*
+	 * The COP_F_CIPHER_FIRST flag predates explicit session
+	 * modes, but the only way it was used was for EtA so allow it
+	 * as long as it is consistent with EtA.
+	 */
+	if (caead->flags & COP_F_CIPHER_FIRST) {
+		if (caead->op != COP_ENCRYPT) {
+			SDT_PROBE1(opencrypto, dev, ioctl, error,  __LINE__);
+			return (EINVAL);
+		}
+	}
 
-	crp = crypto_getreq(2);
-	if (crp == NULL) {
-		error = ENOMEM;
+	cod = cod_alloc(cse, caead->aadlen + caead->len + cse->hashsize, td);
+
+	crp = crypto_getreq(cse->cses, M_WAITOK);
+
+	error = copyin(caead->aad, cod->buf, caead->aadlen);
+	if (error) {
 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		goto bail;
 	}
+	crp->crp_aad_start = 0;
+	crp->crp_aad_length = caead->aadlen;
 
-	if (caead->flags & COP_F_CIPHER_FIRST) {
-		crde = crp->crp_desc;
-		crda = crde->crd_next;
-	} else {
-		crda = crp->crp_desc;
-		crde = crda->crd_next;
-	}
-
-	if ((error = copyin(caead->aad, cod->uio.uio_iov[0].iov_base,
-	    caead->aadlen))) {
+	error = copyin(caead->src, cod->buf + caead->aadlen, caead->len);
+	if (error) {
 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		goto bail;
 	}
+	crp->crp_payload_start = caead->aadlen;
+	crp->crp_payload_length = caead->len;
+	crp->crp_digest_start = caead->aadlen + caead->len;
 
-	if ((error = copyin(caead->src, (char *)cod->uio.uio_iov[0].iov_base +
-	    caead->aadlen, caead->len))) {
+	switch (cse->mode) {
+	case CSP_MODE_AEAD:
+		switch (caead->op) {
+		case COP_ENCRYPT:
+			crp->crp_op = CRYPTO_OP_ENCRYPT |
+			    CRYPTO_OP_COMPUTE_DIGEST;
+			break;
+		case COP_DECRYPT:
+			crp->crp_op = CRYPTO_OP_DECRYPT |
+			    CRYPTO_OP_VERIFY_DIGEST;
+			break;
+		default:
+			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
+			error = EINVAL;
+			goto bail;
+		}
+		break;
+	case CSP_MODE_ETA:
+		switch (caead->op) {
+		case COP_ENCRYPT:
+			crp->crp_op = CRYPTO_OP_ENCRYPT |
+			    CRYPTO_OP_COMPUTE_DIGEST;
+			break;
+		case COP_DECRYPT:
+			crp->crp_op = CRYPTO_OP_DECRYPT |
+			    CRYPTO_OP_VERIFY_DIGEST;
+			break;
+		default:
+			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
+			error = EINVAL;
+			goto bail;
+		}
+		break;
+	default:
 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
+		error = EINVAL;
 		goto bail;
 	}
 
-	/*
-	 * For GCM/CCM, crd_len covers only the AAD.  For other ciphers
-	 * chained with an HMAC, crd_len covers both the AAD and the
-	 * cipher text.
-	 */
-	crda->crd_skip = 0;
-	if (cse->cipher == CRYPTO_AES_NIST_GCM_16 ||
-	    cse->cipher == CRYPTO_AES_CCM_16)
-		crda->crd_len = caead->aadlen;
-	else
-		crda->crd_len = caead->aadlen + caead->len;
-	crda->crd_inject = caead->aadlen + caead->len;
-
-	crda->crd_alg = cse->mac;
-	crda->crd_key = cse->mackey;
-	crda->crd_klen = cse->mackeylen * 8;
-
-	if (caead->op == COP_ENCRYPT)
-		crde->crd_flags |= CRD_F_ENCRYPT;
-	else
-		crde->crd_flags &= ~CRD_F_ENCRYPT;
-	crde->crd_skip = caead->aadlen;
-	crde->crd_len = caead->len;
-	crde->crd_inject = caead->aadlen;
-
-	crde->crd_alg = cse->cipher;
-	crde->crd_key = cse->key;
-	crde->crd_klen = cse->keylen * 8;
-
-	crp->crp_ilen = caead->aadlen + caead->len;
-	crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIMM
-		       | (caead->flags & COP_F_BATCH);
-	crp->crp_uio = &cod->uio;
+	crp->crp_ilen = caead->aadlen + caead->len + cse->hashsize;
+	crp->crp_flags = CRYPTO_F_CBIMM | (caead->flags & COP_F_BATCH);
+	crp->crp_buf = cod->buf;
+	crp->crp_buf_type = CRYPTO_BUF_CONTIG;
 	crp->crp_callback = cryptodev_cb;
-	crp->crp_session = cse->cses;
 	crp->crp_opaque = cod;
 
 	if (caead->iv) {
-		if (caead->ivlen > sizeof(crde->crd_iv)) {
+		/*
+		 * Permit a 16-byte IV for AES-XTS, but only use the
+		 * first 8 bytes as a block number.
+		 */
+		if (cse->mode == CSP_MODE_ETA &&
+		    caead->ivlen == AES_BLOCK_LEN &&
+		    cse->ivsize == AES_XTS_IV_LEN)
+			caead->ivlen = AES_XTS_IV_LEN;
+
+		if (caead->ivlen != cse->ivsize) {
 			error = EINVAL;
 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 			goto bail;
 		}
 
-		if ((error = copyin(caead->iv, crde->crd_iv, caead->ivlen))) {
+		error = copyin(caead->iv, crp->crp_iv, cse->ivsize);
+		if (error) {
 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 			goto bail;
 		}
-		crde->crd_flags |= CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
+		crp->crp_flags |= CRYPTO_F_IV_SEPARATE;
 	} else {
-		crde->crd_flags |= CRD_F_IV_PRESENT;
-		crde->crd_skip += cse->txform->ivsize;
-		crde->crd_len -= cse->txform->ivsize;
+		crp->crp_iv_start = crp->crp_payload_start;
+		crp->crp_payload_start += cse->ivsize;
+		crp->crp_payload_length -= cse->ivsize;
 	}
 
-	if ((error = copyin(caead->tag, (caddr_t)cod->uio.uio_iov[0].iov_base +
-	    caead->len + caead->aadlen, cse->thash->hashsize))) {
+	error = copyin(caead->tag, cod->buf + caead->len + caead->aadlen,
+	    cse->hashsize);
+	if (error) {
 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		goto bail;
 	}
 	cryptodev_warn(cse);
 again:
 	/*
 	 * Let the dispatch run unlocked, then, interlock against the
 	 * callback before checking if the operation completed and going
 	 * to sleep.  This insures drivers don't inherit our lock which
 	 * results in a lock order reversal between crypto_dispatch forced
 	 * entry and the crypto_done callback into us.
 	 */
 	error = crypto_dispatch(crp);
 	if (error != 0) {
 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		goto bail;
 	}
 
 	mtx_lock(&cse->lock);
 	while (!cod->done)
 		mtx_sleep(cod, &cse->lock, PWAIT, "crydev", 0);
 	mtx_unlock(&cse->lock);
 
 	if (crp->crp_etype == EAGAIN) {
 		crp->crp_etype = 0;
 		crp->crp_flags &= ~CRYPTO_F_DONE;
 		cod->done = false;
 		goto again;
 	}
 
 	if (crp->crp_etype != 0) {
 		error = crp->crp_etype;
 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		goto bail;
 	}
 
-	if (caead->dst && (error = copyout(
-	    (caddr_t)cod->uio.uio_iov[0].iov_base + caead->aadlen, caead->dst,
-	    caead->len))) {
-		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
-		goto bail;
+	if (caead->dst != NULL) {
+		error = copyout(cod->buf + caead->aadlen, caead->dst,
+		    caead->len);
+		if (error) {
+			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
+			goto bail;
+		}
 	}
 
-	if ((error = copyout((caddr_t)cod->uio.uio_iov[0].iov_base +
-	    caead->aadlen + caead->len, caead->tag, cse->thash->hashsize))) {
+	error = copyout(cod->buf + caead->aadlen + caead->len, caead->tag,
+	    cse->hashsize);
+	if (error) {
 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		goto bail;
 	}
 
 bail:
 	crypto_freereq(crp);
 	if (cod)
 		cod_free(cod);
 
 	return (error);
 }
 
 static int
 cryptodev_cb(struct cryptop *crp)
 {
 	struct cryptop_data *cod = crp->crp_opaque;
 
 	/*
 	 * Lock to ensure the wakeup() is not missed by the loops
 	 * waiting on cod->done in cryptodev_op() and
 	 * cryptodev_aead().
 	 */
 	mtx_lock(&cod->cse->lock);
 	cod->done = true;
 	mtx_unlock(&cod->cse->lock);
 	wakeup(cod);
 	return (0);
 }
 
-static int
-cryptodevkey_cb(void *op)
+static void
+cryptodevkey_cb(struct cryptkop *krp)
 {
-	struct cryptkop *krp = (struct cryptkop *) op;
 
 	wakeup_one(krp);
-	return (0);
 }
 
 static int
 cryptodev_key(struct crypt_kop *kop)
 {
 	struct cryptkop *krp = NULL;
 	int error = EINVAL;
 	int in, out, size, i;
 
 	if (kop->crk_iparams + kop->crk_oparams > CRK_MAXPARAM) {
 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		return (EFBIG);
 	}
 
 	in = kop->crk_iparams;
 	out = kop->crk_oparams;
 	switch (kop->crk_op) {
 	case CRK_MOD_EXP:
 		if (in == 3 && out == 1)
 			break;
 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		return (EINVAL);
 	case CRK_MOD_EXP_CRT:
 		if (in == 6 && out == 1)
 			break;
 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		return (EINVAL);
 	case CRK_DSA_SIGN:
 		if (in == 5 && out == 2)
 			break;
 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		return (EINVAL);
 	case CRK_DSA_VERIFY:
 		if (in == 7 && out == 0)
 			break;
 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		return (EINVAL);
 	case CRK_DH_COMPUTE_KEY:
 		if (in == 3 && out == 1)
 			break;
 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		return (EINVAL);
 	default:
 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		return (EINVAL);
 	}
 
 	krp = (struct cryptkop *)malloc(sizeof *krp, M_XDATA, M_WAITOK|M_ZERO);
 	if (!krp) {
 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		return (ENOMEM);
 	}
 	krp->krp_op = kop->crk_op;
 	krp->krp_status = kop->crk_status;
 	krp->krp_iparams = kop->crk_iparams;
 	krp->krp_oparams = kop->crk_oparams;
 	krp->krp_crid = kop->crk_crid;
 	krp->krp_status = 0;
-	krp->krp_callback = (int (*) (struct cryptkop *)) cryptodevkey_cb;
+	krp->krp_callback = cryptodevkey_cb;
 
 	for (i = 0; i < CRK_MAXPARAM; i++) {
 		if (kop->crk_param[i].crp_nbits > 65536) {
 			/* Limit is the same as in OpenBSD */
 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 			goto fail;
 		}
 		krp->krp_param[i].crp_nbits = kop->crk_param[i].crp_nbits;
 	}
 	for (i = 0; i < krp->krp_iparams + krp->krp_oparams; i++) {
 		size = (krp->krp_param[i].crp_nbits + 7) / 8;
 		if (size == 0)
 			continue;
 		krp->krp_param[i].crp_p = malloc(size, M_XDATA, M_WAITOK);
 		if (i >= krp->krp_iparams)
 			continue;
 		error = copyin(kop->crk_param[i].crp_p, krp->krp_param[i].crp_p, size);
 		if (error) {
 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 			goto fail;
 		}
 	}
 
 	error = crypto_kdispatch(krp);
 	if (error) {
 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		goto fail;
 	}
 	error = tsleep(krp, PSOCK, "crydev", 0);
 	if (error) {
 		/* XXX can this happen?  if so, how do we recover? */
 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		goto fail;
 	}
 	
-	kop->crk_crid = krp->krp_crid;		/* device that did the work */
+	kop->crk_crid = krp->krp_hid;		/* device that did the work */
 	if (krp->krp_status != 0) {
 		error = krp->krp_status;
 		SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 		goto fail;
 	}
 
 	for (i = krp->krp_iparams; i < krp->krp_iparams + krp->krp_oparams; i++) {
 		size = (krp->krp_param[i].crp_nbits + 7) / 8;
 		if (size == 0)
 			continue;
 		error = copyout(krp->krp_param[i].crp_p, kop->crk_param[i].crp_p, size);
 		if (error) {
 			SDT_PROBE1(opencrypto, dev, ioctl, error, __LINE__);
 			goto fail;
 		}
 	}
 
 fail:
 	if (krp) {
 		kop->crk_status = krp->krp_status;
 		for (i = 0; i < CRK_MAXPARAM; i++) {
 			if (krp->krp_param[i].crp_p)
 				free(krp->krp_param[i].crp_p, M_XDATA);
 		}
 		free(krp, M_XDATA);
 	}
 	return (error);
 }
 
 static int
 cryptodev_find(struct crypt_find_op *find)
 {
 	device_t dev;
 	size_t fnlen = sizeof find->name;
 
 	if (find->crid != -1) {
 		dev = crypto_find_device_byhid(find->crid);
 		if (dev == NULL)
 			return (ENOENT);
 		strncpy(find->name, device_get_nameunit(dev), fnlen);
 		find->name[fnlen - 1] = '\x0';
 	} else {
 		find->name[fnlen - 1] = '\x0';
 		find->crid = crypto_find_driver(find->name);
 		if (find->crid == -1)
 			return (ENOENT);
 	}
 	return (0);
 }
 
 /* ARGSUSED */
 static int
 cryptof_stat(
 	struct file *fp,
 	struct stat *sb,
 	struct ucred *active_cred,
 	struct thread *td)
 {
 
 	return (EOPNOTSUPP);
 }
 
 /* ARGSUSED */
 static int
 cryptof_close(struct file *fp, struct thread *td)
 {
 	struct fcrypt *fcr = fp->f_data;
 	struct csession *cse;
 
 	while ((cse = TAILQ_FIRST(&fcr->csessions))) {
 		TAILQ_REMOVE(&fcr->csessions, cse, next);
 		KASSERT(cse->refs == 1,
 		    ("%s: crypto session %p with %d refs", __func__, cse,
 		    cse->refs));
 		csefree(cse);
 	}
 	free(fcr, M_XDATA);
 	fp->f_data = NULL;
 	return 0;
 }
 
 static int
 cryptof_fill_kinfo(struct file *fp, struct kinfo_file *kif, struct filedesc *fdp)
 {
 
 	kif->kf_type = KF_TYPE_CRYPTO;
 	return (0);
 }
 
 static struct csession *
 csefind(struct fcrypt *fcr, u_int ses)
 {
 	struct csession *cse;
 
 	mtx_lock(&fcr->lock);
 	TAILQ_FOREACH(cse, &fcr->csessions, next) {
 		if (cse->ses == ses) {
 			refcount_acquire(&cse->refs);
 			mtx_unlock(&fcr->lock);
 			return (cse);
 		}
 	}
 	mtx_unlock(&fcr->lock);
 	return (NULL);
 }
 
 static bool
 csedelete(struct fcrypt *fcr, u_int ses)
 {
 	struct csession *cse;
 
 	mtx_lock(&fcr->lock);
 	TAILQ_FOREACH(cse, &fcr->csessions, next) {
 		if (cse->ses == ses) {
 			TAILQ_REMOVE(&fcr->csessions, cse, next);
 			mtx_unlock(&fcr->lock);
 			csefree(cse);
 			return (true);
 		}
 	}
 	mtx_unlock(&fcr->lock);
 	return (false);
 }
 	
 struct csession *
-csecreate(struct fcrypt *fcr, crypto_session_t cses, caddr_t key, u_int64_t keylen,
-    caddr_t mackey, u_int64_t mackeylen, u_int32_t cipher, u_int32_t mac,
-    struct enc_xform *txform, struct auth_hash *thash)
+csecreate(struct fcrypt *fcr, crypto_session_t cses,
+    struct crypto_session_params *csp, struct enc_xform *txform,
+    void *key, struct auth_hash *thash, void *mackey)
 {
 	struct csession *cse;
 
 	cse = malloc(sizeof(struct csession), M_XDATA, M_NOWAIT | M_ZERO);
 	if (cse == NULL)
 		return NULL;
 	mtx_init(&cse->lock, "cryptodev", "crypto session lock", MTX_DEF);
 	refcount_init(&cse->refs, 1);
 	cse->key = key;
-	cse->keylen = keylen/8;
 	cse->mackey = mackey;
-	cse->mackeylen = mackeylen/8;
+	cse->mode = csp->csp_mode;
 	cse->cses = cses;
-	cse->cipher = cipher;
-	cse->mac = mac;
 	cse->txform = txform;
-	cse->thash = thash;
+	if (thash != NULL)
+		cse->hashsize = thash->hashsize;
+	else if (csp->csp_cipher_alg == CRYPTO_AES_NIST_GCM_16)
+		cse->hashsize = AES_GMAC_HASH_LEN;
+	else if (csp->csp_cipher_alg == CRYPTO_AES_CCM_16)
+		cse->hashsize = AES_CBC_MAC_HASH_LEN;
+	cse->ivsize = csp->csp_ivlen;
 	mtx_lock(&fcr->lock);
 	TAILQ_INSERT_TAIL(&fcr->csessions, cse, next);
 	cse->ses = fcr->sesn++;
 	mtx_unlock(&fcr->lock);
 	return (cse);
 }
 
 static void
 csefree(struct csession *cse)
 {
 
 	if (!refcount_release(&cse->refs))
 		return;
 	crypto_freesession(cse->cses);
 	mtx_destroy(&cse->lock);
 	if (cse->key)
 		free(cse->key, M_XDATA);
 	if (cse->mackey)
 		free(cse->mackey, M_XDATA);
 	free(cse, M_XDATA);
 }
 
 static int
 cryptoioctl(struct cdev *dev, u_long cmd, caddr_t data, int flag, struct thread *td)
 {
 	struct file *f;
 	struct fcrypt *fcr;
 	int fd, error;
 
 	switch (cmd) {
 	case CRIOGET:
 		error = falloc_noinstall(td, &f);
 		if (error)
 			break;
 
 		fcr = malloc(sizeof(struct fcrypt), M_XDATA, M_WAITOK | M_ZERO);
 		TAILQ_INIT(&fcr->csessions);
 		mtx_init(&fcr->lock, "fcrypt", NULL, MTX_DEF);
 
 		finit(f, FREAD | FWRITE, DTYPE_CRYPTO, fcr, &cryptofops);
 		error = finstall(td, f, &fd, 0, NULL);
 		if (error) {
 			mtx_destroy(&fcr->lock);
 			free(fcr, M_XDATA);
 		} else
 			*(uint32_t *)data = fd;
 		fdrop(f, td);
 		break;
 	case CRIOFINDDEV:
 		error = cryptodev_find((struct crypt_find_op *)data);
 		break;
 	case CRIOASYMFEAT:
 		error = crypto_getfeat((int *)data);
 		break;
 	default:
 		error = EINVAL;
 		break;
 	}
 	return (error);
 }
 
 static struct cdevsw crypto_cdevsw = {
 	.d_version =	D_VERSION,
 	.d_ioctl =	cryptoioctl,
 	.d_name =	"crypto",
 };
 static struct cdev *crypto_dev;
 
 /*
  * Initialization code, both for static and dynamic loading.
  */
 static int
 cryptodev_modevent(module_t mod, int type, void *unused)
 {
 	switch (type) {
 	case MOD_LOAD:
 		if (bootverbose)
 			printf("crypto: <crypto device>\n");
 		crypto_dev = make_dev(&crypto_cdevsw, 0, 
 				      UID_ROOT, GID_WHEEL, 0666,
 				      "crypto");
 		return 0;
 	case MOD_UNLOAD:
 		/*XXX disallow if active sessions */
 		destroy_dev(crypto_dev);
 		return 0;
 	}
 	return EINVAL;
 }
 
 static moduledata_t cryptodev_mod = {
 	"cryptodev",
 	cryptodev_modevent,
 	0
 };
 MODULE_VERSION(cryptodev, 1);
 DECLARE_MODULE(cryptodev, cryptodev_mod, SI_SUB_PSEUDO, SI_ORDER_ANY);
 MODULE_DEPEND(cryptodev, crypto, 1, 1, 1);
 MODULE_DEPEND(cryptodev, zlib, 1, 1, 1);
Index: head/sys/opencrypto/cryptodev.h
===================================================================
--- head/sys/opencrypto/cryptodev.h	(revision 359373)
+++ head/sys/opencrypto/cryptodev.h	(revision 359374)
@@ -1,577 +1,615 @@
 /*	$FreeBSD$	*/
 /*	$OpenBSD: cryptodev.h,v 1.31 2002/06/11 11:14:29 beck Exp $	*/
 
 /*-
  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
  * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting
  *
  * This code was written by Angelos D. Keromytis in Athens, Greece, in
  * February 2000. Network Security Technologies Inc. (NSTI) kindly
  * supported the development of this code.
  *
  * Copyright (c) 2000 Angelos D. Keromytis
  *
  * Permission to use, copy, and modify this software with or without fee
  * is hereby granted, provided that this entire notice is included in
  * all source code copies of any software which is or includes a copy or
  * modification of this software.
  *
  * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR
  * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY
  * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE
  * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR
  * PURPOSE.
  *
  * Copyright (c) 2001 Theo de Raadt
  * Copyright (c) 2014 The FreeBSD Foundation
  * All rights reserved.
  *
  * Portions of this software were developed by John-Mark Gurney
  * under sponsorship of the FreeBSD Foundation and
  * Rubicon Communications, LLC (Netgate).
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  *
  * 1. Redistributions of source code must retain the above copyright
  *   notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *   notice, this list of conditions and the following disclaimer in the
  *   documentation and/or other materials provided with the distribution.
  * 3. The name of the author may not be used to endorse or promote products
  *   derived from this software without specific prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  * Effort sponsored in part by the Defense Advanced Research Projects
  * Agency (DARPA) and Air Force Research Laboratory, Air Force
  * Materiel Command, USAF, under agreement number F30602-01-2-0537.
  *
  */
 
 #ifndef _CRYPTO_CRYPTO_H_
 #define _CRYPTO_CRYPTO_H_
 
 #include <sys/ioccom.h>
 
 #ifdef _KERNEL
 #include <opencrypto/_cryptodev.h>
 #include <sys/_task.h>
 #endif
 
 /* Some initial values */
 #define CRYPTO_DRIVERS_INITIAL	4
-#define CRYPTO_SW_SESSIONS	32
 
 /* Hash values */
 #define	NULL_HASH_LEN		16
 #define	MD5_HASH_LEN		16
 #define	SHA1_HASH_LEN		20
 #define	RIPEMD160_HASH_LEN	20
 #define	SHA2_224_HASH_LEN	28
 #define	SHA2_256_HASH_LEN	32
 #define	SHA2_384_HASH_LEN	48
 #define	SHA2_512_HASH_LEN	64
 #define	MD5_KPDK_HASH_LEN	16
 #define	SHA1_KPDK_HASH_LEN	20
 #define	AES_GMAC_HASH_LEN	16
 #define	POLY1305_HASH_LEN	16
 #define	AES_CBC_MAC_HASH_LEN	16
 /* Maximum hash algorithm result length */
 #define	HASH_MAX_LEN		SHA2_512_HASH_LEN /* Keep this updated */
 
 #define	MD5_BLOCK_LEN		64
 #define	SHA1_BLOCK_LEN		64
 #define	RIPEMD160_BLOCK_LEN	64
 #define	SHA2_224_BLOCK_LEN	64
 #define	SHA2_256_BLOCK_LEN	64
 #define	SHA2_384_BLOCK_LEN	128
 #define	SHA2_512_BLOCK_LEN	128
 
 /* HMAC values */
 #define	NULL_HMAC_BLOCK_LEN		64
 /* Maximum HMAC block length */
 #define	HMAC_MAX_BLOCK_LEN	SHA2_512_BLOCK_LEN /* Keep this updated */
 #define	HMAC_IPAD_VAL			0x36
 #define	HMAC_OPAD_VAL			0x5C
 /* HMAC Key Length */
 #define	AES_128_GMAC_KEY_LEN		16
 #define	AES_192_GMAC_KEY_LEN		24
 #define	AES_256_GMAC_KEY_LEN		32
 #define	AES_128_CBC_MAC_KEY_LEN		16
 #define	AES_192_CBC_MAC_KEY_LEN		24
 #define	AES_256_CBC_MAC_KEY_LEN		32
 
 #define	POLY1305_KEY_LEN		32
 
 /* Encryption algorithm block sizes */
 #define	NULL_BLOCK_LEN		4	/* IPsec to maintain alignment */
 #define	DES_BLOCK_LEN		8
 #define	DES3_BLOCK_LEN		8
 #define	BLOWFISH_BLOCK_LEN	8
 #define	SKIPJACK_BLOCK_LEN	8
 #define	CAST128_BLOCK_LEN	8
 #define	RIJNDAEL128_BLOCK_LEN	16
 #define	AES_BLOCK_LEN		16
 #define	AES_ICM_BLOCK_LEN	1
 #define	ARC4_BLOCK_LEN		1
 #define	CAMELLIA_BLOCK_LEN	16
 #define	CHACHA20_NATIVE_BLOCK_LEN	64
 #define	EALG_MAX_BLOCK_LEN	CHACHA20_NATIVE_BLOCK_LEN /* Keep this updated */
 
 /* IV Lengths */
 
 #define	ARC4_IV_LEN		1
 #define	AES_GCM_IV_LEN		12
 #define	AES_CCM_IV_LEN		12
 #define	AES_XTS_IV_LEN		8
 #define	AES_XTS_ALPHA		0x87	/* GF(2^128) generator polynomial */
 
 /* Min and Max Encryption Key Sizes */
 #define	NULL_MIN_KEY		0
 #define	NULL_MAX_KEY		256 /* 2048 bits, max key */
 #define	DES_MIN_KEY		8
 #define	DES_MAX_KEY		DES_MIN_KEY
 #define	TRIPLE_DES_MIN_KEY	24
 #define	TRIPLE_DES_MAX_KEY	TRIPLE_DES_MIN_KEY
 #define	BLOWFISH_MIN_KEY	5
 #define	BLOWFISH_MAX_KEY	56 /* 448 bits, max key */
 #define	CAST_MIN_KEY		5
 #define	CAST_MAX_KEY		16
 #define	SKIPJACK_MIN_KEY	10
 #define	SKIPJACK_MAX_KEY	SKIPJACK_MIN_KEY
 #define	RIJNDAEL_MIN_KEY	16
 #define	RIJNDAEL_MAX_KEY	32
 #define	AES_MIN_KEY		RIJNDAEL_MIN_KEY
 #define	AES_MAX_KEY		RIJNDAEL_MAX_KEY
 #define	AES_XTS_MIN_KEY		(2 * AES_MIN_KEY)
 #define	AES_XTS_MAX_KEY		(2 * AES_MAX_KEY)
 #define	ARC4_MIN_KEY		1
 #define	ARC4_MAX_KEY		32
 #define	CAMELLIA_MIN_KEY	8
 #define	CAMELLIA_MAX_KEY	32
 
 /* Maximum hash algorithm result length */
 #define	AALG_MAX_RESULT_LEN	64 /* Keep this updated */
 
 #define	CRYPTO_ALGORITHM_MIN	1
 #define	CRYPTO_DES_CBC		1
 #define	CRYPTO_3DES_CBC		2
 #define	CRYPTO_BLF_CBC		3
 #define	CRYPTO_CAST_CBC		4
 #define	CRYPTO_SKIPJACK_CBC	5
 #define	CRYPTO_MD5_HMAC		6
 #define	CRYPTO_SHA1_HMAC	7
 #define	CRYPTO_RIPEMD160_HMAC	8
 #define	CRYPTO_MD5_KPDK		9
 #define	CRYPTO_SHA1_KPDK	10
 #define	CRYPTO_RIJNDAEL128_CBC	11 /* 128 bit blocksize */
 #define	CRYPTO_AES_CBC		11 /* 128 bit blocksize -- the same as above */
 #define	CRYPTO_ARC4		12
 #define	CRYPTO_MD5		13
 #define	CRYPTO_SHA1		14
 #define	CRYPTO_NULL_HMAC	15
 #define	CRYPTO_NULL_CBC		16
 #define	CRYPTO_DEFLATE_COMP	17 /* Deflate compression algorithm */
 #define	CRYPTO_SHA2_256_HMAC	18
 #define	CRYPTO_SHA2_384_HMAC	19
 #define	CRYPTO_SHA2_512_HMAC	20
 #define	CRYPTO_CAMELLIA_CBC	21
 #define	CRYPTO_AES_XTS		22
 #define	CRYPTO_AES_ICM		23 /* commonly known as CTR mode */
-#define	CRYPTO_AES_NIST_GMAC	24 /* cipher side */
+#define	CRYPTO_AES_NIST_GMAC	24 /* GMAC only */
 #define	CRYPTO_AES_NIST_GCM_16	25 /* 16 byte ICV */
+#ifdef _KERNEL
 #define	CRYPTO_AES_128_NIST_GMAC 26 /* auth side */
 #define	CRYPTO_AES_192_NIST_GMAC 27 /* auth side */
 #define	CRYPTO_AES_256_NIST_GMAC 28 /* auth side */
+#endif
 #define	CRYPTO_BLAKE2B		29 /* Blake2b hash */
 #define	CRYPTO_BLAKE2S		30 /* Blake2s hash */
 #define	CRYPTO_CHACHA20		31 /* Chacha20 stream cipher */
 #define	CRYPTO_SHA2_224_HMAC	32
 #define	CRYPTO_RIPEMD160	33
 #define	CRYPTO_SHA2_224		34
 #define	CRYPTO_SHA2_256		35
 #define	CRYPTO_SHA2_384		36
 #define	CRYPTO_SHA2_512		37
 #define	CRYPTO_POLY1305		38
 #define	CRYPTO_AES_CCM_CBC_MAC	39	/* auth side */
 #define	CRYPTO_AES_CCM_16	40	/* cipher side */
 #define	CRYPTO_ALGORITHM_MAX	40	/* Keep updated - see below */
 
 #define	CRYPTO_ALGO_VALID(x)	((x) >= CRYPTO_ALGORITHM_MIN && \
 				 (x) <= CRYPTO_ALGORITHM_MAX)
 
 /* Algorithm flags */
 #define	CRYPTO_ALG_FLAG_SUPPORTED	0x01 /* Algorithm is supported */
 #define	CRYPTO_ALG_FLAG_RNG_ENABLE	0x02 /* Has HW RNG for DH/DSA */
 #define	CRYPTO_ALG_FLAG_DSA_SHA		0x04 /* Can do SHA on msg */
 
 /*
  * Crypto driver/device flags.  They can set in the crid
  * parameter when creating a session or submitting a key
  * op to affect the device/driver assigned.  If neither
  * of these are specified then the crid is assumed to hold
  * the driver id of an existing (and suitable) device that
  * must be used to satisfy the request.
  */
 #define CRYPTO_FLAG_HARDWARE	0x01000000	/* hardware accelerated */
 #define CRYPTO_FLAG_SOFTWARE	0x02000000	/* software implementation */
 
 /* NB: deprecated */
 struct session_op {
 	u_int32_t	cipher;		/* ie. CRYPTO_DES_CBC */
 	u_int32_t	mac;		/* ie. CRYPTO_MD5_HMAC */
 
 	u_int32_t	keylen;		/* cipher key */
 	c_caddr_t	key;
 	int		mackeylen;	/* mac key */
 	c_caddr_t	mackey;
 
   	u_int32_t	ses;		/* returns: session # */ 
 };
 
 /*
  * session and crypt _op structs are used by userspace programs to interact
  * with /dev/crypto.  Confusingly, the internal kernel interface is named
  * "cryptop" (no underscore).
  */
 struct session2_op {
 	u_int32_t	cipher;		/* ie. CRYPTO_DES_CBC */
 	u_int32_t	mac;		/* ie. CRYPTO_MD5_HMAC */
 
 	u_int32_t	keylen;		/* cipher key */
 	c_caddr_t	key;
 	int		mackeylen;	/* mac key */
 	c_caddr_t	mackey;
 
   	u_int32_t	ses;		/* returns: session # */ 
 	int		crid;		/* driver id + flags (rw) */
 	int		pad[4];		/* for future expansion */
 };
 
 struct crypt_op {
 	u_int32_t	ses;
 	u_int16_t	op;		/* i.e. COP_ENCRYPT */
 #define COP_ENCRYPT	1
 #define COP_DECRYPT	2
 	u_int16_t	flags;
 #define	COP_F_CIPHER_FIRST	0x0001	/* Cipher before MAC. */
 #define	COP_F_BATCH		0x0008	/* Batch op if possible */
 	u_int		len;
 	c_caddr_t	src;		/* become iov[] inside kernel */
 	caddr_t		dst;
 	caddr_t		mac;		/* must be big enough for chosen MAC */
 	c_caddr_t	iv;
 };
 
 /* op and flags the same as crypt_op */
 struct crypt_aead {
 	u_int32_t	ses;
 	u_int16_t	op;		/* i.e. COP_ENCRYPT */
 	u_int16_t	flags;
 	u_int		len;
 	u_int		aadlen;
 	u_int		ivlen;
 	c_caddr_t	src;		/* become iov[] inside kernel */
 	caddr_t		dst;
 	c_caddr_t	aad;		/* additional authenticated data */
 	caddr_t		tag;		/* must fit for chosen TAG length */
 	c_caddr_t	iv;
 };
 
 /*
  * Parameters for looking up a crypto driver/device by
  * device name or by id.  The latter are returned for
  * created sessions (crid) and completed key operations.
  */
 struct crypt_find_op {
 	int		crid;		/* driver id + flags */
 	char		name[32];	/* device/driver name */
 };
 
 /* bignum parameter, in packed bytes, ... */
 struct crparam {
 	caddr_t		crp_p;
 	u_int		crp_nbits;
 };
 
 #define CRK_MAXPARAM	8
 
 struct crypt_kop {
 	u_int		crk_op;		/* ie. CRK_MOD_EXP or other */
 	u_int		crk_status;	/* return status */
 	u_short		crk_iparams;	/* # of input parameters */
 	u_short		crk_oparams;	/* # of output parameters */
 	u_int		crk_crid;	/* NB: only used by CIOCKEY2 (rw) */
 	struct crparam	crk_param[CRK_MAXPARAM];
 };
 #define	CRK_ALGORITM_MIN	0
 #define CRK_MOD_EXP		0
 #define CRK_MOD_EXP_CRT		1
 #define CRK_DSA_SIGN		2
 #define CRK_DSA_VERIFY		3
 #define CRK_DH_COMPUTE_KEY	4
 #define CRK_ALGORITHM_MAX	4 /* Keep updated - see below */
 
 #define CRF_MOD_EXP		(1 << CRK_MOD_EXP)
 #define CRF_MOD_EXP_CRT		(1 << CRK_MOD_EXP_CRT)
 #define CRF_DSA_SIGN		(1 << CRK_DSA_SIGN)
 #define CRF_DSA_VERIFY		(1 << CRK_DSA_VERIFY)
 #define CRF_DH_COMPUTE_KEY	(1 << CRK_DH_COMPUTE_KEY)
 
 /*
  * done against open of /dev/crypto, to get a cloned descriptor.
  * Please use F_SETFD against the cloned descriptor.
  */
 #define	CRIOGET		_IOWR('c', 100, u_int32_t)
 #define	CRIOASYMFEAT	CIOCASYMFEAT
 #define	CRIOFINDDEV	CIOCFINDDEV
 
 /* the following are done against the cloned descriptor */
 #define	CIOCGSESSION	_IOWR('c', 101, struct session_op)
 #define	CIOCFSESSION	_IOW('c', 102, u_int32_t)
 #define CIOCCRYPT	_IOWR('c', 103, struct crypt_op)
 #define CIOCKEY		_IOWR('c', 104, struct crypt_kop)
 #define CIOCASYMFEAT	_IOR('c', 105, u_int32_t)
 #define	CIOCGSESSION2	_IOWR('c', 106, struct session2_op)
 #define	CIOCKEY2	_IOWR('c', 107, struct crypt_kop)
 #define	CIOCFINDDEV	_IOWR('c', 108, struct crypt_find_op)
 #define	CIOCCRYPTAEAD	_IOWR('c', 109, struct crypt_aead)
 
 struct cryptotstat {
 	struct timespec	acc;		/* total accumulated time */
 	struct timespec	min;		/* min time */
 	struct timespec	max;		/* max time */
 	u_int32_t	count;		/* number of observations */
 };
 
 struct cryptostats {
 	u_int32_t	cs_ops;		/* symmetric crypto ops submitted */
 	u_int32_t	cs_errs;	/* symmetric crypto ops that failed */
 	u_int32_t	cs_kops;	/* asymetric/key ops submitted */
 	u_int32_t	cs_kerrs;	/* asymetric/key ops that failed */
 	u_int32_t	cs_intrs;	/* crypto swi thread activations */
 	u_int32_t	cs_rets;	/* crypto return thread activations */
 	u_int32_t	cs_blocks;	/* symmetric op driver block */
 	u_int32_t	cs_kblocks;	/* symmetric op driver block */
 	/*
 	 * When CRYPTO_TIMING is defined at compile time and the
 	 * sysctl debug.crypto is set to 1, the crypto system will
 	 * accumulate statistics about how long it takes to process
 	 * crypto requests at various points during processing.
 	 */
 	struct cryptotstat cs_invoke;	/* crypto_dipsatch -> crypto_invoke */
 	struct cryptotstat cs_done;	/* crypto_invoke -> crypto_done */
 	struct cryptotstat cs_cb;	/* crypto_done -> callback */
 	struct cryptotstat cs_finis;	/* callback -> callback return */
 };
 
 #ifdef _KERNEL
 
+/*
+ * Return values for cryptodev_probesession methods.
+ */
+#define	CRYPTODEV_PROBE_HARDWARE	(-100)
+#define	CRYPTODEV_PROBE_ACCEL_SOFTWARE	(-200)
+#define	CRYPTODEV_PROBE_SOFTWARE	(-500)
+
 #if 0
 #define CRYPTDEB(s, ...) do {						\
 	printf("%s:%d: " s "\n", __FILE__, __LINE__, ## __VA_ARGS__);	\
 } while (0)
 #else
 #define CRYPTDEB(...)	do { } while (0)
 #endif
 
-/* Standard initialization structure beginning */
-struct cryptoini {
-	int		cri_alg;	/* Algorithm to use */
-	int		cri_klen;	/* Key length, in bits */
-	int		cri_mlen;	/* Number of bytes we want from the
-					   entire hash. 0 means all. */
-	caddr_t		cri_key;	/* key to use */
-	u_int8_t	cri_iv[EALG_MAX_BLOCK_LEN];	/* IV to use */
-	struct cryptoini *cri_next;
-};
+struct crypto_session_params {
+	int		csp_mode;	/* Type of operations to perform. */
 
-/* Describe boundaries of a single crypto operation */
-struct cryptodesc {
-	int		crd_skip;	/* How many bytes to ignore from start */
-	int		crd_len;	/* How many bytes to process */
-	int		crd_inject;	/* Where to inject results, if applicable */
-	int		crd_flags;
+#define	CSP_MODE_NONE		0
+#define	CSP_MODE_COMPRESS	1	/* Compression/decompression. */
+#define	CSP_MODE_CIPHER		2	/* Encrypt/decrypt. */
+#define	CSP_MODE_DIGEST		3	/* Compute/verify digest. */
+#define	CSP_MODE_AEAD		4	/* Combined auth/encryption. */
+#define	CSP_MODE_ETA		5	/* IPsec style encrypt-then-auth */
 
-#define	CRD_F_ENCRYPT		0x01	/* Set when doing encryption */
-#define	CRD_F_IV_PRESENT	0x02	/* When encrypting, IV is already in
-					   place, so don't copy. */
-#define	CRD_F_IV_EXPLICIT	0x04	/* IV explicitly provided */
-#define	CRD_F_DSA_SHA_NEEDED	0x08	/* Compute SHA-1 of buffer for DSA */
-#define	CRD_F_COMP		0x0f    /* Set when doing compression */
-#define	CRD_F_KEY_EXPLICIT	0x10	/* Key explicitly provided */
+	int		csp_flags;
 
-	struct cryptoini	CRD_INI; /* Initialization/context data */
-#define	crd_esn		CRD_INI.cri_esn
-#define	crd_iv		CRD_INI.cri_iv
-#define	crd_key		CRD_INI.cri_key
-#define	crd_alg		CRD_INI.cri_alg
-#define	crd_klen	CRD_INI.cri_klen
+	int		csp_ivlen;	 /* IV length in bytes. */
 
-	struct cryptodesc *crd_next;
+	int		csp_cipher_alg;
+	int		csp_cipher_klen; /* Key length in bytes. */
+	const void	*csp_cipher_key;
+
+	int		csp_auth_alg;
+	int		csp_auth_klen;	/* Key length in bytes. */
+	const void	*csp_auth_key;
+	int		csp_auth_mlen;	/* Number of digest bytes to use.
+					   0 means all. */
 };
 
 /* Structure describing complete operation */
 struct cryptop {
 	TAILQ_ENTRY(cryptop) crp_next;
 
 	struct task	crp_task;
 
 	crypto_session_t crp_session;	/* Session */
 	int		crp_ilen;	/* Input data total length */
 	int		crp_olen;	/* Result total length */
 
 	int		crp_etype;	/*
 					 * Error type (zero means no error).
 					 * All error codes except EAGAIN
 					 * indicate possible data corruption (as in,
 					 * the data have been touched). On all
 					 * errors, the crp_session may have changed
 					 * (reset to a new one), so the caller
 					 * should always check and use the new
 					 * value on future requests.
 					 */
 	int		crp_flags;
 
-#define	CRYPTO_F_IMBUF		0x0001	/* Input/output are mbuf chains */
-#define	CRYPTO_F_IOV		0x0002	/* Input/output are uio */
 #define	CRYPTO_F_BATCH		0x0008	/* Batch op if possible */
 #define	CRYPTO_F_CBIMM		0x0010	/* Do callback immediately */
 #define	CRYPTO_F_DONE		0x0020	/* Operation completed */
 #define	CRYPTO_F_CBIFSYNC	0x0040	/* Do CBIMM if op is synchronous */
 #define	CRYPTO_F_ASYNC		0x0080	/* Dispatch crypto jobs on several threads
 					 * if op is synchronous
 					 */
 #define	CRYPTO_F_ASYNC_KEEPORDER	0x0100	/*
 					 * Dispatch the crypto jobs in the same
 					 * order there are submitted. Applied only
 					 * if CRYPTO_F_ASYNC flags is set
 					 */
+#define	CRYPTO_F_IV_SEPARATE	0x0200	/* Use crp_iv[] as IV. */
+#define	CRYPTO_F_IV_GENERATE	0x0400	/* Generate a random IV and store. */
 
+	int		crp_op;
+
 	union {
 		caddr_t		crp_buf;	/* Data to be processed */
 		struct mbuf	*crp_mbuf;
 		struct uio	*crp_uio;
 	};
-	void *		crp_opaque;	/* Opaque pointer, passed along */
-	struct cryptodesc *crp_desc;	/* Linked list of processing descriptors */
+	int		crp_buf_type;	/* Which union member describes data. */
 
+	int		crp_aad_start;	/* Location of AAD. */
+	int		crp_aad_length;	/* 0 => no AAD. */
+	int		crp_iv_start;	/* Location of IV.  IV length is from
+					 * the session.
+					 */
+	int		crp_payload_start; /* Location of ciphertext. */
+	int		crp_payload_length;
+	int		crp_digest_start; /* Location of MAC/tag.  Length is
+					   * from the session.
+					   */
+
+	uint8_t		crp_iv[EALG_MAX_BLOCK_LEN]; /* IV if IV_SEPARATE. */
+
+	const void	*crp_cipher_key; /* New cipher key if non-NULL. */
+	const void	*crp_auth_key;	/* New auth key if non-NULL. */
+
+	void		*crp_opaque;	/* Opaque pointer, passed along */
+
 	int (*crp_callback)(struct cryptop *); /* Callback function */
 
 	struct bintime	crp_tstamp;	/* performance time stamp */
 	uint32_t	crp_seq;	/* used for ordered dispatch */
 	uint32_t	crp_retw_id;	/*
 					 * the return worker to be used,
 					 *  used for ordered dispatch
 					 */
 };
 
 #define	CRYPTOP_ASYNC(crp) \
 	(((crp)->crp_flags & CRYPTO_F_ASYNC) && \
 	crypto_ses2caps((crp)->crp_session) & CRYPTOCAP_F_SYNC)
 #define	CRYPTOP_ASYNC_KEEPORDER(crp) \
 	(CRYPTOP_ASYNC(crp) && \
 	(crp)->crp_flags & CRYPTO_F_ASYNC_KEEPORDER)
 
 #define	CRYPTO_BUF_CONTIG	0x0
-#define	CRYPTO_BUF_IOV		0x1
+#define	CRYPTO_BUF_UIO		0x1
 #define	CRYPTO_BUF_MBUF		0x2
 
-#define	CRYPTO_OP_DECRYPT	0x0
-#define	CRYPTO_OP_ENCRYPT	0x1
+/* Flags in crp_op. */
+#define	CRYPTO_OP_DECRYPT		0x0
+#define	CRYPTO_OP_ENCRYPT		0x1
+#define	CRYPTO_OP_IS_ENCRYPT(op)	((op) & CRYPTO_OP_ENCRYPT)
+#define	CRYPTO_OP_COMPUTE_DIGEST	0x0
+#define	CRYPTO_OP_VERIFY_DIGEST		0x2
+#define	CRYPTO_OP_DECOMPRESS		CRYPTO_OP_DECRYPT
+#define	CRYPTO_OP_COMPRESS		CRYPTO_OP_ENCRYPT
+#define	CRYPTO_OP_IS_COMPRESS(op)	((op) & CRYPTO_OP_COMPRESS)
 
 /*
  * Hints passed to process methods.
  */
 #define	CRYPTO_HINT_MORE	0x1	/* more ops coming shortly */
 
 struct cryptkop {
 	TAILQ_ENTRY(cryptkop) krp_next;
 
 	u_int		krp_op;		/* ie. CRK_MOD_EXP or other */
 	u_int		krp_status;	/* return status */
 	u_short		krp_iparams;	/* # of input parameters */
 	u_short		krp_oparams;	/* # of output parameters */
 	u_int		krp_crid;	/* desired device, etc. */
-	u_int32_t	krp_hid;
+	uint32_t	krp_hid;	/* device used */
 	struct crparam	krp_param[CRK_MAXPARAM];	/* kvm */
-	int		(*krp_callback)(struct cryptkop *);
+	void		(*krp_callback)(struct cryptkop *);
+	struct cryptocap *krp_cap;
 };
 
 uint32_t crypto_ses2hid(crypto_session_t crypto_session);
 uint32_t crypto_ses2caps(crypto_session_t crypto_session);
 void *crypto_get_driver_session(crypto_session_t crypto_session);
+const struct crypto_session_params *crypto_get_params(
+    crypto_session_t crypto_session);
+struct auth_hash *crypto_auth_hash(const struct crypto_session_params *csp);
+struct enc_xform *crypto_cipher(const struct crypto_session_params *csp);
 
 MALLOC_DECLARE(M_CRYPTO_DATA);
 
-extern	int crypto_newsession(crypto_session_t *cses, struct cryptoini *cri, int hard);
+extern	int crypto_newsession(crypto_session_t *cses,
+    const struct crypto_session_params *params, int hard);
 extern	void crypto_freesession(crypto_session_t cses);
 #define	CRYPTOCAP_F_HARDWARE	CRYPTO_FLAG_HARDWARE
 #define	CRYPTOCAP_F_SOFTWARE	CRYPTO_FLAG_SOFTWARE
 #define	CRYPTOCAP_F_SYNC	0x04000000	/* operates synchronously */
 extern	int32_t crypto_get_driverid(device_t dev, size_t session_size,
     int flags);
 extern	int crypto_find_driver(const char *);
 extern	device_t crypto_find_device_byhid(int hid);
 extern	int crypto_getcaps(int hid);
-extern	int crypto_register(u_int32_t driverid, int alg, u_int16_t maxoplen,
-	    u_int32_t flags);
 extern	int crypto_kregister(u_int32_t, int, u_int32_t);
-extern	int crypto_unregister(u_int32_t driverid, int alg);
 extern	int crypto_unregister_all(u_int32_t driverid);
 extern	int crypto_dispatch(struct cryptop *crp);
 extern	int crypto_kdispatch(struct cryptkop *);
 #define	CRYPTO_SYMQ	0x1
 #define	CRYPTO_ASYMQ	0x2
 extern	int crypto_unblock(u_int32_t, int);
 extern	void crypto_done(struct cryptop *crp);
 extern	void crypto_kdone(struct cryptkop *);
 extern	int crypto_getfeat(int *);
 
 extern	void crypto_freereq(struct cryptop *crp);
-extern	struct cryptop *crypto_getreq(int num);
+extern	struct cryptop *crypto_getreq(crypto_session_t cses, int how);
 
 extern	int crypto_usercrypto;		/* userland may do crypto requests */
 extern	int crypto_userasymcrypto;	/* userland may do asym crypto reqs */
 extern	int crypto_devallowsoft;	/* only use hardware crypto */
 
+/* Helper routines for drivers to initialize auth contexts for HMAC. */
+struct auth_hash;
+
+void	hmac_init_ipad(struct auth_hash *axf, const char *key, int klen,
+    void *auth_ctx);
+void	hmac_init_opad(struct auth_hash *axf, const char *key, int klen,
+    void *auth_ctx);
+
 /*
  * Crypto-related utility routines used mainly by drivers.
  *
  * XXX these don't really belong here; but for now they're
  *     kept apart from the rest of the system.
+ *
+ * Similar to m_copyback/data, *_copyback copy data from the 'src'
+ * buffer into the crypto request's data buffer while *_copydata copy
+ * data from the crypto request's data buffer into the the 'dst'
+ * buffer.
  */
 struct uio;
 extern	void cuio_copydata(struct uio* uio, int off, int len, caddr_t cp);
 extern	void cuio_copyback(struct uio* uio, int off, int len, c_caddr_t cp);
 extern	int cuio_getptr(struct uio *uio, int loc, int *off);
 extern	int cuio_apply(struct uio *uio, int off, int len,
 	    int (*f)(void *, void *, u_int), void *arg);
 
 struct mbuf;
 struct iovec;
 extern	int crypto_mbuftoiov(struct mbuf *mbuf, struct iovec **iovptr,
 	    int *cnt, int *allocated);
 
-extern	void crypto_copyback(int flags, caddr_t buf, int off, int size,
-	    c_caddr_t in);
-extern	void crypto_copydata(int flags, caddr_t buf, int off, int size,
-	    caddr_t out);
-extern	int crypto_apply(int flags, caddr_t buf, int off, int len,
+void	crypto_copyback(struct cryptop *crp, int off, int size,
+	    const void *src);
+void	crypto_copydata(struct cryptop *crp, int off, int size, void *dst);
+int	crypto_apply(struct cryptop *crp, int off, int len,
 	    int (*f)(void *, void *, u_int), void *arg);
-
-extern void *crypto_contiguous_subsegment(int, void *, size_t, size_t);
+void	*crypto_contiguous_subsegment(struct cryptop *crp, size_t skip,
+	    size_t len);
 
 #endif /* _KERNEL */
 #endif /* _CRYPTO_CRYPTO_H_ */
Index: head/sys/opencrypto/cryptodev_if.m
===================================================================
--- head/sys/opencrypto/cryptodev_if.m	(revision 359373)
+++ head/sys/opencrypto/cryptodev_if.m	(revision 359374)
@@ -1,73 +1,179 @@
 #-
 # Copyright (c) 2006, Sam Leffler
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
 # are met:
 # 1. Redistributions of source code must retain the above copyright
 #    notice, this list of conditions and the following disclaimer.
 # 2. Redistributions in binary form must reproduce the above copyright
 #    notice, this list of conditions and the following disclaimer in the
 #    documentation and/or other materials provided with the distribution.
 #
 # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 # ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 #
 # $FreeBSD$
 #
 
 #include <sys/malloc.h>
 #include <opencrypto/cryptodev.h>
 
 INTERFACE cryptodev;
 
 CODE {
 	static int null_freesession(device_t dev,
 	    crypto_session_t crypto_session)
 	{
 		return 0;
 	}
 };
 
 /**
- * Crypto driver method to initialize a new session object with the given
- * initialization parameters (cryptoini).  The driver's session memory object
- * is already allocated and zeroed, like driver softcs.  It is accessed with
+ * @brief Probe to see if a crypto driver supports a session.
+ *
+ * The crypto framework invokes this method on each crypto driver when
+ * creating a session for symmetric crypto operations to determine if
+ * the driver supports the algorithms and mode requested by the
+ * session.
+ *
+ * If the driver does not support a session with the requested
+ * parameters, this function should fail with an error.
+ *
+ * If the driver does support a session with the requested parameters,
+ * this function should return a negative value indicating the
+ * priority of this driver.  These negative values should be derived
+ * from one of the CRYPTODEV_PROBE_* constants in
+ * <opencrypto/cryptodev.h>.
+ *
+ * This function's return value is similar to that used by
+ * DEVICE_PROBE(9).  However, a return value of zero is not supported
+ * and should not be used.
+ *
+ * @param dev		the crypto driver device
+ * @param csp		crypto session parameters
+ *
+ * @retval negative	if the driver supports this session - the
+ *			least negative value is used to select the
+ *			driver for the session
+ * @retval EINVAL	if the driver does not support the session
+ * @retval positive	if some other error occurs
+ */
+METHOD int probesession {
+	device_t	dev;
+	const struct crypto_session_params *csp;
+};
+
+/**
+ * @brief Initialize a new crypto session object
+ *
+ * Invoked by the crypto framework to initialize driver-specific data
+ * for a crypto session.  The framework allocates and zeroes the
+ * driver's per-session memory object prior to invoking this method.
+ * The driver is able to access it's per-session memory object via
  * crypto_get_driver_session().
+ *
+ * @param dev		the crypto driver device
+ * @param crypto_session session being initialized
+ * @param csp		crypto session parameters
+ *
+ * @retval 0		success
+ * @retval non-zero	if some kind of error occurred
  */
 METHOD int newsession {
 	device_t	dev;
 	crypto_session_t crypto_session;
-	struct cryptoini *cri;
+	const struct crypto_session_params *csp;
 };
 
 /**
- * Optional crypto driver method to release any additional allocations.  OCF
- * owns session memory itself; it is zeroed before release.
+ * @brief Destroy a crypto session object
+ *
+ * The crypto framework invokes this method when tearing down a crypto
+ * session.  After this callback returns, the frame will explicitly
+ * zero and free the drvier's per-session memory object.  If the
+ * driver requires additional actions to destroy a session, it should
+ * perform those in this method.  If the driver does not require
+ * additional actions it does not need to provide an implementation of
+ * this method.
+ *
+ * @param dev		the crypto driver device
+ * @param crypto_session session being destroyed
  */
 METHOD void freesession {
 	device_t	dev;
 	crypto_session_t crypto_session;
 } DEFAULT null_freesession;
 
+/**
+ * @brief Perform a symmetric crypto operation
+ *
+ * The crypto framework invokes this method for each symmetric crypto
+ * operation performed on a session.  A reference to the containing
+ * session is stored as a member of 'struct cryptop'.  This routine
+ * should not block, but queue the operation if necessary.
+ *
+ * This method may return ERESTART to indicate that any internal
+ * queues are full so the operation should be queued in the crypto
+ * framework and retried in the future.
+ *
+ * To report errors with a crypto operation, 'crp_etype' should be set
+ * and the operation completed by calling 'crypto_done'.  This method
+ * should then return zero.
+ *
+ * @param dev		the crypto driver device
+ * @param op		crypto operation to perform
+ * @param flags		set to CRYPTO_HINT_MORE if additional symmetric
+ *			crypto operations are queued for this driver;
+ *			otherwise set to zero.
+ *
+ * @retval 0		success
+ * @retval ERESTART	internal queue is full
+ */
 METHOD int process {
 	device_t	dev;
 	struct cryptop	*op;
 	int		flags;
 };
 
+/**
+ * @brief Perform an asymmetric crypto operation
+ *
+ * The crypto framework invokes this method for each asymmetric crypto
+ * operation.  Each asymmetric crypto operation should be
+ * self-contained and is not assicated with any persistent session.
+ * This routine should not block, but queue the operation if
+ * necessary.
+ *
+ * This method may return ERESTART to indicate that any internal
+ * queues are full so the operation should be queued in the crypto
+ * framework and retried in the future.
+ *
+ * To report errors with a crypto operation, 'krp_status' should be set
+ * and the operation completed by calling 'crypto_kdone'.  This method
+ * should then return zero.
+ *
+ * @param dev		the crypto driver device
+ * @param op		crypto operation to perform
+ * @param flags		set to CRYPTO_HINT_MORE if additional asymmetric
+ *			crypto operations are queued for this driver;
+ *			otherwise set to zero.
+ *
+ * @retval 0		success
+ * @retval ERESTART	internal queue is full
+ */
 METHOD int kprocess {
 	device_t	dev;
 	struct cryptkop	*op;
 	int		flags;
 };
Index: head/sys/opencrypto/cryptosoft.c
===================================================================
--- head/sys/opencrypto/cryptosoft.c	(revision 359373)
+++ head/sys/opencrypto/cryptosoft.c	(revision 359374)
@@ -1,1418 +1,1519 @@
 /*	$OpenBSD: cryptosoft.c,v 1.35 2002/04/26 08:43:50 deraadt Exp $	*/
 
 /*-
  * The author of this code is Angelos D. Keromytis (angelos@cis.upenn.edu)
  * Copyright (c) 2002-2006 Sam Leffler, Errno Consulting
  *
  * This code was written by Angelos D. Keromytis in Athens, Greece, in
  * February 2000. Network Security Technologies Inc. (NSTI) kindly
  * supported the development of this code.
  *
  * Copyright (c) 2000, 2001 Angelos D. Keromytis
  * Copyright (c) 2014 The FreeBSD Foundation
  * All rights reserved.
  *
  * Portions of this software were developed by John-Mark Gurney
  * under sponsorship of the FreeBSD Foundation and
  * Rubicon Communications, LLC (Netgate).
  *
  * Permission to use, copy, and modify this software with or without fee
  * is hereby granted, provided that this entire notice is included in
  * all source code copies of any software which is or includes a copy or
  * modification of this software.
  *
  * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR
  * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY
  * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE
  * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR
  * PURPOSE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/malloc.h>
 #include <sys/mbuf.h>
 #include <sys/module.h>
 #include <sys/sysctl.h>
 #include <sys/errno.h>
 #include <sys/random.h>
 #include <sys/kernel.h>
 #include <sys/uio.h>
 #include <sys/lock.h>
 #include <sys/rwlock.h>
 #include <sys/endian.h>
 #include <sys/limits.h>
 #include <sys/mutex.h>
 
 #include <crypto/blowfish/blowfish.h>
 #include <crypto/sha1.h>
 #include <opencrypto/rmd160.h>
 #include <opencrypto/cast.h>
 #include <opencrypto/skipjack.h>
 #include <sys/md5.h>
 
 #include <opencrypto/cryptodev.h>
-#include <opencrypto/cryptosoft.h>
 #include <opencrypto/xform.h>
 
 #include <sys/kobj.h>
 #include <sys/bus.h>
 #include "cryptodev_if.h"
 
-_Static_assert(AES_CCM_IV_LEN == AES_GCM_IV_LEN,
-    "AES_GCM_IV_LEN must currently be the same as AES_CCM_IV_LEN");
+struct swcr_auth {
+	void		*sw_ictx;
+	void		*sw_octx;
+	struct auth_hash *sw_axf;
+	uint16_t	sw_mlen;
+	uint16_t	sw_octx_len;
+};
 
-static	int32_t swcr_id;
+struct swcr_encdec {
+	uint8_t		*sw_kschedule;
+	struct enc_xform *sw_exf;
+};
 
-u_int8_t hmac_ipad_buffer[HMAC_MAX_BLOCK_LEN];
-u_int8_t hmac_opad_buffer[HMAC_MAX_BLOCK_LEN];
+struct swcr_compdec {
+	struct comp_algo *sw_cxf;
+};
 
-static	int swcr_encdec(struct cryptodesc *, struct swcr_data *, caddr_t, int);
-static	int swcr_authcompute(struct cryptodesc *, struct swcr_data *, caddr_t, int);
-static	int swcr_authenc(struct cryptop *crp);
-static	int swcr_compdec(struct cryptodesc *, struct swcr_data *, caddr_t, int);
+struct swcr_session {
+	struct mtx	swcr_lock;
+	int	(*swcr_process)(struct swcr_session *, struct cryptop *);
+
+	struct swcr_auth swcr_auth;
+	struct swcr_encdec swcr_encdec;
+	struct swcr_compdec swcr_compdec;
+};
+
+static	int32_t swcr_id;
+
 static	void swcr_freesession(device_t dev, crypto_session_t cses);
 
+/* Used for CRYPTO_NULL_CBC. */
+static int
+swcr_null(struct swcr_session *ses, struct cryptop *crp)
+{
+
+	return (0);
+}
+
 /*
  * Apply a symmetric encryption/decryption algorithm.
  */
 static int
-swcr_encdec(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf,
-    int flags)
+swcr_encdec(struct swcr_session *ses, struct cryptop *crp)
 {
 	unsigned char iv[EALG_MAX_BLOCK_LEN], blk[EALG_MAX_BLOCK_LEN];
 	unsigned char *ivp, *nivp, iv2[EALG_MAX_BLOCK_LEN];
+	const struct crypto_session_params *csp;
+	struct swcr_encdec *sw;
 	struct enc_xform *exf;
 	int i, j, k, blks, ind, count, ivlen;
 	struct uio *uio, uiolcl;
 	struct iovec iovlcl[4];
 	struct iovec *iov;
 	int iovcnt, iovalloc;
 	int error;
+	bool encrypting;
 
 	error = 0;
 
+	sw = &ses->swcr_encdec;
 	exf = sw->sw_exf;
 	blks = exf->blocksize;
 	ivlen = exf->ivsize;
 
 	/* Check for non-padded data */
-	if (crd->crd_len % blks)
+	if ((crp->crp_payload_length % blks) != 0)
 		return EINVAL;
 
-	if (crd->crd_alg == CRYPTO_AES_ICM &&
-	    (crd->crd_flags & CRD_F_IV_EXPLICIT) == 0)
+	if (exf == &enc_xform_aes_icm &&
+	    (crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0)
 		return (EINVAL);
 
-	/* Initialize the IV */
-	if (crd->crd_flags & CRD_F_ENCRYPT) {
-		/* IV explicitly provided ? */
-		if (crd->crd_flags & CRD_F_IV_EXPLICIT)
-			bcopy(crd->crd_iv, iv, ivlen);
-		else
-			arc4rand(iv, ivlen, 0);
+	/* IV explicitly provided ? */
+	if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+		bcopy(crp->crp_iv, iv, ivlen);
+	else if (crp->crp_flags & CRYPTO_F_IV_GENERATE) {
+		arc4rand(iv, ivlen, 0);
+		crypto_copyback(crp, crp->crp_iv_start, ivlen, iv);
+	} else
+		crypto_copydata(crp, crp->crp_iv_start, ivlen, iv);
 
-		/* Do we need to write the IV */
-		if (!(crd->crd_flags & CRD_F_IV_PRESENT))
-			crypto_copyback(flags, buf, crd->crd_inject, ivlen, iv);
-
-	} else {	/* Decryption */
-		/* IV explicitly provided ? */
-		if (crd->crd_flags & CRD_F_IV_EXPLICIT)
-			bcopy(crd->crd_iv, iv, ivlen);
-		else {
-			/* Get IV off buf */
-			crypto_copydata(flags, buf, crd->crd_inject, ivlen, iv);
-		}
-	}
-
-	if (crd->crd_flags & CRD_F_KEY_EXPLICIT) {
-		int error; 
-
+	if (crp->crp_cipher_key != NULL) {
 		if (sw->sw_kschedule)
 			exf->zerokey(&(sw->sw_kschedule));
 
+		csp = crypto_get_params(crp->crp_session);
 		error = exf->setkey(&sw->sw_kschedule,
-				crd->crd_key, crd->crd_klen / 8);
+		    crp->crp_cipher_key, csp->csp_cipher_klen);
 		if (error)
 			return (error);
 	}
 
 	iov = iovlcl;
 	iovcnt = nitems(iovlcl);
 	iovalloc = 0;
 	uio = &uiolcl;
-	if ((flags & CRYPTO_F_IMBUF) != 0) {
-		error = crypto_mbuftoiov((struct mbuf *)buf, &iov, &iovcnt,
+	switch (crp->crp_buf_type) {
+	case CRYPTO_BUF_MBUF:
+		error = crypto_mbuftoiov(crp->crp_mbuf, &iov, &iovcnt,
 		    &iovalloc);
 		if (error)
 			return (error);
 		uio->uio_iov = iov;
 		uio->uio_iovcnt = iovcnt;
-	} else if ((flags & CRYPTO_F_IOV) != 0)
-		uio = (struct uio *)buf;
-	else {
-		iov[0].iov_base = buf;
-		iov[0].iov_len = crd->crd_skip + crd->crd_len;
+		break;
+	case CRYPTO_BUF_UIO:
+		uio = crp->crp_uio;
+		break;
+	case CRYPTO_BUF_CONTIG:
+		iov[0].iov_base = crp->crp_buf;
+		iov[0].iov_len = crp->crp_ilen;
 		uio->uio_iov = iov;
 		uio->uio_iovcnt = 1;
+		break;
 	}
 
 	ivp = iv;
 
 	if (exf->reinit) {
 		/*
 		 * xforms that provide a reinit method perform all IV
 		 * handling themselves.
 		 */
 		exf->reinit(sw->sw_kschedule, iv);
 	}
 
-	count = crd->crd_skip;
+	count = crp->crp_payload_start;
 	ind = cuio_getptr(uio, count, &k);
 	if (ind == -1) {
 		error = EINVAL;
 		goto out;
 	}
 
-	i = crd->crd_len;
+	i = crp->crp_payload_length;
+	encrypting = CRYPTO_OP_IS_ENCRYPT(crp->crp_op);
 
 	while (i > 0) {
 		/*
 		 * If there's insufficient data at the end of
 		 * an iovec, we have to do some copying.
 		 */
 		if (uio->uio_iov[ind].iov_len < k + blks &&
 		    uio->uio_iov[ind].iov_len != k) {
 			cuio_copydata(uio, count, blks, blk);
 
 			/* Actual encryption/decryption */
 			if (exf->reinit) {
-				if (crd->crd_flags & CRD_F_ENCRYPT) {
+				if (encrypting) {
 					exf->encrypt(sw->sw_kschedule,
 					    blk);
 				} else {
 					exf->decrypt(sw->sw_kschedule,
 					    blk);
 				}
-			} else if (crd->crd_flags & CRD_F_ENCRYPT) {
+			} else if (encrypting) {
 				/* XOR with previous block */
 				for (j = 0; j < blks; j++)
 					blk[j] ^= ivp[j];
 
 				exf->encrypt(sw->sw_kschedule, blk);
 
 				/*
 				 * Keep encrypted block for XOR'ing
 				 * with next block
 				 */
 				bcopy(blk, iv, blks);
 				ivp = iv;
 			} else {	/* decrypt */
 				/*	
 				 * Keep encrypted block for XOR'ing
 				 * with next block
 				 */
 				nivp = (ivp == iv) ? iv2 : iv;
 				bcopy(blk, nivp, blks);
 
 				exf->decrypt(sw->sw_kschedule, blk);
 
 				/* XOR with previous block */
 				for (j = 0; j < blks; j++)
 					blk[j] ^= ivp[j];
 
 				ivp = nivp;
 			}
 
 			/* Copy back decrypted block */
 			cuio_copyback(uio, count, blks, blk);
 
 			count += blks;
 
 			/* Advance pointer */
 			ind = cuio_getptr(uio, count, &k);
 			if (ind == -1) {
 				error = EINVAL;
 				goto out;
 			}
 
 			i -= blks;
 
 			/* Could be done... */
 			if (i == 0)
 				break;
 		}
 
 		while (uio->uio_iov[ind].iov_len >= k + blks && i > 0) {
 			uint8_t *idat;
 			size_t nb, rem;
 
 			nb = blks;
 			rem = MIN((size_t)i,
 			    uio->uio_iov[ind].iov_len - (size_t)k);
 			idat = (uint8_t *)uio->uio_iov[ind].iov_base + k;
 
 			if (exf->reinit) {
-				if ((crd->crd_flags & CRD_F_ENCRYPT) != 0 &&
-				    exf->encrypt_multi == NULL)
+				if (encrypting && exf->encrypt_multi == NULL)
 					exf->encrypt(sw->sw_kschedule,
 					    idat);
-				else if ((crd->crd_flags & CRD_F_ENCRYPT) != 0) {
+				else if (encrypting) {
 					nb = rounddown(rem, blks);
 					exf->encrypt_multi(sw->sw_kschedule,
 					    idat, nb);
 				} else if (exf->decrypt_multi == NULL)
 					exf->decrypt(sw->sw_kschedule,
 					    idat);
 				else {
 					nb = rounddown(rem, blks);
 					exf->decrypt_multi(sw->sw_kschedule,
 					    idat, nb);
 				}
-			} else if (crd->crd_flags & CRD_F_ENCRYPT) {
+			} else if (encrypting) {
 				/* XOR with previous block/IV */
 				for (j = 0; j < blks; j++)
 					idat[j] ^= ivp[j];
 
 				exf->encrypt(sw->sw_kschedule, idat);
 				ivp = idat;
 			} else {	/* decrypt */
 				/*
 				 * Keep encrypted block to be used
 				 * in next block's processing.
 				 */
 				nivp = (ivp == iv) ? iv2 : iv;
 				bcopy(idat, nivp, blks);
 
 				exf->decrypt(sw->sw_kschedule, idat);
 
 				/* XOR with previous block/IV */
 				for (j = 0; j < blks; j++)
 					idat[j] ^= ivp[j];
 
 				ivp = nivp;
 			}
 
 			count += nb;
 			k += nb;
 			i -= nb;
 		}
 
 		/*
 		 * Advance to the next iov if the end of the current iov
 		 * is aligned with the end of a cipher block.
 		 * Note that the code is equivalent to calling:
 		 *      ind = cuio_getptr(uio, count, &k);
 		 */
 		if (i > 0 && k == uio->uio_iov[ind].iov_len) {
 			k = 0;
 			ind++;
 			if (ind >= uio->uio_iovcnt) {
 				error = EINVAL;
 				goto out;
 			}
 		}
 	}
 
 out:
 	if (iovalloc)
 		free(iov, M_CRYPTO_DATA);
 
 	return (error);
 }
 
-static int __result_use_check
-swcr_authprepare(struct auth_hash *axf, struct swcr_data *sw, u_char *key,
-    int klen)
+static void
+swcr_authprepare(struct auth_hash *axf, struct swcr_auth *sw,
+    const uint8_t *key, int klen)
 {
-	int k;
 
-	klen /= 8;
-
 	switch (axf->type) {
 	case CRYPTO_MD5_HMAC:
 	case CRYPTO_SHA1_HMAC:
 	case CRYPTO_SHA2_224_HMAC:
 	case CRYPTO_SHA2_256_HMAC:
 	case CRYPTO_SHA2_384_HMAC:
 	case CRYPTO_SHA2_512_HMAC:
 	case CRYPTO_NULL_HMAC:
 	case CRYPTO_RIPEMD160_HMAC:
-		for (k = 0; k < klen; k++)
-			key[k] ^= HMAC_IPAD_VAL;
-	
-		axf->Init(sw->sw_ictx);
-		axf->Update(sw->sw_ictx, key, klen);
-		axf->Update(sw->sw_ictx, hmac_ipad_buffer, axf->blocksize - klen);
-	
-		for (k = 0; k < klen; k++)
-			key[k] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
-	
-		axf->Init(sw->sw_octx);
-		axf->Update(sw->sw_octx, key, klen);
-		axf->Update(sw->sw_octx, hmac_opad_buffer, axf->blocksize - klen);
-	
-		for (k = 0; k < klen; k++)
-			key[k] ^= HMAC_OPAD_VAL;
+		hmac_init_ipad(axf, key, klen, sw->sw_ictx);
+		hmac_init_opad(axf, key, klen, sw->sw_octx);
 		break;
 	case CRYPTO_MD5_KPDK:
 	case CRYPTO_SHA1_KPDK:
 	{
 		/* 
 		 * We need a buffer that can hold an md5 and a sha1 result
 		 * just to throw it away.
 		 * What we do here is the initial part of:
 		 *   ALGO( key, keyfill, .. )
 		 * adding the key to sw_ictx and abusing Final() to get the
 		 * "keyfill" padding.
 		 * In addition we abuse the sw_octx to save the key to have
 		 * it to be able to append it at the end in swcr_authcompute().
 		 */
 		u_char buf[SHA1_RESULTLEN];
 
-		sw->sw_klen = klen;
 		bcopy(key, sw->sw_octx, klen);
 		axf->Init(sw->sw_ictx);
 		axf->Update(sw->sw_ictx, key, klen);
 		axf->Final(buf, sw->sw_ictx);
 		break;
 	}
 	case CRYPTO_POLY1305:
-		if (klen != POLY1305_KEY_LEN) {
-			CRYPTDEB("bad poly1305 key size %d", klen);
-			return EINVAL;
-		}
-		/* FALLTHROUGH */
 	case CRYPTO_BLAKE2B:
 	case CRYPTO_BLAKE2S:
 		axf->Setkey(sw->sw_ictx, key, klen);
 		axf->Init(sw->sw_ictx);
 		break;
 	default:
-		printf("%s: CRD_F_KEY_EXPLICIT flag given, but algorithm %d "
-		    "doesn't use keys.\n", __func__, axf->type);
-		return EINVAL;
+		panic("%s: algorithm %d doesn't use keys", __func__, axf->type);
 	}
-	return 0;
 }
 
 /*
- * Compute keyed-hash authenticator.
+ * Compute or verify hash.
  */
 static int
-swcr_authcompute(struct cryptodesc *crd, struct swcr_data *sw, caddr_t buf,
-    int flags)
+swcr_authcompute(struct swcr_session *ses, struct cryptop *crp)
 {
-	unsigned char aalg[HASH_MAX_LEN];
+	u_char aalg[HASH_MAX_LEN];
+	u_char uaalg[HASH_MAX_LEN];
+	const struct crypto_session_params *csp;
+	struct swcr_auth *sw;
 	struct auth_hash *axf;
 	union authctx ctx;
 	int err;
 
-	if (sw->sw_ictx == 0)
-		return EINVAL;
+	sw = &ses->swcr_auth;
 
 	axf = sw->sw_axf;
 
-	if (crd->crd_flags & CRD_F_KEY_EXPLICIT) {
-		err = swcr_authprepare(axf, sw, crd->crd_key, crd->crd_klen);
-		if (err != 0)
-			return err;
+	if (crp->crp_auth_key != NULL) {
+		csp = crypto_get_params(crp->crp_session);
+		swcr_authprepare(axf, sw, crp->crp_auth_key,
+		    csp->csp_auth_klen);
 	}
 
 	bcopy(sw->sw_ictx, &ctx, axf->ctxsize);
 
-	err = crypto_apply(flags, buf, crd->crd_skip, crd->crd_len,
-	    (int (*)(void *, void *, unsigned int))axf->Update, (caddr_t)&ctx);
+	err = crypto_apply(crp, crp->crp_aad_start, crp->crp_aad_length,
+	    (int (*)(void *, void *, unsigned int))axf->Update, &ctx);
 	if (err)
 		return err;
 
-	switch (sw->sw_alg) {
+	err = crypto_apply(crp, crp->crp_payload_start, crp->crp_payload_length,
+	    (int (*)(void *, void *, unsigned int))axf->Update, &ctx);
+	if (err)
+		return err;
+
+	switch (axf->type) {
 	case CRYPTO_SHA1:
 	case CRYPTO_SHA2_224:
 	case CRYPTO_SHA2_256:
 	case CRYPTO_SHA2_384:
 	case CRYPTO_SHA2_512:
 		axf->Final(aalg, &ctx);
 		break;
 
 	case CRYPTO_MD5_HMAC:
 	case CRYPTO_SHA1_HMAC:
 	case CRYPTO_SHA2_224_HMAC:
 	case CRYPTO_SHA2_256_HMAC:
 	case CRYPTO_SHA2_384_HMAC:
 	case CRYPTO_SHA2_512_HMAC:
 	case CRYPTO_RIPEMD160_HMAC:
 		if (sw->sw_octx == NULL)
 			return EINVAL;
 
 		axf->Final(aalg, &ctx);
 		bcopy(sw->sw_octx, &ctx, axf->ctxsize);
 		axf->Update(&ctx, aalg, axf->hashsize);
 		axf->Final(aalg, &ctx);
 		break;
 
 	case CRYPTO_MD5_KPDK:
 	case CRYPTO_SHA1_KPDK:
 		/* If we have no key saved, return error. */
 		if (sw->sw_octx == NULL)
 			return EINVAL;
 
 		/*
 		 * Add the trailing copy of the key (see comment in
 		 * swcr_authprepare()) after the data:
 		 *   ALGO( .., key, algofill )
 		 * and let Final() do the proper, natural "algofill"
 		 * padding.
 		 */
-		axf->Update(&ctx, sw->sw_octx, sw->sw_klen);
+		axf->Update(&ctx, sw->sw_octx, sw->sw_octx_len);
 		axf->Final(aalg, &ctx);
 		break;
 
 	case CRYPTO_BLAKE2B:
 	case CRYPTO_BLAKE2S:
 	case CRYPTO_NULL_HMAC:
 	case CRYPTO_POLY1305:
 		axf->Final(aalg, &ctx);
 		break;
 	}
 
-	/* Inject the authentication data */
-	crypto_copyback(flags, buf, crd->crd_inject,
-	    sw->sw_mlen == 0 ? axf->hashsize : sw->sw_mlen, aalg);
-	return 0;
+	if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
+		crypto_copydata(crp, crp->crp_digest_start, sw->sw_mlen, uaalg);
+		if (timingsafe_bcmp(aalg, uaalg, sw->sw_mlen) != 0)
+			return (EBADMSG);
+	} else {
+		/* Inject the authentication data */
+		crypto_copyback(crp, crp->crp_digest_start, sw->sw_mlen, aalg);
+	}
+	return (0);
 }
 
 CTASSERT(INT_MAX <= (1ll<<39) - 256);	/* GCM: plain text < 2^39-256 */
 CTASSERT(INT_MAX <= (uint64_t)-1);	/* GCM: associated data <= 2^64-1 */
 
-/*
- * Apply a combined encryption-authentication transformation
- */
 static int
-swcr_authenc(struct cryptop *crp)
+swcr_gmac(struct swcr_session *ses, struct cryptop *crp)
 {
 	uint32_t blkbuf[howmany(EALG_MAX_BLOCK_LEN, sizeof(uint32_t))];
 	u_char *blk = (u_char *)blkbuf;
 	u_char aalg[AALG_MAX_RESULT_LEN];
 	u_char uaalg[AALG_MAX_RESULT_LEN];
 	u_char iv[EALG_MAX_BLOCK_LEN];
 	union authctx ctx;
-	struct swcr_session *ses;
-	struct cryptodesc *crd, *crda = NULL, *crde = NULL;
-	struct swcr_data *sw, *swa, *swe = NULL;
-	struct auth_hash *axf = NULL;
-	struct enc_xform *exf = NULL;
-	caddr_t buf = (caddr_t)crp->crp_buf;
+	struct swcr_auth *swa;
+	struct auth_hash *axf;
 	uint32_t *blkp;
-	int aadlen, blksz, i, ivlen, len, iskip, oskip, r;
-	int isccm = 0;
+	int blksz, i, ivlen, len;
 
-	ivlen = blksz = iskip = oskip = 0;
+	swa = &ses->swcr_auth;
+	axf = swa->sw_axf;
 
-	ses = crypto_get_driver_session(crp->crp_session);
+	bcopy(swa->sw_ictx, &ctx, axf->ctxsize);
+	blksz = axf->blocksize;
 
-	for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
-		for (i = 0; i < nitems(ses->swcr_algorithms) &&
-		    ses->swcr_algorithms[i].sw_alg != crd->crd_alg; i++)
-			;
-		if (i == nitems(ses->swcr_algorithms))
-			return (EINVAL);
+	if (crp->crp_flags & CRYPTO_F_IV_GENERATE)
+		return (EINVAL);
 
-		sw = &ses->swcr_algorithms[i];
-		switch (sw->sw_alg) {
-		case CRYPTO_AES_CCM_16:
-		case CRYPTO_AES_NIST_GCM_16:
-		case CRYPTO_AES_NIST_GMAC:
-			swe = sw;
-			crde = crd;
-			exf = swe->sw_exf;
-			/* AES_CCM_IV_LEN and AES_GCM_IV_LEN are both 12 */
-			ivlen = AES_CCM_IV_LEN;
-			break;
-		case CRYPTO_AES_CCM_CBC_MAC:
-			isccm = 1;
-			/* FALLTHROUGH */
-		case CRYPTO_AES_128_NIST_GMAC:
-		case CRYPTO_AES_192_NIST_GMAC:
-		case CRYPTO_AES_256_NIST_GMAC:
-			swa = sw;
-			crda = crd;
-			axf = swa->sw_axf;
-			if (swa->sw_ictx == 0)
-				return (EINVAL);
-			bcopy(swa->sw_ictx, &ctx, axf->ctxsize);
-			blksz = axf->blocksize;
-			break;
-		default:
-			return (EINVAL);
-		}
+	/* Initialize the IV */
+	ivlen = AES_GCM_IV_LEN;
+	if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+		bcopy(crp->crp_iv, iv, ivlen);
+	else
+		crypto_copydata(crp, crp->crp_iv_start, ivlen, iv);
+
+	axf->Reinit(&ctx, iv, ivlen);
+	for (i = 0; i < crp->crp_payload_length; i += blksz) {
+		len = MIN(crp->crp_payload_length - i, blksz);
+		crypto_copydata(crp, crp->crp_payload_start + i, len, blk);
+		bzero(blk + len, blksz - len);
+		axf->Update(&ctx, blk, blksz);
 	}
-	if (crde == NULL || crda == NULL)
+
+	/* length block */
+	bzero(blk, blksz);
+	blkp = (uint32_t *)blk + 1;
+	*blkp = htobe32(crp->crp_payload_length * 8);
+	axf->Update(&ctx, blk, blksz);
+
+	/* Finalize MAC */
+	axf->Final(aalg, &ctx);
+
+	if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
+		crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen,
+		    uaalg);
+		if (timingsafe_bcmp(aalg, uaalg, swa->sw_mlen) != 0)
+			return (EBADMSG);
+	} else {
+		/* Inject the authentication data */
+		crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, aalg);
+	}
+	return (0);
+}
+
+static int
+swcr_gcm(struct swcr_session *ses, struct cryptop *crp)
+{
+	uint32_t blkbuf[howmany(EALG_MAX_BLOCK_LEN, sizeof(uint32_t))];
+	u_char *blk = (u_char *)blkbuf;
+	u_char aalg[AALG_MAX_RESULT_LEN];
+	u_char uaalg[AALG_MAX_RESULT_LEN];
+	u_char iv[EALG_MAX_BLOCK_LEN];
+	union authctx ctx;
+	struct swcr_auth *swa;
+	struct swcr_encdec *swe;
+	struct auth_hash *axf;
+	struct enc_xform *exf;
+	uint32_t *blkp;
+	int blksz, i, ivlen, len, r;
+
+	swa = &ses->swcr_auth;
+	axf = swa->sw_axf;
+
+	bcopy(swa->sw_ictx, &ctx, axf->ctxsize);
+	blksz = axf->blocksize;
+	
+	swe = &ses->swcr_encdec;
+	exf = swe->sw_exf;
+
+	if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0)
 		return (EINVAL);
-	/*
-	 * We need to make sure that the auth algorithm matches the
-	 * encr algorithm.  Specifically, for AES-GCM must go with
-	 * AES NIST GMAC, and AES-CCM must go with CBC-MAC.
-	 */
-	if (crde->crd_alg == CRYPTO_AES_NIST_GCM_16) {
-		switch (crda->crd_alg) {
-		case CRYPTO_AES_128_NIST_GMAC:
-		case CRYPTO_AES_192_NIST_GMAC:
-		case CRYPTO_AES_256_NIST_GMAC:
-			break;	/* Good! */
-		default:
-			return (EINVAL);	/* Not good! */
+
+	/* Initialize the IV */
+	ivlen = AES_GCM_IV_LEN;
+	bcopy(crp->crp_iv, iv, ivlen);
+
+	/* Supply MAC with IV */
+	axf->Reinit(&ctx, iv, ivlen);
+
+	/* Supply MAC with AAD */
+	for (i = 0; i < crp->crp_aad_length; i += blksz) {
+		len = MIN(crp->crp_aad_length - i, blksz);
+		crypto_copydata(crp, crp->crp_aad_start + i, len, blk);
+		bzero(blk + len, blksz - len);
+		axf->Update(&ctx, blk, blksz);
+	}
+
+	exf->reinit(swe->sw_kschedule, iv);
+
+	/* Do encryption with MAC */
+	for (i = 0; i < crp->crp_payload_length; i += len) {
+		len = MIN(crp->crp_payload_length - i, blksz);
+		if (len < blksz)
+			bzero(blk, blksz);
+		crypto_copydata(crp, crp->crp_payload_start + i, len, blk);
+		if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
+			exf->encrypt(swe->sw_kschedule, blk);
+			axf->Update(&ctx, blk, len);
+			crypto_copyback(crp, crp->crp_payload_start + i, len,
+			    blk);
+		} else {
+			axf->Update(&ctx, blk, len);
 		}
-	} else if (crde->crd_alg == CRYPTO_AES_CCM_16 &&
-	    crda->crd_alg != CRYPTO_AES_CCM_CBC_MAC)
-		return (EINVAL);
+	}
 
-	if ((crde->crd_alg == CRYPTO_AES_NIST_GCM_16 ||
-	     crde->crd_alg == CRYPTO_AES_CCM_16) &&
-	    (crde->crd_flags & CRD_F_IV_EXPLICIT) == 0)
-		return (EINVAL);
+	/* length block */
+	bzero(blk, blksz);
+	blkp = (uint32_t *)blk + 1;
+	*blkp = htobe32(crp->crp_aad_length * 8);
+	blkp = (uint32_t *)blk + 3;
+	*blkp = htobe32(crp->crp_payload_length * 8);
+	axf->Update(&ctx, blk, blksz);
 
-	if (crde->crd_klen != crda->crd_klen)
+	/* Finalize MAC */
+	axf->Final(aalg, &ctx);
+
+	/* Validate tag */
+	if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
+		crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen,
+		    uaalg);
+
+		r = timingsafe_bcmp(aalg, uaalg, swa->sw_mlen);
+		if (r != 0)
+			return (EBADMSG);
+		
+		/* tag matches, decrypt data */
+		for (i = 0; i < crp->crp_payload_length; i += blksz) {
+			len = MIN(crp->crp_payload_length - i, blksz);
+			if (len < blksz)
+				bzero(blk, blksz);
+			crypto_copydata(crp, crp->crp_payload_start + i, len,
+			    blk);
+			exf->decrypt(swe->sw_kschedule, blk);
+			crypto_copyback(crp, crp->crp_payload_start + i, len,
+			    blk);
+		}
+	} else {
+		/* Inject the authentication data */
+		crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen,
+		    aalg);
+	}
+
+	return (0);
+}
+
+static int
+swcr_ccm_cbc_mac(struct swcr_session *ses, struct cryptop *crp)
+{
+	uint32_t blkbuf[howmany(EALG_MAX_BLOCK_LEN, sizeof(uint32_t))];
+	u_char *blk = (u_char *)blkbuf;
+	u_char aalg[AALG_MAX_RESULT_LEN];
+	u_char uaalg[AALG_MAX_RESULT_LEN];
+	u_char iv[EALG_MAX_BLOCK_LEN];
+	union authctx ctx;
+	struct swcr_auth *swa;
+	struct auth_hash *axf;
+	int blksz, i, ivlen, len;
+
+	swa = &ses->swcr_auth;
+	axf = swa->sw_axf;
+
+	bcopy(swa->sw_ictx, &ctx, axf->ctxsize);
+	blksz = axf->blocksize;
+
+	if (crp->crp_flags & CRYPTO_F_IV_GENERATE)
 		return (EINVAL);
 
 	/* Initialize the IV */
-	if (crde->crd_flags & CRD_F_ENCRYPT) {
-		/* IV explicitly provided ? */
-		if (crde->crd_flags & CRD_F_IV_EXPLICIT)
-			bcopy(crde->crd_iv, iv, ivlen);
-		else
-			arc4rand(iv, ivlen, 0);
+	ivlen = AES_CCM_IV_LEN;
+	if (crp->crp_flags & CRYPTO_F_IV_SEPARATE)
+		bcopy(crp->crp_iv, iv, ivlen);
+	else
+		crypto_copydata(crp, crp->crp_iv_start, ivlen, iv);
 
-		/* Do we need to write the IV */
-		if (!(crde->crd_flags & CRD_F_IV_PRESENT))
-			crypto_copyback(crp->crp_flags, buf, crde->crd_inject,
-			    ivlen, iv);
+	/*
+	 * AES CCM-CBC-MAC needs to know the length of both the auth
+	 * data and payload data before doing the auth computation.
+	 */
+	ctx.aes_cbc_mac_ctx.authDataLength = crp->crp_payload_length;
+	ctx.aes_cbc_mac_ctx.cryptDataLength = 0;
 
-	} else {	/* Decryption */
-			/* IV explicitly provided ? */
-		if (crde->crd_flags & CRD_F_IV_EXPLICIT)
-			bcopy(crde->crd_iv, iv, ivlen);
-		else {
-			/* Get IV off buf */
-			crypto_copydata(crp->crp_flags, buf, crde->crd_inject,
-			    ivlen, iv);
-		}
+	axf->Reinit(&ctx, iv, ivlen);
+	for (i = 0; i < crp->crp_payload_length; i += blksz) {
+		len = MIN(crp->crp_payload_length - i, blksz);
+		crypto_copydata(crp, crp->crp_payload_start + i, len, blk);
+		bzero(blk + len, blksz - len);
+		axf->Update(&ctx, blk, blksz);
 	}
 
-	if (swa->sw_alg == CRYPTO_AES_CCM_CBC_MAC) {
-		/*
-		 * AES CCM-CBC needs to know the length of
-		 * both the auth data, and payload data, before
-		 * doing the auth computation.
-		 */
-		ctx.aes_cbc_mac_ctx.authDataLength = crda->crd_len;
-		ctx.aes_cbc_mac_ctx.cryptDataLength = crde->crd_len;
+	/* Finalize MAC */
+	axf->Final(aalg, &ctx);
+
+	if (crp->crp_op & CRYPTO_OP_VERIFY_DIGEST) {
+		crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen,
+		    uaalg);
+		if (timingsafe_bcmp(aalg, uaalg, swa->sw_mlen) != 0)
+			return (EBADMSG);
+	} else {
+		/* Inject the authentication data */
+		crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen, aalg);
 	}
+	return (0);
+}
+
+static int
+swcr_ccm(struct swcr_session *ses, struct cryptop *crp)
+{
+	uint32_t blkbuf[howmany(EALG_MAX_BLOCK_LEN, sizeof(uint32_t))];
+	u_char *blk = (u_char *)blkbuf;
+	u_char aalg[AALG_MAX_RESULT_LEN];
+	u_char uaalg[AALG_MAX_RESULT_LEN];
+	u_char iv[EALG_MAX_BLOCK_LEN];
+	union authctx ctx;
+	struct swcr_auth *swa;
+	struct swcr_encdec *swe;
+	struct auth_hash *axf;
+	struct enc_xform *exf;
+	int blksz, i, ivlen, len, r;
+
+	swa = &ses->swcr_auth;
+	axf = swa->sw_axf;
+
+	bcopy(swa->sw_ictx, &ctx, axf->ctxsize);
+	blksz = axf->blocksize;
+	
+	swe = &ses->swcr_encdec;
+	exf = swe->sw_exf;
+
+	if ((crp->crp_flags & CRYPTO_F_IV_SEPARATE) == 0)
+		return (EINVAL);
+
+	/* Initialize the IV */
+	ivlen = AES_CCM_IV_LEN;
+	bcopy(crp->crp_iv, iv, ivlen);
+
+	/*
+	 * AES CCM-CBC-MAC needs to know the length of both the auth
+	 * data and payload data before doing the auth computation.
+	 */
+	ctx.aes_cbc_mac_ctx.authDataLength = crp->crp_aad_length;
+	ctx.aes_cbc_mac_ctx.cryptDataLength = crp->crp_payload_length;
+
 	/* Supply MAC with IV */
-	if (axf->Reinit)
-		axf->Reinit(&ctx, iv, ivlen);
+	axf->Reinit(&ctx, iv, ivlen);
 
 	/* Supply MAC with AAD */
-	aadlen = crda->crd_len;
-
-	for (i = iskip; i < crda->crd_len; i += blksz) {
-		len = MIN(crda->crd_len - i, blksz - oskip);
-		crypto_copydata(crp->crp_flags, buf, crda->crd_skip + i, len,
-		    blk + oskip);
-		bzero(blk + len + oskip, blksz - len - oskip);
+	for (i = 0; i < crp->crp_aad_length; i += blksz) {
+		len = MIN(crp->crp_aad_length - i, blksz);
+		crypto_copydata(crp, crp->crp_aad_start + i, len, blk);
+		bzero(blk + len, blksz - len);
 		axf->Update(&ctx, blk, blksz);
-		oskip = 0; /* reset initial output offset */
 	}
 
-	if (exf->reinit)
-		exf->reinit(swe->sw_kschedule, iv);
+	exf->reinit(swe->sw_kschedule, iv);
 
 	/* Do encryption/decryption with MAC */
-	for (i = 0; i < crde->crd_len; i += len) {
-		if (exf->encrypt_multi != NULL) {
-			len = rounddown(crde->crd_len - i, blksz);
-			if (len == 0)
-				len = blksz;
-			else
-				len = MIN(len, sizeof(blkbuf));
-		} else
-			len = blksz;
-		len = MIN(crde->crd_len - i, len);
+	for (i = 0; i < crp->crp_payload_length; i += len) {
+		len = MIN(crp->crp_payload_length - i, blksz);
 		if (len < blksz)
 			bzero(blk, blksz);
-		crypto_copydata(crp->crp_flags, buf, crde->crd_skip + i, len,
-		    blk);
-		/*
-		 * One of the problems with CCM+CBC is that the authentication
-		 * is done on the unecncrypted data.  As a result, we have
-		 * to do the authentication update at different times,
-		 * depending on whether it's CCM or not.
-		 */
-		if (crde->crd_flags & CRD_F_ENCRYPT) {
-			if (isccm)
-				axf->Update(&ctx, blk, len);
-			if (exf->encrypt_multi != NULL)
-				exf->encrypt_multi(swe->sw_kschedule, blk,
-				    len);
-			else
-				exf->encrypt(swe->sw_kschedule, blk);
-			if (!isccm)
-				axf->Update(&ctx, blk, len);
-			crypto_copyback(crp->crp_flags, buf,
-			    crde->crd_skip + i, len, blk);
+		crypto_copydata(crp, crp->crp_payload_start + i, len, blk);
+		if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
+			axf->Update(&ctx, blk, len);
+			exf->encrypt(swe->sw_kschedule, blk);
+			crypto_copyback(crp, crp->crp_payload_start + i, len,
+			    blk);
 		} else {
-			if (isccm) {
-				KASSERT(exf->encrypt_multi == NULL,
-				    ("assume CCM is single-block only"));
-				exf->decrypt(swe->sw_kschedule, blk);
-			}
+			/*
+			 * One of the problems with CCM+CBC is that
+			 * the authentication is done on the
+			 * unecncrypted data.  As a result, we have to
+			 * decrypt the data twice: once to generate
+			 * the tag and a second time after the tag is
+			 * verified.
+			 */
+			exf->decrypt(swe->sw_kschedule, blk);
 			axf->Update(&ctx, blk, len);
 		}
 	}
 
-	/* Do any required special finalization */
-	switch (crda->crd_alg) {
-		case CRYPTO_AES_128_NIST_GMAC:
-		case CRYPTO_AES_192_NIST_GMAC:
-		case CRYPTO_AES_256_NIST_GMAC:
-			/* length block */
-			bzero(blk, blksz);
-			blkp = (uint32_t *)blk + 1;
-			*blkp = htobe32(aadlen * 8);
-			blkp = (uint32_t *)blk + 3;
-			*blkp = htobe32(crde->crd_len * 8);
-			axf->Update(&ctx, blk, blksz);
-			break;
-	}
-
 	/* Finalize MAC */
 	axf->Final(aalg, &ctx);
 
 	/* Validate tag */
-	if (!(crde->crd_flags & CRD_F_ENCRYPT)) {
-		crypto_copydata(crp->crp_flags, buf, crda->crd_inject,
-		    axf->hashsize, uaalg);
+	if (!CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
+		crypto_copydata(crp, crp->crp_digest_start, swa->sw_mlen,
+		    uaalg);
 
-		r = timingsafe_bcmp(aalg, uaalg, axf->hashsize);
-		if (r == 0) {
-			/* tag matches, decrypt data */
-			if (isccm) {
-				KASSERT(exf->reinit != NULL,
-				    ("AES-CCM reinit function must be set"));
-				exf->reinit(swe->sw_kschedule, iv);
-			}
-			for (i = 0; i < crde->crd_len; i += blksz) {
-				len = MIN(crde->crd_len - i, blksz);
-				if (len < blksz)
-					bzero(blk, blksz);
-				crypto_copydata(crp->crp_flags, buf,
-				    crde->crd_skip + i, len, blk);
-				exf->decrypt(swe->sw_kschedule, blk);
-				crypto_copyback(crp->crp_flags, buf,
-				    crde->crd_skip + i, len, blk);
-			}
-		} else
+		r = timingsafe_bcmp(aalg, uaalg, swa->sw_mlen);
+		if (r != 0)
 			return (EBADMSG);
+		
+		/* tag matches, decrypt data */
+		exf->reinit(swe->sw_kschedule, iv);
+		for (i = 0; i < crp->crp_payload_length; i += blksz) {
+			len = MIN(crp->crp_payload_length - i, blksz);
+			if (len < blksz)
+				bzero(blk, blksz);
+			crypto_copydata(crp, crp->crp_payload_start + i, len,
+			    blk);
+			exf->decrypt(swe->sw_kschedule, blk);
+			crypto_copyback(crp, crp->crp_payload_start + i, len,
+			    blk);
+		}
 	} else {
 		/* Inject the authentication data */
-		crypto_copyback(crp->crp_flags, buf, crda->crd_inject,
-		    axf->hashsize, aalg);
+		crypto_copyback(crp, crp->crp_digest_start, swa->sw_mlen,
+		    aalg);
 	}
 
 	return (0);
 }
 
 /*
+ * Apply a cipher and a digest to perform EtA.
+ */
+static int
+swcr_eta(struct swcr_session *ses, struct cryptop *crp)
+{
+	int error;
+
+	if (CRYPTO_OP_IS_ENCRYPT(crp->crp_op)) {
+		error = swcr_encdec(ses, crp);
+		if (error == 0)
+			error = swcr_authcompute(ses, crp);
+	} else {
+		error = swcr_authcompute(ses, crp);
+		if (error == 0)
+			error = swcr_encdec(ses, crp);
+	}
+	return (error);
+}
+
+/*
  * Apply a compression/decompression algorithm
  */
 static int
-swcr_compdec(struct cryptodesc *crd, struct swcr_data *sw,
-    caddr_t buf, int flags)
+swcr_compdec(struct swcr_session *ses, struct cryptop *crp)
 {
 	u_int8_t *data, *out;
 	struct comp_algo *cxf;
 	int adj;
 	u_int32_t result;
 
-	cxf = sw->sw_cxf;
+	cxf = ses->swcr_compdec.sw_cxf;
 
 	/* We must handle the whole buffer of data in one time
 	 * then if there is not all the data in the mbuf, we must
 	 * copy in a buffer.
 	 */
 
-	data = malloc(crd->crd_len, M_CRYPTO_DATA,  M_NOWAIT);
+	data = malloc(crp->crp_payload_length, M_CRYPTO_DATA,  M_NOWAIT);
 	if (data == NULL)
 		return (EINVAL);
-	crypto_copydata(flags, buf, crd->crd_skip, crd->crd_len, data);
+	crypto_copydata(crp, crp->crp_payload_start, crp->crp_payload_length,
+	    data);
 
-	if (crd->crd_flags & CRD_F_COMP)
-		result = cxf->compress(data, crd->crd_len, &out);
+	if (CRYPTO_OP_IS_COMPRESS(crp->crp_op))
+		result = cxf->compress(data, crp->crp_payload_length, &out);
 	else
-		result = cxf->decompress(data, crd->crd_len, &out);
+		result = cxf->decompress(data, crp->crp_payload_length, &out);
 
 	free(data, M_CRYPTO_DATA);
 	if (result == 0)
-		return EINVAL;
+		return (EINVAL);
+	crp->crp_olen = result;
 
-	/* Copy back the (de)compressed data. m_copyback is
-	 * extending the mbuf as necessary.
-	 */
-	sw->sw_size = result;
 	/* Check the compressed size when doing compression */
-	if (crd->crd_flags & CRD_F_COMP) {
-		if (result >= crd->crd_len) {
+	if (CRYPTO_OP_IS_COMPRESS(crp->crp_op)) {
+		if (result >= crp->crp_payload_length) {
 			/* Compression was useless, we lost time */
 			free(out, M_CRYPTO_DATA);
-			return 0;
+			return (0);
 		}
 	}
 
-	crypto_copyback(flags, buf, crd->crd_skip, result, out);
-	if (result < crd->crd_len) {
-		adj = result - crd->crd_len;
-		if (flags & CRYPTO_F_IMBUF) {
-			adj = result - crd->crd_len;
-			m_adj((struct mbuf *)buf, adj);
-		} else if (flags & CRYPTO_F_IOV) {
-			struct uio *uio = (struct uio *)buf;
+	/* Copy back the (de)compressed data. m_copyback is
+	 * extending the mbuf as necessary.
+	 */
+	crypto_copyback(crp, crp->crp_payload_start, result, out);
+	if (result < crp->crp_payload_length) {
+		switch (crp->crp_buf_type) {
+		case CRYPTO_BUF_MBUF:
+			adj = result - crp->crp_payload_length;
+			m_adj(crp->crp_mbuf, adj);
+			break;
+		case CRYPTO_BUF_UIO: {
+			struct uio *uio = crp->crp_uio;
 			int ind;
 
-			adj = crd->crd_len - result;
+			adj = crp->crp_payload_length - result;
 			ind = uio->uio_iovcnt - 1;
 
 			while (adj > 0 && ind >= 0) {
 				if (adj < uio->uio_iov[ind].iov_len) {
 					uio->uio_iov[ind].iov_len -= adj;
 					break;
 				}
 
 				adj -= uio->uio_iov[ind].iov_len;
 				uio->uio_iov[ind].iov_len = 0;
 				ind--;
 				uio->uio_iovcnt--;
 			}
+			}
+			break;
 		}
 	}
 	free(out, M_CRYPTO_DATA);
 	return 0;
 }
 
-/*
- * Generate a new software session.
- */
 static int
-swcr_newsession(device_t dev, crypto_session_t cses, struct cryptoini *cri)
+swcr_setup_encdec(struct swcr_session *ses,
+    const struct crypto_session_params *csp)
 {
-	struct swcr_session *ses;
-	struct swcr_data *swd;
-	struct auth_hash *axf;
+	struct swcr_encdec *swe;
 	struct enc_xform *txf;
-	struct comp_algo *cxf;
-	size_t i;
-	int len;
 	int error;
 
-	if (cses == NULL || cri == NULL)
-		return EINVAL;
+	swe = &ses->swcr_encdec;
+	txf = crypto_cipher(csp);
+	MPASS(txf->ivsize == csp->csp_ivlen);
+	if (csp->csp_cipher_key != NULL) {
+		error = txf->setkey(&swe->sw_kschedule,
+		    csp->csp_cipher_key, csp->csp_cipher_klen);
+		if (error)
+			return (error);
+	}
+	swe->sw_exf = txf;
+	return (0);
+}
 
-	ses = crypto_get_driver_session(cses);
-	mtx_init(&ses->swcr_lock, "swcr session lock", NULL, MTX_DEF);
+static int
+swcr_setup_auth(struct swcr_session *ses,
+    const struct crypto_session_params *csp)
+{
+	struct swcr_auth *swa;
+	struct auth_hash *axf;
 
-	for (i = 0; cri != NULL && i < nitems(ses->swcr_algorithms); i++) {
-		swd = &ses->swcr_algorithms[i];
+	swa = &ses->swcr_auth;
 
-		switch (cri->cri_alg) {
-		case CRYPTO_DES_CBC:
-			txf = &enc_xform_des;
-			goto enccommon;
-		case CRYPTO_3DES_CBC:
-			txf = &enc_xform_3des;
-			goto enccommon;
-		case CRYPTO_BLF_CBC:
-			txf = &enc_xform_blf;
-			goto enccommon;
-		case CRYPTO_CAST_CBC:
-			txf = &enc_xform_cast5;
-			goto enccommon;
-		case CRYPTO_SKIPJACK_CBC:
-			txf = &enc_xform_skipjack;
-			goto enccommon;
-		case CRYPTO_RIJNDAEL128_CBC:
-			txf = &enc_xform_rijndael128;
-			goto enccommon;
-		case CRYPTO_AES_XTS:
-			txf = &enc_xform_aes_xts;
-			goto enccommon;
-		case CRYPTO_AES_ICM:
-			txf = &enc_xform_aes_icm;
-			goto enccommon;
-		case CRYPTO_AES_NIST_GCM_16:
-			txf = &enc_xform_aes_nist_gcm;
-			goto enccommon;
-		case CRYPTO_AES_CCM_16:
-			txf = &enc_xform_ccm;
-			goto enccommon;
-		case CRYPTO_AES_NIST_GMAC:
-			txf = &enc_xform_aes_nist_gmac;
-			swd->sw_exf = txf;
-			break;
-		case CRYPTO_CAMELLIA_CBC:
-			txf = &enc_xform_camellia;
-			goto enccommon;
-		case CRYPTO_NULL_CBC:
-			txf = &enc_xform_null;
-			goto enccommon;
-		case CRYPTO_CHACHA20:
-			txf = &enc_xform_chacha20;
-			goto enccommon;
-		enccommon:
-			if (cri->cri_key != NULL) {
-				error = txf->setkey(&swd->sw_kschedule,
-				    cri->cri_key, cri->cri_klen / 8);
-				if (error) {
-					swcr_freesession(dev, cses);
-					return error;
-				}
-			}
-			swd->sw_exf = txf;
-			break;
+	axf = crypto_auth_hash(csp);
+	swa->sw_axf = axf;
+	if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize)
+		return (EINVAL);
+	if (csp->csp_auth_mlen == 0)
+		swa->sw_mlen = axf->hashsize;
+	else
+		swa->sw_mlen = csp->csp_auth_mlen;
+	swa->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, M_NOWAIT);
+	if (swa->sw_ictx == NULL)
+		return (ENOBUFS);
 	
-		case CRYPTO_MD5_HMAC:
-			axf = &auth_hash_hmac_md5;
-			goto authcommon;
-		case CRYPTO_SHA1_HMAC:
-			axf = &auth_hash_hmac_sha1;
-			goto authcommon;
-		case CRYPTO_SHA2_224_HMAC:
-			axf = &auth_hash_hmac_sha2_224;
-			goto authcommon;
-		case CRYPTO_SHA2_256_HMAC:
-			axf = &auth_hash_hmac_sha2_256;
-			goto authcommon;
-		case CRYPTO_SHA2_384_HMAC:
-			axf = &auth_hash_hmac_sha2_384;
-			goto authcommon;
-		case CRYPTO_SHA2_512_HMAC:
-			axf = &auth_hash_hmac_sha2_512;
-			goto authcommon;
-		case CRYPTO_NULL_HMAC:
-			axf = &auth_hash_null;
-			goto authcommon;
-		case CRYPTO_RIPEMD160_HMAC:
-			axf = &auth_hash_hmac_ripemd_160;
-		authcommon:
-			swd->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA,
-			    M_NOWAIT);
-			if (swd->sw_ictx == NULL) {
-				swcr_freesession(dev, cses);
-				return ENOBUFS;
-			}
-	
-			swd->sw_octx = malloc(axf->ctxsize, M_CRYPTO_DATA,
-			    M_NOWAIT);
-			if (swd->sw_octx == NULL) {
-				swcr_freesession(dev, cses);
-				return ENOBUFS;
-			}
+	switch (csp->csp_auth_alg) {
+	case CRYPTO_MD5_HMAC:
+	case CRYPTO_SHA1_HMAC:
+	case CRYPTO_SHA2_224_HMAC:
+	case CRYPTO_SHA2_256_HMAC:
+	case CRYPTO_SHA2_384_HMAC:
+	case CRYPTO_SHA2_512_HMAC:
+	case CRYPTO_NULL_HMAC:
+	case CRYPTO_RIPEMD160_HMAC:
+		swa->sw_octx_len = axf->ctxsize;
+		swa->sw_octx = malloc(swa->sw_octx_len, M_CRYPTO_DATA,
+		    M_NOWAIT);
+		if (swa->sw_octx == NULL)
+			return (ENOBUFS);
 
-			if (cri->cri_key != NULL) {
-				error = swcr_authprepare(axf, swd,
-				    cri->cri_key, cri->cri_klen);
-				if (error != 0) {
-					swcr_freesession(dev, cses);
-					return error;
-				}
-			}
+		if (csp->csp_auth_key != NULL) {
+			swcr_authprepare(axf, swa, csp->csp_auth_key,
+			    csp->csp_auth_klen);
+		}
 
-			swd->sw_mlen = cri->cri_mlen;
-			swd->sw_axf = axf;
-			break;
-	
-		case CRYPTO_MD5_KPDK:
-			axf = &auth_hash_key_md5;
-			goto auth2common;
-	
-		case CRYPTO_SHA1_KPDK:
-			axf = &auth_hash_key_sha1;
-		auth2common:
-			swd->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA,
-			    M_NOWAIT);
-			if (swd->sw_ictx == NULL) {
-				swcr_freesession(dev, cses);
-				return ENOBUFS;
-			}
-	
-			swd->sw_octx = malloc(cri->cri_klen / 8,
-			    M_CRYPTO_DATA, M_NOWAIT);
-			if (swd->sw_octx == NULL) {
-				swcr_freesession(dev, cses);
-				return ENOBUFS;
-			}
+		if (csp->csp_mode == CSP_MODE_DIGEST)
+			ses->swcr_process = swcr_authcompute;
+		break;
+	case CRYPTO_MD5_KPDK:
+	case CRYPTO_SHA1_KPDK:
+		swa->sw_octx_len = csp->csp_auth_klen;
+		swa->sw_octx = malloc(swa->sw_octx_len, M_CRYPTO_DATA,
+		    M_NOWAIT);
+		if (swa->sw_octx == NULL)
+			return (ENOBUFS);
 
-			/* Store the key so we can "append" it to the payload */
-			if (cri->cri_key != NULL) {
-				error = swcr_authprepare(axf, swd,
-				    cri->cri_key, cri->cri_klen);
-				if (error != 0) {
-					swcr_freesession(dev, cses);
-					return error;
-				}
-			}
+		/* Store the key so we can "append" it to the payload */
+		if (csp->csp_auth_key != NULL) {
+			swcr_authprepare(axf, swa, csp->csp_auth_key,
+			    csp->csp_auth_klen);
+		}
 
-			swd->sw_mlen = cri->cri_mlen;
-			swd->sw_axf = axf;
-			break;
+		if (csp->csp_mode == CSP_MODE_DIGEST)
+			ses->swcr_process = swcr_authcompute;
+		break;
 #ifdef notdef
-		case CRYPTO_MD5:
-			axf = &auth_hash_md5;
-			goto auth3common;
+	case CRYPTO_MD5:
 #endif
+	case CRYPTO_SHA1:
+	case CRYPTO_SHA2_224:
+	case CRYPTO_SHA2_256:
+	case CRYPTO_SHA2_384:
+	case CRYPTO_SHA2_512:
+		axf->Init(swa->sw_ictx);
+		if (csp->csp_mode == CSP_MODE_DIGEST)
+			ses->swcr_process = swcr_authcompute;
+		break;
+	case CRYPTO_AES_NIST_GMAC:
+		axf->Init(swa->sw_ictx);
+		axf->Setkey(swa->sw_ictx, csp->csp_auth_key,
+		    csp->csp_auth_klen);
+		if (csp->csp_mode == CSP_MODE_DIGEST)
+			ses->swcr_process = swcr_gmac;
+		break;
+	case CRYPTO_POLY1305:
+	case CRYPTO_BLAKE2B:
+	case CRYPTO_BLAKE2S:
+		/*
+		 * Blake2b and Blake2s support an optional key but do
+		 * not require one.
+		 */
+		if (csp->csp_auth_klen == 0 || csp->csp_auth_key != NULL)
+			axf->Setkey(swa->sw_ictx, csp->csp_auth_key,
+			    csp->csp_auth_klen);
+		axf->Init(swa->sw_ictx);
+		if (csp->csp_mode == CSP_MODE_DIGEST)
+			ses->swcr_process = swcr_authcompute;
+		break;
+	case CRYPTO_AES_CCM_CBC_MAC:
+		axf->Init(swa->sw_ictx);
+		axf->Setkey(swa->sw_ictx, csp->csp_auth_key,
+		    csp->csp_auth_klen);
+		if (csp->csp_mode == CSP_MODE_DIGEST)
+			ses->swcr_process = swcr_ccm_cbc_mac;
+		break;
+	}
 
-		case CRYPTO_SHA1:
-			axf = &auth_hash_sha1;
-			goto auth3common;
-		case CRYPTO_SHA2_224:
-			axf = &auth_hash_sha2_224;
-			goto auth3common;
-		case CRYPTO_SHA2_256:
-			axf = &auth_hash_sha2_256;
-			goto auth3common;
-		case CRYPTO_SHA2_384:
-			axf = &auth_hash_sha2_384;
-			goto auth3common;
-		case CRYPTO_SHA2_512:
-			axf = &auth_hash_sha2_512;
+	return (0);
+}
 
-		auth3common:
-			swd->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA,
-			    M_NOWAIT);
-			if (swd->sw_ictx == NULL) {
-				swcr_freesession(dev, cses);
-				return ENOBUFS;
-			}
+static int
+swcr_setup_gcm(struct swcr_session *ses,
+    const struct crypto_session_params *csp)
+{
+	struct swcr_encdec *swe;
+	struct swcr_auth *swa;
+	struct enc_xform *txf;
+	struct auth_hash *axf;
+	int error;
 
-			axf->Init(swd->sw_ictx);
-			swd->sw_mlen = cri->cri_mlen;
-			swd->sw_axf = axf;
-			break;
+	if (csp->csp_ivlen != AES_GCM_IV_LEN)
+		return (EINVAL);
 
-		case CRYPTO_AES_CCM_CBC_MAC:
-			switch (cri->cri_klen) {
-			case 128:
-				axf = &auth_hash_ccm_cbc_mac_128;
-				break;
-			case 192:
-				axf = &auth_hash_ccm_cbc_mac_192;
-				break;
-			case 256:
-				axf = &auth_hash_ccm_cbc_mac_256;
-				break;
-			default:
-				swcr_freesession(dev, cses);
-				return EINVAL;
-			}
-			goto auth4common;
-		case CRYPTO_AES_128_NIST_GMAC:
-			axf = &auth_hash_nist_gmac_aes_128;
-			goto auth4common;
+	/* First, setup the auth side. */
+	swa = &ses->swcr_auth;
+	switch (csp->csp_cipher_klen * 8) {
+	case 128:
+		axf = &auth_hash_nist_gmac_aes_128;
+		break;
+	case 192:
+		axf = &auth_hash_nist_gmac_aes_192;
+		break;
+	case 256:
+		axf = &auth_hash_nist_gmac_aes_256;
+		break;
+	default:
+		return (EINVAL);
+	}
+	swa->sw_axf = axf;
+	if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize)
+		return (EINVAL);
+	if (csp->csp_auth_mlen == 0)
+		swa->sw_mlen = axf->hashsize;
+	else
+		swa->sw_mlen = csp->csp_auth_mlen;
+	swa->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, M_NOWAIT);
+	if (swa->sw_ictx == NULL)
+		return (ENOBUFS);
+	axf->Init(swa->sw_ictx);
+	if (csp->csp_cipher_key != NULL)
+		axf->Setkey(swa->sw_ictx, csp->csp_cipher_key,
+		    csp->csp_cipher_klen);
 
-		case CRYPTO_AES_192_NIST_GMAC:
-			axf = &auth_hash_nist_gmac_aes_192;
-			goto auth4common;
+	/* Second, setup the cipher side. */
+	swe = &ses->swcr_encdec;
+	txf = &enc_xform_aes_nist_gcm;
+	if (csp->csp_cipher_key != NULL) {
+		error = txf->setkey(&swe->sw_kschedule,
+		    csp->csp_cipher_key, csp->csp_cipher_klen);
+		if (error)
+			return (error);
+	}
+	swe->sw_exf = txf;
 
-		case CRYPTO_AES_256_NIST_GMAC:
-			axf = &auth_hash_nist_gmac_aes_256;
-		auth4common:
-			len = cri->cri_klen / 8;
-			if (len != 16 && len != 24 && len != 32) {
-				swcr_freesession(dev, cses);
-				return EINVAL;
-			}
+	return (0);
+}
 
-			swd->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA,
-			    M_NOWAIT);
-			if (swd->sw_ictx == NULL) {
-				swcr_freesession(dev, cses);
-				return ENOBUFS;
-			}
-			axf->Init(swd->sw_ictx);
-			axf->Setkey(swd->sw_ictx, cri->cri_key, len);
-			swd->sw_axf = axf;
-			break;
+static int
+swcr_setup_ccm(struct swcr_session *ses,
+    const struct crypto_session_params *csp)
+{
+	struct swcr_encdec *swe;
+	struct swcr_auth *swa;
+	struct enc_xform *txf;
+	struct auth_hash *axf;
+	int error;
 
-		case CRYPTO_BLAKE2B:
-			axf = &auth_hash_blake2b;
-			goto auth5common;
-		case CRYPTO_BLAKE2S:
-			axf = &auth_hash_blake2s;
-			goto auth5common;
-		case CRYPTO_POLY1305:
-			axf = &auth_hash_poly1305;
-		auth5common:
-			swd->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA,
-			    M_NOWAIT);
-			if (swd->sw_ictx == NULL) {
-				swcr_freesession(dev, cses);
-				return ENOBUFS;
-			}
-			axf->Setkey(swd->sw_ictx, cri->cri_key,
-			    cri->cri_klen / 8);
-			axf->Init(swd->sw_ictx);
-			swd->sw_axf = axf;
+	if (csp->csp_ivlen != AES_CCM_IV_LEN)
+		return (EINVAL);
+
+	/* First, setup the auth side. */
+	swa = &ses->swcr_auth;
+	switch (csp->csp_cipher_klen * 8) {
+	case 128:
+		axf = &auth_hash_ccm_cbc_mac_128;
+		break;
+	case 192:
+		axf = &auth_hash_ccm_cbc_mac_192;
+		break;
+	case 256:
+		axf = &auth_hash_ccm_cbc_mac_256;
+		break;
+	default:
+		return (EINVAL);
+	}
+	swa->sw_axf = axf;
+	if (csp->csp_auth_mlen < 0 || csp->csp_auth_mlen > axf->hashsize)
+		return (EINVAL);
+	if (csp->csp_auth_mlen == 0)
+		swa->sw_mlen = axf->hashsize;
+	else
+		swa->sw_mlen = csp->csp_auth_mlen;
+	swa->sw_ictx = malloc(axf->ctxsize, M_CRYPTO_DATA, M_NOWAIT);
+	if (swa->sw_ictx == NULL)
+		return (ENOBUFS);
+	axf->Init(swa->sw_ictx);
+	if (csp->csp_cipher_key != NULL)
+		axf->Setkey(swa->sw_ictx, csp->csp_cipher_key,
+		    csp->csp_cipher_klen);
+
+	/* Second, setup the cipher side. */
+	swe = &ses->swcr_encdec;
+	txf = &enc_xform_ccm;
+	if (csp->csp_cipher_key != NULL) {
+		error = txf->setkey(&swe->sw_kschedule,
+		    csp->csp_cipher_key, csp->csp_cipher_klen);
+		if (error)
+			return (error);
+	}
+	swe->sw_exf = txf;
+
+	return (0);
+}
+
+static bool
+swcr_auth_supported(const struct crypto_session_params *csp)
+{
+	struct auth_hash *axf;
+
+	axf = crypto_auth_hash(csp);
+	if (axf == NULL)
+		return (false);
+	switch (csp->csp_auth_alg) {
+	case CRYPTO_MD5_HMAC:
+	case CRYPTO_SHA1_HMAC:
+	case CRYPTO_SHA2_224_HMAC:
+	case CRYPTO_SHA2_256_HMAC:
+	case CRYPTO_SHA2_384_HMAC:
+	case CRYPTO_SHA2_512_HMAC:
+	case CRYPTO_NULL_HMAC:
+	case CRYPTO_RIPEMD160_HMAC:
+	case CRYPTO_MD5_KPDK:
+	case CRYPTO_SHA1_KPDK:
+		break;
+	case CRYPTO_AES_NIST_GMAC:
+		switch (csp->csp_auth_klen * 8) {
+		case 128:
+		case 192:
+		case 256:
 			break;
+		default:
+			return (false);
+		}
+		if (csp->csp_auth_key == NULL)
+			return (false);
+		if (csp->csp_ivlen != AES_GCM_IV_LEN)
+			return (false);
+		break;
+	case CRYPTO_POLY1305:
+		if (csp->csp_auth_klen != POLY1305_KEY_LEN)
+			return (false);
+		break;
+	case CRYPTO_AES_CCM_CBC_MAC:
+		switch (csp->csp_auth_klen * 8) {
+		case 128:
+		case 192:
+		case 256:
+			break;
+		default:
+			return (false);
+		}
+		if (csp->csp_auth_key == NULL)
+			return (false);
+		if (csp->csp_ivlen != AES_CCM_IV_LEN)
+			return (false);
+		break;
+	}
+	return (true);
+}
 
+static bool
+swcr_cipher_supported(const struct crypto_session_params *csp)
+{
+	struct enc_xform *txf;
+
+	txf = crypto_cipher(csp);
+	if (txf == NULL)
+		return (false);
+	if (csp->csp_cipher_alg != CRYPTO_NULL_CBC &&
+	    txf->ivsize != csp->csp_ivlen)
+		return (false);
+	return (true);
+}
+
+static int
+swcr_probesession(device_t dev, const struct crypto_session_params *csp)
+{
+
+	if (csp->csp_flags != 0)
+		return (EINVAL);
+	switch (csp->csp_mode) {
+	case CSP_MODE_COMPRESS:
+		switch (csp->csp_cipher_alg) {
 		case CRYPTO_DEFLATE_COMP:
-			cxf = &comp_algo_deflate;
-			swd->sw_cxf = cxf;
 			break;
 		default:
-			swcr_freesession(dev, cses);
-			return EINVAL;
+			return (EINVAL);
 		}
-	
-		swd->sw_alg = cri->cri_alg;
-		cri = cri->cri_next;
-		ses->swcr_nalgs++;
-	}
+		break;
+	case CSP_MODE_CIPHER:
+		switch (csp->csp_cipher_alg) {
+		case CRYPTO_AES_NIST_GCM_16:
+		case CRYPTO_AES_CCM_16:
+			return (EINVAL);
+		default:
+			if (!swcr_cipher_supported(csp))
+				return (EINVAL);
+			break;
+		}
+		break;
+	case CSP_MODE_DIGEST:
+		if (!swcr_auth_supported(csp))
+			return (EINVAL);
+		break;
+	case CSP_MODE_AEAD:
+		switch (csp->csp_cipher_alg) {
+		case CRYPTO_AES_NIST_GCM_16:
+		case CRYPTO_AES_CCM_16:
+			break;
+		default:
+			return (EINVAL);
+		}
+		break;
+	case CSP_MODE_ETA:
+		/* AEAD algorithms cannot be used for EtA. */
+		switch (csp->csp_cipher_alg) {
+		case CRYPTO_AES_NIST_GCM_16:
+		case CRYPTO_AES_CCM_16:
+			return (EINVAL);
+		}
+		switch (csp->csp_auth_alg) {
+		case CRYPTO_AES_NIST_GMAC:
+		case CRYPTO_AES_CCM_CBC_MAC:
+			return (EINVAL);
+		}
 
-	if (cri != NULL) {
-		CRYPTDEB("Bogus session request for three or more algorithms");
-		return EINVAL;
+		if (!swcr_cipher_supported(csp) ||
+		    !swcr_auth_supported(csp))
+			return (EINVAL);
+		break;
+	default:
+		return (EINVAL);
 	}
-	return 0;
+
+	return (CRYPTODEV_PROBE_SOFTWARE);
 }
 
-static void
-swcr_freesession(device_t dev, crypto_session_t cses)
+/*
+ * Generate a new software session.
+ */
+static int
+swcr_newsession(device_t dev, crypto_session_t cses,
+    const struct crypto_session_params *csp)
 {
 	struct swcr_session *ses;
-	struct swcr_data *swd;
-	struct enc_xform *txf;
-	struct auth_hash *axf;
-	size_t i;
+	struct swcr_encdec *swe;
+	struct swcr_auth *swa;
+	struct comp_algo *cxf;
+	int error;
 
 	ses = crypto_get_driver_session(cses);
+	mtx_init(&ses->swcr_lock, "swcr session lock", NULL, MTX_DEF);
 
-	mtx_destroy(&ses->swcr_lock);
-	for (i = 0; i < nitems(ses->swcr_algorithms); i++) {
-		swd = &ses->swcr_algorithms[i];
-
-		switch (swd->sw_alg) {
-		case CRYPTO_DES_CBC:
-		case CRYPTO_3DES_CBC:
-		case CRYPTO_BLF_CBC:
-		case CRYPTO_CAST_CBC:
-		case CRYPTO_SKIPJACK_CBC:
-		case CRYPTO_RIJNDAEL128_CBC:
-		case CRYPTO_AES_XTS:
-		case CRYPTO_AES_ICM:
-		case CRYPTO_AES_NIST_GCM_16:
-		case CRYPTO_AES_NIST_GMAC:
-		case CRYPTO_CAMELLIA_CBC:
+	error = 0;
+	swe = &ses->swcr_encdec;
+	swa = &ses->swcr_auth;
+	switch (csp->csp_mode) {
+	case CSP_MODE_COMPRESS:
+		switch (csp->csp_cipher_alg) {
+		case CRYPTO_DEFLATE_COMP:
+			cxf = &comp_algo_deflate;
+			break;
+#ifdef INVARIANTS
+		default:
+			panic("bad compression algo");
+#endif
+		}
+		ses->swcr_compdec.sw_cxf = cxf;
+		ses->swcr_process = swcr_compdec;
+		break;
+	case CSP_MODE_CIPHER:
+		switch (csp->csp_cipher_alg) {
 		case CRYPTO_NULL_CBC:
-		case CRYPTO_CHACHA20:
+			ses->swcr_process = swcr_null;
+			break;
+#ifdef INVARIANTS
+		case CRYPTO_AES_NIST_GCM_16:
 		case CRYPTO_AES_CCM_16:
-			txf = swd->sw_exf;
-
-			if (swd->sw_kschedule)
-				txf->zerokey(&(swd->sw_kschedule));
+			panic("bad cipher algo");
+#endif
+		default:
+			error = swcr_setup_encdec(ses, csp);
+			if (error == 0)
+				ses->swcr_process = swcr_encdec;
+		}
+		break;
+	case CSP_MODE_DIGEST:
+		error = swcr_setup_auth(ses, csp);
+		break;
+	case CSP_MODE_AEAD:
+		switch (csp->csp_cipher_alg) {
+		case CRYPTO_AES_NIST_GCM_16:
+			error = swcr_setup_gcm(ses, csp);
+			if (error == 0)
+				ses->swcr_process = swcr_gcm;
 			break;
-
-		case CRYPTO_MD5_HMAC:
-		case CRYPTO_SHA1_HMAC:
-		case CRYPTO_SHA2_224_HMAC:
-		case CRYPTO_SHA2_256_HMAC:
-		case CRYPTO_SHA2_384_HMAC:
-		case CRYPTO_SHA2_512_HMAC:
-		case CRYPTO_RIPEMD160_HMAC:
-		case CRYPTO_NULL_HMAC:
+		case CRYPTO_AES_CCM_16:
+			error = swcr_setup_ccm(ses, csp);
+			if (error == 0)
+				ses->swcr_process = swcr_ccm;
+			break;
+#ifdef INVARIANTS
+		default:
+			panic("bad aead algo");
+#endif
+		}
+		break;
+	case CSP_MODE_ETA:
+#ifdef INVARIANTS
+		switch (csp->csp_cipher_alg) {
+		case CRYPTO_AES_NIST_GCM_16:
+		case CRYPTO_AES_CCM_16:
+			panic("bad eta cipher algo");
+		}
+		switch (csp->csp_auth_alg) {
+		case CRYPTO_AES_NIST_GMAC:
 		case CRYPTO_AES_CCM_CBC_MAC:
-			axf = swd->sw_axf;
+			panic("bad eta auth algo");
+		}
+#endif
 
-			if (swd->sw_ictx) {
-				bzero(swd->sw_ictx, axf->ctxsize);
-				free(swd->sw_ictx, M_CRYPTO_DATA);
-			}
-			if (swd->sw_octx) {
-				bzero(swd->sw_octx, axf->ctxsize);
-				free(swd->sw_octx, M_CRYPTO_DATA);
-			}
+		error = swcr_setup_auth(ses, csp);
+		if (error)
 			break;
+		if (csp->csp_cipher_alg == CRYPTO_NULL_CBC) {
+			/* Effectively degrade to digest mode. */
+			ses->swcr_process = swcr_authcompute;
+			break;
+		}
 
-		case CRYPTO_MD5_KPDK:
-		case CRYPTO_SHA1_KPDK:
-			axf = swd->sw_axf;
+		error = swcr_setup_encdec(ses, csp);
+		if (error == 0)
+			ses->swcr_process = swcr_eta;
+		break;
+	default:
+		error = EINVAL;
+	}
 
-			if (swd->sw_ictx) {
-				bzero(swd->sw_ictx, axf->ctxsize);
-				free(swd->sw_ictx, M_CRYPTO_DATA);
-			}
-			if (swd->sw_octx) {
-				bzero(swd->sw_octx, swd->sw_klen);
-				free(swd->sw_octx, M_CRYPTO_DATA);
-			}
-			break;
+	if (error)
+		swcr_freesession(dev, cses);
+	return (error);
+}
 
-		case CRYPTO_BLAKE2B:
-		case CRYPTO_BLAKE2S:
-		case CRYPTO_MD5:
-		case CRYPTO_POLY1305:
-		case CRYPTO_SHA1:
-		case CRYPTO_SHA2_224:
-		case CRYPTO_SHA2_256:
-		case CRYPTO_SHA2_384:
-		case CRYPTO_SHA2_512:
-		case CRYPTO_AES_128_NIST_GMAC:
-		case CRYPTO_AES_192_NIST_GMAC:
-		case CRYPTO_AES_256_NIST_GMAC:
-			axf = swd->sw_axf;
+static void
+swcr_freesession(device_t dev, crypto_session_t cses)
+{
+	struct swcr_session *ses;
+	struct swcr_auth *swa;
+	struct enc_xform *txf;
+	struct auth_hash *axf;
 
-			if (swd->sw_ictx) {
-				explicit_bzero(swd->sw_ictx, axf->ctxsize);
-				free(swd->sw_ictx, M_CRYPTO_DATA);
-			}
-			break;
+	ses = crypto_get_driver_session(cses);
 
-		case CRYPTO_DEFLATE_COMP:
-			/* Nothing to do */
-			break;
+	mtx_destroy(&ses->swcr_lock);
+
+	txf = ses->swcr_encdec.sw_exf;
+	if (txf != NULL) {
+		if (ses->swcr_encdec.sw_kschedule != NULL)
+			txf->zerokey(&(ses->swcr_encdec.sw_kschedule));
+	}
+
+	axf = ses->swcr_auth.sw_axf;
+	if (axf != NULL) {
+		swa = &ses->swcr_auth;
+		if (swa->sw_ictx != NULL) {
+			explicit_bzero(swa->sw_ictx, axf->ctxsize);
+			free(swa->sw_ictx, M_CRYPTO_DATA);
 		}
+		if (swa->sw_octx != NULL) {
+			explicit_bzero(swa->sw_octx, swa->sw_octx_len);
+			free(swa->sw_octx, M_CRYPTO_DATA);
+		}
 	}
 }
 
 /*
  * Process a software request.
  */
 static int
 swcr_process(device_t dev, struct cryptop *crp, int hint)
 {
-	struct swcr_session *ses = NULL;
-	struct cryptodesc *crd;
-	struct swcr_data *sw;
-	size_t i;
+	struct swcr_session *ses;
 
-	/* Sanity check */
-	if (crp == NULL)
-		return EINVAL;
-
-	if (crp->crp_desc == NULL || crp->crp_buf == NULL) {
-		crp->crp_etype = EINVAL;
-		goto done;
-	}
-
 	ses = crypto_get_driver_session(crp->crp_session);
 	mtx_lock(&ses->swcr_lock);
 
-	/* Go through crypto descriptors, processing as we go */
-	for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
-		/*
-		 * Find the crypto context.
-		 *
-		 * XXX Note that the logic here prevents us from having
-		 * XXX the same algorithm multiple times in a session
-		 * XXX (or rather, we can but it won't give us the right
-		 * XXX results). To do that, we'd need some way of differentiating
-		 * XXX between the various instances of an algorithm (so we can
-		 * XXX locate the correct crypto context).
-		 */
-		for (i = 0; i < nitems(ses->swcr_algorithms) &&
-		    ses->swcr_algorithms[i].sw_alg != crd->crd_alg; i++)
-			;
+	crp->crp_etype = ses->swcr_process(ses, crp);
 
-		/* No such context ? */
-		if (i == nitems(ses->swcr_algorithms)) {
-			crp->crp_etype = EINVAL;
-			goto done;
-		}
-		sw = &ses->swcr_algorithms[i];
-		switch (sw->sw_alg) {
-		case CRYPTO_DES_CBC:
-		case CRYPTO_3DES_CBC:
-		case CRYPTO_BLF_CBC:
-		case CRYPTO_CAST_CBC:
-		case CRYPTO_SKIPJACK_CBC:
-		case CRYPTO_RIJNDAEL128_CBC:
-		case CRYPTO_AES_XTS:
-		case CRYPTO_AES_ICM:
-		case CRYPTO_CAMELLIA_CBC:
-		case CRYPTO_CHACHA20:
-			if ((crp->crp_etype = swcr_encdec(crd, sw,
-			    crp->crp_buf, crp->crp_flags)) != 0)
-				goto done;
-			break;
-		case CRYPTO_NULL_CBC:
-			crp->crp_etype = 0;
-			break;
-		case CRYPTO_MD5_HMAC:
-		case CRYPTO_SHA1_HMAC:
-		case CRYPTO_SHA2_224_HMAC:
-		case CRYPTO_SHA2_256_HMAC:
-		case CRYPTO_SHA2_384_HMAC:
-		case CRYPTO_SHA2_512_HMAC:
-		case CRYPTO_RIPEMD160_HMAC:
-		case CRYPTO_NULL_HMAC:
-		case CRYPTO_MD5_KPDK:
-		case CRYPTO_SHA1_KPDK:
-		case CRYPTO_MD5:
-		case CRYPTO_SHA1:
-		case CRYPTO_SHA2_224:
-		case CRYPTO_SHA2_256:
-		case CRYPTO_SHA2_384:
-		case CRYPTO_SHA2_512:
-		case CRYPTO_BLAKE2B:
-		case CRYPTO_BLAKE2S:
-		case CRYPTO_POLY1305:
-			if ((crp->crp_etype = swcr_authcompute(crd, sw,
-			    crp->crp_buf, crp->crp_flags)) != 0)
-				goto done;
-			break;
-
-		case CRYPTO_AES_NIST_GCM_16:
-		case CRYPTO_AES_NIST_GMAC:
-		case CRYPTO_AES_128_NIST_GMAC:
-		case CRYPTO_AES_192_NIST_GMAC:
-		case CRYPTO_AES_256_NIST_GMAC:
-		case CRYPTO_AES_CCM_16:
-		case CRYPTO_AES_CCM_CBC_MAC:
-			crp->crp_etype = swcr_authenc(crp);
-			goto done;
-
-		case CRYPTO_DEFLATE_COMP:
-			if ((crp->crp_etype = swcr_compdec(crd, sw, 
-			    crp->crp_buf, crp->crp_flags)) != 0)
-				goto done;
-			else
-				crp->crp_olen = (int)sw->sw_size;
-			break;
-
-		default:
-			/* Unknown/unsupported algorithm */
-			crp->crp_etype = EINVAL;
-			goto done;
-		}
-	}
-
-done:
-	if (ses)
-		mtx_unlock(&ses->swcr_lock);
+	mtx_unlock(&ses->swcr_lock);
 	crypto_done(crp);
-	return 0;
+	return (0);
 }
 
 static void
 swcr_identify(driver_t *drv, device_t parent)
 {
 	/* NB: order 10 is so we get attached after h/w devices */
 	if (device_find_child(parent, "cryptosoft", -1) == NULL &&
 	    BUS_ADD_CHILD(parent, 10, "cryptosoft", 0) == 0)
 		panic("cryptosoft: could not attach");
 }
 
 static int
 swcr_probe(device_t dev)
 {
 	device_set_desc(dev, "software crypto");
 	return (BUS_PROBE_NOWILDCARD);
 }
 
 static int
 swcr_attach(device_t dev)
 {
-	memset(hmac_ipad_buffer, HMAC_IPAD_VAL, HMAC_MAX_BLOCK_LEN);
-	memset(hmac_opad_buffer, HMAC_OPAD_VAL, HMAC_MAX_BLOCK_LEN);
 
 	swcr_id = crypto_get_driverid(dev, sizeof(struct swcr_session),
 			CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_SYNC);
 	if (swcr_id < 0) {
 		device_printf(dev, "cannot initialize!");
-		return ENOMEM;
+		return (ENXIO);
 	}
-#define	REGISTER(alg) \
-	crypto_register(swcr_id, alg, 0,0)
-	REGISTER(CRYPTO_DES_CBC);
-	REGISTER(CRYPTO_3DES_CBC);
-	REGISTER(CRYPTO_BLF_CBC);
-	REGISTER(CRYPTO_CAST_CBC);
-	REGISTER(CRYPTO_SKIPJACK_CBC);
-	REGISTER(CRYPTO_NULL_CBC);
-	REGISTER(CRYPTO_MD5_HMAC);
-	REGISTER(CRYPTO_SHA1_HMAC);
-	REGISTER(CRYPTO_SHA2_224_HMAC);
-	REGISTER(CRYPTO_SHA2_256_HMAC);
-	REGISTER(CRYPTO_SHA2_384_HMAC);
-	REGISTER(CRYPTO_SHA2_512_HMAC);
-	REGISTER(CRYPTO_RIPEMD160_HMAC);
-	REGISTER(CRYPTO_NULL_HMAC);
-	REGISTER(CRYPTO_MD5_KPDK);
-	REGISTER(CRYPTO_SHA1_KPDK);
-	REGISTER(CRYPTO_MD5);
-	REGISTER(CRYPTO_SHA1);
-	REGISTER(CRYPTO_SHA2_224);
-	REGISTER(CRYPTO_SHA2_256);
-	REGISTER(CRYPTO_SHA2_384);
-	REGISTER(CRYPTO_SHA2_512);
-	REGISTER(CRYPTO_RIJNDAEL128_CBC);
-	REGISTER(CRYPTO_AES_XTS);
-	REGISTER(CRYPTO_AES_ICM);
-	REGISTER(CRYPTO_AES_NIST_GCM_16);
-	REGISTER(CRYPTO_AES_NIST_GMAC);
-	REGISTER(CRYPTO_AES_128_NIST_GMAC);
-	REGISTER(CRYPTO_AES_192_NIST_GMAC);
-	REGISTER(CRYPTO_AES_256_NIST_GMAC);
- 	REGISTER(CRYPTO_CAMELLIA_CBC);
-	REGISTER(CRYPTO_DEFLATE_COMP);
-	REGISTER(CRYPTO_BLAKE2B);
-	REGISTER(CRYPTO_BLAKE2S);
-	REGISTER(CRYPTO_CHACHA20);
-	REGISTER(CRYPTO_AES_CCM_16);
-	REGISTER(CRYPTO_AES_CCM_CBC_MAC);
-	REGISTER(CRYPTO_POLY1305);
-#undef REGISTER
 
-	return 0;
+	return (0);
 }
 
 static int
 swcr_detach(device_t dev)
 {
 	crypto_unregister_all(swcr_id);
 	return 0;
 }
 
 static device_method_t swcr_methods[] = {
 	DEVMETHOD(device_identify,	swcr_identify),
 	DEVMETHOD(device_probe,		swcr_probe),
 	DEVMETHOD(device_attach,	swcr_attach),
 	DEVMETHOD(device_detach,	swcr_detach),
 
+	DEVMETHOD(cryptodev_probesession, swcr_probesession),
 	DEVMETHOD(cryptodev_newsession,	swcr_newsession),
 	DEVMETHOD(cryptodev_freesession,swcr_freesession),
 	DEVMETHOD(cryptodev_process,	swcr_process),
 
 	{0, 0},
 };
 
 static driver_t swcr_driver = {
 	"cryptosoft",
 	swcr_methods,
 	0,		/* NB: no softc */
 };
 static devclass_t swcr_devclass;
 
 /*
  * NB: We explicitly reference the crypto module so we
  * get the necessary ordering when built as a loadable
  * module.  This is required because we bundle the crypto
  * module code together with the cryptosoft driver (otherwise
  * normal module dependencies would handle things).
  */
 extern int crypto_modevent(struct module *, int, void *);
 /* XXX where to attach */
 DRIVER_MODULE(cryptosoft, nexus, swcr_driver, swcr_devclass, crypto_modevent,0);
 MODULE_VERSION(cryptosoft, 1);
 MODULE_DEPEND(cryptosoft, crypto, 1, 1, 1);
Index: head/sys/opencrypto/ktls_ocf.c
===================================================================
--- head/sys/opencrypto/ktls_ocf.c	(revision 359373)
+++ head/sys/opencrypto/ktls_ocf.c	(revision 359374)
@@ -1,433 +1,404 @@
 /*-
  * SPDX-License-Identifier: BSD-2-Clause
  *
  * Copyright (c) 2019 Netflix Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/counter.h>
 #include <sys/endian.h>
 #include <sys/kernel.h>
 #include <sys/ktls.h>
 #include <sys/lock.h>
 #include <sys/malloc.h>
 #include <sys/module.h>
 #include <sys/mutex.h>
 #include <sys/sysctl.h>
 #include <sys/uio.h>
 #include <opencrypto/cryptodev.h>
 
 struct ocf_session {
 	crypto_session_t sid;
-	int crda_alg;
 	struct mtx lock;
 };
 
 struct ocf_operation {
 	struct ocf_session *os;
 	bool done;
 	struct iovec iov[0];
 };
 
 static MALLOC_DEFINE(M_KTLS_OCF, "ktls_ocf", "OCF KTLS");
 
 SYSCTL_DECL(_kern_ipc_tls);
 SYSCTL_DECL(_kern_ipc_tls_stats);
 
 static SYSCTL_NODE(_kern_ipc_tls_stats, OID_AUTO, ocf,
     CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
     "Kernel TLS offload via OCF stats");
 
 static counter_u64_t ocf_tls12_gcm_crypts;
 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls12_gcm_crypts,
     CTLFLAG_RD, &ocf_tls12_gcm_crypts,
     "Total number of OCF TLS 1.2 GCM encryption operations");
 
 static counter_u64_t ocf_tls13_gcm_crypts;
 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, tls13_gcm_crypts,
     CTLFLAG_RD, &ocf_tls13_gcm_crypts,
     "Total number of OCF TLS 1.3 GCM encryption operations");
 
 static counter_u64_t ocf_retries;
 SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, retries, CTLFLAG_RD,
     &ocf_retries,
     "Number of OCF encryption operation retries");
 
 static int
 ktls_ocf_callback(struct cryptop *crp)
 {
 	struct ocf_operation *oo;
 
 	oo = crp->crp_opaque;
 	mtx_lock(&oo->os->lock);
 	oo->done = true;
 	mtx_unlock(&oo->os->lock);
 	wakeup(oo);
 	return (0);
 }
 
 static int
 ktls_ocf_tls12_gcm_encrypt(struct ktls_session *tls,
     const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov,
     struct iovec *outiov, int iovcnt, uint64_t seqno,
     uint8_t record_type __unused)
 {
 	struct uio uio;
 	struct tls_aead_data ad;
-	struct tls_nonce_data nd;
-	struct cryptodesc *crde, *crda;
 	struct cryptop *crp;
 	struct ocf_session *os;
 	struct ocf_operation *oo;
 	struct iovec *iov;
 	int i, error;
 	uint16_t tls_comp_len;
 
 	os = tls->cipher;
 
 	oo = malloc(sizeof(*oo) + (iovcnt + 2) * sizeof(*iov), M_KTLS_OCF,
 	    M_WAITOK | M_ZERO);
 	oo->os = os;
 	iov = oo->iov;
 
-	crp = crypto_getreq(2);
-	if (crp == NULL) {
-		free(oo, M_KTLS_OCF);
-		return (ENOMEM);
-	}
+	crp = crypto_getreq(os->sid, M_WAITOK);
 
 	/* Setup the IV. */
-	memcpy(nd.fixed, tls->params.iv, TLS_AEAD_GCM_LEN);
-	memcpy(&nd.seq, hdr + 1, sizeof(nd.seq));
+	memcpy(crp->crp_iv, tls->params.iv, TLS_AEAD_GCM_LEN);
+	memcpy(crp->crp_iv + TLS_AEAD_GCM_LEN, hdr + 1, sizeof(uint64_t));
 
 	/* Setup the AAD. */
 	tls_comp_len = ntohs(hdr->tls_length) -
-	    (AES_GMAC_HASH_LEN + sizeof(nd.seq));
+	    (AES_GMAC_HASH_LEN + sizeof(uint64_t));
 	ad.seq = htobe64(seqno);
 	ad.type = hdr->tls_type;
 	ad.tls_vmajor = hdr->tls_vmajor;
 	ad.tls_vminor = hdr->tls_vminor;
 	ad.tls_length = htons(tls_comp_len);
 	iov[0].iov_base = &ad;
 	iov[0].iov_len = sizeof(ad);
 	uio.uio_resid = sizeof(ad);
 
 	/*
 	 * OCF always does encryption in place, so copy the data if
 	 * needed.  Ugh.
 	 */
 	for (i = 0; i < iovcnt; i++) {
 		iov[i + 1] = outiov[i];
 		if (iniov[i].iov_base != outiov[i].iov_base)
 			memcpy(outiov[i].iov_base, iniov[i].iov_base,
 			    outiov[i].iov_len);
 		uio.uio_resid += outiov[i].iov_len;
 	}
 
 	iov[iovcnt + 1].iov_base = trailer;
 	iov[iovcnt + 1].iov_len = AES_GMAC_HASH_LEN;
 	uio.uio_resid += AES_GMAC_HASH_LEN;
 
 	uio.uio_iov = iov;
 	uio.uio_iovcnt = iovcnt + 2;
 	uio.uio_offset = 0;
 	uio.uio_segflg = UIO_SYSSPACE;
 	uio.uio_td = curthread;
 
-	crp->crp_session = os->sid;
-	crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIMM;
+	crp->crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST;
+	crp->crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE;
+	crp->crp_buf_type = CRYPTO_BUF_UIO;
 	crp->crp_uio = &uio;
 	crp->crp_ilen = uio.uio_resid;
 	crp->crp_opaque = oo;
 	crp->crp_callback = ktls_ocf_callback;
 
-	crde = crp->crp_desc;
-	crda = crde->crd_next;
+	crp->crp_aad_start = 0;
+	crp->crp_aad_length = sizeof(ad);
+	crp->crp_payload_start = sizeof(ad);
+	crp->crp_payload_length = crp->crp_ilen -
+	    (sizeof(ad) + AES_GMAC_HASH_LEN);
+	crp->crp_digest_start = crp->crp_ilen - AES_GMAC_HASH_LEN;
 
-	crda->crd_alg = os->crda_alg;
-	crda->crd_skip = 0;
-	crda->crd_len = sizeof(ad);
-	crda->crd_inject = crp->crp_ilen - AES_GMAC_HASH_LEN;
-
-	crde->crd_alg = CRYPTO_AES_NIST_GCM_16;
-	crde->crd_skip = sizeof(ad);
-	crde->crd_len = crp->crp_ilen - (sizeof(ad) + AES_GMAC_HASH_LEN);
-	crde->crd_flags = CRD_F_ENCRYPT | CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
-	memcpy(crde->crd_iv, &nd, sizeof(nd));
-
 	counter_u64_add(ocf_tls12_gcm_crypts, 1);
 	for (;;) {
 		error = crypto_dispatch(crp);
 		if (error)
 			break;
 
 		mtx_lock(&os->lock);
 		while (!oo->done)
 			mtx_sleep(oo, &os->lock, 0, "ocfktls", 0);
 		mtx_unlock(&os->lock);
 
 		if (crp->crp_etype != EAGAIN) {
 			error = crp->crp_etype;
 			break;
 		}
 
 		crp->crp_etype = 0;
 		crp->crp_flags &= ~CRYPTO_F_DONE;
 		oo->done = false;
 		counter_u64_add(ocf_retries, 1);
 	}
 
 	crypto_freereq(crp);
 	free(oo, M_KTLS_OCF);
 	return (error);
 }
 
 static int
 ktls_ocf_tls13_gcm_encrypt(struct ktls_session *tls,
     const struct tls_record_layer *hdr, uint8_t *trailer, struct iovec *iniov,
     struct iovec *outiov, int iovcnt, uint64_t seqno, uint8_t record_type)
 {
 	struct uio uio;
 	struct tls_aead_data_13 ad;
 	char nonce[12];
-	struct cryptodesc *crde, *crda;
 	struct cryptop *crp;
 	struct ocf_session *os;
 	struct ocf_operation *oo;
 	struct iovec *iov;
 	int i, error;
 
 	os = tls->cipher;
 
 	oo = malloc(sizeof(*oo) + (iovcnt + 2) * sizeof(*iov), M_KTLS_OCF,
 	    M_WAITOK | M_ZERO);
 	oo->os = os;
 	iov = oo->iov;
 
-	crp = crypto_getreq(2);
-	if (crp == NULL) {
-		free(oo, M_KTLS_OCF);
-		return (ENOMEM);
-	}
+	crp = crypto_getreq(os->sid, M_WAITOK);
 
 	/* Setup the nonce. */
 	memcpy(nonce, tls->params.iv, tls->params.iv_len);
 	*(uint64_t *)(nonce + 4) ^= htobe64(seqno);
 
 	/* Setup the AAD. */
 	ad.type = hdr->tls_type;
 	ad.tls_vmajor = hdr->tls_vmajor;
 	ad.tls_vminor = hdr->tls_vminor;
 	ad.tls_length = hdr->tls_length;
 	iov[0].iov_base = &ad;
 	iov[0].iov_len = sizeof(ad);
 	uio.uio_resid = sizeof(ad);
 
 	/*
 	 * OCF always does encryption in place, so copy the data if
 	 * needed.  Ugh.
 	 */
 	for (i = 0; i < iovcnt; i++) {
 		iov[i + 1] = outiov[i];
 		if (iniov[i].iov_base != outiov[i].iov_base)
 			memcpy(outiov[i].iov_base, iniov[i].iov_base,
 			    outiov[i].iov_len);
 		uio.uio_resid += outiov[i].iov_len;
 	}
 
 	trailer[0] = record_type;
 	iov[iovcnt + 1].iov_base = trailer;
 	iov[iovcnt + 1].iov_len = AES_GMAC_HASH_LEN + 1;
 	uio.uio_resid += AES_GMAC_HASH_LEN + 1;
 
 	uio.uio_iov = iov;
 	uio.uio_iovcnt = iovcnt + 2;
 	uio.uio_offset = 0;
 	uio.uio_segflg = UIO_SYSSPACE;
 	uio.uio_td = curthread;
 
-	crp->crp_session = os->sid;
-	crp->crp_flags = CRYPTO_F_IOV | CRYPTO_F_CBIMM;
+	crp->crp_op = CRYPTO_OP_ENCRYPT | CRYPTO_OP_COMPUTE_DIGEST;
+	crp->crp_flags = CRYPTO_F_CBIMM | CRYPTO_F_IV_SEPARATE;
+	crp->crp_buf_type = CRYPTO_BUF_UIO;
 	crp->crp_uio = &uio;
 	crp->crp_ilen = uio.uio_resid;
 	crp->crp_opaque = oo;
 	crp->crp_callback = ktls_ocf_callback;
 
-	crde = crp->crp_desc;
-	crda = crde->crd_next;
+	crp->crp_aad_start = 0;
+	crp->crp_aad_length = sizeof(ad);
+	crp->crp_payload_start = sizeof(ad);
+	crp->crp_payload_length = crp->crp_ilen -
+	    (sizeof(ad) + AES_GMAC_HASH_LEN);
+	crp->crp_digest_start = crp->crp_ilen - AES_GMAC_HASH_LEN;
+	memcpy(crp->crp_iv, nonce, sizeof(nonce));
 
-	crda->crd_alg = os->crda_alg;
-	crda->crd_skip = 0;
-	crda->crd_len = sizeof(ad);
-	crda->crd_inject = crp->crp_ilen - AES_GMAC_HASH_LEN;
-
-	crde->crd_alg = CRYPTO_AES_NIST_GCM_16;
-	crde->crd_skip = sizeof(ad);
-	crde->crd_len = crp->crp_ilen - (sizeof(ad) + AES_GMAC_HASH_LEN);
-	crde->crd_flags = CRD_F_ENCRYPT | CRD_F_IV_EXPLICIT | CRD_F_IV_PRESENT;
-	memcpy(crde->crd_iv, nonce, sizeof(nonce));
-
 	counter_u64_add(ocf_tls13_gcm_crypts, 1);
 	for (;;) {
 		error = crypto_dispatch(crp);
 		if (error)
 			break;
 
 		mtx_lock(&os->lock);
 		while (!oo->done)
 			mtx_sleep(oo, &os->lock, 0, "ocfktls", 0);
 		mtx_unlock(&os->lock);
 
 		if (crp->crp_etype != EAGAIN) {
 			error = crp->crp_etype;
 			break;
 		}
 
 		crp->crp_etype = 0;
 		crp->crp_flags &= ~CRYPTO_F_DONE;
 		oo->done = false;
 		counter_u64_add(ocf_retries, 1);
 	}
 
 	crypto_freereq(crp);
 	free(oo, M_KTLS_OCF);
 	return (error);
 }
 
 static void
 ktls_ocf_free(struct ktls_session *tls)
 {
 	struct ocf_session *os;
 
 	os = tls->cipher;
+	crypto_freesession(os->sid);
 	mtx_destroy(&os->lock);
 	explicit_bzero(os, sizeof(*os));
 	free(os, M_KTLS_OCF);
 }
 
 static int
 ktls_ocf_try(struct socket *so, struct ktls_session *tls)
 {
-	struct cryptoini cria, crie;
+	struct crypto_session_params csp;
 	struct ocf_session *os;
 	int error;
 
-	memset(&cria, 0, sizeof(cria));
-	memset(&crie, 0, sizeof(crie));
+	memset(&csp, 0, sizeof(csp));
 
 	switch (tls->params.cipher_algorithm) {
 	case CRYPTO_AES_NIST_GCM_16:
 		switch (tls->params.cipher_key_len) {
 		case 128 / 8:
-			cria.cri_alg = CRYPTO_AES_128_NIST_GMAC;
-			break;
 		case 256 / 8:
-			cria.cri_alg = CRYPTO_AES_256_NIST_GMAC;
 			break;
 		default:
 			return (EINVAL);
 		}
-		cria.cri_key = tls->params.cipher_key;
-		cria.cri_klen = tls->params.cipher_key_len * 8;
+		csp.csp_mode = CSP_MODE_AEAD;
+		csp.csp_cipher_alg = CRYPTO_AES_NIST_GCM_16;
+		csp.csp_cipher_key = tls->params.cipher_key;
+		csp.csp_cipher_klen = tls->params.cipher_key_len;
+		csp.csp_ivlen = AES_GCM_IV_LEN;
 		break;
 	default:
 		return (EPROTONOSUPPORT);
 	}
 
 	/* Only TLS 1.2 and 1.3 are supported. */
 	if (tls->params.tls_vmajor != TLS_MAJOR_VER_ONE ||
 	    tls->params.tls_vminor < TLS_MINOR_VER_TWO ||
 	    tls->params.tls_vminor > TLS_MINOR_VER_THREE)
 		return (EPROTONOSUPPORT);
 
 	os = malloc(sizeof(*os), M_KTLS_OCF, M_NOWAIT | M_ZERO);
 	if (os == NULL)
 		return (ENOMEM);
 
-	crie.cri_alg = tls->params.cipher_algorithm;
-	crie.cri_key = tls->params.cipher_key;
-	crie.cri_klen = tls->params.cipher_key_len * 8;
-
-	crie.cri_next = &cria;
-	error = crypto_newsession(&os->sid, &crie,
+	error = crypto_newsession(&os->sid, &csp,
 	    CRYPTO_FLAG_HARDWARE | CRYPTO_FLAG_SOFTWARE);
 	if (error) {
 		free(os, M_KTLS_OCF);
 		return (error);
 	}
 
-	os->crda_alg = cria.cri_alg;
 	mtx_init(&os->lock, "ktls_ocf", NULL, MTX_DEF);
 	tls->cipher = os;
 	if (tls->params.tls_vminor == TLS_MINOR_VER_THREE)
 		tls->sw_encrypt = ktls_ocf_tls13_gcm_encrypt;
 	else
 		tls->sw_encrypt = ktls_ocf_tls12_gcm_encrypt;
 	tls->free = ktls_ocf_free;
 	return (0);
 }
 
 struct ktls_crypto_backend ocf_backend = {
 	.name = "OCF",
 	.prio = 5,
 	.api_version = KTLS_API_VERSION,
 	.try = ktls_ocf_try,
 };
 
 static int
 ktls_ocf_modevent(module_t mod, int what, void *arg)
 {
 	int error;
 
 	switch (what) {
 	case MOD_LOAD:
 		ocf_tls12_gcm_crypts = counter_u64_alloc(M_WAITOK);
 		ocf_tls13_gcm_crypts = counter_u64_alloc(M_WAITOK);
 		ocf_retries = counter_u64_alloc(M_WAITOK);
 		return (ktls_crypto_backend_register(&ocf_backend));
 	case MOD_UNLOAD:
 		error = ktls_crypto_backend_deregister(&ocf_backend);
 		if (error)
 			return (error);
 		counter_u64_free(ocf_tls12_gcm_crypts);
 		counter_u64_free(ocf_tls13_gcm_crypts);
 		counter_u64_free(ocf_retries);
 		return (0);
 	default:
 		return (EOPNOTSUPP);
 	}
 }
 
 static moduledata_t ktls_ocf_moduledata = {
 	"ktls_ocf",
 	ktls_ocf_modevent,
 	NULL
 };
 
 DECLARE_MODULE(ktls_ocf, ktls_ocf_moduledata, SI_SUB_PROTO_END, SI_ORDER_ANY);
Index: head/sys/opencrypto/xform_gmac.c
===================================================================
--- head/sys/opencrypto/xform_gmac.c	(revision 359373)
+++ head/sys/opencrypto/xform_gmac.c	(revision 359374)
@@ -1,99 +1,99 @@
 /*	$OpenBSD: xform.c,v 1.16 2001/08/28 12:20:43 ben Exp $	*/
 /*-
  * The authors of this code are John Ioannidis (ji@tla.org),
  * Angelos D. Keromytis (kermit@csd.uch.gr),
  * Niels Provos (provos@physnet.uni-hamburg.de) and
  * Damien Miller (djm@mindrot.org).
  *
  * This code was written by John Ioannidis for BSD/OS in Athens, Greece,
  * in November 1995.
  *
  * Ported to OpenBSD and NetBSD, with additional transforms, in December 1996,
  * by Angelos D. Keromytis.
  *
  * Additional transforms and features in 1997 and 1998 by Angelos D. Keromytis
  * and Niels Provos.
  *
  * Additional features in 1999 by Angelos D. Keromytis.
  *
  * AES XTS implementation in 2008 by Damien Miller
  *
  * Copyright (C) 1995, 1996, 1997, 1998, 1999 by John Ioannidis,
  * Angelos D. Keromytis and Niels Provos.
  *
  * Copyright (C) 2001, Angelos D. Keromytis.
  *
  * Copyright (C) 2008, Damien Miller
  * Copyright (c) 2014 The FreeBSD Foundation
  * All rights reserved.
  *
  * Portions of this software were developed by John-Mark Gurney
  * under sponsorship of the FreeBSD Foundation and
  * Rubicon Communications, LLC (Netgate).
  *
  * Permission to use, copy, and modify this software with or without fee
  * is hereby granted, provided that this entire notice is included in
  * all copies of any software which is or includes a copy or
  * modification of this software.
  * You may use this code under the GNU public license if you so wish. Please
  * contribute changes back to the authors under this freer than GPL license
  * so that we may further the use of strong encryption without limitations to
  * all.
  *
  * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR
  * IMPLIED WARRANTY. IN PARTICULAR, NONE OF THE AUTHORS MAKES ANY
  * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE
  * MERCHANTABILITY OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR
  * PURPOSE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include <opencrypto/gmac.h>
 #include <opencrypto/xform_auth.h>
 
 /* Encryption instances */
 struct enc_xform enc_xform_aes_nist_gmac = {
 	CRYPTO_AES_NIST_GMAC, "AES-GMAC",
 	AES_ICM_BLOCK_LEN, AES_GCM_IV_LEN, AES_MIN_KEY, AES_MAX_KEY,
 	NULL,
 	NULL,
 	NULL,
 	NULL,
 	NULL,
 };
 
 /* Authentication instances */
 struct auth_hash auth_hash_nist_gmac_aes_128 = {
-	CRYPTO_AES_128_NIST_GMAC, "GMAC-AES-128",
+	CRYPTO_AES_NIST_GMAC, "GMAC-AES-128",
 	AES_128_GMAC_KEY_LEN, AES_GMAC_HASH_LEN, sizeof(struct aes_gmac_ctx),
 	GMAC_BLOCK_LEN,
 	(void (*)(void *)) AES_GMAC_Init,
 	(void (*)(void *, const u_int8_t *, u_int16_t)) AES_GMAC_Setkey,
 	(void (*)(void *, const u_int8_t *, u_int16_t)) AES_GMAC_Reinit,
 	(int  (*)(void *, const u_int8_t *, u_int16_t)) AES_GMAC_Update,
 	(void (*)(u_int8_t *, void *)) AES_GMAC_Final
 };
 
 struct auth_hash auth_hash_nist_gmac_aes_192 = {
-	CRYPTO_AES_192_NIST_GMAC, "GMAC-AES-192",
+	CRYPTO_AES_NIST_GMAC, "GMAC-AES-192",
 	AES_192_GMAC_KEY_LEN, AES_GMAC_HASH_LEN, sizeof(struct aes_gmac_ctx),
 	GMAC_BLOCK_LEN,
 	(void (*)(void *)) AES_GMAC_Init,
 	(void (*)(void *, const u_int8_t *, u_int16_t)) AES_GMAC_Setkey,
 	(void (*)(void *, const u_int8_t *, u_int16_t)) AES_GMAC_Reinit,
 	(int  (*)(void *, const u_int8_t *, u_int16_t)) AES_GMAC_Update,
 	(void (*)(u_int8_t *, void *)) AES_GMAC_Final
 };
 
 struct auth_hash auth_hash_nist_gmac_aes_256 = {
-	CRYPTO_AES_256_NIST_GMAC, "GMAC-AES-256",
+	CRYPTO_AES_NIST_GMAC, "GMAC-AES-256",
 	AES_256_GMAC_KEY_LEN, AES_GMAC_HASH_LEN, sizeof(struct aes_gmac_ctx),
 	GMAC_BLOCK_LEN,
 	(void (*)(void *)) AES_GMAC_Init,
 	(void (*)(void *, const u_int8_t *, u_int16_t)) AES_GMAC_Setkey,
 	(void (*)(void *, const u_int8_t *, u_int16_t)) AES_GMAC_Reinit,
 	(int  (*)(void *, const u_int8_t *, u_int16_t)) AES_GMAC_Update,
 	(void (*)(u_int8_t *, void *)) AES_GMAC_Final
 };
Index: head/sys/sys/bus_dma.h
===================================================================
--- head/sys/sys/bus_dma.h	(revision 359373)
+++ head/sys/sys/bus_dma.h	(revision 359374)
@@ -1,328 +1,336 @@
 /*	$NetBSD: bus.h,v 1.12 1997/10/01 08:25:15 fvdl Exp $	*/
 
 /*-
  * SPDX-License-Identifier: (BSD-2-Clause-NetBSD AND BSD-4-Clause)
  *
  * Copyright (c) 1996, 1997 The NetBSD Foundation, Inc.
  * All rights reserved.
  *
  * This code is derived from software contributed to The NetBSD Foundation
  * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
  * NASA Ames Research Center.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  */
 
 /*-
  * Copyright (c) 1996 Charles M. Hannum.  All rights reserved.
  * Copyright (c) 1996 Christopher G. Demetriou.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  * 3. All advertising materials mentioning features or use of this software
  *    must display the following acknowledgement:
  *      This product includes software developed by Christopher G. Demetriou
  *	for the NetBSD Project.
  * 4. The name of the author may not be used to endorse or promote products
  *    derived from this software without specific prior written permission
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 /* $FreeBSD$ */
 
 #ifndef _BUS_DMA_H_
 #define _BUS_DMA_H_
 
 #ifdef _KERNEL
 #include <sys/_bus_dma.h>
 #endif
 
 /*
  * Machine independent interface for mapping physical addresses to peripheral
  * bus 'physical' addresses, and assisting with DMA operations.
  *
  * XXX This file is always included from <machine/bus_dma.h> and should not
  *     (yet) be included directly.
  */
 
 /*
  * Flags used in various bus DMA methods.
  */
 #define	BUS_DMA_WAITOK		0x00	/* safe to sleep (pseudo-flag) */
 #define	BUS_DMA_NOWAIT		0x01	/* not safe to sleep */
 #define	BUS_DMA_ALLOCNOW	0x02	/* perform resource allocation now */
 #define	BUS_DMA_COHERENT	0x04	/* hint: map memory in a coherent way */
 #define	BUS_DMA_ZERO		0x08	/* allocate zero'ed memory */
 #define	BUS_DMA_BUS1		0x10	/* placeholders for bus functions... */
 #define	BUS_DMA_BUS2		0x20
 #define	BUS_DMA_BUS3		0x40
 #define	BUS_DMA_BUS4		0x80
 
 /*
  * The following two flags are non-standard or specific to only certain
  * architectures
  */
 #define	BUS_DMA_NOWRITE		0x100
 #define	BUS_DMA_NOCACHE		0x200
 
 /*
  * The following flag is a DMA tag hint that the page offset of the
  * loaded kernel virtual address must be preserved in the first
  * physical segment address, when the KVA is loaded into DMA.
  */
 #define	BUS_DMA_KEEP_PG_OFFSET	0x400
 
 #define	BUS_DMA_LOAD_MBUF	0x800
 
 /* Forwards needed by prototypes below. */
 union ccb;
 struct bio;
+struct cryptop;
 struct mbuf;
 struct memdesc;
 struct pmap;
 struct uio;
 
 /*
  * Operations performed by bus_dmamap_sync().
  */
 #define	BUS_DMASYNC_PREREAD	1
 #define	BUS_DMASYNC_POSTREAD	2
 #define	BUS_DMASYNC_PREWRITE	4
 #define	BUS_DMASYNC_POSTWRITE	8
 
 /*
  *	bus_dma_segment_t
  *
  *	Describes a single contiguous DMA transaction.  Values
  *	are suitable for programming into DMA registers.
  */
 typedef struct bus_dma_segment {
 	bus_addr_t	ds_addr;	/* DMA address */
 	bus_size_t	ds_len;		/* length of transfer */
 } bus_dma_segment_t;
 
 #ifdef _KERNEL
 /*
  * A function that returns 1 if the address cannot be accessed by
  * a device and 0 if it can be.
  */
 typedef int bus_dma_filter_t(void *, bus_addr_t);
 
 /*
  * Generic helper function for manipulating mutexes.
  */
 void busdma_lock_mutex(void *arg, bus_dma_lock_op_t op);
 
 /*
  * Allocate a device specific dma_tag encapsulating the constraints of
  * the parent tag in addition to other restrictions specified:
  *
  *	alignment:	Alignment for segments.
  *	boundary:	Boundary that segments cannot cross.
  *	lowaddr:	Low restricted address that cannot appear in a mapping.
  *	highaddr:	High restricted address that cannot appear in a mapping.
  *	filtfunc:	An optional function to further test if an address
  *			within the range of lowaddr and highaddr cannot appear
  *			in a mapping.
  *	filtfuncarg:	An argument that will be passed to filtfunc in addition
  *			to the address to test.
  *	maxsize:	Maximum mapping size supported by this tag.
  *	nsegments:	Number of discontinuities allowed in maps.
  *	maxsegsz:	Maximum size of a segment in the map.
  *	flags:		Bus DMA flags.
  *	lockfunc:	An optional function to handle driver-defined lock
  *			operations.
  *	lockfuncarg:	An argument that will be passed to lockfunc in addition
  *			to the lock operation.
  *	dmat:		A pointer to set to a valid dma tag should the return
  *			value of this function indicate success.
  */
 /* XXX Should probably allow specification of alignment */
 int bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
 		       bus_addr_t boundary, bus_addr_t lowaddr,
 		       bus_addr_t highaddr, bus_dma_filter_t *filtfunc,
 		       void *filtfuncarg, bus_size_t maxsize, int nsegments,
 		       bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc,
 		       void *lockfuncarg, bus_dma_tag_t *dmat);
 
 /* Functions for creating and cloning tags via a template */
 typedef struct {
 	bus_dma_tag_t		parent;
 	bus_size_t		alignment;
 	bus_addr_t		boundary;
 	bus_addr_t		lowaddr;
 	bus_addr_t		highaddr;
 	bus_size_t		maxsize;
 	int			nsegments;
 	bus_size_t		maxsegsize;
 	int			flags;
 	bus_dma_lock_t		*lockfunc;
 	void			*lockfuncarg;
 } bus_dma_tag_template_t;
 void bus_dma_template_init(bus_dma_tag_template_t *t, bus_dma_tag_t parent);
 int bus_dma_template_tag(bus_dma_tag_template_t *t, bus_dma_tag_t *dmat);
 void bus_dma_template_clone(bus_dma_tag_template_t *t, bus_dma_tag_t dmat);
 
 /*
  * Set the memory domain to be used for allocations.
  *
  * Automatic for PCI devices.  Must be set prior to creating maps or
  * allocating memory.
  */
 int bus_dma_tag_set_domain(bus_dma_tag_t dmat, int domain);
 
 int bus_dma_tag_destroy(bus_dma_tag_t dmat);
 
 /*
  * A function that processes a successfully loaded dma map or an error
  * from a delayed load map.
  */
 typedef void bus_dmamap_callback_t(void *, bus_dma_segment_t *, int, int);
 
 /*
  * Like bus_dmamap_callback but includes map size in bytes.  This is
  * defined as a separate interface to maintain compatibility for users
  * of bus_dmamap_callback_t--at some point these interfaces should be merged.
  */
 typedef void bus_dmamap_callback2_t(void *, bus_dma_segment_t *, int, bus_size_t, int);
 
 /*
  * Map the buffer buf into bus space using the dmamap map.
  */
 int bus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,
 		    bus_size_t buflen, bus_dmamap_callback_t *callback,
 		    void *callback_arg, int flags);
 
 /*
  * Like bus_dmamap_load but for mbufs.  Note the use of the
  * bus_dmamap_callback2_t interface.
  */
 int bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map,
 			 struct mbuf *mbuf,
 			 bus_dmamap_callback2_t *callback, void *callback_arg,
 			 int flags);
 
 int bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map,
 			    struct mbuf *mbuf, bus_dma_segment_t *segs,
 			    int *nsegs, int flags);
 
 /*
  * Like bus_dmamap_load but for uios.  Note the use of the
  * bus_dmamap_callback2_t interface.
  */
 int bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map,
 			struct uio *ui,
 			bus_dmamap_callback2_t *callback, void *callback_arg,
 			int flags);
 
 /*
  * Like bus_dmamap_load but for cam control blocks.
  */
 int bus_dmamap_load_ccb(bus_dma_tag_t dmat, bus_dmamap_t map, union ccb *ccb,
 			bus_dmamap_callback_t *callback, void *callback_arg,
 			int flags);
 
 /*
  * Like bus_dmamap_load but for bios.
  */
 int bus_dmamap_load_bio(bus_dma_tag_t dmat, bus_dmamap_t map, struct bio *bio,
 			bus_dmamap_callback_t *callback, void *callback_arg,
 			int flags);
+
+/*
+ * Like bus_dmamap_load but for crypto ops.
+ */
+int bus_dmamap_load_crp(bus_dma_tag_t dmat, bus_dmamap_t map,
+			struct cryptop *crp, bus_dmamap_callback_t *callback,
+			void *callback_arg, int flags);
 
 /*
  * Loads any memory descriptor.
  */
 int bus_dmamap_load_mem(bus_dma_tag_t dmat, bus_dmamap_t map,
 			struct memdesc *mem, bus_dmamap_callback_t *callback,
 			void *callback_arg, int flags);
 
 /*
  * Placeholder for use by busdma implementations which do not benefit
  * from optimized procedure to load an array of vm_page_t.  Falls back
  * to do _bus_dmamap_load_phys() in loop.
  */
 int bus_dmamap_load_ma_triv(bus_dma_tag_t dmat, bus_dmamap_t map,
     struct vm_page **ma, bus_size_t tlen, int ma_offs, int flags,
     bus_dma_segment_t *segs, int *segp);
 
 #ifdef WANT_INLINE_DMAMAP
 #define BUS_DMAMAP_OP static inline
 #else
 #define BUS_DMAMAP_OP
 #endif
 
 /*
  * Allocate a handle for mapping from kva/uva/physical
  * address space into bus device space.
  */
 BUS_DMAMAP_OP int bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp);
 
 /*
  * Destroy a handle for mapping from kva/uva/physical
  * address space into bus device space.
  */
 BUS_DMAMAP_OP int bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map);
 
 /*
  * Allocate a piece of memory that can be efficiently mapped into
  * bus device space based on the constraints listed in the dma tag.
  * A dmamap to for use with dmamap_load is also allocated.
  */
 BUS_DMAMAP_OP int bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
 		     bus_dmamap_t *mapp);
 
 /*
  * Free a piece of memory and its allocated dmamap, that was allocated
  * via bus_dmamem_alloc.
  */
 BUS_DMAMAP_OP void bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map);
 
 /*
  * Perform a synchronization operation on the given map. If the map
  * is NULL we have a fully IO-coherent system.
  */
 BUS_DMAMAP_OP void bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t dmamap, bus_dmasync_op_t op);
 
 /*
  * Release the mapping held by map.
  */
 BUS_DMAMAP_OP void bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t dmamap);
 
 #undef BUS_DMAMAP_OP
 #endif /* _KERNEL */
 
 #endif /* _BUS_DMA_H_ */
Index: head/sys/sys/param.h
===================================================================
--- head/sys/sys/param.h	(revision 359373)
+++ head/sys/sys/param.h	(revision 359374)
@@ -1,368 +1,368 @@
 /*-
  * SPDX-License-Identifier: BSD-3-Clause
  *
  * Copyright (c) 1982, 1986, 1989, 1993
  *	The Regents of the University of California.  All rights reserved.
  * (c) UNIX System Laboratories, Inc.
  * All or some portions of this file are derived from material licensed
  * to the University of California by American Telephone and Telegraph
  * Co. or Unix System Laboratories, Inc. and are reproduced herein with
  * the permission of UNIX System Laboratories, Inc.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  * 3. Neither the name of the University nor the names of its contributors
  *    may be used to endorse or promote products derived from this software
  *    without specific prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
  *	@(#)param.h	8.3 (Berkeley) 4/4/95
  * $FreeBSD$
  */
 
 #ifndef _SYS_PARAM_H_
 #define _SYS_PARAM_H_
 
 #include <sys/_null.h>
 
 #define	BSD	199506		/* System version (year & month). */
 #define BSD4_3	1
 #define BSD4_4	1
 
 /*
  * __FreeBSD_version numbers are documented in the Porter's Handbook.
  * If you bump the version for any reason, you should update the documentation
  * there.
  * Currently this lives here in the doc/ repository:
  *
  *	head/en_US.ISO8859-1/books/porters-handbook/versions/chapter.xml
  *
  * scheme is:  <major><two digit minor>Rxx
  *		'R' is in the range 0 to 4 if this is a release branch or
  *		X.0-CURRENT before releng/X.0 is created, otherwise 'R' is
  *		in the range 5 to 9.
  */
 #undef __FreeBSD_version
-#define __FreeBSD_version 1300086	/* Master, propagated to newvers */
+#define __FreeBSD_version 1300087	/* Master, propagated to newvers */
 
 /*
  * __FreeBSD_kernel__ indicates that this system uses the kernel of FreeBSD,
  * which by definition is always true on FreeBSD. This macro is also defined
  * on other systems that use the kernel of FreeBSD, such as GNU/kFreeBSD.
  *
  * It is tempting to use this macro in userland code when we want to enable
  * kernel-specific routines, and in fact it's fine to do this in code that
  * is part of FreeBSD itself.  However, be aware that as presence of this
  * macro is still not widespread (e.g. older FreeBSD versions, 3rd party
  * compilers, etc), it is STRONGLY DISCOURAGED to check for this macro in
  * external applications without also checking for __FreeBSD__ as an
  * alternative.
  */
 #undef __FreeBSD_kernel__
 #define __FreeBSD_kernel__
 
 #if defined(_KERNEL) || defined(IN_RTLD)
 #define	P_OSREL_SIGWAIT			700000
 #define	P_OSREL_SIGSEGV			700004
 #define	P_OSREL_MAP_ANON		800104
 #define	P_OSREL_MAP_FSTRICT		1100036
 #define	P_OSREL_SHUTDOWN_ENOTCONN	1100077
 #define	P_OSREL_MAP_GUARD		1200035
 #define	P_OSREL_WRFSBASE		1200041
 #define	P_OSREL_CK_CYLGRP		1200046
 #define	P_OSREL_VMTOTAL64		1200054
 #define	P_OSREL_CK_SUPERBLOCK		1300000
 #define	P_OSREL_CK_INODE		1300005
 #define	P_OSREL_POWERPC_NEW_AUX_ARGS	1300070
 
 #define	P_OSREL_MAJOR(x)		((x) / 100000)
 #endif
 
 #ifndef LOCORE
 #include <sys/types.h>
 #endif
 
 /*
  * Machine-independent constants (some used in following include files).
  * Redefined constants are from POSIX 1003.1 limits file.
  *
  * MAXCOMLEN should be >= sizeof(ac_comm) (see <acct.h>)
  */
 #include <sys/syslimits.h>
 
 #define	MAXCOMLEN	19		/* max command name remembered */
 #define	MAXINTERP	PATH_MAX	/* max interpreter file name length */
 #define	MAXLOGNAME	33		/* max login name length (incl. NUL) */
 #define	MAXUPRC		CHILD_MAX	/* max simultaneous processes */
 #define	NCARGS		ARG_MAX		/* max bytes for an exec function */
 #define	NGROUPS		(NGROUPS_MAX+1)	/* max number groups */
 #define	NOFILE		OPEN_MAX	/* max open files per process */
 #define	NOGROUP		65535		/* marker for empty group set member */
 #define MAXHOSTNAMELEN	256		/* max hostname size */
 #define SPECNAMELEN	255		/* max length of devicename */
 
 /* More types and definitions used throughout the kernel. */
 #ifdef _KERNEL
 #include <sys/cdefs.h>
 #include <sys/errno.h>
 #ifndef LOCORE
 #include <sys/time.h>
 #include <sys/priority.h>
 #endif
 
 #ifndef FALSE
 #define	FALSE	0
 #endif
 #ifndef TRUE
 #define	TRUE	1
 #endif
 #endif
 
 #ifndef _KERNEL
 /* Signals. */
 #include <sys/signal.h>
 #endif
 
 /* Machine type dependent parameters. */
 #include <machine/param.h>
 #ifndef _KERNEL
 #include <sys/limits.h>
 #endif
 
 #ifndef DEV_BSHIFT
 #define	DEV_BSHIFT	9		/* log2(DEV_BSIZE) */
 #endif
 #define	DEV_BSIZE	(1<<DEV_BSHIFT)
 
 #ifndef BLKDEV_IOSIZE
 #define BLKDEV_IOSIZE  PAGE_SIZE	/* default block device I/O size */
 #endif
 #ifndef DFLTPHYS
 #define DFLTPHYS	(64 * 1024)	/* default max raw I/O transfer size */
 #endif
 #ifndef MAXPHYS
 #define MAXPHYS		(128 * 1024)	/* max raw I/O transfer size */
 #endif
 #ifndef MAXDUMPPGS
 #define MAXDUMPPGS	(DFLTPHYS/PAGE_SIZE)
 #endif
 
 /*
  * Constants related to network buffer management.
  * MCLBYTES must be no larger than PAGE_SIZE.
  */
 #ifndef	MSIZE
 #define	MSIZE		256		/* size of an mbuf */
 #endif
 
 #ifndef	MCLSHIFT
 #define MCLSHIFT	11		/* convert bytes to mbuf clusters */
 #endif	/* MCLSHIFT */
 
 #define MCLBYTES	(1 << MCLSHIFT)	/* size of an mbuf cluster */
 
 #if PAGE_SIZE < 2048
 #define	MJUMPAGESIZE	MCLBYTES
 #elif PAGE_SIZE <= 8192
 #define	MJUMPAGESIZE	PAGE_SIZE
 #else
 #define	MJUMPAGESIZE	(8 * 1024)
 #endif
 
 #define	MJUM9BYTES	(9 * 1024)	/* jumbo cluster 9k */
 #define	MJUM16BYTES	(16 * 1024)	/* jumbo cluster 16k */
 
 /*
  * Some macros for units conversion
  */
 
 /* clicks to bytes */
 #ifndef ctob
 #define ctob(x)	((x)<<PAGE_SHIFT)
 #endif
 
 /* bytes to clicks */
 #ifndef btoc
 #define btoc(x)	(((vm_offset_t)(x)+PAGE_MASK)>>PAGE_SHIFT)
 #endif
 
 /*
  * btodb() is messy and perhaps slow because `bytes' may be an off_t.  We
  * want to shift an unsigned type to avoid sign extension and we don't
  * want to widen `bytes' unnecessarily.  Assume that the result fits in
  * a daddr_t.
  */
 #ifndef btodb
 #define btodb(bytes)	 		/* calculates (bytes / DEV_BSIZE) */ \
 	(sizeof (bytes) > sizeof(long) \
 	 ? (daddr_t)((unsigned long long)(bytes) >> DEV_BSHIFT) \
 	 : (daddr_t)((unsigned long)(bytes) >> DEV_BSHIFT))
 #endif
 
 #ifndef dbtob
 #define dbtob(db)			/* calculates (db * DEV_BSIZE) */ \
 	((off_t)(db) << DEV_BSHIFT)
 #endif
 
 #define	PRIMASK	0x0ff
 #define	PCATCH	0x100		/* OR'd with pri for tsleep to check signals */
 #define	PDROP	0x200	/* OR'd with pri to stop re-entry of interlock mutex */
 
 #define	NZERO	0		/* default "nice" */
 
 #define	NBBY	8		/* number of bits in a byte */
 #define	NBPW	sizeof(int)	/* number of bytes per word (integer) */
 
 #define	CMASK	022		/* default file mask: S_IWGRP|S_IWOTH */
 
 #define	NODEV	(dev_t)(-1)	/* non-existent device */
 
 /*
  * File system parameters and macros.
  *
  * MAXBSIZE -	Filesystems are made out of blocks of at most MAXBSIZE bytes
  *		per block.  MAXBSIZE may be made larger without effecting
  *		any existing filesystems as long as it does not exceed MAXPHYS,
  *		and may be made smaller at the risk of not being able to use
  *		filesystems which require a block size exceeding MAXBSIZE.
  *
  * MAXBCACHEBUF - Maximum size of a buffer in the buffer cache.  This must
  *		be >= MAXBSIZE and can be set differently for different
  *		architectures by defining it in <machine/param.h>.
  *		Making this larger allows NFS to do larger reads/writes.
  *
  * BKVASIZE -	Nominal buffer space per buffer, in bytes.  BKVASIZE is the
  *		minimum KVM memory reservation the kernel is willing to make.
  *		Filesystems can of course request smaller chunks.  Actual
  *		backing memory uses a chunk size of a page (PAGE_SIZE).
  *		The default value here can be overridden on a per-architecture
  *		basis by defining it in <machine/param.h>.
  *
  *		If you make BKVASIZE too small you risk seriously fragmenting
  *		the buffer KVM map which may slow things down a bit.  If you
  *		make it too big the kernel will not be able to optimally use
  *		the KVM memory reserved for the buffer cache and will wind
  *		up with too-few buffers.
  *
  *		The default is 16384, roughly 2x the block size used by a
  *		normal UFS filesystem.
  */
 #define MAXBSIZE	65536	/* must be power of 2 */
 #ifndef	MAXBCACHEBUF
 #define	MAXBCACHEBUF	MAXBSIZE /* must be a power of 2 >= MAXBSIZE */
 #endif
 #ifndef	BKVASIZE
 #define BKVASIZE	16384	/* must be power of 2 */
 #endif
 #define BKVAMASK	(BKVASIZE-1)
 
 /*
  * MAXPATHLEN defines the longest permissible path length after expanding
  * symbolic links. It is used to allocate a temporary buffer from the buffer
  * pool in which to do the name expansion, hence should be a power of two,
  * and must be less than or equal to MAXBSIZE.  MAXSYMLINKS defines the
  * maximum number of symbolic links that may be expanded in a path name.
  * It should be set high enough to allow all legitimate uses, but halt
  * infinite loops reasonably quickly.
  */
 #define	MAXPATHLEN	PATH_MAX
 #define MAXSYMLINKS	32
 
 /* Bit map related macros. */
 #define	setbit(a,i)	(((unsigned char *)(a))[(i)/NBBY] |= 1<<((i)%NBBY))
 #define	clrbit(a,i)	(((unsigned char *)(a))[(i)/NBBY] &= ~(1<<((i)%NBBY)))
 #define	isset(a,i)							\
 	(((const unsigned char *)(a))[(i)/NBBY] & (1<<((i)%NBBY)))
 #define	isclr(a,i)							\
 	((((const unsigned char *)(a))[(i)/NBBY] & (1<<((i)%NBBY))) == 0)
 
 /* Macros for counting and rounding. */
 #ifndef howmany
 #define	howmany(x, y)	(((x)+((y)-1))/(y))
 #endif
 #define	nitems(x)	(sizeof((x)) / sizeof((x)[0]))
 #define	rounddown(x, y)	(((x)/(y))*(y))
 #define	rounddown2(x, y) ((x)&(~((y)-1)))          /* if y is power of two */
 #define	roundup(x, y)	((((x)+((y)-1))/(y))*(y))  /* to any y */
 #define	roundup2(x, y)	(((x)+((y)-1))&(~((y)-1))) /* if y is powers of two */
 #define powerof2(x)	((((x)-1)&(x))==0)
 
 /* Macros for min/max. */
 #define	MIN(a,b) (((a)<(b))?(a):(b))
 #define	MAX(a,b) (((a)>(b))?(a):(b))
 
 #ifdef _KERNEL
 /*
  * Basic byte order function prototypes for non-inline functions.
  */
 #ifndef LOCORE
 #ifndef _BYTEORDER_PROTOTYPED
 #define	_BYTEORDER_PROTOTYPED
 __BEGIN_DECLS
 __uint32_t	 htonl(__uint32_t);
 __uint16_t	 htons(__uint16_t);
 __uint32_t	 ntohl(__uint32_t);
 __uint16_t	 ntohs(__uint16_t);
 __END_DECLS
 #endif
 #endif
 
 #ifndef _BYTEORDER_FUNC_DEFINED
 #define	_BYTEORDER_FUNC_DEFINED
 #define	htonl(x)	__htonl(x)
 #define	htons(x)	__htons(x)
 #define	ntohl(x)	__ntohl(x)
 #define	ntohs(x)	__ntohs(x)
 #endif /* !_BYTEORDER_FUNC_DEFINED */
 #endif /* _KERNEL */
 
 /*
  * Scale factor for scaled integers used to count %cpu time and load avgs.
  *
  * The number of CPU `tick's that map to a unique `%age' can be expressed
  * by the formula (1 / (2 ^ (FSHIFT - 11))).  The maximum load average that
  * can be calculated (assuming 32 bits) can be closely approximated using
  * the formula (2 ^ (2 * (16 - FSHIFT))) for (FSHIFT < 15).
  *
  * For the scheduler to maintain a 1:1 mapping of CPU `tick' to `%age',
  * FSHIFT must be at least 11; this gives us a maximum load avg of ~1024.
  */
 #define	FSHIFT	11		/* bits to right of fixed binary point */
 #define FSCALE	(1<<FSHIFT)
 
 #define dbtoc(db)			/* calculates devblks to pages */ \
 	((db + (ctodb(1) - 1)) >> (PAGE_SHIFT - DEV_BSHIFT))
 
 #define ctodb(db)			/* calculates pages to devblks */ \
 	((db) << (PAGE_SHIFT - DEV_BSHIFT))
 
 /*
  * Old spelling of __containerof().
  */
 #define	member2struct(s, m, x)						\
 	((struct s *)(void *)((char *)(x) - offsetof(struct s, m)))
 
 /*
  * Access a variable length array that has been declared as a fixed
  * length array.
  */
 #define __PAST_END(array, offset) (((__typeof__(*(array)) *)(array))[offset])
 
 #endif	/* _SYS_PARAM_H_ */
Index: head/tests/sys/opencrypto/cryptodev.py
===================================================================
--- head/tests/sys/opencrypto/cryptodev.py	(revision 359373)
+++ head/tests/sys/opencrypto/cryptodev.py	(revision 359374)
@@ -1,725 +1,724 @@
 #!/usr/local/bin/python2
 #
 # Copyright (c) 2014 The FreeBSD Foundation
 # Copyright 2014 John-Mark Gurney
 # All rights reserved.
 # Copyright 2019 Enji Cooper
 #
 # This software was developed by John-Mark Gurney under
 # the sponsorship from the FreeBSD Foundation.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
 # are met:
 # 1.  Redistributions of source code must retain the above copyright
 #     notice, this list of conditions and the following disclaimer.
 # 2.  Redistributions in binary form must reproduce the above copyright
 #     notice, this list of conditions and the following disclaimer in the
 #     documentation and/or other materials provided with the distribution.
 #
 # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 #
 # $FreeBSD$
 #
 
 from __future__ import print_function
 import array
 import binascii
 from fcntl import ioctl
 import os
 import platform
 import random
 import signal
 from struct import pack as _pack
 import sys
 import time
 
 import dpkt
 
 from cryptodevh import *
 
 __all__ = [ 'Crypto', 'MismatchError', ]
 
 class FindOp(dpkt.Packet):
     __byte_order__ = '@'
     __hdr__ = (
         ('crid', 'i',   0),
         ('name', '32s', 0),
     )
 
 class SessionOp(dpkt.Packet):
     __byte_order__ = '@'
     __hdr__ = (
         ('cipher',    'I', 0),
         ('mac',       'I', 0),
         ('keylen',    'I', 0),
         ('key',       'P', 0),
         ('mackeylen', 'i', 0),
         ('mackey',    'P', 0),
         ('ses',       'I', 0),
     )
 
 class SessionOp2(dpkt.Packet):
     __byte_order__ = '@'
     __hdr__ = (
         ('cipher',    'I', 0),
         ('mac',       'I', 0),
         ('keylen',    'I', 0),
         ('key',       'P', 0),
         ('mackeylen', 'i', 0),
         ('mackey',    'P', 0),
         ('ses',       'I', 0),
         ('crid',      'i', 0),
         ('pad0',      'i', 0),
         ('pad1',      'i', 0),
         ('pad2',      'i', 0),
         ('pad3',      'i', 0),
     )
 
 class CryptOp(dpkt.Packet):
     __byte_order__ = '@'
     __hdr__ = (
         ('ses',   'I', 0),
         ('op',    'H', 0),
         ('flags', 'H', 0),
         ('len',   'I', 0),
         ('src',   'P', 0),
         ('dst',   'P', 0),
         ('mac',   'P', 0),
         ('iv',    'P', 0),
     )
 
 class CryptAEAD(dpkt.Packet):
     __byte_order__ = '@'
     __hdr__ = (
         ('ses',    'I', 0),
         ('op',     'H', 0),
         ('flags',  'H', 0),
         ('len',    'I', 0),
         ('aadlen', 'I', 0),
         ('ivlen',  'I', 0),
         ('src',    'P', 0),
         ('dst',    'P', 0),
         ('aad',    'P', 0),
         ('tag',    'P', 0),
         ('iv',     'P', 0),
     )
 
 # h2py.py can't handle multiarg macros
 CRIOGET = 3221513060
 CIOCGSESSION = 3224396645
 CIOCFSESSION = 2147771238
 CIOCKEY = 3230688104
 CIOCASYMFEAT = 1074029417
 CIOCKEY2 = 3230688107
 CIOCFINDDEV = 3223610220
 if platform.architecture()[0] == '64bit':
     CIOCGSESSION2 = 3225445226
     CIOCCRYPT = 3224396647
     CIOCCRYPTAEAD = 3225445229
 else:
     CIOCGSESSION2 = 3224396650
     CIOCCRYPT = 3223085927
     CIOCCRYPTAEAD = 3223872365
 
 def _getdev():
     buf = array.array('I', [0])
     fd = os.open('/dev/crypto', os.O_RDWR)
     try:
         ioctl(fd, CRIOGET, buf, 1)
     finally:
         os.close(fd)
 
     return buf[0]
 
 _cryptodev = _getdev()
 
 def str_to_ascii(val):
     if sys.version_info[0] >= 3:
         if isinstance(val, str):
             return val.encode("ascii")
     return val
 
 def _findop(crid, name):
     fop = FindOp()
     fop.crid = crid
     fop.name = str_to_ascii(name)
     s = array.array('B', fop.pack_hdr())
     ioctl(_cryptodev, CIOCFINDDEV, s, 1)
     fop.unpack(s)
 
     try:
         idx = fop.name.index(b'\x00')
         name = fop.name[:idx]
     except ValueError:
         name = fop.name
 
     return fop.crid, name
 
 def array_tobytes(array_obj):
     if sys.version_info[:2] >= (3, 2):
         return array_obj.tobytes()
     return array_obj.tostring()
 
 class Crypto:
     @staticmethod
     def findcrid(name):
         return _findop(-1, name)[0]
 
     @staticmethod
     def getcridname(crid):
         return _findop(crid, '')[1]
 
     def __init__(self, cipher=0, key=None, mac=0, mackey=None,
         crid=CRYPTOCAP_F_SOFTWARE | CRYPTOCAP_F_HARDWARE, maclen=None):
         self._ses = None
         self._maclen = maclen
         ses = SessionOp2()
         ses.cipher = cipher
         ses.mac = mac
 
         if key is not None:
             ses.keylen = len(key)
             k = array.array('B', key)
             ses.key = k.buffer_info()[0]
         else:
             self.key = None
 
         if mackey is not None:
             ses.mackeylen = len(mackey)
             mk = array.array('B', mackey)
             ses.mackey = mk.buffer_info()[0]
 
         if not cipher and not mac:
             raise ValueError('one of cipher or mac MUST be specified.')
         ses.crid = crid
         #print(ses)
         s = array.array('B', ses.pack_hdr())
         #print(s)
         ioctl(_cryptodev, CIOCGSESSION2, s, 1)
         ses.unpack(s)
 
         self._ses = ses.ses
 
     def __del__(self):
         if self._ses is None:
             return
 
         try:
             ioctl(_cryptodev, CIOCFSESSION, _pack('I', self._ses))
         except TypeError:
             pass
         self._ses = None
 
     def _doop(self, op, src, iv):
         cop = CryptOp()
         cop.ses = self._ses
         cop.op = op
         cop.flags = 0
         cop.len = len(src)
         s = array.array('B', src)
         cop.src = cop.dst = s.buffer_info()[0]
         if self._maclen is not None:
             m = array.array('B', [0] * self._maclen)
             cop.mac = m.buffer_info()[0]
         ivbuf = array.array('B', str_to_ascii(iv))
         cop.iv = ivbuf.buffer_info()[0]
 
         #print('cop:', cop)
         ioctl(_cryptodev, CIOCCRYPT, bytes(cop))
 
         s = array_tobytes(s)
         if self._maclen is not None:
             return s, array_tobytes(m)
 
         return s
 
     def _doaead(self, op, src, aad, iv, tag=None):
         caead = CryptAEAD()
         caead.ses = self._ses
         caead.op = op
         caead.flags = CRD_F_IV_EXPLICIT
         caead.flags = 0
         src = str_to_ascii(src)
         caead.len = len(src)
         s = array.array('B', src)
         caead.src = caead.dst = s.buffer_info()[0]
         aad = str_to_ascii(aad)
         caead.aadlen = len(aad)
         saad = array.array('B', aad)
         caead.aad = saad.buffer_info()[0]
 
         if self._maclen is None:
             raise ValueError('must have a tag length')
 
         tag = str_to_ascii(tag)
         if tag is None:
             tag = array.array('B', [0] * self._maclen)
         else:
             assert len(tag) == self._maclen, \
                 '%d != %d' % (len(tag), self._maclen)
             tag = array.array('B', tag)
 
         caead.tag = tag.buffer_info()[0]
 
         ivbuf = array.array('B', iv)
         caead.ivlen = len(iv)
         caead.iv = ivbuf.buffer_info()[0]
 
         ioctl(_cryptodev, CIOCCRYPTAEAD, bytes(caead))
 
         s = array_tobytes(s)
 
         return s, array_tobytes(tag)
 
     def perftest(self, op, size, timeo=3):
         inp = array.array('B', (random.randint(0, 255) for x in range(size)))
         inp = str_to_ascii(inp)
         out = array.array('B', inp)
 
         # prep ioctl
         cop = CryptOp()
         cop.ses = self._ses
         cop.op = op
         cop.flags = 0
         cop.len = len(inp)
         s = array.array('B', inp)
         cop.src = s.buffer_info()[0]
         cop.dst = out.buffer_info()[0]
         if self._maclen is not None:
             m = array.array('B', [0] * self._maclen)
             cop.mac = m.buffer_info()[0]
         ivbuf = array.array('B', (random.randint(0, 255) for x in range(16)))
         cop.iv = ivbuf.buffer_info()[0]
 
         exit = [ False ]
         def alarmhandle(a, b, exit=exit):
             exit[0] = True
 
         oldalarm = signal.signal(signal.SIGALRM, alarmhandle)
         signal.alarm(timeo)
 
         start = time.time()
         reps = 0
         cop = bytes(cop)
         while not exit[0]:
             ioctl(_cryptodev, CIOCCRYPT, cop)
             reps += 1
 
         end = time.time()
 
         signal.signal(signal.SIGALRM, oldalarm)
 
         print('time:', end - start)
         print('perf MB/sec:', (reps * size) / (end - start) / 1024 / 1024)
 
     def encrypt(self, data, iv, aad=None):
         if aad is None:
             return self._doop(COP_ENCRYPT, data, iv)
         else:
             return self._doaead(COP_ENCRYPT, data, aad,
                 iv)
 
     def decrypt(self, data, iv, aad=None, tag=None):
         if aad is None:
             return self._doop(COP_DECRYPT, data, iv)
         else:
             return self._doaead(COP_DECRYPT, data, aad,
                 iv, tag=tag)
 
 class MismatchError(Exception):
     pass
 
 class KATParser:
     def __init__(self, fname, fields):
         self.fields = set(fields)
         self._pending = None
         self.fname = fname
         self.fp = None
 
     def __enter__(self):
         self.fp = open(self.fname)
         return self
 
     def __exit__(self, exc_type, exc_value, exc_tb):
         if self.fp is not None:
             self.fp.close()
 
     def __iter__(self):
         return self
 
     def __next__(self):
         while True:
             didread = False
             if self._pending is not None:
                 i = self._pending
                 self._pending = None
             else:
                 i = self.fp.readline()
                 didread = True
 
             if didread and not i:
                 return
 
             if not i.startswith('#') and i.strip():
                 break
 
         if i[0] == '[':
             yield i[1:].split(']', 1)[0], self.fielditer()
         else:
             raise ValueError('unknown line: %r' % repr(i))
 
     def eatblanks(self):
         while True:
             line = self.fp.readline()
             if line == '':
                 break
 
             line = line.strip()
             if line:
                 break
 
         return line
 
     def fielditer(self):
         while True:
             values = {}
 
             line = self.eatblanks()
             if not line or line[0] == '[':
                 self._pending = line
                 return
 
             while True:
                 try:
                     f, v = line.split(' =')
                 except:
                     if line == 'FAIL':
                         f, v = 'FAIL', ''
                     else:
                         print('line:', repr(line))
                         raise
                 v = v.strip()
 
                 if f in values:
                     raise ValueError('already present: %r' % repr(f))
                 values[f] = v
                 line = self.fp.readline().strip()
                 if not line:
                     break
 
             # we should have everything
             remain = self.fields.copy() - set(values.keys())
             # XXX - special case GCM decrypt
             if remain and not ('FAIL' in values and 'PT' in remain):
                 raise ValueError('not all fields found: %r' % repr(remain))
 
             yield values
 
 # The CCM files use a bit of a different syntax that doesn't quite fit
 # the generic KATParser.  In particular, some keys are set globally at
 # the start of the file, and some are set globally at the start of a
 # section.
 class KATCCMParser:
     def __init__(self, fname):
         self._pending = None
         self.fname = fname
         self.fp = None
 
     def __enter__(self):
         self.fp = open(self.fname)
         self.read_globals()
         return self
 
     def __exit__(self, exc_type, exc_value, exc_tb):
         if self.fp is not None:
             self.fp.close()
 
     def read_globals(self):
         self.global_values = {}
         while True:
             line = self.fp.readline()
             if not line:
                 return
             if line[0] == '#' or not line.strip():
                 continue
             if line[0] == '[':
                 self._pending = line
                 return
 
             try:
                 f, v = line.split(' =')
             except:
                 print('line:', repr(line))
                 raise
 
             v = v.strip()
 
             if f in self.global_values:
                 raise ValueError('already present: %r' % repr(f))
             self.global_values[f] = v
 
     def read_section_values(self, kwpairs):
         self.section_values = self.global_values.copy()
         for pair in kwpairs.split(', '):
             f, v = pair.split(' = ')
             if f in self.section_values:
                 raise ValueError('already present: %r' % repr(f))
             self.section_values[f] = v
 
         while True:
             line = self.fp.readline()
             if not line:
                 return
             if line[0] == '#' or not line.strip():
                 continue
             if line[0] == '[':
                 self._pending = line
                 return
 
             try:
                 f, v = line.split(' =')
             except:
                 print('line:', repr(line))
                 raise
 
             if f == 'Count':
                 self._pending = line
                 return
 
             v = v.strip()
 
             if f in self.section_values:
                 raise ValueError('already present: %r' % repr(f))
             self.section_values[f] = v
 
     def __iter__(self):
         return self
 
     def __next__(self):
         while True:
             if self._pending:
                 line = self._pending
                 self._pending = None
             else:
                 line = self.fp.readline()
                 if not line:
                     return
 
             if (line and line[0] == '#') or not line.strip():
                 continue
 
             if line[0] == '[':
                 section = line[1:].split(']', 1)[0]
                 self.read_section_values(section)
                 continue
 
             values = self.section_values.copy()
 
             while True:
                 try:
                     f, v = line.split(' =')
                 except:
                     print('line:', repr(line))
                     raise
                 v = v.strip()
 
                 if f in values:
                     raise ValueError('already present: %r' % repr(f))
                 values[f] = v
                 line = self.fp.readline().strip()
                 if not line:
                     break
 
             yield values
 
 def _spdechex(s):
     return binascii.hexlify(''.join(s.split()))
 
 if sys.version_info[0] < 3:
     KATCCMParser.next = KATCCMParser.__next__
     KATParser.next = KATParser.__next__
 
 if __name__ == '__main__':
     if True:
         try:
             crid = Crypto.findcrid('aesni0')
             print('aesni:', crid)
         except IOError:
             print('aesni0 not found')
 
         for i in range(10):
             try:
                 name = Crypto.getcridname(i)
                 print('%2d: %r' % (i, repr(name)))
             except IOError:
                 pass
     elif False:
         columns = [ 'COUNT', 'DataUnitLen', 'Key', 'DataUnitSeqNumber', 'PT', 'CT' ]
         fname = '/usr/home/jmg/aesni.testing/format tweak value input - data unit seq no/XTSGenAES128.rsp'
         with KATParser(fname, columns) as kp:
             for mode, ni in kp:
                 print(i, ni)
                 for j in ni:
                     print(j)
     elif False:
         key = _spdechex('c939cc13397c1d37de6ae0e1cb7c423c')
         iv = _spdechex('00000000000000000000000000000001')
         pt = _spdechex('ab3cabed693a32946055524052afe3c9cb49664f09fc8b7da824d924006b7496353b8c1657c5dec564d8f38d7432e1de35aae9d95590e66278d4acce883e51abaf94977fcd3679660109a92bf7b2973ccd547f065ec6cee4cb4a72a5e9f45e615d920d76cb34cba482467b3e21422a7242e7d931330c0fbf465c3a3a46fae943029fd899626dda542750a1eee253df323c6ef1573f1c8c156613e2ea0a6cdbf2ae9701020be2d6a83ecb7f3f9d8e')
         #pt = _spdechex('00000000000000000000000000000000')
         ct = _spdechex('f42c33853ecc5ce2949865fdb83de3bff1089e9360c94f830baebfaff72836ab5236f77212f1e7396c8c54ac73d81986375a6e9e299cfeca5ba051ed25e8d1affa5beaf6c1d2b45e90802408f2ced21663497e906de5f29341e5e52ddfea5363d628b3eb7806835e17bae051b3a6da3f8e2941fe44384eac17a9d298d2c331ca8320c775b5d53263a5e905059d891b21dede2d8110fd427c7bd5a9a274ddb47b1945ee79522203b6e297d0e399ef')
 
         c = Crypto(CRYPTO_AES_ICM, key)
         enc = c.encrypt(pt, iv)
 
         print('enc:', binascii.hexlify(enc))
         print(' ct:', binascii.hexlify(ct))
 
         assert ct == enc
 
         dec = c.decrypt(ct, iv)
 
         print('dec:', binascii.hexlify(dec))
         print(' pt:', binascii.hexlify(pt))
 
         assert pt == dec
     elif False:
         key = _spdechex('c939cc13397c1d37de6ae0e1cb7c423c')
         iv = _spdechex('00000000000000000000000000000001')
         pt = _spdechex('ab3cabed693a32946055524052afe3c9cb49664f09fc8b7da824d924006b7496353b8c1657c5dec564d8f38d7432e1de35aae9d95590e66278d4acce883e51abaf94977fcd3679660109a92bf7b2973ccd547f065ec6cee4cb4a72a5e9f45e615d920d76cb34cba482467b3e21422a7242e7d931330c0fbf465c3a3a46fae943029fd899626dda542750a1eee253df323c6ef1573f1c8c156613e2ea0a6cdbf2ae9701020be2d6a83ecb7f3f9d8e0a3f')
         #pt = _spdechex('00000000000000000000000000000000')
         ct = _spdechex('f42c33853ecc5ce2949865fdb83de3bff1089e9360c94f830baebfaff72836ab5236f77212f1e7396c8c54ac73d81986375a6e9e299cfeca5ba051ed25e8d1affa5beaf6c1d2b45e90802408f2ced21663497e906de5f29341e5e52ddfea5363d628b3eb7806835e17bae051b3a6da3f8e2941fe44384eac17a9d298d2c331ca8320c775b5d53263a5e905059d891b21dede2d8110fd427c7bd5a9a274ddb47b1945ee79522203b6e297d0e399ef3768')
 
         c = Crypto(CRYPTO_AES_ICM, key)
         enc = c.encrypt(pt, iv)
 
         print('enc:', binascii.hexlify(enc))
         print(' ct:', binascii.hexlify(ct))
 
         assert ct == enc
 
         dec = c.decrypt(ct, iv)
 
         print('dec:', binascii.hexlify(dec))
         print(' pt:', binascii.hexlify(pt))
 
         assert pt == dec
     elif False:
         key = _spdechex('c939cc13397c1d37de6ae0e1cb7c423c')
         iv = _spdechex('6eba2716ec0bd6fa5cdef5e6d3a795bc')
         pt = _spdechex('ab3cabed693a32946055524052afe3c9cb49664f09fc8b7da824d924006b7496353b8c1657c5dec564d8f38d7432e1de35aae9d95590e66278d4acce883e51abaf94977fcd3679660109a92bf7b2973ccd547f065ec6cee4cb4a72a5e9f45e615d920d76cb34cba482467b3e21422a7242e7d931330c0fbf465c3a3a46fae943029fd899626dda542750a1eee253df323c6ef1573f1c8c156613e2ea0a6cdbf2ae9701020be2d6a83ecb7f3f9d8e0a3f')
         ct = _spdechex('f1f81f12e72e992dbdc304032705dc75dc3e4180eff8ee4819906af6aee876d5b00b7c36d282a445ce3620327be481e8e53a8e5a8e5ca9abfeb2281be88d12ffa8f46d958d8224738c1f7eea48bda03edbf9adeb900985f4fa25648b406d13a886c25e70cfdecdde0ad0f2991420eb48a61c64fd797237cf2798c2675b9bb744360b0a3f329ac53bbceb4e3e7456e6514f1a9d2f06c236c31d0f080b79c15dce1096357416602520daa098b17d1af427')
         c = Crypto(CRYPTO_AES_CBC, key)
 
         enc = c.encrypt(pt, iv)
 
         print('enc:', binascii.hexlify(enc))
         print(' ct:', binascii.hexlify(ct))
 
         assert ct == enc
 
         dec = c.decrypt(ct, iv)
 
         print('dec:', binascii.hexlify(dec))
         print(' pt:', binascii.hexlify(pt))
 
         assert pt == dec
     elif False:
         key = _spdechex('c939cc13397c1d37de6ae0e1cb7c423c')
         iv = _spdechex('b3d8cc017cbb89b39e0f67e2')
         pt = _spdechex('c3b3c41f113a31b73d9a5cd4321030')
         aad = _spdechex('24825602bd12a984e0092d3e448eda5f')
         ct = _spdechex('93fe7d9e9bfd10348a5606e5cafa7354')
         ct = _spdechex('93fe7d9e9bfd10348a5606e5cafa73')
         tag = _spdechex('0032a1dc85f1c9786925a2e71d8272dd')
         tag = _spdechex('8d11a0929cb3fbe1fef01a4a38d5f8ea')
 
-        c = Crypto(CRYPTO_AES_NIST_GCM_16, key,
-            mac=CRYPTO_AES_128_NIST_GMAC, mackey=key)
+        c = Crypto(CRYPTO_AES_NIST_GCM_16, key)
 
         enc, enctag = c.encrypt(pt, iv, aad=aad)
 
         print('enc:', binascii.hexlify(enc))
         print(' ct:', binascii.hexlify(ct))
 
         assert enc == ct
 
         print('etg:', binascii.hexlify(enctag))
         print('tag:', binascii.hexlify(tag))
         assert enctag == tag
 
         # Make sure we get EBADMSG
         #enctag = enctag[:-1] + 'a'
         dec, dectag = c.decrypt(ct, iv, aad=aad, tag=enctag)
 
         print('dec:', binascii.hexlify(dec))
         print(' pt:', binascii.hexlify(pt))
 
         assert dec == pt
 
         print('dtg:', binascii.hexlify(dectag))
         print('tag:', binascii.hexlify(tag))
 
         assert dectag == tag
     elif False:
         key = _spdechex('c939cc13397c1d37de6ae0e1cb7c423c')
         iv = _spdechex('b3d8cc017cbb89b39e0f67e2')
         key = key + iv[:4]
         iv = iv[4:]
         pt = _spdechex('c3b3c41f113a31b73d9a5cd432103069')
         aad = _spdechex('24825602bd12a984e0092d3e448eda5f')
         ct = _spdechex('93fe7d9e9bfd10348a5606e5cafa7354')
         tag = _spdechex('0032a1dc85f1c9786925a2e71d8272dd')
 
-        c = Crypto(CRYPTO_AES_GCM_16, key, mac=CRYPTO_AES_128_GMAC, mackey=key)
+        c = Crypto(CRYPTO_AES_GCM_16, key)
 
         enc, enctag = c.encrypt(pt, iv, aad=aad)
 
         print('enc:', binascii.hexlify(enc))
         print(' ct:', binascii.hexlify(ct))
 
         assert enc == ct
 
         print('etg:', binascii.hexlify(enctag))
         print('tag:', binascii.hexlify(tag))
         assert enctag == tag
     elif False:
         for i in range(100000):
             c = Crypto(CRYPTO_AES_XTS, binascii.unhexlify('1bbfeadf539daedcae33ced497343f3ca1f2474ad932b903997d44707db41382'))
             data = binascii.unhexlify('52a42bca4e9425a25bbc8c8bf6129dec')
             ct = binascii.unhexlify('517e602becd066b65fa4f4f56ddfe240')
             iv = _pack('QQ', 71, 0)
 
             enc = c.encrypt(data, iv)
             assert enc == ct
     elif True:
         c = Crypto(CRYPTO_AES_XTS, binascii.unhexlify('1bbfeadf539daedcae33ced497343f3ca1f2474ad932b903997d44707db41382'))
         data = binascii.unhexlify('52a42bca4e9425a25bbc8c8bf6129dec')
         ct = binascii.unhexlify('517e602becd066b65fa4f4f56ddfe240')
         iv = _pack('QQ', 71, 0)
 
         enc = c.encrypt(data, iv)
         assert enc == ct
 
         dec = c.decrypt(enc, iv)
         assert dec == data
 
         #c.perftest(COP_ENCRYPT, 192*1024, reps=30000)
 
     else:
         key = binascii.unhexlify('1bbfeadf539daedcae33ced497343f3ca1f2474ad932b903997d44707db41382')
         print('XTS %d testing:' % (len(key) * 8))
         c = Crypto(CRYPTO_AES_XTS, key)
         for i in [ 8192, 192*1024]:
             print('block size: %d' % i)
             c.perftest(COP_ENCRYPT, i)
             c.perftest(COP_DECRYPT, i)
Index: head/tests/sys/opencrypto/cryptodevh.py
===================================================================
--- head/tests/sys/opencrypto/cryptodevh.py	(revision 359373)
+++ head/tests/sys/opencrypto/cryptodevh.py	(revision 359374)
@@ -1,262 +1,259 @@
 # $FreeBSD$
 # Generated by h2py from stdin
 
 # Included from sys/ioccom.h
 IOCPARM_SHIFT = 13
 IOCPARM_MASK = ((1 << IOCPARM_SHIFT) - 1)
 def IOCPARM_LEN(x): return (((x) >> 16) & IOCPARM_MASK)
 
 def IOCBASECMD(x): return ((x) & ~(IOCPARM_MASK << 16))
 
 def IOCGROUP(x): return (((x) >> 8) & 0xff)
 
 IOCPARM_MAX = (1 << IOCPARM_SHIFT)
 IOC_VOID = 0x20000000
 IOC_OUT = 0x40000000
 IOC_IN = 0x80000000
 IOC_INOUT = (IOC_IN|IOC_OUT)
 IOC_DIRMASK = (IOC_VOID|IOC_OUT|IOC_IN)
 
 # Included from sys/cdefs.h
 def __has_feature(x): return 0
 
 def __has_include(x): return 0
 
 def __has_builtin(x): return 0
 
 __GNUCLIKE_ASM = 3
 __GNUCLIKE_ASM = 2
 __GNUCLIKE___TYPEOF = 1
 __GNUCLIKE___OFFSETOF = 1
 __GNUCLIKE___SECTION = 1
 __GNUCLIKE_CTOR_SECTION_HANDLING = 1
 __GNUCLIKE_BUILTIN_CONSTANT_P = 1
 __GNUCLIKE_BUILTIN_VARARGS = 1
 __GNUCLIKE_BUILTIN_STDARG = 1
 __GNUCLIKE_BUILTIN_VAALIST = 1
 __GNUC_VA_LIST_COMPATIBILITY = 1
 __GNUCLIKE_BUILTIN_NEXT_ARG = 1
 __GNUCLIKE_BUILTIN_MEMCPY = 1
 __CC_SUPPORTS_INLINE = 1
 __CC_SUPPORTS___INLINE = 1
 __CC_SUPPORTS___INLINE__ = 1
 __CC_SUPPORTS___FUNC__ = 1
 __CC_SUPPORTS_WARNING = 1
 __CC_SUPPORTS_VARADIC_XXX = 1
 __CC_SUPPORTS_DYNAMIC_ARRAY_INIT = 1
 def __P(protos): return protos		 
 
 def __STRING(x): return #x		 
 
 def __XSTRING(x): return __STRING(x)	 
 
 def __P(protos): return ()		 
 
 def __STRING(x): return "x"
 
 def __aligned(x): return __attribute__((__aligned__(x)))
 
 def __section(x): return __attribute__((__section__(x)))
 
 def __aligned(x): return __attribute__((__aligned__(x)))
 
 def __section(x): return __attribute__((__section__(x)))
 
 def _Alignas(x): return alignas(x)
 
 def _Alignas(x): return __aligned(x)
 
 def _Alignof(x): return alignof(x)
 
 def _Alignof(x): return __alignof(x)
 
 def __nonnull(x): return __attribute__((__nonnull__(x)))
 
 def __predict_true(exp): return __builtin_expect((exp), 1)
 
 def __predict_false(exp): return __builtin_expect((exp), 0)
 
 def __predict_true(exp): return (exp)
 
 def __predict_false(exp): return (exp)
 
 def __format_arg(fmtarg): return __attribute__((__format_arg__ (fmtarg)))
 
 def __GLOBL(sym): return __GLOBL1(sym)
 
 def __FBSDID(s): return __IDSTRING(__CONCAT(__rcsid_,__LINE__),s)
 
 def __RCSID(s): return __IDSTRING(__CONCAT(__rcsid_,__LINE__),s)
 
 def __RCSID_SOURCE(s): return __IDSTRING(__CONCAT(__rcsid_source_,__LINE__),s)
 
 def __SCCSID(s): return __IDSTRING(__CONCAT(__sccsid_,__LINE__),s)
 
 def __COPYRIGHT(s): return __IDSTRING(__CONCAT(__copyright_,__LINE__),s)
 
 _POSIX_C_SOURCE = 199009
 _POSIX_C_SOURCE = 199209
 __XSI_VISIBLE = 700
 _POSIX_C_SOURCE = 200809
 __XSI_VISIBLE = 600
 _POSIX_C_SOURCE = 200112
 __XSI_VISIBLE = 500
 _POSIX_C_SOURCE = 199506
 _POSIX_C_SOURCE = 198808
 __POSIX_VISIBLE = 200809
 __ISO_C_VISIBLE = 1999
 __POSIX_VISIBLE = 200112
 __ISO_C_VISIBLE = 1999
 __POSIX_VISIBLE = 199506
 __ISO_C_VISIBLE = 1990
 __POSIX_VISIBLE = 199309
 __ISO_C_VISIBLE = 1990
 __POSIX_VISIBLE = 199209
 __ISO_C_VISIBLE = 1990
 __POSIX_VISIBLE = 199009
 __ISO_C_VISIBLE = 1990
 __POSIX_VISIBLE = 198808
 __ISO_C_VISIBLE = 0
 __POSIX_VISIBLE = 0
 __XSI_VISIBLE = 0
 __BSD_VISIBLE = 0
 __ISO_C_VISIBLE = 1990
 __POSIX_VISIBLE = 0
 __XSI_VISIBLE = 0
 __BSD_VISIBLE = 0
 __ISO_C_VISIBLE = 1999
 __POSIX_VISIBLE = 0
 __XSI_VISIBLE = 0
 __BSD_VISIBLE = 0
 __ISO_C_VISIBLE = 2011
 __POSIX_VISIBLE = 200809
 __XSI_VISIBLE = 700
 __BSD_VISIBLE = 1
 __ISO_C_VISIBLE = 2011
 __NO_TLS = 1
 CRYPTO_DRIVERS_INITIAL = 4
 CRYPTO_SW_SESSIONS = 32
 NULL_HASH_LEN = 16
 MD5_HASH_LEN = 16
 SHA1_HASH_LEN = 20
 RIPEMD160_HASH_LEN = 20
 SHA2_256_HASH_LEN = 32
 SHA2_384_HASH_LEN = 48
 SHA2_512_HASH_LEN = 64
 MD5_KPDK_HASH_LEN = 16
 SHA1_KPDK_HASH_LEN = 20
 HASH_MAX_LEN = SHA2_512_HASH_LEN
 NULL_HMAC_BLOCK_LEN = 64
 MD5_HMAC_BLOCK_LEN = 64
 SHA1_HMAC_BLOCK_LEN = 64
 RIPEMD160_HMAC_BLOCK_LEN = 64
 SHA2_256_HMAC_BLOCK_LEN = 64
 SHA2_384_HMAC_BLOCK_LEN = 128
 SHA2_512_HMAC_BLOCK_LEN = 128
 HMAC_MAX_BLOCK_LEN = SHA2_512_HMAC_BLOCK_LEN
 HMAC_IPAD_VAL = 0x36
 HMAC_OPAD_VAL = 0x5C
 NULL_BLOCK_LEN = 4
 DES_BLOCK_LEN = 8
 DES3_BLOCK_LEN = 8
 BLOWFISH_BLOCK_LEN = 8
 SKIPJACK_BLOCK_LEN = 8
 CAST128_BLOCK_LEN = 8
 RIJNDAEL128_BLOCK_LEN = 16
 AES_BLOCK_LEN = RIJNDAEL128_BLOCK_LEN
 CAMELLIA_BLOCK_LEN = 16
 EALG_MAX_BLOCK_LEN = AES_BLOCK_LEN
 AALG_MAX_RESULT_LEN = 64
 CRYPTO_ALGORITHM_MIN = 1
 CRYPTO_DES_CBC = 1
 CRYPTO_3DES_CBC = 2
 CRYPTO_BLF_CBC = 3
 CRYPTO_CAST_CBC = 4
 CRYPTO_SKIPJACK_CBC = 5
 CRYPTO_MD5_HMAC = 6
 CRYPTO_SHA1_HMAC = 7
 CRYPTO_RIPEMD160_HMAC = 8
 CRYPTO_MD5_KPDK = 9
 CRYPTO_SHA1_KPDK = 10
 CRYPTO_RIJNDAEL128_CBC = 11
 CRYPTO_AES_CBC = 11
 CRYPTO_ARC4 = 12
 CRYPTO_MD5 = 13
 CRYPTO_SHA1 = 14
 CRYPTO_NULL_HMAC = 15
 CRYPTO_NULL_CBC = 16
 CRYPTO_DEFLATE_COMP = 17
 CRYPTO_SHA2_256_HMAC = 18
 CRYPTO_SHA2_384_HMAC = 19
 CRYPTO_SHA2_512_HMAC = 20
 CRYPTO_CAMELLIA_CBC = 21
 CRYPTO_AES_XTS = 22
 CRYPTO_AES_ICM = 23
 CRYPTO_AES_NIST_GMAC = 24
 CRYPTO_AES_NIST_GCM_16 = 25
-CRYPTO_AES_128_NIST_GMAC = 26
-CRYPTO_AES_192_NIST_GMAC = 27
-CRYPTO_AES_256_NIST_GMAC = 28
 CRYPTO_BLAKE2B = 29
 CRYPTO_BLAKE2S = 30
 CRYPTO_CHACHA20 = 31
 CRYPTO_SHA2_224_HMAC = 32
 CRYPTO_RIPEMD160 = 33
 CRYPTO_SHA2_224 = 34
 CRYPTO_SHA2_256 = 35
 CRYPTO_SHA2_384 = 36
 CRYPTO_SHA2_512 = 37
 CRYPTO_POLY1305 = 38
 CRYPTO_AES_CCM_CBC_MAC = 39
 CRYPTO_AES_CCM_16 = 40
 CRYPTO_ALGORITHM_MAX = 40
 CRYPTO_ALG_FLAG_SUPPORTED = 0x01
 CRYPTO_ALG_FLAG_RNG_ENABLE = 0x02
 CRYPTO_ALG_FLAG_DSA_SHA = 0x04
 CRYPTO_FLAG_HARDWARE = 0x01000000
 CRYPTO_FLAG_SOFTWARE = 0x02000000
 COP_ENCRYPT = 1
 COP_DECRYPT = 2
 COP_F_BATCH = 0x0008
 CRK_MAXPARAM = 8
 CRK_ALGORITM_MIN = 0
 CRK_MOD_EXP = 0
 CRK_MOD_EXP_CRT = 1
 CRK_DSA_SIGN = 2
 CRK_DSA_VERIFY = 3
 CRK_DH_COMPUTE_KEY = 4
 CRK_ALGORITHM_MAX = 4
 CRF_MOD_EXP = (1 << CRK_MOD_EXP)
 CRF_MOD_EXP_CRT = (1 << CRK_MOD_EXP_CRT)
 CRF_DSA_SIGN = (1 << CRK_DSA_SIGN)
 CRF_DSA_VERIFY = (1 << CRK_DSA_VERIFY)
 CRF_DH_COMPUTE_KEY = (1 << CRK_DH_COMPUTE_KEY)
 CRD_F_ENCRYPT = 0x01
 CRD_F_IV_PRESENT = 0x02
 CRD_F_IV_EXPLICIT = 0x04
 CRD_F_DSA_SHA_NEEDED = 0x08
 CRD_F_COMP = 0x0f
 CRD_F_KEY_EXPLICIT = 0x10
 CRYPTO_F_IMBUF = 0x0001
 CRYPTO_F_IOV = 0x0002
 CRYPTO_F_BATCH = 0x0008
 CRYPTO_F_CBIMM = 0x0010
 CRYPTO_F_DONE = 0x0020
 CRYPTO_F_CBIFSYNC = 0x0040
 CRYPTO_BUF_CONTIG = 0x0
 CRYPTO_BUF_IOV = 0x1
 CRYPTO_BUF_MBUF = 0x2
 CRYPTO_OP_DECRYPT = 0x0
 CRYPTO_OP_ENCRYPT = 0x1
 CRYPTO_HINT_MORE = 0x1
 def CRYPTO_SESID2HID(_sid): return (((_sid) >> 32) & 0x00ffffff)
 
 def CRYPTO_SESID2CAPS(_sid): return (((_sid) >> 32) & 0xff000000)
 
 def CRYPTO_SESID2LID(_sid): return (((u_int32_t) (_sid)) & 0xffffffff)
 
 CRYPTOCAP_F_HARDWARE = CRYPTO_FLAG_HARDWARE
 CRYPTOCAP_F_SOFTWARE = CRYPTO_FLAG_SOFTWARE
 CRYPTOCAP_F_SYNC = 0x04000000
 CRYPTO_SYMQ = 0x1
 CRYPTO_ASYMQ = 0x2
Index: head/tests/sys/opencrypto/cryptotest.py
===================================================================
--- head/tests/sys/opencrypto/cryptotest.py	(revision 359373)
+++ head/tests/sys/opencrypto/cryptotest.py	(revision 359374)
@@ -1,508 +1,502 @@
 #!/usr/local/bin/python2
 #
 # Copyright (c) 2014 The FreeBSD Foundation
 # All rights reserved.
 # Copyright 2019 Enji Cooper
 #
 # This software was developed by John-Mark Gurney under
 # the sponsorship from the FreeBSD Foundation.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
 # are met:
 # 1.  Redistributions of source code must retain the above copyright
 #     notice, this list of conditions and the following disclaimer.
 # 2.  Redistributions in binary form must reproduce the above copyright
 #     notice, this list of conditions and the following disclaimer in the
 #     documentation and/or other materials provided with the distribution.
 #
 # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 # SUCH DAMAGE.
 #
 # $FreeBSD$
 #
 
 from __future__ import print_function
 
 import binascii
 import errno
 import cryptodev
 import itertools
 import os
 import struct
 import unittest
 from cryptodev import *
 from glob import iglob
 
 katdir = '/usr/local/share/nist-kat'
 
 def katg(base, glob):
     assert os.path.exists(katdir), "Please 'pkg install nist-kat'"
     if not os.path.exists(os.path.join(katdir, base)):
         raise unittest.SkipTest("Missing %s test vectors" % (base))
     return iglob(os.path.join(katdir, base, glob))
 
 aesmodules = [ 'cryptosoft0', 'aesni0', 'armv8crypto0', 'ccr0', 'ccp0' ]
 desmodules = [ 'cryptosoft0', ]
 shamodules = [ 'cryptosoft0', 'aesni0', 'armv8crypto0', 'ccr0', 'ccp0' ]
 
 def GenTestCase(cname):
     try:
         crid = cryptodev.Crypto.findcrid(cname)
     except IOError:
         return None
 
     class GendCryptoTestCase(unittest.TestCase):
         ###############
         ##### AES #####
         ###############
         @unittest.skipIf(cname not in aesmodules, 'skipping AES-XTS on %s' % (cname))
         def test_xts(self):
             for i in katg('XTSTestVectors/format tweak value input - data unit seq no', '*.rsp'):
                 self.runXTS(i, cryptodev.CRYPTO_AES_XTS)
 
         @unittest.skipIf(cname not in aesmodules, 'skipping AES-CBC on %s' % (cname))
         def test_cbc(self):
             for i in katg('KAT_AES', 'CBC[GKV]*.rsp'):
                 self.runCBC(i)
 
         @unittest.skipIf(cname not in aesmodules, 'skipping AES-CCM on %s' % (cname))
         def test_ccm(self):
             for i in katg('ccmtestvectors', 'V*.rsp'):
                 self.runCCMEncrypt(i)
 
             for i in katg('ccmtestvectors', 'D*.rsp'):
                 self.runCCMDecrypt(i)
 
         @unittest.skipIf(cname not in aesmodules, 'skipping AES-GCM on %s' % (cname))
         def test_gcm(self):
             for i in katg('gcmtestvectors', 'gcmEncrypt*'):
                 self.runGCM(i, 'ENCRYPT')
 
             for i in katg('gcmtestvectors', 'gcmDecrypt*'):
                 self.runGCM(i, 'DECRYPT')
 
-        _gmacsizes = { 32: cryptodev.CRYPTO_AES_256_NIST_GMAC,
-            24: cryptodev.CRYPTO_AES_192_NIST_GMAC,
-            16: cryptodev.CRYPTO_AES_128_NIST_GMAC,
-        }
         def runGCM(self, fname, mode):
             curfun = None
             if mode == 'ENCRYPT':
                 swapptct = False
                 curfun = Crypto.encrypt
             elif mode == 'DECRYPT':
                 swapptct = True
                 curfun = Crypto.decrypt
             else:
                 raise RuntimeError('unknown mode: %r' % repr(mode))
 
             columns = [ 'Count', 'Key', 'IV', 'CT', 'AAD', 'Tag', 'PT', ]
             with cryptodev.KATParser(fname, columns) as parser:
                 self.runGCMWithParser(parser, mode)
 
         def runGCMWithParser(self, parser, mode):
             for _, lines in next(parser):
                 for data in lines:
                     curcnt = int(data['Count'])
                     cipherkey = binascii.unhexlify(data['Key'])
                     iv = binascii.unhexlify(data['IV'])
                     aad = binascii.unhexlify(data['AAD'])
                     tag = binascii.unhexlify(data['Tag'])
                     if 'FAIL' not in data:
                         pt = binascii.unhexlify(data['PT'])
                     ct = binascii.unhexlify(data['CT'])
 
                     if len(iv) != 12:
                         # XXX - isn't supported
                         continue
 
                     try:
                         c = Crypto(cryptodev.CRYPTO_AES_NIST_GCM_16,
-                            cipherkey,
-                            mac=self._gmacsizes[len(cipherkey)],
-                            mackey=cipherkey, crid=crid,
+                            cipherkey, crid=crid,
                             maclen=16)
                     except EnvironmentError as e:
                         # Can't test algorithms the driver does not support.
                         if e.errno != errno.EOPNOTSUPP:
                             raise
                         continue
 
                     if mode == 'ENCRYPT':
                         try:
                             rct, rtag = c.encrypt(pt, iv, aad)
                         except EnvironmentError as e:
                             # Can't test inputs the driver does not support.
                             if e.errno != errno.EINVAL:
                                 raise
                             continue
                         rtag = rtag[:len(tag)]
                         data['rct'] = binascii.hexlify(rct)
                         data['rtag'] = binascii.hexlify(rtag)
                         self.assertEqual(rct, ct, repr(data))
                         self.assertEqual(rtag, tag, repr(data))
                     else:
                         if len(tag) != 16:
                             continue
                         args = (ct, iv, aad, tag)
                         if 'FAIL' in data:
                             self.assertRaises(IOError,
                                 c.decrypt, *args)
                         else:
                             try:
                                 rpt, rtag = c.decrypt(*args)
                             except EnvironmentError as e:
                                 # Can't test inputs the driver does not support.
                                 if e.errno != errno.EINVAL:
                                     raise
                                 continue
                             data['rpt'] = binascii.hexlify(rpt)
                             data['rtag'] = binascii.hexlify(rtag)
                             self.assertEqual(rpt, pt,
                                 repr(data))
 
         def runCBC(self, fname):
             columns = [ 'COUNT', 'KEY', 'IV', 'PLAINTEXT', 'CIPHERTEXT', ]
             with cryptodev.KATParser(fname, columns) as parser:
                 self.runCBCWithParser(parser)
 
         def runCBCWithParser(self, parser):
             curfun = None
             for mode, lines in next(parser):
                 if mode == 'ENCRYPT':
                     swapptct = False
                     curfun = Crypto.encrypt
                 elif mode == 'DECRYPT':
                     swapptct = True
                     curfun = Crypto.decrypt
                 else:
                     raise RuntimeError('unknown mode: %r' % repr(mode))
 
                 for data in lines:
                     curcnt = int(data['COUNT'])
                     cipherkey = binascii.unhexlify(data['KEY'])
                     iv = binascii.unhexlify(data['IV'])
                     pt = binascii.unhexlify(data['PLAINTEXT'])
                     ct = binascii.unhexlify(data['CIPHERTEXT'])
 
                     if swapptct:
                         pt, ct = ct, pt
                     # run the fun
                     c = Crypto(cryptodev.CRYPTO_AES_CBC, cipherkey, crid=crid)
                     r = curfun(c, pt, iv)
                     self.assertEqual(r, ct)
 
         def runXTS(self, fname, meth):
             columns = [ 'COUNT', 'DataUnitLen', 'Key', 'DataUnitSeqNumber', 'PT',
                         'CT']
             with cryptodev.KATParser(fname, columns) as parser:
                 self.runXTSWithParser(parser, meth)
 
         def runXTSWithParser(self, parser, meth):
             curfun = None
             for mode, lines in next(parser):
                 if mode == 'ENCRYPT':
                     swapptct = False
                     curfun = Crypto.encrypt
                 elif mode == 'DECRYPT':
                     swapptct = True
                     curfun = Crypto.decrypt
                 else:
                     raise RuntimeError('unknown mode: %r' % repr(mode))
 
                 for data in lines:
                     curcnt = int(data['COUNT'])
                     nbits = int(data['DataUnitLen'])
                     cipherkey = binascii.unhexlify(data['Key'])
                     iv = struct.pack('QQ', int(data['DataUnitSeqNumber']), 0)
                     pt = binascii.unhexlify(data['PT'])
                     ct = binascii.unhexlify(data['CT'])
 
                     if nbits % 128 != 0:
                         # XXX - mark as skipped
                         continue
                     if swapptct:
                         pt, ct = ct, pt
                     # run the fun
                     try:
                         c = Crypto(meth, cipherkey, crid=crid)
                         r = curfun(c, pt, iv)
                     except EnvironmentError as e:
                         # Can't test hashes the driver does not support.
                         if e.errno != errno.EOPNOTSUPP:
                             raise
                         continue
                     self.assertEqual(r, ct)
 
         def runCCMEncrypt(self, fname):
             with cryptodev.KATCCMParser(fname) as parser:
                 self.runCCMEncryptWithParser(parser)
 
         def runCCMEncryptWithParser(self, parser):
             for data in next(parser):
                 Nlen = int(data['Nlen'])
                 if Nlen != 12:
                     # OCF only supports 12 byte IVs
                     continue
                 key = binascii.unhexlify(data['Key'])
                 nonce = binascii.unhexlify(data['Nonce'])
                 Alen = int(data['Alen'])
                 if Alen != 0:
                     aad = binascii.unhexlify(data['Adata'])
                 else:
                     aad = None
                 payload = binascii.unhexlify(data['Payload'])
                 ct = binascii.unhexlify(data['CT'])
 
                 try:
                     c = Crypto(crid=crid,
                         cipher=cryptodev.CRYPTO_AES_CCM_16,
                         key=key,
                         mac=cryptodev.CRYPTO_AES_CCM_CBC_MAC,
                         mackey=key, maclen=16)
                     r, tag = Crypto.encrypt(c, payload,
                         nonce, aad)
                 except EnvironmentError as e:
                     if e.errno != errno.EOPNOTSUPP:
                         raise
                     continue
 
                 out = r + tag
                 self.assertEqual(out, ct,
                     "Count " + data['Count'] + " Actual: " + \
                     repr(binascii.hexlify(out)) + " Expected: " + \
                     repr(data) + " on " + cname)
 
         def runCCMDecrypt(self, fname):
             with cryptodev.KATCCMParser(fname) as parser:
                 self.runCCMDecryptWithParser(parser)
 
         def runCCMDecryptWithParser(self, parser):
             # XXX: Note that all of the current CCM
             # decryption test vectors use IV and tag sizes
             # that aren't supported by OCF none of the
             # tests are actually ran.
             for data in next(parser):
                 Nlen = int(data['Nlen'])
                 if Nlen != 12:
                     # OCF only supports 12 byte IVs
                     continue
                 Tlen = int(data['Tlen'])
                 if Tlen != 16:
                     # OCF only supports 16 byte tags
                     continue
                 key = binascii.unhexlify(data['Key'])
                 nonce = binascii.unhexlify(data['Nonce'])
                 Alen = int(data['Alen'])
                 if Alen != 0:
                     aad = binascii.unhexlify(data['Adata'])
                 else:
                     aad = None
                 ct = binascii.unhexlify(data['CT'])
                 tag = ct[-16:]
                 ct = ct[:-16]
 
                 try:
                     c = Crypto(crid=crid,
                         cipher=cryptodev.CRYPTO_AES_CCM_16,
                         key=key,
                         mac=cryptodev.CRYPTO_AES_CCM_CBC_MAC,
                         mackey=key, maclen=16)
                 except EnvironmentError as e:
                     if e.errno != errno.EOPNOTSUPP:
                         raise
                     continue
 
                 if data['Result'] == 'Fail':
                     self.assertRaises(IOError,
                         c.decrypt, payload, nonce, aad, tag)
                 else:
                     r = Crypto.decrypt(c, payload, nonce,
                         aad, tag)
 
                     payload = binascii.unhexlify(data['Payload'])
                     plen = int(data('Plen'))
                     payload = payload[:plen]
                     self.assertEqual(r, payload,
                         "Count " + data['Count'] + \
                         " Actual: " + repr(binascii.hexlify(r)) + \
                         " Expected: " + repr(data) + \
                         " on " + cname)
 
         ###############
         ##### DES #####
         ###############
         @unittest.skipIf(cname not in desmodules, 'skipping DES on %s' % (cname))
         def test_tdes(self):
             for i in katg('KAT_TDES', 'TCBC[a-z]*.rsp'):
                 self.runTDES(i)
 
         def runTDES(self, fname):
             columns = [ 'COUNT', 'KEYs', 'IV', 'PLAINTEXT', 'CIPHERTEXT', ]
             with cryptodev.KATParser(fname, columns) as parser:
                 self.runTDESWithParser(parser)
 
         def runTDESWithParser(self, parser):
             curfun = None
             for mode, lines in next(parser):
                 if mode == 'ENCRYPT':
                     swapptct = False
                     curfun = Crypto.encrypt
                 elif mode == 'DECRYPT':
                     swapptct = True
                     curfun = Crypto.decrypt
                 else:
                     raise RuntimeError('unknown mode: %r' % repr(mode))
 
                 for data in lines:
                     curcnt = int(data['COUNT'])
                     key = data['KEYs'] * 3
                     cipherkey = binascii.unhexlify(key)
                     iv = binascii.unhexlify(data['IV'])
                     pt = binascii.unhexlify(data['PLAINTEXT'])
                     ct = binascii.unhexlify(data['CIPHERTEXT'])
 
                     if swapptct:
                         pt, ct = ct, pt
                     # run the fun
                     c = Crypto(cryptodev.CRYPTO_3DES_CBC, cipherkey, crid=crid)
                     r = curfun(c, pt, iv)
                     self.assertEqual(r, ct)
 
         ###############
         ##### SHA #####
         ###############
         @unittest.skipIf(cname not in shamodules, 'skipping SHA on %s' % str(cname))
         def test_sha(self):
             for i in katg('shabytetestvectors', 'SHA*Msg.rsp'):
                 self.runSHA(i)
 
         def runSHA(self, fname):
             # Skip SHA512_(224|256) tests
             if fname.find('SHA512_') != -1:
                 return
             columns = [ 'Len', 'Msg', 'MD' ]
             with cryptodev.KATParser(fname, columns) as parser:
                 self.runSHAWithParser(parser)
 
         def runSHAWithParser(self, parser):
             for hashlength, lines in next(parser):
                 # E.g., hashlength will be "L=20" (bytes)
                 hashlen = int(hashlength.split("=")[1])
 
                 if hashlen == 20:
                     alg = cryptodev.CRYPTO_SHA1
                 elif hashlen == 28:
                     alg = cryptodev.CRYPTO_SHA2_224
                 elif hashlen == 32:
                     alg = cryptodev.CRYPTO_SHA2_256
                 elif hashlen == 48:
                     alg = cryptodev.CRYPTO_SHA2_384
                 elif hashlen == 64:
                     alg = cryptodev.CRYPTO_SHA2_512
                 else:
                     # Skip unsupported hashes
                     # Slurp remaining input in section
                     for data in lines:
                         continue
                     continue
 
                 for data in lines:
                     msg = binascii.unhexlify(data['Msg'])
                     msg = msg[:int(data['Len'])]
                     md = binascii.unhexlify(data['MD'])
 
                     try:
                         c = Crypto(mac=alg, crid=crid,
                             maclen=hashlen)
                     except EnvironmentError as e:
                         # Can't test hashes the driver does not support.
                         if e.errno != errno.EOPNOTSUPP:
                             raise
                         continue
 
                     _, r = c.encrypt(msg, iv="")
 
                     self.assertEqual(r, md, "Actual: " + \
                         repr(binascii.hexlify(r)) + " Expected: " + repr(data) + " on " + cname)
 
         @unittest.skipIf(cname not in shamodules, 'skipping SHA-HMAC on %s' % str(cname))
         def test_sha1hmac(self):
             for i in katg('hmactestvectors', 'HMAC.rsp'):
                 self.runSHA1HMAC(i)
 
         def runSHA1HMAC(self, fname):
             columns = [ 'Count', 'Klen', 'Tlen', 'Key', 'Msg', 'Mac' ]
             with cryptodev.KATParser(fname, columns) as parser:
                 self.runSHA1HMACWithParser(parser)
 
         def runSHA1HMACWithParser(self, parser):
             for hashlength, lines in next(parser):
                 # E.g., hashlength will be "L=20" (bytes)
                 hashlen = int(hashlength.split("=")[1])
 
                 blocksize = None
                 if hashlen == 20:
                     alg = cryptodev.CRYPTO_SHA1_HMAC
                     blocksize = 64
                 elif hashlen == 28:
                     alg = cryptodev.CRYPTO_SHA2_224_HMAC
                     blocksize = 64
                 elif hashlen == 32:
                     alg = cryptodev.CRYPTO_SHA2_256_HMAC
                     blocksize = 64
                 elif hashlen == 48:
                     alg = cryptodev.CRYPTO_SHA2_384_HMAC
                     blocksize = 128
                 elif hashlen == 64:
                     alg = cryptodev.CRYPTO_SHA2_512_HMAC
                     blocksize = 128
                 else:
                     # Skip unsupported hashes
                     # Slurp remaining input in section
                     for data in lines:
                         continue
                     continue
 
                 for data in lines:
                     key = binascii.unhexlify(data['Key'])
                     msg = binascii.unhexlify(data['Msg'])
                     mac = binascii.unhexlify(data['Mac'])
                     tlen = int(data['Tlen'])
 
                     if len(key) > blocksize:
                         continue
 
                     try:
                         c = Crypto(mac=alg, mackey=key,
                             crid=crid, maclen=hashlen)
                     except EnvironmentError as e:
                         # Can't test hashes the driver does not support.
                         if e.errno != errno.EOPNOTSUPP:
                             raise
                         continue
 
                     _, r = c.encrypt(msg, iv="")
 
                     self.assertEqual(r[:tlen], mac, "Actual: " + \
                         repr(binascii.hexlify(r)) + " Expected: " + repr(data))
 
     return GendCryptoTestCase
 
 cryptosoft = GenTestCase('cryptosoft0')
 aesni = GenTestCase('aesni0')
 armv8crypto = GenTestCase('armv8crypto0')
 ccr = GenTestCase('ccr0')
 ccp = GenTestCase('ccp0')
 
 if __name__ == '__main__':
     unittest.main()
Index: head/tools/tools/crypto/cryptocheck.c
===================================================================
--- head/tools/tools/crypto/cryptocheck.c	(revision 359373)
+++ head/tools/tools/crypto/cryptocheck.c	(revision 359374)
@@ -1,1502 +1,1634 @@
 /*-
  * Copyright (c) 2017 Chelsio Communications, Inc.
  * All rights reserved.
  * Written by: John Baldwin <jhb@FreeBSD.org>
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 /*-
  * Copyright (c) 2004 Sam Leffler, Errno Consulting
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer,
  *    without modification.
  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  *    similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any
  *    redistribution must be conditioned upon including a substantially
  *    similar Disclaimer requirement for further binary redistribution.
  * 3. Neither the names of the above-listed copyright holders nor the names
  *    of any contributors may be used to endorse or promote products derived
  *    from this software without specific prior written permission.
  *
  * NO WARRANTY
  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY
  * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
  * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY,
  * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
  * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  * THE POSSIBILITY OF SUCH DAMAGES.
  *
  * $FreeBSD$
  */
 
 /*
  * A different tool for checking hardware crypto support.  Whereas
  * cryptotest is focused on simple performance numbers, this tool is
  * focused on correctness.  For each crypto operation, it performs the
  * operation once in software via OpenSSL and a second time via
  * OpenCrypto and compares the results.
  *
  * cryptocheck [-vz] [-A aad length] [-a algorithm] [-d dev] [size ...]
  *
  * Options:
  *	-v	Verbose.
  *	-z	Run all algorithms on a variety of buffer sizes.
  *
  * Supported algorithms:
  *	all		Run all tests
  *	hash		Run all hash tests
  *	mac		Run all mac tests
  *	cipher		Run all cipher tests
  *	eta		Run all encrypt-then-authenticate tests
  *	aead		Run all authenticated encryption with associated data
  *			tests
  *
  * Hashes:
  *	sha1		SHA-1
  *	sha224		224-bit SHA-2
  *	sha256		256-bit SHA-2
  *	sha384		384-bit SHA-2
  *	sha512		512-bit	SHA-2
  *	blake2b		Blake2-B
  *	blake2s		Blake2-S
  *
  * MACs:
  *	sha1hmac	SHA-1 HMAC
  *	sha224hmac	224-bit SHA-2 HMAC
  *	sha256hmac	256-bit SHA-2 HMAC
  *	sha384hmac	384-bit SHA-2 HMAC
  *	sha512hmac	512-bit	SHA-2 HMAC
+ *	gmac		128-bit GMAC
+ *	gmac192		192-bit GMAC
+ *	gmac256		256-bit GMAC
  *
  * Ciphers:
  *	aes-cbc		128-bit AES-CBC
  *	aes-cbc192	192-bit	AES-CBC
  *	aes-cbc256	256-bit AES-CBC
  *	aes-ctr		128-bit AES-CTR
  *	aes-ctr192	192-bit AES-CTR
  *	aes-ctr256	256-bit AES-CTR
  *	aes-xts		128-bit AES-XTS
  *	aes-xts256	256-bit AES-XTS
  *	chacha20
  *
  * Encrypt then Authenticate:
  *	<cipher>+<mac>
  *
  * Authenticated Encryption with Associated Data:
  *	aes-gcm		128-bit AES-GCM
  *	aes-gcm192	192-bit AES-GCM
  *	aes-gcm256	256-bit AES-GCM
  *	aes-ccm		128-bit AES-CCM
  *	aes-ccm192	192-bit AES-CCM
  *	aes-ccm256	256-bit AES-CCM
  */
 
 #include <sys/param.h>
 #include <sys/sysctl.h>
 #include <assert.h>
 #include <err.h>
 #include <fcntl.h>
 #include <libutil.h>
 #include <stdbool.h>
 #include <stdio.h>
 #include <string.h>
 #include <unistd.h>
 
 #include <openssl/err.h>
 #include <openssl/hmac.h>
 
 #include <crypto/cryptodev.h>
 
 struct ocf_session {
 	int fd;
 	int ses;
 	int crid;
 };
 
 const struct alg {
 	const char *name;
 	int cipher;
 	int mac;
-	enum { T_HASH, T_HMAC, T_CIPHER, T_ETA, T_AEAD } type;
+	enum { T_HASH, T_HMAC, T_GMAC, T_CIPHER, T_ETA, T_AEAD } type;
 	const EVP_CIPHER *(*evp_cipher)(void);
 	const EVP_MD *(*evp_md)(void);
 } algs[] = {
 	{ .name = "sha1", .mac = CRYPTO_SHA1, .type = T_HASH,
 	  .evp_md = EVP_sha1 },
 	{ .name = "sha224", .mac = CRYPTO_SHA2_224, .type = T_HASH,
 	  .evp_md = EVP_sha224 },
 	{ .name = "sha256", .mac = CRYPTO_SHA2_256, .type = T_HASH,
 	  .evp_md = EVP_sha256 },
 	{ .name = "sha384", .mac = CRYPTO_SHA2_384, .type = T_HASH,
 	  .evp_md = EVP_sha384 },
 	{ .name = "sha512", .mac = CRYPTO_SHA2_512, .type = T_HASH,
 	  .evp_md = EVP_sha512 },
 	{ .name = "sha1hmac", .mac = CRYPTO_SHA1_HMAC, .type = T_HMAC,
 	  .evp_md = EVP_sha1 },
 	{ .name = "sha224hmac", .mac = CRYPTO_SHA2_224_HMAC, .type = T_HMAC,
 	  .evp_md = EVP_sha224 },
 	{ .name = "sha256hmac", .mac = CRYPTO_SHA2_256_HMAC, .type = T_HMAC,
 	  .evp_md = EVP_sha256 },
 	{ .name = "sha384hmac", .mac = CRYPTO_SHA2_384_HMAC, .type = T_HMAC,
 	  .evp_md = EVP_sha384 },
 	{ .name = "sha512hmac", .mac = CRYPTO_SHA2_512_HMAC, .type = T_HMAC,
 	  .evp_md = EVP_sha512 },
 	{ .name = "blake2b", .mac = CRYPTO_BLAKE2B, .type = T_HASH,
 	  .evp_md = EVP_blake2b512 },
 	{ .name = "blake2s", .mac = CRYPTO_BLAKE2S, .type = T_HASH,
 	  .evp_md = EVP_blake2s256 },
+	{ .name = "gmac", .mac = CRYPTO_AES_NIST_GMAC, .type = T_GMAC,
+	  .evp_cipher = EVP_aes_128_gcm },
+	{ .name = "gmac192", .mac = CRYPTO_AES_NIST_GMAC, .type = T_GMAC,
+	  .evp_cipher = EVP_aes_192_gcm },
+	{ .name = "gmac256", .mac = CRYPTO_AES_NIST_GMAC, .type = T_GMAC,
+	  .evp_cipher = EVP_aes_256_gcm },
 	{ .name = "aes-cbc", .cipher = CRYPTO_AES_CBC, .type = T_CIPHER,
 	  .evp_cipher = EVP_aes_128_cbc },
 	{ .name = "aes-cbc192", .cipher = CRYPTO_AES_CBC, .type = T_CIPHER,
 	  .evp_cipher = EVP_aes_192_cbc },
 	{ .name = "aes-cbc256", .cipher = CRYPTO_AES_CBC, .type = T_CIPHER,
 	  .evp_cipher = EVP_aes_256_cbc },
 	{ .name = "aes-ctr", .cipher = CRYPTO_AES_ICM, .type = T_CIPHER,
 	  .evp_cipher = EVP_aes_128_ctr },
 	{ .name = "aes-ctr192", .cipher = CRYPTO_AES_ICM, .type = T_CIPHER,
 	  .evp_cipher = EVP_aes_192_ctr },
 	{ .name = "aes-ctr256", .cipher = CRYPTO_AES_ICM, .type = T_CIPHER,
 	  .evp_cipher = EVP_aes_256_ctr },
 	{ .name = "aes-xts", .cipher = CRYPTO_AES_XTS, .type = T_CIPHER,
 	  .evp_cipher = EVP_aes_128_xts },
 	{ .name = "aes-xts256", .cipher = CRYPTO_AES_XTS, .type = T_CIPHER,
 	  .evp_cipher = EVP_aes_256_xts },
 	{ .name = "chacha20", .cipher = CRYPTO_CHACHA20, .type = T_CIPHER,
 	  .evp_cipher = EVP_chacha20 },
-	{ .name = "aes-gcm", .cipher = CRYPTO_AES_NIST_GCM_16,
-	  .mac = CRYPTO_AES_128_NIST_GMAC, .type = T_AEAD,
+	{ .name = "aes-gcm", .cipher = CRYPTO_AES_NIST_GCM_16, .type = T_AEAD,
 	  .evp_cipher = EVP_aes_128_gcm },
 	{ .name = "aes-gcm192", .cipher = CRYPTO_AES_NIST_GCM_16,
-	  .mac = CRYPTO_AES_192_NIST_GMAC, .type = T_AEAD,
-	  .evp_cipher = EVP_aes_192_gcm },
+	  .type = T_AEAD, .evp_cipher = EVP_aes_192_gcm },
 	{ .name = "aes-gcm256", .cipher = CRYPTO_AES_NIST_GCM_16,
-	  .mac = CRYPTO_AES_256_NIST_GMAC, .type = T_AEAD,
-	  .evp_cipher = EVP_aes_256_gcm },
-	{ .name = "aes-ccm", .cipher = CRYPTO_AES_CCM_16,
-	  .mac = CRYPTO_AES_CCM_CBC_MAC, .type = T_AEAD,
+	  .type = T_AEAD, .evp_cipher = EVP_aes_256_gcm },
+	{ .name = "aes-ccm", .cipher = CRYPTO_AES_CCM_16, .type = T_AEAD,
 	  .evp_cipher = EVP_aes_128_ccm },
-	{ .name = "aes-ccm192", .cipher = CRYPTO_AES_CCM_16,
-	  .mac = CRYPTO_AES_CCM_CBC_MAC, .type = T_AEAD,
+	{ .name = "aes-ccm192", .cipher = CRYPTO_AES_CCM_16, .type = T_AEAD,
 	  .evp_cipher = EVP_aes_192_ccm },
-	{ .name = "aes-ccm256", .cipher = CRYPTO_AES_CCM_16,
-	  .mac = CRYPTO_AES_CCM_CBC_MAC, .type = T_AEAD,
+	{ .name = "aes-ccm256", .cipher = CRYPTO_AES_CCM_16, .type = T_AEAD,
 	  .evp_cipher = EVP_aes_256_ccm },
 };
 
 static bool verbose;
 static int crid;
 static size_t aad_len;
 
 static void
 usage(void)
 {
 	fprintf(stderr,
 	    "usage: cryptocheck [-z] [-a algorithm] [-d dev] [size ...]\n");
 	exit(1);
 }
 
 static const struct alg *
 find_alg(const char *name)
 {
 	u_int i;
 
 	for (i = 0; i < nitems(algs); i++)
 		if (strcasecmp(algs[i].name, name) == 0)
 			return (&algs[i]);
 	return (NULL);
 }
 
 static struct alg *
 build_eta(const struct alg *cipher, const struct alg *mac)
 {
 	struct alg *eta;
 	char *name;
 
 	assert(cipher->type == T_CIPHER);
 	assert(mac->type == T_HMAC);
 	eta = calloc(1, sizeof(*eta));
 	asprintf(&name, "%s+%s", cipher->name, mac->name);
 	eta->name = name;
 	eta->cipher = cipher->cipher;
 	eta->mac = mac->mac;
 	eta->type = T_ETA;
 	eta->evp_cipher = cipher->evp_cipher;
 	eta->evp_md = mac->evp_md;
 	return (eta);
 }
 
 static void
 free_eta(struct alg *eta)
 {
 	free(__DECONST(char *, eta->name));
 	free(eta);
 }
 
 static struct alg *
 build_eta_name(const char *name)
 {
 	const struct alg *cipher, *mac;
 	const char *mac_name;
 	char *cp, *cipher_name;
 
 	cp = strchr(name, '+');
 	cipher_name = strndup(name, cp - name);
 	mac_name = cp + 1;
 	cipher = find_alg(cipher_name);
 	free(cipher_name);
 	if (cipher == NULL || cipher->type != T_CIPHER)
 		errx(1, "Invalid cipher %s", cipher_name);
 	mac = find_alg(mac_name);
 	if (mac == NULL || mac->type != T_HMAC)
 		errx(1, "Invalid hmac %s", mac_name);
 	return (build_eta(cipher, mac));
 }
 
 static int
 devcrypto(void)
 {
 	static int fd = -1;
 
 	if (fd < 0) {
 		fd = open("/dev/crypto", O_RDWR | O_CLOEXEC, 0);
 		if (fd < 0)
 			err(1, "/dev/crypto");
 	}
 	return (fd);
 }
 
 /*
  * Called on exit to change kern.cryptodevallowsoft back to 0
  */
 #define CRYPT_SOFT_ALLOW	"kern.cryptodevallowsoft"
 
 static void
 reset_user_soft(void)
 {
 	int off = 0;
 	sysctlbyname(CRYPT_SOFT_ALLOW, NULL, NULL, &off, sizeof(off));
 }
 
 static void
 enable_user_soft(void)
 {
 	int curstate;
 	int on = 1;
 	size_t cursize = sizeof(curstate);
 
 	if (sysctlbyname(CRYPT_SOFT_ALLOW, &curstate, &cursize,
 		&on, sizeof(on)) == 0) {
 		if (curstate == 0)
 			atexit(reset_user_soft);
 	}
 }
 
 static int
 crlookup(const char *devname)
 {
 	struct crypt_find_op find;
 
 	if (strncmp(devname, "soft", 4) == 0) {
 		enable_user_soft();
 		return CRYPTO_FLAG_SOFTWARE;
 	}
 
 	find.crid = -1;
 	strlcpy(find.name, devname, sizeof(find.name));
 	if (ioctl(devcrypto(), CIOCFINDDEV, &find) == -1)
 		err(1, "ioctl(CIOCFINDDEV)");
 	return (find.crid);
 }
 
 const char *
 crfind(int crid)
 {
 	static struct crypt_find_op find;
 
 	if (crid == CRYPTO_FLAG_SOFTWARE)
 		return ("soft");
 	else if (crid == CRYPTO_FLAG_HARDWARE)
 		return ("unknown");
 
 	bzero(&find, sizeof(find));
 	find.crid = crid;
 	if (ioctl(devcrypto(), CRIOFINDDEV, &find) == -1)
 		err(1, "ioctl(CIOCFINDDEV): crid %d", crid);
 	return (find.name);
 }
 
 static int
 crget(void)
 {
 	int fd;
 
 	if (ioctl(devcrypto(), CRIOGET, &fd) == -1)
 		err(1, "ioctl(CRIOGET)");
 	if (fcntl(fd, F_SETFD, 1) == -1)
 		err(1, "fcntl(F_SETFD) (crget)");
 	return fd;
 }
 
 static char
 rdigit(void)
 {
 	const char a[] = {
 		0x10,0x54,0x11,0x48,0x45,0x12,0x4f,0x13,0x49,0x53,0x14,0x41,
 		0x15,0x16,0x4e,0x55,0x54,0x17,0x18,0x4a,0x4f,0x42,0x19,0x01
 	};
 	return 0x20+a[random()%nitems(a)];
 }
 
 static char *
 alloc_buffer(size_t len)
 {
 	char *buf;
 	size_t i;
 
 	buf = malloc(len);
 	for (i = 0; i < len; i++)
 		buf[i] = rdigit();
 	return (buf);
 }
 
 static char *
 generate_iv(size_t len, const struct alg *alg)
 {
 	char *iv;
 
 	iv = alloc_buffer(len);
 	switch (alg->cipher) {
 	case CRYPTO_AES_ICM:
 		/* Clear the low 32 bits of the IV to hold the counter. */
 		iv[len - 4] = 0;
 		iv[len - 3] = 0;
 		iv[len - 2] = 0;
 		iv[len - 1] = 0;
 		break;
 	case CRYPTO_AES_XTS:
 		/*
 		 * Clear the low 64-bits to only store a 64-bit block
 		 * number.
 		 */
 		iv[len - 8] = 0;
 		iv[len - 7] = 0;
 		iv[len - 6] = 0;
 		iv[len - 5] = 0;
 		iv[len - 4] = 0;
 		iv[len - 3] = 0;
 		iv[len - 2] = 0;
 		iv[len - 1] = 0;
 		break;
 	}
 	return (iv);
 }
 
 static void
 ocf_init_sop(struct session2_op *sop)
 {
 	memset(sop, 0, sizeof(*sop));
 	sop->crid = crid;
 }
 
 static bool
 ocf_init_session(struct session2_op *sop, const char *type, const char *name,
     struct ocf_session *ses)
 {
 	int fd;
 
 	fd = crget();
 	if (ioctl(fd, CIOCGSESSION2, sop) < 0) {
 		warn("cryptodev %s %s not supported for device %s",
 		    type, name, crfind(crid));
 		close(fd);
 		ses->fd = -1;
 		return (false);
 	}
 	ses->fd = fd;
 	ses->ses = sop->ses;
 	ses->crid = sop->crid;
 	return (true);
 }
 
 static void
 ocf_destroy_session(struct ocf_session *ses)
 {
 	if (ses->fd == -1)
 		return;
 
 	if (ioctl(ses->fd, CIOCFSESSION, &ses->ses) < 0)
 		warn("ioctl(CIOCFSESSION)");
 
 	close(ses->fd);
 }
 
 static void
 ocf_init_cop(const struct ocf_session *ses, struct crypt_op *cop)
 {
 	memset(cop, 0, sizeof(*cop));
 	cop->ses = ses->ses;
 }
 
 static void
 ocf_init_caead(const struct ocf_session *ses, struct crypt_aead *caead)
 {
 	memset(caead, 0, sizeof(*caead));
 	caead->ses = ses->ses;
 }
 
 static bool
 ocf_hash(const struct alg *alg, const char *buffer, size_t size, char *digest,
     int *cridp)
 {
 	struct ocf_session ses;
 	struct session2_op sop;
 	struct crypt_op cop;
 	int error;
 
 	ocf_init_sop(&sop);
 	sop.mac = alg->mac;
 	if (!ocf_init_session(&sop, "HASH", alg->name, &ses))
 		return (false);
 
 	ocf_init_cop(&ses, &cop);
 	cop.op = 0;
 	cop.len = size;
 	cop.src = (char *)buffer;
 	cop.mac = digest;
 
 	if (ioctl(ses.fd, CIOCCRYPT, &cop) < 0) {
 		warn("cryptodev %s (%zu) HASH failed for device %s", alg->name,
 		    size, crfind(crid));
 		ocf_destroy_session(&ses);
 		return (false);
 	}
 
 	*cridp = ses.crid;
 	ocf_destroy_session(&ses);
 	return (true);
 }
 
 static void
 openssl_hash(const struct alg *alg, const EVP_MD *md, const void *buffer,
     size_t size, void *digest_out, unsigned *digest_sz_out)
 {
 	EVP_MD_CTX *mdctx;
 	const char *errs;
 	int rc;
 
 	errs = "";
 
 	mdctx = EVP_MD_CTX_create();
 	if (mdctx == NULL)
 		goto err_out;
 
 	rc = EVP_DigestInit_ex(mdctx, md, NULL);
 	if (rc != 1)
 		goto err_out;
 
 	rc = EVP_DigestUpdate(mdctx, buffer, size);
 	if (rc != 1)
 		goto err_out;
 
 	rc = EVP_DigestFinal_ex(mdctx, digest_out, digest_sz_out);
 	if (rc != 1)
 		goto err_out;
 
 	EVP_MD_CTX_destroy(mdctx);
 	return;
 
 err_out:
 	errx(1, "OpenSSL %s HASH failed%s: %s", alg->name, errs,
 	    ERR_error_string(ERR_get_error(), NULL));
 }
 
 static void
 run_hash_test(const struct alg *alg, size_t size)
 {
 	const EVP_MD *md;
 	char *buffer;
 	u_int digest_len;
 	int crid;
 	char control_digest[EVP_MAX_MD_SIZE], test_digest[EVP_MAX_MD_SIZE];
 
 	memset(control_digest, 0x3c, sizeof(control_digest));
 	memset(test_digest, 0x3c, sizeof(test_digest));
 
 	md = alg->evp_md();
 	assert(EVP_MD_size(md) <= sizeof(control_digest));
 
 	buffer = alloc_buffer(size);
 
 	/* OpenSSL HASH. */
 	digest_len = sizeof(control_digest);
 	openssl_hash(alg, md, buffer, size, control_digest, &digest_len);
 
 	/* cryptodev HASH. */
 	if (!ocf_hash(alg, buffer, size, test_digest, &crid))
 		goto out;
 	if (memcmp(control_digest, test_digest, sizeof(control_digest)) != 0) {
 		if (memcmp(control_digest, test_digest, EVP_MD_size(md)) == 0)
 			printf("%s (%zu) mismatch in trailer:\n",
 			    alg->name, size);
 		else
 			printf("%s (%zu) mismatch:\n", alg->name, size);
 		printf("control:\n");
 		hexdump(control_digest, sizeof(control_digest), NULL, 0);
 		printf("test (cryptodev device %s):\n", crfind(crid));
 		hexdump(test_digest, sizeof(test_digest), NULL, 0);
 		goto out;
 	}
 
 	if (verbose)
 		printf("%s (%zu) matched (cryptodev device %s)\n",
 		    alg->name, size, crfind(crid));
 
 out:
 	free(buffer);
 }
 
 static bool
 ocf_hmac(const struct alg *alg, const char *buffer, size_t size,
     const char *key, size_t key_len, char *digest, int *cridp)
 {
 	struct ocf_session ses;
 	struct session2_op sop;
 	struct crypt_op cop;
 
 	ocf_init_sop(&sop);
 	sop.mackeylen = key_len;
 	sop.mackey = (char *)key;
 	sop.mac = alg->mac;
 	if (!ocf_init_session(&sop, "HMAC", alg->name, &ses))
 		return (false);
 
 	ocf_init_cop(&ses, &cop);
 	cop.op = 0;
 	cop.len = size;
 	cop.src = (char *)buffer;
 	cop.mac = digest;
 
 	if (ioctl(ses.fd, CIOCCRYPT, &cop) < 0) {
 		warn("cryptodev %s (%zu) HMAC failed for device %s", alg->name,
 		    size, crfind(crid));
 		ocf_destroy_session(&ses);
 		return (false);
 	}
 
 	*cridp = ses.crid;
 	ocf_destroy_session(&ses);
 	return (true);
 }
 
 static void
 run_hmac_test(const struct alg *alg, size_t size)
 {
 	const EVP_MD *md;
 	char *key, *buffer;
 	u_int key_len, digest_len;
 	int crid;
 	char control_digest[EVP_MAX_MD_SIZE], test_digest[EVP_MAX_MD_SIZE];
 
 	memset(control_digest, 0x3c, sizeof(control_digest));
 	memset(test_digest, 0x3c, sizeof(test_digest));
 
 	md = alg->evp_md();
 	key_len = EVP_MD_size(md);
 	assert(EVP_MD_size(md) <= sizeof(control_digest));
 
 	key = alloc_buffer(key_len);
 	buffer = alloc_buffer(size);
 
 	/* OpenSSL HMAC. */
 	digest_len = sizeof(control_digest);
 	if (HMAC(md, key, key_len, (u_char *)buffer, size,
 	    (u_char *)control_digest, &digest_len) == NULL)
 		errx(1, "OpenSSL %s (%zu) HMAC failed: %s", alg->name,
 		    size, ERR_error_string(ERR_get_error(), NULL));
 
 	/* cryptodev HMAC. */
 	if (!ocf_hmac(alg, buffer, size, key, key_len, test_digest, &crid))
 		goto out;
 	if (memcmp(control_digest, test_digest, sizeof(control_digest)) != 0) {
 		if (memcmp(control_digest, test_digest, EVP_MD_size(md)) == 0)
 			printf("%s (%zu) mismatch in trailer:\n",
 			    alg->name, size);
 		else
 			printf("%s (%zu) mismatch:\n", alg->name, size);
 		printf("control:\n");
 		hexdump(control_digest, sizeof(control_digest), NULL, 0);
 		printf("test (cryptodev device %s):\n", crfind(crid));
 		hexdump(test_digest, sizeof(test_digest), NULL, 0);
 		goto out;
 	}
 
 	if (verbose)
 		printf("%s (%zu) matched (cryptodev device %s)\n",
 		    alg->name, size, crfind(crid));
 
 out:
 	free(buffer);
 	free(key);
 }
 
 static void
 openssl_cipher(const struct alg *alg, const EVP_CIPHER *cipher, const char *key,
     const char *iv, const char *input, char *output, size_t size, int enc)
 {
 	EVP_CIPHER_CTX *ctx;
 	int outl, total;
 
 	ctx = EVP_CIPHER_CTX_new();
 	if (ctx == NULL)
 		errx(1, "OpenSSL %s (%zu) ctx new failed: %s", alg->name,
 		    size, ERR_error_string(ERR_get_error(), NULL));
 	if (EVP_CipherInit_ex(ctx, cipher, NULL, (const u_char *)key,
 	    (const u_char *)iv, enc) != 1)
 		errx(1, "OpenSSL %s (%zu) ctx init failed: %s", alg->name,
 		    size, ERR_error_string(ERR_get_error(), NULL));
 	EVP_CIPHER_CTX_set_padding(ctx, 0);
 	if (EVP_CipherUpdate(ctx, (u_char *)output, &outl,
 	    (const u_char *)input, size) != 1)
 		errx(1, "OpenSSL %s (%zu) cipher update failed: %s", alg->name,
 		    size, ERR_error_string(ERR_get_error(), NULL));
 	total = outl;
 	if (EVP_CipherFinal_ex(ctx, (u_char *)output + outl, &outl) != 1)
 		errx(1, "OpenSSL %s (%zu) cipher final failed: %s", alg->name,
 		    size, ERR_error_string(ERR_get_error(), NULL));
 	total += outl;
 	if (total != size)
 		errx(1, "OpenSSL %s (%zu) cipher size mismatch: %d", alg->name,
 		    size, total);
 	EVP_CIPHER_CTX_free(ctx);
 }
 
 static bool
 ocf_init_cipher_session(const struct alg *alg, const char *key, size_t key_len,
     struct ocf_session *ses)
 {
 	struct session2_op sop;
 
 	ocf_init_sop(&sop);
 	sop.keylen = key_len;
 	sop.key = (char *)key;
 	sop.cipher = alg->cipher;
 	return (ocf_init_session(&sop, "cipher", alg->name, ses));
 }
 
 static bool
 ocf_cipher(const struct ocf_session *ses, const struct alg *alg, const char *iv,
     const char *input, char *output, size_t size, int op)
 {
 	struct crypt_op cop;
 
 	ocf_init_cop(ses, &cop);
 	cop.op = op;
 	cop.len = size;
 	cop.src = (char *)input;
 	cop.dst = output;
 	cop.iv = (char *)iv;
 
 	if (ioctl(ses->fd, CIOCCRYPT, &cop) < 0) {
 		warn("cryptodev %s (%zu) cipher failed for device %s",
 		    alg->name, size, crfind(crid));
 		return (false);
 	}
 
 	return (true);
 }
 
 static void
 run_cipher_test(const struct alg *alg, size_t size)
 {
 	struct ocf_session ses;
 	const EVP_CIPHER *cipher;
 	char *buffer, *cleartext, *ciphertext;
 	char *iv, *key;
 	u_int iv_len, key_len;
 
 	cipher = alg->evp_cipher();
 	if (size % EVP_CIPHER_block_size(cipher) != 0) {
 		if (verbose)
 			printf(
 			    "%s (%zu): invalid buffer size (block size %d)\n",
 			    alg->name, size, EVP_CIPHER_block_size(cipher));
 		return;
 	}
 
 	key_len = EVP_CIPHER_key_length(cipher);
 	iv_len = EVP_CIPHER_iv_length(cipher);
 
 	key = alloc_buffer(key_len);
 	iv = generate_iv(iv_len, alg);
 	cleartext = alloc_buffer(size);
 	buffer = malloc(size);
 	ciphertext = malloc(size);
 
 	/* OpenSSL cipher. */
 	openssl_cipher(alg, cipher, key, iv, cleartext, ciphertext, size, 1);
 	if (size > 0 && memcmp(cleartext, ciphertext, size) == 0)
 		errx(1, "OpenSSL %s (%zu): cipher text unchanged", alg->name,
 		    size);
 	openssl_cipher(alg, cipher, key, iv, ciphertext, buffer, size, 0);
 	if (memcmp(cleartext, buffer, size) != 0) {
 		printf("OpenSSL %s (%zu): cipher mismatch:", alg->name, size);
 		printf("original:\n");
 		hexdump(cleartext, size, NULL, 0);
 		printf("decrypted:\n");
 		hexdump(buffer, size, NULL, 0);
 		exit(1);
 	}
 
 	if (!ocf_init_cipher_session(alg, key, key_len, &ses))
 		goto out;
 
 	/* OCF encrypt. */
 	if (!ocf_cipher(&ses, alg, iv, cleartext, buffer, size, COP_ENCRYPT))
 		goto out;
 	if (memcmp(ciphertext, buffer, size) != 0) {
 		printf("%s (%zu) encryption mismatch:\n", alg->name, size);
 		printf("control:\n");
 		hexdump(ciphertext, size, NULL, 0);
 		printf("test (cryptodev device %s):\n", crfind(ses.crid));
 		hexdump(buffer, size, NULL, 0);
 		goto out;
 	}
 
 	/* OCF decrypt. */
 	if (!ocf_cipher(&ses, alg, iv, ciphertext, buffer, size, COP_DECRYPT))
 		goto out;
 	if (memcmp(cleartext, buffer, size) != 0) {
 		printf("%s (%zu) decryption mismatch:\n", alg->name, size);
 		printf("control:\n");
 		hexdump(cleartext, size, NULL, 0);
 		printf("test (cryptodev device %s):\n", crfind(ses.crid));
 		hexdump(buffer, size, NULL, 0);
 		goto out;
 	}
 
 	if (verbose)
 		printf("%s (%zu) matched (cryptodev device %s)\n",
 		    alg->name, size, crfind(ses.crid));
 
 out:
 	ocf_destroy_session(&ses);
 	free(ciphertext);
 	free(buffer);
 	free(cleartext);
 	free(iv);
 	free(key);
 }
 
 static bool
 ocf_init_eta_session(const struct alg *alg, const char *cipher_key,
     size_t cipher_key_len, const char *auth_key, size_t auth_key_len,
     struct ocf_session *ses)
 {
 	struct session2_op sop;
 
 	ocf_init_sop(&sop);
 	sop.keylen = cipher_key_len;
 	sop.key = (char *)cipher_key;
 	sop.cipher = alg->cipher;
 	sop.mackeylen = auth_key_len;
 	sop.mackey = (char *)auth_key;
 	sop.mac = alg->mac;
 	return (ocf_init_session(&sop, "ETA", alg->name, ses));
 }
 
-static bool
+static int
 ocf_eta(const struct ocf_session *ses, const struct alg *alg, const char *iv,
     size_t iv_len, const char *aad, size_t aad_len, const char *input,
     char *output, size_t size, char *digest, int op)
 {
 	int ret;
 
 	if (aad_len != 0) {
 		struct crypt_aead caead;
 
 		ocf_init_caead(ses, &caead);
 		caead.op = op;
-		caead.flags = op == COP_ENCRYPT ? COP_F_CIPHER_FIRST : 0;
 		caead.len = size;
 		caead.aadlen = aad_len;
 		caead.ivlen = iv_len;
 		caead.src = (char *)input;
 		caead.dst = output;
 		caead.aad = (char *)aad;
 		caead.tag = digest;
 		caead.iv = (char *)iv;
 
 		ret = ioctl(ses->fd, CIOCCRYPTAEAD, &caead);
 	} else {
 		struct crypt_op cop;
 
 		ocf_init_cop(ses, &cop);
 		cop.op = op;
-		cop.flags = op == COP_ENCRYPT ? COP_F_CIPHER_FIRST : 0;
 		cop.len = size;
 		cop.src = (char *)input;
 		cop.dst = output;
 		cop.mac = digest;
 		cop.iv = (char *)iv;
 
 		ret = ioctl(ses->fd, CIOCCRYPT, &cop);
 	}
 
-	if (ret < 0) {
-		warn("cryptodev %s (%zu) ETA failed for device %s",
-		    alg->name, size, crfind(crid));
-		return (false);
-	}
-
-	return (true);
+	if (ret < 0)
+		return (errno);
+	return (0);
 }
 
 static void
 run_eta_test(const struct alg *alg, size_t size)
 {
 	struct ocf_session ses;
 	const EVP_CIPHER *cipher;
 	const EVP_MD *md;
 	char *aad, *buffer, *cleartext, *ciphertext;
 	char *iv, *auth_key, *cipher_key;
-	u_int i, iv_len, auth_key_len, cipher_key_len, digest_len;
+	u_int iv_len, auth_key_len, cipher_key_len, digest_len;
+	int error;
 	char control_digest[EVP_MAX_MD_SIZE], test_digest[EVP_MAX_MD_SIZE];
 
 	cipher = alg->evp_cipher();
 	if (size % EVP_CIPHER_block_size(cipher) != 0) {
 		if (verbose)
 			printf(
 			    "%s (%zu): invalid buffer size (block size %d)\n",
 			    alg->name, size, EVP_CIPHER_block_size(cipher));
 		return;
 	}
 
 	memset(control_digest, 0x3c, sizeof(control_digest));
 	memset(test_digest, 0x3c, sizeof(test_digest));
 
 	md = alg->evp_md();
 
 	cipher_key_len = EVP_CIPHER_key_length(cipher);
 	iv_len = EVP_CIPHER_iv_length(cipher);
 	auth_key_len = EVP_MD_size(md);
 
 	cipher_key = alloc_buffer(cipher_key_len);
 	iv = generate_iv(iv_len, alg);
 	auth_key = alloc_buffer(auth_key_len);
 	cleartext = alloc_buffer(aad_len + size);
 	buffer = malloc(aad_len + size);
 	ciphertext = malloc(aad_len + size);
 
 	/* OpenSSL encrypt + HMAC. */
 	if (aad_len != 0)
 		memcpy(ciphertext, cleartext, aad_len);
 	openssl_cipher(alg, cipher, cipher_key, iv, cleartext + aad_len,
 	    ciphertext + aad_len, size, 1);
 	if (size > 0 && memcmp(cleartext + aad_len, ciphertext + aad_len,
 	    size) == 0)
 		errx(1, "OpenSSL %s (%zu): cipher text unchanged", alg->name,
 		    size);
 	digest_len = sizeof(control_digest);
 	if (HMAC(md, auth_key, auth_key_len, (u_char *)ciphertext,
 	    aad_len + size, (u_char *)control_digest, &digest_len) == NULL)
 		errx(1, "OpenSSL %s (%zu) HMAC failed: %s", alg->name,
 		    size, ERR_error_string(ERR_get_error(), NULL));
 
 	if (!ocf_init_eta_session(alg, cipher_key, cipher_key_len, auth_key,
 	    auth_key_len, &ses))
 		goto out;
 
 	/* OCF encrypt + HMAC. */
-	if (!ocf_eta(&ses, alg, iv, iv_len,
+	error = ocf_eta(&ses, alg, iv, iv_len,
 	    aad_len != 0 ? cleartext : NULL, aad_len, cleartext + aad_len,
-	    buffer + aad_len, size, test_digest, COP_ENCRYPT))
+	    buffer + aad_len, size, test_digest, COP_ENCRYPT);
+	if (error != 0) {
+		warnc(error, "cryptodev %s (%zu) ETA failed for device %s",
+		    alg->name, size, crfind(ses.crid));
 		goto out;
+	}
 	if (memcmp(ciphertext + aad_len, buffer + aad_len, size) != 0) {
 		printf("%s (%zu) encryption mismatch:\n", alg->name, size);
 		printf("control:\n");
 		hexdump(ciphertext + aad_len, size, NULL, 0);
 		printf("test (cryptodev device %s):\n", crfind(ses.crid));
 		hexdump(buffer + aad_len, size, NULL, 0);
 		goto out;
 	}
 	if (memcmp(control_digest, test_digest, sizeof(control_digest)) != 0) {
 		if (memcmp(control_digest, test_digest, EVP_MD_size(md)) == 0)
 			printf("%s (%zu) enc hash mismatch in trailer:\n",
 			    alg->name, size);
 		else
 			printf("%s (%zu) enc hash mismatch:\n", alg->name,
 			    size);
 		printf("control:\n");
 		hexdump(control_digest, sizeof(control_digest), NULL, 0);
 		printf("test (cryptodev device %s):\n", crfind(ses.crid));
 		hexdump(test_digest, sizeof(test_digest), NULL, 0);
 		goto out;
 	}
 
 	/* OCF HMAC + decrypt. */
-	if (!ocf_eta(&ses, alg, iv, iv_len,
+	error = ocf_eta(&ses, alg, iv, iv_len,
 	    aad_len != 0 ? ciphertext : NULL, aad_len, ciphertext + aad_len,
-	    buffer + aad_len, size, test_digest, COP_DECRYPT))
+	    buffer + aad_len, size, test_digest, COP_DECRYPT);
+	if (error != 0) {
+		warnc(error, "cryptodev %s (%zu) ETA failed for device %s",
+		    alg->name, size, crfind(ses.crid));
 		goto out;
+	}
 	if (memcmp(cleartext + aad_len, buffer + aad_len, size) != 0) {
 		printf("%s (%zu) decryption mismatch:\n", alg->name, size);
 		printf("control:\n");
 		hexdump(cleartext, size, NULL, 0);
 		printf("test (cryptodev device %s):\n", crfind(ses.crid));
 		hexdump(buffer, size, NULL, 0);
 		goto out;
 	}
 
+	/* Verify OCF HMAC + decrypt fails with busted MAC. */
+	test_digest[0] ^= 0x1;
+	error = ocf_eta(&ses, alg, iv, iv_len,
+	    aad_len != 0 ? ciphertext : NULL, aad_len, ciphertext + aad_len,
+	    buffer + aad_len, size, test_digest, COP_DECRYPT);
+	if (error != EBADMSG) {
+		if (error != 0)
+			warnc(error,
+		    "cryptodev %s (%zu) corrupt tag failed for device %s",
+			    alg->name, size, crfind(ses.crid));
+		else
+			warnx(
+		    "cryptodev %s (%zu) corrupt tag didn't fail for device %s",
+			    alg->name, size, crfind(ses.crid));
+		goto out;
+	}
+
 	if (verbose)
 		printf("%s (%zu) matched (cryptodev device %s)\n",
 		    alg->name, size, crfind(ses.crid));
 
 out:
 	ocf_destroy_session(&ses);
 	free(ciphertext);
 	free(buffer);
 	free(cleartext);
 	free(auth_key);
 	free(iv);
 	free(cipher_key);
 }
 
 static void
+openssl_gmac(const struct alg *alg, const EVP_CIPHER *cipher, const char *key,
+    const char *iv, const char *input, size_t size, char *tag)
+{
+	EVP_CIPHER_CTX *ctx;
+	int outl;
+
+	ctx = EVP_CIPHER_CTX_new();
+	if (ctx == NULL)
+		errx(1, "OpenSSL %s (%zu) ctx new failed: %s", alg->name,
+		    size, ERR_error_string(ERR_get_error(), NULL));
+	if (EVP_EncryptInit_ex(ctx, cipher, NULL, (const u_char *)key,
+	    (const u_char *)iv) != 1)
+		errx(1, "OpenSSL %s (%zu) ctx init failed: %s", alg->name,
+		    size, ERR_error_string(ERR_get_error(), NULL));
+	EVP_CIPHER_CTX_set_padding(ctx, 0);
+	if (EVP_EncryptUpdate(ctx, NULL, &outl, (const u_char *)input,
+		size) != 1)
+		errx(1, "OpenSSL %s (%zu) update failed: %s",
+		    alg->name, size, ERR_error_string(ERR_get_error(), NULL));
+	if (EVP_EncryptFinal_ex(ctx, NULL, &outl) != 1)
+		errx(1, "OpenSSL %s (%zu) final failed: %s", alg->name,
+		    size, ERR_error_string(ERR_get_error(), NULL));
+	if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, AES_GMAC_HASH_LEN,
+	    tag) != 1)
+		errx(1, "OpenSSL %s (%zu) get tag failed: %s", alg->name,
+		    size, ERR_error_string(ERR_get_error(), NULL));
+	EVP_CIPHER_CTX_free(ctx);
+}
+
+static bool
+ocf_gmac(const struct alg *alg, const char *input, size_t size, const char *key,
+    size_t key_len, const char *iv, char *tag, int *cridp)
+{
+	struct ocf_session ses;
+	struct session2_op sop;
+	struct crypt_op cop;
+
+	ocf_init_sop(&sop);
+	sop.mackeylen = key_len;
+	sop.mackey = (char *)key;
+	sop.mac = alg->mac;
+	if (!ocf_init_session(&sop, "GMAC", alg->name, &ses))
+		return (false);
+
+	ocf_init_cop(&ses, &cop);
+	cop.op = 0;
+	cop.len = size;
+	cop.src = (char *)input;
+	cop.mac = tag;
+	cop.iv = iv;
+
+	if (ioctl(ses.fd, CIOCCRYPT, &cop) < 0) {
+		warn("cryptodev %s (%zu) failed for device %s", alg->name,
+		    size, crfind(crid));
+		ocf_destroy_session(&ses);
+		return (false);
+	}
+
+	*cridp = ses.crid;
+	ocf_destroy_session(&ses);
+	return (true);
+}
+
+static void
+run_gmac_test(const struct alg *alg, size_t size)
+{
+	const EVP_CIPHER *cipher;
+	char *iv, *key, *buffer;
+	u_int iv_len, key_len, digest_len;
+	int crid;
+	char control_tag[AES_GMAC_HASH_LEN], test_tag[AES_GMAC_HASH_LEN];
+
+	cipher = alg->evp_cipher();
+
+	memset(control_tag, 0x3c, sizeof(control_tag));
+	memset(test_tag, 0x3c, sizeof(test_tag));
+
+	key_len = EVP_CIPHER_key_length(cipher);
+	iv_len = EVP_CIPHER_iv_length(cipher);
+
+	key = alloc_buffer(key_len);
+	iv = generate_iv(iv_len, alg);
+	buffer = alloc_buffer(size);
+
+	/* OpenSSL GMAC. */
+	openssl_gmac(alg, cipher, key, iv, buffer, size, control_tag);
+
+	/* OCF GMAC. */
+	if (!ocf_gmac(alg, buffer, size, key, key_len, iv, test_tag, &crid))
+		goto out;
+	if (memcmp(control_tag, test_tag, sizeof(control_tag)) != 0) {
+		printf("%s (%zu) mismatch:\n", alg->name, size);
+		printf("control:\n");
+		hexdump(control_tag, sizeof(control_tag), NULL, 0);
+		printf("test (cryptodev device %s):\n", crfind(crid));
+		hexdump(test_tag, sizeof(test_tag), NULL, 0);
+		goto out;
+	}
+
+	if (verbose)
+		printf("%s (%zu) matched (cryptodev device %s)\n",
+		    alg->name, size, crfind(crid));
+
+out:
+	free(buffer);
+	free(key);
+}
+
+static void
 openssl_gcm_encrypt(const struct alg *alg, const EVP_CIPHER *cipher,
     const char *key, const char *iv, const char *aad, size_t aad_len,
     const char *input, char *output, size_t size, char *tag)
 {
 	EVP_CIPHER_CTX *ctx;
 	int outl, total;
 
 	ctx = EVP_CIPHER_CTX_new();
 	if (ctx == NULL)
 		errx(1, "OpenSSL %s (%zu) ctx new failed: %s", alg->name,
 		    size, ERR_error_string(ERR_get_error(), NULL));
 	if (EVP_EncryptInit_ex(ctx, cipher, NULL, (const u_char *)key,
 	    (const u_char *)iv) != 1)
 		errx(1, "OpenSSL %s (%zu) ctx init failed: %s", alg->name,
 		    size, ERR_error_string(ERR_get_error(), NULL));
 	EVP_CIPHER_CTX_set_padding(ctx, 0);
 	if (aad != NULL) {
 		if (EVP_EncryptUpdate(ctx, NULL, &outl, (const u_char *)aad,
 		    aad_len) != 1)
 			errx(1, "OpenSSL %s (%zu) aad update failed: %s",
 			    alg->name, size,
 			    ERR_error_string(ERR_get_error(), NULL));
 	}
 	if (EVP_EncryptUpdate(ctx, (u_char *)output, &outl,
 	    (const u_char *)input, size) != 1)
 		errx(1, "OpenSSL %s (%zu) encrypt update failed: %s", alg->name,
 		    size, ERR_error_string(ERR_get_error(), NULL));
 	total = outl;
 	if (EVP_EncryptFinal_ex(ctx, (u_char *)output + outl, &outl) != 1)
 		errx(1, "OpenSSL %s (%zu) encrypt final failed: %s", alg->name,
 		    size, ERR_error_string(ERR_get_error(), NULL));
 	total += outl;
 	if (total != size)
 		errx(1, "OpenSSL %s (%zu) encrypt size mismatch: %d", alg->name,
 		    size, total);
 	if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, AES_GMAC_HASH_LEN,
 	    tag) != 1)
 		errx(1, "OpenSSL %s (%zu) get tag failed: %s", alg->name,
 		    size, ERR_error_string(ERR_get_error(), NULL));
 	EVP_CIPHER_CTX_free(ctx);
 }
 
 #ifdef notused
 static bool
 openssl_gcm_decrypt(const struct alg *alg, const EVP_CIPHER *cipher,
     const char *key, const char *iv, const char *aad, size_t aad_len,
     const char *input, char *output, size_t size, char *tag)
 {
 	EVP_CIPHER_CTX *ctx;
 	int outl, total;
 	bool valid;
 
 	ctx = EVP_CIPHER_CTX_new();
 	if (ctx == NULL)
 		errx(1, "OpenSSL %s (%zu) ctx new failed: %s", alg->name,
 		    size, ERR_error_string(ERR_get_error(), NULL));
 	if (EVP_DecryptInit_ex(ctx, cipher, NULL, (const u_char *)key,
 	    (const u_char *)iv) != 1)
 		errx(1, "OpenSSL %s (%zu) ctx init failed: %s", alg->name,
 		    size, ERR_error_string(ERR_get_error(), NULL));
 	EVP_CIPHER_CTX_set_padding(ctx, 0);
 	if (aad != NULL) {
 		if (EVP_DecryptUpdate(ctx, NULL, &outl, (const u_char *)aad,
 		    aad_len) != 1)
 			errx(1, "OpenSSL %s (%zu) aad update failed: %s",
 			    alg->name, size,
 			    ERR_error_string(ERR_get_error(), NULL));
 	}
 	if (EVP_DecryptUpdate(ctx, (u_char *)output, &outl,
 	    (const u_char *)input, size) != 1)
 		errx(1, "OpenSSL %s (%zu) decrypt update failed: %s", alg->name,
 		    size, ERR_error_string(ERR_get_error(), NULL));
 	total = outl;
 	if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, AES_GMAC_HASH_LEN,
 	    tag) != 1)
 		errx(1, "OpenSSL %s (%zu) get tag failed: %s", alg->name,
 		    size, ERR_error_string(ERR_get_error(), NULL));
 	valid = (EVP_DecryptFinal_ex(ctx, (u_char *)output + outl, &outl) != 1);
 	total += outl;
 	if (total != size)
 		errx(1, "OpenSSL %s (%zu) decrypt size mismatch: %d", alg->name,
 		    size, total);
 	EVP_CIPHER_CTX_free(ctx);
 	return (valid);
 }
 #endif
 
 static void
 openssl_ccm_encrypt(const struct alg *alg, const EVP_CIPHER *cipher,
     const char *key, const char *iv, size_t iv_len, const char *aad,
     size_t aad_len, const char *input, char *output, size_t size, char *tag)
 {
 	EVP_CIPHER_CTX *ctx;
 	int outl, total;
 
 	ctx = EVP_CIPHER_CTX_new();
 	if (ctx == NULL)
 		errx(1, "OpenSSL %s (%zu) ctx new failed: %s", alg->name,
 		    size, ERR_error_string(ERR_get_error(), NULL));
 	if (EVP_EncryptInit_ex(ctx, cipher, NULL, NULL, NULL) != 1)
 		errx(1, "OpenSSL %s (%zu) ctx init failed: %s", alg->name,
 		    size, ERR_error_string(ERR_get_error(), NULL));
 	if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_IVLEN, iv_len, NULL) != 1)
 		errx(1, "OpenSSL %s (%zu) setting iv length failed: %s", alg->name,
 		    size, ERR_error_string(ERR_get_error(), NULL));
 	if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG, AES_CBC_MAC_HASH_LEN, NULL) != 1)
 		errx(1, "OpenSSL %s (%zu) setting tag length failed: %s", alg->name,
 		     size, ERR_error_string(ERR_get_error(), NULL));
 	if (EVP_EncryptInit_ex(ctx, NULL, NULL, (const u_char *)key,
 	    (const u_char *)iv) != 1)
 		errx(1, "OpenSSL %s (%zu) ctx init failed: %s", alg->name,
 		    size, ERR_error_string(ERR_get_error(), NULL));
 	if (EVP_EncryptUpdate(ctx, NULL, &outl, NULL, size) != 1)
 		errx(1, "OpenSSL %s (%zu) unable to set data length: %s", alg->name,
 		     size, ERR_error_string(ERR_get_error(), NULL));
 
 	if (aad != NULL) {
 		if (EVP_EncryptUpdate(ctx, NULL, &outl, (const u_char *)aad,
 		    aad_len) != 1)
 			errx(1, "OpenSSL %s (%zu) aad update failed: %s",
 			    alg->name, size,
 			    ERR_error_string(ERR_get_error(), NULL));
 	}
 	if (EVP_EncryptUpdate(ctx, (u_char *)output, &outl,
 	    (const u_char *)input, size) != 1)
 		errx(1, "OpenSSL %s (%zu) encrypt update failed: %s", alg->name,
 		    size, ERR_error_string(ERR_get_error(), NULL));
 	total = outl;
 	if (EVP_EncryptFinal_ex(ctx, (u_char *)output + outl, &outl) != 1)
 		errx(1, "OpenSSL %s (%zu) encrypt final failed: %s", alg->name,
 		    size, ERR_error_string(ERR_get_error(), NULL));
 	total += outl;
 	if (total != size)
 		errx(1, "OpenSSL %s (%zu) encrypt size mismatch: %d", alg->name,
 		    size, total);
 	if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_GET_TAG, AES_CBC_MAC_HASH_LEN,
 	    tag) != 1)
 		errx(1, "OpenSSL %s (%zu) get tag failed: %s", alg->name,
 		    size, ERR_error_string(ERR_get_error(), NULL));
 	EVP_CIPHER_CTX_free(ctx);
 }
 
 static bool
 ocf_init_aead_session(const struct alg *alg, const char *key, size_t key_len,
     struct ocf_session *ses)
 {
 	struct session2_op sop;
 
 	ocf_init_sop(&sop);
 	sop.keylen = key_len;
 	sop.key = (char *)key;
 	sop.cipher = alg->cipher;
-	sop.mackeylen = key_len;
-	sop.mackey = (char *)key;
-	sop.mac = alg->mac;
 	return (ocf_init_session(&sop, "AEAD", alg->name, ses));
 }
 
 static int
 ocf_aead(const struct ocf_session *ses, const struct alg *alg, const char *iv,
     size_t iv_len, const char *aad, size_t aad_len, const char *input,
     char *output, size_t size, char *tag, int op)
 {
 	struct crypt_aead caead;
 
 	ocf_init_caead(ses, &caead);
 	caead.op = op;
 	caead.len = size;
 	caead.aadlen = aad_len;
 	caead.ivlen = iv_len;
 	caead.src = (char *)input;
 	caead.dst = output;
 	caead.aad = (char *)aad;
 	caead.tag = tag;
 	caead.iv = (char *)iv;
 
 	if (ioctl(ses->fd, CIOCCRYPTAEAD, &caead) < 0)
 		return (errno);
 	return (0);
 }
 
 #define	AEAD_MAX_TAG_LEN	MAX(AES_GMAC_HASH_LEN, AES_CBC_MAC_HASH_LEN)
 
 static void
 run_aead_test(const struct alg *alg, size_t size)
 {
 	struct ocf_session ses;
 	const EVP_CIPHER *cipher;
 	char *aad, *buffer, *cleartext, *ciphertext;
 	char *iv, *key;
 	u_int iv_len, key_len;
 	int error;
 	char control_tag[AEAD_MAX_TAG_LEN], test_tag[AEAD_MAX_TAG_LEN];
 
 	cipher = alg->evp_cipher();
 	if (size % EVP_CIPHER_block_size(cipher) != 0) {
 		if (verbose)
 			printf(
 			    "%s (%zu): invalid buffer size (block size %d)\n",
 			    alg->name, size, EVP_CIPHER_block_size(cipher));
 		return;
 	}
 
 	memset(control_tag, 0x3c, sizeof(control_tag));
 	memset(test_tag, 0x3c, sizeof(test_tag));
 
 	key_len = EVP_CIPHER_key_length(cipher);
 	iv_len = EVP_CIPHER_iv_length(cipher);
 
 	/*
 	 * AES-CCM can have varying IV lengths; however, for the moment
 	 * we only support AES_CCM_IV_LEN (12).  So if the sizes are
 	 * different, we'll fail.
 	 */
 	if (EVP_CIPHER_mode(cipher) == EVP_CIPH_CCM_MODE &&
 	    iv_len != AES_CCM_IV_LEN) {
 		if (verbose)
 			printf("OpenSSL CCM IV length (%d) != AES_CCM_IV_LEN",
 			    iv_len);
 		return;
 	}
 
 	key = alloc_buffer(key_len);
 	iv = generate_iv(iv_len, alg);
 	cleartext = alloc_buffer(size);
 	buffer = malloc(size);
 	ciphertext = malloc(size);
 	if (aad_len != 0)
 		aad = alloc_buffer(aad_len);
 	else
 		aad = NULL;
 
 	/* OpenSSL encrypt */
 	if (EVP_CIPHER_mode(cipher) == EVP_CIPH_CCM_MODE)
 		openssl_ccm_encrypt(alg, cipher, key, iv, iv_len, aad,
 		    aad_len, cleartext, ciphertext, size, control_tag);
 	else
 		openssl_gcm_encrypt(alg, cipher, key, iv, aad, aad_len,
 		    cleartext, ciphertext, size, control_tag);
 
 	if (!ocf_init_aead_session(alg, key, key_len, &ses))
 		goto out;
 
 	/* OCF encrypt */
 	error = ocf_aead(&ses, alg, iv, iv_len, aad, aad_len, cleartext, buffer,
 	    size, test_tag, COP_ENCRYPT);
 	if (error != 0) {
 		warnc(error, "cryptodev %s (%zu) failed for device %s",
 		    alg->name, size, crfind(ses.crid));
 		goto out;
 	}
 	if (memcmp(ciphertext, buffer, size) != 0) {
 		printf("%s (%zu) encryption mismatch:\n", alg->name, size);
 		printf("control:\n");
 		hexdump(ciphertext, size, NULL, 0);
 		printf("test (cryptodev device %s):\n", crfind(crid));
 		hexdump(buffer, size, NULL, 0);
 		goto out;
 	}
 	if (memcmp(control_tag, test_tag, sizeof(control_tag)) != 0) {
 		printf("%s (%zu) enc tag mismatch:\n", alg->name, size);
 		printf("control:\n");
 		hexdump(control_tag, sizeof(control_tag), NULL, 0);
 		printf("test (cryptodev device %s):\n", crfind(crid));
 		hexdump(test_tag, sizeof(test_tag), NULL, 0);
 		goto out;
 	}
 
 	/* OCF decrypt */
 	error = ocf_aead(&ses, alg, iv, iv_len, aad, aad_len, ciphertext,
 	    buffer, size, control_tag, COP_DECRYPT);
 	if (error != 0) {
 		warnc(error, "cryptodev %s (%zu) failed for device %s",
 		    alg->name, size, crfind(ses.crid));
 		goto out;
 	}
 	if (memcmp(cleartext, buffer, size) != 0) {
 		printf("%s (%zu) decryption mismatch:\n", alg->name, size);
 		printf("control:\n");
 		hexdump(cleartext, size, NULL, 0);
 		printf("test (cryptodev device %s):\n", crfind(crid));
 		hexdump(buffer, size, NULL, 0);
 		goto out;
 	}
 
 	/* Verify OCF decrypt fails with busted tag. */
 	test_tag[0] ^= 0x1;
 	error = ocf_aead(&ses, alg, iv, iv_len, aad, aad_len, ciphertext,
 	    buffer, size, test_tag, COP_DECRYPT);
 	if (error != EBADMSG) {
 		if (error != 0)
 			warnc(error,
 		    "cryptodev %s (%zu) corrupt tag failed for device %s",
 			    alg->name, size, crfind(ses.crid));
 		else
 			warnx(
 		    "cryptodev %s (%zu) corrupt tag didn't fail for device %s",
 			    alg->name, size, crfind(ses.crid));
 		goto out;
 	}
 
 	if (verbose)
 		printf("%s (%zu) matched (cryptodev device %s)\n",
 		    alg->name, size, crfind(ses.crid));
 
 out:
 	ocf_destroy_session(&ses);
 	free(aad);
 	free(ciphertext);
 	free(buffer);
 	free(cleartext);
 	free(iv);
 	free(key);
 }
 
 static void
 run_test(const struct alg *alg, size_t size)
 {
 
 	switch (alg->type) {
 	case T_HASH:
 		run_hash_test(alg, size);
 		break;
 	case T_HMAC:
 		run_hmac_test(alg, size);
 		break;
+	case T_GMAC:
+		run_gmac_test(alg, size);
+		break;
 	case T_CIPHER:
 		run_cipher_test(alg, size);
 		break;
 	case T_ETA:
 		run_eta_test(alg, size);
 		break;
 	case T_AEAD:
 		run_aead_test(alg, size);
 		break;
 	}
 }
 
 static void
 run_test_sizes(const struct alg *alg, size_t *sizes, u_int nsizes)
 {
 	u_int i;
 
 	for (i = 0; i < nsizes; i++)
 		run_test(alg, sizes[i]);
 }
 
 static void
 run_hash_tests(size_t *sizes, u_int nsizes)
 {
 	u_int i;
 
 	for (i = 0; i < nitems(algs); i++)
 		if (algs[i].type == T_HASH)
 			run_test_sizes(&algs[i], sizes, nsizes);
 }
 
 static void
 run_mac_tests(size_t *sizes, u_int nsizes)
 {
 	u_int i;
 
 	for (i = 0; i < nitems(algs); i++)
-		if (algs[i].type == T_HMAC)
+		if (algs[i].type == T_HMAC || algs[i].type == T_GMAC)
 			run_test_sizes(&algs[i], sizes, nsizes);
 }
 
 static void
 run_cipher_tests(size_t *sizes, u_int nsizes)
 {
 	u_int i;
 
 	for (i = 0; i < nitems(algs); i++)
 		if (algs[i].type == T_CIPHER)
 			run_test_sizes(&algs[i], sizes, nsizes);
 }
 
 static void
 run_eta_tests(size_t *sizes, u_int nsizes)
 {
 	const struct alg *cipher, *mac;
 	struct alg *eta;
 	u_int i, j;
 
 	for (i = 0; i < nitems(algs); i++) {
 		cipher = &algs[i];
 		if (cipher->type != T_CIPHER)
 			continue;
 		for (j = 0; j < nitems(algs); j++) {
 			mac = &algs[j];
 			if (mac->type != T_HMAC)
 				continue;
 			eta = build_eta(cipher, mac);
 			run_test_sizes(eta, sizes, nsizes);
 			free_eta(eta);
 		}
 	}
 }
 
 static void
 run_aead_tests(size_t *sizes, u_int nsizes)
 {
 	u_int i;
 
 	for (i = 0; i < nitems(algs); i++)
 		if (algs[i].type == T_AEAD)
 			run_test_sizes(&algs[i], sizes, nsizes);
 }
 
 int
 main(int ac, char **av)
 {
 	const char *algname;
 	const struct alg *alg;
 	struct alg *eta;
 	size_t sizes[128];
 	u_int i, nsizes;
 	bool testall;
 	int ch;
 
 	algname = NULL;
 	crid = CRYPTO_FLAG_HARDWARE;
 	testall = false;
 	verbose = false;
 	while ((ch = getopt(ac, av, "A:a:d:vz")) != -1)
 		switch (ch) {
 		case 'A':
 			aad_len = atoi(optarg);
 			break;
 		case 'a':
 			algname = optarg;
 			break;
 		case 'd':
 			crid = crlookup(optarg);
 			break;
 		case 'v':
 			verbose = true;
 			break;
 		case 'z':
 			testall = true;
 			break;
 		default:
 			usage();
 		}
 	ac -= optind;
 	av += optind;
 	nsizes = 0;
 	while (ac > 0) {
 		char *cp;
 
 		if (nsizes >= nitems(sizes)) {
 			warnx("Too many sizes, ignoring extras");
 			break;
 		}
 		sizes[nsizes] = strtol(av[0], &cp, 0);
 		if (*cp != '\0')
 			errx(1, "Bad size %s", av[0]);
 		nsizes++;
 		ac--;
 		av++;
 	}
 
 	if (algname == NULL)
 		errx(1, "Algorithm required");
 	if (nsizes == 0) {
 		sizes[0] = 16;
 		nsizes++;
 		if (testall) {
 			while (sizes[nsizes - 1] * 2 < 240 * 1024) {
 				assert(nsizes < nitems(sizes));
 				sizes[nsizes] = sizes[nsizes - 1] * 2;
 				nsizes++;
 			}
 			if (sizes[nsizes - 1] < 240 * 1024) {
 				assert(nsizes < nitems(sizes));
 				sizes[nsizes] = 240 * 1024;
 				nsizes++;
 			}
 		}
 	}
 
 	if (strcasecmp(algname, "hash") == 0)
 		run_hash_tests(sizes, nsizes);
 	else if (strcasecmp(algname, "mac") == 0)
 		run_mac_tests(sizes, nsizes);
 	else if (strcasecmp(algname, "cipher") == 0)
 		run_cipher_tests(sizes, nsizes);
 	else if (strcasecmp(algname, "eta") == 0)
 		run_eta_tests(sizes, nsizes);
 	else if (strcasecmp(algname, "aead") == 0)
 		run_aead_tests(sizes, nsizes);
 	else if (strcasecmp(algname, "all") == 0) {
 		run_hash_tests(sizes, nsizes);
 		run_mac_tests(sizes, nsizes);
 		run_cipher_tests(sizes, nsizes);
 		run_eta_tests(sizes, nsizes);
 		run_aead_tests(sizes, nsizes);
 	} else if (strchr(algname, '+') != NULL) {
 		eta = build_eta_name(algname);
 		run_test_sizes(eta, sizes, nsizes);
 		free_eta(eta);
 	} else {
 		alg = find_alg(algname);
 		if (alg == NULL)
 			errx(1, "Invalid algorithm %s", algname);
 		run_test_sizes(alg, sizes, nsizes);
 	}
 
 	return (0);
 }