Index: projects/release-debugdist/Makefile.inc1
===================================================================
--- projects/release-debugdist/Makefile.inc1	(revision 267343)
+++ projects/release-debugdist/Makefile.inc1	(revision 267344)
@@ -1,1974 +1,1981 @@
 #
 # $FreeBSD$
 #
 # Make command line options:
 #	-DNO_CLEANDIR run ${MAKE} clean, instead of ${MAKE} cleandir
 #	-DNO_CLEAN do not clean at all
 #	-DDB_FROM_SRC use the user/group databases in src/etc instead of
 #	    the system database when installing.
 #	-DNO_SHARE do not go into share subdir
 #	-DKERNFAST define NO_KERNEL{CONFIG,CLEAN,DEPEND,OBJ}
 #	-DNO_KERNELCONFIG do not run config in ${MAKE} buildkernel
 #	-DNO_KERNELCLEAN do not run ${MAKE} clean in ${MAKE} buildkernel
 #	-DNO_KERNELDEPEND do not run ${MAKE} depend in ${MAKE} buildkernel
 #	-DNO_KERNELOBJ do not run ${MAKE} obj in ${MAKE} buildkernel
 #	-DNO_PORTSUPDATE do not update ports in ${MAKE} update
 #	-DNO_ROOT install without using root privilege
 #	-DNO_DOCUPDATE do not update doc in ${MAKE} update
 #	-DWITHOUT_CTF do not run the DTrace CTF conversion tools on built objects
 #	LOCAL_DIRS="list of dirs" to add additional dirs to the SUBDIR list
 #	LOCAL_LIB_DIRS="list of dirs" to add additional dirs to libraries target
 #	LOCAL_MTREE="list of mtree files" to process to allow local directories
 #	    to be created before files are installed
 #	LOCAL_TOOL_DIRS="list of dirs" to add additional dirs to the build-tools
 #	    list
 #	METALOG="path to metadata log" to write permission and ownership
 #	    when NO_ROOT is set.  (default: ${DESTDIR}/METALOG)
 #	TARGET="machine" to crossbuild world for a different machine type
 #	TARGET_ARCH= may be required when a TARGET supports multiple endians
 #	BUILDENV_SHELL= shell to launch for the buildenv target (def:/bin/sh)
 #	WORLD_FLAGS= additional flags to pass to make(1) during buildworld
 #	KERNEL_FLAGS= additional flags to pass to make(1) during buildkernel
 
 #
 # The intended user-driven targets are:
 # buildworld  - rebuild *everything*, including glue to help do upgrades
 # installworld- install everything built by "buildworld"
 # doxygen     - build API documentation of the kernel
 # update      - convenient way to update your source tree (eg: svn/svnup)
 #
 # Standard targets (not defined here) are documented in the makefiles in
 # /usr/share/mk.  These include:
 #		obj depend all install clean cleandepend cleanobj
 
 .if !defined(TARGET) || !defined(TARGET_ARCH)
 .error "Both TARGET and TARGET_ARCH must be defined."
 .endif
 
 .include "share/mk/src.opts.mk"
 .include <bsd.arch.inc.mk>
 .include <bsd.compiler.mk>
 
 # We must do share/info early so that installation of info `dir'
 # entries works correctly.  Do it first since it is less likely to
 # grow dependencies on include and lib than vice versa.
 #
 # We must do lib/ and libexec/ before bin/, because if installworld
 # installs a new /bin/sh, the 'make' command will *immediately*
 # use that new version.  And the new (dynamically-linked) /bin/sh
 # will expect to find appropriate libraries in /lib and /libexec.
 #
 SRCDIR?=	${.CURDIR}
 .if defined(SUBDIR_OVERRIDE)
 SUBDIR=	${SUBDIR_OVERRIDE}
 .else
 SUBDIR=	share/info lib libexec
 SUBDIR+=bin
 .if ${MK_GAMES} != "no"
 SUBDIR+=games
 .endif
 .if ${MK_CDDL} != "no"
 SUBDIR+=cddl
 .endif
 SUBDIR+=gnu include
 .if ${MK_KERBEROS} != "no"
 SUBDIR+=kerberos5
 .endif
 .if ${MK_RESCUE} != "no"
 SUBDIR+=rescue
 .endif
 SUBDIR+=sbin
 .if ${MK_CRYPT} != "no"
 SUBDIR+=secure
 .endif
 .if !defined(NO_SHARE)
 SUBDIR+=share
 .endif
 SUBDIR+=sys usr.bin usr.sbin
 .if ${MK_TESTS} != "no"
 SUBDIR+=	tests
 .endif
 .if ${MK_OFED} != "no"
 SUBDIR+=contrib/ofed
 .endif
 #
 # We must do etc/ last for install/distribute to work.
 #
 SUBDIR+=etc
 
 # These are last, since it is nice to at least get the base system
 # rebuilt before you do them.
 .for _DIR in ${LOCAL_LIB_DIRS} ${LOCAL_DIRS}
 .if exists(${.CURDIR}/${_DIR}/Makefile)
 SUBDIR+= ${_DIR}
 .endif
 .endfor
 .endif
 
 .if defined(NOCLEAN)
 NO_CLEAN=	${NOCLEAN}
 .endif
 .if defined(NO_CLEANDIR)
 CLEANDIR=	clean cleandepend
 .else
 CLEANDIR=	cleandir
 .endif
 
 LOCAL_TOOL_DIRS?=
 
 BUILDENV_SHELL?=/bin/sh
 
 SVN?=		/usr/local/bin/svn
 SVNFLAGS?=	-r HEAD
 
 MAKEOBJDIRPREFIX?=	/usr/obj
 .if !defined(OSRELDATE)
 .if exists(/usr/include/osreldate.h)
 OSRELDATE!=	awk '/^\#define[[:space:]]*__FreeBSD_version/ { print $$3 }' \
 		/usr/include/osreldate.h
 .else
 OSRELDATE=	0
 .endif
 .endif
 
 .if !defined(VERSION)
 REVISION!=	${MAKE} -C ${SRCDIR}/release -V REVISION
 BRANCH!=	${MAKE} -C ${SRCDIR}/release -V BRANCH
 SRCRELDATE!=	awk '/^\#define[[:space:]]*__FreeBSD_version/ { print $$3 }' \
 		${SRCDIR}/sys/sys/param.h
 VERSION=	FreeBSD ${REVISION}-${BRANCH:C/-p[0-9]+$//} ${TARGET_ARCH} ${SRCRELDATE}
 .endif
 
 KNOWN_ARCHES?=	amd64 arm armeb/arm armv6/arm armv6hf/arm i386 i386/pc98 ia64 mips mipsel/mips mips64el/mips mips64/mips mipsn32el/mips mipsn32/mips powerpc powerpc64/powerpc sparc64
 .if ${TARGET} == ${TARGET_ARCH}
 _t=		${TARGET}
 .else
 _t=		${TARGET_ARCH}/${TARGET}
 .endif
 .for _t in ${_t}
 .if empty(KNOWN_ARCHES:M${_t})
 .error Unknown target ${TARGET_ARCH}:${TARGET}.
 .endif
 .endfor
 
 .if ${TARGET} == ${MACHINE}
 TARGET_CPUTYPE?=${CPUTYPE}
 .else
 TARGET_CPUTYPE?=
 .endif
 
 .if !empty(TARGET_CPUTYPE)
 _TARGET_CPUTYPE=${TARGET_CPUTYPE}
 .else
 _TARGET_CPUTYPE=dummy
 .endif
 _CPUTYPE!=	MAKEFLAGS= CPUTYPE=${_TARGET_CPUTYPE} ${MAKE} \
 		-f /dev/null -m ${.CURDIR}/share/mk -V CPUTYPE
 .if ${_CPUTYPE} != ${_TARGET_CPUTYPE}
 .error CPUTYPE global should be set with ?=.
 .endif
 .if make(buildworld)
 BUILD_ARCH!=	uname -p
 .if ${MACHINE_ARCH} != ${BUILD_ARCH}
 .error To cross-build, set TARGET_ARCH.
 .endif
 .endif
 .if ${MACHINE} == ${TARGET} && ${MACHINE_ARCH} == ${TARGET_ARCH} && !defined(CROSS_BUILD_TESTING)
 OBJTREE=	${MAKEOBJDIRPREFIX}
 .else
 OBJTREE=	${MAKEOBJDIRPREFIX}/${TARGET}.${TARGET_ARCH}
 .endif
 WORLDTMP=	${OBJTREE}${.CURDIR}/tmp
 # /usr/games added for fortune which depend on strfile
 BPATH=		${WORLDTMP}/legacy/usr/sbin:${WORLDTMP}/legacy/usr/bin:${WORLDTMP}/legacy/usr/games:${WORLDTMP}/legacy/bin
 XPATH=		${WORLDTMP}/usr/sbin:${WORLDTMP}/usr/bin:${WORLDTMP}/usr/games
 STRICTTMPPATH=	${BPATH}:${XPATH}
 TMPPATH=	${STRICTTMPPATH}:${PATH}
 
 #
 # Avoid running mktemp(1) unless actually needed.
 # It may not be functional, e.g., due to new ABI
 # when in the middle of installing over this system.
 #
 .if make(distributeworld) || make(installworld)
 INSTALLTMP!=	/usr/bin/mktemp -d -u -t install
 .endif
 
 #
 # Building a world goes through the following stages
 #
 # 1. legacy stage [BMAKE]
 #	This stage is responsible for creating compatibility
 #	shims that are needed by the bootstrap-tools,
 #	build-tools and cross-tools stages.
 # 1. bootstrap-tools stage [BMAKE]
 #	This stage is responsible for creating programs that
 #	are needed for backward compatibility reasons. They
 #	are not built as cross-tools.
 # 2. build-tools stage [TMAKE]
 #	This stage is responsible for creating the object
 #	tree and building any tools that are needed during
 #	the build process.
 # 3. cross-tools stage [XMAKE]
 #	This stage is responsible for creating any tools that
 #	are needed for cross-builds. A cross-compiler is one
 #	of them.
 # 4. world stage [WMAKE]
 #	This stage actually builds the world.
 # 5. install stage (optional) [IMAKE]
 #	This stage installs a previously built world.
 #
 
 BOOTSTRAPPING?=	0
 
 # Common environment for world related stages
 CROSSENV=	MAKEOBJDIRPREFIX=${OBJTREE} \
 		MACHINE_ARCH=${TARGET_ARCH} \
 		MACHINE=${TARGET} \
 		CPUTYPE=${TARGET_CPUTYPE}
 .if ${MK_GROFF} != "no"
 CROSSENV+=	GROFF_BIN_PATH=${WORLDTMP}/legacy/usr/bin \
 		GROFF_FONT_PATH=${WORLDTMP}/legacy/usr/share/groff_font \
 		GROFF_TMAC_PATH=${WORLDTMP}/legacy/usr/share/tmac
 .endif
 .if defined(TARGET_CFLAGS)
 CROSSENV+=	${TARGET_CFLAGS}
 .endif
 
 # bootstrap-tools stage
 BMAKEENV=	INSTALL="sh ${.CURDIR}/tools/install.sh" \
 		PATH=${BPATH}:${PATH} \
 		WORLDTMP=${WORLDTMP} \
 		VERSION="${VERSION}" \
 		MAKEFLAGS="-m ${.CURDIR}/tools/build/mk ${.MAKEFLAGS}"
 BMAKE=		MAKEOBJDIRPREFIX=${WORLDTMP} \
 		${BMAKEENV} ${MAKE} ${WORLD_FLAGS} -f Makefile.inc1 \
 		DESTDIR= \
 		BOOTSTRAPPING=${OSRELDATE} \
 		SSP_CFLAGS= MK_PIE=no \
 		MK_HTML=no MK_INFO=no NO_LINT=yes MK_MAN=no \
 		-DNO_PIC MK_PROFILE=no -DNO_SHARED \
 		-DNO_CPU_CFLAGS MK_WARNS=no MK_CTF=no \
 		MK_CLANG_FULL=no MK_LLDB=no MK_TESTS=no
 
 # build-tools stage
 TMAKE=		MAKEOBJDIRPREFIX=${OBJTREE} \
 		${BMAKEENV} ${MAKE} ${WORLD_FLAGS} -f Makefile.inc1 \
 		TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \
 		DESTDIR= \
 		BOOTSTRAPPING=${OSRELDATE} \
 		SSP_CFLAGS= MK_PIE=no \
 		-DNO_LINT \
 		-DNO_CPU_CFLAGS MK_WARNS=no MK_CTF=no MK_CLANG_FULL=no MK_LLDB=no MK_TESTS=no
 
 # cross-tools stage
 XMAKE=		TOOLS_PREFIX=${WORLDTMP} ${BMAKE} \
 		TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \
 		MK_GDB=no MK_TESTS=no
 
 # kernel-tools stage
 KTMAKEENV=	INSTALL="sh ${.CURDIR}/tools/install.sh" \
 		PATH=${BPATH}:${PATH} \
 		WORLDTMP=${WORLDTMP} \
 		VERSION="${VERSION}"
 KTMAKE=		TOOLS_PREFIX=${WORLDTMP} MAKEOBJDIRPREFIX=${WORLDTMP} \
 		${KTMAKEENV} ${MAKE} ${WORLD_FLAGS} -f Makefile.inc1 \
 		DESTDIR= \
 		BOOTSTRAPPING=${OSRELDATE} \
 		SSP_CFLAGS= MK_PIE=no \
 		MK_HTML=no MK_INFO=no -DNO_LINT MK_MAN=no \
 		-DNO_PIC MK_PROFILE=no -DNO_SHARED \
 		-DNO_CPU_CFLAGS MK_WARNS=no MK_CTF=no
 
 # world stage
 WMAKEENV=	${CROSSENV} \
 		_SHLIBDIRPREFIX=${WORLDTMP} \
 		_LDSCRIPTROOT= \
 		VERSION="${VERSION}" \
 		INSTALL="sh ${.CURDIR}/tools/install.sh" \
 		PATH=${TMPPATH}
 
 # make hierarchy
 HMAKE=		PATH=${TMPPATH} ${MAKE} LOCAL_MTREE=${LOCAL_MTREE:Q}
 .if defined(NO_ROOT)
 HMAKE+=		PATH=${TMPPATH} METALOG=${METALOG} -DNO_ROOT
 .endif
 
 .if ${MK_CDDL} == "no"
 WMAKEENV+=	MK_CTF=no
 .endif
 
 .if defined(CROSS_TOOLCHAIN_PREFIX)
 CROSS_COMPILER_PREFIX?=${CROSS_TOOLCHAIN_PREFIX}
 CROSS_BINUTILS_PREFIX?=${CROSS_TOOLCHAIN_PREFIX}
 .endif
 XCOMPILERS=	CC CXX CPP
 .for COMPILER in ${XCOMPILERS}
 .if defined(CROSS_COMPILER_PREFIX)
 X${COMPILER}?=	${CROSS_COMPILER_PREFIX}${${COMPILER}}
 .else
 X${COMPILER}?=	${${COMPILER}}
 .endif
 .endfor
 XBINUTILS=	AS AR LD NM OBJDUMP RANLIB STRINGS
 .for BINUTIL in ${XBINUTILS}
 .if defined(CROSS_BINUTILS_PREFIX)
 X${BINUTIL}?=	${CROSS_BINUTILS_PREFIX}${${BINUTIL}}
 .else
 X${BINUTIL}?=	${${BINUTIL}}
 .endif
 .endfor
 WMAKEENV+=	CC="${XCC} ${XFLAGS}" CXX="${XCXX} ${XFLAGS}" \
 		CPP="${XCPP} ${XFLAGS}" \
 		AS="${XAS}" AR="${XAR}" LD="${XLD}" NM=${XNM} \
 		OBJDUMP=${XOBJDUMP} RANLIB=${XRANLIB} STRINGS=${XSTRINGS}
 
 .if ${XCC:M/*}
 XFLAGS=		--sysroot=${WORLDTMP}
 .if defined(CROSS_BINUTILS_PREFIX)
 # In the case of xdev-build tools, CROSS_BINUTILS_PREFIX won't be a
 # directory, but the compiler will look in the right place for it's
 # tools so we don't need to tell it where to look.
 .if exists(${CROSS_BINUTILS_PREFIX})
 XFLAGS+=	-B${CROSS_BINUTILS_PREFIX}
 .endif
 .else
 XFLAGS+=	-B${WORLDTMP}/usr/bin
 .endif
 .if ${TARGET} == "arm" && ${MK_ARM_EABI} != "no"
 .if ${TARGET_ARCH:M*eb*} == ""
 TARGET_ABI=	gnueabi
 .elif ${TARGET_ARCH} == "armv6hf"
 TARGET_ABI=	gnueabihf
 .endif
 .endif
 TARGET_ABI?=	unknown
 TARGET_TRIPLE?=	${TARGET_ARCH:C/amd64/x86_64/}-${TARGET_ABI}-freebsd11.0
 XFLAGS+=	-target ${TARGET_TRIPLE}
 .endif
 
 WMAKE=		${WMAKEENV} ${MAKE} ${WORLD_FLAGS} -f Makefile.inc1 DESTDIR=${WORLDTMP}
 
 .if ${TARGET_ARCH} == "amd64" || ${TARGET_ARCH} == "powerpc64"
 # 32 bit world
 LIB32TMP=	${OBJTREE}${.CURDIR}/lib32
 
 .if ${TARGET_ARCH} == "amd64"
 .if empty(TARGET_CPUTYPE)
 LIB32CPUFLAGS=	-march=i686 -mmmx -msse -msse2
 .else
 LIB32CPUFLAGS=	-march=${TARGET_CPUTYPE}
 .endif
 LIB32WMAKEENV=	MACHINE=i386 MACHINE_ARCH=i386 \
 		MACHINE_CPU="i686 mmx sse sse2"
 LIB32WMAKEFLAGS=	\
 		AS="${AS} --32" \
 		LD="${LD} -m elf_i386_fbsd -Y P,${LIB32TMP}/usr/lib32"
 
 .elif ${TARGET_ARCH} == "powerpc64"
 .if empty(TARGET_CPUTYPE)
 LIB32CPUFLAGS=	-mcpu=powerpc
 .else
 LIB32CPUFLAGS=	-mcpu=${TARGET_CPUTYPE}
 .endif
 LIB32WMAKEENV=	MACHINE=powerpc MACHINE_ARCH=powerpc
 LIB32WMAKEFLAGS=	\
 		LD="${LD} -m elf32ppc_fbsd"
 .endif
 
 
 LIB32FLAGS=	-m32 ${LIB32CPUFLAGS} -DCOMPAT_32BIT \
 		-isystem ${LIB32TMP}/usr/include/ \
 		-L${LIB32TMP}/usr/lib32 \
 		-B${LIB32TMP}/usr/lib32
 .if ${XCC:M/*}
 LIB32FLAGS+=		--sysroot=${WORLDTMP}
 .endif
 
 # Yes, the flags are redundant.
 LIB32WMAKEENV+=	MAKEOBJDIRPREFIX=${OBJTREE}/lib32 \
 		_SHLIBDIRPREFIX=${LIB32TMP} \
 		_LDSCRIPTROOT=${LIB32TMP} \
 		VERSION="${VERSION}" \
 		INSTALL="sh ${.CURDIR}/tools/install.sh" \
 		PATH=${TMPPATH} \
 		LIBDIR=/usr/lib32 \
 		SHLIBDIR=/usr/lib32 \
 		LIBPRIVATEDIR=/usr/lib32/private
 LIB32WMAKEFLAGS+= CC="${XCC} ${LIB32FLAGS}" \
 		CXX="${XCXX} ${LIB32FLAGS}" \
 		DESTDIR=${LIB32TMP} \
 		-DCOMPAT_32BIT \
 		-DLIBRARIES_ONLY \
 		-DNO_CPU_CFLAGS \
 		MK_CTF=no \
 		-DNO_LINT \
 		MK_TESTS=no
 
 LIB32WMAKE=	${LIB32WMAKEENV} ${MAKE} ${LIB32WMAKEFLAGS} \
 		MK_MAN=no MK_INFO=no MK_HTML=no
 LIB32IMAKE=	${LIB32WMAKE:NINSTALL=*:NDESTDIR=*:N_LDSCRIPTROOT=*} \
 		MK_TOOLCHAIN=no ${IMAKE_INSTALL}
 .endif
 
 IMAKEENV=	${CROSSENV:N_LDSCRIPTROOT=*}
 IMAKE=		${IMAKEENV} ${MAKE} -f Makefile.inc1 \
 		${IMAKE_INSTALL} ${IMAKE_MTREE}
 .if empty(.MAKEFLAGS:M-n)
 IMAKEENV+=	PATH=${STRICTTMPPATH}:${INSTALLTMP} \
 		LD_LIBRARY_PATH=${INSTALLTMP} \
 		PATH_LOCALE=${INSTALLTMP}/locale
 IMAKE+=		__MAKE_SHELL=${INSTALLTMP}/sh
 .else
 IMAKEENV+=	PATH=${TMPPATH}:${INSTALLTMP}
 .endif
 .if defined(DB_FROM_SRC)
 INSTALLFLAGS+=	-N ${.CURDIR}/etc
 MTREEFLAGS+=	-N ${.CURDIR}/etc
 .endif
 _INSTALL_DDIR=	${DESTDIR}/${DISTDIR}
 INSTALL_DDIR=	${_INSTALL_DDIR:S://:/:g:C:/$::}
 .if defined(NO_ROOT)
 METALOG?=	${DESTDIR}/${DISTDIR}/METALOG
 IMAKE+=		-DNO_ROOT METALOG=${METALOG}
 INSTALLFLAGS+=	-U -M ${METALOG} -D ${INSTALL_DDIR}
 MTREEFLAGS+=	-W
 .endif
 .if defined(DB_FROM_SRC) || defined(NO_ROOT)
 IMAKE_INSTALL=	INSTALL="install ${INSTALLFLAGS}"
 IMAKE_MTREE=	MTREE_CMD="mtree ${MTREEFLAGS}"
 .endif
 
 # kernel stage
 KMAKEENV=	${WMAKEENV}
 KMAKE=		${KMAKEENV} ${MAKE} ${.MAKEFLAGS} ${KERNEL_FLAGS} KERNEL=${INSTKERNNAME}
 
 #
 # buildworld
 #
 # Attempt to rebuild the entire system, with reasonable chance of
 # success, regardless of how old your existing system is.
 #
 _worldtmp:
 .if ${.CURDIR:C/[^,]//g} != ""
 #	The m4 build of sendmail files doesn't like it if ',' is used
 #	anywhere in the path of it's files.
 	@echo
 	@echo "*** Error: path to source tree contains a comma ','"
 	@echo
 	false
 .endif
 	@echo
 	@echo "--------------------------------------------------------------"
 	@echo ">>> Rebuilding the temporary build tree"
 	@echo "--------------------------------------------------------------"
 .if !defined(NO_CLEAN)
 	rm -rf ${WORLDTMP}
 .if defined(LIB32TMP)
 	rm -rf ${LIB32TMP}
 .endif
 .else
 	rm -rf ${WORLDTMP}/legacy/usr/include
 #	XXX - These three can depend on any header file.
 	rm -f ${OBJTREE}${.CURDIR}/usr.bin/kdump/ioctl.c
 	rm -f ${OBJTREE}${.CURDIR}/usr.bin/kdump/kdump_subr.c
 	rm -f ${OBJTREE}${.CURDIR}/usr.bin/truss/ioctl.c
 .endif
 .for _dir in \
     lib usr legacy/bin legacy/usr
 	mkdir -p ${WORLDTMP}/${_dir}
 .endfor
 	mtree -deU -f ${.CURDIR}/etc/mtree/BSD.usr.dist \
 	    -p ${WORLDTMP}/legacy/usr >/dev/null
 .if ${MK_GROFF} != "no"
 	mtree -deU -f ${.CURDIR}/etc/mtree/BSD.groff.dist \
 	    -p ${WORLDTMP}/legacy/usr >/dev/null
 .endif
 	mtree -deU -f ${.CURDIR}/etc/mtree/BSD.usr.dist \
 	    -p ${WORLDTMP}/usr >/dev/null
 	mtree -deU -f ${.CURDIR}/etc/mtree/BSD.include.dist \
 	    -p ${WORLDTMP}/usr/include >/dev/null
 	ln -sf ${.CURDIR}/sys ${WORLDTMP}
 .if ${MK_DEBUG_FILES} != "no"
 	# We could instead disable debug files for these build stages
 	mtree -deU -f ${.CURDIR}/etc/mtree/BSD.debug.dist \
 	    -p ${WORLDTMP}/legacy/usr/lib >/dev/null
 	mtree -deU -f ${.CURDIR}/etc/mtree/BSD.debug.dist \
 	    -p ${WORLDTMP}/usr/lib >/dev/null
 .endif
 .if ${MK_TESTS} != "no"
 	mtree -deU -f ${.CURDIR}/etc/mtree/BSD.tests.dist \
 	    -p ${WORLDTMP}/usr >/dev/null
 .endif
 .for _mtree in ${LOCAL_MTREE}
 	mtree -deU -f ${.CURDIR}/${_mtree} -p ${WORLDTMP} > /dev/null
 .endfor
 _legacy:
 	@echo
 	@echo "--------------------------------------------------------------"
 	@echo ">>> stage 1.1: legacy release compatibility shims"
 	@echo "--------------------------------------------------------------"
 	${_+_}cd ${.CURDIR}; ${BMAKE} legacy
 _bootstrap-tools:
 	@echo
 	@echo "--------------------------------------------------------------"
 	@echo ">>> stage 1.2: bootstrap tools"
 	@echo "--------------------------------------------------------------"
 	${_+_}cd ${.CURDIR}; ${BMAKE} bootstrap-tools
 _cleanobj:
 .if !defined(NO_CLEAN)
 	@echo
 	@echo "--------------------------------------------------------------"
 	@echo ">>> stage 2.1: cleaning up the object tree"
 	@echo "--------------------------------------------------------------"
 	${_+_}cd ${.CURDIR}; ${WMAKE} ${CLEANDIR:S/^/par-/}
 .if defined(LIB32TMP)
 	${_+_}cd ${.CURDIR}; ${LIB32WMAKE} -f Makefile.inc1 ${CLEANDIR:S/^/par-/}
 .endif
 .endif
 _obj:
 	@echo
 	@echo "--------------------------------------------------------------"
 	@echo ">>> stage 2.2: rebuilding the object tree"
 	@echo "--------------------------------------------------------------"
 	${_+_}cd ${.CURDIR}; ${WMAKE} par-obj
 _build-tools:
 	@echo
 	@echo "--------------------------------------------------------------"
 	@echo ">>> stage 2.3: build tools"
 	@echo "--------------------------------------------------------------"
 	${_+_}cd ${.CURDIR}; ${TMAKE} build-tools
 _cross-tools:
 	@echo
 	@echo "--------------------------------------------------------------"
 	@echo ">>> stage 3: cross tools"
 	@echo "--------------------------------------------------------------"
 	${_+_}cd ${.CURDIR}; ${XMAKE} cross-tools
 	${_+_}cd ${.CURDIR}; ${XMAKE} kernel-tools
 _includes:
 	@echo
 	@echo "--------------------------------------------------------------"
 	@echo ">>> stage 4.1: building includes"
 	@echo "--------------------------------------------------------------"
 	${_+_}cd ${.CURDIR}; ${WMAKE} SHARED=symlinks par-includes
 _libraries:
 	@echo
 	@echo "--------------------------------------------------------------"
 	@echo ">>> stage 4.2: building libraries"
 	@echo "--------------------------------------------------------------"
 	${_+_}cd ${.CURDIR}; \
 	    ${WMAKE} -DNO_FSCHG MK_HTML=no MK_INFO=no -DNO_LINT MK_MAN=no \
 	    MK_PROFILE=no  MK_TESTS=no libraries
 _depend:
 	@echo
 	@echo "--------------------------------------------------------------"
 	@echo ">>> stage 4.3: make dependencies"
 	@echo "--------------------------------------------------------------"
 	${_+_}cd ${.CURDIR}; ${WMAKE} par-depend
 everything:
 	@echo
 	@echo "--------------------------------------------------------------"
 	@echo ">>> stage 4.4: building everything"
 	@echo "--------------------------------------------------------------"
 	${_+_}cd ${.CURDIR}; ${WMAKE} par-all
 .if defined(LIB32TMP)
 build32:
 	@echo
 	@echo "--------------------------------------------------------------"
 	@echo ">>> stage 5.1: building 32 bit shim libraries"
 	@echo "--------------------------------------------------------------"
 	mkdir -p ${LIB32TMP}/usr/include
 	mtree -deU -f ${.CURDIR}/etc/mtree/BSD.usr.dist \
 	    -p ${LIB32TMP}/usr >/dev/null
 	mtree -deU -f ${.CURDIR}/etc/mtree/BSD.include.dist \
 	    -p ${LIB32TMP}/usr/include >/dev/null
 .if ${MK_DEBUG_FILES} != "no"
 	mtree -deU -f ${.CURDIR}/etc/mtree/BSD.debug.dist \
 	    -p ${LIB32TMP}/usr/lib >/dev/null
 .endif
 	mkdir -p ${WORLDTMP}
 	ln -sf ${.CURDIR}/sys ${WORLDTMP}
 .for _t in obj includes
 	cd ${.CURDIR}/include; ${LIB32WMAKE} DIRPRFX=include/ ${_t}
 	cd ${.CURDIR}/lib; ${LIB32WMAKE} DIRPRFX=lib/ ${_t}
 .if ${MK_CDDL} != "no"
 	cd ${.CURDIR}/cddl/lib; ${LIB32WMAKE} DIRPRFX=cddl/lib/ ${_t}
 .endif
 	cd ${.CURDIR}/gnu/lib; ${LIB32WMAKE} DIRPRFX=gnu/lib/ ${_t}
 .if ${MK_CRYPT} != "no"
 	cd ${.CURDIR}/secure/lib; ${LIB32WMAKE} DIRPRFX=secure/lib/ ${_t}
 .endif
 .if ${MK_KERBEROS} != "no"
 	cd ${.CURDIR}/kerberos5/lib; ${LIB32WMAKE} DIRPRFX=kerberos5/lib ${_t}
 .endif
 .endfor
 .for _dir in usr.bin/lex/lib
 	cd ${.CURDIR}/${_dir}; ${LIB32WMAKE} DIRPRFX=${_dir}/ obj
 .endfor
 .for _dir in lib/ncurses/ncurses lib/ncurses/ncursesw lib/libmagic
 	cd ${.CURDIR}/${_dir}; \
 	    WORLDTMP=${WORLDTMP} \
 	    MAKEFLAGS="-m ${.CURDIR}/tools/build/mk ${.MAKEFLAGS}" \
 	    MAKEOBJDIRPREFIX=${OBJTREE}/lib32 ${MAKE} SSP_CFLAGS= DESTDIR= \
 	    DIRPRFX=${_dir}/ -DNO_LINT -DNO_CPU_CFLAGS MK_WARNS=no MK_CTF=no \
 	    build-tools
 .endfor
 	cd ${.CURDIR}; \
 	    ${LIB32WMAKE} -f Makefile.inc1 libraries
 .for _t in obj depend all
 	cd ${.CURDIR}/libexec/rtld-elf; PROG=ld-elf32.so.1 ${LIB32WMAKE} \
 	    DIRPRFX=libexec/rtld-elf/ ${_t}
 	cd ${.CURDIR}/usr.bin/ldd; PROG=ldd32 ${LIB32WMAKE} \
 	    DIRPRFX=usr.bin/ldd ${_t}
 .endfor
 
 distribute32 install32:
 	cd ${.CURDIR}/lib; ${LIB32IMAKE} ${.TARGET:S/32$//}
 .if ${MK_CDDL} != "no"
 	cd ${.CURDIR}/cddl/lib; ${LIB32IMAKE} ${.TARGET:S/32$//}
 .endif
 	cd ${.CURDIR}/gnu/lib; ${LIB32IMAKE} ${.TARGET:S/32$//}
 .if ${MK_CRYPT} != "no"
 	cd ${.CURDIR}/secure/lib; ${LIB32IMAKE} ${.TARGET:S/32$//}
 .endif
 .if ${MK_KERBEROS} != "no"
 	cd ${.CURDIR}/kerberos5/lib; ${LIB32IMAKE} ${.TARGET:S/32$//}
 .endif
 	cd ${.CURDIR}/libexec/rtld-elf; \
 	    PROG=ld-elf32.so.1 ${LIB32IMAKE} ${.TARGET:S/32$//}
 	cd ${.CURDIR}/usr.bin/ldd; PROG=ldd32 ${LIB32IMAKE} ${.TARGET:S/32$//}
 .endif
 
 WMAKE_TGTS=
 .if !defined(SUBDIR_OVERRIDE)
 WMAKE_TGTS+=	_worldtmp _legacy _bootstrap-tools
 .endif
 WMAKE_TGTS+=	_cleanobj _obj _build-tools
 .if !defined(SUBDIR_OVERRIDE)
 WMAKE_TGTS+=	_cross-tools
 .endif
 WMAKE_TGTS+=	_includes _libraries _depend everything
 .if defined(LIB32TMP) && ${MK_LIB32} != "no"
 WMAKE_TGTS+=	build32
 .endif
 
 buildworld: buildworld_prologue ${WMAKE_TGTS} buildworld_epilogue
 .ORDER: buildworld_prologue ${WMAKE_TGTS} buildworld_epilogue
 
 buildworld_prologue:
 	@echo "--------------------------------------------------------------"
 	@echo ">>> World build started on `LC_ALL=C date`"
 	@echo "--------------------------------------------------------------"
 
 buildworld_epilogue:
 	@echo
 	@echo "--------------------------------------------------------------"
 	@echo ">>> World build completed on `LC_ALL=C date`"
 	@echo "--------------------------------------------------------------"
 
 #
 # We need to have this as a target because the indirection between Makefile
 # and Makefile.inc1 causes the correct PATH to be used, rather than a
 # modification of the current environment's PATH.  In addition, we need
 # to quote multiword values.
 #
 buildenvvars:
 	@echo ${WMAKEENV:Q}
 
 buildenv:
 	@echo Entering world for ${TARGET_ARCH}:${TARGET}
 	@cd ${.CURDIR} && env ${WMAKEENV} ${BUILDENV_SHELL} || true
 
 TOOLCHAIN_TGTS=	${WMAKE_TGTS:N_depend:Neverything:Nbuild32}
 toolchain: ${TOOLCHAIN_TGTS}
 kernel-toolchain: ${TOOLCHAIN_TGTS:N_includes:N_libraries}
 
 #
 # installcheck
 #
 # Checks to be sure system is ready for installworld/installkernel.
 #
 installcheck: _installcheck_world _installcheck_kernel
 _installcheck_world:
 _installcheck_kernel:
 
 #
 # Require DESTDIR to be set if installing for a different architecture or
 # using the user/group database in the source tree.
 #
 .if ${TARGET_ARCH} != ${MACHINE_ARCH} || ${TARGET} != ${MACHINE} || \
     defined(DB_FROM_SRC)
 .if !make(distributeworld)
 _installcheck_world: __installcheck_DESTDIR
 _installcheck_kernel: __installcheck_DESTDIR
 __installcheck_DESTDIR:
 .if !defined(DESTDIR) || empty(DESTDIR)
 	@echo "ERROR: Please set DESTDIR!"; \
 	false
 .endif
 .endif
 .endif
 
 .if !defined(DB_FROM_SRC)
 #
 # Check for missing UIDs/GIDs.
 #
 CHECK_UIDS=	auditdistd
 CHECK_GIDS=	audit
 .if ${MK_SENDMAIL} != "no"
 CHECK_UIDS+=	smmsp
 CHECK_GIDS+=	smmsp
 .endif
 .if ${MK_PF} != "no"
 CHECK_UIDS+=	proxy
 CHECK_GIDS+=	proxy authpf
 .endif
 .if ${MK_UNBOUND} != "no"
 CHECK_UIDS+=	unbound
 CHECK_GIDS+=	unbound
 .endif
 _installcheck_world: __installcheck_UGID
 __installcheck_UGID:
 .for uid in ${CHECK_UIDS}
 	@if ! `id -u ${uid} >/dev/null 2>&1`; then \
 		echo "ERROR: Required ${uid} user is missing, see /usr/src/UPDATING."; \
 		false; \
 	fi
 .endfor
 .for gid in ${CHECK_GIDS}
 	@if ! `find / -prune -group ${gid} >/dev/null 2>&1`; then \
 		echo "ERROR: Required ${gid} group is missing, see /usr/src/UPDATING."; \
 		false; \
 	fi
 .endfor
 .endif
 
 #
 # Required install tools to be saved in a scratch dir for safety.
 #
 .if ${MK_INFO} != "no"
 _install-info=	install-info
 .endif
 .if ${MK_ZONEINFO} != "no"
 _zoneinfo=	zic tzsetup
 .endif
 
 ITOOLS=	[ awk cap_mkdb cat chflags chmod chown \
 	date echo egrep find grep id install ${_install-info} \
 	ln lockf make mkdir mtree mv pwd_mkdb \
 	rm sed services_mkdb sh sysctl test true uname wc ${_zoneinfo}
 
 #
 # distributeworld
 #
 # Distributes everything compiled by a `buildworld'.
 #
 # installworld
 #
 # Installs everything compiled by a 'buildworld'.
 #
 
 # Non-base distributions produced by the base system
 EXTRA_DISTRIBUTIONS=	doc
 .if ${MK_GAMES} != "no"
 EXTRA_DISTRIBUTIONS+=	games
 .endif
 .if defined(LIB32TMP) && ${MK_LIB32} != "no"
 EXTRA_DISTRIBUTIONS+=	lib32
 .endif
+.if ${MK_TESTS} != "no"
+EXTRA_DISTRIBUTIONS+=	tests
+.endif
 
 DEBUG_DISTRIBUTIONS=
 .if ${MK_DEBUG_FILES} != "no"
 . for dist in base ${EXTRA_DISTRIBUTIONS:S,doc,,}
 DEBUG_DISTRIBUTIONS+=	${dist}
 . endfor
 .endif
 
 MTREE_MAGIC?=	mtree 2.0
 
 distributeworld installworld: _installcheck_world
 	mkdir -p ${INSTALLTMP}
 	progs=$$(for prog in ${ITOOLS}; do \
 		if progpath=`which $$prog`; then \
 			echo $$progpath; \
 		else \
 			echo "Required tool $$prog not found in PATH." >&2; \
 			exit 1; \
 		fi; \
 	    done); \
 	libs=$$(ldd -f "%o %p\n" -f "%o %p\n" $$progs 2>/dev/null | sort -u | \
 	    while read line; do \
 		set -- $$line; \
 		if [ "$$2 $$3" != "not found" ]; then \
 			echo $$2; \
 		else \
 			echo "Required library $$1 not found." >&2; \
 			exit 1; \
 		fi; \
 	    done); \
 	cp $$libs $$progs ${INSTALLTMP}
 	cp -R $${PATH_LOCALE:-"/usr/share/locale"} ${INSTALLTMP}/locale
 .if defined(NO_ROOT)
 	echo "#${MTREE_MAGIC}" > ${METALOG}
 .endif
 .if make(distributeworld)
 .for dist in ${EXTRA_DISTRIBUTIONS}
 	-mkdir ${DESTDIR}/${DISTDIR}/${dist}
 	mtree -deU -f ${.CURDIR}/etc/mtree/BSD.root.dist \
 	    -p ${DESTDIR}/${DISTDIR}/${dist} >/dev/null
 	mtree -deU -f ${.CURDIR}/etc/mtree/BSD.usr.dist \
 	    -p ${DESTDIR}/${DISTDIR}/${dist}/usr >/dev/null
 	mtree -deU -f ${.CURDIR}/etc/mtree/BSD.include.dist \
 	    -p ${DESTDIR}/${DISTDIR}/${dist}/usr/include >/dev/null
 .if ${MK_DEBUG_FILES} != "no"
 	mtree -deU -f ${.CURDIR}/etc/mtree/BSD.debug.dist \
 	    -p ${DESTDIR}/${DISTDIR}/${dist}/usr/lib >/dev/null
+.endif
+.if ${MK_TESTS} != "no" && ${dist} == "tests"
+	mtree -deU -f ${.CURDIR}/etc/mtree/BSD.tests.dist \
+	    -p ${DESTDIR}/${DISTDIR}/${dist}/usr >/dev/null
 .endif
 .if defined(NO_ROOT)
 	${IMAKEENV} mtree -C -f ${.CURDIR}/etc/mtree/BSD.root.dist | \
 	    sed -e 's#^\./#./${dist}/#' >> ${METALOG}
 	${IMAKEENV} mtree -C -f ${.CURDIR}/etc/mtree/BSD.usr.dist | \
 	    sed -e 's#^\./#./${dist}/usr/#' >> ${METALOG}
 	${IMAKEENV} mtree -C -f ${.CURDIR}/etc/mtree/BSD.include.dist | \
 	    sed -e 's#^\./#./${dist}/usr/include/#' >> ${METALOG}
 .endif
 .endfor
 	-mkdir ${DESTDIR}/${DISTDIR}/base
 	cd ${.CURDIR}/etc; ${CROSSENV} PATH=${TMPPATH} ${MAKE} \
 	    METALOG=${METALOG} ${IMAKE_INSTALL} ${IMAKE_MTREE} \
 	    DISTBASE=/base DESTDIR=${DESTDIR}/${DISTDIR}/base \
 	    LOCAL_MTREE=${LOCAL_MTREE:Q} distrib-dirs
 .endif
 	${_+_}cd ${.CURDIR}; ${IMAKE} re${.TARGET:S/world$//}; \
 	    ${IMAKEENV} rm -rf ${INSTALLTMP}
 .if make(distributeworld)
 .for dist in ${EXTRA_DISTRIBUTIONS}
 	find ${DESTDIR}/${DISTDIR}/${dist} -mindepth 1 -empty -delete
 .endfor
 .if defined(NO_ROOT)
 .for dist in base ${EXTRA_DISTRIBUTIONS}
 	@# For each file that exists in this dist, print the corresponding
 	@# line from the METALOG.  This relies on the fact that
 	@# a line containing only the filename will sort immediatly before
 	@# the relevant mtree line.
 	cd ${DESTDIR}/${DISTDIR}; \
 	find ./${dist} | sort -u ${METALOG} - | \
 	awk 'BEGIN { print "#${MTREE_MAGIC}" } !/ type=/ { file = $$1 } / type=/ { if ($$1 == file) { sub(/^\.\/${dist}\//, "./"); print } }' > \
 	${DESTDIR}/${DISTDIR}/${dist}.meta
 .endfor
 .if ${MK_DEBUG_FILES} != "no"
 . for dist in ${DEBUG_DISTRIBUTIONS}
 	@# For each file that exists in this dist, print the corresponding
 	@# line from the METALOG.  This relies on the fact that
 	@# a line containing only the filename will sort immediatly before
 	@# the relevant mtree line.
 	cd ${DESTDIR}/${DISTDIR}; \
 	find ./${dist}/usr/lib/debug | sort -u ${METALOG} - | \
 	awk 'BEGIN { print "#${MTREE_MAGIC}" } !/ type=/ { file = $$1 } / type=/ { if ($$1 == file) { sub(/^\.\/${dist}\//, "./"); print } }' > \
 	${DESTDIR}/${DISTDIR}/${dist}.debug.meta
 . endfor
 .endif
 .endif
 .endif
 
 packageworld:
 .for dist in base ${EXTRA_DISTRIBUTIONS}
 .if defined(NO_ROOT)
 	${_+_}cd ${DESTDIR}/${DISTDIR}/${dist}; \
 	    tar cvJf ${DESTDIR}/${DISTDIR}/${dist}.txz \
 	    --exclude usr/lib/debug \
 	    @${DESTDIR}/${DISTDIR}/${dist}.meta
 .else
 	${_+_}cd ${DESTDIR}/${DISTDIR}/${dist}; \
 	    tar cvJf ${DESTDIR}/${DISTDIR}/${dist}.txz \
 	    --exclude usr/lib/debug .
 .endif
 .endfor
 
 .if ${MK_DEBUG_FILES} != "no"
 . for dist in ${DEBUG_DISTRIBUTIONS}
 .  if defined(NO_ROOT)
 	${_+_}cd ${DESTDIR}/${DISTDIR}/${dist}; \
 	    tar cvJf ${DESTDIR}/${DISTDIR}/${dist}_debug.txz \
 	    @${DESTDIR}/${DISTDIR}/${dist}.debug.meta
 .  else
 	${_+_}cd ${DESTDIR}/${DISTDIR}/${dist}; \
 	    tar cvJfL ${DESTDIR}/${DISTDIR}/${dist}_debug.txz \
 	    usr/lib/debug
 .  endif
 . endfor
 .endif
 
 #
 # reinstall
 #
 # If you have a build server, you can NFS mount the source and obj directories
 # and do a 'make reinstall' on the *client* to install new binaries from the
 # most recent server build.
 #
 reinstall:
 	@echo "--------------------------------------------------------------"
 	@echo ">>> Making hierarchy"
 	@echo "--------------------------------------------------------------"
 	${_+_}cd ${.CURDIR}; ${MAKE} -f Makefile.inc1 \
 	    LOCAL_MTREE=${LOCAL_MTREE:Q} hierarchy
 	@echo
 	@echo "--------------------------------------------------------------"
 	@echo ">>> Installing everything"
 	@echo "--------------------------------------------------------------"
 	${_+_}cd ${.CURDIR}; ${MAKE} -f Makefile.inc1 install
 .if defined(LIB32TMP) && ${MK_LIB32} != "no"
 	${_+_}cd ${.CURDIR}; ${MAKE} -f Makefile.inc1 install32
 .endif
 
 redistribute:
 	@echo "--------------------------------------------------------------"
 	@echo ">>> Distributing everything"
 	@echo "--------------------------------------------------------------"
 	${_+_}cd ${.CURDIR}; ${MAKE} -f Makefile.inc1 distribute
 .if defined(LIB32TMP) && ${MK_LIB32} != "no"
 	${_+_}cd ${.CURDIR}; ${MAKE} -f Makefile.inc1 distribute32 \
 	    DISTRIBUTION=lib32
 .endif
 
 distrib-dirs distribution:
 	cd ${.CURDIR}/etc; ${CROSSENV} PATH=${TMPPATH} ${MAKE} \
 	    ${IMAKE_INSTALL} ${IMAKE_MTREE} METALOG=${METALOG} ${.TARGET}
 
 #
 # buildkernel and installkernel
 #
 # Which kernels to build and/or install is specified by setting
 # KERNCONF. If not defined a GENERIC kernel is built/installed.
 # Only the existing (depending TARGET) config files are used
 # for building kernels and only the first of these is designated
 # as the one being installed.
 #
 # Note that we have to use TARGET instead of TARGET_ARCH when
 # we're in kernel-land. Since only TARGET_ARCH is (expected) to
 # be set to cross-build, we have to make sure TARGET is set
 # properly.
 
 .if defined(KERNFAST)
 NO_KERNELCLEAN=	t
 NO_KERNELCONFIG=	t
 NO_KERNELDEPEND=	t
 NO_KERNELOBJ=		t
 # Shortcut for KERNCONF=Blah -DKERNFAST is now KERNFAST=Blah
 .if !defined(KERNCONF) && ${KERNFAST} != "1"
 KERNCONF=${KERNFAST}
 .endif
 .endif
 .if ${TARGET_ARCH} == "powerpc64"
 KERNCONF?=	GENERIC64
 .else
 KERNCONF?=	GENERIC
 .endif
 INSTKERNNAME?=	kernel
 
 KERNSRCDIR?=	${.CURDIR}/sys
 KRNLCONFDIR=	${KERNSRCDIR}/${TARGET}/conf
 KRNLOBJDIR=	${OBJTREE}${KERNSRCDIR}
 KERNCONFDIR?=	${KRNLCONFDIR}
 
 BUILDKERNELS=
 INSTALLKERNEL=
 .for _kernel in ${KERNCONF}
 .if exists(${KERNCONFDIR}/${_kernel})
 BUILDKERNELS+=	${_kernel}
 .if empty(INSTALLKERNEL)
 INSTALLKERNEL= ${_kernel}
 .endif
 .endif
 .endfor
 
 buildkernel ${WMAKE_TGTS} ${.ALLTARGETS:M_*}: .MAKE
 
 #
 # buildkernel
 #
 # Builds all kernels defined by BUILDKERNELS.
 #
 buildkernel:
 .if empty(BUILDKERNELS)
 	@echo "ERROR: Missing kernel configuration file(s) (${KERNCONF})."; \
 	false
 .endif
 	@echo
 .for _kernel in ${BUILDKERNELS}
 	@echo "--------------------------------------------------------------"
 	@echo ">>> Kernel build for ${_kernel} started on `LC_ALL=C date`"
 	@echo "--------------------------------------------------------------"
 	@echo "===> ${_kernel}"
 	mkdir -p ${KRNLOBJDIR}
 .if !defined(NO_KERNELCONFIG)
 	@echo
 	@echo "--------------------------------------------------------------"
 	@echo ">>> stage 1: configuring the kernel"
 	@echo "--------------------------------------------------------------"
 	cd ${KRNLCONFDIR}; \
 		PATH=${TMPPATH} \
 		    config ${CONFIGARGS} -d ${KRNLOBJDIR}/${_kernel} \
 			-I '${KERNCONFDIR}' '${KERNCONFDIR}/${_kernel}'
 .endif
 .if !defined(NO_CLEAN) && !defined(NO_KERNELCLEAN)
 	@echo
 	@echo "--------------------------------------------------------------"
 	@echo ">>> stage 2.1: cleaning up the object tree"
 	@echo "--------------------------------------------------------------"
 	cd ${KRNLOBJDIR}/${_kernel}; ${KMAKE} ${CLEANDIR}
 .endif
 .if !defined(NO_KERNELOBJ)
 	@echo
 	@echo "--------------------------------------------------------------"
 	@echo ">>> stage 2.2: rebuilding the object tree"
 	@echo "--------------------------------------------------------------"
 	cd ${KRNLOBJDIR}/${_kernel}; ${KMAKE} obj
 .endif
 	@echo
 	@echo "--------------------------------------------------------------"
 	@echo ">>> stage 2.3: build tools"
 	@echo "--------------------------------------------------------------"
 	${_+_}cd ${.CURDIR}; ${KTMAKE} kernel-tools
 .if !defined(NO_KERNELDEPEND)
 	@echo
 	@echo "--------------------------------------------------------------"
 	@echo ">>> stage 3.1: making dependencies"
 	@echo "--------------------------------------------------------------"
 	cd ${KRNLOBJDIR}/${_kernel}; ${KMAKE} depend -DNO_MODULES_OBJ
 .endif
 	@echo
 	@echo "--------------------------------------------------------------"
 	@echo ">>> stage 3.2: building everything"
 	@echo "--------------------------------------------------------------"
 	cd ${KRNLOBJDIR}/${_kernel}; ${KMAKE} all -DNO_MODULES_OBJ
 	@echo "--------------------------------------------------------------"
 	@echo ">>> Kernel build for ${_kernel} completed on `LC_ALL=C date`"
 	@echo "--------------------------------------------------------------"
 .endfor
 
 #
 # installkernel, etc.
 #
 # Install the kernel defined by INSTALLKERNEL
 #
 installkernel installkernel.debug \
 reinstallkernel reinstallkernel.debug: _installcheck_kernel
 .if empty(INSTALLKERNEL)
 	@echo "ERROR: No kernel \"${KERNCONF}\" to install."; \
 	false
 .endif
 	@echo "--------------------------------------------------------------"
 	@echo ">>> Installing kernel ${INSTALLKERNEL}"
 	@echo "--------------------------------------------------------------"
 	cd ${KRNLOBJDIR}/${INSTALLKERNEL}; \
 	    ${CROSSENV} PATH=${TMPPATH} \
 	    ${MAKE} ${IMAKE_INSTALL} KERNEL=${INSTKERNNAME} ${.TARGET:S/kernel//}
 
 distributekernel distributekernel.debug:
 .if empty(INSTALLKERNEL)
 	@echo "ERROR: No kernel \"${KERNCONF}\" to install."; \
 	false
 .endif
 	mkdir -p ${DESTDIR}/${DISTDIR}
 .if defined(NO_ROOT)
 	echo "#${MTREE_MAGIC}" > ${DESTDIR}/${DISTDIR}/kernel.premeta
 .endif
 	cd ${KRNLOBJDIR}/${INSTALLKERNEL}; \
 	    ${IMAKEENV} ${IMAKE_INSTALL:S/METALOG/kernel.premeta/} \
 	    ${IMAKE_MTREE} PATH=${TMPPATH} ${MAKE} KERNEL=${INSTKERNNAME} \
 	    DESTDIR=${INSTALL_DDIR}/kernel \
 	    ${.TARGET:S/distributekernel/install/}
 .if defined(NO_ROOT)
 	sed -e 's|^./kernel|.|' ${DESTDIR}/${DISTDIR}/kernel.premeta > \
 	    ${DESTDIR}/${DISTDIR}/kernel.meta
 .endif
 .for _kernel in ${BUILDKERNELS:S/${INSTALLKERNEL}//}
 .if defined(NO_ROOT)
 	echo "#${MTREE_MAGIC}" > ${DESTDIR}/${DISTDIR}/kernel.${_kernel}.premeta
 .endif
 	cd ${KRNLOBJDIR}/${_kernel}; \
 	    ${IMAKEENV} ${IMAKE_INSTALL:S/METALOG/kernel.${_kernel}.premeta/} \
 	    ${IMAKE_MTREE} PATH=${TMPPATH} ${MAKE} \
 	    KERNEL=${INSTKERNNAME}.${_kernel} \
 	    DESTDIR=${INSTALL_DDIR}/kernel.${_kernel} \
 	    ${.TARGET:S/distributekernel/install/}
 .if defined(NO_ROOT)
 	sed -e 's|^./kernel|.|' \
 	    ${DESTDIR}/${DISTDIR}/kernel.${_kernel}.premeta > \
 	    ${DESTDIR}/${DISTDIR}/kernel.${_kernel}.meta
 .endif
 .endfor
 
 packagekernel:
 .if defined(NO_ROOT)
 	cd ${DESTDIR}/${DISTDIR}/kernel; \
 	    tar cvJf ${DESTDIR}/${DISTDIR}/kernel.txz \
 	    @${DESTDIR}/${DISTDIR}/kernel.meta
 .for _kernel in ${BUILDKERNELS:S/${INSTALLKERNEL}//}
 	cd ${DESTDIR}/${DISTDIR}/kernel.${_kernel}; \
 	    tar cvJf ${DESTDIR}/${DISTDIR}/kernel.${_kernel}.txz \
 	    @${DESTDIR}/${DISTDIR}/kernel.${_kernel}.meta
 .endfor
 .else
 	cd ${DESTDIR}/${DISTDIR}/kernel; \
 	    tar cvJf ${DESTDIR}/${DISTDIR}/kernel.txz .
 .for _kernel in ${BUILDKERNELS:S/${INSTALLKERNEL}//}
 	cd ${DESTDIR}/${DISTDIR}/kernel.${_kernel}; \
 	    tar cvJf ${DESTDIR}/${DISTDIR}/kernel.${_kernel}.txz .
 .endfor
 .endif
 
 #
 # doxygen
 #
 # Build the API documentation with doxygen
 #
 doxygen:
 	@if [ ! -x `/usr/bin/which doxygen` ]; then \
 		echo "You need doxygen (devel/doxygen) to generate the API documentation of the kernel." | /usr/bin/fmt; \
 		exit 1; \
 	fi
 	cd ${.CURDIR}/tools/kerneldoc/subsys && ${MAKE} obj all
 
 #
 # update
 #
 # Update the source tree(s), by running svn/svnup to update to the
 # latest copy.
 #
 update:
 .if (defined(CVS_UPDATE) || defined(SUP_UPDATE)) && !defined(SVN_UPDATE)
 	@echo "--------------------------------------------------------------"
 	@echo "CVS_UPDATE and SUP_UPDATE are no longer supported."
 	@echo "Please see: https://wiki.freebsd.org/CvsIsDeprecated"
 	@echo "--------------------------------------------------------------"
 	@exit 1
 .endif
 .if defined(SVN_UPDATE)
 	@echo "--------------------------------------------------------------"
 	@echo ">>> Updating ${.CURDIR} using Subversion"
 	@echo "--------------------------------------------------------------"
 	@(cd ${.CURDIR} && ${SVN} update ${SVNFLAGS})
 .endif
 
 #
 # ------------------------------------------------------------------------
 #
 # From here onwards are utility targets used by the 'make world' and
 # related targets.  If your 'world' breaks, you may like to try to fix
 # the problem and manually run the following targets to attempt to
 # complete the build.  Beware, this is *not* guaranteed to work, you
 # need to have a pretty good grip on the current state of the system
 # to attempt to manually finish it.  If in doubt, 'make world' again.
 #
 
 #
 # legacy: Build compatibility shims for the next three targets
 #
 legacy:
 .if ${BOOTSTRAPPING} < 800107 && ${BOOTSTRAPPING} != 0
 	@echo "ERROR: Source upgrades from versions prior to 8.0 not supported."; \
 	false
 .endif
 .for _tool in tools/build
 	${_+_}@${ECHODIR} "===> ${_tool} (obj,includes,depend,all,install)"; \
 	    cd ${.CURDIR}/${_tool} && \
 	    ${MAKE} DIRPRFX=${_tool}/ obj && \
 	    ${MAKE} DIRPRFX=${_tool}/ DESTDIR=${MAKEOBJDIRPREFIX}/legacy includes && \
 	    ${MAKE} DIRPRFX=${_tool}/ depend && \
 	    ${MAKE} DIRPRFX=${_tool}/ all && \
 	    ${MAKE} DIRPRFX=${_tool}/ DESTDIR=${MAKEOBJDIRPREFIX}/legacy install
 .endfor
 
 #
 # bootstrap-tools: Build tools needed for compatibility
 #
 .if ${MK_GAMES} != "no"
 _strfile=	games/fortune/strfile
 .endif
 
 .if ${MK_CXX} != "no"
 _gperf=		gnu/usr.bin/gperf
 .endif
 
 .if ${MK_GROFF} != "no"
 _groff=		gnu/usr.bin/groff
 .endif
 
 .if ${BOOTSTRAPPING} < 900002
 _sed=		usr.bin/sed
 .endif
 
 .if ${BOOTSTRAPPING} < 900006
 _lex=		usr.bin/lex
 .endif
 
 .if ${BOOTSTRAPPING} < 1000002
 _m4=		usr.bin/m4
 .endif
 
 .if ${BOOTSTRAPPING} < 1000013
 _yacc=		lib/liby \
 		usr.bin/yacc
 .endif
 
 .if ${BOOTSTRAPPING} < 1000014
 _crunch=	usr.sbin/crunch
 .endif
 
 .if ${BOOTSTRAPPING} < 1000026
 _nmtree=	lib/libnetbsd \
 		usr.sbin/nmtree
 .endif
 
 .if ${BOOTSTRAPPING} < 1000027
 _cat=		bin/cat
 .endif
 
 .if ${BOOTSTRAPPING} >= 900040 && ${BOOTSTRAPPING} < 900041
 _awk=		usr.bin/awk
 .endif
 
 .if ${MK_BSNMP} != "no" && !exists(/usr/sbin/gensnmptree)
 _gensnmptree=	usr.sbin/bsnmpd/gensnmptree
 .endif
 
 # We need to build tlbgen when we're building clang either as
 # the bootstrap compiler, or as the part of the normal build.
 .if ${MK_CLANG_BOOTSTRAP} != "no" || ${MK_CLANG} != "no"
 _clang_tblgen= \
 	lib/clang/libllvmsupport \
 	lib/clang/libllvmtablegen \
 	usr.bin/clang/tblgen \
 	usr.bin/clang/clang-tblgen
 .endif
 
 # dtrace tools are required for older bootstrap env and cross-build
 # pre libdwarf
 .if ${MK_CDDL} != "no" && (${BOOTSTRAPPING} < 1100006 \
       || (${MACHINE} != ${TARGET} || ${MACHINE_ARCH} != ${TARGET_ARCH}))
 _dtrace_tools= cddl/usr.bin/sgsmsg cddl/lib/libctf lib/libelf \
     lib/libdwarf cddl/usr.bin/ctfconvert cddl/usr.bin/ctfmerge
 .endif
 
 # Default to building the GPL DTC, but build the BSDL one if users explicitly
 # request it.
 _dtc= usr.bin/dtc
 .if ${MK_GPL_DTC} != "no"
 _dtc= gnu/usr.bin/dtc
 .endif
 
 .if ${MK_KERBEROS} != "no"
 _kerberos5_bootstrap_tools= \
 	kerberos5/tools/make-roken \
 	kerberos5/lib/libroken \
 	kerberos5/lib/libvers \
 	kerberos5/tools/asn1_compile \
 	kerberos5/tools/slc \
 	usr.bin/compile_et
 .endif
 
 #	Please document (add comment) why something is in 'bootstrap-tools'.
 #	Try to bound the building of the bootstrap-tool to just the
 #	FreeBSD versions that need the tool built at this stage of the build.
 bootstrap-tools: .MAKE
 .for _tool in \
     ${_clang_tblgen} \
     ${_kerberos5_bootstrap_tools} \
     ${_dtrace_tools} \
     ${_strfile} \
     ${_gperf} \
     ${_groff} \
     ${_dtc} \
     ${_awk} \
     ${_cat} \
     usr.bin/lorder \
     usr.bin/makewhatis \
     usr.bin/rpcgen \
     ${_sed} \
     ${_yacc} \
     ${_m4} \
     ${_lex} \
     lib/libmd \
     usr.bin/xinstall \
     ${_gensnmptree} \
     usr.sbin/config \
     ${_crunch} \
     ${_nmtree}
 	${_+_}@${ECHODIR} "===> ${_tool} (obj,depend,all,install)"; \
 		cd ${.CURDIR}/${_tool} && \
 		${MAKE} DIRPRFX=${_tool}/ obj && \
 		${MAKE} DIRPRFX=${_tool}/ depend && \
 		${MAKE} DIRPRFX=${_tool}/ all && \
 		${MAKE} DIRPRFX=${_tool}/ DESTDIR=${MAKEOBJDIRPREFIX}/legacy install
 .endfor
 
 #
 # build-tools: Build special purpose build tools
 #
 .if !defined(NO_SHARE)
 _share=	share/syscons/scrnmaps
 .endif
 
 .if ${MK_GCC} != "no"
 _gcc_tools= gnu/usr.bin/cc/cc_tools
 .endif
 
 .if ${MK_RESCUE} != "no"
 _rescue= rescue/rescue
 .endif
 
 build-tools: .MAKE
 .for _tool in \
     bin/csh \
     bin/sh \
     ${_rescue} \
     ${LOCAL_TOOL_DIRS} \
     lib/ncurses/ncurses \
     lib/ncurses/ncursesw \
     ${_share} \
     usr.bin/awk \
     lib/libmagic \
     usr.bin/mkesdb_static \
     usr.bin/mkcsmapper_static \
     usr.bin/vi/catalog
 	${_+_}@${ECHODIR} "===> ${_tool} (obj,build-tools)"; \
 		cd ${.CURDIR}/${_tool} && \
 		${MAKE} DIRPRFX=${_tool}/ obj && \
 		${MAKE} DIRPRFX=${_tool}/ build-tools
 .endfor
 .for _tool in \
     ${_gcc_tools}
 	${_+_}@${ECHODIR} "===> ${_tool} (obj,depend,all)"; \
 		cd ${.CURDIR}/${_tool} && \
 		${MAKE} DIRPRFX=${_tool}/ obj && \
 		${MAKE} DIRPRFX=${_tool}/ depend && \
 		${MAKE} DIRPRFX=${_tool}/ all
 .endfor
 
 #
 # kernel-tools: Build kernel-building tools
 #
 kernel-tools: .MAKE
 	mkdir -p ${MAKEOBJDIRPREFIX}/usr
 	mtree -deU -f ${.CURDIR}/etc/mtree/BSD.usr.dist \
 	    -p ${MAKEOBJDIRPREFIX}/usr >/dev/null
 
 #
 # cross-tools: Build cross-building tools
 #
 .if !defined(TARGET_ARCH) && defined(XDEV_ARCH)
 TARGET_ARCH=	${XDEV_ARCH}
 .endif
 .if ${TARGET_ARCH} != ${MACHINE_ARCH}
 .if ${TARGET_ARCH} == "amd64" || ${TARGET_ARCH} == "i386"
 _btxld=		usr.sbin/btxld
 .endif
 .endif
 .if ${TARGET_ARCH} != ${MACHINE_ARCH}
 .if ${MK_RESCUE} != "no" || defined(RELEASEDIR)
 _crunchide=	usr.sbin/crunch/crunchide
 .endif
 .if ${TARGET_ARCH} == "i386" && defined(RELEASEDIR)
 _kgzip=		usr.sbin/kgzip
 .endif
 .endif
 
 # If we're given an XAS, don't build binutils.
 .if ${XAS:M/*} == "" && ${MK_BINUTILS_BOOTSTRAP} != "no"
 _binutils=	gnu/usr.bin/binutils
 .endif
 
 # If an full path to an external cross compiler is given, don't build
 # a cross compiler.
 .if ${XCC:M/*} == "" && ${MK_CROSS_COMPILER} != "no"
 .if ${MK_CLANG_BOOTSTRAP} != "no"
 _clang=		usr.bin/clang
 _clang_libs=	lib/clang
 .endif
 .if ${MK_GCC_BOOTSTRAP} != "no"
 _cc=		gnu/usr.bin/cc
 .endif
 .endif
 
 cross-tools: .MAKE
 .for _tool in \
     ${_clang_libs} \
     ${_clang} \
     ${_binutils} \
     ${_cc} \
     usr.bin/xlint/lint1 usr.bin/xlint/lint2 usr.bin/xlint/xlint \
     ${_btxld} \
     ${_crunchide} \
     ${_kgzip}
 	${_+_}@${ECHODIR} "===> ${_tool} (obj,depend,all,install)"; \
 		cd ${.CURDIR}/${_tool} && \
 		${MAKE} DIRPRFX=${_tool}/ obj && \
 		${MAKE} DIRPRFX=${_tool}/ depend && \
 		${MAKE} DIRPRFX=${_tool}/ all && \
 		${MAKE} DIRPRFX=${_tool}/ DESTDIR=${MAKEOBJDIRPREFIX} install
 .endfor
 
 #
 # hierarchy - ensure that all the needed directories are present
 #
 hierarchy hier:
 	cd ${.CURDIR}/etc && ${HMAKE} distrib-dirs
 
 #
 # libraries - build all libraries, and install them under ${DESTDIR}.
 #
 # The list of libraries with dependents (${_prebuild_libs}) and their
 # interdependencies (__L) are built automatically by the
 # ${.CURDIR}/tools/make_libdeps.sh script.
 #
 libraries: .MAKE
 	cd ${.CURDIR} && \
 	    ${MAKE} -f Makefile.inc1 _prereq_libs && \
 	    ${MAKE} -f Makefile.inc1 _startup_libs && \
 	    ${MAKE} -f Makefile.inc1 _prebuild_libs && \
 	    ${MAKE} -f Makefile.inc1 _generic_libs
 
 #
 # static libgcc.a prerequisite for shared libc
 #
 _prereq_libs= gnu/lib/libssp/libssp_nonshared gnu/lib/libgcc lib/libcompiler_rt
 
 # These dependencies are not automatically generated:
 #
 # gnu/lib/csu, gnu/lib/libgcc, lib/csu and lib/libc must be built before
 # all shared libraries for ELF.
 #
 _startup_libs=	gnu/lib/csu
 .if exists(${.CURDIR}/lib/csu/${MACHINE_ARCH}-elf)
 _startup_libs+=	lib/csu/${MACHINE_ARCH}-elf
 .elif exists(${.CURDIR}/lib/csu/${MACHINE_ARCH})
 _startup_libs+=	lib/csu/${MACHINE_ARCH}
 .else
 _startup_libs+=	lib/csu/${MACHINE_CPUARCH}
 .endif
 _startup_libs+=	gnu/lib/libgcc
 _startup_libs+=	lib/libcompiler_rt
 _startup_libs+=	lib/libc
 _startup_libs+=	lib/libc_nonshared
 .if ${MK_LIBCPLUSPLUS} != "no"
 _startup_libs+=	lib/libcxxrt
 .endif
 
 gnu/lib/libgcc__L: lib/libc__L
 gnu/lib/libgcc__L: lib/libc_nonshared__L
 .if ${MK_LIBCPLUSPLUS} != "no"
 lib/libcxxrt__L: gnu/lib/libgcc__L
 .endif
 
 _prebuild_libs=	${_kerberos5_lib_libasn1} \
 		${_kerberos5_lib_libhdb} \
 		${_kerberos5_lib_libheimbase} \
 		${_kerberos5_lib_libheimntlm} \
 		${_kerberos5_lib_libheimsqlite} \
 		${_kerberos5_lib_libheimipcc} \
 		${_kerberos5_lib_libhx509} ${_kerberos5_lib_libkrb5} \
 		${_kerberos5_lib_libroken} \
 		${_kerberos5_lib_libwind} \
 		${_lib_atf} \
 		lib/libbz2 ${_libcom_err} lib/libcrypt \
 		lib/libelf lib/libexpat \
 		${_lib_libgssapi} \
 		lib/libkiconv lib/libkvm lib/liblzma lib/libmd lib/libnv \
 		${_lib_libcapsicum} \
 		lib/ncurses/ncurses lib/ncurses/ncursesw \
 		lib/libopie lib/libpam ${_lib_libthr} \
 		lib/libradius lib/libsbuf lib/libtacplus \
 		${_cddl_lib_libumem} ${_cddl_lib_libnvpair} \
 		${_cddl_lib_libavl} \
 		${_cddl_lib_libzfs_core} \
 		lib/libutil lib/libpjdlog ${_lib_libypclnt} lib/libz lib/msun \
 		${_secure_lib_libcrypto} ${_lib_libldns} \
 		${_secure_lib_libssh} ${_secure_lib_libssl}
 .if ${MK_GNUCXX} != "no" && ${MK_CXX} != "no"
 _prebuild_libs+= gnu/lib/libstdc++ gnu/lib/libsupc++
 gnu/lib/libstdc++__L: lib/msun__L
 .endif
 
 .if defined(WITH_ATF) || ${MK_TESTS} != "no"
 .if !defined(WITH_ATF)
 # Ensure that the ATF libraries will be built during make libraries, even
 # though they will have WITHOUT_TESTS
 MAKE+=		-DWITH_ATF
 .endif
 _lib_atf=	lib/atf
 .endif
 
 .if ${MK_LIBTHR} != "no"
 _lib_libthr=	lib/libthr
 .endif
 
 .if ${MK_OFED} != "no"
 _ofed_lib=	contrib/ofed/usr.lib/
 .endif
 
 .if ${MK_CASPER} != "no"
 _lib_libcapsicum=lib/libcapsicum
 .endif
 
 lib/libcapsicum__L: lib/libnv__L
 lib/libpjdlog__L: lib/libutil__L
 
 _generic_libs=	${_cddl_lib} gnu/lib ${_kerberos5_lib} lib ${_secure_lib} usr.bin/lex/lib ${_ofed_lib}
 .for _DIR in ${LOCAL_LIB_DIRS}
 .if exists(${.CURDIR}/${_DIR}/Makefile)
 _generic_libs+= ${_DIR}
 .endif
 .endfor
 
 lib/libopie__L lib/libtacplus__L: lib/libmd__L
 
 .if ${MK_CDDL} != "no"
 _cddl_lib_libumem= cddl/lib/libumem
 _cddl_lib_libnvpair= cddl/lib/libnvpair
 _cddl_lib_libavl= cddl/lib/libavl
 _cddl_lib_libzfs_core= cddl/lib/libzfs_core
 _cddl_lib= cddl/lib
 cddl/lib/libzfs_core__L: cddl/lib/libnvpair__L
 .endif
 
 .if ${MK_CRYPT} != "no"
 .if ${MK_OPENSSL} != "no"
 _secure_lib_libcrypto= secure/lib/libcrypto
 _secure_lib_libssl= secure/lib/libssl
 lib/libradius__L secure/lib/libssl__L: secure/lib/libcrypto__L
 .if ${MK_LDNS} != "no"
 _lib_libldns= lib/libldns
 lib/libldns__L: secure/lib/libcrypto__L
 .endif
 .if ${MK_OPENSSH} != "no"
 _secure_lib_libssh= secure/lib/libssh
 secure/lib/libssh__L: lib/libz__L secure/lib/libcrypto__L lib/libcrypt__L
 .if ${MK_LDNS} != "no"
 secure/lib/libssh__L: lib/libldns__L
 .endif
 .if ${MK_KERBEROS_SUPPORT} != "no"
 secure/lib/libssh__L: lib/libgssapi__L kerberos5/lib/libkrb5__L \
     kerberos5/lib/libhx509__L kerberos5/lib/libasn1__L lib/libcom_err__L \
     lib/libmd__L kerberos5/lib/libroken__L
 .endif
 .endif
 .endif
 _secure_lib=	secure/lib
 .endif
 
 .if ${MK_KERBEROS} != "no"
 kerberos5/lib/libasn1__L: lib/libcom_err__L kerberos5/lib/libroken__L
 kerberos5/lib/libhdb__L: kerberos5/lib/libasn1__L lib/libcom_err__L \
     kerberos5/lib/libkrb5__L kerberos5/lib/libroken__L \
     kerberos5/lib/libwind__L kerberos5/lib/libheimsqlite__L 
 kerberos5/lib/libheimntlm__L: secure/lib/libcrypto__L kerberos5/lib/libkrb5__L \
     kerberos5/lib/libroken__L lib/libcom_err__L
 kerberos5/lib/libhx509__L: kerberos5/lib/libasn1__L lib/libcom_err__L \
     secure/lib/libcrypto__L kerberos5/lib/libroken__L kerberos5/lib/libwind__L
 kerberos5/lib/libkrb5__L: kerberos5/lib/libasn1__L lib/libcom_err__L \
     lib/libcrypt__L secure/lib/libcrypto__L kerberos5/lib/libhx509__L \
     kerberos5/lib/libroken__L kerberos5/lib/libwind__L \
     kerberos5/lib/libheimbase__L kerberos5/lib/libheimipcc__L
 kerberos5/lib/libroken__L: lib/libcrypt__L
 kerberos5/lib/libwind__L: kerberos5/lib/libroken__L lib/libcom_err__L
 kerberos5/lib/libheimbase__L: lib/libthr__L
 kerberos5/lib/libheimipcc__L: kerberos5/lib/libroken__L kerberos5/lib/libheimbase__L lib/libthr__L
 kerberos5/lib/libheimsqlite__L: lib/libthr__L
 .endif
 
 .if ${MK_GSSAPI} != "no"
 _lib_libgssapi=	lib/libgssapi
 .endif
 
 .if ${MK_KERBEROS} != "no"
 _kerberos5_lib=	kerberos5/lib
 _kerberos5_lib_libasn1= kerberos5/lib/libasn1
 _kerberos5_lib_libhdb= kerberos5/lib/libhdb
 _kerberos5_lib_libheimbase= kerberos5/lib/libheimbase
 _kerberos5_lib_libkrb5= kerberos5/lib/libkrb5
 _kerberos5_lib_libhx509= kerberos5/lib/libhx509
 _kerberos5_lib_libroken= kerberos5/lib/libroken
 _kerberos5_lib_libheimntlm= kerberos5/lib/libheimntlm
 _kerberos5_lib_libheimsqlite= kerberos5/lib/libheimsqlite
 _kerberos5_lib_libheimipcc= kerberos5/lib/libheimipcc
 _kerberos5_lib_libwind= kerberos5/lib/libwind
 _libcom_err= lib/libcom_err
 .endif
 
 .if ${MK_NIS} != "no"
 _lib_libypclnt=	lib/libypclnt
 .endif
 
 .if ${MK_OPENSSL} == "no"
 lib/libradius__L: lib/libmd__L
 .endif
 
 .for _lib in ${_prereq_libs}
 ${_lib}__PL: .PHONY .MAKE
 .if exists(${.CURDIR}/${_lib})
 	${_+_}@${ECHODIR} "===> ${_lib} (obj,depend,all,install)"; \
 		cd ${.CURDIR}/${_lib} && \
 		${MAKE} MK_TESTS=no DIRPRFX=${_lib}/ obj && \
 		${MAKE} MK_TESTS=no DIRPRFX=${_lib}/ depend && \
 		${MAKE} MK_TESTS=no MK_PROFILE=no -DNO_PIC \
 		    DIRPRFX=${_lib}/ all && \
 		${MAKE} MK_TESTS=no MK_PROFILE=no -DNO_PIC \
 		    DIRPRFX=${_lib}/ install
 .endif
 .endfor
 
 .for _lib in ${_startup_libs} ${_prebuild_libs:Nlib/libpam} ${_generic_libs}
 ${_lib}__L: .PHONY .MAKE
 .if exists(${.CURDIR}/${_lib})
 	${_+_}@${ECHODIR} "===> ${_lib} (obj,depend,all,install)"; \
 		cd ${.CURDIR}/${_lib} && \
 		${MAKE} MK_TESTS=no DIRPRFX=${_lib}/ obj && \
 		${MAKE} MK_TESTS=no DIRPRFX=${_lib}/ depend && \
 		${MAKE} MK_TESTS=no DIRPRFX=${_lib}/ all && \
 		${MAKE} MK_TESTS=no DIRPRFX=${_lib}/ install
 .endif
 .endfor
 
 # libpam is special: we need to build static PAM modules before
 # static PAM library, and dynamic PAM library before dynamic PAM
 # modules.
 lib/libpam__L: .PHONY .MAKE
 	${_+_}@${ECHODIR} "===> lib/libpam (obj,depend,all,install)"; \
 		cd ${.CURDIR}/lib/libpam && \
 		${MAKE} MK_TESTS=no DIRPRFX=lib/libpam/ obj && \
 		${MAKE} MK_TESTS=no DIRPRFX=lib/libpam/ depend && \
 		${MAKE} MK_TESTS=no DIRPRFX=lib/libpam/ \
 		    -D_NO_LIBPAM_SO_YET all && \
 		${MAKE} MK_TESTS=no DIRPRFX=lib/libpam/ \
 		    -D_NO_LIBPAM_SO_YET install
 
 _prereq_libs: ${_prereq_libs:S/$/__PL/}
 _startup_libs: ${_startup_libs:S/$/__L/}
 _prebuild_libs: ${_prebuild_libs:S/$/__L/}
 _generic_libs: ${_generic_libs:S/$/__L/}
 
 .for __target in all clean cleandepend cleandir depend includes obj
 .for entry in ${SUBDIR}
 ${entry}.${__target}__D: .PHONY .MAKE
 	${_+_}@set -e; if test -d ${.CURDIR}/${entry}.${MACHINE_ARCH}; then \
 		${ECHODIR} "===> ${DIRPRFX}${entry}.${MACHINE_ARCH} (${__target})"; \
 		edir=${entry}.${MACHINE_ARCH}; \
 		cd ${.CURDIR}/$${edir}; \
 	else \
 		${ECHODIR} "===> ${DIRPRFX}${entry} (${__target})"; \
 		edir=${entry}; \
 		cd ${.CURDIR}/$${edir}; \
 	fi; \
 	${MAKE} ${__target} DIRPRFX=${DIRPRFX}$${edir}/
 .endfor
 par-${__target}: ${SUBDIR:S/$/.${__target}__D/}
 .endfor
 
 .include <bsd.subdir.mk>
 
 .if make(check-old) || make(check-old-dirs) || \
     make(check-old-files) || make(check-old-libs) || \
     make(delete-old) || make(delete-old-dirs) || \
     make(delete-old-files) || make(delete-old-libs)
 
 #
 # check for / delete old files section
 #
 
 .include "ObsoleteFiles.inc"
 
 OLD_LIBS_MESSAGE="Please be sure no application still uses those libraries, \
 else you can not start such an application. Consult UPDATING for more \
 information regarding how to cope with the removal/revision bump of a \
 specific library."
 
 .if !defined(BATCH_DELETE_OLD_FILES)
 RM_I=-i
 .else
 RM_I=-v
 .endif
 
 delete-old-files:
 	@echo ">>> Removing old files (only deletes safe to delete libs)"
 # Ask for every old file if the user really wants to remove it.
 # It's annoying, but better safe than sorry.
 # NB: We cannot pass the list of OLD_FILES as a parameter because the
 # argument list will get too long. Using .for/.endfor make "loops" will make
 # the Makefile parser segfault.
 	@exec 3<&0; \
 	cd ${.CURDIR}; \
 	${MAKE} -f ${.CURDIR}/Makefile.inc1 ${.MAKEFLAGS} ${.TARGET} \
 	    -V OLD_FILES -V "OLD_FILES:Musr/share/*.gz:R" | xargs -n1 | \
 	while read file; do \
 		if [ -f "${DESTDIR}/$${file}" -o -L "${DESTDIR}/$${file}" ]; then \
 			chflags noschg "${DESTDIR}/$${file}" 2>/dev/null || true; \
 			rm ${RM_I} "${DESTDIR}/$${file}" <&3; \
 		fi; \
 	done
 # Remove catpages without corresponding manpages.
 	@exec 3<&0; \
 	find ${DESTDIR}/usr/share/man/cat* ! -type d | \
 	sed -ep -e's:${DESTDIR}/usr/share/man/cat:${DESTDIR}/usr/share/man/man:' | \
 	while read catpage; do \
 		read manpage; \
 		if [ ! -e "$${manpage}" ]; then \
 			rm ${RM_I} $${catpage} <&3; \
 	        fi; \
 	done
 	@echo ">>> Old files removed"
 
 check-old-files:
 	@echo ">>> Checking for old files"
 	@cd ${.CURDIR}; \
 	${MAKE} -f ${.CURDIR}/Makefile.inc1 ${.MAKEFLAGS} ${.TARGET} \
 	    -V OLD_FILES -V "OLD_FILES:Musr/share/*.gz:R" | xargs -n1 | \
 	while read file; do \
 		if [ -f "${DESTDIR}/$${file}" -o -L "${DESTDIR}/$${file}" ]; then \
 		 	echo "${DESTDIR}/$${file}"; \
 		fi; \
 	done
 # Check for catpages without corresponding manpages.
 	@find ${DESTDIR}/usr/share/man/cat* ! -type d | \
 	sed -ep -e's:${DESTDIR}/usr/share/man/cat:${DESTDIR}/usr/share/man/man:' | \
 	while read catpage; do \
 		read manpage; \
 		if [ ! -e "$${manpage}" ]; then \
 			echo $${catpage}; \
 	        fi; \
 	done
 
 delete-old-libs:
 	@echo ">>> Removing old libraries"
 	@echo "${OLD_LIBS_MESSAGE}" | fmt
 	@exec 3<&0; \
 	cd ${.CURDIR}; \
 	${MAKE} -f ${.CURDIR}/Makefile.inc1 ${.MAKEFLAGS} ${.TARGET} \
 	    -V OLD_LIBS | xargs -n1 | \
 	while read file; do \
 		if [ -f "${DESTDIR}/$${file}" -o -L "${DESTDIR}/$${file}" ]; then \
 			chflags noschg "${DESTDIR}/$${file}" 2>/dev/null || true; \
 			rm ${RM_I} "${DESTDIR}/$${file}" <&3; \
 		fi; \
 		for ext in debug symbols; do \
 		  if ! [ -e "${DESTDIR}/$${file}" ] && [ -f \
 		      "${DESTDIR}${DEBUGDIR}/$${file}.$${ext}" ]; then \
 			  rm ${RM_I} "${DESTDIR}${DEBUGDIR}/$${file}.$${ext}" \
 			      <&3; \
 		  fi; \
 		done; \
 	done
 	@echo ">>> Old libraries removed"
 
 check-old-libs:
 	@echo ">>> Checking for old libraries"
 	@cd ${.CURDIR}; \
 	${MAKE} -f ${.CURDIR}/Makefile.inc1 ${.MAKEFLAGS} ${.TARGET} \
 	    -V OLD_LIBS | xargs -n1 | \
 	while read file; do \
 		if [ -f "${DESTDIR}/$${file}" -o -L "${DESTDIR}/$${file}" ]; then \
 			echo "${DESTDIR}/$${file}"; \
 		fi; \
 		for ext in debug symbols; do \
 		  if [ -f "${DESTDIR}${DEBUGDIR}/$${file}.$${ext}" ]; then \
 			  echo "${DESTDIR}${DEBUGDIR}/$${file}.$${ext}"; \
 		  fi; \
 		done; \
 	done
 
 delete-old-dirs:
 	@echo ">>> Removing old directories"
 	@cd ${.CURDIR}; \
 	${MAKE} -f ${.CURDIR}/Makefile.inc1 ${.MAKEFLAGS} ${.TARGET} \
 	    -V OLD_DIRS | xargs -n1 | sort -r | \
 	while read dir; do \
 		if [ -d "${DESTDIR}/$${dir}" ]; then \
 			rmdir -v "${DESTDIR}/$${dir}" || true; \
 		elif [ -L "${DESTDIR}/$${dir}" ]; then \
 			echo "${DESTDIR}/$${dir} is a link, please remove everything manually."; \
 		fi; \
 	done
 	@echo ">>> Old directories removed"
 
 check-old-dirs:
 	@echo ">>> Checking for old directories"
 	@cd ${.CURDIR}; \
 	${MAKE} -f ${.CURDIR}/Makefile.inc1 ${.MAKEFLAGS} ${.TARGET} \
 	    -V OLD_DIRS | xargs -n1 | \
 	while read dir; do \
 		if [ -d "${DESTDIR}/$${dir}" ]; then \
 			echo "${DESTDIR}/$${dir}"; \
 		elif [ -L "${DESTDIR}/$${dir}" ]; then \
 			echo "${DESTDIR}/$${dir} is a link, please remove everything manually."; \
 		fi; \
 	done
 
 delete-old: delete-old-files delete-old-dirs
 	@echo "To remove old libraries run '${MAKE} delete-old-libs'."
 
 check-old: check-old-files check-old-libs check-old-dirs
 	@echo "To remove old files and directories run '${MAKE} delete-old'."
 	@echo "To remove old libraries run '${MAKE} delete-old-libs'."
 
 .endif
 
 #
 # showconfig - show build configuration.
 #
 showconfig:
 	@${MAKE} -n -f src.opts.mk -V dummy -dg1 2>&1 | grep ^MK_ | sort
 
 .if !empty(KRNLOBJDIR) && !empty(KERNCONF)
 DTBOUTPUTPATH= ${KRNLOBJDIR}/${KERNCONF}/
 
 .if !defined(FDT_DTS_FILE) || empty(FDT_DTS_FILE)
 .if exists(${KERNCONFDIR}/${KERNCONF})
 FDT_DTS_FILE!= awk 'BEGIN {FS="="} /^makeoptions[[:space:]]+FDT_DTS_FILE/ {print $$2}' \
 	'${KERNCONFDIR}/${KERNCONF}' ; echo
 .endif
 .endif
 
 .endif
 
 .if !defined(DTBOUTPUTPATH) || !exists(${DTBOUTPUTPATH})
 DTBOUTPUTPATH= ${.CURDIR}
 .endif
 
 #
 # Build 'standalone' Device Tree Blob
 #
 builddtb:
 	@PATH=${TMPPATH} MACHINE=${TARGET} \
 	${.CURDIR}/sys/tools/fdt/make_dtb.sh ${.CURDIR}/sys \
 	    "${FDT_DTS_FILE}" ${DTBOUTPUTPATH}
 
 ###############
 
 .if defined(XDEV) && defined(XDEV_ARCH)
 
 .if ${XDEV} == ${MACHINE} && ${XDEV_ARCH} == ${MACHINE_ARCH}
 XDEV_CPUTYPE?=${CPUTYPE}
 .else
 XDEV_CPUTYPE?=${TARGET_CPUTYPE}
 .endif
 
 NOFUN=-DNO_FSCHG MK_HTML=no MK_INFO=no -DNO_LINT \
 	MK_MAN=no MK_NLS=no MK_PROFILE=no \
 	MK_KERBEROS=no MK_RESCUE=no MK_TESTS=no MK_WARNS=no \
 	TARGET=${XDEV} TARGET_ARCH=${XDEV_ARCH} \
 	CPUTYPE=${XDEV_CPUTYPE}
 
 XDDIR=${XDEV_ARCH}-freebsd
 XDTP?=/usr/${XDDIR}
 .if ${XDTP:N/*}
 .error XDTP variable should be an absolute path
 .endif
 
 CDBENV=MAKEOBJDIRPREFIX=${MAKEOBJDIRPREFIX}/${XDDIR} \
 	INSTALL="sh ${.CURDIR}/tools/install.sh"
 CDENV= ${CDBENV} \
 	_SHLIBDIRPREFIX=${XDDESTDIR} \
 	TOOLS_PREFIX=${XDTP}
 CD2CFLAGS=-isystem ${XDDESTDIR}/usr/include -L${XDDESTDIR}/usr/lib \
 	--sysroot=${XDDESTDIR}/ -B${XDDESTDIR}/usr/libexec \
 	-B${XDDESTDIR}/usr/bin -B${XDDESTDIR}/usr/lib
 CD2ENV=${CDENV} CC="${CC} ${CD2CFLAGS}" CXX="${CXX} ${CD2CFLAGS}" \
 	CPP="${CPP} ${CD2CFLAGS}" \
 	MACHINE=${XDEV} MACHINE_ARCH=${XDEV_ARCH}
 
 CDTMP=	${MAKEOBJDIRPREFIX}/${XDDIR}/${.CURDIR}/tmp
 CDMAKE=${CDENV} PATH=${CDTMP}/usr/bin:${PATH} ${MAKE} ${NOFUN}
 CD2MAKE=${CD2ENV} PATH=${CDTMP}/usr/bin:${XDDESTDIR}/usr/bin:${PATH} ${MAKE} ${NOFUN}
 XDDESTDIR=${DESTDIR}/${XDTP}
 .if !defined(OSREL)
 OSREL!= uname -r | sed -e 's/[-(].*//'
 .endif
 
 .ORDER: xdev-build xdev-install
 xdev: xdev-build xdev-install
 
 .ORDER: _xb-worldtmp _xb-bootstrap-tools _xb-build-tools _xb-cross-tools
 xdev-build: _xb-worldtmp _xb-bootstrap-tools _xb-build-tools _xb-cross-tools
 
 _xb-worldtmp:
 	mkdir -p ${CDTMP}/usr
 	mtree -deU -f ${.CURDIR}/etc/mtree/BSD.usr.dist \
 	    -p ${CDTMP}/usr >/dev/null
 
 _xb-bootstrap-tools:
 .for _tool in \
     ${_clang_tblgen}
 	${_+_}@${ECHODIR} "===> ${_tool} (obj,depend,all,install)"; \
 	cd ${.CURDIR}/${_tool} && \
 	${CDMAKE} DIRPRFX=${_tool}/ obj && \
 	${CDMAKE} DIRPRFX=${_tool}/ depend && \
 	${CDMAKE} DIRPRFX=${_tool}/ all && \
 	${CDMAKE} DIRPRFX=${_tool}/ DESTDIR=${CDTMP} install
 .endfor
 
 _xb-build-tools:
 	${_+_}@cd ${.CURDIR}; \
 	${CDBENV} ${MAKE} -f Makefile.inc1 ${NOFUN} build-tools
 
 _xb-cross-tools:
 .for _tool in \
     ${_binutils} \
     usr.bin/ar \
     ${_clang_libs} \
     ${_clang} \
     ${_cc}
 	${_+_}@${ECHODIR} "===> xdev ${_tool} (obj,depend,all)"; \
 	cd ${.CURDIR}/${_tool} && \
 	${CDMAKE} DIRPRFX=${_tool}/ obj && \
 	${CDMAKE} DIRPRFX=${_tool}/ depend && \
 	${CDMAKE} DIRPRFX=${_tool}/ all
 .endfor
 
 _xi-mtree:
 	${_+_}@${ECHODIR} "mtree populating ${XDDESTDIR}"
 	mkdir -p ${XDDESTDIR}
 	mtree -deU -f ${.CURDIR}/etc/mtree/BSD.root.dist \
 	    -p ${XDDESTDIR} >/dev/null
 	mtree -deU -f ${.CURDIR}/etc/mtree/BSD.usr.dist \
 	    -p ${XDDESTDIR}/usr >/dev/null
 	mtree -deU -f ${.CURDIR}/etc/mtree/BSD.include.dist \
 	    -p ${XDDESTDIR}/usr/include >/dev/null
 
 .ORDER: xdev-build _xi-mtree _xi-cross-tools _xi-includes _xi-libraries _xi-links
 xdev-install: xdev-build _xi-mtree _xi-cross-tools _xi-includes _xi-libraries _xi-links
 
 _xi-cross-tools:
 	@echo "_xi-cross-tools"
 .for _tool in \
     ${_binutils} \
     usr.bin/ar \
     ${_clang_libs} \
     ${_clang} \
     ${_cc}
 	${_+_}@${ECHODIR} "===> xdev ${_tool} (install)"; \
 	cd ${.CURDIR}/${_tool}; \
 	${CDMAKE} DIRPRFX=${_tool}/ install DESTDIR=${XDDESTDIR}
 .endfor
 
 _xi-includes:
 	${_+_}cd ${.CURDIR}; ${CD2MAKE} -f Makefile.inc1 par-includes \
 		DESTDIR=${XDDESTDIR}
 
 _xi-libraries:
 	${_+_}cd ${.CURDIR}; ${CD2MAKE} -f Makefile.inc1 libraries \
 		DESTDIR=${XDDESTDIR}
 
 _xi-links:
 	${_+_}cd ${XDDESTDIR}/usr/bin; \
        mkdir -p ../../../../usr/bin; \
 		for i in *; do \
 			ln -sf ../../${XDTP}/usr/bin/$$i \
 			    ../../../../usr/bin/${XDDIR}-$$i; \
 			ln -sf ../../${XDTP}/usr/bin/$$i \
 			    ../../../../usr/bin/${XDDIR}${OSREL}-$$i; \
 		done
 .else
 xdev xdev-build xdev-install:
 	@echo "*** Error: Both XDEV and XDEV_ARCH must be defined for \"${.TARGET}\" target"
 .endif
Index: projects/release-debugdist/etc/Makefile
===================================================================
--- projects/release-debugdist/etc/Makefile	(revision 267343)
+++ projects/release-debugdist/etc/Makefile	(revision 267344)
@@ -1,400 +1,403 @@
 #	from: @(#)Makefile	5.11 (Berkeley) 5/21/91
 # $FreeBSD$
 
 .include <src.opts.mk>
 
 .if ${MK_SENDMAIL} != "no"
 SUBDIR=	sendmail
 .endif
 
 .if ${MK_TESTS} != "no"
 SUBDIR+=tests
 .endif
 
 BIN1=	crontab \
 	devd.conf \
 	devfs.conf \
 	ddb.conf \
 	dhclient.conf \
 	disktab \
 	fbtab \
 	ftpusers \
 	gettytab \
 	group \
 	hosts \
 	hosts.allow \
 	hosts.equiv \
 	inetd.conf \
 	libalias.conf \
 	libmap.conf \
 	login.access \
 	login.conf \
 	mac.conf \
 	motd \
 	netconfig \
 	network.subr \
 	networks \
 	newsyslog.conf \
 	nsswitch.conf \
 	phones \
 	profile \
 	protocols \
 	rc \
 	rc.bsdextended \
 	rc.firewall \
 	rc.initdiskless \
 	rc.sendmail \
 	rc.shutdown \
 	rc.subr \
 	remote \
 	rpc \
 	services \
 	shells \
 	sysctl.conf \
 	syslog.conf \
 	termcap.small
 
 .if ${MACHINE_ARCH} == "amd64" || ${MACHINE_ARCH} == "powerpc64"
 BIN1+=	libmap32.conf
 .endif
 
 .if exists(${.CURDIR}/etc.${MACHINE}/ttys)
 BIN1+=	etc.${MACHINE}/ttys
 .elif exists(${.CURDIR}/etc.${MACHINE_ARCH}/ttys)
 BIN1+=	etc.${MACHINE_ARCH}/ttys
 .elif exists(${.CURDIR}/etc.${MACHINE_CPUARCH}/ttys)
 BIN1+=	etc.${MACHINE_CPUARCH}/ttys
 .else
 .error etc.MACHINE/ttys missing
 .endif
 
 OPENBSMDIR=			${.CURDIR}/../contrib/openbsm
 BSM_ETC_OPEN_FILES=		${OPENBSMDIR}/etc/audit_class \
 				${OPENBSMDIR}/etc/audit_event
 BSM_ETC_RESTRICTED_FILES=	${OPENBSMDIR}/etc/audit_control \
 				${OPENBSMDIR}/etc/audit_user
 BSM_ETC_EXEC_FILES=		${OPENBSMDIR}/etc/audit_warn
 BSM_ETC_DIR=			${DESTDIR}/etc/security
 
 # NB: keep these sorted by MK_* knobs
 
 .if ${MK_AMD} != "no"
 BIN1+= amd.map
 .endif
 
 .if ${MK_APM} != "no"
 BIN1+= apmd.conf
 .endif
 
 .if ${MK_BSNMP} != "no"
 BIN1+= snmpd.config
 .endif
 
 .if ${MK_FREEBSD_UPDATE} != "no"
 BIN1+= freebsd-update.conf
 .endif
 
 .if ${MK_LOCATE} != "no"
 BIN1+=	${.CURDIR}/../usr.bin/locate/locate/locate.rc
 .endif
 
 .if ${MK_LPR} != "no"
 BIN1+=	hosts.lpd printcap
 .endif
 
 .if ${MK_MAIL} != "no"
 BIN1+=	${.CURDIR}/../usr.bin/mail/misc/mail.rc
 .endif
 
 .if ${MK_NTP} != "no"
 BIN1+=	ntp.conf
 .endif
 
 .if ${MK_OPENSSH} != "no"
 SSH=	${.CURDIR}/../crypto/openssh/ssh_config \
 	${.CURDIR}/../crypto/openssh/sshd_config \
 	${.CURDIR}/../crypto/openssh/moduli
 .endif
 .if ${MK_OPENSSL} != "no"
 SSL=	${.CURDIR}/../crypto/openssl/apps/openssl.cnf
 .endif
 
 .if ${MK_NS_CACHING} != "no"
 BIN1+= nscd.conf
 .endif
 
 .if ${MK_PORTSNAP} != "no"
 BIN1+= portsnap.conf
 .endif
 
 .if ${MK_PF} != "no"
 BIN1+= pf.os
 .endif
 
 .if ${MK_TCSH} != "no"
 BIN1+= csh.cshrc csh.login csh.logout
 .endif
 
 .if ${MK_WIRELESS} != "no"
 BIN1+= regdomain.xml
 .endif
 
 # -rwxr-xr-x root:wheel, for the new cron root:wheel
 BIN2=	netstart pccard_ether rc.suspend rc.resume
 
 MTREE=	BSD.include.dist BSD.root.dist BSD.usr.dist BSD.var.dist
 .if ${MK_TESTS} != "no"
 MTREE+=	BSD.tests.dist
 .endif
 .if ${MK_SENDMAIL} != "no"
 MTREE+=	BSD.sendmail.dist
 .endif
 .if ${MK_DEBUG_FILES} != "no"
 MTREE+=	BSD.debug.dist
 .endif
 
 PPPCNF=	ppp.conf
 
 .if ${MK_SENDMAIL} == "no"
 ETCMAIL=mailer.conf aliases
 .else
 ETCMAIL=Makefile README mailer.conf access.sample virtusertable.sample \
 	mailertable.sample aliases
 .endif
 
 # Special top level files for FreeBSD
 FREEBSD=COPYRIGHT
 
 # Sanitize DESTDIR
 DESTDIR:=	${DESTDIR:C://*:/:g}
 
 afterinstall:
 .if ${MK_MAN} != "no"
 	${_+_}cd ${.CURDIR}/../share/man; ${MAKE} makedb
 .endif
 
 distribute:
-	${_+_}cd ${.CURDIR} ; ${MAKE} install DESTDIR=${DISTDIR}/${DISTRIBUTION}
+	# Avoid installing tests here; "make distribution" will do this and
+	# correctly place them in the right location.
+	${_+_}cd ${.CURDIR} ; ${MAKE} MK_TESTS=no install \
+	    DESTDIR=${DISTDIR}/${DISTRIBUTION}
 	${_+_}cd ${.CURDIR} ; ${MAKE} distribution DESTDIR=${DISTDIR}/${DISTRIBUTION}
 
 .include <bsd.endian.mk>
 .if ${TARGET_ENDIANNESS} == "1234"
 CAP_MKDB_ENDIAN?= -l
 PWD_MKDB_ENDIAN?= -L
 .elif ${TARGET_ENDIANNESS} == "4321"
 CAP_MKDB_ENDIAN?= -b
 PWD_MKDB_ENDIAN?= -B
 .else
 CAP_MKDB_ENDIAN?=
 PWD_MKDB_ENDIAN?=
 .endif
 
 .if defined(NO_ROOT)
 METALOG.add?=	cat -l >> ${METALOG}
 .endif
 
 distribution:
 .if !defined(DESTDIR)
 	@echo "set DESTDIR before running \"make ${.TARGET}\""
 	@false
 .endif
 	cd ${.CURDIR}; \
 	    ${INSTALL} -o ${BINOWN} -g ${BINGRP} -m 644 \
 		${BIN1} ${DESTDIR}/etc; \
 	    cap_mkdb ${CAP_MKDB_ENDIAN} ${DESTDIR}/etc/login.conf; \
 	    ${INSTALL} -o ${BINOWN} -g ${BINGRP} -m 755 \
 		${BIN2} ${DESTDIR}/etc; \
 	    ${INSTALL} -o ${BINOWN} -g ${BINGRP} -m 600 \
 		master.passwd nsmb.conf opieaccess ${DESTDIR}/etc;
 .if ${MK_AT} == "no"
 	sed -i "" -e 's;.*/usr/libexec/atrun;#&;' ${DESTDIR}/etc/crontab
 .endif
 .if ${MK_TCSH} == "no"
 	sed -i "" -e 's;/bin/csh;/bin/sh;' ${DESTDIR}/etc/master.passwd
 .endif
 	pwd_mkdb ${PWD_MKDB_ENDIAN} -i -p -d ${DESTDIR}/etc \
 	    ${DESTDIR}/etc/master.passwd
 .if defined(NO_ROOT)
 	( \
 		echo "./etc/login.conf.db type=file mode=0644 uname=root gname=wheel"; \
 		echo "./etc/passwd type=file mode=0644 uname=root gname=wheel"; \
 		echo "./etc/pwd.db type=file mode=0644 uname=root gname=wheel"; \
 		echo "./etc/spwd.db type=file mode=0600 uname=root gname=wheel"; \
 	) | ${METALOG.add}
 .endif
 .if ${MK_BLUETOOTH} != "no"
 	${_+_}cd ${.CURDIR}/bluetooth; ${MAKE} install
 .endif
 .if ${MK_CASPER} != "no"
 	${_+_}cd ${.CURDIR}/casper; ${MAKE} install
 .endif
 	${_+_}cd ${.CURDIR}/defaults; ${MAKE} install
 	${_+_}cd ${.CURDIR}/devd; ${MAKE} install
 .if ${MK_DMAGENT} != "no"
 	${_+_}cd ${.CURDIR}/dma; ${MAKE} install
 .endif
 	${_+_}cd ${.CURDIR}/gss; ${MAKE} install
 	${_+_}cd ${.CURDIR}/periodic; ${MAKE} install
 	${_+_}cd ${.CURDIR}/pkg; ${MAKE} install
 	${_+_}cd ${.CURDIR}/rc.d; ${MAKE} install
 	${_+_}cd ${.CURDIR}/../gnu/usr.bin/send-pr; ${MAKE} etc-gnats-freefall
 	${_+_}cd ${.CURDIR}/../share/termcap; ${MAKE} etc-termcap
 	${_+_}cd ${.CURDIR}/../usr.sbin/rmt; ${MAKE} etc-rmt
 	${_+_}cd ${.CURDIR}/pam.d; ${MAKE} install
 	cd ${.CURDIR}; ${INSTALL} -o ${BINOWN} -g ${BINGRP} -m 0444 \
 	    ${BSM_ETC_OPEN_FILES} ${BSM_ETC_DIR}
 	cd ${.CURDIR}; ${INSTALL} -o ${BINOWN} -g ${BINGRP} -m 0600 \
 	    ${BSM_ETC_RESTRICTED_FILES} ${BSM_ETC_DIR}
 	cd ${.CURDIR}; ${INSTALL} -o ${BINOWN} -g ${BINGRP} -m 0500 \
 	    ${BSM_ETC_EXEC_FILES} ${BSM_ETC_DIR}
 .if ${MK_UNBOUND} != "no"
 	if [ ! -e ${DESTDIR}/etc/unbound ]; then \
 		${INSTALL_SYMLINK} ../var/unbound ${DESTDIR}/etc/unbound; \
 	fi
 .endif
 .if ${MK_SENDMAIL} != "no"
 	${_+_}cd ${.CURDIR}/sendmail; ${MAKE} distribution
 .endif
 .if ${MK_OPENSSH} != "no"
 	cd ${.CURDIR}; ${INSTALL} -o ${BINOWN} -g ${BINGRP} -m 644 \
 	    ${SSH} ${DESTDIR}/etc/ssh
 .endif
 .if ${MK_OPENSSL} != "no"
 	cd ${.CURDIR}; ${INSTALL} -o ${BINOWN} -g ${BINGRP} -m 644 \
 	    ${SSL} ${DESTDIR}/etc/ssl
 .endif
 .if ${MK_KERBEROS} != "no"
 	cd ${.CURDIR}/root; \
 	    ${INSTALL} -o ${BINOWN} -g ${BINGRP} -m 644 \
 		dot.k5login ${DESTDIR}/root/.k5login;
 .endif
 	cd ${.CURDIR}/root; \
 	    ${INSTALL} -o ${BINOWN} -g ${BINGRP} -m 644 \
 		dot.profile ${DESTDIR}/root/.profile; \
 	    rm -f ${DESTDIR}/.profile; \
 	    ln ${DESTDIR}/root/.profile ${DESTDIR}/.profile
 .if ${MK_TCSH} != "no"
 	cd ${.CURDIR}/root; \
 	    ${INSTALL} -o ${BINOWN} -g ${BINGRP} -m 644 \
 		dot.cshrc ${DESTDIR}/root/.cshrc; \
 	    ${INSTALL} -o ${BINOWN} -g ${BINGRP} -m 644 \
 		dot.login ${DESTDIR}/root/.login; \
 	    rm -f ${DESTDIR}/.cshrc; \
 	    ln ${DESTDIR}/root/.cshrc ${DESTDIR}/.cshrc
 .endif
 	cd ${.CURDIR}/mtree; ${INSTALL} -o ${BINOWN} -g ${BINGRP} -m 444 \
 	    ${MTREE} ${DESTDIR}/etc/mtree
 .if ${MK_PPP} != "no"
 	cd ${.CURDIR}/ppp; ${INSTALL} -o ${BINOWN} -g ${BINGRP} -m 600 \
 	    ${PPPCNF} ${DESTDIR}/etc/ppp
 .endif
 .if ${MK_MAIL} != "no"
 	cd ${.CURDIR}/mail; ${INSTALL} -o ${BINOWN} -g ${BINGRP} -m 644 \
 	    ${ETCMAIL} ${DESTDIR}/etc/mail
 	if [ -d ${DESTDIR}/etc/mail -a -f ${DESTDIR}/etc/mail/aliases -a \
 	      ! -f ${DESTDIR}/etc/aliases ]; then \
 		ln -s mail/aliases ${DESTDIR}/etc/aliases; \
 	fi
 .endif
 	${INSTALL} -o ${BINOWN} -g operator -m 664 /dev/null \
 	    ${DESTDIR}/etc/dumpdates
 	${INSTALL} -o nobody -g ${BINGRP} -m 644 /dev/null \
 	    ${DESTDIR}/var/db/locate.database
 	${INSTALL} -o ${BINOWN} -g ${BINGRP} -m 644 ${.CURDIR}/minfree \
 	    ${DESTDIR}/var/crash
 	cd ${.CURDIR}/..; ${INSTALL} -o ${BINOWN} -g ${BINGRP} -m 444 \
 		${FREEBSD} ${DESTDIR}/
 .if ${MK_BOOT} != "no"
 .if exists(${.CURDIR}/../sys/${MACHINE}/conf/GENERIC.hints)
 	${INSTALL} -o ${BINOWN} -g ${BINGRP} -m 444 \
 	    ${.CURDIR}/../sys/${MACHINE}/conf/GENERIC.hints \
 	    ${DESTDIR}/boot/device.hints
 .endif
 .endif
 .if ${MK_NIS} == "no"
 	sed -i "" -e 's/.*_compat:/# &/' -e 's/compat$$/files/' \
 		${DESTDIR}/etc/nsswitch.conf
 .endif
 
 MTREE_CMD?=	mtree
 
 MTREES=		mtree/BSD.root.dist		/		\
 		mtree/BSD.var.dist		/var		\
 		mtree/BSD.usr.dist		/usr		\
 		mtree/BSD.include.dist		/usr/include
 .if ${MK_DEBUG_FILES} != "no"
 MTREES+=	mtree/BSD.debug.dist		/usr/lib
 .endif
 .if ${MK_GROFF} != "no"
 MTREES+=	mtree/BSD.groff.dist		/usr
 .endif
 .if ${MK_TESTS} != "no"
 MTREES+=	mtree/BSD.tests.dist		/usr
 .endif
 .if ${MK_SENDMAIL} != "no"
 MTREES+=	mtree/BSD.sendmail.dist		/
 .endif
 .for mtree in ${LOCAL_MTREE}
 MTREES+=	../${mtree}			/
 .endfor
 
 distrib-dirs: ${MTREES:N/*}
 	@set ${MTREES}; \
 	while test $$# -ge 2; do \
 		m=${.CURDIR}/$$1; \
 		shift; \
 		d=${DESTDIR}$$1; \
 		shift; \
 		${ECHO} ${MTREE_CMD} -deU ${MTREE_FOLLOWS_SYMLINKS} \
 		    -f $$m -p $$d; \
 		${MTREE_CMD} -deU ${MTREE_FOLLOWS_SYMLINKS} -f $$m -p $$d; \
 	done; true
 .if defined(NO_ROOT)
 	@set ${MTREES}; \
 	while test $$# -ge 2; do \
 		m=${.CURDIR}/$$1; \
 		shift; \
 		d=$$1; \
 		test "$$d" == "/" && d=""; \
 		d=${DISTBASE}$$d; \
 		shift; \
 		${ECHO} "${MTREE_CMD:N-W} -C -f $$m -K uname,gname | " \
 		    "sed s#^\.#.$$d# | ${METALOG.add}" ; \
 		${MTREE_CMD:N-W} -C -f $$m -K uname,gname | sed s#^\.#.$$d# | \
 		    ${METALOG.add} ; \
 	done; true
 .endif
 	${INSTALL_SYMLINK} usr/src/sys ${DESTDIR}/sys
 	cd ${DESTDIR}/usr/share/man; \
 	for mandir in man*; do \
 		${INSTALL_SYMLINK} ../$$mandir \
 		    ${DESTDIR}/usr/share/man/en.ISO8859-1/; \
 		${INSTALL_SYMLINK} ../$$mandir \
 		    ${DESTDIR}/usr/share/man/en.UTF-8/; \
 	done
 	cd ${DESTDIR}/usr/share/openssl/man; \
 	for mandir in man*; do \
 		${INSTALL_SYMLINK} ../$$mandir \
 		    ${DESTDIR}/usr/share/openssl/man/en.ISO8859-1/; \
 	done
 	set - `grep "^[a-zA-Z]" ${.CURDIR}/man.alias`; \
 	while [ $$# -gt 0 ] ; do \
 		${INSTALL_SYMLINK} "$$2" "${DESTDIR}/usr/share/man/$$1"; \
 		${INSTALL_SYMLINK} "$$2" \
 		    "${DESTDIR}/usr/share/openssl/man/$$1"; \
 		shift; shift; \
 	done
 	set - `grep "^[a-zA-Z]" ${.CURDIR}/nls.alias`; \
 	while [ $$# -gt 0 ] ; do \
 		${INSTALL_SYMLINK} "$$2" "${DESTDIR}/usr/share/nls/$$1"; \
 		shift; shift; \
 	done
 
 etc-examples:
 	cd ${.CURDIR}; ${INSTALL} -o ${BINOWN} -g ${BINGRP} -m 444 \
 	    ${BIN1} ${BIN2} nsmb.conf opieaccess \
 	    ${DESTDIR}/usr/share/examples/etc
 	${_+_}cd ${.CURDIR}/defaults; ${MAKE} install \
 	    DESTDIR=${DESTDIR}/usr/share/examples
 
 .include <bsd.prog.mk>
Index: projects/release-debugdist/lib/clang/clang.build.mk
===================================================================
--- projects/release-debugdist/lib/clang/clang.build.mk	(revision 267343)
+++ projects/release-debugdist/lib/clang/clang.build.mk	(revision 267344)
@@ -1,230 +1,241 @@
 # $FreeBSD$
 
 .include <src.opts.mk>
 
 CLANG_SRCS=	${LLVM_SRCS}/tools/clang
 
 CFLAGS+=	-I${LLVM_SRCS}/include -I${CLANG_SRCS}/include \
 		-I${LLVM_SRCS}/${SRCDIR} ${INCDIR:C/^/-I${LLVM_SRCS}\//} -I. \
 		-I${LLVM_SRCS}/../../lib/clang/include \
 		-DLLVM_ON_UNIX -DLLVM_ON_FREEBSD \
 		-D__STDC_LIMIT_MACROS -D__STDC_CONSTANT_MACROS #-DNDEBUG
 
 .if ${MK_CLANG_FULL} != "no"
 CFLAGS+=	-DCLANG_ENABLE_ARCMT \
 		-DCLANG_ENABLE_REWRITER \
 		-DCLANG_ENABLE_STATIC_ANALYZER
 .endif # MK_CLANG_FULL
 
 # LLVM is not strict aliasing safe as of 12/31/2011
 CFLAGS+=	-fno-strict-aliasing
 
 TARGET_ARCH?=	${MACHINE_ARCH}
 BUILD_ARCH?=	${MACHINE_ARCH}
 
 .if (${TARGET_ARCH} == "arm" || ${TARGET_ARCH} == "armv6") && \
     ${MK_ARM_EABI} != "no"
 TARGET_ABI=	gnueabi
 .elif ${TARGET_ARCH} == "armv6hf"
 TARGET_ABI=	gnueabihf
 .else
 TARGET_ABI=	unknown
 .endif
 
 TARGET_TRIPLE?=	${TARGET_ARCH:C/amd64/x86_64/:C/armv6hf/armv6/}-${TARGET_ABI}-freebsd11.0
 BUILD_TRIPLE?=	${BUILD_ARCH:C/amd64/x86_64/:C/armv6hf/armv6/}-unknown-freebsd11.0
 CFLAGS+=	-DLLVM_DEFAULT_TARGET_TRIPLE=\"${TARGET_TRIPLE}\" \
 		-DLLVM_HOST_TRIPLE=\"${BUILD_TRIPLE}\" \
 		-DDEFAULT_SYSROOT=\"${TOOLS_PREFIX}\"
 CXXFLAGS+=	-fno-exceptions -fno-rtti
 
 .PATH:	${LLVM_SRCS}/${SRCDIR}
 
 TBLGEN?=	tblgen
 CLANG_TBLGEN?=	clang-tblgen
 
 Intrinsics.inc.h: ${LLVM_SRCS}/include/llvm/IR/Intrinsics.td
 	${TBLGEN} -gen-intrinsic \
 	    -I ${LLVM_SRCS}/include -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} \
 	    ${LLVM_SRCS}/include/llvm/IR/Intrinsics.td
 .for arch in \
 	ARM/ARM Mips/Mips PowerPC/PPC Sparc/Sparc X86/X86
 . for hdr in \
 	AsmMatcher/-gen-asm-matcher \
 	AsmWriter1/-gen-asm-writer,-asmwriternum=1 \
 	AsmWriter/-gen-asm-writer \
 	CallingConv/-gen-callingconv \
 	CodeEmitter/-gen-emitter \
 	DAGISel/-gen-dag-isel \
 	DisassemblerTables/-gen-disassembler \
 	FastISel/-gen-fast-isel \
 	InstrInfo/-gen-instr-info \
 	MCCodeEmitter/-gen-emitter,-mc-emitter \
 	MCPseudoLowering/-gen-pseudo-lowering \
 	RegisterInfo/-gen-register-info \
 	SubtargetInfo/-gen-subtarget
 ${arch:T}Gen${hdr:H:C/$/.inc.h/}: ${LLVM_SRCS}/lib/Target/${arch:H}/${arch:T}.td
 	${TBLGEN} ${hdr:T:C/,/ /g} \
 	    -I ${LLVM_SRCS}/include -I ${LLVM_SRCS}/lib/Target/${arch:H} \
 	    -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} \
 	    ${LLVM_SRCS}/lib/Target/${arch:H}/${arch:T}.td
 . endfor
 .endfor
 
 Attrs.inc.h: ${CLANG_SRCS}/include/clang/Basic/Attr.td
 	${CLANG_TBLGEN} -gen-clang-attr-classes \
 	    -I ${CLANG_SRCS}/include -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} \
-	    ${.ALLSRC}
+	    ${CLANG_SRCS}/include/clang/Basic/Attr.td
 
 AttrDump.inc.h: ${CLANG_SRCS}/include/clang/Basic/Attr.td
 	${CLANG_TBLGEN} -gen-clang-attr-dump \
 	    -I ${CLANG_SRCS}/include -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} \
-	    ${.ALLSRC}
+	    ${CLANG_SRCS}/include/clang/Basic/Attr.td
 
 AttrIdentifierArg.inc.h: ${CLANG_SRCS}/include/clang/Basic/Attr.td
 	${CLANG_TBLGEN} -gen-clang-attr-identifier-arg-list \
 	    -I ${CLANG_SRCS}/include -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} \
-	    ${.ALLSRC}
+	    ${CLANG_SRCS}/include/clang/Basic/Attr.td
 
 AttrImpl.inc.h: ${CLANG_SRCS}/include/clang/Basic/Attr.td
 	${CLANG_TBLGEN} -gen-clang-attr-impl \
 	    -I ${CLANG_SRCS}/include -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} \
-	    ${.ALLSRC}
+	    ${CLANG_SRCS}/include/clang/Basic/Attr.td
 
 AttrLateParsed.inc.h: ${CLANG_SRCS}/include/clang/Basic/Attr.td
 	${CLANG_TBLGEN} -gen-clang-attr-late-parsed-list \
 	    -I ${CLANG_SRCS}/include -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} \
-	    ${.ALLSRC}
+	    ${CLANG_SRCS}/include/clang/Basic/Attr.td
 
 AttrList.inc.h: ${CLANG_SRCS}/include/clang/Basic/Attr.td
 	${CLANG_TBLGEN} -gen-clang-attr-list \
 	    -I ${CLANG_SRCS}/include -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} \
-	    ${.ALLSRC}
+	    ${CLANG_SRCS}/include/clang/Basic/Attr.td
 
 AttrParsedAttrImpl.inc.h: ${CLANG_SRCS}/include/clang/Basic/Attr.td
 	${CLANG_TBLGEN} -gen-clang-attr-parsed-attr-impl \
 	    -I ${CLANG_SRCS}/include -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} \
-	    ${.ALLSRC}
+	    ${CLANG_SRCS}/include/clang/Basic/Attr.td
 
 AttrParsedAttrKinds.inc.h: ${CLANG_SRCS}/include/clang/Basic/Attr.td
 	${CLANG_TBLGEN} -gen-clang-attr-parsed-attr-kinds \
 	    -I ${CLANG_SRCS}/include -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} \
-	    ${.ALLSRC}
+	    ${CLANG_SRCS}/include/clang/Basic/Attr.td
 
 AttrParsedAttrList.inc.h: ${CLANG_SRCS}/include/clang/Basic/Attr.td
 	${CLANG_TBLGEN} -gen-clang-attr-parsed-attr-list \
 	    -I ${CLANG_SRCS}/include -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} \
-	    ${.ALLSRC}
+	    ${CLANG_SRCS}/include/clang/Basic/Attr.td
 
 AttrPCHRead.inc.h: ${CLANG_SRCS}/include/clang/Basic/Attr.td
 	${CLANG_TBLGEN} -gen-clang-attr-pch-read \
 	    -I ${CLANG_SRCS}/include -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} \
-	    ${.ALLSRC}
+	    ${CLANG_SRCS}/include/clang/Basic/Attr.td
 
 AttrPCHWrite.inc.h: ${CLANG_SRCS}/include/clang/Basic/Attr.td
 	${CLANG_TBLGEN} -gen-clang-attr-pch-write \
 	    -I ${CLANG_SRCS}/include -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} \
-	    ${.ALLSRC}
+	    ${CLANG_SRCS}/include/clang/Basic/Attr.td
 
 AttrSpellings.inc.h: ${CLANG_SRCS}/include/clang/Basic/Attr.td
 	${CLANG_TBLGEN} -gen-clang-attr-spelling-list \
 	    -I ${CLANG_SRCS}/include -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} \
-	    ${.ALLSRC}
+	    ${CLANG_SRCS}/include/clang/Basic/Attr.td
 
 AttrSpellingListIndex.inc.h: ${CLANG_SRCS}/include/clang/Basic/Attr.td
 	${CLANG_TBLGEN} -gen-clang-attr-spelling-index \
 	    -I ${CLANG_SRCS}/include -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} \
-	    ${.ALLSRC}
+	    ${CLANG_SRCS}/include/clang/Basic/Attr.td
 
 AttrTemplateInstantiate.inc.h: ${CLANG_SRCS}/include/clang/Basic/Attr.td
 	${CLANG_TBLGEN} -gen-clang-attr-template-instantiate \
 	    -I ${CLANG_SRCS}/include -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} \
-	    ${.ALLSRC}
+	    ${CLANG_SRCS}/include/clang/Basic/Attr.td
 
 AttrTypeArg.inc.h: ${CLANG_SRCS}/include/clang/Basic/Attr.td
 	${CLANG_TBLGEN} -gen-clang-attr-type-arg-list \
 	    -I ${CLANG_SRCS}/include -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} \
-	    ${.ALLSRC}
+	    ${CLANG_SRCS}/include/clang/Basic/Attr.td
 
 CommentCommandInfo.inc.h: ${CLANG_SRCS}/include/clang/AST/CommentCommands.td
 	${CLANG_TBLGEN} -gen-clang-comment-command-info \
-	    -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} ${.ALLSRC}
+	    -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} \
+	    ${CLANG_SRCS}/include/clang/AST/CommentCommands.td
 
 CommentCommandList.inc.h: ${CLANG_SRCS}/include/clang/AST/CommentCommands.td
 	${CLANG_TBLGEN} -gen-clang-comment-command-list \
-	    -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} ${.ALLSRC}
+	    -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} \
+	    ${CLANG_SRCS}/include/clang/AST/CommentCommands.td
 
 CommentHTMLNamedCharacterReferences.inc.h: \
 	${CLANG_SRCS}/include/clang/AST/CommentHTMLNamedCharacterReferences.td
 	${CLANG_TBLGEN} -gen-clang-comment-html-named-character-references \
-	    -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} ${.ALLSRC}
+	    -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} \
+	    ${CLANG_SRCS}/include/clang/AST/CommentHTMLNamedCharacterReferences.td
 
 CommentHTMLTags.inc.h: ${CLANG_SRCS}/include/clang/AST/CommentHTMLTags.td
 	${CLANG_TBLGEN} -gen-clang-comment-html-tags \
-	    -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} ${.ALLSRC}
+	    -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} \
+	    ${CLANG_SRCS}/include/clang/AST/CommentHTMLTags.td
 
 CommentHTMLTagsProperties.inc.h: \
 	${CLANG_SRCS}/include/clang/AST/CommentHTMLTags.td
 	${CLANG_TBLGEN} -gen-clang-comment-html-tags-properties \
-	    -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} ${.ALLSRC}
+	    -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} \
+	    ${CLANG_SRCS}/include/clang/AST/CommentHTMLTags.td
 
 CommentNodes.inc.h: ${CLANG_SRCS}/include/clang/Basic/CommentNodes.td
 	${CLANG_TBLGEN} -gen-clang-comment-nodes \
-	    -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} ${.ALLSRC}
+	    -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} \
+	    ${CLANG_SRCS}/include/clang/Basic/CommentNodes.td
 
 DeclNodes.inc.h: ${CLANG_SRCS}/include/clang/Basic/DeclNodes.td
 	${CLANG_TBLGEN} -gen-clang-decl-nodes \
-	    -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} ${.ALLSRC}
+	    -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} \
+	    ${CLANG_SRCS}/include/clang/Basic/DeclNodes.td
 
 StmtNodes.inc.h: ${CLANG_SRCS}/include/clang/Basic/StmtNodes.td
 	${CLANG_TBLGEN} -gen-clang-stmt-nodes \
-	    -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} ${.ALLSRC}
+	    -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} \
+	    ${CLANG_SRCS}/include/clang/Basic/StmtNodes.td
 
 arm_neon.h: ${CLANG_SRCS}/include/clang/Basic/arm_neon.td
 	${CLANG_TBLGEN} -gen-arm-neon \
-	    -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} ${.ALLSRC}
+	    -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} \
+	    ${CLANG_SRCS}/include/clang/Basic/arm_neon.td
 
 arm_neon.inc.h: ${CLANG_SRCS}/include/clang/Basic/arm_neon.td
 	${CLANG_TBLGEN} -gen-arm-neon-sema \
-	    -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} ${.ALLSRC}
+	    -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} \
+	    ${CLANG_SRCS}/include/clang/Basic/arm_neon.td
 
 DiagnosticGroups.inc.h: ${CLANG_SRCS}/include/clang/Basic/Diagnostic.td
 	${CLANG_TBLGEN} -gen-clang-diag-groups \
 	    -I ${CLANG_SRCS}/include/clang/Basic -d ${.TARGET:C/\.h$/.d/} \
-	    -o ${.TARGET} ${.ALLSRC}
+	    -o ${.TARGET} ${CLANG_SRCS}/include/clang/Basic/Diagnostic.td
 
 DiagnosticIndexName.inc.h: ${CLANG_SRCS}/include/clang/Basic/Diagnostic.td
 	${CLANG_TBLGEN} -gen-clang-diags-index-name \
 	    -I ${CLANG_SRCS}/include/clang/Basic -d ${.TARGET:C/\.h$/.d/} \
-	    -o ${.TARGET} ${.ALLSRC}
+	    -o ${.TARGET} ${CLANG_SRCS}/include/clang/Basic/Diagnostic.td
 
 .for hdr in AST Analysis Comment Common Driver Frontend Lex Parse Sema Serialization
 Diagnostic${hdr}Kinds.inc.h: ${CLANG_SRCS}/include/clang/Basic/Diagnostic.td
 	${CLANG_TBLGEN} -gen-clang-diags-defs -clang-component=${hdr} \
 	    -I ${CLANG_SRCS}/include/clang/Basic -d ${.TARGET:C/\.h$/.d/} \
-	    -o ${.TARGET} ${.ALLSRC}
+	    -o ${.TARGET} ${CLANG_SRCS}/include/clang/Basic/Diagnostic.td
 .endfor
 
 Options.inc.h: ${CLANG_SRCS}/include/clang/Driver/Options.td
 	${TBLGEN} -gen-opt-parser-defs \
 	    -I ${LLVM_SRCS}/include -I ${CLANG_SRCS}/include/clang/Driver \
-	    -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} ${.ALLSRC}
+	    -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} \
+	    ${CLANG_SRCS}/include/clang/Driver/Options.td
 
 CC1AsOptions.inc.h: ${CLANG_SRCS}/include/clang/Driver/CC1AsOptions.td
 	${TBLGEN} -gen-opt-parser-defs \
 	    -I ${LLVM_SRCS}/include -I ${CLANG_SRCS}/include/clang/Driver \
-	    -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} ${.ALLSRC}
+	    -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} \
+	    ${CLANG_SRCS}/include/clang/Driver/CC1AsOptions.td
 
-Checkers.inc.h: ${CLANG_SRCS}/lib/StaticAnalyzer/Checkers/Checkers.td \
-	    ${CLANG_SRCS}/include/clang/StaticAnalyzer/Checkers/CheckerBase.td
+Checkers.inc.h: ${CLANG_SRCS}/lib/StaticAnalyzer/Checkers/Checkers.td
 	${CLANG_TBLGEN} -gen-clang-sa-checkers \
 	    -I ${CLANG_SRCS}/include -d ${.TARGET:C/\.h$/.d/} -o ${.TARGET} \
 	    ${CLANG_SRCS}/lib/StaticAnalyzer/Checkers/Checkers.td
 
 .for dep in ${TGHDRS:C/$/.inc.d/}
 . sinclude "${dep}"
 .endfor
 
 SRCS+=		${TGHDRS:C/$/.inc.h/}
 DPADD+=		${TGHDRS:C/$/.inc.h/}
 CLEANFILES+=	${TGHDRS:C/$/.inc.h/} ${TGHDRS:C/$/.inc.d/}
Index: projects/release-debugdist/release/Makefile
===================================================================
--- projects/release-debugdist/release/Makefile	(revision 267343)
+++ projects/release-debugdist/release/Makefile	(revision 267344)
@@ -1,286 +1,290 @@
 # $FreeBSD$
 #
 # Makefile for building releases and release media.
 #
 # User-driven targets:
 #  cdrom: Builds release CD-ROM media (disc1.iso)
 #  dvdrom: Builds release DVD-ROM media (dvd1.iso)
 #  memstick: Builds memory stick image (memstick.img)
 #  mini-memstick: Builds minimal memory stick image (mini-memstick.img)
 #  ftp: Sets up FTP distribution area (ftp)
 #  release: Build all media and FTP distribution area
 #  install: Copies all release media into ${DESTDIR}
 #
 # Variables affecting the build process:
 #  WORLDDIR: location of src tree -- must have built world and default kernel
 #            (by default, the directory above this one)
 #  PORTSDIR: location of ports tree to distribute (default: /usr/ports)
 #  DOCDIR:   location of doc tree (default: /usr/doc)
 #  NOPKG:    if set, do not distribute third-party packages
 #  NOPORTS:  if set, do not distribute ports tree
 #  NOSRC:    if set, do not distribute source tree
 #  NODOC:    if set, do not generate release documentation
 #  WITH_DVD: if set, generate dvd1.iso
 #  WITH_COMPRESSED_IMAGES: if set, compress installation images with xz(1)
 #		(uncompressed images are not removed)
 #  TARGET/TARGET_ARCH: architecture of built release
 #
 
 WORLDDIR?=	${.CURDIR}/..
 PORTSDIR?=	/usr/ports
 DOCDIR?=	/usr/doc
 RELNOTES_LANG?= en_US.ISO8859-1
 XZCMD?=		/usr/bin/xz
 
 .if !defined(TARGET) || empty(TARGET)
 TARGET=		${MACHINE}
 .endif
 .if !defined(TARGET_ARCH) || empty(TARGET_ARCH)
 .if ${TARGET} == ${MACHINE}
 TARGET_ARCH=	${MACHINE_ARCH}
 .else
 TARGET_ARCH=	${TARGET}
 .endif
 .endif
 IMAKE=		${MAKE} TARGET_ARCH=${TARGET_ARCH} TARGET=${TARGET}
 DISTDIR=	dist
 
 # Define OSRELEASE by using newvars.sh
 .if !defined(OSRELEASE) || empty(OSRELEASE)
 .for _V in TYPE BRANCH REVISION
 ${_V}!=	eval $$(awk '/^${_V}=/{print}' ${.CURDIR}/../sys/conf/newvers.sh); echo $$${_V}
 .endfor
 .for _V in ${TARGET_ARCH}
 .if !empty(TARGET:M${_V})
 OSRELEASE=	${TYPE}-${REVISION}-${BRANCH}-${TARGET}
 .else
 OSRELEASE=	${TYPE}-${REVISION}-${BRANCH}-${TARGET}-${TARGET_ARCH}
 .endif
 .endfor
 .endif
 
 .if !exists(${DOCDIR})
 NODOC= true
 .endif
 .if !exists(${PORTSDIR})
 NOPORTS= true
 .endif
 
 EXTRA_PACKAGES=
 .if !defined(NOPORTS)
 EXTRA_PACKAGES+= ports.txz
 .endif
 .if !defined(NOSRC)
 EXTRA_PACKAGES+= src.txz
 .endif
 .if !defined(NODOC)
 EXTRA_PACKAGES+= reldoc
 .endif
 
 RELEASE_TARGETS= ftp
 IMAGES=
 .if exists(${.CURDIR}/${TARGET}/mkisoimages.sh)
 RELEASE_TARGETS+= cdrom
 IMAGES+=	disc1.iso bootonly.iso
 . if defined(WITH_DVD) && !empty(WITH_DVD)
 RELEASE_TARGETS+= dvdrom
 IMAGES+=	dvd1.iso
 . endif
 .endif
 .if exists(${.CURDIR}/${TARGET}/make-memstick.sh)
 RELEASE_TARGETS+= memstick.img
 RELEASE_TARGETS+= mini-memstick.img
 IMAGES+=	memstick.img
 IMAGES+=	mini-memstick.img
 .endif
 
 CLEANFILES=	packagesystem *.txz MANIFEST system ${IMAGES}
 .if defined(WITH_COMPRESSED_IMAGES) && !empty(WITH_COMPRESSED_IMAGES)
 . for I in ${IMAGES}
 CLEANFILES+=	${I}.xz
 . endfor
 .endif
+.if defined(WITH_DVD) && !empty(WITH_DVD)
+CLEANFILES+=	pkg-stage
+.endif
 CLEANDIRS=	dist ftp release bootonly dvd
 beforeclean:
 	chflags -R noschg .
 .include <bsd.obj.mk>
 clean: beforeclean
 
 base.txz:
 	mkdir -p ${DISTDIR}
 	cd ${WORLDDIR} && ${IMAKE} distributeworld DISTDIR=${.OBJDIR}/${DISTDIR}
 # Set up mergemaster root database
 	sh ${.CURDIR}/scripts/mm-mtree.sh -m ${WORLDDIR} -F \
 	    "TARGET_ARCH=${TARGET_ARCH} TARGET=${TARGET}" -D "${.OBJDIR}/${DISTDIR}/base"
 	etcupdate extract -B -M "TARGET_ARCH=${TARGET_ARCH} TARGET=${TARGET}" \
 	    -s ${WORLDDIR} -d "${.OBJDIR}/${DISTDIR}/base/var/db/etcupdate"
 # Package all components
 	cd ${WORLDDIR} && ${IMAKE} packageworld DISTDIR=${.OBJDIR}/${DISTDIR}
 	mv ${DISTDIR}/*.txz .
 
 kernel.txz:
 	mkdir -p ${DISTDIR}
 	cd ${WORLDDIR} && ${IMAKE} distributekernel packagekernel DISTDIR=${.OBJDIR}/${DISTDIR}
 	mv ${DISTDIR}/kernel*.txz .
 
 src.txz:
 	mkdir -p ${DISTDIR}/usr
 	ln -fs ${WORLDDIR} ${DISTDIR}/usr/src
 	cd ${DISTDIR} && tar cLvJf ${.OBJDIR}/src.txz --exclude .svn --exclude .zfs \
 	    --exclude .git --exclude @ --exclude usr/src/release/dist usr/src
 
 ports.txz:
 	mkdir -p ${DISTDIR}/usr
 	ln -fs ${PORTSDIR} ${DISTDIR}/usr/ports
 	cd ${DISTDIR} && tar cLvJf ${.OBJDIR}/ports.txz \
 	    --exclude .git --exclude .svn \
 	    --exclude usr/ports/distfiles --exclude usr/ports/packages \
 	    --exclude 'usr/ports/INDEX*' --exclude work usr/ports
 
 reldoc:
 	cd ${.CURDIR}/doc && ${MAKE} all install clean 'FORMATS=html txt' \
 	    INSTALL_COMPRESSED='' URLS_ABSOLUTE=YES DOCDIR=${.OBJDIR}/rdoc
 	mkdir -p reldoc
 .for i in hardware readme relnotes errata
 	ln -f rdoc/${RELNOTES_LANG}/${i}/article.txt reldoc/${i:tu}.TXT
 	ln -f rdoc/${RELNOTES_LANG}/${i}/article.html reldoc/${i:tu}.HTM
 .endfor
 	cp rdoc/${RELNOTES_LANG}/readme/docbook.css reldoc
 
 system: packagesystem
 # Install system
 	mkdir -p release
 	cd ${WORLDDIR} && ${IMAKE} installkernel installworld distribution \
 		DESTDIR=${.OBJDIR}/release WITHOUT_RESCUE=1 WITHOUT_KERNEL_SYMBOLS=1 \
-		WITHOUT_PROFILE=1 WITHOUT_SENDMAIL=1 WITHOUT_ATF=1 WITHOUT_LIB32=1
+		WITHOUT_PROFILE=1 WITHOUT_SENDMAIL=1 MK_TESTS=no WITHOUT_LIB32=1
 # Copy distfiles
 	mkdir -p release/usr/freebsd-dist
 	for dist in MANIFEST $$(ls *.txz | grep -v 'debug'); \
 	    do cp $${dist} release/usr/freebsd-dist; \
 	done
 # Copy documentation, if generated
 .if !defined(NODOC)
 	cp reldoc/* release
 .endif
 # Set up installation environment
 	ln -fs /tmp/bsdinstall_etc/resolv.conf release/etc/resolv.conf
 	echo sendmail_enable=\"NONE\" > release/etc/rc.conf
 	echo hostid_enable=\"NO\" >> release/etc/rc.conf
 	echo debug.witness.trace=0 >> release/etc/sysctl.conf
 	cp ${.CURDIR}/rc.local release/etc
 	touch ${.TARGET}
 # Apply a hammer to remove *.debug files, if they exist.
 .if defined(WITH_DEBUG_FILES)
 	find ${.OBJDIR}/release -mindepth 1 -name \*.debug -delete
 .endif
 
 bootonly: packagesystem
 # Install system
 	mkdir -p bootonly
 	cd ${WORLDDIR} && ${IMAKE} installkernel installworld distribution \
 	    DESTDIR=${.OBJDIR}/bootonly WITHOUT_AMD=1 WITHOUT_AT=1 \
 	    WITHOUT_GAMES=1 WITHOUT_GROFF=1 \
 	    WITHOUT_INSTALLLIB=1 WITHOUT_LIB32=1 WITHOUT_MAIL=1 \
 	    WITHOUT_NCP=1 WITHOUT_TOOLCHAIN=1 WITHOUT_PROFILE=1 \
 	    WITHOUT_INSTALLIB=1 WITHOUT_RESCUE=1 WITHOUT_DICT=1 \
-	    WITHOUT_KERNEL_SYMBOLS=1
+	    WITHOUT_KERNEL_SYMBOLS=1 MK_TESTS=no
 # Copy manifest only (no distfiles) to get checksums
 	mkdir -p bootonly/usr/freebsd-dist
 	cp MANIFEST bootonly/usr/freebsd-dist
 # Copy documentation, if generated
 .if !defined(NODOC)
 	cp reldoc/* bootonly
 .endif
 # Set up installation environment
 	ln -fs /tmp/bsdinstall_etc/resolv.conf bootonly/etc/resolv.conf
 	echo sendmail_enable=\"NONE\" > bootonly/etc/rc.conf
 	echo hostid_enable=\"NO\" >> bootonly/etc/rc.conf
 	echo debug.witness.trace=0 >> bootonly/etc/sysctl.conf
 	cp ${.CURDIR}/rc.local bootonly/etc
 # Apply a hammer to remove *.debug files, if they exist.
 .if defined(WITH_DEBUG_FILES)
 	find ${.OBJDIR}/${.TARGET} -mindepth 1 -name \*.debug -delete
 .endif
 
 dvd:
 # Install system
 	mkdir -p ${.TARGET}
 	cd ${WORLDDIR} && ${IMAKE} installkernel installworld distribution \
-		DESTDIR=${.OBJDIR}/${.TARGET} WITHOUT_RESCUE=1 WITHOUT_KERNEL_SYMBOLS=1
+		DESTDIR=${.OBJDIR}/${.TARGET} WITHOUT_RESCUE=1 WITHOUT_KERNEL_SYMBOLS=1 \
+		MK_TESTS=no
 # Copy distfiles
 	mkdir -p ${.TARGET}/usr/freebsd-dist
 	for dist in MANIFEST $$(ls *.txz | grep -v 'debug'); \
 	    do cp $${dist} ${.TARGET}/usr/freebsd-dist; \
 	done
 # Copy documentation, if generated
 .if !defined(NODOC)
 	cp reldoc/* ${.TARGET}
 .endif
 # Set up installation environment
 	ln -fs /tmp/bsdinstall_etc/resolv.conf ${.TARGET}/etc/resolv.conf
 	echo sendmail_enable=\"NONE\" > ${.TARGET}/etc/rc.conf
 	echo hostid_enable=\"NO\" >> ${.TARGET}/etc/rc.conf
 	echo debug.witness.trace=0 >> ${.TARGET}/etc/sysctl.conf
 	cp ${.CURDIR}/rc.local ${.TARGET}/etc
 	touch ${.TARGET}
 # Apply a hammer to remove *.debug files, if they exist.
 .if defined(WITH_DEBUG_FILES)
 	find ${.OBJDIR}/${.TARGET} -mindepth 1 -name \*.debug -delete
 .endif
 
 release.iso: disc1.iso
 disc1.iso: system
 	sh ${.CURDIR}/${TARGET}/mkisoimages.sh -b FreeBSD_Install ${.TARGET} release
 
 dvd1.iso: dvd pkg-stage
 	sh ${.CURDIR}/${TARGET}/mkisoimages.sh -b FreeBSD_Install ${.TARGET} dvd
 
 bootonly.iso: bootonly
 	sh ${.CURDIR}/${TARGET}/mkisoimages.sh -b FreeBSD_Install ${.TARGET} bootonly
 
 memstick: memstick.img
 memstick.img: system
 	sh ${.CURDIR}/${TARGET}/make-memstick.sh release ${.TARGET}
 
 mini-memstick: mini-memstick.img
 mini-memstick.img: system
 	sh ${.CURDIR}/${TARGET}/make-memstick.sh bootonly ${.TARGET}
 
 packagesystem: base.txz kernel.txz ${EXTRA_PACKAGES}
 	sh ${.CURDIR}/scripts/make-manifest.sh *.txz > MANIFEST
 	touch ${.TARGET}
 
 pkg-stage:
 .if !defined(NOPKG)
 	env REPOS_DIR=${.CURDIR}/pkg_repos/ \
 		sh ${.CURDIR}/scripts/pkg-stage.sh
 	mkdir -p ${.OBJDIR}/dvd/packages/repos/
 	cp ${.CURDIR}/scripts/FreeBSD_install_cdrom.conf \
 		${.OBJDIR}/dvd/packages/repos/
 .endif
 	touch ${.TARGET}
 
 cdrom: disc1.iso bootonly.iso
 dvdrom: dvd1.iso
 ftp: packagesystem
 	rm -rf ftp
 	mkdir -p ftp
 	cp *.txz MANIFEST ftp
 
 release:
 	${MAKE} -C ${.CURDIR} ${.MAKEFLAGS} obj
 	${MAKE} -C ${.CURDIR} ${.MAKEFLAGS} ${RELEASE_TARGETS}
 
 install:
 .if defined(DESTDIR) && !empty(DESTDIR)
 	mkdir -p ${DESTDIR}
 .endif
 	cp -a ftp ${DESTDIR}/
 .for I in ${IMAGES}
 	cp -p ${I} ${DESTDIR}/${OSRELEASE}-${I}
 . if defined(WITH_COMPRESSED_IMAGES) && !empty(WITH_COMPRESSED_IMAGES)
 	${XZCMD} -k ${DESTDIR}/${OSRELEASE}-${I}
 . endif
 .endfor
 	cd ${DESTDIR} && sha256 ${OSRELEASE}* > ${DESTDIR}/CHECKSUM.SHA256
 	cd ${DESTDIR} && md5 ${OSRELEASE}* > ${DESTDIR}/CHECKSUM.MD5
Index: projects/release-debugdist/share/mk/bsd.subdir.mk
===================================================================
--- projects/release-debugdist/share/mk/bsd.subdir.mk	(revision 267343)
+++ projects/release-debugdist/share/mk/bsd.subdir.mk	(revision 267344)
@@ -1,130 +1,132 @@
 #	from: @(#)bsd.subdir.mk	5.9 (Berkeley) 2/1/91
 # $FreeBSD$
 #
 # The include file <bsd.subdir.mk> contains the default targets
 # for building subdirectories.
 #
 # For all of the directories listed in the variable SUBDIRS, the
 # specified directory will be visited and the target made. There is
 # also a default target which allows the command "make subdir" where
 # subdir is any directory listed in the variable SUBDIRS.
 #
 #
 # +++ variables +++
 #
 # DISTRIBUTION	Name of distribution. [base]
 #
 # SUBDIR	A list of subdirectories that should be built as well.
 #		Each of the targets will execute the same target in the
 #		subdirectories.
 #
 # +++ targets +++
 #
 #	distribute:
 # 		This is a variant of install, which will
 # 		put the stuff into the right "distribution".
 #
 #	afterinstall, all, all-man, beforeinstall, checkdpadd, clean,
 #	cleandepend, cleandir, cleanilinks depend, install, lint,
 #	maninstall, manlint, obj, objlink, realinstall, regress, tags
 #
 
 .if !target(__<bsd.subdir.mk>__)
 __<bsd.subdir.mk>__:
 
 .include <bsd.init.mk>
 
 DISTRIBUTION?=	base
 .if !target(distribute)
 distribute: .MAKE
 .for dist in ${DISTRIBUTION}
 	${_+_}cd ${.CURDIR}; \
 	    ${MAKE} install -DNO_SUBDIR DESTDIR=${DISTDIR}/${dist} SHARED=copies
 .endfor
 .endif
 
 _SUBDIR: .USE .MAKE
 .if defined(SUBDIR) && !empty(SUBDIR) && !defined(NO_SUBDIR)
 	@${_+_}set -e; for entry in ${SUBDIR:N.WAIT}; do \
 		if test -d ${.CURDIR}/$${entry}.${MACHINE_ARCH}; then \
 			${ECHODIR} "===> ${DIRPRFX}$${entry}.${MACHINE_ARCH} (${.TARGET:S,realinstall,install,:S,^_sub.,,})"; \
 			edir=$${entry}.${MACHINE_ARCH}; \
 			cd ${.CURDIR}/$${edir}; \
 		else \
 			${ECHODIR} "===> ${DIRPRFX}$$entry (${.TARGET:S,realinstall,install,:S,^_sub.,,})"; \
 			edir=$${entry}; \
 			cd ${.CURDIR}/$${edir}; \
 		fi; \
 		${MAKE} ${.TARGET:S,realinstall,install,:S,^_sub.,,} \
 		    DIRPRFX=${DIRPRFX}$$edir/; \
 	done
 .endif
 
 ${SUBDIR:N.WAIT}: .PHONY .MAKE
 	${_+_}@if test -d ${.TARGET}.${MACHINE_ARCH}; then \
 		cd ${.CURDIR}/${.TARGET}.${MACHINE_ARCH}; \
 	else \
 		cd ${.CURDIR}/${.TARGET}; \
 	fi; \
 	${MAKE} all
 
 # Work around parsing of .if nested in .for by putting .WAIT string into a var.
 __wait= .WAIT
 .for __target in all all-man checkdpadd clean cleandepend cleandir \
     cleanilinks depend distribute lint maninstall manlint obj objlink \
     realinstall regress tags ${SUBDIR_TARGETS}
 .ifdef SUBDIR_PARALLEL
 __subdir_targets=
 .for __dir in ${SUBDIR}
 .if ${__wait} == ${__dir}
 __subdir_targets+= .WAIT
 .else
 __subdir_targets+= ${__target}_subdir_${__dir}
 ${__target}_subdir_${__dir}: .MAKE
+.if !defined(NO_SUBDIR)
 	@${_+_}set -e; \
 		if test -d ${.CURDIR}/${__dir}.${MACHINE_ARCH}; then \
 			${ECHODIR} "===> ${DIRPRFX}${__dir}.${MACHINE_ARCH} (${__target:realinstall=install})"; \
 			edir=${__dir}.${MACHINE_ARCH}; \
 			cd ${.CURDIR}/$${edir}; \
 		else \
 			${ECHODIR} "===> ${DIRPRFX}${__dir} (${__target:realinstall=install})"; \
 			edir=${__dir}; \
 			cd ${.CURDIR}/$${edir}; \
 		fi; \
 		${MAKE} ${__target:realinstall=install} \
 		    DIRPRFX=${DIRPRFX}$$edir/
+.endif
 .endif
 .endfor
 ${__target}: ${__subdir_targets}
 .else
 ${__target}: _sub.${__target}
 _sub.${__target}: _SUBDIR
 .endif
 .endfor
 
 .for __target in files includes
 .for __stage in build install
 ${__stage}${__target}:
 .if make(${__stage}${__target})
 ${__stage}${__target}: _sub.${__stage}${__target}
 _sub.${__stage}${__target}: _SUBDIR
 .endif
 .endfor
 .if !target(${__target})
 ${__target}: .MAKE
 	${_+_}set -e; cd ${.CURDIR}; ${MAKE} build${__target}; ${MAKE} install${__target}
 .endif
 .endfor
 
 .if !target(install)
 .if !target(beforeinstall)
 beforeinstall:
 .endif
 .if !target(afterinstall)
 afterinstall:
 .endif
 install: beforeinstall realinstall afterinstall
 .ORDER: beforeinstall realinstall afterinstall
 .endif
 
 .endif
Index: projects/release-debugdist/share/mk/bsd.test.mk
===================================================================
--- projects/release-debugdist/share/mk/bsd.test.mk	(revision 267343)
+++ projects/release-debugdist/share/mk/bsd.test.mk	(revision 267344)
@@ -1,88 +1,97 @@
 # $FreeBSD$
 #
 # Generic build infrastructure for test programs.
 #
 # This is the only public file that should be included by Makefiles when
 # tests are to be built.  All other *.test.mk files are internal and not
 # to be included directly.
 
 .include <bsd.init.mk>
 
 __<bsd.test.mk>__:
 
 # List of subdirectories containing tests into which to recurse.  This has the
 # same semantics as SUBDIR at build-time.  However, the directories listed here
 # get registered into the run-time test suite definitions so that the test
 # engines know to recurse into these directories.
 #
 # In other words: list here any directories that contain test programs but use
 # SUBDIR for directories that may contain helper binaries and/or data files.
 TESTS_SUBDIRS?=
 
 # If defined, indicates that the tests built by the Makefile are not part of
 # the FreeBSD Test Suite.  The implication of this is that the tests won't be
 # installed under /usr/tests/ and that Kyua won't be able to run them.
 #NOT_FOR_TEST_SUITE=
 
 # List of variables to pass to the tests at run-time via the environment.
 TESTS_ENV?=
 
+# Force all tests in a separate distribution file.
+#
+# We want this to be the case even when the distribution name is already
+# overriden.  For example: we want the tests for programs in the 'games'
+# distribution to end up in the 'tests' distribution; the test programs
+# themselves have all the necessary logic to detect that the games are not
+# installed and thus won't cause false negatives.
+DISTRIBUTION:=	tests
+
 # Ordered list of directories to construct the PATH for the tests.
 TESTS_PATH+= ${DESTDIR}/bin ${DESTDIR}/sbin \
              ${DESTDIR}/usr/bin ${DESTDIR}/usr/sbin
 TESTS_ENV+= PATH=${TESTS_PATH:tW:C/ +/:/g}
 
 # Ordered list of directories to construct the LD_LIBRARY_PATH for the tests.
 TESTS_LD_LIBRARY_PATH+= ${DESTDIR}/lib ${DESTDIR}/usr/lib
 TESTS_ENV+= LD_LIBRARY_PATH=${TESTS_LD_LIBRARY_PATH:tW:C/ +/:/g}
 
 # List of all tests being built.  The various *.test.mk modules extend this
 # variable as needed.
 _TESTS=
 
 # Pull in the definitions of all supported test interfaces.
 .include <atf.test.mk>
 .include <plain.test.mk>
 .include <tap.test.mk>
 
 .if !empty(TESTS_SUBDIRS)
 SUBDIR+= ${TESTS_SUBDIRS}
 .endif
 
 # it is rare for test cases to have man pages
 .if !defined(MAN)
 MAN=
 .endif
 
 # tell progs.mk we might want to install things
 PROG_VARS+= BINDIR
 PROGS_TARGETS+= install
 
 .if !defined(NOT_FOR_TEST_SUITE)
 .include <suite.test.mk>
 .endif
 
 .if !target(realtest)
 realtest: .PHONY
 	@echo "$@ not defined; skipping"
 .endif
 
 test: .PHONY
 .ORDER: beforetest realtest
 test: beforetest realtest
 
 .if target(aftertest)
 .ORDER: realtest aftertest
 test: aftertest
 .endif
 
 .if !empty(SUBDIR)
 .include <bsd.subdir.mk>
 .endif
 
 .if !empty(PROGS) || !empty(PROGS_CXX) || !empty(SCRIPTS)
 .include <bsd.progs.mk>
 .endif
 .include <bsd.files.mk>
 
 .include <bsd.obj.mk>
Index: projects/release-debugdist/share
===================================================================
--- projects/release-debugdist/share	(revision 267343)
+++ projects/release-debugdist/share	(revision 267344)

Property changes on: projects/release-debugdist/share
___________________________________________________________________
Modified: svn:mergeinfo
## -0,0 +0,1 ##
   Merged /head/share:r267325-267343
Index: projects/release-debugdist/sys/amd64/include/vmm.h
===================================================================
--- projects/release-debugdist/sys/amd64/include/vmm.h	(revision 267343)
+++ projects/release-debugdist/sys/amd64/include/vmm.h	(revision 267344)
@@ -1,494 +1,493 @@
 /*-
  * Copyright (c) 2011 NetApp, Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
  * $FreeBSD$
  */
 
 #ifndef _VMM_H_
 #define	_VMM_H_
 
 enum vm_suspend_how {
 	VM_SUSPEND_NONE,
 	VM_SUSPEND_RESET,
 	VM_SUSPEND_POWEROFF,
 	VM_SUSPEND_HALT,
 	VM_SUSPEND_LAST
 };
 
+/*
+ * Identifiers for architecturally defined registers.
+ */
+enum vm_reg_name {
+	VM_REG_GUEST_RAX,
+	VM_REG_GUEST_RBX,
+	VM_REG_GUEST_RCX,
+	VM_REG_GUEST_RDX,
+	VM_REG_GUEST_RSI,
+	VM_REG_GUEST_RDI,
+	VM_REG_GUEST_RBP,
+	VM_REG_GUEST_R8,
+	VM_REG_GUEST_R9,
+	VM_REG_GUEST_R10,
+	VM_REG_GUEST_R11,
+	VM_REG_GUEST_R12,
+	VM_REG_GUEST_R13,
+	VM_REG_GUEST_R14,
+	VM_REG_GUEST_R15,
+	VM_REG_GUEST_CR0,
+	VM_REG_GUEST_CR3,
+	VM_REG_GUEST_CR4,
+	VM_REG_GUEST_DR7,
+	VM_REG_GUEST_RSP,
+	VM_REG_GUEST_RIP,
+	VM_REG_GUEST_RFLAGS,
+	VM_REG_GUEST_ES,
+	VM_REG_GUEST_CS,
+	VM_REG_GUEST_SS,
+	VM_REG_GUEST_DS,
+	VM_REG_GUEST_FS,
+	VM_REG_GUEST_GS,
+	VM_REG_GUEST_LDTR,
+	VM_REG_GUEST_TR,
+	VM_REG_GUEST_IDTR,
+	VM_REG_GUEST_GDTR,
+	VM_REG_GUEST_EFER,
+	VM_REG_GUEST_CR2,
+	VM_REG_LAST
+};
+
+enum x2apic_state {
+	X2APIC_DISABLED,
+	X2APIC_ENABLED,
+	X2APIC_STATE_LAST
+};
+
 #ifdef _KERNEL
 
 #define	VM_MAX_NAMELEN	32
 
 struct vm;
 struct vm_exception;
 struct vm_memory_segment;
 struct seg_desc;
 struct vm_exit;
 struct vm_run;
 struct vhpet;
 struct vioapic;
 struct vlapic;
 struct vmspace;
 struct vm_object;
 struct pmap;
 
-enum vm_reg_name;
-enum x2apic_state;
-
 typedef int	(*vmm_init_func_t)(int ipinum);
 typedef int	(*vmm_cleanup_func_t)(void);
 typedef void	(*vmm_resume_func_t)(void);
 typedef void *	(*vmi_init_func_t)(struct vm *vm, struct pmap *pmap);
 typedef int	(*vmi_run_func_t)(void *vmi, int vcpu, register_t rip,
 				  struct pmap *pmap, void *rendezvous_cookie,
 				  void *suspend_cookie);
 typedef void	(*vmi_cleanup_func_t)(void *vmi);
 typedef int	(*vmi_get_register_t)(void *vmi, int vcpu, int num,
 				      uint64_t *retval);
 typedef int	(*vmi_set_register_t)(void *vmi, int vcpu, int num,
 				      uint64_t val);
 typedef int	(*vmi_get_desc_t)(void *vmi, int vcpu, int num,
 				  struct seg_desc *desc);
 typedef int	(*vmi_set_desc_t)(void *vmi, int vcpu, int num,
 				  struct seg_desc *desc);
 typedef int	(*vmi_get_cap_t)(void *vmi, int vcpu, int num, int *retval);
 typedef int	(*vmi_set_cap_t)(void *vmi, int vcpu, int num, int val);
 typedef struct vmspace * (*vmi_vmspace_alloc)(vm_offset_t min, vm_offset_t max);
 typedef void	(*vmi_vmspace_free)(struct vmspace *vmspace);
 typedef struct vlapic * (*vmi_vlapic_init)(void *vmi, int vcpu);
 typedef void	(*vmi_vlapic_cleanup)(void *vmi, struct vlapic *vlapic);
 
 struct vmm_ops {
 	vmm_init_func_t		init;		/* module wide initialization */
 	vmm_cleanup_func_t	cleanup;
 	vmm_resume_func_t	resume;
 
 	vmi_init_func_t		vminit;		/* vm-specific initialization */
 	vmi_run_func_t		vmrun;
 	vmi_cleanup_func_t	vmcleanup;
 	vmi_get_register_t	vmgetreg;
 	vmi_set_register_t	vmsetreg;
 	vmi_get_desc_t		vmgetdesc;
 	vmi_set_desc_t		vmsetdesc;
 	vmi_get_cap_t		vmgetcap;
 	vmi_set_cap_t		vmsetcap;
 	vmi_vmspace_alloc	vmspace_alloc;
 	vmi_vmspace_free	vmspace_free;
 	vmi_vlapic_init		vlapic_init;
 	vmi_vlapic_cleanup	vlapic_cleanup;
 };
 
 extern struct vmm_ops vmm_ops_intel;
 extern struct vmm_ops vmm_ops_amd;
 
 int vm_create(const char *name, struct vm **retvm);
 void vm_destroy(struct vm *vm);
 int vm_reinit(struct vm *vm);
 const char *vm_name(struct vm *vm);
 int vm_malloc(struct vm *vm, vm_paddr_t gpa, size_t len);
 int vm_map_mmio(struct vm *vm, vm_paddr_t gpa, size_t len, vm_paddr_t hpa);
 int vm_unmap_mmio(struct vm *vm, vm_paddr_t gpa, size_t len);
 void *vm_gpa_hold(struct vm *, vm_paddr_t gpa, size_t len, int prot,
 		  void **cookie);
 void vm_gpa_release(void *cookie);
 int vm_gpabase2memseg(struct vm *vm, vm_paddr_t gpabase,
 	      struct vm_memory_segment *seg);
 int vm_get_memobj(struct vm *vm, vm_paddr_t gpa, size_t len,
 		  vm_offset_t *offset, struct vm_object **object);
 boolean_t vm_mem_allocated(struct vm *vm, vm_paddr_t gpa);
 int vm_get_register(struct vm *vm, int vcpu, int reg, uint64_t *retval);
 int vm_set_register(struct vm *vm, int vcpu, int reg, uint64_t val);
 int vm_get_seg_desc(struct vm *vm, int vcpu, int reg,
 		    struct seg_desc *ret_desc);
 int vm_set_seg_desc(struct vm *vm, int vcpu, int reg,
 		    struct seg_desc *desc);
 int vm_run(struct vm *vm, struct vm_run *vmrun);
 int vm_suspend(struct vm *vm, enum vm_suspend_how how);
 int vm_inject_nmi(struct vm *vm, int vcpu);
 int vm_nmi_pending(struct vm *vm, int vcpuid);
 void vm_nmi_clear(struct vm *vm, int vcpuid);
 int vm_inject_extint(struct vm *vm, int vcpu);
 int vm_extint_pending(struct vm *vm, int vcpuid);
 void vm_extint_clear(struct vm *vm, int vcpuid);
 uint64_t *vm_guest_msrs(struct vm *vm, int cpu);
 struct vlapic *vm_lapic(struct vm *vm, int cpu);
 struct vioapic *vm_ioapic(struct vm *vm);
 struct vhpet *vm_hpet(struct vm *vm);
 int vm_get_capability(struct vm *vm, int vcpu, int type, int *val);
 int vm_set_capability(struct vm *vm, int vcpu, int type, int val);
 int vm_get_x2apic_state(struct vm *vm, int vcpu, enum x2apic_state *state);
 int vm_set_x2apic_state(struct vm *vm, int vcpu, enum x2apic_state state);
 int vm_apicid2vcpuid(struct vm *vm, int apicid);
 int vm_activate_cpu(struct vm *vm, int vcpu);
 cpuset_t vm_active_cpus(struct vm *vm);
 cpuset_t vm_suspended_cpus(struct vm *vm);
 struct vm_exit *vm_exitinfo(struct vm *vm, int vcpuid);
 void vm_exit_suspended(struct vm *vm, int vcpuid, uint64_t rip);
+void vm_exit_rendezvous(struct vm *vm, int vcpuid, uint64_t rip);
+void vm_exit_astpending(struct vm *vm, int vcpuid, uint64_t rip);
 
 /*
  * Rendezvous all vcpus specified in 'dest' and execute 'func(arg)'.
  * The rendezvous 'func(arg)' is not allowed to do anything that will
  * cause the thread to be put to sleep.
  *
  * If the rendezvous is being initiated from a vcpu context then the
  * 'vcpuid' must refer to that vcpu, otherwise it should be set to -1.
  *
  * The caller cannot hold any locks when initiating the rendezvous.
  *
  * The implementation of this API may cause vcpus other than those specified
  * by 'dest' to be stalled. The caller should not rely on any vcpus making
  * forward progress when the rendezvous is in progress.
  */
 typedef void (*vm_rendezvous_func_t)(struct vm *vm, int vcpuid, void *arg);
 void vm_smp_rendezvous(struct vm *vm, int vcpuid, cpuset_t dest,
     vm_rendezvous_func_t func, void *arg);
 
 static __inline int
 vcpu_rendezvous_pending(void *rendezvous_cookie)
 {
 
 	return (*(uintptr_t *)rendezvous_cookie != 0);
 }
 
 static __inline int
 vcpu_suspended(void *suspend_cookie)
 {
 
 	return (*(int *)suspend_cookie);
 }
 
 /*
  * Return 1 if device indicated by bus/slot/func is supposed to be a
  * pci passthrough device.
  *
  * Return 0 otherwise.
  */
 int vmm_is_pptdev(int bus, int slot, int func);
 
 void *vm_iommu_domain(struct vm *vm);
 
 enum vcpu_state {
 	VCPU_IDLE,
 	VCPU_FROZEN,
 	VCPU_RUNNING,
 	VCPU_SLEEPING,
 };
 
 int vcpu_set_state(struct vm *vm, int vcpu, enum vcpu_state state,
     bool from_idle);
 enum vcpu_state vcpu_get_state(struct vm *vm, int vcpu, int *hostcpu);
 
 static int __inline
 vcpu_is_running(struct vm *vm, int vcpu, int *hostcpu)
 {
 	return (vcpu_get_state(vm, vcpu, hostcpu) == VCPU_RUNNING);
 }
 
 void *vcpu_stats(struct vm *vm, int vcpu);
 void vcpu_notify_event(struct vm *vm, int vcpuid, bool lapic_intr);
 struct vmspace *vm_get_vmspace(struct vm *vm);
 int vm_assign_pptdev(struct vm *vm, int bus, int slot, int func);
 int vm_unassign_pptdev(struct vm *vm, int bus, int slot, int func);
 struct vatpic *vm_atpic(struct vm *vm);
 struct vatpit *vm_atpit(struct vm *vm);
 
 /*
  * Inject exception 'vme' into the guest vcpu. This function returns 0 on
  * success and non-zero on failure.
  *
  * Wrapper functions like 'vm_inject_gp()' should be preferred to calling
  * this function directly because they enforce the trap-like or fault-like
  * behavior of an exception.
  *
  * This function should only be called in the context of the thread that is
  * executing this vcpu.
  */
 int vm_inject_exception(struct vm *vm, int vcpuid, struct vm_exception *vme);
 
 /*
  * Returns 0 if there is no exception pending for this vcpu. Returns 1 if an
  * exception is pending and also updates 'vme'. The pending exception is
  * cleared when this function returns.
  *
  * This function should only be called in the context of the thread that is
  * executing this vcpu.
  */
 int vm_exception_pending(struct vm *vm, int vcpuid, struct vm_exception *vme);
 
 void vm_inject_gp(struct vm *vm, int vcpuid); /* general protection fault */
 void vm_inject_ud(struct vm *vm, int vcpuid); /* undefined instruction fault */
 void vm_inject_pf(struct vm *vm, int vcpuid, int error_code, uint64_t cr2);
 
 enum vm_reg_name vm_segment_name(int seg_encoding);
 
 #endif	/* KERNEL */
 
 #define	VM_MAXCPU	16			/* maximum virtual cpus */
 
 /*
- * Identifiers for architecturally defined registers.
- */
-enum vm_reg_name {
-	VM_REG_GUEST_RAX,
-	VM_REG_GUEST_RBX,
-	VM_REG_GUEST_RCX,
-	VM_REG_GUEST_RDX,
-	VM_REG_GUEST_RSI,
-	VM_REG_GUEST_RDI,
-	VM_REG_GUEST_RBP,
-	VM_REG_GUEST_R8,
-	VM_REG_GUEST_R9,
-	VM_REG_GUEST_R10,
-	VM_REG_GUEST_R11,
-	VM_REG_GUEST_R12,
-	VM_REG_GUEST_R13,
-	VM_REG_GUEST_R14,
-	VM_REG_GUEST_R15,
-	VM_REG_GUEST_CR0,
-	VM_REG_GUEST_CR3,
-	VM_REG_GUEST_CR4,
-	VM_REG_GUEST_DR7,
-	VM_REG_GUEST_RSP,
-	VM_REG_GUEST_RIP,
-	VM_REG_GUEST_RFLAGS,
-	VM_REG_GUEST_ES,
-	VM_REG_GUEST_CS,
-	VM_REG_GUEST_SS,
-	VM_REG_GUEST_DS,
-	VM_REG_GUEST_FS,
-	VM_REG_GUEST_GS,
-	VM_REG_GUEST_LDTR,
-	VM_REG_GUEST_TR,
-	VM_REG_GUEST_IDTR,
-	VM_REG_GUEST_GDTR,
-	VM_REG_GUEST_EFER,
-	VM_REG_GUEST_CR2,
-	VM_REG_LAST
-};
-
-/*
  * Identifiers for optional vmm capabilities
  */
 enum vm_cap_type {
 	VM_CAP_HALT_EXIT,
 	VM_CAP_MTRAP_EXIT,
 	VM_CAP_PAUSE_EXIT,
 	VM_CAP_UNRESTRICTED_GUEST,
 	VM_CAP_ENABLE_INVPCID,
 	VM_CAP_MAX
-};
-
-enum x2apic_state {
-	X2APIC_DISABLED,
-	X2APIC_ENABLED,
-	X2APIC_STATE_LAST
 };
 
 enum vm_intr_trigger {
 	EDGE_TRIGGER,
 	LEVEL_TRIGGER
 };
 	
 /*
  * The 'access' field has the format specified in Table 21-2 of the Intel
  * Architecture Manual vol 3b.
  *
  * XXX The contents of the 'access' field are architecturally defined except
  * bit 16 - Segment Unusable.
  */
 struct seg_desc {
 	uint64_t	base;
 	uint32_t	limit;
 	uint32_t	access;
 };
 #define	SEG_DESC_TYPE(desc)		((desc)->access & 0x001f)
 #define	SEG_DESC_PRESENT(desc)		((desc)->access & 0x0080)
 #define	SEG_DESC_DEF32(desc)		((desc)->access & 0x4000)
 #define	SEG_DESC_GRANULARITY(desc)	((desc)->access & 0x8000)
 #define	SEG_DESC_UNUSABLE(desc)		((desc)->access & 0x10000)
 
 enum vm_cpu_mode {
 	CPU_MODE_COMPATIBILITY,		/* IA-32E mode (CS.L = 0) */
 	CPU_MODE_64BIT,			/* IA-32E mode (CS.L = 1) */
 };
 
 enum vm_paging_mode {
 	PAGING_MODE_FLAT,
 	PAGING_MODE_32,
 	PAGING_MODE_PAE,
 	PAGING_MODE_64,
 };
 
 struct vm_guest_paging {
 	uint64_t	cr3;
 	int		cpl;
 	enum vm_cpu_mode cpu_mode;
 	enum vm_paging_mode paging_mode;
 };
 
 /*
  * The data structures 'vie' and 'vie_op' are meant to be opaque to the
  * consumers of instruction decoding. The only reason why their contents
  * need to be exposed is because they are part of the 'vm_exit' structure.
  */
 struct vie_op {
 	uint8_t		op_byte;	/* actual opcode byte */
 	uint8_t		op_type;	/* type of operation (e.g. MOV) */
 	uint16_t	op_flags;
 };
 
 #define	VIE_INST_SIZE	15
 struct vie {
 	uint8_t		inst[VIE_INST_SIZE];	/* instruction bytes */
 	uint8_t		num_valid;		/* size of the instruction */
 	uint8_t		num_processed;
 
 	uint8_t		rex_w:1,		/* REX prefix */
 			rex_r:1,
 			rex_x:1,
 			rex_b:1,
 			rex_present:1;
 
 	uint8_t		mod:2,			/* ModRM byte */
 			reg:4,
 			rm:4;
 
 	uint8_t		ss:2,			/* SIB byte */
 			index:4,
 			base:4;
 
 	uint8_t		disp_bytes;
 	uint8_t		imm_bytes;
 
 	uint8_t		scale;
 	int		base_register;		/* VM_REG_GUEST_xyz */
 	int		index_register;		/* VM_REG_GUEST_xyz */
 
 	int64_t		displacement;		/* optional addr displacement */
 	int64_t		immediate;		/* optional immediate operand */
 
 	uint8_t		decoded;	/* set to 1 if successfully decoded */
 
 	struct vie_op	op;			/* opcode description */
 };
 
 enum vm_exitcode {
 	VM_EXITCODE_INOUT,
 	VM_EXITCODE_VMX,
 	VM_EXITCODE_BOGUS,
 	VM_EXITCODE_RDMSR,
 	VM_EXITCODE_WRMSR,
 	VM_EXITCODE_HLT,
 	VM_EXITCODE_MTRAP,
 	VM_EXITCODE_PAUSE,
 	VM_EXITCODE_PAGING,
 	VM_EXITCODE_INST_EMUL,
 	VM_EXITCODE_SPINUP_AP,
 	VM_EXITCODE_DEPRECATED1,	/* used to be SPINDOWN_CPU */
 	VM_EXITCODE_RENDEZVOUS,
 	VM_EXITCODE_IOAPIC_EOI,
 	VM_EXITCODE_SUSPENDED,
 	VM_EXITCODE_INOUT_STR,
 	VM_EXITCODE_MAX
 };
 
 struct vm_inout {
 	uint16_t	bytes:3;	/* 1 or 2 or 4 */
 	uint16_t	in:1;
 	uint16_t	string:1;
 	uint16_t	rep:1;
 	uint16_t	port;
 	uint32_t	eax;		/* valid for out */
 };
 
 struct vm_inout_str {
 	struct vm_inout	inout;		/* must be the first element */
 	struct vm_guest_paging paging;
 	uint64_t	rflags;
 	uint64_t	cr0;
 	uint64_t	index;
 	uint64_t	count;		/* rep=1 (%rcx), rep=0 (1) */
 	int		addrsize;
 	enum vm_reg_name seg_name;
 	struct seg_desc seg_desc;
 };
 
 struct vm_exit {
 	enum vm_exitcode	exitcode;
 	int			inst_length;	/* 0 means unknown */
 	uint64_t		rip;
 	union {
 		struct vm_inout	inout;
 		struct vm_inout_str inout_str;
 		struct {
 			uint64_t	gpa;
 			int		fault_type;
 		} paging;
 		struct {
 			uint64_t	gpa;
 			uint64_t	gla;
 			struct vm_guest_paging paging;
 			struct vie	vie;
 		} inst_emul;
 		/*
 		 * VMX specific payload. Used when there is no "better"
 		 * exitcode to represent the VM-exit.
 		 */
 		struct {
 			int		status;		/* vmx inst status */
 			/*
 			 * 'exit_reason' and 'exit_qualification' are valid
 			 * only if 'status' is zero.
 			 */
 			uint32_t	exit_reason;
 			uint64_t	exit_qualification;
 			/*
 			 * 'inst_error' and 'inst_type' are valid
 			 * only if 'status' is non-zero.
 			 */
 			int		inst_type;
 			int		inst_error;
 		} vmx;
 		struct {
 			uint32_t	code;		/* ecx value */
 			uint64_t	wval;
 		} msr;
 		struct {
 			int		vcpu;
 			uint64_t	rip;
 		} spinup_ap;
 		struct {
 			uint64_t	rflags;
 		} hlt;
 		struct {
 			int		vector;
 		} ioapic_eoi;
 		struct {
 			enum vm_suspend_how how;
 		} suspended;
 	} u;
 };
 
 #endif	/* _VMM_H_ */
Index: projects/release-debugdist/sys/amd64/vmm/intel/vmx.c
===================================================================
--- projects/release-debugdist/sys/amd64/vmm/intel/vmx.c	(revision 267343)
+++ projects/release-debugdist/sys/amd64/vmm/intel/vmx.c	(revision 267344)
@@ -1,3134 +1,3110 @@
 /*-
  * Copyright (c) 2011 NetApp, Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
  * $FreeBSD$
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/smp.h>
 #include <sys/kernel.h>
 #include <sys/malloc.h>
 #include <sys/pcpu.h>
 #include <sys/proc.h>
 #include <sys/sysctl.h>
 
 #include <vm/vm.h>
 #include <vm/pmap.h>
 
 #include <machine/psl.h>
 #include <machine/cpufunc.h>
 #include <machine/md_var.h>
 #include <machine/segments.h>
 #include <machine/smp.h>
 #include <machine/specialreg.h>
 #include <machine/vmparam.h>
 
 #include <machine/vmm.h>
 #include <machine/vmm_dev.h>
 #include <machine/vmm_instruction_emul.h>
 #include "vmm_host.h"
 #include "vmm_ioport.h"
 #include "vmm_ipi.h"
 #include "vmm_msr.h"
 #include "vmm_ktr.h"
 #include "vmm_stat.h"
 #include "vatpic.h"
 #include "vlapic.h"
 #include "vlapic_priv.h"
 
 #include "vmx_msr.h"
 #include "ept.h"
 #include "vmx_cpufunc.h"
 #include "vmx.h"
 #include "x86.h"
 #include "vmx_controls.h"
 
 #define	PINBASED_CTLS_ONE_SETTING					\
 	(PINBASED_EXTINT_EXITING	|				\
 	 PINBASED_NMI_EXITING		|				\
 	 PINBASED_VIRTUAL_NMI)
 #define	PINBASED_CTLS_ZERO_SETTING	0
 
 #define PROCBASED_CTLS_WINDOW_SETTING					\
 	(PROCBASED_INT_WINDOW_EXITING	|				\
 	 PROCBASED_NMI_WINDOW_EXITING)
 
 #define	PROCBASED_CTLS_ONE_SETTING 					\
 	(PROCBASED_SECONDARY_CONTROLS	|				\
 	 PROCBASED_IO_EXITING		|				\
 	 PROCBASED_MSR_BITMAPS		|				\
 	 PROCBASED_CTLS_WINDOW_SETTING	|				\
 	 PROCBASED_CR8_LOAD_EXITING	|				\
 	 PROCBASED_CR8_STORE_EXITING)
 #define	PROCBASED_CTLS_ZERO_SETTING	\
 	(PROCBASED_CR3_LOAD_EXITING |	\
 	PROCBASED_CR3_STORE_EXITING |	\
 	PROCBASED_IO_BITMAPS)
 
 #define	PROCBASED_CTLS2_ONE_SETTING	PROCBASED2_ENABLE_EPT
 #define	PROCBASED_CTLS2_ZERO_SETTING	0
 
 #define VM_EXIT_CTLS_ONE_SETTING_NO_PAT					\
 	(VM_EXIT_HOST_LMA			|			\
 	VM_EXIT_SAVE_EFER			|			\
 	VM_EXIT_LOAD_EFER)
 
 #define	VM_EXIT_CTLS_ONE_SETTING					\
 	(VM_EXIT_CTLS_ONE_SETTING_NO_PAT       	|			\
 	VM_EXIT_ACKNOWLEDGE_INTERRUPT		|			\
 	VM_EXIT_SAVE_PAT			|			\
 	VM_EXIT_LOAD_PAT)
 #define	VM_EXIT_CTLS_ZERO_SETTING	VM_EXIT_SAVE_DEBUG_CONTROLS
 
 #define	VM_ENTRY_CTLS_ONE_SETTING_NO_PAT	VM_ENTRY_LOAD_EFER
 
 #define	VM_ENTRY_CTLS_ONE_SETTING					\
 	(VM_ENTRY_CTLS_ONE_SETTING_NO_PAT     	|			\
 	VM_ENTRY_LOAD_PAT)
 #define	VM_ENTRY_CTLS_ZERO_SETTING					\
 	(VM_ENTRY_LOAD_DEBUG_CONTROLS		|			\
 	VM_ENTRY_INTO_SMM			|			\
 	VM_ENTRY_DEACTIVATE_DUAL_MONITOR)
 
 #define	guest_msr_rw(vmx, msr) \
 	msr_bitmap_change_access((vmx)->msr_bitmap, (msr), MSR_BITMAP_ACCESS_RW)
 
 #define	guest_msr_ro(vmx, msr) \
     msr_bitmap_change_access((vmx)->msr_bitmap, (msr), MSR_BITMAP_ACCESS_READ)
 
 #define	HANDLED		1
 #define	UNHANDLED	0
 
 static MALLOC_DEFINE(M_VMX, "vmx", "vmx");
 static MALLOC_DEFINE(M_VLAPIC, "vlapic", "vlapic");
 
 SYSCTL_DECL(_hw_vmm);
 SYSCTL_NODE(_hw_vmm, OID_AUTO, vmx, CTLFLAG_RW, NULL, NULL);
 
 int vmxon_enabled[MAXCPU];
 static char vmxon_region[MAXCPU][PAGE_SIZE] __aligned(PAGE_SIZE);
 
 static uint32_t pinbased_ctls, procbased_ctls, procbased_ctls2;
 static uint32_t exit_ctls, entry_ctls;
 
 static uint64_t cr0_ones_mask, cr0_zeros_mask;
 SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_ones_mask, CTLFLAG_RD,
 	     &cr0_ones_mask, 0, NULL);
 SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_zeros_mask, CTLFLAG_RD,
 	     &cr0_zeros_mask, 0, NULL);
 
 static uint64_t cr4_ones_mask, cr4_zeros_mask;
 SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_ones_mask, CTLFLAG_RD,
 	     &cr4_ones_mask, 0, NULL);
 SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_zeros_mask, CTLFLAG_RD,
 	     &cr4_zeros_mask, 0, NULL);
 
 static int vmx_no_patmsr;
 
 static int vmx_initialized;
 SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, initialized, CTLFLAG_RD,
 	   &vmx_initialized, 0, "Intel VMX initialized");
 
 /*
  * Optional capabilities
  */
 static int cap_halt_exit;
 static int cap_pause_exit;
 static int cap_unrestricted_guest;
 static int cap_monitor_trap;
 static int cap_invpcid;
 
 static int virtual_interrupt_delivery;
 SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, virtual_interrupt_delivery, CTLFLAG_RD,
     &virtual_interrupt_delivery, 0, "APICv virtual interrupt delivery support");
 
 static int posted_interrupts;
 SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, posted_interrupts, CTLFLAG_RD,
     &posted_interrupts, 0, "APICv posted interrupt support");
 
 static int pirvec;
 SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, posted_interrupt_vector, CTLFLAG_RD,
     &pirvec, 0, "APICv posted interrupt vector");
 
 static struct unrhdr *vpid_unr;
 static u_int vpid_alloc_failed;
 SYSCTL_UINT(_hw_vmm_vmx, OID_AUTO, vpid_alloc_failed, CTLFLAG_RD,
 	    &vpid_alloc_failed, 0, NULL);
 
 /*
  * Use the last page below 4GB as the APIC access address. This address is
  * occupied by the boot firmware so it is guaranteed that it will not conflict
  * with a page in system memory.
  */
 #define	APIC_ACCESS_ADDRESS	0xFFFFF000
 
 static int vmx_getdesc(void *arg, int vcpu, int reg, struct seg_desc *desc);
 static int vmx_getreg(void *arg, int vcpu, int reg, uint64_t *retval);
 static void vmx_inject_pir(struct vlapic *vlapic);
 
 #ifdef KTR
 static const char *
 exit_reason_to_str(int reason)
 {
 	static char reasonbuf[32];
 
 	switch (reason) {
 	case EXIT_REASON_EXCEPTION:
 		return "exception";
 	case EXIT_REASON_EXT_INTR:
 		return "extint";
 	case EXIT_REASON_TRIPLE_FAULT:
 		return "triplefault";
 	case EXIT_REASON_INIT:
 		return "init";
 	case EXIT_REASON_SIPI:
 		return "sipi";
 	case EXIT_REASON_IO_SMI:
 		return "iosmi";
 	case EXIT_REASON_SMI:
 		return "smi";
 	case EXIT_REASON_INTR_WINDOW:
 		return "intrwindow";
 	case EXIT_REASON_NMI_WINDOW:
 		return "nmiwindow";
 	case EXIT_REASON_TASK_SWITCH:
 		return "taskswitch";
 	case EXIT_REASON_CPUID:
 		return "cpuid";
 	case EXIT_REASON_GETSEC:
 		return "getsec";
 	case EXIT_REASON_HLT:
 		return "hlt";
 	case EXIT_REASON_INVD:
 		return "invd";
 	case EXIT_REASON_INVLPG:
 		return "invlpg";
 	case EXIT_REASON_RDPMC:
 		return "rdpmc";
 	case EXIT_REASON_RDTSC:
 		return "rdtsc";
 	case EXIT_REASON_RSM:
 		return "rsm";
 	case EXIT_REASON_VMCALL:
 		return "vmcall";
 	case EXIT_REASON_VMCLEAR:
 		return "vmclear";
 	case EXIT_REASON_VMLAUNCH:
 		return "vmlaunch";
 	case EXIT_REASON_VMPTRLD:
 		return "vmptrld";
 	case EXIT_REASON_VMPTRST:
 		return "vmptrst";
 	case EXIT_REASON_VMREAD:
 		return "vmread";
 	case EXIT_REASON_VMRESUME:
 		return "vmresume";
 	case EXIT_REASON_VMWRITE:
 		return "vmwrite";
 	case EXIT_REASON_VMXOFF:
 		return "vmxoff";
 	case EXIT_REASON_VMXON:
 		return "vmxon";
 	case EXIT_REASON_CR_ACCESS:
 		return "craccess";
 	case EXIT_REASON_DR_ACCESS:
 		return "draccess";
 	case EXIT_REASON_INOUT:
 		return "inout";
 	case EXIT_REASON_RDMSR:
 		return "rdmsr";
 	case EXIT_REASON_WRMSR:
 		return "wrmsr";
 	case EXIT_REASON_INVAL_VMCS:
 		return "invalvmcs";
 	case EXIT_REASON_INVAL_MSR:
 		return "invalmsr";
 	case EXIT_REASON_MWAIT:
 		return "mwait";
 	case EXIT_REASON_MTF:
 		return "mtf";
 	case EXIT_REASON_MONITOR:
 		return "monitor";
 	case EXIT_REASON_PAUSE:
 		return "pause";
 	case EXIT_REASON_MCE:
 		return "mce";
 	case EXIT_REASON_TPR:
 		return "tpr";
 	case EXIT_REASON_APIC_ACCESS:
 		return "apic-access";
 	case EXIT_REASON_GDTR_IDTR:
 		return "gdtridtr";
 	case EXIT_REASON_LDTR_TR:
 		return "ldtrtr";
 	case EXIT_REASON_EPT_FAULT:
 		return "eptfault";
 	case EXIT_REASON_EPT_MISCONFIG:
 		return "eptmisconfig";
 	case EXIT_REASON_INVEPT:
 		return "invept";
 	case EXIT_REASON_RDTSCP:
 		return "rdtscp";
 	case EXIT_REASON_VMX_PREEMPT:
 		return "vmxpreempt";
 	case EXIT_REASON_INVVPID:
 		return "invvpid";
 	case EXIT_REASON_WBINVD:
 		return "wbinvd";
 	case EXIT_REASON_XSETBV:
 		return "xsetbv";
 	case EXIT_REASON_APIC_WRITE:
 		return "apic-write";
 	default:
 		snprintf(reasonbuf, sizeof(reasonbuf), "%d", reason);
 		return (reasonbuf);
 	}
 }
 #endif	/* KTR */
 
 static int
 vmx_allow_x2apic_msrs(struct vmx *vmx)
 {
 	int i, error;
 
 	error = 0;
 
 	/*
 	 * Allow readonly access to the following x2APIC MSRs from the guest.
 	 */
 	error += guest_msr_ro(vmx, MSR_APIC_ID);
 	error += guest_msr_ro(vmx, MSR_APIC_VERSION);
 	error += guest_msr_ro(vmx, MSR_APIC_LDR);
 	error += guest_msr_ro(vmx, MSR_APIC_SVR);
 
 	for (i = 0; i < 8; i++)
 		error += guest_msr_ro(vmx, MSR_APIC_ISR0 + i);
 
 	for (i = 0; i < 8; i++)
 		error += guest_msr_ro(vmx, MSR_APIC_TMR0 + i);
 	
 	for (i = 0; i < 8; i++)
 		error += guest_msr_ro(vmx, MSR_APIC_IRR0 + i);
 
 	error += guest_msr_ro(vmx, MSR_APIC_ESR);
 	error += guest_msr_ro(vmx, MSR_APIC_LVT_TIMER);
 	error += guest_msr_ro(vmx, MSR_APIC_LVT_THERMAL);
 	error += guest_msr_ro(vmx, MSR_APIC_LVT_PCINT);
 	error += guest_msr_ro(vmx, MSR_APIC_LVT_LINT0);
 	error += guest_msr_ro(vmx, MSR_APIC_LVT_LINT1);
 	error += guest_msr_ro(vmx, MSR_APIC_LVT_ERROR);
 	error += guest_msr_ro(vmx, MSR_APIC_ICR_TIMER);
 	error += guest_msr_ro(vmx, MSR_APIC_DCR_TIMER);
 	error += guest_msr_ro(vmx, MSR_APIC_ICR);
 
 	/*
 	 * Allow TPR, EOI and SELF_IPI MSRs to be read and written by the guest.
 	 *
 	 * These registers get special treatment described in the section
 	 * "Virtualizing MSR-Based APIC Accesses".
 	 */
 	error += guest_msr_rw(vmx, MSR_APIC_TPR);
 	error += guest_msr_rw(vmx, MSR_APIC_EOI);
 	error += guest_msr_rw(vmx, MSR_APIC_SELF_IPI);
 
 	return (error);
 }
 
 u_long
 vmx_fix_cr0(u_long cr0)
 {
 
 	return ((cr0 | cr0_ones_mask) & ~cr0_zeros_mask);
 }
 
 u_long
 vmx_fix_cr4(u_long cr4)
 {
 
 	return ((cr4 | cr4_ones_mask) & ~cr4_zeros_mask);
 }
 
 static void
 vpid_free(int vpid)
 {
 	if (vpid < 0 || vpid > 0xffff)
 		panic("vpid_free: invalid vpid %d", vpid);
 
 	/*
 	 * VPIDs [0,VM_MAXCPU] are special and are not allocated from
 	 * the unit number allocator.
 	 */
 
 	if (vpid > VM_MAXCPU)
 		free_unr(vpid_unr, vpid);
 }
 
 static void
 vpid_alloc(uint16_t *vpid, int num)
 {
 	int i, x;
 
 	if (num <= 0 || num > VM_MAXCPU)
 		panic("invalid number of vpids requested: %d", num);
 
 	/*
 	 * If the "enable vpid" execution control is not enabled then the
 	 * VPID is required to be 0 for all vcpus.
 	 */
 	if ((procbased_ctls2 & PROCBASED2_ENABLE_VPID) == 0) {
 		for (i = 0; i < num; i++)
 			vpid[i] = 0;
 		return;
 	}
 
 	/*
 	 * Allocate a unique VPID for each vcpu from the unit number allocator.
 	 */
 	for (i = 0; i < num; i++) {
 		x = alloc_unr(vpid_unr);
 		if (x == -1)
 			break;
 		else
 			vpid[i] = x;
 	}
 
 	if (i < num) {
 		atomic_add_int(&vpid_alloc_failed, 1);
 
 		/*
 		 * If the unit number allocator does not have enough unique
 		 * VPIDs then we need to allocate from the [1,VM_MAXCPU] range.
 		 *
 		 * These VPIDs are not be unique across VMs but this does not
 		 * affect correctness because the combined mappings are also
 		 * tagged with the EP4TA which is unique for each VM.
 		 *
 		 * It is still sub-optimal because the invvpid will invalidate
 		 * combined mappings for a particular VPID across all EP4TAs.
 		 */
 		while (i-- > 0)
 			vpid_free(vpid[i]);
 
 		for (i = 0; i < num; i++)
 			vpid[i] = i + 1;
 	}
 }
 
 static void
 vpid_init(void)
 {
 	/*
 	 * VPID 0 is required when the "enable VPID" execution control is
 	 * disabled.
 	 *
 	 * VPIDs [1,VM_MAXCPU] are used as the "overflow namespace" when the
 	 * unit number allocator does not have sufficient unique VPIDs to
 	 * satisfy the allocation.
 	 *
 	 * The remaining VPIDs are managed by the unit number allocator.
 	 */
 	vpid_unr = new_unrhdr(VM_MAXCPU + 1, 0xffff, NULL);
 }
 
 static void
 msr_save_area_init(struct msr_entry *g_area, int *g_count)
 {
 	int cnt;
 
 	static struct msr_entry guest_msrs[] = {
 		{ MSR_KGSBASE, 0, 0 },
 	};
 
 	cnt = sizeof(guest_msrs) / sizeof(guest_msrs[0]);
 	if (cnt > GUEST_MSR_MAX_ENTRIES)
 		panic("guest msr save area overrun");
 	bcopy(guest_msrs, g_area, sizeof(guest_msrs));
 	*g_count = cnt;
 }
 
 static void
 vmx_disable(void *arg __unused)
 {
 	struct invvpid_desc invvpid_desc = { 0 };
 	struct invept_desc invept_desc = { 0 };
 
 	if (vmxon_enabled[curcpu]) {
 		/*
 		 * See sections 25.3.3.3 and 25.3.3.4 in Intel Vol 3b.
 		 *
 		 * VMXON or VMXOFF are not required to invalidate any TLB
 		 * caching structures. This prevents potential retention of
 		 * cached information in the TLB between distinct VMX episodes.
 		 */
 		invvpid(INVVPID_TYPE_ALL_CONTEXTS, invvpid_desc);
 		invept(INVEPT_TYPE_ALL_CONTEXTS, invept_desc);
 		vmxoff();
 	}
 	load_cr4(rcr4() & ~CR4_VMXE);
 }
 
 static int
 vmx_cleanup(void)
 {
 	
 	if (pirvec != 0)
 		vmm_ipi_free(pirvec);
 
 	if (vpid_unr != NULL) {
 		delete_unrhdr(vpid_unr);
 		vpid_unr = NULL;
 	}
 
 	smp_rendezvous(NULL, vmx_disable, NULL, NULL);
 
 	return (0);
 }
 
 static void
 vmx_enable(void *arg __unused)
 {
 	int error;
 	uint64_t feature_control;
 
 	feature_control = rdmsr(MSR_IA32_FEATURE_CONTROL);
 	if ((feature_control & IA32_FEATURE_CONTROL_LOCK) == 0 ||
 	    (feature_control & IA32_FEATURE_CONTROL_VMX_EN) == 0) {
 		wrmsr(MSR_IA32_FEATURE_CONTROL,
 		    feature_control | IA32_FEATURE_CONTROL_VMX_EN |
 		    IA32_FEATURE_CONTROL_LOCK);
 	}
 
 	load_cr4(rcr4() | CR4_VMXE);
 
 	*(uint32_t *)vmxon_region[curcpu] = vmx_revision();
 	error = vmxon(vmxon_region[curcpu]);
 	if (error == 0)
 		vmxon_enabled[curcpu] = 1;
 }
 
 static void
 vmx_restore(void)
 {
 
 	if (vmxon_enabled[curcpu])
 		vmxon(vmxon_region[curcpu]);
 }
 
 static int
 vmx_init(int ipinum)
 {
 	int error, use_tpr_shadow;
 	uint64_t basic, fixed0, fixed1, feature_control;
 	uint32_t tmp, procbased2_vid_bits;
 
 	/* CPUID.1:ECX[bit 5] must be 1 for processor to support VMX */
 	if (!(cpu_feature2 & CPUID2_VMX)) {
 		printf("vmx_init: processor does not support VMX operation\n");
 		return (ENXIO);
 	}
 
 	/*
 	 * Verify that MSR_IA32_FEATURE_CONTROL lock and VMXON enable bits
 	 * are set (bits 0 and 2 respectively).
 	 */
 	feature_control = rdmsr(MSR_IA32_FEATURE_CONTROL);
 	if ((feature_control & IA32_FEATURE_CONTROL_LOCK) == 1 &&
 	    (feature_control & IA32_FEATURE_CONTROL_VMX_EN) == 0) {
 		printf("vmx_init: VMX operation disabled by BIOS\n");
 		return (ENXIO);
 	}
 
 	/*
 	 * Verify capabilities MSR_VMX_BASIC:
 	 * - bit 54 indicates support for INS/OUTS decoding
 	 */
 	basic = rdmsr(MSR_VMX_BASIC);
 	if ((basic & (1UL << 54)) == 0) {
 		printf("vmx_init: processor does not support desired basic "
 		    "capabilities\n");
 		return (EINVAL);
 	}
 
 	/* Check support for primary processor-based VM-execution controls */
 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
 			       MSR_VMX_TRUE_PROCBASED_CTLS,
 			       PROCBASED_CTLS_ONE_SETTING,
 			       PROCBASED_CTLS_ZERO_SETTING, &procbased_ctls);
 	if (error) {
 		printf("vmx_init: processor does not support desired primary "
 		       "processor-based controls\n");
 		return (error);
 	}
 
 	/* Clear the processor-based ctl bits that are set on demand */
 	procbased_ctls &= ~PROCBASED_CTLS_WINDOW_SETTING;
 
 	/* Check support for secondary processor-based VM-execution controls */
 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
 			       MSR_VMX_PROCBASED_CTLS2,
 			       PROCBASED_CTLS2_ONE_SETTING,
 			       PROCBASED_CTLS2_ZERO_SETTING, &procbased_ctls2);
 	if (error) {
 		printf("vmx_init: processor does not support desired secondary "
 		       "processor-based controls\n");
 		return (error);
 	}
 
 	/* Check support for VPID */
 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2,
 			       PROCBASED2_ENABLE_VPID, 0, &tmp);
 	if (error == 0)
 		procbased_ctls2 |= PROCBASED2_ENABLE_VPID;
 
 	/* Check support for pin-based VM-execution controls */
 	error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS,
 			       MSR_VMX_TRUE_PINBASED_CTLS,
 			       PINBASED_CTLS_ONE_SETTING,
 			       PINBASED_CTLS_ZERO_SETTING, &pinbased_ctls);
 	if (error) {
 		printf("vmx_init: processor does not support desired "
 		       "pin-based controls\n");
 		return (error);
 	}
 
 	/* Check support for VM-exit controls */
 	error = vmx_set_ctlreg(MSR_VMX_EXIT_CTLS, MSR_VMX_TRUE_EXIT_CTLS,
 			       VM_EXIT_CTLS_ONE_SETTING,
 			       VM_EXIT_CTLS_ZERO_SETTING,
 			       &exit_ctls);
 	if (error) {
 		/* Try again without the PAT MSR bits */
 		error = vmx_set_ctlreg(MSR_VMX_EXIT_CTLS,
 				       MSR_VMX_TRUE_EXIT_CTLS,
 				       VM_EXIT_CTLS_ONE_SETTING_NO_PAT,
 				       VM_EXIT_CTLS_ZERO_SETTING,
 				       &exit_ctls);
 		if (error) {
 			printf("vmx_init: processor does not support desired "
 			       "exit controls\n");
 			return (error);
 		} else {
 			if (bootverbose)
 				printf("vmm: PAT MSR access not supported\n");
 			guest_msr_valid(MSR_PAT);
 			vmx_no_patmsr = 1;
 		}
 	}
 
 	/* Check support for VM-entry controls */
 	if (!vmx_no_patmsr) {
 		error = vmx_set_ctlreg(MSR_VMX_ENTRY_CTLS,
 				       MSR_VMX_TRUE_ENTRY_CTLS,
 				       VM_ENTRY_CTLS_ONE_SETTING,
 				       VM_ENTRY_CTLS_ZERO_SETTING,
 				       &entry_ctls);
 	} else {
 		error = vmx_set_ctlreg(MSR_VMX_ENTRY_CTLS,
 				       MSR_VMX_TRUE_ENTRY_CTLS,
 				       VM_ENTRY_CTLS_ONE_SETTING_NO_PAT,
 				       VM_ENTRY_CTLS_ZERO_SETTING,
 				       &entry_ctls);
 	}
 
 	if (error) {
 		printf("vmx_init: processor does not support desired "
 		       "entry controls\n");
 		       return (error);
 	}
 
 	/*
 	 * Check support for optional features by testing them
 	 * as individual bits
 	 */
 	cap_halt_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
 					MSR_VMX_TRUE_PROCBASED_CTLS,
 					PROCBASED_HLT_EXITING, 0,
 					&tmp) == 0);
 
 	cap_monitor_trap = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
 					MSR_VMX_PROCBASED_CTLS,
 					PROCBASED_MTF, 0,
 					&tmp) == 0);
 
 	cap_pause_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
 					 MSR_VMX_TRUE_PROCBASED_CTLS,
 					 PROCBASED_PAUSE_EXITING, 0,
 					 &tmp) == 0);
 
 	cap_unrestricted_guest = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
 					MSR_VMX_PROCBASED_CTLS2,
 					PROCBASED2_UNRESTRICTED_GUEST, 0,
 				        &tmp) == 0);
 
 	cap_invpcid = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2,
 	    MSR_VMX_PROCBASED_CTLS2, PROCBASED2_ENABLE_INVPCID, 0,
 	    &tmp) == 0);
 
 	/*
 	 * Check support for virtual interrupt delivery.
 	 */
 	procbased2_vid_bits = (PROCBASED2_VIRTUALIZE_APIC_ACCESSES |
 	    PROCBASED2_VIRTUALIZE_X2APIC_MODE |
 	    PROCBASED2_APIC_REGISTER_VIRTUALIZATION |
 	    PROCBASED2_VIRTUAL_INTERRUPT_DELIVERY);
 
 	use_tpr_shadow = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS,
 	    MSR_VMX_TRUE_PROCBASED_CTLS, PROCBASED_USE_TPR_SHADOW, 0,
 	    &tmp) == 0);
 
 	error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2,
 	    procbased2_vid_bits, 0, &tmp);
 	if (error == 0 && use_tpr_shadow) {
 		virtual_interrupt_delivery = 1;
 		TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_vid",
 		    &virtual_interrupt_delivery);
 	}
 
 	if (virtual_interrupt_delivery) {
 		procbased_ctls |= PROCBASED_USE_TPR_SHADOW;
 		procbased_ctls2 |= procbased2_vid_bits;
 		procbased_ctls2 &= ~PROCBASED2_VIRTUALIZE_X2APIC_MODE;
 
 		/*
 		 * No need to emulate accesses to %CR8 if virtual
 		 * interrupt delivery is enabled.
 		 */
 		procbased_ctls &= ~PROCBASED_CR8_LOAD_EXITING;
 		procbased_ctls &= ~PROCBASED_CR8_STORE_EXITING;
 
 		/*
 		 * Check for Posted Interrupts only if Virtual Interrupt
 		 * Delivery is enabled.
 		 */
 		error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS,
 		    MSR_VMX_TRUE_PINBASED_CTLS, PINBASED_POSTED_INTERRUPT, 0,
 		    &tmp);
 		if (error == 0) {
 			pirvec = vmm_ipi_alloc();
 			if (pirvec == 0) {
 				if (bootverbose) {
 					printf("vmx_init: unable to allocate "
 					    "posted interrupt vector\n");
 				}
 			} else {
 				posted_interrupts = 1;
 				TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_pir",
 				    &posted_interrupts);
 			}
 		}
 	}
 
 	if (posted_interrupts)
 		    pinbased_ctls |= PINBASED_POSTED_INTERRUPT;
 
 	/* Initialize EPT */
 	error = ept_init(ipinum);
 	if (error) {
 		printf("vmx_init: ept initialization failed (%d)\n", error);
 		return (error);
 	}
 
 	/*
 	 * Stash the cr0 and cr4 bits that must be fixed to 0 or 1
 	 */
 	fixed0 = rdmsr(MSR_VMX_CR0_FIXED0);
 	fixed1 = rdmsr(MSR_VMX_CR0_FIXED1);
 	cr0_ones_mask = fixed0 & fixed1;
 	cr0_zeros_mask = ~fixed0 & ~fixed1;
 
 	/*
 	 * CR0_PE and CR0_PG can be set to zero in VMX non-root operation
 	 * if unrestricted guest execution is allowed.
 	 */
 	if (cap_unrestricted_guest)
 		cr0_ones_mask &= ~(CR0_PG | CR0_PE);
 
 	/*
 	 * Do not allow the guest to set CR0_NW or CR0_CD.
 	 */
 	cr0_zeros_mask |= (CR0_NW | CR0_CD);
 
 	fixed0 = rdmsr(MSR_VMX_CR4_FIXED0);
 	fixed1 = rdmsr(MSR_VMX_CR4_FIXED1);
 	cr4_ones_mask = fixed0 & fixed1;
 	cr4_zeros_mask = ~fixed0 & ~fixed1;
 
 	vpid_init();
 
 	/* enable VMX operation */
 	smp_rendezvous(NULL, vmx_enable, NULL, NULL);
 
 	vmx_initialized = 1;
 
 	return (0);
 }
 
 static void
 vmx_trigger_hostintr(int vector)
 {
 	uintptr_t func;
 	struct gate_descriptor *gd;
 
 	gd = &idt[vector];
 
 	KASSERT(vector >= 32 && vector <= 255, ("vmx_trigger_hostintr: "
 	    "invalid vector %d", vector));
 	KASSERT(gd->gd_p == 1, ("gate descriptor for vector %d not present",
 	    vector));
 	KASSERT(gd->gd_type == SDT_SYSIGT, ("gate descriptor for vector %d "
 	    "has invalid type %d", vector, gd->gd_type));
 	KASSERT(gd->gd_dpl == SEL_KPL, ("gate descriptor for vector %d "
 	    "has invalid dpl %d", vector, gd->gd_dpl));
 	KASSERT(gd->gd_selector == GSEL(GCODE_SEL, SEL_KPL), ("gate descriptor "
 	    "for vector %d has invalid selector %d", vector, gd->gd_selector));
 	KASSERT(gd->gd_ist == 0, ("gate descriptor for vector %d has invalid "
 	    "IST %d", vector, gd->gd_ist));
 
 	func = ((long)gd->gd_hioffset << 16 | gd->gd_looffset);
 	vmx_call_isr(func);
 }
 
 static int
 vmx_setup_cr_shadow(int which, struct vmcs *vmcs, uint32_t initial)
 {
 	int error, mask_ident, shadow_ident;
 	uint64_t mask_value;
 
 	if (which != 0 && which != 4)
 		panic("vmx_setup_cr_shadow: unknown cr%d", which);
 
 	if (which == 0) {
 		mask_ident = VMCS_CR0_MASK;
 		mask_value = cr0_ones_mask | cr0_zeros_mask;
 		shadow_ident = VMCS_CR0_SHADOW;
 	} else {
 		mask_ident = VMCS_CR4_MASK;
 		mask_value = cr4_ones_mask | cr4_zeros_mask;
 		shadow_ident = VMCS_CR4_SHADOW;
 	}
 
 	error = vmcs_setreg(vmcs, 0, VMCS_IDENT(mask_ident), mask_value);
 	if (error)
 		return (error);
 
 	error = vmcs_setreg(vmcs, 0, VMCS_IDENT(shadow_ident), initial);
 	if (error)
 		return (error);
 
 	return (0);
 }
 #define	vmx_setup_cr0_shadow(vmcs,init)	vmx_setup_cr_shadow(0, (vmcs), (init))
 #define	vmx_setup_cr4_shadow(vmcs,init)	vmx_setup_cr_shadow(4, (vmcs), (init))
 
 static void *
 vmx_vminit(struct vm *vm, pmap_t pmap)
 {
 	uint16_t vpid[VM_MAXCPU];
 	int i, error, guest_msr_count;
 	struct vmx *vmx;
 	struct vmcs *vmcs;
 
 	vmx = malloc(sizeof(struct vmx), M_VMX, M_WAITOK | M_ZERO);
 	if ((uintptr_t)vmx & PAGE_MASK) {
 		panic("malloc of struct vmx not aligned on %d byte boundary",
 		      PAGE_SIZE);
 	}
 	vmx->vm = vm;
 
 	vmx->eptp = eptp(vtophys((vm_offset_t)pmap->pm_pml4));
 
 	/*
 	 * Clean up EPTP-tagged guest physical and combined mappings
 	 *
 	 * VMX transitions are not required to invalidate any guest physical
 	 * mappings. So, it may be possible for stale guest physical mappings
 	 * to be present in the processor TLBs.
 	 *
 	 * Combined mappings for this EP4TA are also invalidated for all VPIDs.
 	 */
 	ept_invalidate_mappings(vmx->eptp);
 
 	msr_bitmap_initialize(vmx->msr_bitmap);
 
 	/*
 	 * It is safe to allow direct access to MSR_GSBASE and MSR_FSBASE.
 	 * The guest FSBASE and GSBASE are saved and restored during
 	 * vm-exit and vm-entry respectively. The host FSBASE and GSBASE are
 	 * always restored from the vmcs host state area on vm-exit.
 	 *
 	 * The SYSENTER_CS/ESP/EIP MSRs are identical to FS/GSBASE in
 	 * how they are saved/restored so can be directly accessed by the
 	 * guest.
 	 *
 	 * Guest KGSBASE is saved and restored in the guest MSR save area.
 	 * Host KGSBASE is restored before returning to userland from the pcb.
 	 * There will be a window of time when we are executing in the host
 	 * kernel context with a value of KGSBASE from the guest. This is ok
 	 * because the value of KGSBASE is inconsequential in kernel context.
 	 *
 	 * MSR_EFER is saved and restored in the guest VMCS area on a
 	 * VM exit and entry respectively. It is also restored from the
 	 * host VMCS area on a VM exit.
 	 *
 	 * The TSC MSR is exposed read-only. Writes are disallowed as that
 	 * will impact the host TSC.
 	 * XXX Writes would be implemented with a wrmsr trap, and
 	 * then modifying the TSC offset in the VMCS.
 	 */
 	if (guest_msr_rw(vmx, MSR_GSBASE) ||
 	    guest_msr_rw(vmx, MSR_FSBASE) ||
 	    guest_msr_rw(vmx, MSR_SYSENTER_CS_MSR) ||
 	    guest_msr_rw(vmx, MSR_SYSENTER_ESP_MSR) ||
 	    guest_msr_rw(vmx, MSR_SYSENTER_EIP_MSR) ||
 	    guest_msr_rw(vmx, MSR_KGSBASE) ||
 	    guest_msr_rw(vmx, MSR_EFER) ||
 	    guest_msr_ro(vmx, MSR_TSC))
 		panic("vmx_vminit: error setting guest msr access");
 
 	/*
 	 * MSR_PAT is saved and restored in the guest VMCS are on a VM exit
 	 * and entry respectively. It is also restored from the host VMCS
 	 * area on a VM exit. However, if running on a system with no
 	 * MSR_PAT save/restore support, leave access disabled so accesses
 	 * will be trapped.
 	 */
 	if (!vmx_no_patmsr && guest_msr_rw(vmx, MSR_PAT))
 		panic("vmx_vminit: error setting guest pat msr access");
 
 	vpid_alloc(vpid, VM_MAXCPU);
 
 	if (virtual_interrupt_delivery) {
 		error = vm_map_mmio(vm, DEFAULT_APIC_BASE, PAGE_SIZE,
 		    APIC_ACCESS_ADDRESS);
 		/* XXX this should really return an error to the caller */
 		KASSERT(error == 0, ("vm_map_mmio(apicbase) error %d", error));
 	}
 
 	for (i = 0; i < VM_MAXCPU; i++) {
 		vmcs = &vmx->vmcs[i];
 		vmcs->identifier = vmx_revision();
 		error = vmclear(vmcs);
 		if (error != 0) {
 			panic("vmx_vminit: vmclear error %d on vcpu %d\n",
 			      error, i);
 		}
 
 		error = vmcs_init(vmcs);
 		KASSERT(error == 0, ("vmcs_init error %d", error));
 
 		VMPTRLD(vmcs);
 		error = 0;
 		error += vmwrite(VMCS_HOST_RSP, (u_long)&vmx->ctx[i]);
 		error += vmwrite(VMCS_EPTP, vmx->eptp);
 		error += vmwrite(VMCS_PIN_BASED_CTLS, pinbased_ctls);
 		error += vmwrite(VMCS_PRI_PROC_BASED_CTLS, procbased_ctls);
 		error += vmwrite(VMCS_SEC_PROC_BASED_CTLS, procbased_ctls2);
 		error += vmwrite(VMCS_EXIT_CTLS, exit_ctls);
 		error += vmwrite(VMCS_ENTRY_CTLS, entry_ctls);
 		error += vmwrite(VMCS_MSR_BITMAP, vtophys(vmx->msr_bitmap));
 		error += vmwrite(VMCS_VPID, vpid[i]);
 		if (virtual_interrupt_delivery) {
 			error += vmwrite(VMCS_APIC_ACCESS, APIC_ACCESS_ADDRESS);
 			error += vmwrite(VMCS_VIRTUAL_APIC,
 			    vtophys(&vmx->apic_page[i]));
 			error += vmwrite(VMCS_EOI_EXIT0, 0);
 			error += vmwrite(VMCS_EOI_EXIT1, 0);
 			error += vmwrite(VMCS_EOI_EXIT2, 0);
 			error += vmwrite(VMCS_EOI_EXIT3, 0);
 		}
 		if (posted_interrupts) {
 			error += vmwrite(VMCS_PIR_VECTOR, pirvec);
 			error += vmwrite(VMCS_PIR_DESC,
 			    vtophys(&vmx->pir_desc[i]));
 		}
 		VMCLEAR(vmcs);
 		KASSERT(error == 0, ("vmx_vminit: error customizing the vmcs"));
 
 		vmx->cap[i].set = 0;
 		vmx->cap[i].proc_ctls = procbased_ctls;
 		vmx->cap[i].proc_ctls2 = procbased_ctls2;
 
 		vmx->state[i].lastcpu = -1;
 		vmx->state[i].vpid = vpid[i];
 
 		msr_save_area_init(vmx->guest_msrs[i], &guest_msr_count);
 
 		error = vmcs_set_msr_save(vmcs, vtophys(vmx->guest_msrs[i]),
 		    guest_msr_count);
 		if (error != 0)
 			panic("vmcs_set_msr_save error %d", error);
 
 		/*
 		 * Set up the CR0/4 shadows, and init the read shadow
 		 * to the power-on register value from the Intel Sys Arch.
 		 *  CR0 - 0x60000010
 		 *  CR4 - 0
 		 */
 		error = vmx_setup_cr0_shadow(vmcs, 0x60000010);
 		if (error != 0)
 			panic("vmx_setup_cr0_shadow %d", error);
 
 		error = vmx_setup_cr4_shadow(vmcs, 0);
 		if (error != 0)
 			panic("vmx_setup_cr4_shadow %d", error);
 
 		vmx->ctx[i].pmap = pmap;
 	}
 
 	return (vmx);
 }
 
 static int
 vmx_handle_cpuid(struct vm *vm, int vcpu, struct vmxctx *vmxctx)
 {
 	int handled, func;
 	
 	func = vmxctx->guest_rax;
 
 	handled = x86_emulate_cpuid(vm, vcpu,
 				    (uint32_t*)(&vmxctx->guest_rax),
 				    (uint32_t*)(&vmxctx->guest_rbx),
 				    (uint32_t*)(&vmxctx->guest_rcx),
 				    (uint32_t*)(&vmxctx->guest_rdx));
 	return (handled);
 }
 
 static __inline void
 vmx_run_trace(struct vmx *vmx, int vcpu)
 {
 #ifdef KTR
 	VCPU_CTR1(vmx->vm, vcpu, "Resume execution at %#lx", vmcs_guest_rip());
 #endif
 }
 
 static __inline void
 vmx_exit_trace(struct vmx *vmx, int vcpu, uint64_t rip, uint32_t exit_reason,
 	       int handled)
 {
 #ifdef KTR
 	VCPU_CTR3(vmx->vm, vcpu, "%s %s vmexit at 0x%0lx",
 		 handled ? "handled" : "unhandled",
 		 exit_reason_to_str(exit_reason), rip);
 #endif
 }
 
 static __inline void
 vmx_astpending_trace(struct vmx *vmx, int vcpu, uint64_t rip)
 {
 #ifdef KTR
 	VCPU_CTR1(vmx->vm, vcpu, "astpending vmexit at 0x%0lx", rip);
 #endif
 }
 
 static VMM_STAT_INTEL(VCPU_INVVPID_SAVED, "Number of vpid invalidations saved");
 
 static void
 vmx_set_pcpu_defaults(struct vmx *vmx, int vcpu, pmap_t pmap)
 {
 	struct vmxstate *vmxstate;
 	struct invvpid_desc invvpid_desc;
 
 	vmxstate = &vmx->state[vcpu];
 	if (vmxstate->lastcpu == curcpu)
 		return;
 
 	vmxstate->lastcpu = curcpu;
 
 	vmm_stat_incr(vmx->vm, vcpu, VCPU_MIGRATIONS, 1);
 
 	vmcs_write(VMCS_HOST_TR_BASE, vmm_get_host_trbase());
 	vmcs_write(VMCS_HOST_GDTR_BASE, vmm_get_host_gdtrbase());
 	vmcs_write(VMCS_HOST_GS_BASE, vmm_get_host_gsbase());
 
 	/*
 	 * If we are using VPIDs then invalidate all mappings tagged with 'vpid'
 	 *
 	 * We do this because this vcpu was executing on a different host
 	 * cpu when it last ran. We do not track whether it invalidated
 	 * mappings associated with its 'vpid' during that run. So we must
 	 * assume that the mappings associated with 'vpid' on 'curcpu' are
 	 * stale and invalidate them.
 	 *
 	 * Note that we incur this penalty only when the scheduler chooses to
 	 * move the thread associated with this vcpu between host cpus.
 	 *
 	 * Note also that this will invalidate mappings tagged with 'vpid'
 	 * for "all" EP4TAs.
 	 */
 	if (vmxstate->vpid != 0) {
 		if (pmap->pm_eptgen == vmx->eptgen[curcpu]) {
 			invvpid_desc._res1 = 0;
 			invvpid_desc._res2 = 0;
 			invvpid_desc.vpid = vmxstate->vpid;
 			invvpid_desc.linear_addr = 0;
 			invvpid(INVVPID_TYPE_SINGLE_CONTEXT, invvpid_desc);
 		} else {
 			/*
 			 * The invvpid can be skipped if an invept is going to
 			 * be performed before entering the guest. The invept
 			 * will invalidate combined mappings tagged with
 			 * 'vmx->eptp' for all vpids.
 			 */
 			vmm_stat_incr(vmx->vm, vcpu, VCPU_INVVPID_SAVED, 1);
 		}
 	}
 }
 
 /*
  * We depend on 'procbased_ctls' to have the Interrupt Window Exiting bit set.
  */
 CTASSERT((PROCBASED_CTLS_ONE_SETTING & PROCBASED_INT_WINDOW_EXITING) != 0);
 
 static void __inline
 vmx_set_int_window_exiting(struct vmx *vmx, int vcpu)
 {
 
 	if ((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) == 0) {
 		vmx->cap[vcpu].proc_ctls |= PROCBASED_INT_WINDOW_EXITING;
 		vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
 		VCPU_CTR0(vmx->vm, vcpu, "Enabling interrupt window exiting");
 	}
 }
 
 static void __inline
 vmx_clear_int_window_exiting(struct vmx *vmx, int vcpu)
 {
 
 	KASSERT((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0,
 	    ("intr_window_exiting not set: %#x", vmx->cap[vcpu].proc_ctls));
 	vmx->cap[vcpu].proc_ctls &= ~PROCBASED_INT_WINDOW_EXITING;
 	vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
 	VCPU_CTR0(vmx->vm, vcpu, "Disabling interrupt window exiting");
 }
 
 static void __inline
 vmx_set_nmi_window_exiting(struct vmx *vmx, int vcpu)
 {
 
 	if ((vmx->cap[vcpu].proc_ctls & PROCBASED_NMI_WINDOW_EXITING) == 0) {
 		vmx->cap[vcpu].proc_ctls |= PROCBASED_NMI_WINDOW_EXITING;
 		vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
 		VCPU_CTR0(vmx->vm, vcpu, "Enabling NMI window exiting");
 	}
 }
 
 static void __inline
 vmx_clear_nmi_window_exiting(struct vmx *vmx, int vcpu)
 {
 
 	KASSERT((vmx->cap[vcpu].proc_ctls & PROCBASED_NMI_WINDOW_EXITING) != 0,
 	    ("nmi_window_exiting not set %#x", vmx->cap[vcpu].proc_ctls));
 	vmx->cap[vcpu].proc_ctls &= ~PROCBASED_NMI_WINDOW_EXITING;
 	vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls);
 	VCPU_CTR0(vmx->vm, vcpu, "Disabling NMI window exiting");
 }
 
 #define	NMI_BLOCKING	(VMCS_INTERRUPTIBILITY_NMI_BLOCKING |		\
 			 VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING)
 #define	HWINTR_BLOCKING	(VMCS_INTERRUPTIBILITY_STI_BLOCKING |		\
 			 VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING)
 
 static void
 vmx_inject_nmi(struct vmx *vmx, int vcpu)
 {
 	uint32_t gi, info;
 
 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
 	KASSERT((gi & NMI_BLOCKING) == 0, ("vmx_inject_nmi: invalid guest "
 	    "interruptibility-state %#x", gi));
 
 	info = vmcs_read(VMCS_ENTRY_INTR_INFO);
 	KASSERT((info & VMCS_INTR_VALID) == 0, ("vmx_inject_nmi: invalid "
 	    "VM-entry interruption information %#x", info));
 
 	/*
 	 * Inject the virtual NMI. The vector must be the NMI IDT entry
 	 * or the VMCS entry check will fail.
 	 */
 	info = IDT_NMI | VMCS_INTR_T_NMI | VMCS_INTR_VALID;
 	vmcs_write(VMCS_ENTRY_INTR_INFO, info);
 
 	VCPU_CTR0(vmx->vm, vcpu, "Injecting vNMI");
 
 	/* Clear the request */
 	vm_nmi_clear(vmx->vm, vcpu);
 }
 
 static void
 vmx_inject_interrupts(struct vmx *vmx, int vcpu, struct vlapic *vlapic)
 {
 	struct vm_exception exc;
 	int vector, need_nmi_exiting, extint_pending;
 	uint64_t rflags;
 	uint32_t gi, info;
 
 	if (vm_exception_pending(vmx->vm, vcpu, &exc)) {
 		KASSERT(exc.vector >= 0 && exc.vector < 32,
 		    ("%s: invalid exception vector %d", __func__, exc.vector));
 
 		info = vmcs_read(VMCS_ENTRY_INTR_INFO);
 		KASSERT((info & VMCS_INTR_VALID) == 0, ("%s: cannot inject "
 		     "pending exception %d: %#x", __func__, exc.vector, info));
 
 		info = exc.vector | VMCS_INTR_T_HWEXCEPTION | VMCS_INTR_VALID;
 		if (exc.error_code_valid) {
 			info |= VMCS_INTR_DEL_ERRCODE;
 			vmcs_write(VMCS_ENTRY_EXCEPTION_ERROR, exc.error_code);
 		}
 		vmcs_write(VMCS_ENTRY_INTR_INFO, info);
 	}
 
 	if (vm_nmi_pending(vmx->vm, vcpu)) {
 		/*
 		 * If there are no conditions blocking NMI injection then
 		 * inject it directly here otherwise enable "NMI window
 		 * exiting" to inject it as soon as we can.
 		 *
 		 * We also check for STI_BLOCKING because some implementations
 		 * don't allow NMI injection in this case. If we are running
 		 * on a processor that doesn't have this restriction it will
 		 * immediately exit and the NMI will be injected in the
 		 * "NMI window exiting" handler.
 		 */
 		need_nmi_exiting = 1;
 		gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
 		if ((gi & (HWINTR_BLOCKING | NMI_BLOCKING)) == 0) {
 			info = vmcs_read(VMCS_ENTRY_INTR_INFO);
 			if ((info & VMCS_INTR_VALID) == 0) {
 				vmx_inject_nmi(vmx, vcpu);
 				need_nmi_exiting = 0;
 			} else {
 				VCPU_CTR1(vmx->vm, vcpu, "Cannot inject NMI "
 				    "due to VM-entry intr info %#x", info);
 			}
 		} else {
 			VCPU_CTR1(vmx->vm, vcpu, "Cannot inject NMI due to "
 			    "Guest Interruptibility-state %#x", gi);
 		}
 
 		if (need_nmi_exiting)
 			vmx_set_nmi_window_exiting(vmx, vcpu);
 	}
 
 	extint_pending = vm_extint_pending(vmx->vm, vcpu);
 
 	if (!extint_pending && virtual_interrupt_delivery) {
 		vmx_inject_pir(vlapic);
 		return;
 	}
 
 	/*
 	 * If interrupt-window exiting is already in effect then don't bother
 	 * checking for pending interrupts. This is just an optimization and
 	 * not needed for correctness.
 	 */
 	if ((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0) {
 		VCPU_CTR0(vmx->vm, vcpu, "Skip interrupt injection due to "
 		    "pending int_window_exiting");
 		return;
 	}
 
 	if (!extint_pending) {
 		/* Ask the local apic for a vector to inject */
 		if (!vlapic_pending_intr(vlapic, &vector))
 			return;
 	} else {
 		/* Ask the legacy pic for a vector to inject */
 		vatpic_pending_intr(vmx->vm, &vector);
 	}
 
 	KASSERT(vector >= 32 && vector <= 255, ("invalid vector %d", vector));
 
 	/* Check RFLAGS.IF and the interruptibility state of the guest */
 	rflags = vmcs_read(VMCS_GUEST_RFLAGS);
 	if ((rflags & PSL_I) == 0) {
 		VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to "
 		    "rflags %#lx", vector, rflags);
 		goto cantinject;
 	}
 
 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
 	if (gi & HWINTR_BLOCKING) {
 		VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to "
 		    "Guest Interruptibility-state %#x", vector, gi);
 		goto cantinject;
 	}
 
 	info = vmcs_read(VMCS_ENTRY_INTR_INFO);
 	if (info & VMCS_INTR_VALID) {
 		/*
 		 * This is expected and could happen for multiple reasons:
 		 * - A vectoring VM-entry was aborted due to astpending
 		 * - A VM-exit happened during event injection.
 		 * - An exception was injected above.
 		 * - An NMI was injected above or after "NMI window exiting"
 		 */
 		VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to "
 		    "VM-entry intr info %#x", vector, info);
 		goto cantinject;
 	}
 
 	/* Inject the interrupt */
 	info = VMCS_INTR_T_HWINTR | VMCS_INTR_VALID;
 	info |= vector;
 	vmcs_write(VMCS_ENTRY_INTR_INFO, info);
 
 	if (!extint_pending) {
 		/* Update the Local APIC ISR */
 		vlapic_intr_accepted(vlapic, vector);
 	} else {
 		vm_extint_clear(vmx->vm, vcpu);
 		vatpic_intr_accepted(vmx->vm, vector);
 
 		/*
 		 * After we accepted the current ExtINT the PIC may
 		 * have posted another one.  If that is the case, set
 		 * the Interrupt Window Exiting execution control so
 		 * we can inject that one too.
 		 *
 		 * Also, interrupt window exiting allows us to inject any
 		 * pending APIC vector that was preempted by the ExtINT
 		 * as soon as possible. This applies both for the software
 		 * emulated vlapic and the hardware assisted virtual APIC.
 		 */
 		vmx_set_int_window_exiting(vmx, vcpu);
 	}
 
 	VCPU_CTR1(vmx->vm, vcpu, "Injecting hwintr at vector %d", vector);
 
 	return;
 
 cantinject:
 	/*
 	 * Set the Interrupt Window Exiting execution control so we can inject
 	 * the interrupt as soon as blocking condition goes away.
 	 */
 	vmx_set_int_window_exiting(vmx, vcpu);
 }
 
 /*
  * If the Virtual NMIs execution control is '1' then the logical processor
  * tracks virtual-NMI blocking in the Guest Interruptibility-state field of
  * the VMCS. An IRET instruction in VMX non-root operation will remove any
  * virtual-NMI blocking.
  *
  * This unblocking occurs even if the IRET causes a fault. In this case the
  * hypervisor needs to restore virtual-NMI blocking before resuming the guest.
  */
 static void
 vmx_restore_nmi_blocking(struct vmx *vmx, int vcpuid)
 {
 	uint32_t gi;
 
 	VCPU_CTR0(vmx->vm, vcpuid, "Restore Virtual-NMI blocking");
 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
 	gi |= VMCS_INTERRUPTIBILITY_NMI_BLOCKING;
 	vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi);
 }
 
 static void
 vmx_clear_nmi_blocking(struct vmx *vmx, int vcpuid)
 {
 	uint32_t gi;
 
 	VCPU_CTR0(vmx->vm, vcpuid, "Clear Virtual-NMI blocking");
 	gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY);
 	gi &= ~VMCS_INTERRUPTIBILITY_NMI_BLOCKING;
 	vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi);
 }
 
 static int
 vmx_emulate_xsetbv(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
 {
 	struct vmxctx *vmxctx;
 	uint64_t xcrval;
 	const struct xsave_limits *limits;
 
 	vmxctx = &vmx->ctx[vcpu];
 	limits = vmm_get_xsave_limits();
 
 	/*
 	 * Note that the processor raises a GP# fault on its own if
 	 * xsetbv is executed for CPL != 0, so we do not have to
 	 * emulate that fault here.
 	 */
 
 	/* Only xcr0 is supported. */
 	if (vmxctx->guest_rcx != 0) {
 		vm_inject_gp(vmx->vm, vcpu);
 		return (HANDLED);
 	}
 
 	/* We only handle xcr0 if both the host and guest have XSAVE enabled. */
 	if (!limits->xsave_enabled || !(vmcs_read(VMCS_GUEST_CR4) & CR4_XSAVE)) {
 		vm_inject_ud(vmx->vm, vcpu);
 		return (HANDLED);
 	}
 
 	xcrval = vmxctx->guest_rdx << 32 | (vmxctx->guest_rax & 0xffffffff);
 	if ((xcrval & ~limits->xcr0_allowed) != 0) {
 		vm_inject_gp(vmx->vm, vcpu);
 		return (HANDLED);
 	}
 
 	if (!(xcrval & XFEATURE_ENABLED_X87)) {
 		vm_inject_gp(vmx->vm, vcpu);
 		return (HANDLED);
 	}
 
 	/* AVX (YMM_Hi128) requires SSE. */
 	if (xcrval & XFEATURE_ENABLED_AVX &&
 	    (xcrval & XFEATURE_AVX) != XFEATURE_AVX) {
 		vm_inject_gp(vmx->vm, vcpu);
 		return (HANDLED);
 	}
 
 	/*
 	 * AVX512 requires base AVX (YMM_Hi128) as well as OpMask,
 	 * ZMM_Hi256, and Hi16_ZMM.
 	 */
 	if (xcrval & XFEATURE_AVX512 &&
 	    (xcrval & (XFEATURE_AVX512 | XFEATURE_AVX)) !=
 	    (XFEATURE_AVX512 | XFEATURE_AVX)) {
 		vm_inject_gp(vmx->vm, vcpu);
 		return (HANDLED);
 	}
 
 	/*
 	 * Intel MPX requires both bound register state flags to be
 	 * set.
 	 */
 	if (((xcrval & XFEATURE_ENABLED_BNDREGS) != 0) !=
 	    ((xcrval & XFEATURE_ENABLED_BNDCSR) != 0)) {
 		vm_inject_gp(vmx->vm, vcpu);
 		return (HANDLED);
 	}
 
 	/*
 	 * This runs "inside" vmrun() with the guest's FPU state, so
 	 * modifying xcr0 directly modifies the guest's xcr0, not the
 	 * host's.
 	 */
 	load_xcr(0, xcrval);
 	return (HANDLED);
 }
 
 static uint64_t
 vmx_get_guest_reg(struct vmx *vmx, int vcpu, int ident)
 {
 	const struct vmxctx *vmxctx;
 
 	vmxctx = &vmx->ctx[vcpu];
 
 	switch (ident) {
 	case 0:
 		return (vmxctx->guest_rax);
 	case 1:
 		return (vmxctx->guest_rcx);
 	case 2:
 		return (vmxctx->guest_rdx);
 	case 3:
 		return (vmxctx->guest_rbx);
 	case 4:
 		return (vmcs_read(VMCS_GUEST_RSP));
 	case 5:
 		return (vmxctx->guest_rbp);
 	case 6:
 		return (vmxctx->guest_rsi);
 	case 7:
 		return (vmxctx->guest_rdi);
 	case 8:
 		return (vmxctx->guest_r8);
 	case 9:
 		return (vmxctx->guest_r9);
 	case 10:
 		return (vmxctx->guest_r10);
 	case 11:
 		return (vmxctx->guest_r11);
 	case 12:
 		return (vmxctx->guest_r12);
 	case 13:
 		return (vmxctx->guest_r13);
 	case 14:
 		return (vmxctx->guest_r14);
 	case 15:
 		return (vmxctx->guest_r15);
 	default:
 		panic("invalid vmx register %d", ident);
 	}
 }
 
 static void
 vmx_set_guest_reg(struct vmx *vmx, int vcpu, int ident, uint64_t regval)
 {
 	struct vmxctx *vmxctx;
 
 	vmxctx = &vmx->ctx[vcpu];
 
 	switch (ident) {
 	case 0:
 		vmxctx->guest_rax = regval;
 		break;
 	case 1:
 		vmxctx->guest_rcx = regval;
 		break;
 	case 2:
 		vmxctx->guest_rdx = regval;
 		break;
 	case 3:
 		vmxctx->guest_rbx = regval;
 		break;
 	case 4:
 		vmcs_write(VMCS_GUEST_RSP, regval);
 		break;
 	case 5:
 		vmxctx->guest_rbp = regval;
 		break;
 	case 6:
 		vmxctx->guest_rsi = regval;
 		break;
 	case 7:
 		vmxctx->guest_rdi = regval;
 		break;
 	case 8:
 		vmxctx->guest_r8 = regval;
 		break;
 	case 9:
 		vmxctx->guest_r9 = regval;
 		break;
 	case 10:
 		vmxctx->guest_r10 = regval;
 		break;
 	case 11:
 		vmxctx->guest_r11 = regval;
 		break;
 	case 12:
 		vmxctx->guest_r12 = regval;
 		break;
 	case 13:
 		vmxctx->guest_r13 = regval;
 		break;
 	case 14:
 		vmxctx->guest_r14 = regval;
 		break;
 	case 15:
 		vmxctx->guest_r15 = regval;
 		break;
 	default:
 		panic("invalid vmx register %d", ident);
 	}
 }
 
 static int
 vmx_emulate_cr0_access(struct vmx *vmx, int vcpu, uint64_t exitqual)
 {
 	uint64_t crval, regval;
 
 	/* We only handle mov to %cr0 at this time */
 	if ((exitqual & 0xf0) != 0x00)
 		return (UNHANDLED);
 
 	regval = vmx_get_guest_reg(vmx, vcpu, (exitqual >> 8) & 0xf);
 
 	vmcs_write(VMCS_CR0_SHADOW, regval);
 
 	crval = regval | cr0_ones_mask;
 	crval &= ~cr0_zeros_mask;
 	vmcs_write(VMCS_GUEST_CR0, crval);
 
 	if (regval & CR0_PG) {
 		uint64_t efer, entry_ctls;
 
 		/*
 		 * If CR0.PG is 1 and EFER.LME is 1 then EFER.LMA and
 		 * the "IA-32e mode guest" bit in VM-entry control must be
 		 * equal.
 		 */
 		efer = vmcs_read(VMCS_GUEST_IA32_EFER);
 		if (efer & EFER_LME) {
 			efer |= EFER_LMA;
 			vmcs_write(VMCS_GUEST_IA32_EFER, efer);
 			entry_ctls = vmcs_read(VMCS_ENTRY_CTLS);
 			entry_ctls |= VM_ENTRY_GUEST_LMA;
 			vmcs_write(VMCS_ENTRY_CTLS, entry_ctls);
 		}
 	}
 
 	return (HANDLED);
 }
 
 static int
 vmx_emulate_cr4_access(struct vmx *vmx, int vcpu, uint64_t exitqual)
 {
 	uint64_t crval, regval;
 
 	/* We only handle mov to %cr4 at this time */
 	if ((exitqual & 0xf0) != 0x00)
 		return (UNHANDLED);
 
 	regval = vmx_get_guest_reg(vmx, vcpu, (exitqual >> 8) & 0xf);
 
 	vmcs_write(VMCS_CR4_SHADOW, regval);
 
 	crval = regval | cr4_ones_mask;
 	crval &= ~cr4_zeros_mask;
 	vmcs_write(VMCS_GUEST_CR4, crval);
 
 	return (HANDLED);
 }
 
 static int
 vmx_emulate_cr8_access(struct vmx *vmx, int vcpu, uint64_t exitqual)
 {
 	struct vlapic *vlapic;
 	uint64_t cr8;
 	int regnum;
 
 	/* We only handle mov %cr8 to/from a register at this time. */
 	if ((exitqual & 0xe0) != 0x00) {
 		return (UNHANDLED);
 	}
 
 	vlapic = vm_lapic(vmx->vm, vcpu);
 	regnum = (exitqual >> 8) & 0xf;
 	if (exitqual & 0x10) {
 		cr8 = vlapic_get_cr8(vlapic);
 		vmx_set_guest_reg(vmx, vcpu, regnum, cr8);
 	} else {
 		cr8 = vmx_get_guest_reg(vmx, vcpu, regnum);
 		vlapic_set_cr8(vlapic, cr8);
 	}
 
 	return (HANDLED);
 }
 
 /*
  * From section "Guest Register State" in the Intel SDM: CPL = SS.DPL
  */
 static int
 vmx_cpl(void)
 {
 	uint32_t ssar;
 
 	ssar = vmcs_read(VMCS_GUEST_SS_ACCESS_RIGHTS);
 	return ((ssar >> 5) & 0x3);
 }
 
 static enum vm_cpu_mode
 vmx_cpu_mode(void)
 {
 
 	if (vmcs_read(VMCS_GUEST_IA32_EFER) & EFER_LMA)
 		return (CPU_MODE_64BIT);
 	else
 		return (CPU_MODE_COMPATIBILITY);
 }
 
 static enum vm_paging_mode
 vmx_paging_mode(void)
 {
 
 	if (!(vmcs_read(VMCS_GUEST_CR0) & CR0_PG))
 		return (PAGING_MODE_FLAT);
 	if (!(vmcs_read(VMCS_GUEST_CR4) & CR4_PAE))
 		return (PAGING_MODE_32);
 	if (vmcs_read(VMCS_GUEST_IA32_EFER) & EFER_LME)
 		return (PAGING_MODE_64);
 	else
 		return (PAGING_MODE_PAE);
 }
 
 static uint64_t
 inout_str_index(struct vmx *vmx, int vcpuid, int in)
 {
 	uint64_t val;
 	int error;
 	enum vm_reg_name reg;
 
 	reg = in ? VM_REG_GUEST_RDI : VM_REG_GUEST_RSI;
 	error = vmx_getreg(vmx, vcpuid, reg, &val);
 	KASSERT(error == 0, ("%s: vmx_getreg error %d", __func__, error));
 	return (val);
 }
 
 static uint64_t
 inout_str_count(struct vmx *vmx, int vcpuid, int rep)
 {
 	uint64_t val;
 	int error;
 
 	if (rep) {
 		error = vmx_getreg(vmx, vcpuid, VM_REG_GUEST_RCX, &val);
 		KASSERT(!error, ("%s: vmx_getreg error %d", __func__, error));
 	} else {
 		val = 1;
 	}
 	return (val);
 }
 
 static int
 inout_str_addrsize(uint32_t inst_info)
 {
 	uint32_t size;
 
 	size = (inst_info >> 7) & 0x7;
 	switch (size) {
 	case 0:
 		return (2);	/* 16 bit */
 	case 1:
 		return (4);	/* 32 bit */
 	case 2:
 		return (8);	/* 64 bit */
 	default:
 		panic("%s: invalid size encoding %d", __func__, size);
 	}
 }
 
 static void
 inout_str_seginfo(struct vmx *vmx, int vcpuid, uint32_t inst_info, int in,
     struct vm_inout_str *vis)
 {
 	int error, s;
 
 	if (in) {
 		vis->seg_name = VM_REG_GUEST_ES;
 	} else {
 		s = (inst_info >> 15) & 0x7;
 		vis->seg_name = vm_segment_name(s);
 	}
 
 	error = vmx_getdesc(vmx, vcpuid, vis->seg_name, &vis->seg_desc);
 	KASSERT(error == 0, ("%s: vmx_getdesc error %d", __func__, error));
 
 	/* XXX modify svm.c to update bit 16 of seg_desc.access (unusable) */
 }
 
 static void
 vmx_paging_info(struct vm_guest_paging *paging)
 {
 	paging->cr3 = vmcs_guest_cr3();
 	paging->cpl = vmx_cpl();
 	paging->cpu_mode = vmx_cpu_mode();
 	paging->paging_mode = vmx_paging_mode();
 }
 
 static void
 vmexit_inst_emul(struct vm_exit *vmexit, uint64_t gpa, uint64_t gla)
 {
 	vmexit->exitcode = VM_EXITCODE_INST_EMUL;
 	vmexit->u.inst_emul.gpa = gpa;
 	vmexit->u.inst_emul.gla = gla;
 	vmx_paging_info(&vmexit->u.inst_emul.paging);
 }
 
 static int
 ept_fault_type(uint64_t ept_qual)
 {
 	int fault_type;
 
 	if (ept_qual & EPT_VIOLATION_DATA_WRITE)
 		fault_type = VM_PROT_WRITE;
 	else if (ept_qual & EPT_VIOLATION_INST_FETCH)
 		fault_type = VM_PROT_EXECUTE;
 	else
 		fault_type= VM_PROT_READ;
 
 	return (fault_type);
 }
 
 static boolean_t
 ept_emulation_fault(uint64_t ept_qual)
 {
 	int read, write;
 
 	/* EPT fault on an instruction fetch doesn't make sense here */
 	if (ept_qual & EPT_VIOLATION_INST_FETCH)
 		return (FALSE);
 
 	/* EPT fault must be a read fault or a write fault */
 	read = ept_qual & EPT_VIOLATION_DATA_READ ? 1 : 0;
 	write = ept_qual & EPT_VIOLATION_DATA_WRITE ? 1 : 0;
 	if ((read | write) == 0)
 		return (FALSE);
 
 	/*
 	 * The EPT violation must have been caused by accessing a
 	 * guest-physical address that is a translation of a guest-linear
 	 * address.
 	 */
 	if ((ept_qual & EPT_VIOLATION_GLA_VALID) == 0 ||
 	    (ept_qual & EPT_VIOLATION_XLAT_VALID) == 0) {
 		return (FALSE);
 	}
 
 	return (TRUE);
 }
 
 static __inline int
 apic_access_virtualization(struct vmx *vmx, int vcpuid)
 {
 	uint32_t proc_ctls2;
 
 	proc_ctls2 = vmx->cap[vcpuid].proc_ctls2;
 	return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_APIC_ACCESSES) ? 1 : 0);
 }
 
 static __inline int
 x2apic_virtualization(struct vmx *vmx, int vcpuid)
 {
 	uint32_t proc_ctls2;
 
 	proc_ctls2 = vmx->cap[vcpuid].proc_ctls2;
 	return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_X2APIC_MODE) ? 1 : 0);
 }
 
 static int
 vmx_handle_apic_write(struct vmx *vmx, int vcpuid, struct vlapic *vlapic,
     uint64_t qual)
 {
 	int error, handled, offset;
 	uint32_t *apic_regs, vector;
 	bool retu;
 
 	handled = HANDLED;
 	offset = APIC_WRITE_OFFSET(qual);
 
 	if (!apic_access_virtualization(vmx, vcpuid)) {
 		/*
 		 * In general there should not be any APIC write VM-exits
 		 * unless APIC-access virtualization is enabled.
 		 *
 		 * However self-IPI virtualization can legitimately trigger
 		 * an APIC-write VM-exit so treat it specially.
 		 */
 		if (x2apic_virtualization(vmx, vcpuid) &&
 		    offset == APIC_OFFSET_SELF_IPI) {
 			apic_regs = (uint32_t *)(vlapic->apic_page);
 			vector = apic_regs[APIC_OFFSET_SELF_IPI / 4];
 			vlapic_self_ipi_handler(vlapic, vector);
 			return (HANDLED);
 		} else
 			return (UNHANDLED);
 	}
 
 	switch (offset) {
 	case APIC_OFFSET_ID:
 		vlapic_id_write_handler(vlapic);
 		break;
 	case APIC_OFFSET_LDR:
 		vlapic_ldr_write_handler(vlapic);
 		break;
 	case APIC_OFFSET_DFR:
 		vlapic_dfr_write_handler(vlapic);
 		break;
 	case APIC_OFFSET_SVR:
 		vlapic_svr_write_handler(vlapic);
 		break;
 	case APIC_OFFSET_ESR:
 		vlapic_esr_write_handler(vlapic);
 		break;
 	case APIC_OFFSET_ICR_LOW:
 		retu = false;
 		error = vlapic_icrlo_write_handler(vlapic, &retu);
 		if (error != 0 || retu)
 			handled = UNHANDLED;
 		break;
 	case APIC_OFFSET_CMCI_LVT:
 	case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
 		vlapic_lvt_write_handler(vlapic, offset);
 		break;
 	case APIC_OFFSET_TIMER_ICR:
 		vlapic_icrtmr_write_handler(vlapic);
 		break;
 	case APIC_OFFSET_TIMER_DCR:
 		vlapic_dcr_write_handler(vlapic);
 		break;
 	default:
 		handled = UNHANDLED;
 		break;
 	}
 	return (handled);
 }
 
 static bool
 apic_access_fault(struct vmx *vmx, int vcpuid, uint64_t gpa)
 {
 
 	if (apic_access_virtualization(vmx, vcpuid) &&
 	    (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + PAGE_SIZE))
 		return (true);
 	else
 		return (false);
 }
 
 static int
 vmx_handle_apic_access(struct vmx *vmx, int vcpuid, struct vm_exit *vmexit)
 {
 	uint64_t qual;
 	int access_type, offset, allowed;
 
 	if (!apic_access_virtualization(vmx, vcpuid))
 		return (UNHANDLED);
 
 	qual = vmexit->u.vmx.exit_qualification;
 	access_type = APIC_ACCESS_TYPE(qual);
 	offset = APIC_ACCESS_OFFSET(qual);
 
 	allowed = 0;
 	if (access_type == 0) {
 		/*
 		 * Read data access to the following registers is expected.
 		 */
 		switch (offset) {
 		case APIC_OFFSET_APR:
 		case APIC_OFFSET_PPR:
 		case APIC_OFFSET_RRR:
 		case APIC_OFFSET_CMCI_LVT:
 		case APIC_OFFSET_TIMER_CCR:
 			allowed = 1;
 			break;
 		default:
 			break;
 		}
 	} else if (access_type == 1) {
 		/*
 		 * Write data access to the following registers is expected.
 		 */
 		switch (offset) {
 		case APIC_OFFSET_VER:
 		case APIC_OFFSET_APR:
 		case APIC_OFFSET_PPR:
 		case APIC_OFFSET_RRR:
 		case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
 		case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
 		case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
 		case APIC_OFFSET_CMCI_LVT:
 		case APIC_OFFSET_TIMER_CCR:
 			allowed = 1;
 			break;
 		default:
 			break;
 		}
 	}
 
 	if (allowed) {
 		vmexit_inst_emul(vmexit, DEFAULT_APIC_BASE + offset,
 		    VIE_INVALID_GLA);
 	}
 
 	/*
 	 * Regardless of whether the APIC-access is allowed this handler
 	 * always returns UNHANDLED:
 	 * - if the access is allowed then it is handled by emulating the
 	 *   instruction that caused the VM-exit (outside the critical section)
 	 * - if the access is not allowed then it will be converted to an
 	 *   exitcode of VM_EXITCODE_VMX and will be dealt with in userland.
 	 */
 	return (UNHANDLED);
 }
 
 static int
 vmx_exit_process(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
 {
 	int error, handled, in;
 	struct vmxctx *vmxctx;
 	struct vlapic *vlapic;
 	struct vm_inout_str *vis;
 	uint32_t eax, ecx, edx, idtvec_info, idtvec_err, intr_info, inst_info;
 	uint32_t reason;
 	uint64_t qual, gpa;
 	bool retu;
 
 	CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_VIRTUAL_NMI) != 0);
 	CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_NMI_EXITING) != 0);
 
 	handled = UNHANDLED;
 	vmxctx = &vmx->ctx[vcpu];
 
 	qual = vmexit->u.vmx.exit_qualification;
 	reason = vmexit->u.vmx.exit_reason;
 	vmexit->exitcode = VM_EXITCODE_BOGUS;
 
 	vmm_stat_incr(vmx->vm, vcpu, VMEXIT_COUNT, 1);
 
 	/*
 	 * VM exits that could be triggered during event injection on the
 	 * previous VM entry need to be handled specially by re-injecting
 	 * the event.
 	 *
 	 * See "Information for VM Exits During Event Delivery" in Intel SDM
 	 * for details.
 	 */
 	switch (reason) {
 	case EXIT_REASON_EPT_FAULT:
 	case EXIT_REASON_EPT_MISCONFIG:
 	case EXIT_REASON_APIC_ACCESS:
 	case EXIT_REASON_TASK_SWITCH:
 	case EXIT_REASON_EXCEPTION:
 		idtvec_info = vmcs_idt_vectoring_info();
 		if (idtvec_info & VMCS_IDT_VEC_VALID) {
 			idtvec_info &= ~(1 << 12); /* clear undefined bit */
 			vmcs_write(VMCS_ENTRY_INTR_INFO, idtvec_info);
 			if (idtvec_info & VMCS_IDT_VEC_ERRCODE_VALID) {
 				idtvec_err = vmcs_idt_vectoring_err();
 				vmcs_write(VMCS_ENTRY_EXCEPTION_ERROR,
 				    idtvec_err);
 			}
 			/*
 			 * If 'virtual NMIs' are being used and the VM-exit
 			 * happened while injecting an NMI during the previous
 			 * VM-entry, then clear "blocking by NMI" in the Guest
 			 * Interruptibility-state.
 			 */
 			if ((idtvec_info & VMCS_INTR_T_MASK) ==
 			    VMCS_INTR_T_NMI) {
 				 vmx_clear_nmi_blocking(vmx, vcpu);
 			}
 			vmcs_write(VMCS_ENTRY_INST_LENGTH, vmexit->inst_length);
 		}
 	default:
 		idtvec_info = 0;
 		break;
 	}
 
 	switch (reason) {
 	case EXIT_REASON_CR_ACCESS:
 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CR_ACCESS, 1);
 		switch (qual & 0xf) {
 		case 0:
 			handled = vmx_emulate_cr0_access(vmx, vcpu, qual);
 			break;
 		case 4:
 			handled = vmx_emulate_cr4_access(vmx, vcpu, qual);
 			break;
 		case 8:
 			handled = vmx_emulate_cr8_access(vmx, vcpu, qual);
 			break;
 		}
 		break;
 	case EXIT_REASON_RDMSR:
 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_RDMSR, 1);
 		retu = false;
 		ecx = vmxctx->guest_rcx;
 		VCPU_CTR1(vmx->vm, vcpu, "rdmsr 0x%08x", ecx);
 		error = emulate_rdmsr(vmx->vm, vcpu, ecx, &retu);
 		if (error) {
 			vmexit->exitcode = VM_EXITCODE_RDMSR;
 			vmexit->u.msr.code = ecx;
 		} else if (!retu) {
 			handled = HANDLED;
 		} else {
 			/* Return to userspace with a valid exitcode */
 			KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS,
 			    ("emulate_wrmsr retu with bogus exitcode"));
 		}
 		break;
 	case EXIT_REASON_WRMSR:
 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_WRMSR, 1);
 		retu = false;
 		eax = vmxctx->guest_rax;
 		ecx = vmxctx->guest_rcx;
 		edx = vmxctx->guest_rdx;
 		VCPU_CTR2(vmx->vm, vcpu, "wrmsr 0x%08x value 0x%016lx",
 		    ecx, (uint64_t)edx << 32 | eax);
 		error = emulate_wrmsr(vmx->vm, vcpu, ecx,
 		    (uint64_t)edx << 32 | eax, &retu);
 		if (error) {
 			vmexit->exitcode = VM_EXITCODE_WRMSR;
 			vmexit->u.msr.code = ecx;
 			vmexit->u.msr.wval = (uint64_t)edx << 32 | eax;
 		} else if (!retu) {
 			handled = HANDLED;
 		} else {
 			/* Return to userspace with a valid exitcode */
 			KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS,
 			    ("emulate_wrmsr retu with bogus exitcode"));
 		}
 		break;
 	case EXIT_REASON_HLT:
 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_HLT, 1);
 		vmexit->exitcode = VM_EXITCODE_HLT;
 		vmexit->u.hlt.rflags = vmcs_read(VMCS_GUEST_RFLAGS);
 		break;
 	case EXIT_REASON_MTF:
 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_MTRAP, 1);
 		vmexit->exitcode = VM_EXITCODE_MTRAP;
 		break;
 	case EXIT_REASON_PAUSE:
 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_PAUSE, 1);
 		vmexit->exitcode = VM_EXITCODE_PAUSE;
 		break;
 	case EXIT_REASON_INTR_WINDOW:
 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INTR_WINDOW, 1);
 		vmx_clear_int_window_exiting(vmx, vcpu);
 		return (1);
 	case EXIT_REASON_EXT_INTR:
 		/*
 		 * External interrupts serve only to cause VM exits and allow
 		 * the host interrupt handler to run.
 		 *
 		 * If this external interrupt triggers a virtual interrupt
 		 * to a VM, then that state will be recorded by the
 		 * host interrupt handler in the VM's softc. We will inject
 		 * this virtual interrupt during the subsequent VM enter.
 		 */
 		intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
 
 		/*
 		 * XXX: Ignore this exit if VMCS_INTR_VALID is not set.
 		 * This appears to be a bug in VMware Fusion?
 		 */
 		if (!(intr_info & VMCS_INTR_VALID))
 			return (1);
 		KASSERT((intr_info & VMCS_INTR_VALID) != 0 &&
 		    (intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_HWINTR,
 		    ("VM exit interruption info invalid: %#x", intr_info));
 		vmx_trigger_hostintr(intr_info & 0xff);
 
 		/*
 		 * This is special. We want to treat this as an 'handled'
 		 * VM-exit but not increment the instruction pointer.
 		 */
 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXTINT, 1);
 		return (1);
 	case EXIT_REASON_NMI_WINDOW:
 		/* Exit to allow the pending virtual NMI to be injected */
 		if (vm_nmi_pending(vmx->vm, vcpu))
 			vmx_inject_nmi(vmx, vcpu);
 		vmx_clear_nmi_window_exiting(vmx, vcpu);
 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_NMI_WINDOW, 1);
 		return (1);
 	case EXIT_REASON_INOUT:
 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INOUT, 1);
 		vmexit->exitcode = VM_EXITCODE_INOUT;
 		vmexit->u.inout.bytes = (qual & 0x7) + 1;
 		vmexit->u.inout.in = in = (qual & 0x8) ? 1 : 0;
 		vmexit->u.inout.string = (qual & 0x10) ? 1 : 0;
 		vmexit->u.inout.rep = (qual & 0x20) ? 1 : 0;
 		vmexit->u.inout.port = (uint16_t)(qual >> 16);
 		vmexit->u.inout.eax = (uint32_t)(vmxctx->guest_rax);
 		if (vmexit->u.inout.string) {
 			inst_info = vmcs_read(VMCS_EXIT_INSTRUCTION_INFO);
 			vmexit->exitcode = VM_EXITCODE_INOUT_STR;
 			vis = &vmexit->u.inout_str;
 			vmx_paging_info(&vis->paging);
 			vis->rflags = vmcs_read(VMCS_GUEST_RFLAGS);
 			vis->cr0 = vmcs_read(VMCS_GUEST_CR0);
 			vis->index = inout_str_index(vmx, vcpu, in);
 			vis->count = inout_str_count(vmx, vcpu, vis->inout.rep);
 			vis->addrsize = inout_str_addrsize(inst_info);
 			inout_str_seginfo(vmx, vcpu, inst_info, in, vis);
 		}
 		break;
 	case EXIT_REASON_CPUID:
 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CPUID, 1);
 		handled = vmx_handle_cpuid(vmx->vm, vcpu, vmxctx);
 		break;
 	case EXIT_REASON_EXCEPTION:
 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXCEPTION, 1);
 		intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
 		KASSERT((intr_info & VMCS_INTR_VALID) != 0,
 		    ("VM exit interruption info invalid: %#x", intr_info));
 
 		/*
 		 * If Virtual NMIs control is 1 and the VM-exit is due to a
 		 * fault encountered during the execution of IRET then we must
 		 * restore the state of "virtual-NMI blocking" before resuming
 		 * the guest.
 		 *
 		 * See "Resuming Guest Software after Handling an Exception".
 		 */
 		if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 &&
 		    (intr_info & 0xff) != IDT_DF &&
 		    (intr_info & EXIT_QUAL_NMIUDTI) != 0)
 			vmx_restore_nmi_blocking(vmx, vcpu);
 
 		/*
 		 * The NMI has already been handled in vmx_exit_handle_nmi().
 		 */
 		if ((intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_NMI)
 			return (1);
 		break;
 	case EXIT_REASON_EPT_FAULT:
 		/*
 		 * If 'gpa' lies within the address space allocated to
 		 * memory then this must be a nested page fault otherwise
 		 * this must be an instruction that accesses MMIO space.
 		 */
 		gpa = vmcs_gpa();
 		if (vm_mem_allocated(vmx->vm, gpa) ||
 		    apic_access_fault(vmx, vcpu, gpa)) {
 			vmexit->exitcode = VM_EXITCODE_PAGING;
 			vmexit->u.paging.gpa = gpa;
 			vmexit->u.paging.fault_type = ept_fault_type(qual);
 			vmm_stat_incr(vmx->vm, vcpu, VMEXIT_NESTED_FAULT, 1);
 		} else if (ept_emulation_fault(qual)) {
 			vmexit_inst_emul(vmexit, gpa, vmcs_gla());
 			vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INST_EMUL, 1);
 		}
 		/*
 		 * If Virtual NMIs control is 1 and the VM-exit is due to an
 		 * EPT fault during the execution of IRET then we must restore
 		 * the state of "virtual-NMI blocking" before resuming.
 		 *
 		 * See description of "NMI unblocking due to IRET" in
 		 * "Exit Qualification for EPT Violations".
 		 */
 		if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 &&
 		    (qual & EXIT_QUAL_NMIUDTI) != 0)
 			vmx_restore_nmi_blocking(vmx, vcpu);
 		break;
 	case EXIT_REASON_VIRTUALIZED_EOI:
 		vmexit->exitcode = VM_EXITCODE_IOAPIC_EOI;
 		vmexit->u.ioapic_eoi.vector = qual & 0xFF;
 		vmexit->inst_length = 0;	/* trap-like */
 		break;
 	case EXIT_REASON_APIC_ACCESS:
 		handled = vmx_handle_apic_access(vmx, vcpu, vmexit);
 		break;
 	case EXIT_REASON_APIC_WRITE:
 		/*
 		 * APIC-write VM exit is trap-like so the %rip is already
 		 * pointing to the next instruction.
 		 */
 		vmexit->inst_length = 0;
 		vlapic = vm_lapic(vmx->vm, vcpu);
 		handled = vmx_handle_apic_write(vmx, vcpu, vlapic, qual);
 		break;
 	case EXIT_REASON_XSETBV:
 		handled = vmx_emulate_xsetbv(vmx, vcpu, vmexit);
 		break;
 	default:
 		vmm_stat_incr(vmx->vm, vcpu, VMEXIT_UNKNOWN, 1);
 		break;
 	}
 
 	if (handled) {
 		/*
 		 * It is possible that control is returned to userland
 		 * even though we were able to handle the VM exit in the
 		 * kernel.
 		 *
 		 * In such a case we want to make sure that the userland
 		 * restarts guest execution at the instruction *after*
 		 * the one we just processed. Therefore we update the
 		 * guest rip in the VMCS and in 'vmexit'.
 		 */
 		vmexit->rip += vmexit->inst_length;
 		vmexit->inst_length = 0;
 		vmcs_write(VMCS_GUEST_RIP, vmexit->rip);
 	} else {
 		if (vmexit->exitcode == VM_EXITCODE_BOGUS) {
 			/*
 			 * If this VM exit was not claimed by anybody then
 			 * treat it as a generic VMX exit.
 			 */
 			vmexit->exitcode = VM_EXITCODE_VMX;
 			vmexit->u.vmx.status = VM_SUCCESS;
 			vmexit->u.vmx.inst_type = 0;
 			vmexit->u.vmx.inst_error = 0;
 		} else {
 			/*
 			 * The exitcode and collateral have been populated.
 			 * The VM exit will be processed further in userland.
 			 */
 		}
 	}
 	return (handled);
 }
 
-static __inline int
-vmx_exit_astpending(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
-{
-
-	vmexit->rip = vmcs_guest_rip();
-	vmexit->inst_length = 0;
-	vmexit->exitcode = VM_EXITCODE_BOGUS;
-	vmx_astpending_trace(vmx, vcpu, vmexit->rip);
-	vmm_stat_incr(vmx->vm, vcpu, VMEXIT_ASTPENDING, 1);
-
-	return (HANDLED);
-}
-
-static __inline int
-vmx_exit_rendezvous(struct vmx *vmx, int vcpu, struct vm_exit *vmexit)
-{
-
-	vmexit->rip = vmcs_guest_rip();
-	vmexit->inst_length = 0;
-	vmexit->exitcode = VM_EXITCODE_RENDEZVOUS;
-	vmm_stat_incr(vmx->vm, vcpu, VMEXIT_RENDEZVOUS, 1);
-
-	return (UNHANDLED);
-}
-
-static __inline int
+static __inline void
 vmx_exit_inst_error(struct vmxctx *vmxctx, int rc, struct vm_exit *vmexit)
 {
 
 	KASSERT(vmxctx->inst_fail_status != VM_SUCCESS,
 	    ("vmx_exit_inst_error: invalid inst_fail_status %d",
 	    vmxctx->inst_fail_status));
 
 	vmexit->inst_length = 0;
 	vmexit->exitcode = VM_EXITCODE_VMX;
 	vmexit->u.vmx.status = vmxctx->inst_fail_status;
 	vmexit->u.vmx.inst_error = vmcs_instruction_error();
 	vmexit->u.vmx.exit_reason = ~0;
 	vmexit->u.vmx.exit_qualification = ~0;
 
 	switch (rc) {
 	case VMX_VMRESUME_ERROR:
 	case VMX_VMLAUNCH_ERROR:
 	case VMX_INVEPT_ERROR:
 		vmexit->u.vmx.inst_type = rc;
 		break;
 	default:
 		panic("vm_exit_inst_error: vmx_enter_guest returned %d", rc);
 	}
-
-	return (UNHANDLED);
 }
 
 /*
  * If the NMI-exiting VM execution control is set to '1' then an NMI in
  * non-root operation causes a VM-exit. NMI blocking is in effect so it is
  * sufficient to simply vector to the NMI handler via a software interrupt.
  * However, this must be done before maskable interrupts are enabled
  * otherwise the "iret" issued by an interrupt handler will incorrectly
  * clear NMI blocking.
  */
 static __inline void
 vmx_exit_handle_nmi(struct vmx *vmx, int vcpuid, struct vm_exit *vmexit)
 {
 	uint32_t intr_info;
 
 	KASSERT((read_rflags() & PSL_I) == 0, ("interrupts enabled"));
 
 	if (vmexit->u.vmx.exit_reason != EXIT_REASON_EXCEPTION)
 		return;
 
 	intr_info = vmcs_read(VMCS_EXIT_INTR_INFO);
 	KASSERT((intr_info & VMCS_INTR_VALID) != 0,
 	    ("VM exit interruption info invalid: %#x", intr_info));
 
 	if ((intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_NMI) {
 		KASSERT((intr_info & 0xff) == IDT_NMI, ("VM exit due "
 		    "to NMI has invalid vector: %#x", intr_info));
 		VCPU_CTR0(vmx->vm, vcpuid, "Vectoring to NMI handler");
 		__asm __volatile("int $2");
 	}
 }
 
 static int
 vmx_run(void *arg, int vcpu, register_t startrip, pmap_t pmap,
     void *rendezvous_cookie, void *suspend_cookie)
 {
 	int rc, handled, launched;
 	struct vmx *vmx;
 	struct vm *vm;
 	struct vmxctx *vmxctx;
 	struct vmcs *vmcs;
 	struct vm_exit *vmexit;
 	struct vlapic *vlapic;
 	uint64_t rip;
 	uint32_t exit_reason;
 
 	vmx = arg;
 	vm = vmx->vm;
 	vmcs = &vmx->vmcs[vcpu];
 	vmxctx = &vmx->ctx[vcpu];
 	vlapic = vm_lapic(vm, vcpu);
 	vmexit = vm_exitinfo(vm, vcpu);
 	launched = 0;
 
 	KASSERT(vmxctx->pmap == pmap,
 	    ("pmap %p different than ctx pmap %p", pmap, vmxctx->pmap));
 
 	VMPTRLD(vmcs);
 
 	/*
 	 * XXX
 	 * We do this every time because we may setup the virtual machine
 	 * from a different process than the one that actually runs it.
 	 *
 	 * If the life of a virtual machine was spent entirely in the context
 	 * of a single process we could do this once in vmx_vminit().
 	 */
 	vmcs_write(VMCS_HOST_CR3, rcr3());
 
 	vmcs_write(VMCS_GUEST_RIP, startrip);
 	vmx_set_pcpu_defaults(vmx, vcpu, pmap);
 	do {
+		handled = UNHANDLED;
+
 		/*
 		 * Interrupts are disabled from this point on until the
 		 * guest starts executing. This is done for the following
 		 * reasons:
 		 *
 		 * If an AST is asserted on this thread after the check below,
 		 * then the IPI_AST notification will not be lost, because it
 		 * will cause a VM exit due to external interrupt as soon as
 		 * the guest state is loaded.
 		 *
 		 * A posted interrupt after 'vmx_inject_interrupts()' will
 		 * not be "lost" because it will be held pending in the host
 		 * APIC because interrupts are disabled. The pending interrupt
 		 * will be recognized as soon as the guest state is loaded.
 		 *
 		 * The same reasoning applies to the IPI generated by
 		 * pmap_invalidate_ept().
 		 */
 		disable_intr();
 		if (vcpu_suspended(suspend_cookie)) {
 			enable_intr();
 			vm_exit_suspended(vmx->vm, vcpu, vmcs_guest_rip());
-			handled = UNHANDLED;
 			break;
 		}
 
 		if (vcpu_rendezvous_pending(rendezvous_cookie)) {
 			enable_intr();
-			handled = vmx_exit_rendezvous(vmx, vcpu, vmexit);
+			vm_exit_rendezvous(vmx->vm, vcpu, vmcs_guest_rip());
 			break;
 		}
 
 		if (curthread->td_flags & (TDF_ASTPENDING | TDF_NEEDRESCHED)) {
 			enable_intr();
-			handled = vmx_exit_astpending(vmx, vcpu, vmexit);
+			vm_exit_astpending(vmx->vm, vcpu, vmcs_guest_rip());
+			vmx_astpending_trace(vmx, vcpu, vmexit->rip);
+			handled = HANDLED;
 			break;
 		}
 
 		vmx_inject_interrupts(vmx, vcpu, vlapic);
 		vmx_run_trace(vmx, vcpu);
 		rc = vmx_enter_guest(vmxctx, vmx, launched);
 
 		/* Collect some information for VM exit processing */
 		vmexit->rip = rip = vmcs_guest_rip();
 		vmexit->inst_length = vmexit_instruction_length();
 		vmexit->u.vmx.exit_reason = exit_reason = vmcs_exit_reason();
 		vmexit->u.vmx.exit_qualification = vmcs_exit_qualification();
 
 		if (rc == VMX_GUEST_VMEXIT) {
 			vmx_exit_handle_nmi(vmx, vcpu, vmexit);
 			enable_intr();
 			handled = vmx_exit_process(vmx, vcpu, vmexit);
 		} else {
 			enable_intr();
-			handled = vmx_exit_inst_error(vmxctx, rc, vmexit);
+			vmx_exit_inst_error(vmxctx, rc, vmexit);
 		}
 		launched = 1;
 		vmx_exit_trace(vmx, vcpu, rip, exit_reason, handled);
 	} while (handled);
 
 	/*
 	 * If a VM exit has been handled then the exitcode must be BOGUS
 	 * If a VM exit is not handled then the exitcode must not be BOGUS
 	 */
 	if ((handled && vmexit->exitcode != VM_EXITCODE_BOGUS) ||
 	    (!handled && vmexit->exitcode == VM_EXITCODE_BOGUS)) {
 		panic("Mismatch between handled (%d) and exitcode (%d)",
 		      handled, vmexit->exitcode);
 	}
 
 	if (!handled)
 		vmm_stat_incr(vm, vcpu, VMEXIT_USERSPACE, 1);
 
 	VCPU_CTR1(vm, vcpu, "returning from vmx_run: exitcode %d",
 	    vmexit->exitcode);
 
 	VMCLEAR(vmcs);
 	return (0);
 }
 
 static void
 vmx_vmcleanup(void *arg)
 {
 	int i;
 	struct vmx *vmx = arg;
 
 	if (apic_access_virtualization(vmx, 0))
 		vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE);
 
 	for (i = 0; i < VM_MAXCPU; i++)
 		vpid_free(vmx->state[i].vpid);
 
 	free(vmx, M_VMX);
 
 	return;
 }
 
 static register_t *
 vmxctx_regptr(struct vmxctx *vmxctx, int reg)
 {
 
 	switch (reg) {
 	case VM_REG_GUEST_RAX:
 		return (&vmxctx->guest_rax);
 	case VM_REG_GUEST_RBX:
 		return (&vmxctx->guest_rbx);
 	case VM_REG_GUEST_RCX:
 		return (&vmxctx->guest_rcx);
 	case VM_REG_GUEST_RDX:
 		return (&vmxctx->guest_rdx);
 	case VM_REG_GUEST_RSI:
 		return (&vmxctx->guest_rsi);
 	case VM_REG_GUEST_RDI:
 		return (&vmxctx->guest_rdi);
 	case VM_REG_GUEST_RBP:
 		return (&vmxctx->guest_rbp);
 	case VM_REG_GUEST_R8:
 		return (&vmxctx->guest_r8);
 	case VM_REG_GUEST_R9:
 		return (&vmxctx->guest_r9);
 	case VM_REG_GUEST_R10:
 		return (&vmxctx->guest_r10);
 	case VM_REG_GUEST_R11:
 		return (&vmxctx->guest_r11);
 	case VM_REG_GUEST_R12:
 		return (&vmxctx->guest_r12);
 	case VM_REG_GUEST_R13:
 		return (&vmxctx->guest_r13);
 	case VM_REG_GUEST_R14:
 		return (&vmxctx->guest_r14);
 	case VM_REG_GUEST_R15:
 		return (&vmxctx->guest_r15);
 	case VM_REG_GUEST_CR2:
 		return (&vmxctx->guest_cr2);
 	default:
 		break;
 	}
 	return (NULL);
 }
 
 static int
 vmxctx_getreg(struct vmxctx *vmxctx, int reg, uint64_t *retval)
 {
 	register_t *regp;
 
 	if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) {
 		*retval = *regp;
 		return (0);
 	} else
 		return (EINVAL);
 }
 
 static int
 vmxctx_setreg(struct vmxctx *vmxctx, int reg, uint64_t val)
 {
 	register_t *regp;
 
 	if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) {
 		*regp = val;
 		return (0);
 	} else
 		return (EINVAL);
 }
 
 static int
 vmx_shadow_reg(int reg)
 {
 	int shreg;
 
 	shreg = -1;
 
 	switch (reg) {
 	case VM_REG_GUEST_CR0:
 		shreg = VMCS_CR0_SHADOW;
                 break;
         case VM_REG_GUEST_CR4:
 		shreg = VMCS_CR4_SHADOW;
 		break;
 	default:
 		break;
 	}
 
 	return (shreg);
 }
 
 static int
 vmx_getreg(void *arg, int vcpu, int reg, uint64_t *retval)
 {
 	int running, hostcpu;
 	struct vmx *vmx = arg;
 
 	running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
 	if (running && hostcpu != curcpu)
 		panic("vmx_getreg: %s%d is running", vm_name(vmx->vm), vcpu);
 
 	if (vmxctx_getreg(&vmx->ctx[vcpu], reg, retval) == 0)
 		return (0);
 
 	return (vmcs_getreg(&vmx->vmcs[vcpu], running, reg, retval));
 }
 
 static int
 vmx_setreg(void *arg, int vcpu, int reg, uint64_t val)
 {
 	int error, hostcpu, running, shadow;
 	uint64_t ctls;
 	struct vmx *vmx = arg;
 
 	running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
 	if (running && hostcpu != curcpu)
 		panic("vmx_setreg: %s%d is running", vm_name(vmx->vm), vcpu);
 
 	if (vmxctx_setreg(&vmx->ctx[vcpu], reg, val) == 0)
 		return (0);
 
 	error = vmcs_setreg(&vmx->vmcs[vcpu], running, reg, val);
 
 	if (error == 0) {
 		/*
 		 * If the "load EFER" VM-entry control is 1 then the
 		 * value of EFER.LMA must be identical to "IA-32e mode guest"
 		 * bit in the VM-entry control.
 		 */
 		if ((entry_ctls & VM_ENTRY_LOAD_EFER) != 0 &&
 		    (reg == VM_REG_GUEST_EFER)) {
 			vmcs_getreg(&vmx->vmcs[vcpu], running,
 				    VMCS_IDENT(VMCS_ENTRY_CTLS), &ctls);
 			if (val & EFER_LMA)
 				ctls |= VM_ENTRY_GUEST_LMA;
 			else
 				ctls &= ~VM_ENTRY_GUEST_LMA;
 			vmcs_setreg(&vmx->vmcs[vcpu], running,
 				    VMCS_IDENT(VMCS_ENTRY_CTLS), ctls);
 		}
 
 		shadow = vmx_shadow_reg(reg);
 		if (shadow > 0) {
 			/*
 			 * Store the unmodified value in the shadow
 			 */			
 			error = vmcs_setreg(&vmx->vmcs[vcpu], running,
 				    VMCS_IDENT(shadow), val);
 		}
 	}
 
 	return (error);
 }
 
 static int
 vmx_getdesc(void *arg, int vcpu, int reg, struct seg_desc *desc)
 {
 	int hostcpu, running;
 	struct vmx *vmx = arg;
 
 	running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
 	if (running && hostcpu != curcpu)
 		panic("vmx_getdesc: %s%d is running", vm_name(vmx->vm), vcpu);
 
 	return (vmcs_getdesc(&vmx->vmcs[vcpu], running, reg, desc));
 }
 
 static int
 vmx_setdesc(void *arg, int vcpu, int reg, struct seg_desc *desc)
 {
 	int hostcpu, running;
 	struct vmx *vmx = arg;
 
 	running = vcpu_is_running(vmx->vm, vcpu, &hostcpu);
 	if (running && hostcpu != curcpu)
 		panic("vmx_setdesc: %s%d is running", vm_name(vmx->vm), vcpu);
 
 	return (vmcs_setdesc(&vmx->vmcs[vcpu], running, reg, desc));
 }
 
 static int
 vmx_getcap(void *arg, int vcpu, int type, int *retval)
 {
 	struct vmx *vmx = arg;
 	int vcap;
 	int ret;
 
 	ret = ENOENT;
 
 	vcap = vmx->cap[vcpu].set;
 
 	switch (type) {
 	case VM_CAP_HALT_EXIT:
 		if (cap_halt_exit)
 			ret = 0;
 		break;
 	case VM_CAP_PAUSE_EXIT:
 		if (cap_pause_exit)
 			ret = 0;
 		break;
 	case VM_CAP_MTRAP_EXIT:
 		if (cap_monitor_trap)
 			ret = 0;
 		break;
 	case VM_CAP_UNRESTRICTED_GUEST:
 		if (cap_unrestricted_guest)
 			ret = 0;
 		break;
 	case VM_CAP_ENABLE_INVPCID:
 		if (cap_invpcid)
 			ret = 0;
 		break;
 	default:
 		break;
 	}
 
 	if (ret == 0)
 		*retval = (vcap & (1 << type)) ? 1 : 0;
 
 	return (ret);
 }
 
 static int
 vmx_setcap(void *arg, int vcpu, int type, int val)
 {
 	struct vmx *vmx = arg;
 	struct vmcs *vmcs = &vmx->vmcs[vcpu];
 	uint32_t baseval;
 	uint32_t *pptr;
 	int error;
 	int flag;
 	int reg;
 	int retval;
 
 	retval = ENOENT;
 	pptr = NULL;
 
 	switch (type) {
 	case VM_CAP_HALT_EXIT:
 		if (cap_halt_exit) {
 			retval = 0;
 			pptr = &vmx->cap[vcpu].proc_ctls;
 			baseval = *pptr;
 			flag = PROCBASED_HLT_EXITING;
 			reg = VMCS_PRI_PROC_BASED_CTLS;
 		}
 		break;
 	case VM_CAP_MTRAP_EXIT:
 		if (cap_monitor_trap) {
 			retval = 0;
 			pptr = &vmx->cap[vcpu].proc_ctls;
 			baseval = *pptr;
 			flag = PROCBASED_MTF;
 			reg = VMCS_PRI_PROC_BASED_CTLS;
 		}
 		break;
 	case VM_CAP_PAUSE_EXIT:
 		if (cap_pause_exit) {
 			retval = 0;
 			pptr = &vmx->cap[vcpu].proc_ctls;
 			baseval = *pptr;
 			flag = PROCBASED_PAUSE_EXITING;
 			reg = VMCS_PRI_PROC_BASED_CTLS;
 		}
 		break;
 	case VM_CAP_UNRESTRICTED_GUEST:
 		if (cap_unrestricted_guest) {
 			retval = 0;
 			pptr = &vmx->cap[vcpu].proc_ctls2;
 			baseval = *pptr;
 			flag = PROCBASED2_UNRESTRICTED_GUEST;
 			reg = VMCS_SEC_PROC_BASED_CTLS;
 		}
 		break;
 	case VM_CAP_ENABLE_INVPCID:
 		if (cap_invpcid) {
 			retval = 0;
 			pptr = &vmx->cap[vcpu].proc_ctls2;
 			baseval = *pptr;
 			flag = PROCBASED2_ENABLE_INVPCID;
 			reg = VMCS_SEC_PROC_BASED_CTLS;
 		}
 		break;
 	default:
 		break;
 	}
 
 	if (retval == 0) {
 		if (val) {
 			baseval |= flag;
 		} else {
 			baseval &= ~flag;
 		}
 		VMPTRLD(vmcs);
 		error = vmwrite(reg, baseval);
 		VMCLEAR(vmcs);
 
 		if (error) {
 			retval = error;
 		} else {
 			/*
 			 * Update optional stored flags, and record
 			 * setting
 			 */
 			if (pptr != NULL) {
 				*pptr = baseval;
 			}
 
 			if (val) {
 				vmx->cap[vcpu].set |= (1 << type);
 			} else {
 				vmx->cap[vcpu].set &= ~(1 << type);
 			}
 		}
 	}
 
         return (retval);
 }
 
 struct vlapic_vtx {
 	struct vlapic	vlapic;
 	struct pir_desc	*pir_desc;
 	struct vmx	*vmx;
 };
 
 #define	VMX_CTR_PIR(vm, vcpuid, pir_desc, notify, vector, level, msg)	\
 do {									\
 	VCPU_CTR2(vm, vcpuid, msg " assert %s-triggered vector %d",	\
 	    level ? "level" : "edge", vector);				\
 	VCPU_CTR1(vm, vcpuid, msg " pir0 0x%016lx", pir_desc->pir[0]);	\
 	VCPU_CTR1(vm, vcpuid, msg " pir1 0x%016lx", pir_desc->pir[1]);	\
 	VCPU_CTR1(vm, vcpuid, msg " pir2 0x%016lx", pir_desc->pir[2]);	\
 	VCPU_CTR1(vm, vcpuid, msg " pir3 0x%016lx", pir_desc->pir[3]);	\
 	VCPU_CTR1(vm, vcpuid, msg " notify: %s", notify ? "yes" : "no");\
 } while (0)
 
 /*
  * vlapic->ops handlers that utilize the APICv hardware assist described in
  * Chapter 29 of the Intel SDM.
  */
 static int
 vmx_set_intr_ready(struct vlapic *vlapic, int vector, bool level)
 {
 	struct vlapic_vtx *vlapic_vtx;
 	struct pir_desc *pir_desc;
 	uint64_t mask;
 	int idx, notify;
 
 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
 	pir_desc = vlapic_vtx->pir_desc;
 
 	/*
 	 * Keep track of interrupt requests in the PIR descriptor. This is
 	 * because the virtual APIC page pointed to by the VMCS cannot be
 	 * modified if the vcpu is running.
 	 */
 	idx = vector / 64;
 	mask = 1UL << (vector % 64);
 	atomic_set_long(&pir_desc->pir[idx], mask);
 	notify = atomic_cmpset_long(&pir_desc->pending, 0, 1);
 
 	VMX_CTR_PIR(vlapic->vm, vlapic->vcpuid, pir_desc, notify, vector,
 	    level, "vmx_set_intr_ready");
 	return (notify);
 }
 
 static int
 vmx_pending_intr(struct vlapic *vlapic, int *vecptr)
 {
 	struct vlapic_vtx *vlapic_vtx;
 	struct pir_desc *pir_desc;
 	struct LAPIC *lapic;
 	uint64_t pending, pirval;
 	uint32_t ppr, vpr;
 	int i;
 
 	/*
 	 * This function is only expected to be called from the 'HLT' exit
 	 * handler which does not care about the vector that is pending.
 	 */
 	KASSERT(vecptr == NULL, ("vmx_pending_intr: vecptr must be NULL"));
 
 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
 	pir_desc = vlapic_vtx->pir_desc;
 
 	pending = atomic_load_acq_long(&pir_desc->pending);
 	if (!pending)
 		return (0);	/* common case */
 
 	/*
 	 * If there is an interrupt pending then it will be recognized only
 	 * if its priority is greater than the processor priority.
 	 *
 	 * Special case: if the processor priority is zero then any pending
 	 * interrupt will be recognized.
 	 */
 	lapic = vlapic->apic_page;
 	ppr = lapic->ppr & 0xf0;
 	if (ppr == 0)
 		return (1);
 
 	VCPU_CTR1(vlapic->vm, vlapic->vcpuid, "HLT with non-zero PPR %d",
 	    lapic->ppr);
 
 	for (i = 3; i >= 0; i--) {
 		pirval = pir_desc->pir[i];
 		if (pirval != 0) {
 			vpr = (i * 64 + flsl(pirval) - 1) & 0xf0;
 			return (vpr > ppr);
 		}
 	}
 	return (0);
 }
 
 static void
 vmx_intr_accepted(struct vlapic *vlapic, int vector)
 {
 
 	panic("vmx_intr_accepted: not expected to be called");
 }
 
 static void
 vmx_set_tmr(struct vlapic *vlapic, int vector, bool level)
 {
 	struct vlapic_vtx *vlapic_vtx;
 	struct vmx *vmx;
 	struct vmcs *vmcs;
 	uint64_t mask, val;
 
 	KASSERT(vector >= 0 && vector <= 255, ("invalid vector %d", vector));
 	KASSERT(!vcpu_is_running(vlapic->vm, vlapic->vcpuid, NULL),
 	    ("vmx_set_tmr: vcpu cannot be running"));
 
 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
 	vmx = vlapic_vtx->vmx;
 	vmcs = &vmx->vmcs[vlapic->vcpuid];
 	mask = 1UL << (vector % 64);
 
 	VMPTRLD(vmcs);
 	val = vmcs_read(VMCS_EOI_EXIT(vector));
 	if (level)
 		val |= mask;
 	else
 		val &= ~mask;
 	vmcs_write(VMCS_EOI_EXIT(vector), val);
 	VMCLEAR(vmcs);
 }
 
 static void
 vmx_enable_x2apic_mode(struct vlapic *vlapic)
 {
 	struct vmx *vmx;
 	struct vmcs *vmcs;
 	uint32_t proc_ctls2;
 	int vcpuid, error;
 
 	vcpuid = vlapic->vcpuid;
 	vmx = ((struct vlapic_vtx *)vlapic)->vmx;
 	vmcs = &vmx->vmcs[vcpuid];
 
 	proc_ctls2 = vmx->cap[vcpuid].proc_ctls2;
 	KASSERT((proc_ctls2 & PROCBASED2_VIRTUALIZE_APIC_ACCESSES) != 0,
 	    ("%s: invalid proc_ctls2 %#x", __func__, proc_ctls2));
 
 	proc_ctls2 &= ~PROCBASED2_VIRTUALIZE_APIC_ACCESSES;
 	proc_ctls2 |= PROCBASED2_VIRTUALIZE_X2APIC_MODE;
 	vmx->cap[vcpuid].proc_ctls2 = proc_ctls2;
 
 	VMPTRLD(vmcs);
 	vmcs_write(VMCS_SEC_PROC_BASED_CTLS, proc_ctls2);
 	VMCLEAR(vmcs);
 
 	if (vlapic->vcpuid == 0) {
 		/*
 		 * The nested page table mappings are shared by all vcpus
 		 * so unmap the APIC access page just once.
 		 */
 		error = vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE);
 		KASSERT(error == 0, ("%s: vm_unmap_mmio error %d",
 		    __func__, error));
 
 		/*
 		 * The MSR bitmap is shared by all vcpus so modify it only
 		 * once in the context of vcpu 0.
 		 */
 		error = vmx_allow_x2apic_msrs(vmx);
 		KASSERT(error == 0, ("%s: vmx_allow_x2apic_msrs error %d",
 		    __func__, error));
 	}
 }
 
 static void
 vmx_post_intr(struct vlapic *vlapic, int hostcpu)
 {
 
 	ipi_cpu(hostcpu, pirvec);
 }
 
 /*
  * Transfer the pending interrupts in the PIR descriptor to the IRR
  * in the virtual APIC page.
  */
 static void
 vmx_inject_pir(struct vlapic *vlapic)
 {
 	struct vlapic_vtx *vlapic_vtx;
 	struct pir_desc *pir_desc;
 	struct LAPIC *lapic;
 	uint64_t val, pirval;
 	int rvi, pirbase = -1;
 	uint16_t intr_status_old, intr_status_new;
 
 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
 	pir_desc = vlapic_vtx->pir_desc;
 	if (atomic_cmpset_long(&pir_desc->pending, 1, 0) == 0) {
 		VCPU_CTR0(vlapic->vm, vlapic->vcpuid, "vmx_inject_pir: "
 		    "no posted interrupt pending");
 		return;
 	}
 
 	pirval = 0;
 	pirbase = -1;
 	lapic = vlapic->apic_page;
 
 	val = atomic_readandclear_long(&pir_desc->pir[0]);
 	if (val != 0) {
 		lapic->irr0 |= val;
 		lapic->irr1 |= val >> 32;
 		pirbase = 0;
 		pirval = val;
 	}
 
 	val = atomic_readandclear_long(&pir_desc->pir[1]);
 	if (val != 0) {
 		lapic->irr2 |= val;
 		lapic->irr3 |= val >> 32;
 		pirbase = 64;
 		pirval = val;
 	}
 
 	val = atomic_readandclear_long(&pir_desc->pir[2]);
 	if (val != 0) {
 		lapic->irr4 |= val;
 		lapic->irr5 |= val >> 32;
 		pirbase = 128;
 		pirval = val;
 	}
 
 	val = atomic_readandclear_long(&pir_desc->pir[3]);
 	if (val != 0) {
 		lapic->irr6 |= val;
 		lapic->irr7 |= val >> 32;
 		pirbase = 192;
 		pirval = val;
 	}
 
 	VLAPIC_CTR_IRR(vlapic, "vmx_inject_pir");
 
 	/*
 	 * Update RVI so the processor can evaluate pending virtual
 	 * interrupts on VM-entry.
 	 *
 	 * It is possible for pirval to be 0 here, even though the
 	 * pending bit has been set. The scenario is:
 	 * CPU-Y is sending a posted interrupt to CPU-X, which
 	 * is running a guest and processing posted interrupts in h/w.
 	 * CPU-X will eventually exit and the state seen in s/w is
 	 * the pending bit set, but no PIR bits set.
 	 *
 	 *      CPU-X                      CPU-Y
 	 *   (vm running)                (host running)
 	 *   rx posted interrupt
 	 *   CLEAR pending bit
 	 *				 SET PIR bit
 	 *   READ/CLEAR PIR bits
 	 *				 SET pending bit
 	 *   (vm exit)
 	 *   pending bit set, PIR 0
 	 */
 	if (pirval != 0) {
 		rvi = pirbase + flsl(pirval) - 1;
 		intr_status_old = vmcs_read(VMCS_GUEST_INTR_STATUS);
 		intr_status_new = (intr_status_old & 0xFF00) | rvi;
 		if (intr_status_new > intr_status_old) {
 			vmcs_write(VMCS_GUEST_INTR_STATUS, intr_status_new);
 			VCPU_CTR2(vlapic->vm, vlapic->vcpuid, "vmx_inject_pir: "
 			    "guest_intr_status changed from 0x%04x to 0x%04x",
 			    intr_status_old, intr_status_new);
 		}
 	}
 }
 
 static struct vlapic *
 vmx_vlapic_init(void *arg, int vcpuid)
 {
 	struct vmx *vmx;
 	struct vlapic *vlapic;
 	struct vlapic_vtx *vlapic_vtx;
 	
 	vmx = arg;
 
 	vlapic = malloc(sizeof(struct vlapic_vtx), M_VLAPIC, M_WAITOK | M_ZERO);
 	vlapic->vm = vmx->vm;
 	vlapic->vcpuid = vcpuid;
 	vlapic->apic_page = (struct LAPIC *)&vmx->apic_page[vcpuid];
 
 	vlapic_vtx = (struct vlapic_vtx *)vlapic;
 	vlapic_vtx->pir_desc = &vmx->pir_desc[vcpuid];
 	vlapic_vtx->vmx = vmx;
 
 	if (virtual_interrupt_delivery) {
 		vlapic->ops.set_intr_ready = vmx_set_intr_ready;
 		vlapic->ops.pending_intr = vmx_pending_intr;
 		vlapic->ops.intr_accepted = vmx_intr_accepted;
 		vlapic->ops.set_tmr = vmx_set_tmr;
 		vlapic->ops.enable_x2apic_mode = vmx_enable_x2apic_mode;
 	}
 
 	if (posted_interrupts)
 		vlapic->ops.post_intr = vmx_post_intr;
 
 	vlapic_init(vlapic);
 
 	return (vlapic);
 }
 
 static void
 vmx_vlapic_cleanup(void *arg, struct vlapic *vlapic)
 {
 
 	vlapic_cleanup(vlapic);
 	free(vlapic, M_VLAPIC);
 }
 
 struct vmm_ops vmm_ops_intel = {
 	vmx_init,
 	vmx_cleanup,
 	vmx_restore,
 	vmx_vminit,
 	vmx_run,
 	vmx_vmcleanup,
 	vmx_getreg,
 	vmx_setreg,
 	vmx_getdesc,
 	vmx_setdesc,
 	vmx_getcap,
 	vmx_setcap,
 	ept_vmspace_alloc,
 	ept_vmspace_free,
 	vmx_vlapic_init,
 	vmx_vlapic_cleanup,
 };
Index: projects/release-debugdist/sys/amd64/vmm/vmm.c
===================================================================
--- projects/release-debugdist/sys/amd64/vmm/vmm.c	(revision 267343)
+++ projects/release-debugdist/sys/amd64/vmm/vmm.c	(revision 267344)
@@ -1,1968 +1,1994 @@
 /*-
  * Copyright (c) 2011 NetApp, Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
  * $FreeBSD$
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/module.h>
 #include <sys/sysctl.h>
 #include <sys/malloc.h>
 #include <sys/pcpu.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <sys/proc.h>
 #include <sys/rwlock.h>
 #include <sys/sched.h>
 #include <sys/smp.h>
 #include <sys/systm.h>
 
 #include <vm/vm.h>
 #include <vm/vm_object.h>
 #include <vm/vm_page.h>
 #include <vm/pmap.h>
 #include <vm/vm_map.h>
 #include <vm/vm_extern.h>
 #include <vm/vm_param.h>
 
 #include <machine/cpu.h>
 #include <machine/vm.h>
 #include <machine/pcb.h>
 #include <machine/smp.h>
 #include <x86/psl.h>
 #include <x86/apicreg.h>
 #include <machine/vmparam.h>
 
 #include <machine/vmm.h>
 #include <machine/vmm_dev.h>
 #include <machine/vmm_instruction_emul.h>
 
 #include "vmm_ioport.h"
 #include "vmm_ktr.h"
 #include "vmm_host.h"
 #include "vmm_mem.h"
 #include "vmm_util.h"
 #include "vatpic.h"
 #include "vatpit.h"
 #include "vhpet.h"
 #include "vioapic.h"
 #include "vlapic.h"
 #include "vmm_msr.h"
 #include "vmm_ipi.h"
 #include "vmm_stat.h"
 #include "vmm_lapic.h"
 
 #include "io/ppt.h"
 #include "io/iommu.h"
 
 struct vlapic;
 
 /*
  * Initialization:
  * (a) allocated when vcpu is created
  * (i) initialized when vcpu is created and when it is reinitialized
  * (o) initialized the first time the vcpu is created
  * (x) initialized before use
  */
 struct vcpu {
 	struct mtx 	mtx;		/* (o) protects 'state' and 'hostcpu' */
 	enum vcpu_state	state;		/* (o) vcpu state */
 	int		hostcpu;	/* (o) vcpu's host cpu */
 	struct vlapic	*vlapic;	/* (i) APIC device model */
 	enum x2apic_state x2apic_state;	/* (i) APIC mode */
 	int		nmi_pending;	/* (i) NMI pending */
 	int		extint_pending;	/* (i) INTR pending */
 	struct vm_exception exception;	/* (x) exception collateral */
 	int	exception_pending;	/* (i) exception pending */
 	struct savefpu	*guestfpu;	/* (a,i) guest fpu state */
 	uint64_t	guest_xcr0;	/* (i) guest %xcr0 register */
 	void		*stats;		/* (a,i) statistics */
 	uint64_t guest_msrs[VMM_MSR_NUM]; /* (i) emulated MSRs */
 	struct vm_exit	exitinfo;	/* (x) exit reason and collateral */
 };
 
 #define	vcpu_lock_initialized(v) mtx_initialized(&((v)->mtx))
 #define	vcpu_lock_init(v)	mtx_init(&((v)->mtx), "vcpu lock", 0, MTX_SPIN)
 #define	vcpu_lock(v)		mtx_lock_spin(&((v)->mtx))
 #define	vcpu_unlock(v)		mtx_unlock_spin(&((v)->mtx))
 #define	vcpu_assert_locked(v)	mtx_assert(&((v)->mtx), MA_OWNED)
 
 struct mem_seg {
 	vm_paddr_t	gpa;
 	size_t		len;
 	boolean_t	wired;
 	vm_object_t	object;
 };
 #define	VM_MAX_MEMORY_SEGMENTS	2
 
 /*
  * Initialization:
  * (o) initialized the first time the VM is created
  * (i) initialized when VM is created and when it is reinitialized
  * (x) initialized before use
  */
 struct vm {
 	void		*cookie;		/* (i) cpu-specific data */
 	void		*iommu;			/* (x) iommu-specific data */
 	struct vhpet	*vhpet;			/* (i) virtual HPET */
 	struct vioapic	*vioapic;		/* (i) virtual ioapic */
 	struct vatpic	*vatpic;		/* (i) virtual atpic */
 	struct vatpit	*vatpit;		/* (i) virtual atpit */
 	volatile cpuset_t active_cpus;		/* (i) active vcpus */
 	int		suspend;		/* (i) stop VM execution */
 	volatile cpuset_t suspended_cpus; 	/* (i) suspended vcpus */
 	volatile cpuset_t halted_cpus;		/* (x) cpus in a hard halt */
 	cpuset_t	rendezvous_req_cpus;	/* (x) rendezvous requested */
 	cpuset_t	rendezvous_done_cpus;	/* (x) rendezvous finished */
 	void		*rendezvous_arg;	/* (x) rendezvous func/arg */
 	vm_rendezvous_func_t rendezvous_func;
 	struct mtx	rendezvous_mtx;		/* (o) rendezvous lock */
 	int		num_mem_segs;		/* (o) guest memory segments */
 	struct mem_seg	mem_segs[VM_MAX_MEMORY_SEGMENTS];
 	struct vmspace	*vmspace;		/* (o) guest's address space */
 	char		name[VM_MAX_NAMELEN];	/* (o) virtual machine name */
 	struct vcpu	vcpu[VM_MAXCPU];	/* (i) guest vcpus */
 };
 
 static int vmm_initialized;
 
 static struct vmm_ops *ops;
 #define	VMM_INIT(num)	(ops != NULL ? (*ops->init)(num) : 0)
 #define	VMM_CLEANUP()	(ops != NULL ? (*ops->cleanup)() : 0)
 #define	VMM_RESUME()	(ops != NULL ? (*ops->resume)() : 0)
 
 #define	VMINIT(vm, pmap) (ops != NULL ? (*ops->vminit)(vm, pmap): NULL)
 #define	VMRUN(vmi, vcpu, rip, pmap, rptr, sptr) \
 	(ops != NULL ? (*ops->vmrun)(vmi, vcpu, rip, pmap, rptr, sptr) : ENXIO)
 #define	VMCLEANUP(vmi)	(ops != NULL ? (*ops->vmcleanup)(vmi) : NULL)
 #define	VMSPACE_ALLOC(min, max) \
 	(ops != NULL ? (*ops->vmspace_alloc)(min, max) : NULL)
 #define	VMSPACE_FREE(vmspace) \
 	(ops != NULL ? (*ops->vmspace_free)(vmspace) : ENXIO)
 #define	VMGETREG(vmi, vcpu, num, retval)		\
 	(ops != NULL ? (*ops->vmgetreg)(vmi, vcpu, num, retval) : ENXIO)
 #define	VMSETREG(vmi, vcpu, num, val)		\
 	(ops != NULL ? (*ops->vmsetreg)(vmi, vcpu, num, val) : ENXIO)
 #define	VMGETDESC(vmi, vcpu, num, desc)		\
 	(ops != NULL ? (*ops->vmgetdesc)(vmi, vcpu, num, desc) : ENXIO)
 #define	VMSETDESC(vmi, vcpu, num, desc)		\
 	(ops != NULL ? (*ops->vmsetdesc)(vmi, vcpu, num, desc) : ENXIO)
 #define	VMGETCAP(vmi, vcpu, num, retval)	\
 	(ops != NULL ? (*ops->vmgetcap)(vmi, vcpu, num, retval) : ENXIO)
 #define	VMSETCAP(vmi, vcpu, num, val)		\
 	(ops != NULL ? (*ops->vmsetcap)(vmi, vcpu, num, val) : ENXIO)
 #define	VLAPIC_INIT(vmi, vcpu)			\
 	(ops != NULL ? (*ops->vlapic_init)(vmi, vcpu) : NULL)
 #define	VLAPIC_CLEANUP(vmi, vlapic)		\
 	(ops != NULL ? (*ops->vlapic_cleanup)(vmi, vlapic) : NULL)
 
 #define	fpu_start_emulating()	load_cr0(rcr0() | CR0_TS)
 #define	fpu_stop_emulating()	clts()
 
 static MALLOC_DEFINE(M_VM, "vm", "vm");
 CTASSERT(VMM_MSR_NUM <= 64);	/* msr_mask can keep track of up to 64 msrs */
 
 /* statistics */
 static VMM_STAT(VCPU_TOTAL_RUNTIME, "vcpu total runtime");
 
 SYSCTL_NODE(_hw, OID_AUTO, vmm, CTLFLAG_RW, NULL, NULL);
 
 /*
  * Halt the guest if all vcpus are executing a HLT instruction with
  * interrupts disabled.
  */
 static int halt_detection_enabled = 1;
 TUNABLE_INT("hw.vmm.halt_detection", &halt_detection_enabled);
 SYSCTL_INT(_hw_vmm, OID_AUTO, halt_detection, CTLFLAG_RDTUN,
     &halt_detection_enabled, 0,
     "Halt VM if all vcpus execute HLT with interrupts disabled");
 
 static int vmm_ipinum;
 SYSCTL_INT(_hw_vmm, OID_AUTO, ipinum, CTLFLAG_RD, &vmm_ipinum, 0,
     "IPI vector used for vcpu notifications");
 
 static void
 vcpu_cleanup(struct vm *vm, int i, bool destroy)
 {
 	struct vcpu *vcpu = &vm->vcpu[i];
 
 	VLAPIC_CLEANUP(vm->cookie, vcpu->vlapic);
 	if (destroy) {
 		vmm_stat_free(vcpu->stats);	
 		fpu_save_area_free(vcpu->guestfpu);
 	}
 }
 
 static void
 vcpu_init(struct vm *vm, int vcpu_id, bool create)
 {
 	struct vcpu *vcpu;
 
 	KASSERT(vcpu_id >= 0 && vcpu_id < VM_MAXCPU,
 	    ("vcpu_init: invalid vcpu %d", vcpu_id));
 	  
 	vcpu = &vm->vcpu[vcpu_id];
 
 	if (create) {
 		KASSERT(!vcpu_lock_initialized(vcpu), ("vcpu %d already "
 		    "initialized", vcpu_id));
 		vcpu_lock_init(vcpu);
 		vcpu->state = VCPU_IDLE;
 		vcpu->hostcpu = NOCPU;
 		vcpu->guestfpu = fpu_save_area_alloc();
 		vcpu->stats = vmm_stat_alloc();
 	}
 
 	vcpu->vlapic = VLAPIC_INIT(vm->cookie, vcpu_id);
 	vm_set_x2apic_state(vm, vcpu_id, X2APIC_DISABLED);
 	vcpu->nmi_pending = 0;
 	vcpu->extint_pending = 0;
 	vcpu->exception_pending = 0;
 	vcpu->guest_xcr0 = XFEATURE_ENABLED_X87;
 	fpu_save_area_reset(vcpu->guestfpu);
 	vmm_stat_init(vcpu->stats);
 	guest_msrs_init(vm, vcpu_id);
 }
 
 struct vm_exit *
 vm_exitinfo(struct vm *vm, int cpuid)
 {
 	struct vcpu *vcpu;
 
 	if (cpuid < 0 || cpuid >= VM_MAXCPU)
 		panic("vm_exitinfo: invalid cpuid %d", cpuid);
 
 	vcpu = &vm->vcpu[cpuid];
 
 	return (&vcpu->exitinfo);
 }
 
 static void
 vmm_resume(void)
 {
 	VMM_RESUME();
 }
 
 static int
 vmm_init(void)
 {
 	int error;
 
 	vmm_host_state_init();
 
 	vmm_ipinum = vmm_ipi_alloc();
 	if (vmm_ipinum == 0)
 		vmm_ipinum = IPI_AST;
 
 	error = vmm_mem_init();
 	if (error)
 		return (error);
 	
 	if (vmm_is_intel())
 		ops = &vmm_ops_intel;
 	else if (vmm_is_amd())
 		ops = &vmm_ops_amd;
 	else
 		return (ENXIO);
 
 	vmm_msr_init();
 	vmm_resume_p = vmm_resume;
 
 	return (VMM_INIT(vmm_ipinum));
 }
 
 static int
 vmm_handler(module_t mod, int what, void *arg)
 {
 	int error;
 
 	switch (what) {
 	case MOD_LOAD:
 		vmmdev_init();
 		if (ppt_avail_devices() > 0)
 			iommu_init();
 		error = vmm_init();
 		if (error == 0)
 			vmm_initialized = 1;
 		break;
 	case MOD_UNLOAD:
 		error = vmmdev_cleanup();
 		if (error == 0) {
 			vmm_resume_p = NULL;
 			iommu_cleanup();
 			if (vmm_ipinum != IPI_AST)
 				vmm_ipi_free(vmm_ipinum);
 			error = VMM_CLEANUP();
 			/*
 			 * Something bad happened - prevent new
 			 * VMs from being created
 			 */
 			if (error)
 				vmm_initialized = 0;
 		}
 		break;
 	default:
 		error = 0;
 		break;
 	}
 	return (error);
 }
 
 static moduledata_t vmm_kmod = {
 	"vmm",
 	vmm_handler,
 	NULL
 };
 
 /*
  * vmm initialization has the following dependencies:
  *
  * - iommu initialization must happen after the pci passthru driver has had
  *   a chance to attach to any passthru devices (after SI_SUB_CONFIGURE).
  *
  * - VT-x initialization requires smp_rendezvous() and therefore must happen
  *   after SMP is fully functional (after SI_SUB_SMP).
  */
 DECLARE_MODULE(vmm, vmm_kmod, SI_SUB_SMP + 1, SI_ORDER_ANY);
 MODULE_VERSION(vmm, 1);
 
 static void
 vm_init(struct vm *vm, bool create)
 {
 	int i;
 
 	vm->cookie = VMINIT(vm, vmspace_pmap(vm->vmspace));
 	vm->iommu = NULL;
 	vm->vioapic = vioapic_init(vm);
 	vm->vhpet = vhpet_init(vm);
 	vm->vatpic = vatpic_init(vm);
 	vm->vatpit = vatpit_init(vm);
 
 	CPU_ZERO(&vm->active_cpus);
 
 	vm->suspend = 0;
 	CPU_ZERO(&vm->suspended_cpus);
 
 	for (i = 0; i < VM_MAXCPU; i++)
 		vcpu_init(vm, i, create);
 }
 
 int
 vm_create(const char *name, struct vm **retvm)
 {
 	struct vm *vm;
 	struct vmspace *vmspace;
 
 	/*
 	 * If vmm.ko could not be successfully initialized then don't attempt
 	 * to create the virtual machine.
 	 */
 	if (!vmm_initialized)
 		return (ENXIO);
 
 	if (name == NULL || strlen(name) >= VM_MAX_NAMELEN)
 		return (EINVAL);
 
 	vmspace = VMSPACE_ALLOC(VM_MIN_ADDRESS, VM_MAXUSER_ADDRESS);
 	if (vmspace == NULL)
 		return (ENOMEM);
 
 	vm = malloc(sizeof(struct vm), M_VM, M_WAITOK | M_ZERO);
 	strcpy(vm->name, name);
 	vm->num_mem_segs = 0;
 	vm->vmspace = vmspace;
 	mtx_init(&vm->rendezvous_mtx, "vm rendezvous lock", 0, MTX_DEF);
 
 	vm_init(vm, true);
 
 	*retvm = vm;
 	return (0);
 }
 
 static void
 vm_free_mem_seg(struct vm *vm, struct mem_seg *seg)
 {
 
 	if (seg->object != NULL)
 		vmm_mem_free(vm->vmspace, seg->gpa, seg->len);
 
 	bzero(seg, sizeof(*seg));
 }
 
 static void
 vm_cleanup(struct vm *vm, bool destroy)
 {
 	int i;
 
 	ppt_unassign_all(vm);
 
 	if (vm->iommu != NULL)
 		iommu_destroy_domain(vm->iommu);
 
 	vatpit_cleanup(vm->vatpit);
 	vhpet_cleanup(vm->vhpet);
 	vatpic_cleanup(vm->vatpic);
 	vioapic_cleanup(vm->vioapic);
 
 	for (i = 0; i < VM_MAXCPU; i++)
 		vcpu_cleanup(vm, i, destroy);
 
 	VMCLEANUP(vm->cookie);
 
 	if (destroy) {
 		for (i = 0; i < vm->num_mem_segs; i++)
 			vm_free_mem_seg(vm, &vm->mem_segs[i]);
 
 		vm->num_mem_segs = 0;
 
 		VMSPACE_FREE(vm->vmspace);
 		vm->vmspace = NULL;
 	}
 }
 
 void
 vm_destroy(struct vm *vm)
 {
 	vm_cleanup(vm, true);
 	free(vm, M_VM);
 }
 
 int
 vm_reinit(struct vm *vm)
 {
 	int error;
 
 	/*
 	 * A virtual machine can be reset only if all vcpus are suspended.
 	 */
 	if (CPU_CMP(&vm->suspended_cpus, &vm->active_cpus) == 0) {
 		vm_cleanup(vm, false);
 		vm_init(vm, false);
 		error = 0;
 	} else {
 		error = EBUSY;
 	}
 
 	return (error);
 }
 
 const char *
 vm_name(struct vm *vm)
 {
 	return (vm->name);
 }
 
 int
 vm_map_mmio(struct vm *vm, vm_paddr_t gpa, size_t len, vm_paddr_t hpa)
 {
 	vm_object_t obj;
 
 	if ((obj = vmm_mmio_alloc(vm->vmspace, gpa, len, hpa)) == NULL)
 		return (ENOMEM);
 	else
 		return (0);
 }
 
 int
 vm_unmap_mmio(struct vm *vm, vm_paddr_t gpa, size_t len)
 {
 
 	vmm_mmio_free(vm->vmspace, gpa, len);
 	return (0);
 }
 
 boolean_t
 vm_mem_allocated(struct vm *vm, vm_paddr_t gpa)
 {
 	int i;
 	vm_paddr_t gpabase, gpalimit;
 
 	for (i = 0; i < vm->num_mem_segs; i++) {
 		gpabase = vm->mem_segs[i].gpa;
 		gpalimit = gpabase + vm->mem_segs[i].len;
 		if (gpa >= gpabase && gpa < gpalimit)
 			return (TRUE);		/* 'gpa' is regular memory */
 	}
 
 	if (ppt_is_mmio(vm, gpa))
 		return (TRUE);			/* 'gpa' is pci passthru mmio */
 
 	return (FALSE);
 }
 
 int
 vm_malloc(struct vm *vm, vm_paddr_t gpa, size_t len)
 {
 	int available, allocated;
 	struct mem_seg *seg;
 	vm_object_t object;
 	vm_paddr_t g;
 
 	if ((gpa & PAGE_MASK) || (len & PAGE_MASK) || len == 0)
 		return (EINVAL);
 	
 	available = allocated = 0;
 	g = gpa;
 	while (g < gpa + len) {
 		if (vm_mem_allocated(vm, g))
 			allocated++;
 		else
 			available++;
 
 		g += PAGE_SIZE;
 	}
 
 	/*
 	 * If there are some allocated and some available pages in the address
 	 * range then it is an error.
 	 */
 	if (allocated && available)
 		return (EINVAL);
 
 	/*
 	 * If the entire address range being requested has already been
 	 * allocated then there isn't anything more to do.
 	 */
 	if (allocated && available == 0)
 		return (0);
 
 	if (vm->num_mem_segs >= VM_MAX_MEMORY_SEGMENTS)
 		return (E2BIG);
 
 	seg = &vm->mem_segs[vm->num_mem_segs];
 
 	if ((object = vmm_mem_alloc(vm->vmspace, gpa, len)) == NULL)
 		return (ENOMEM);
 
 	seg->gpa = gpa;
 	seg->len = len;
 	seg->object = object;
 	seg->wired = FALSE;
 
 	vm->num_mem_segs++;
 
 	return (0);
 }
 
 static void
 vm_gpa_unwire(struct vm *vm)
 {
 	int i, rv;
 	struct mem_seg *seg;
 
 	for (i = 0; i < vm->num_mem_segs; i++) {
 		seg = &vm->mem_segs[i];
 		if (!seg->wired)
 			continue;
 
 		rv = vm_map_unwire(&vm->vmspace->vm_map,
 				   seg->gpa, seg->gpa + seg->len,
 				   VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES);
 		KASSERT(rv == KERN_SUCCESS, ("vm(%s) memory segment "
 		    "%#lx/%ld could not be unwired: %d",
 		    vm_name(vm), seg->gpa, seg->len, rv));
 
 		seg->wired = FALSE;
 	}
 }
 
 static int
 vm_gpa_wire(struct vm *vm)
 {
 	int i, rv;
 	struct mem_seg *seg;
 
 	for (i = 0; i < vm->num_mem_segs; i++) {
 		seg = &vm->mem_segs[i];
 		if (seg->wired)
 			continue;
 
 		/* XXX rlimits? */
 		rv = vm_map_wire(&vm->vmspace->vm_map,
 				 seg->gpa, seg->gpa + seg->len,
 				 VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES);
 		if (rv != KERN_SUCCESS)
 			break;
 
 		seg->wired = TRUE;
 	}
 
 	if (i < vm->num_mem_segs) {
 		/*
 		 * Undo the wiring before returning an error.
 		 */
 		vm_gpa_unwire(vm);
 		return (EAGAIN);
 	}
 
 	return (0);
 }
 
 static void
 vm_iommu_modify(struct vm *vm, boolean_t map)
 {
 	int i, sz;
 	vm_paddr_t gpa, hpa;
 	struct mem_seg *seg;
 	void *vp, *cookie, *host_domain;
 
 	sz = PAGE_SIZE;
 	host_domain = iommu_host_domain();
 
 	for (i = 0; i < vm->num_mem_segs; i++) {
 		seg = &vm->mem_segs[i];
 		KASSERT(seg->wired, ("vm(%s) memory segment %#lx/%ld not wired",
 		    vm_name(vm), seg->gpa, seg->len));
 
 		gpa = seg->gpa;
 		while (gpa < seg->gpa + seg->len) {
 			vp = vm_gpa_hold(vm, gpa, PAGE_SIZE, VM_PROT_WRITE,
 					 &cookie);
 			KASSERT(vp != NULL, ("vm(%s) could not map gpa %#lx",
 			    vm_name(vm), gpa));
 
 			vm_gpa_release(cookie);
 
 			hpa = DMAP_TO_PHYS((uintptr_t)vp);
 			if (map) {
 				iommu_create_mapping(vm->iommu, gpa, hpa, sz);
 				iommu_remove_mapping(host_domain, hpa, sz);
 			} else {
 				iommu_remove_mapping(vm->iommu, gpa, sz);
 				iommu_create_mapping(host_domain, hpa, hpa, sz);
 			}
 
 			gpa += PAGE_SIZE;
 		}
 	}
 
 	/*
 	 * Invalidate the cached translations associated with the domain
 	 * from which pages were removed.
 	 */
 	if (map)
 		iommu_invalidate_tlb(host_domain);
 	else
 		iommu_invalidate_tlb(vm->iommu);
 }
 
 #define	vm_iommu_unmap(vm)	vm_iommu_modify((vm), FALSE)
 #define	vm_iommu_map(vm)	vm_iommu_modify((vm), TRUE)
 
 int
 vm_unassign_pptdev(struct vm *vm, int bus, int slot, int func)
 {
 	int error;
 
 	error = ppt_unassign_device(vm, bus, slot, func);
 	if (error)
 		return (error);
 
 	if (ppt_assigned_devices(vm) == 0) {
 		vm_iommu_unmap(vm);
 		vm_gpa_unwire(vm);
 	}
 	return (0);
 }
 
 int
 vm_assign_pptdev(struct vm *vm, int bus, int slot, int func)
 {
 	int error;
 	vm_paddr_t maxaddr;
 
 	/*
 	 * Virtual machines with pci passthru devices get special treatment:
 	 * - the guest physical memory is wired
 	 * - the iommu is programmed to do the 'gpa' to 'hpa' translation
 	 *
 	 * We need to do this before the first pci passthru device is attached.
 	 */
 	if (ppt_assigned_devices(vm) == 0) {
 		KASSERT(vm->iommu == NULL,
 		    ("vm_assign_pptdev: iommu must be NULL"));
 		maxaddr = vmm_mem_maxaddr();
 		vm->iommu = iommu_create_domain(maxaddr);
 
 		error = vm_gpa_wire(vm);
 		if (error)
 			return (error);
 
 		vm_iommu_map(vm);
 	}
 
 	error = ppt_assign_device(vm, bus, slot, func);
 	return (error);
 }
 
 void *
 vm_gpa_hold(struct vm *vm, vm_paddr_t gpa, size_t len, int reqprot,
 	    void **cookie)
 {
 	int count, pageoff;
 	vm_page_t m;
 
 	pageoff = gpa & PAGE_MASK;
 	if (len > PAGE_SIZE - pageoff)
 		panic("vm_gpa_hold: invalid gpa/len: 0x%016lx/%lu", gpa, len);
 
 	count = vm_fault_quick_hold_pages(&vm->vmspace->vm_map,
 	    trunc_page(gpa), PAGE_SIZE, reqprot, &m, 1);
 
 	if (count == 1) {
 		*cookie = m;
 		return ((void *)(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)) + pageoff));
 	} else {
 		*cookie = NULL;
 		return (NULL);
 	}
 }
 
 void
 vm_gpa_release(void *cookie)
 {
 	vm_page_t m = cookie;
 
 	vm_page_lock(m);
 	vm_page_unhold(m);
 	vm_page_unlock(m);
 }
 
 int
 vm_gpabase2memseg(struct vm *vm, vm_paddr_t gpabase,
 		  struct vm_memory_segment *seg)
 {
 	int i;
 
 	for (i = 0; i < vm->num_mem_segs; i++) {
 		if (gpabase == vm->mem_segs[i].gpa) {
 			seg->gpa = vm->mem_segs[i].gpa;
 			seg->len = vm->mem_segs[i].len;
 			seg->wired = vm->mem_segs[i].wired;
 			return (0);
 		}
 	}
 	return (-1);
 }
 
 int
 vm_get_memobj(struct vm *vm, vm_paddr_t gpa, size_t len,
 	      vm_offset_t *offset, struct vm_object **object)
 {
 	int i;
 	size_t seg_len;
 	vm_paddr_t seg_gpa;
 	vm_object_t seg_obj;
 
 	for (i = 0; i < vm->num_mem_segs; i++) {
 		if ((seg_obj = vm->mem_segs[i].object) == NULL)
 			continue;
 
 		seg_gpa = vm->mem_segs[i].gpa;
 		seg_len = vm->mem_segs[i].len;
 
 		if (gpa >= seg_gpa && gpa < seg_gpa + seg_len) {
 			*offset = gpa - seg_gpa;
 			*object = seg_obj;
 			vm_object_reference(seg_obj);
 			return (0);
 		}
 	}
 
 	return (EINVAL);
 }
 
 int
 vm_get_register(struct vm *vm, int vcpu, int reg, uint64_t *retval)
 {
 
 	if (vcpu < 0 || vcpu >= VM_MAXCPU)
 		return (EINVAL);
 
 	if (reg >= VM_REG_LAST)
 		return (EINVAL);
 
 	return (VMGETREG(vm->cookie, vcpu, reg, retval));
 }
 
 int
 vm_set_register(struct vm *vm, int vcpu, int reg, uint64_t val)
 {
 
 	if (vcpu < 0 || vcpu >= VM_MAXCPU)
 		return (EINVAL);
 
 	if (reg >= VM_REG_LAST)
 		return (EINVAL);
 
 	return (VMSETREG(vm->cookie, vcpu, reg, val));
 }
 
 static boolean_t
 is_descriptor_table(int reg)
 {
 
 	switch (reg) {
 	case VM_REG_GUEST_IDTR:
 	case VM_REG_GUEST_GDTR:
 		return (TRUE);
 	default:
 		return (FALSE);
 	}
 }
 
 static boolean_t
 is_segment_register(int reg)
 {
 	
 	switch (reg) {
 	case VM_REG_GUEST_ES:
 	case VM_REG_GUEST_CS:
 	case VM_REG_GUEST_SS:
 	case VM_REG_GUEST_DS:
 	case VM_REG_GUEST_FS:
 	case VM_REG_GUEST_GS:
 	case VM_REG_GUEST_TR:
 	case VM_REG_GUEST_LDTR:
 		return (TRUE);
 	default:
 		return (FALSE);
 	}
 }
 
 int
 vm_get_seg_desc(struct vm *vm, int vcpu, int reg,
 		struct seg_desc *desc)
 {
 
 	if (vcpu < 0 || vcpu >= VM_MAXCPU)
 		return (EINVAL);
 
 	if (!is_segment_register(reg) && !is_descriptor_table(reg))
 		return (EINVAL);
 
 	return (VMGETDESC(vm->cookie, vcpu, reg, desc));
 }
 
 int
 vm_set_seg_desc(struct vm *vm, int vcpu, int reg,
 		struct seg_desc *desc)
 {
 	if (vcpu < 0 || vcpu >= VM_MAXCPU)
 		return (EINVAL);
 
 	if (!is_segment_register(reg) && !is_descriptor_table(reg))
 		return (EINVAL);
 
 	return (VMSETDESC(vm->cookie, vcpu, reg, desc));
 }
 
 static void
 restore_guest_fpustate(struct vcpu *vcpu)
 {
 
 	/* flush host state to the pcb */
 	fpuexit(curthread);
 
 	/* restore guest FPU state */
 	fpu_stop_emulating();
 	fpurestore(vcpu->guestfpu);
 
 	/* restore guest XCR0 if XSAVE is enabled in the host */
 	if (rcr4() & CR4_XSAVE)
 		load_xcr(0, vcpu->guest_xcr0);
 
 	/*
 	 * The FPU is now "dirty" with the guest's state so turn on emulation
 	 * to trap any access to the FPU by the host.
 	 */
 	fpu_start_emulating();
 }
 
 static void
 save_guest_fpustate(struct vcpu *vcpu)
 {
 
 	if ((rcr0() & CR0_TS) == 0)
 		panic("fpu emulation not enabled in host!");
 
 	/* save guest XCR0 and restore host XCR0 */
 	if (rcr4() & CR4_XSAVE) {
 		vcpu->guest_xcr0 = rxcr(0);
 		load_xcr(0, vmm_get_host_xcr0());
 	}
 
 	/* save guest FPU state */
 	fpu_stop_emulating();
 	fpusave(vcpu->guestfpu);
 	fpu_start_emulating();
 }
 
 static VMM_STAT(VCPU_IDLE_TICKS, "number of ticks vcpu was idle");
 
 static int
 vcpu_set_state_locked(struct vcpu *vcpu, enum vcpu_state newstate,
     bool from_idle)
 {
 	int error;
 
 	vcpu_assert_locked(vcpu);
 
 	/*
 	 * State transitions from the vmmdev_ioctl() must always begin from
 	 * the VCPU_IDLE state. This guarantees that there is only a single
 	 * ioctl() operating on a vcpu at any point.
 	 */
 	if (from_idle) {
 		while (vcpu->state != VCPU_IDLE)
 			msleep_spin(&vcpu->state, &vcpu->mtx, "vmstat", hz);
 	} else {
 		KASSERT(vcpu->state != VCPU_IDLE, ("invalid transition from "
 		    "vcpu idle state"));
 	}
 
 	if (vcpu->state == VCPU_RUNNING) {
 		KASSERT(vcpu->hostcpu == curcpu, ("curcpu %d and hostcpu %d "
 		    "mismatch for running vcpu", curcpu, vcpu->hostcpu));
 	} else {
 		KASSERT(vcpu->hostcpu == NOCPU, ("Invalid hostcpu %d for a "
 		    "vcpu that is not running", vcpu->hostcpu));
 	}
 
 	/*
 	 * The following state transitions are allowed:
 	 * IDLE -> FROZEN -> IDLE
 	 * FROZEN -> RUNNING -> FROZEN
 	 * FROZEN -> SLEEPING -> FROZEN
 	 */
 	switch (vcpu->state) {
 	case VCPU_IDLE:
 	case VCPU_RUNNING:
 	case VCPU_SLEEPING:
 		error = (newstate != VCPU_FROZEN);
 		break;
 	case VCPU_FROZEN:
 		error = (newstate == VCPU_FROZEN);
 		break;
 	default:
 		error = 1;
 		break;
 	}
 
 	if (error)
 		return (EBUSY);
 
 	vcpu->state = newstate;
 	if (newstate == VCPU_RUNNING)
 		vcpu->hostcpu = curcpu;
 	else
 		vcpu->hostcpu = NOCPU;
 
 	if (newstate == VCPU_IDLE)
 		wakeup(&vcpu->state);
 
 	return (0);
 }
 
 static void
 vcpu_require_state(struct vm *vm, int vcpuid, enum vcpu_state newstate)
 {
 	int error;
 
 	if ((error = vcpu_set_state(vm, vcpuid, newstate, false)) != 0)
 		panic("Error %d setting state to %d\n", error, newstate);
 }
 
 static void
 vcpu_require_state_locked(struct vcpu *vcpu, enum vcpu_state newstate)
 {
 	int error;
 
 	if ((error = vcpu_set_state_locked(vcpu, newstate, false)) != 0)
 		panic("Error %d setting state to %d", error, newstate);
 }
 
 static void
 vm_set_rendezvous_func(struct vm *vm, vm_rendezvous_func_t func)
 {
 
 	KASSERT(mtx_owned(&vm->rendezvous_mtx), ("rendezvous_mtx not locked"));
 
 	/*
 	 * Update 'rendezvous_func' and execute a write memory barrier to
 	 * ensure that it is visible across all host cpus. This is not needed
 	 * for correctness but it does ensure that all the vcpus will notice
 	 * that the rendezvous is requested immediately.
 	 */
 	vm->rendezvous_func = func;
 	wmb();
 }
 
 #define	RENDEZVOUS_CTR0(vm, vcpuid, fmt)				\
 	do {								\
 		if (vcpuid >= 0)					\
 			VCPU_CTR0(vm, vcpuid, fmt);			\
 		else							\
 			VM_CTR0(vm, fmt);				\
 	} while (0)
 
 static void
 vm_handle_rendezvous(struct vm *vm, int vcpuid)
 {
 
 	KASSERT(vcpuid == -1 || (vcpuid >= 0 && vcpuid < VM_MAXCPU),
 	    ("vm_handle_rendezvous: invalid vcpuid %d", vcpuid));
 
 	mtx_lock(&vm->rendezvous_mtx);
 	while (vm->rendezvous_func != NULL) {
 		/* 'rendezvous_req_cpus' must be a subset of 'active_cpus' */
 		CPU_AND(&vm->rendezvous_req_cpus, &vm->active_cpus);
 
 		if (vcpuid != -1 &&
 		    CPU_ISSET(vcpuid, &vm->rendezvous_req_cpus) &&
 		    !CPU_ISSET(vcpuid, &vm->rendezvous_done_cpus)) {
 			VCPU_CTR0(vm, vcpuid, "Calling rendezvous func");
 			(*vm->rendezvous_func)(vm, vcpuid, vm->rendezvous_arg);
 			CPU_SET(vcpuid, &vm->rendezvous_done_cpus);
 		}
 		if (CPU_CMP(&vm->rendezvous_req_cpus,
 		    &vm->rendezvous_done_cpus) == 0) {
 			VCPU_CTR0(vm, vcpuid, "Rendezvous completed");
 			vm_set_rendezvous_func(vm, NULL);
 			wakeup(&vm->rendezvous_func);
 			break;
 		}
 		RENDEZVOUS_CTR0(vm, vcpuid, "Wait for rendezvous completion");
 		mtx_sleep(&vm->rendezvous_func, &vm->rendezvous_mtx, 0,
 		    "vmrndv", 0);
 	}
 	mtx_unlock(&vm->rendezvous_mtx);
 }
 
 /*
  * Emulate a guest 'hlt' by sleeping until the vcpu is ready to run.
  */
 static int
 vm_handle_hlt(struct vm *vm, int vcpuid, bool intr_disabled, bool *retu)
 {
 	struct vcpu *vcpu;
 	const char *wmesg;
 	int t, vcpu_halted, vm_halted;
 
 	KASSERT(!CPU_ISSET(vcpuid, &vm->halted_cpus), ("vcpu already halted"));
 
 	vcpu = &vm->vcpu[vcpuid];
 	vcpu_halted = 0;
 	vm_halted = 0;
 
 	vcpu_lock(vcpu);
 	while (1) {
 		/*
 		 * Do a final check for pending NMI or interrupts before
 		 * really putting this thread to sleep. Also check for
 		 * software events that would cause this vcpu to wakeup.
 		 *
 		 * These interrupts/events could have happened after the
 		 * vcpu returned from VMRUN() and before it acquired the
 		 * vcpu lock above.
 		 */
 		if (vm->rendezvous_func != NULL || vm->suspend)
 			break;
 		if (vm_nmi_pending(vm, vcpuid))
 			break;
 		if (!intr_disabled) {
 			if (vm_extint_pending(vm, vcpuid) ||
 			    vlapic_pending_intr(vcpu->vlapic, NULL)) {
 				break;
 			}
 		}
 
 		/*
 		 * Some Linux guests implement "halt" by having all vcpus
 		 * execute HLT with interrupts disabled. 'halted_cpus' keeps
 		 * track of the vcpus that have entered this state. When all
 		 * vcpus enter the halted state the virtual machine is halted.
 		 */
 		if (intr_disabled) {
 			wmesg = "vmhalt";
 			VCPU_CTR0(vm, vcpuid, "Halted");
 			if (!vcpu_halted && halt_detection_enabled) {
 				vcpu_halted = 1;
 				CPU_SET_ATOMIC(vcpuid, &vm->halted_cpus);
 			}
 			if (CPU_CMP(&vm->halted_cpus, &vm->active_cpus) == 0) {
 				vm_halted = 1;
 				break;
 			}
 		} else {
 			wmesg = "vmidle";
 		}
 
 		t = ticks;
 		vcpu_require_state_locked(vcpu, VCPU_SLEEPING);
 		msleep_spin(vcpu, &vcpu->mtx, wmesg, 0);
 		vcpu_require_state_locked(vcpu, VCPU_FROZEN);
 		vmm_stat_incr(vm, vcpuid, VCPU_IDLE_TICKS, ticks - t);
 	}
 
 	if (vcpu_halted)
 		CPU_CLR_ATOMIC(vcpuid, &vm->halted_cpus);
 
 	vcpu_unlock(vcpu);
 
 	if (vm_halted)
 		vm_suspend(vm, VM_SUSPEND_HALT);
 
 	return (0);
 }
 
 static int
 vm_handle_paging(struct vm *vm, int vcpuid, bool *retu)
 {
 	int rv, ftype;
 	struct vm_map *map;
 	struct vcpu *vcpu;
 	struct vm_exit *vme;
 
 	vcpu = &vm->vcpu[vcpuid];
 	vme = &vcpu->exitinfo;
 
 	ftype = vme->u.paging.fault_type;
 	KASSERT(ftype == VM_PROT_READ ||
 	    ftype == VM_PROT_WRITE || ftype == VM_PROT_EXECUTE,
 	    ("vm_handle_paging: invalid fault_type %d", ftype));
 
 	if (ftype == VM_PROT_READ || ftype == VM_PROT_WRITE) {
 		rv = pmap_emulate_accessed_dirty(vmspace_pmap(vm->vmspace),
 		    vme->u.paging.gpa, ftype);
 		if (rv == 0)
 			goto done;
 	}
 
 	map = &vm->vmspace->vm_map;
 	rv = vm_fault(map, vme->u.paging.gpa, ftype, VM_FAULT_NORMAL);
 
 	VCPU_CTR3(vm, vcpuid, "vm_handle_paging rv = %d, gpa = %#lx, "
 	    "ftype = %d", rv, vme->u.paging.gpa, ftype);
 
 	if (rv != KERN_SUCCESS)
 		return (EFAULT);
 done:
 	/* restart execution at the faulting instruction */
 	vme->inst_length = 0;
 
 	return (0);
 }
 
 static int
 vm_handle_inst_emul(struct vm *vm, int vcpuid, bool *retu)
 {
 	struct vie *vie;
 	struct vcpu *vcpu;
 	struct vm_exit *vme;
 	uint64_t gla, gpa;
 	struct vm_guest_paging *paging;
 	mem_region_read_t mread;
 	mem_region_write_t mwrite;
 	int error;
 
 	vcpu = &vm->vcpu[vcpuid];
 	vme = &vcpu->exitinfo;
 
 	gla = vme->u.inst_emul.gla;
 	gpa = vme->u.inst_emul.gpa;
 	vie = &vme->u.inst_emul.vie;
 	paging = &vme->u.inst_emul.paging;
 
 	vie_init(vie);
 
 	/* Fetch, decode and emulate the faulting instruction */
 	error = vmm_fetch_instruction(vm, vcpuid, paging, vme->rip,
 	    vme->inst_length, vie);
 	if (error == 1)
 		return (0);		/* Resume guest to handle page fault */
 	else if (error == -1)
 		return (EFAULT);
 	else if (error != 0)
 		panic("%s: vmm_fetch_instruction error %d", __func__, error);
 
 	if (vmm_decode_instruction(vm, vcpuid, gla, paging->cpu_mode, vie) != 0)
 		return (EFAULT);
 
 	/* return to userland unless this is an in-kernel emulated device */
 	if (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + PAGE_SIZE) {
 		mread = lapic_mmio_read;
 		mwrite = lapic_mmio_write;
 	} else if (gpa >= VIOAPIC_BASE && gpa < VIOAPIC_BASE + VIOAPIC_SIZE) {
 		mread = vioapic_mmio_read;
 		mwrite = vioapic_mmio_write;
 	} else if (gpa >= VHPET_BASE && gpa < VHPET_BASE + VHPET_SIZE) {
 		mread = vhpet_mmio_read;
 		mwrite = vhpet_mmio_write;
 	} else {
 		*retu = true;
 		return (0);
 	}
 
 	error = vmm_emulate_instruction(vm, vcpuid, gpa, vie, mread, mwrite,
 	    retu);
 
 	return (error);
 }
 
 static int
 vm_handle_suspend(struct vm *vm, int vcpuid, bool *retu)
 {
 	int i, done;
 	struct vcpu *vcpu;
 
 	done = 0;
 	vcpu = &vm->vcpu[vcpuid];
 
 	CPU_SET_ATOMIC(vcpuid, &vm->suspended_cpus);
 
 	/*
 	 * Wait until all 'active_cpus' have suspended themselves.
 	 *
 	 * Since a VM may be suspended at any time including when one or
 	 * more vcpus are doing a rendezvous we need to call the rendezvous
 	 * handler while we are waiting to prevent a deadlock.
 	 */
 	vcpu_lock(vcpu);
 	while (1) {
 		if (CPU_CMP(&vm->suspended_cpus, &vm->active_cpus) == 0) {
 			VCPU_CTR0(vm, vcpuid, "All vcpus suspended");
 			break;
 		}
 
 		if (vm->rendezvous_func == NULL) {
 			VCPU_CTR0(vm, vcpuid, "Sleeping during suspend");
 			vcpu_require_state_locked(vcpu, VCPU_SLEEPING);
 			msleep_spin(vcpu, &vcpu->mtx, "vmsusp", hz);
 			vcpu_require_state_locked(vcpu, VCPU_FROZEN);
 		} else {
 			VCPU_CTR0(vm, vcpuid, "Rendezvous during suspend");
 			vcpu_unlock(vcpu);
 			vm_handle_rendezvous(vm, vcpuid);
 			vcpu_lock(vcpu);
 		}
 	}
 	vcpu_unlock(vcpu);
 
 	/*
 	 * Wakeup the other sleeping vcpus and return to userspace.
 	 */
 	for (i = 0; i < VM_MAXCPU; i++) {
 		if (CPU_ISSET(i, &vm->suspended_cpus)) {
 			vcpu_notify_event(vm, i, false);
 		}
 	}
 
 	*retu = true;
 	return (0);
 }
 
 int
 vm_suspend(struct vm *vm, enum vm_suspend_how how)
 {
 	int i;
 
 	if (how <= VM_SUSPEND_NONE || how >= VM_SUSPEND_LAST)
 		return (EINVAL);
 
 	if (atomic_cmpset_int(&vm->suspend, 0, how) == 0) {
 		VM_CTR2(vm, "virtual machine already suspended %d/%d",
 		    vm->suspend, how);
 		return (EALREADY);
 	}
 
 	VM_CTR1(vm, "virtual machine successfully suspended %d", how);
 
 	/*
 	 * Notify all active vcpus that they are now suspended.
 	 */
 	for (i = 0; i < VM_MAXCPU; i++) {
 		if (CPU_ISSET(i, &vm->active_cpus))
 			vcpu_notify_event(vm, i, false);
 	}
 
 	return (0);
 }
 
 void
 vm_exit_suspended(struct vm *vm, int vcpuid, uint64_t rip)
 {
 	struct vm_exit *vmexit;
 
 	KASSERT(vm->suspend > VM_SUSPEND_NONE && vm->suspend < VM_SUSPEND_LAST,
 	    ("vm_exit_suspended: invalid suspend type %d", vm->suspend));
 
 	vmexit = vm_exitinfo(vm, vcpuid);
 	vmexit->rip = rip;
 	vmexit->inst_length = 0;
 	vmexit->exitcode = VM_EXITCODE_SUSPENDED;
 	vmexit->u.suspended.how = vm->suspend;
 }
 
+void
+vm_exit_rendezvous(struct vm *vm, int vcpuid, uint64_t rip)
+{
+	struct vm_exit *vmexit;
+
+	KASSERT(vm->rendezvous_func != NULL, ("rendezvous not in progress"));
+
+	vmexit = vm_exitinfo(vm, vcpuid);
+	vmexit->rip = rip;
+	vmexit->inst_length = 0;
+	vmexit->exitcode = VM_EXITCODE_RENDEZVOUS;
+	vmm_stat_incr(vm, vcpuid, VMEXIT_RENDEZVOUS, 1);
+}
+
+void
+vm_exit_astpending(struct vm *vm, int vcpuid, uint64_t rip)
+{
+	struct vm_exit *vmexit;
+
+	vmexit = vm_exitinfo(vm, vcpuid);
+	vmexit->rip = rip;
+	vmexit->inst_length = 0;
+	vmexit->exitcode = VM_EXITCODE_BOGUS;
+	vmm_stat_incr(vm, vcpuid, VMEXIT_ASTPENDING, 1);
+}
+
 int
 vm_run(struct vm *vm, struct vm_run *vmrun)
 {
 	int error, vcpuid;
 	struct vcpu *vcpu;
 	struct pcb *pcb;
 	uint64_t tscval, rip;
 	struct vm_exit *vme;
 	bool retu, intr_disabled;
 	pmap_t pmap;
 	void *rptr, *sptr;
 
 	vcpuid = vmrun->cpuid;
 
 	if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
 		return (EINVAL);
 
 	if (!CPU_ISSET(vcpuid, &vm->active_cpus))
 		return (EINVAL);
 
 	if (CPU_ISSET(vcpuid, &vm->suspended_cpus))
 		return (EINVAL);
 
 	rptr = &vm->rendezvous_func;
 	sptr = &vm->suspend;
 	pmap = vmspace_pmap(vm->vmspace);
 	vcpu = &vm->vcpu[vcpuid];
 	vme = &vcpu->exitinfo;
 	rip = vmrun->rip;
 restart:
 	critical_enter();
 
 	KASSERT(!CPU_ISSET(curcpu, &pmap->pm_active),
 	    ("vm_run: absurd pm_active"));
 
 	tscval = rdtsc();
 
 	pcb = PCPU_GET(curpcb);
 	set_pcb_flags(pcb, PCB_FULL_IRET);
 
 	restore_guest_msrs(vm, vcpuid);	
 	restore_guest_fpustate(vcpu);
 
 	vcpu_require_state(vm, vcpuid, VCPU_RUNNING);
 	error = VMRUN(vm->cookie, vcpuid, rip, pmap, rptr, sptr);
 	vcpu_require_state(vm, vcpuid, VCPU_FROZEN);
 
 	save_guest_fpustate(vcpu);
 	restore_host_msrs(vm, vcpuid);
 
 	vmm_stat_incr(vm, vcpuid, VCPU_TOTAL_RUNTIME, rdtsc() - tscval);
 
 	critical_exit();
 
 	if (error == 0) {
 		retu = false;
 		switch (vme->exitcode) {
 		case VM_EXITCODE_SUSPENDED:
 			error = vm_handle_suspend(vm, vcpuid, &retu);
 			break;
 		case VM_EXITCODE_IOAPIC_EOI:
 			vioapic_process_eoi(vm, vcpuid,
 			    vme->u.ioapic_eoi.vector);
 			break;
 		case VM_EXITCODE_RENDEZVOUS:
 			vm_handle_rendezvous(vm, vcpuid);
 			error = 0;
 			break;
 		case VM_EXITCODE_HLT:
 			intr_disabled = ((vme->u.hlt.rflags & PSL_I) == 0);
 			error = vm_handle_hlt(vm, vcpuid, intr_disabled, &retu);
 			break;
 		case VM_EXITCODE_PAGING:
 			error = vm_handle_paging(vm, vcpuid, &retu);
 			break;
 		case VM_EXITCODE_INST_EMUL:
 			error = vm_handle_inst_emul(vm, vcpuid, &retu);
 			break;
 		case VM_EXITCODE_INOUT:
 		case VM_EXITCODE_INOUT_STR:
 			error = vm_handle_inout(vm, vcpuid, vme, &retu);
 			break;
 		default:
 			retu = true;	/* handled in userland */
 			break;
 		}
 	}
 
 	if (error == 0 && retu == false) {
 		rip = vme->rip + vme->inst_length;
 		goto restart;
 	}
 
 	/* copy the exit information */
 	bcopy(vme, &vmrun->vm_exit, sizeof(struct vm_exit));
 	return (error);
 }
 
 int
 vm_inject_exception(struct vm *vm, int vcpuid, struct vm_exception *exception)
 {
 	struct vcpu *vcpu;
 
 	if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
 		return (EINVAL);
 
 	if (exception->vector < 0 || exception->vector >= 32)
 		return (EINVAL);
 
 	vcpu = &vm->vcpu[vcpuid];
 
 	if (vcpu->exception_pending) {
 		VCPU_CTR2(vm, vcpuid, "Unable to inject exception %d due to "
 		    "pending exception %d", exception->vector,
 		    vcpu->exception.vector);
 		return (EBUSY);
 	}
 
 	vcpu->exception_pending = 1;
 	vcpu->exception = *exception;
 	VCPU_CTR1(vm, vcpuid, "Exception %d pending", exception->vector);
 	return (0);
 }
 
 int
 vm_exception_pending(struct vm *vm, int vcpuid, struct vm_exception *exception)
 {
 	struct vcpu *vcpu;
 	int pending;
 
 	KASSERT(vcpuid >= 0 && vcpuid < VM_MAXCPU, ("invalid vcpu %d", vcpuid));
 
 	vcpu = &vm->vcpu[vcpuid];
 	pending = vcpu->exception_pending;
 	if (pending) {
 		vcpu->exception_pending = 0;
 		*exception = vcpu->exception;
 		VCPU_CTR1(vm, vcpuid, "Exception %d delivered",
 		    exception->vector);
 	}
 	return (pending);
 }
 
 static void
 vm_inject_fault(struct vm *vm, int vcpuid, struct vm_exception *exception)
 {
 	struct vm_exit *vmexit;
 	int error;
 
 	error = vm_inject_exception(vm, vcpuid, exception);
 	KASSERT(error == 0, ("vm_inject_exception error %d", error));
 
 	/*
 	 * A fault-like exception allows the instruction to be restarted
 	 * after the exception handler returns.
 	 *
 	 * By setting the inst_length to 0 we ensure that the instruction
 	 * pointer remains at the faulting instruction.
 	 */
 	vmexit = vm_exitinfo(vm, vcpuid);
 	vmexit->inst_length = 0;
 }
 
 void
 vm_inject_pf(struct vm *vm, int vcpuid, int error_code, uint64_t cr2)
 {
 	struct vm_exception pf = {
 		.vector = IDT_PF,
 		.error_code_valid = 1,
 		.error_code = error_code
 	};
 	int error;
 
 	VCPU_CTR2(vm, vcpuid, "Injecting page fault: error_code %#x, cr2 %#lx",
 	    error_code, cr2);
 
 	error = vm_set_register(vm, vcpuid, VM_REG_GUEST_CR2, cr2);
 	KASSERT(error == 0, ("vm_set_register(cr2) error %d", error));
 
 	vm_inject_fault(vm, vcpuid, &pf);
 }
 
 void
 vm_inject_gp(struct vm *vm, int vcpuid)
 {
 	struct vm_exception gpf = {
 		.vector = IDT_GP,
 		.error_code_valid = 1,
 		.error_code = 0
 	};
 
 	vm_inject_fault(vm, vcpuid, &gpf);
 }
 
 void
 vm_inject_ud(struct vm *vm, int vcpuid)
 {
 	struct vm_exception udf = {
 		.vector = IDT_UD,
 		.error_code_valid = 0
 	};
 
 	vm_inject_fault(vm, vcpuid, &udf);
 }
 
 static VMM_STAT(VCPU_NMI_COUNT, "number of NMIs delivered to vcpu");
 
 int
 vm_inject_nmi(struct vm *vm, int vcpuid)
 {
 	struct vcpu *vcpu;
 
 	if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
 		return (EINVAL);
 
 	vcpu = &vm->vcpu[vcpuid];
 
 	vcpu->nmi_pending = 1;
 	vcpu_notify_event(vm, vcpuid, false);
 	return (0);
 }
 
 int
 vm_nmi_pending(struct vm *vm, int vcpuid)
 {
 	struct vcpu *vcpu;
 
 	if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
 		panic("vm_nmi_pending: invalid vcpuid %d", vcpuid);
 
 	vcpu = &vm->vcpu[vcpuid];
 
 	return (vcpu->nmi_pending);
 }
 
 void
 vm_nmi_clear(struct vm *vm, int vcpuid)
 {
 	struct vcpu *vcpu;
 
 	if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
 		panic("vm_nmi_pending: invalid vcpuid %d", vcpuid);
 
 	vcpu = &vm->vcpu[vcpuid];
 
 	if (vcpu->nmi_pending == 0)
 		panic("vm_nmi_clear: inconsistent nmi_pending state");
 
 	vcpu->nmi_pending = 0;
 	vmm_stat_incr(vm, vcpuid, VCPU_NMI_COUNT, 1);
 }
 
 static VMM_STAT(VCPU_EXTINT_COUNT, "number of ExtINTs delivered to vcpu");
 
 int
 vm_inject_extint(struct vm *vm, int vcpuid)
 {
 	struct vcpu *vcpu;
 
 	if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
 		return (EINVAL);
 
 	vcpu = &vm->vcpu[vcpuid];
 
 	vcpu->extint_pending = 1;
 	vcpu_notify_event(vm, vcpuid, false);
 	return (0);
 }
 
 int
 vm_extint_pending(struct vm *vm, int vcpuid)
 {
 	struct vcpu *vcpu;
 
 	if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
 		panic("vm_extint_pending: invalid vcpuid %d", vcpuid);
 
 	vcpu = &vm->vcpu[vcpuid];
 
 	return (vcpu->extint_pending);
 }
 
 void
 vm_extint_clear(struct vm *vm, int vcpuid)
 {
 	struct vcpu *vcpu;
 
 	if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
 		panic("vm_extint_pending: invalid vcpuid %d", vcpuid);
 
 	vcpu = &vm->vcpu[vcpuid];
 
 	if (vcpu->extint_pending == 0)
 		panic("vm_extint_clear: inconsistent extint_pending state");
 
 	vcpu->extint_pending = 0;
 	vmm_stat_incr(vm, vcpuid, VCPU_EXTINT_COUNT, 1);
 }
 
 int
 vm_get_capability(struct vm *vm, int vcpu, int type, int *retval)
 {
 	if (vcpu < 0 || vcpu >= VM_MAXCPU)
 		return (EINVAL);
 
 	if (type < 0 || type >= VM_CAP_MAX)
 		return (EINVAL);
 
 	return (VMGETCAP(vm->cookie, vcpu, type, retval));
 }
 
 int
 vm_set_capability(struct vm *vm, int vcpu, int type, int val)
 {
 	if (vcpu < 0 || vcpu >= VM_MAXCPU)
 		return (EINVAL);
 
 	if (type < 0 || type >= VM_CAP_MAX)
 		return (EINVAL);
 
 	return (VMSETCAP(vm->cookie, vcpu, type, val));
 }
 
 uint64_t *
 vm_guest_msrs(struct vm *vm, int cpu)
 {
 	return (vm->vcpu[cpu].guest_msrs);
 }
 
 struct vlapic *
 vm_lapic(struct vm *vm, int cpu)
 {
 	return (vm->vcpu[cpu].vlapic);
 }
 
 struct vioapic *
 vm_ioapic(struct vm *vm)
 {
 
 	return (vm->vioapic);
 }
 
 struct vhpet *
 vm_hpet(struct vm *vm)
 {
 
 	return (vm->vhpet);
 }
 
 boolean_t
 vmm_is_pptdev(int bus, int slot, int func)
 {
 	int found, i, n;
 	int b, s, f;
 	char *val, *cp, *cp2;
 
 	/*
 	 * XXX
 	 * The length of an environment variable is limited to 128 bytes which
 	 * puts an upper limit on the number of passthru devices that may be
 	 * specified using a single environment variable.
 	 *
 	 * Work around this by scanning multiple environment variable
 	 * names instead of a single one - yuck!
 	 */
 	const char *names[] = { "pptdevs", "pptdevs2", "pptdevs3", NULL };
 
 	/* set pptdevs="1/2/3 4/5/6 7/8/9 10/11/12" */
 	found = 0;
 	for (i = 0; names[i] != NULL && !found; i++) {
 		cp = val = getenv(names[i]);
 		while (cp != NULL && *cp != '\0') {
 			if ((cp2 = strchr(cp, ' ')) != NULL)
 				*cp2 = '\0';
 
 			n = sscanf(cp, "%d/%d/%d", &b, &s, &f);
 			if (n == 3 && bus == b && slot == s && func == f) {
 				found = 1;
 				break;
 			}
 		
 			if (cp2 != NULL)
 				*cp2++ = ' ';
 
 			cp = cp2;
 		}
 		freeenv(val);
 	}
 	return (found);
 }
 
 void *
 vm_iommu_domain(struct vm *vm)
 {
 
 	return (vm->iommu);
 }
 
 int
 vcpu_set_state(struct vm *vm, int vcpuid, enum vcpu_state newstate,
     bool from_idle)
 {
 	int error;
 	struct vcpu *vcpu;
 
 	if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
 		panic("vm_set_run_state: invalid vcpuid %d", vcpuid);
 
 	vcpu = &vm->vcpu[vcpuid];
 
 	vcpu_lock(vcpu);
 	error = vcpu_set_state_locked(vcpu, newstate, from_idle);
 	vcpu_unlock(vcpu);
 
 	return (error);
 }
 
 enum vcpu_state
 vcpu_get_state(struct vm *vm, int vcpuid, int *hostcpu)
 {
 	struct vcpu *vcpu;
 	enum vcpu_state state;
 
 	if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
 		panic("vm_get_run_state: invalid vcpuid %d", vcpuid);
 
 	vcpu = &vm->vcpu[vcpuid];
 
 	vcpu_lock(vcpu);
 	state = vcpu->state;
 	if (hostcpu != NULL)
 		*hostcpu = vcpu->hostcpu;
 	vcpu_unlock(vcpu);
 
 	return (state);
 }
 
 int
 vm_activate_cpu(struct vm *vm, int vcpuid)
 {
 
 	if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
 		return (EINVAL);
 
 	if (CPU_ISSET(vcpuid, &vm->active_cpus))
 		return (EBUSY);
 
 	VCPU_CTR0(vm, vcpuid, "activated");
 	CPU_SET_ATOMIC(vcpuid, &vm->active_cpus);
 	return (0);
 }
 
 cpuset_t
 vm_active_cpus(struct vm *vm)
 {
 
 	return (vm->active_cpus);
 }
 
 cpuset_t
 vm_suspended_cpus(struct vm *vm)
 {
 
 	return (vm->suspended_cpus);
 }
 
 void *
 vcpu_stats(struct vm *vm, int vcpuid)
 {
 
 	return (vm->vcpu[vcpuid].stats);
 }
 
 int
 vm_get_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state *state)
 {
 	if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
 		return (EINVAL);
 
 	*state = vm->vcpu[vcpuid].x2apic_state;
 
 	return (0);
 }
 
 int
 vm_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state)
 {
 	if (vcpuid < 0 || vcpuid >= VM_MAXCPU)
 		return (EINVAL);
 
 	if (state >= X2APIC_STATE_LAST)
 		return (EINVAL);
 
 	vm->vcpu[vcpuid].x2apic_state = state;
 
 	vlapic_set_x2apic_state(vm, vcpuid, state);
 
 	return (0);
 }
 
 /*
  * This function is called to ensure that a vcpu "sees" a pending event
  * as soon as possible:
  * - If the vcpu thread is sleeping then it is woken up.
  * - If the vcpu is running on a different host_cpu then an IPI will be directed
  *   to the host_cpu to cause the vcpu to trap into the hypervisor.
  */
 void
 vcpu_notify_event(struct vm *vm, int vcpuid, bool lapic_intr)
 {
 	int hostcpu;
 	struct vcpu *vcpu;
 
 	vcpu = &vm->vcpu[vcpuid];
 
 	vcpu_lock(vcpu);
 	hostcpu = vcpu->hostcpu;
 	if (vcpu->state == VCPU_RUNNING) {
 		KASSERT(hostcpu != NOCPU, ("vcpu running on invalid hostcpu"));
 		if (hostcpu != curcpu) {
 			if (lapic_intr) {
 				vlapic_post_intr(vcpu->vlapic, hostcpu,
 				    vmm_ipinum);
 			} else {
 				ipi_cpu(hostcpu, vmm_ipinum);
 			}
 		} else {
 			/*
 			 * If the 'vcpu' is running on 'curcpu' then it must
 			 * be sending a notification to itself (e.g. SELF_IPI).
 			 * The pending event will be picked up when the vcpu
 			 * transitions back to guest context.
 			 */
 		}
 	} else {
 		KASSERT(hostcpu == NOCPU, ("vcpu state %d not consistent "
 		    "with hostcpu %d", vcpu->state, hostcpu));
 		if (vcpu->state == VCPU_SLEEPING)
 			wakeup_one(vcpu);
 	}
 	vcpu_unlock(vcpu);
 }
 
 struct vmspace *
 vm_get_vmspace(struct vm *vm)
 {
 
 	return (vm->vmspace);
 }
 
 int
 vm_apicid2vcpuid(struct vm *vm, int apicid)
 {
 	/*
 	 * XXX apic id is assumed to be numerically identical to vcpu id
 	 */
 	return (apicid);
 }
 
 void
 vm_smp_rendezvous(struct vm *vm, int vcpuid, cpuset_t dest,
     vm_rendezvous_func_t func, void *arg)
 {
 	int i;
 
 	/*
 	 * Enforce that this function is called without any locks
 	 */
 	WITNESS_WARN(WARN_PANIC, NULL, "vm_smp_rendezvous");
 	KASSERT(vcpuid == -1 || (vcpuid >= 0 && vcpuid < VM_MAXCPU),
 	    ("vm_smp_rendezvous: invalid vcpuid %d", vcpuid));
 
 restart:
 	mtx_lock(&vm->rendezvous_mtx);
 	if (vm->rendezvous_func != NULL) {
 		/*
 		 * If a rendezvous is already in progress then we need to
 		 * call the rendezvous handler in case this 'vcpuid' is one
 		 * of the targets of the rendezvous.
 		 */
 		RENDEZVOUS_CTR0(vm, vcpuid, "Rendezvous already in progress");
 		mtx_unlock(&vm->rendezvous_mtx);
 		vm_handle_rendezvous(vm, vcpuid);
 		goto restart;
 	}
 	KASSERT(vm->rendezvous_func == NULL, ("vm_smp_rendezvous: previous "
 	    "rendezvous is still in progress"));
 
 	RENDEZVOUS_CTR0(vm, vcpuid, "Initiating rendezvous");
 	vm->rendezvous_req_cpus = dest;
 	CPU_ZERO(&vm->rendezvous_done_cpus);
 	vm->rendezvous_arg = arg;
 	vm_set_rendezvous_func(vm, func);
 	mtx_unlock(&vm->rendezvous_mtx);
 
 	/*
 	 * Wake up any sleeping vcpus and trigger a VM-exit in any running
 	 * vcpus so they handle the rendezvous as soon as possible.
 	 */
 	for (i = 0; i < VM_MAXCPU; i++) {
 		if (CPU_ISSET(i, &dest))
 			vcpu_notify_event(vm, i, false);
 	}
 
 	vm_handle_rendezvous(vm, vcpuid);
 }
 
 struct vatpic *
 vm_atpic(struct vm *vm)
 {
 	return (vm->vatpic);
 }
 
 struct vatpit *
 vm_atpit(struct vm *vm)
 {
 	return (vm->vatpit);
 }
 
 enum vm_reg_name
 vm_segment_name(int seg)
 {
 	static enum vm_reg_name seg_names[] = {
 		VM_REG_GUEST_ES,
 		VM_REG_GUEST_CS,
 		VM_REG_GUEST_SS,
 		VM_REG_GUEST_DS,
 		VM_REG_GUEST_FS,
 		VM_REG_GUEST_GS
 	};
 
 	KASSERT(seg >= 0 && seg < nitems(seg_names),
 	    ("%s: invalid segment encoding %d", __func__, seg));
 	return (seg_names[seg]);
 }
Index: projects/release-debugdist/sys/arm/versatile/versatile_clcd.c
===================================================================
--- projects/release-debugdist/sys/arm/versatile/versatile_clcd.c	(revision 267343)
+++ projects/release-debugdist/sys/arm/versatile/versatile_clcd.c	(revision 267344)
@@ -1,961 +1,959 @@
 /*
  * Copyright (c) 2012 Oleksandr Tymoshenko <gonzo@freebsd.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/bus.h>
 #include <sys/kernel.h>
 #include <sys/module.h>
 #include <sys/malloc.h>
 #include <sys/rman.h>
 #include <sys/fbio.h>
 #include <sys/consio.h>
 #include <sys/kdb.h>
 
 #include <machine/bus.h>
 #include <machine/cpu.h>
 #include <machine/intr.h>
 
 #include <dev/fdt/fdt_common.h>
 #include <dev/ofw/openfirm.h>
 #include <dev/ofw/ofw_bus.h>
 #include <dev/ofw/ofw_bus_subr.h>
 
 #include <dev/fb/fbreg.h>
 #include <dev/syscons/syscons.h>
 
 #include <machine/bus.h>
 #include <machine/fdt.h>
 
 #define	PL110_VENDOR_ARM926PXP	1
 
 #define	MEM_SYS		0
 #define	MEM_CLCD	1
 #define MEM_REGIONS	2
 
 #define	SYS_CLCD		0x00
 #define		SYS_CLCD_CLCDID_SHIFT	0x08
 #define		SYS_CLCD_CLCDID_MASK	0x1f
 #define		SYS_CLCD_PWR3V5VSWITCH	(1 << 4)
 #define		SYS_CLCD_VDDPOSSWITCH 	(1 << 3)
 #define		SYS_CLCD_NLCDIOON  	(1 << 2)
 #define		SYS_CLCD_LCD_MODE_MASK	0x03
 
 #define	CLCD_MODE_RGB888	0x0
 #define	CLCD_MODE_RGB555	0x01
 #define	CLCD_MODE_RBG565	0x02
 #define	CLCD_MODE_RGB565	0x03
 
 #define	CLCDC_TIMING0		0x00
 #define	CLCDC_TIMING1		0x04
 #define	CLCDC_TIMING2		0x08
 #define	CLCDC_TIMING3		0x0C
 #define	CLCDC_TIMING3		0x0C
 #define	CLCDC_UPBASE		0x10
 #define	CLCDC_LPBASE		0x14
 #ifdef PL110_VENDOR_ARM926PXP
 #define	CLCDC_CONTROL		0x18
 #define	CLCDC_IMSC		0x1C
 #else
 #define	CLCDC_IMSC		0x18
 #define	CLCDC_CONTROL		0x1C
 #endif
 #define		CONTROL_WATERMARK	(1 << 16)
 #define		CONTROL_VCOMP_VS	(0 << 12)
 #define		CONTROL_VCOMP_BP	(1 << 12)
 #define		CONTROL_VCOMP_SAV	(2 << 12)
 #define		CONTROL_VCOMP_FP	(3 << 12)
 #define		CONTROL_PWR		(1 << 11)
 #define		CONTROL_BEPO		(1 << 10)
 #define		CONTROL_BEBO		(1 << 9)
 #define		CONTROL_BGR		(1 << 8)
 #define		CONTROL_DUAL		(1 << 7)
 #define		CONTROL_MONO8		(1 << 6)
 #define		CONTROL_TFT		(1 << 5)
 #define		CONTROL_BW		(1 << 4)
 #define		CONTROL_BPP1		(0x00 << 1)
 #define		CONTROL_BPP2		(0x01 << 1)
 #define		CONTROL_BPP4		(0x02 << 1)
 #define		CONTROL_BPP8		(0x03 << 1)
 #define		CONTROL_BPP16		(0x04 << 1)
 #define		CONTROL_BPP24		(0x05 << 1)
 #define		CONTROL_EN	(1 << 0)
 #define	CLCDC_RIS		0x20
 #define	CLCDC_MIS		0x24
 #define		INTR_MBERR		(1 << 4)
 #define		INTR_VCOMP		(1 << 3)
 #define		INTR_LNB		(1 << 2)
 #define		INTR_FUF		(1 << 1)
 #define	CLCDC_ICR		0x28
 
 #ifdef DEBUG
 #define dprintf(fmt, args...) do { printf("%s(): ", __func__);   \
     printf(fmt,##args); } while (0)
 #else
 #define dprintf(fmt, args...)
 #endif
 
 #define	versatile_clcdc_sys_read_4(sc, reg)	\
 	bus_read_4((sc)->mem_res[MEM_SYS], (reg))
 #define	versatile_clcdc_sys_write_4(sc, reg, val)	\
 	bus_write_4((sc)->mem_res[MEM_SYS], (reg), (val))
 
 #define	versatile_clcdc_read_4(sc, reg)	\
 	bus_read_4((sc)->mem_res[MEM_CLCD], (reg))
 #define	versatile_clcdc_write_4(sc, reg, val)	\
 	bus_write_4((sc)->mem_res[MEM_CLCD], (reg), (val))
 
 struct versatile_clcdc_softc {
 	struct resource*	mem_res[MEM_REGIONS];
 
 	struct mtx		mtx;
 
 	int			width;
 	int			height;
 	int			mode;
 
 	bus_dma_tag_t		dma_tag;
 	bus_dmamap_t		dma_map;
 	bus_addr_t		fb_phys;
 	uint8_t			*fb_base;
 
 };
 
 struct video_adapter_softc {
 	/* Videoadpater part */
 	video_adapter_t	va;
 	int		console;
 
 	intptr_t	fb_addr;
 	unsigned int	fb_size;
 
 	unsigned int	height;
 	unsigned int	width;
 	unsigned int	depth;
 	unsigned int	stride;
 
 	unsigned int	xmargin;
 	unsigned int	ymargin;
 
 	unsigned char	*font;
 	int		initialized;
 };
 
 struct argb {
 	uint8_t		a;
 	uint8_t		r;
 	uint8_t		g;
 	uint8_t		b;
 };
 
 static struct argb versatilefb_palette[16] = {
 	{0x00, 0x00, 0x00, 0x00},
 	{0x00, 0x00, 0x00, 0xaa},
 	{0x00, 0x00, 0xaa, 0x00},
 	{0x00, 0x00, 0xaa, 0xaa},
 	{0x00, 0xaa, 0x00, 0x00},
 	{0x00, 0xaa, 0x00, 0xaa},
 	{0x00, 0xaa, 0x55, 0x00},
 	{0x00, 0xaa, 0xaa, 0xaa},
 	{0x00, 0x55, 0x55, 0x55},
 	{0x00, 0x55, 0x55, 0xff},
 	{0x00, 0x55, 0xff, 0x55},
 	{0x00, 0x55, 0xff, 0xff},
 	{0x00, 0xff, 0x55, 0x55},
 	{0x00, 0xff, 0x55, 0xff},
 	{0x00, 0xff, 0xff, 0x55},
 	{0x00, 0xff, 0xff, 0xff}
 };
 
 /* mouse pointer from dev/syscons/scgfbrndr.c */
 static u_char mouse_pointer[16] = {
         0x00, 0x40, 0x60, 0x70, 0x78, 0x7c, 0x7e, 0x68,
         0x0c, 0x0c, 0x06, 0x06, 0x00, 0x00, 0x00, 0x00
 };
 
 #define FB_WIDTH		640
 #define FB_HEIGHT		480
 #define FB_DEPTH		16
 
 #define	VERSATILE_FONT_HEIGHT	16
 
 static struct video_adapter_softc va_softc;
 
 static struct resource_spec versatile_clcdc_mem_spec[] = {
 	{ SYS_RES_MEMORY, 0, RF_ACTIVE },
 	{ SYS_RES_MEMORY, 1, RF_ACTIVE },
 	{ -1, 0, 0 }
 };
 
 static int versatilefb_configure(int);
 static void versatilefb_update_margins(video_adapter_t *adp);
 
 static void
 versatile_fb_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int err)
 {
 	bus_addr_t *addr;
 
 	if (err)
 		return;
 
 	addr = (bus_addr_t*)arg;
 	*addr = segs[0].ds_addr;
 }
 
 static int
 versatile_clcdc_probe(device_t dev)
 {
 
 	if (!ofw_bus_status_okay(dev))
 		return (ENXIO);
 
 	if (ofw_bus_is_compatible(dev, "arm,pl110")) {
 		device_set_desc(dev, "PL110 CLCD controller");
 		return (BUS_PROBE_DEFAULT);
 	}
 
 	return (ENXIO);
 }
 
 static int
 versatile_clcdc_attach(device_t dev)
 {
 	struct versatile_clcdc_softc *sc = device_get_softc(dev);
 	struct video_adapter_softc *va_sc = &va_softc;
 	int err;
 	uint32_t reg;
 	int clcdid;
 	int dma_size;
 
 	/* Request memory resources */
 	err = bus_alloc_resources(dev, versatile_clcdc_mem_spec,
 		sc->mem_res);
 	if (err) {
 		device_printf(dev, "Error: could not allocate memory resources\n");
 		return (ENXIO);
 	}
 
 	reg = versatile_clcdc_sys_read_4(sc, SYS_CLCD);
 	clcdid = (reg >> SYS_CLCD_CLCDID_SHIFT) & SYS_CLCD_CLCDID_MASK;
 	switch (clcdid) {
 		case 31:
 			device_printf(dev, "QEMU VGA 640x480\n");
 			sc->width = 640;
 			sc->height = 480;
 			break;
 		default:
 			device_printf(dev, "Unsupported: %d\n", clcdid);
 			goto fail;
 	}
 
 	reg &= ~SYS_CLCD_LCD_MODE_MASK;
 	reg |= CLCD_MODE_RGB565;
 	sc->mode = CLCD_MODE_RGB565;
 	versatile_clcdc_sys_write_4(sc, SYS_CLCD, reg);
 	dma_size = sc->width*sc->height*2;
 
 	/*
 	 * Power on LCD
 	 */
 	reg |= SYS_CLCD_PWR3V5VSWITCH | SYS_CLCD_NLCDIOON;
 	versatile_clcdc_sys_write_4(sc, SYS_CLCD, reg);
 
 	/*
 	 * XXX: hardcoded timing for VGA. For other modes/panels
 	 * we need to keep table of timing register values
 	 */
 	/*
 	 * XXX: set SYS_OSC1 
 	 */
 	versatile_clcdc_write_4(sc, CLCDC_TIMING0, 0x3F1F3F9C);
 	versatile_clcdc_write_4(sc, CLCDC_TIMING1, 0x090B61DF);
 	versatile_clcdc_write_4(sc, CLCDC_TIMING2, 0x067F1800);
 	/* XXX: timing 3? */
 
 	/*
 	 * Now allocate framebuffer memory
 	 */
 	err = bus_dma_tag_create(
 	    bus_get_dma_tag(dev),
 	    4, 0,		/* alignment, boundary */
 	    BUS_SPACE_MAXADDR_32BIT,	/* lowaddr */
 	    BUS_SPACE_MAXADDR,		/* highaddr */
 	    NULL, NULL,			/* filter, filterarg */
 	    dma_size, 1,		/* maxsize, nsegments */
 	    dma_size, 0,		/* maxsegsize, flags */
 	    NULL, NULL,			/* lockfunc, lockarg */
 	    &sc->dma_tag);
 
 	err = bus_dmamem_alloc(sc->dma_tag, (void **)&sc->fb_base,
 	    0, &sc->dma_map);
 	if (err) {
 		device_printf(dev, "cannot allocate framebuffer\n");
 		goto fail;
 	}
 
 	err = bus_dmamap_load(sc->dma_tag, sc->dma_map, sc->fb_base,
 	    dma_size, versatile_fb_dmamap_cb, &sc->fb_phys, BUS_DMA_NOWAIT);
 
 	if (err) {
 		device_printf(dev, "cannot load DMA map\n");
 		goto fail;
 	}
 
 	/* Make sure it's blank */
 	memset(sc->fb_base, 0x00, dma_size);
 
 	versatile_clcdc_write_4(sc, CLCDC_UPBASE, sc->fb_phys);
 
 	err = (sc_attach_unit(device_get_unit(dev),
 	    device_get_flags(dev) | SC_AUTODETECT_KBD));
 
 	if (err) {
 		device_printf(dev, "failed to attach syscons\n");
 		goto fail;
 	}
 
 	/*
 	 * XXX: hardcoded for VGA
 	 */
 	reg = CONTROL_VCOMP_BP | CONTROL_TFT | CONTROL_BGR | CONTROL_EN;
 	reg |= CONTROL_BPP16;
 	versatile_clcdc_write_4(sc, CLCDC_CONTROL, reg);
 	DELAY(20);
 	reg |= CONTROL_PWR;
 	versatile_clcdc_write_4(sc, CLCDC_CONTROL, reg);
 
 	va_sc->fb_addr = (vm_offset_t)sc->fb_base;
 	va_sc->fb_size = dma_size;
 	va_sc->width = sc->width;
 	va_sc->height = sc->height;
 	va_sc->depth = 16;
 	va_sc->stride = sc->width * 2;
 	versatilefb_update_margins(&va_sc->va);
 
 	return (0);
 
 fail:
 	if (sc->fb_base)
 		bus_dmamem_free(sc->dma_tag, sc->fb_base, sc->dma_map);
-	if (sc->dma_map)
-		bus_dmamap_destroy(sc->dma_tag, sc->dma_map);
 	if (sc->dma_tag)
 		bus_dma_tag_destroy(sc->dma_tag);
 	return (err);
 }
 
 static device_method_t versatile_clcdc_methods[] = {
 	DEVMETHOD(device_probe,		versatile_clcdc_probe),
 	DEVMETHOD(device_attach,	versatile_clcdc_attach),
 
 	DEVMETHOD_END
 };
 
 static driver_t versatile_clcdc_driver = {
 	"clcdc",
 	versatile_clcdc_methods,
 	sizeof(struct versatile_clcdc_softc),
 };
 
 static devclass_t versatile_clcdc_devclass;
 
 DRIVER_MODULE(versatile_clcdc, simplebus, versatile_clcdc_driver, versatile_clcdc_devclass, 0, 0);
 
 /*
  * Video driver routines and glue.
  */
 static vi_probe_t		versatilefb_probe;
 static vi_init_t		versatilefb_init;
 static vi_get_info_t		versatilefb_get_info;
 static vi_query_mode_t		versatilefb_query_mode;
 static vi_set_mode_t		versatilefb_set_mode;
 static vi_save_font_t		versatilefb_save_font;
 static vi_load_font_t		versatilefb_load_font;
 static vi_show_font_t		versatilefb_show_font;
 static vi_save_palette_t	versatilefb_save_palette;
 static vi_load_palette_t	versatilefb_load_palette;
 static vi_set_border_t		versatilefb_set_border;
 static vi_save_state_t		versatilefb_save_state;
 static vi_load_state_t		versatilefb_load_state;
 static vi_set_win_org_t		versatilefb_set_win_org;
 static vi_read_hw_cursor_t	versatilefb_read_hw_cursor;
 static vi_set_hw_cursor_t	versatilefb_set_hw_cursor;
 static vi_set_hw_cursor_shape_t	versatilefb_set_hw_cursor_shape;
 static vi_blank_display_t	versatilefb_blank_display;
 static vi_mmap_t		versatilefb_mmap;
 static vi_ioctl_t		versatilefb_ioctl;
 static vi_clear_t		versatilefb_clear;
 static vi_fill_rect_t		versatilefb_fill_rect;
 static vi_bitblt_t		versatilefb_bitblt;
 static vi_diag_t		versatilefb_diag;
 static vi_save_cursor_palette_t	versatilefb_save_cursor_palette;
 static vi_load_cursor_palette_t	versatilefb_load_cursor_palette;
 static vi_copy_t		versatilefb_copy;
 static vi_putp_t		versatilefb_putp;
 static vi_putc_t		versatilefb_putc;
 static vi_puts_t		versatilefb_puts;
 static vi_putm_t		versatilefb_putm;
 
 static video_switch_t versatilefbvidsw = {
 	.probe			= versatilefb_probe,
 	.init			= versatilefb_init,
 	.get_info		= versatilefb_get_info,
 	.query_mode		= versatilefb_query_mode,
 	.set_mode		= versatilefb_set_mode,
 	.save_font		= versatilefb_save_font,
 	.load_font		= versatilefb_load_font,
 	.show_font		= versatilefb_show_font,
 	.save_palette		= versatilefb_save_palette,
 	.load_palette		= versatilefb_load_palette,
 	.set_border		= versatilefb_set_border,
 	.save_state		= versatilefb_save_state,
 	.load_state		= versatilefb_load_state,
 	.set_win_org		= versatilefb_set_win_org,
 	.read_hw_cursor		= versatilefb_read_hw_cursor,
 	.set_hw_cursor		= versatilefb_set_hw_cursor,
 	.set_hw_cursor_shape	= versatilefb_set_hw_cursor_shape,
 	.blank_display		= versatilefb_blank_display,
 	.mmap			= versatilefb_mmap,
 	.ioctl			= versatilefb_ioctl,
 	.clear			= versatilefb_clear,
 	.fill_rect		= versatilefb_fill_rect,
 	.bitblt			= versatilefb_bitblt,
 	.diag			= versatilefb_diag,
 	.save_cursor_palette	= versatilefb_save_cursor_palette,
 	.load_cursor_palette	= versatilefb_load_cursor_palette,
 	.copy			= versatilefb_copy,
 	.putp			= versatilefb_putp,
 	.putc			= versatilefb_putc,
 	.puts			= versatilefb_puts,
 	.putm			= versatilefb_putm,
 };
 
 VIDEO_DRIVER(versatilefb, versatilefbvidsw, versatilefb_configure);
 
 static vr_init_t clcdr_init;
 static vr_clear_t clcdr_clear;
 static vr_draw_border_t clcdr_draw_border;
 static vr_draw_t clcdr_draw;
 static vr_set_cursor_t clcdr_set_cursor;
 static vr_draw_cursor_t clcdr_draw_cursor;
 static vr_blink_cursor_t clcdr_blink_cursor;
 static vr_set_mouse_t clcdr_set_mouse;
 static vr_draw_mouse_t clcdr_draw_mouse;
 
 /*
  * We use our own renderer; this is because we must emulate a hardware
  * cursor.
  */
 static sc_rndr_sw_t clcdrend = {
 	clcdr_init,
 	clcdr_clear,
 	clcdr_draw_border,
 	clcdr_draw,
 	clcdr_set_cursor,
 	clcdr_draw_cursor,
 	clcdr_blink_cursor,
 	clcdr_set_mouse,
 	clcdr_draw_mouse
 };
 
 RENDERER(versatilefb, 0, clcdrend, gfb_set);
 RENDERER_MODULE(versatilefb, gfb_set);
 
 static void
 clcdr_init(scr_stat* scp)
 {
 }
 
 static void
 clcdr_clear(scr_stat* scp, int c, int attr)
 {
 }
 
 static void
 clcdr_draw_border(scr_stat* scp, int color)
 {
 }
 
 static void
 clcdr_draw(scr_stat* scp, int from, int count, int flip)
 {
 	video_adapter_t* adp = scp->sc->adp;
 	int i, c, a;
 
 	if (!flip) {
 		/* Normal printing */
 		vidd_puts(adp, from, (uint16_t*)sc_vtb_pointer(&scp->vtb, from), count);
 	} else {	
 		/* This is for selections and such: invert the color attribute */
 		for (i = count; i-- > 0; ++from) {
 			c = sc_vtb_getc(&scp->vtb, from);
 			a = sc_vtb_geta(&scp->vtb, from) >> 8;
 			vidd_putc(adp, from, c, (a >> 4) | ((a & 0xf) << 4));
 		}
 	}
 }
 
 static void
 clcdr_set_cursor(scr_stat* scp, int base, int height, int blink)
 {
 }
 
 static void
 clcdr_draw_cursor(scr_stat* scp, int off, int blink, int on, int flip)
 {
 	video_adapter_t* adp = scp->sc->adp;
 	struct video_adapter_softc *sc;
 	int row, col;
 	uint8_t *addr;
 	int i,j;
 
 	sc = (struct video_adapter_softc *)adp;
 
 	if (scp->curs_attr.height <= 0)
 		return;
 
 	if (sc->fb_addr == 0)
 		return;
 
 	if (off >= adp->va_info.vi_width * adp->va_info.vi_height)
 		return;
 
 	/* calculate the coordinates in the video buffer */
 	row = (off / adp->va_info.vi_width) * adp->va_info.vi_cheight;
 	col = (off % adp->va_info.vi_width) * adp->va_info.vi_cwidth;
 
 	addr = (uint8_t *)sc->fb_addr
 	    + (row + sc->ymargin)*(sc->stride)
 	    + (sc->depth/8) * (col + sc->xmargin);
 
 	/* our cursor consists of simply inverting the char under it */
 	for (i = 0; i < adp->va_info.vi_cheight; i++) {
 		for (j = 0; j < adp->va_info.vi_cwidth; j++) {
 
 			addr[2*j] ^= 0xff;
 			addr[2*j + 1] ^= 0xff;
 		}
 
 		addr += sc->stride;
 	}
 }
 
 static void
 clcdr_blink_cursor(scr_stat* scp, int at, int flip)
 {
 }
 
 static void
 clcdr_set_mouse(scr_stat* scp)
 {
 }
 
 static void
 clcdr_draw_mouse(scr_stat* scp, int x, int y, int on)
 {
 	vidd_putm(scp->sc->adp, x, y, mouse_pointer, 0xffffffff, 16, 8);
 
 }
 
 static uint16_t versatilefb_static_window[ROW*COL];
 extern u_char dflt_font_16[];
 
 /*
  * Update videoadapter settings after changing resolution
  */
 static void
 versatilefb_update_margins(video_adapter_t *adp)
 {
 	struct video_adapter_softc *sc;
 	video_info_t *vi;
 
 	sc = (struct video_adapter_softc *)adp;
 	vi = &adp->va_info;
 
 	sc->xmargin = (sc->width - (vi->vi_width * vi->vi_cwidth)) / 2;
 	sc->ymargin = (sc->height - (vi->vi_height * vi->vi_cheight))/2;
 }
 
 static int
 versatilefb_configure(int flags)
 {
 	struct video_adapter_softc *va_sc;
 
 	va_sc = &va_softc;
 
 	if (va_sc->initialized)
 		return (0);
 
 	va_sc->width = FB_WIDTH;
 	va_sc->height = FB_HEIGHT;
 	va_sc->depth = FB_DEPTH;
 
 	versatilefb_init(0, &va_sc->va, 0);
 
 	va_sc->initialized = 1;
 
 	return (0);
 }
 
 static int
 versatilefb_probe(int unit, video_adapter_t **adp, void *arg, int flags)
 {
 
 	return (0);
 }
 
 static int
 versatilefb_init(int unit, video_adapter_t *adp, int flags)
 {
 	struct video_adapter_softc *sc;
 	video_info_t *vi;
 
 	sc = (struct video_adapter_softc *)adp;
 	vi = &adp->va_info;
 
 	vid_init_struct(adp, "versatilefb", -1, unit);
 
 	sc->font = dflt_font_16;
 	vi->vi_cheight = VERSATILE_FONT_HEIGHT;
 	vi->vi_cwidth = 8;
 
 	vi->vi_width = sc->width/8;
 	vi->vi_height = sc->height/vi->vi_cheight;
 
 	/*
 	 * Clamp width/height to syscons maximums
 	 */
 	if (vi->vi_width > COL)
 		vi->vi_width = COL;
 	if (vi->vi_height > ROW)
 		vi->vi_height = ROW;
 
 	sc->xmargin = (sc->width - (vi->vi_width * vi->vi_cwidth)) / 2;
 	sc->ymargin = (sc->height - (vi->vi_height * vi->vi_cheight))/2;
 
 
 	adp->va_window = (vm_offset_t) versatilefb_static_window;
 	adp->va_flags |= V_ADP_FONT /* | V_ADP_COLOR | V_ADP_MODECHANGE */;
 
 	vid_register(&sc->va);
 
 	return (0);
 }
 
 static int
 versatilefb_get_info(video_adapter_t *adp, int mode, video_info_t *info)
 {
 	bcopy(&adp->va_info, info, sizeof(*info));
 	return (0);
 }
 
 static int
 versatilefb_query_mode(video_adapter_t *adp, video_info_t *info)
 {
 	return (0);
 }
 
 static int
 versatilefb_set_mode(video_adapter_t *adp, int mode)
 {
 	return (0);
 }
 
 static int
 versatilefb_save_font(video_adapter_t *adp, int page, int size, int width,
     u_char *data, int c, int count)
 {
 	return (0);
 }
 
 static int
 versatilefb_load_font(video_adapter_t *adp, int page, int size, int width,
     u_char *data, int c, int count)
 {
 	struct video_adapter_softc *sc = (struct video_adapter_softc *)adp;
 
 	sc->font = data;
 
 	return (0);
 }
 
 static int
 versatilefb_show_font(video_adapter_t *adp, int page)
 {
 	return (0);
 }
 
 static int
 versatilefb_save_palette(video_adapter_t *adp, u_char *palette)
 {
 	return (0);
 }
 
 static int
 versatilefb_load_palette(video_adapter_t *adp, u_char *palette)
 {
 	return (0);
 }
 
 static int
 versatilefb_set_border(video_adapter_t *adp, int border)
 {
 	return (versatilefb_blank_display(adp, border));
 }
 
 static int
 versatilefb_save_state(video_adapter_t *adp, void *p, size_t size)
 {
 	return (0);
 }
 
 static int
 versatilefb_load_state(video_adapter_t *adp, void *p)
 {
 	return (0);
 }
 
 static int
 versatilefb_set_win_org(video_adapter_t *adp, off_t offset)
 {
 	return (0);
 }
 
 static int
 versatilefb_read_hw_cursor(video_adapter_t *adp, int *col, int *row)
 {
 	*col = *row = 0;
 
 	return (0);
 }
 
 static int
 versatilefb_set_hw_cursor(video_adapter_t *adp, int col, int row)
 {
 
 	return (0);
 }
 
 static int
 versatilefb_set_hw_cursor_shape(video_adapter_t *adp, int base, int height,
     int celsize, int blink)
 {
 	return (0);
 }
 
 static int
 versatilefb_blank_display(video_adapter_t *adp, int mode)
 {
 
 	struct video_adapter_softc *sc;
 
 	sc = (struct video_adapter_softc *)adp;
 	if (sc && sc->fb_addr)
 		memset((void*)sc->fb_addr, 0, sc->fb_size);
 
 	return (0);
 }
 
 static int
 versatilefb_mmap(video_adapter_t *adp, vm_ooffset_t offset, vm_paddr_t *paddr,
     int prot, vm_memattr_t *memattr)
 {
 	struct video_adapter_softc *sc;
 
 	sc = (struct video_adapter_softc *)adp;
 
 	/*
 	 * This might be a legacy VGA mem request: if so, just point it at the
 	 * framebuffer, since it shouldn't be touched
 	 */
 	if (offset < sc->stride*sc->height) {
 		*paddr = sc->fb_addr + offset;
 		return (0);
 	}
 
 	return (EINVAL);
 }
 
 static int
 versatilefb_ioctl(video_adapter_t *adp, u_long cmd, caddr_t data)
 {
 
 	return (0);
 }
 
 static int
 versatilefb_clear(video_adapter_t *adp)
 {
 
 	return (versatilefb_blank_display(adp, 0));
 }
 
 static int
 versatilefb_fill_rect(video_adapter_t *adp, int val, int x, int y, int cx, int cy)
 {
 
 	return (0);
 }
 
 static int
 versatilefb_bitblt(video_adapter_t *adp, ...)
 {
 
 	return (0);
 }
 
 static int
 versatilefb_diag(video_adapter_t *adp, int level)
 {
 
 	return (0);
 }
 
 static int
 versatilefb_save_cursor_palette(video_adapter_t *adp, u_char *palette)
 {
 
 	return (0);
 }
 
 static int
 versatilefb_load_cursor_palette(video_adapter_t *adp, u_char *palette)
 {
 
 	return (0);
 }
 
 static int
 versatilefb_copy(video_adapter_t *adp, vm_offset_t src, vm_offset_t dst, int n)
 {
 
 	return (0);
 }
 
 static int
 versatilefb_putp(video_adapter_t *adp, vm_offset_t off, uint32_t p, uint32_t a,
     int size, int bpp, int bit_ltor, int byte_ltor)
 {
 
 	return (0);
 }
 
 static int
 versatilefb_putc(video_adapter_t *adp, vm_offset_t off, uint8_t c, uint8_t a)
 {
 	struct video_adapter_softc *sc;
 	int row;
 	int col;
 	int i, j, k;
 	uint8_t *addr;
 	u_char *p;
 	uint8_t fg, bg, color;
 	uint16_t rgb;
 
 	sc = (struct video_adapter_softc *)adp;
 
 	if (sc->fb_addr == 0)
 		return (0);
 
 	if (off >= adp->va_info.vi_width * adp->va_info.vi_height)
 		return (0);
 
 	row = (off / adp->va_info.vi_width) * adp->va_info.vi_cheight;
 	col = (off % adp->va_info.vi_width) * adp->va_info.vi_cwidth;
 	p = sc->font + c*VERSATILE_FONT_HEIGHT;
 	addr = (uint8_t *)sc->fb_addr
 	    + (row + sc->ymargin)*(sc->stride)
 	    + (sc->depth/8) * (col + sc->xmargin);
 
 	fg = a & 0xf ;
 	bg = (a >> 4) & 0xf;
 
 	for (i = 0; i < VERSATILE_FONT_HEIGHT; i++) {
 		for (j = 0, k = 7; j < 8; j++, k--) {
 			if ((p[i] & (1 << k)) == 0)
 				color = bg;
 			else
 				color = fg;
 
 			switch (sc->depth) {
 			case 16:
 				rgb = (versatilefb_palette[color].r >> 3) << 11;
 				rgb |= (versatilefb_palette[color].g >> 2) << 5;
 				rgb |= (versatilefb_palette[color].b >> 3);
 				addr[2*j] = rgb & 0xff;
 				addr[2*j + 1] = (rgb >> 8) & 0xff;
 			default:
 				/* Not supported yet */
 				break;
 			}
 		}
 
 		addr += (sc->stride);
 	}
 
         return (0);
 }
 
 static int
 versatilefb_puts(video_adapter_t *adp, vm_offset_t off, u_int16_t *s, int len)
 {
 	int i;
 
 	for (i = 0; i < len; i++) 
 		versatilefb_putc(adp, off + i, s[i] & 0xff, (s[i] & 0xff00) >> 8);
 
 	return (0);
 }
 
 static int
 versatilefb_putm(video_adapter_t *adp, int x, int y, uint8_t *pixel_image,
     uint32_t pixel_mask, int size, int width)
 {
 
 	return (0);
 }
 
 /*
  * Define a stub keyboard driver in case one hasn't been
  * compiled into the kernel
  */
 #include <sys/kbio.h>
 #include <dev/kbd/kbdreg.h>
 
 static int dummy_kbd_configure(int flags);
 
 keyboard_switch_t bcmdummysw;
 
 static int
 dummy_kbd_configure(int flags)
 {
 
 	return (0);
 }
 KEYBOARD_DRIVER(bcmdummy, bcmdummysw, dummy_kbd_configure);
Index: projects/release-debugdist/sys/conf/options
===================================================================
--- projects/release-debugdist/sys/conf/options	(revision 267343)
+++ projects/release-debugdist/sys/conf/options	(revision 267344)
@@ -1,921 +1,922 @@
 # $FreeBSD$
 #
 #        On the handling of kernel options
 #
 # All kernel options should be listed in NOTES, with suitable
 # descriptions.  Negative options (options that make some code not
 # compile) should be commented out; LINT (generated from NOTES) should
 # compile as much code as possible.  Try to structure option-using
 # code so that a single option only switch code on, or only switch
 # code off, to make it possible to have a full compile-test.  If
 # necessary, you can check for COMPILING_LINT to get maximum code
 # coverage.
 #
 # All new options shall also be listed in either "conf/options" or
 # "conf/options.<machine>".  Options that affect a single source-file
 # <xxx>.[c|s] should be directed into "opt_<xxx>.h", while options
 # that affect multiple files should either go in "opt_global.h" if
 # this is a kernel-wide option (used just about everywhere), or in
 # "opt_<option-name-in-lower-case>.h" if it affects only some files.
 # Note that the effect of listing only an option without a
 # header-file-name in conf/options (and cousins) is that the last
 # convention is followed.
 #
 # This handling scheme is not yet fully implemented.
 #
 #
 # Format of this file:
 # Option name	filename
 #
 # If filename is missing, the default is
 # opt_<name-of-option-in-lower-case>.h
 
 AAC_DEBUG		opt_aac.h
 AACRAID_DEBUG		opt_aacraid.h
 AHC_ALLOW_MEMIO		opt_aic7xxx.h
 AHC_TMODE_ENABLE	opt_aic7xxx.h
 AHC_DUMP_EEPROM		opt_aic7xxx.h
 AHC_DEBUG		opt_aic7xxx.h
 AHC_DEBUG_OPTS		opt_aic7xxx.h
 AHC_REG_PRETTY_PRINT	opt_aic7xxx.h
 AHD_DEBUG		opt_aic79xx.h
 AHD_DEBUG_OPTS		opt_aic79xx.h
 AHD_TMODE_ENABLE	opt_aic79xx.h	
 AHD_REG_PRETTY_PRINT	opt_aic79xx.h
 ADW_ALLOW_MEMIO		opt_adw.h
 
 TWA_DEBUG		opt_twa.h
 TWA_FLASH_FIRMWARE	opt_twa.h
 
 # Debugging options.
 ALT_BREAK_TO_DEBUGGER	opt_kdb.h
 BREAK_TO_DEBUGGER	opt_kdb.h
 DDB
 DDB_BUFR_SIZE	opt_ddb.h
 DDB_CAPTURE_DEFAULTBUFSIZE	opt_ddb.h
 DDB_CAPTURE_MAXBUFSIZE	opt_ddb.h
 DDB_CTF		opt_ddb.h
 DDB_NUMSYM	opt_ddb.h
 GDB
 KDB		opt_global.h
 KDB_TRACE	opt_kdb.h
 KDB_UNATTENDED	opt_kdb.h
 KLD_DEBUG	opt_kld.h
 SYSCTL_DEBUG	opt_sysctl.h
 EARLY_PRINTF	opt_global.h
 TEXTDUMP_PREFERRED	opt_ddb.h
 TEXTDUMP_VERBOSE	opt_ddb.h
 
 # Miscellaneous options.
 ADAPTIVE_LOCKMGRS
 ALQ
 ALTERA_SDCARD_FAST_SIM	opt_altera_sdcard.h
 ATSE_CFI_HACK	opt_cfi.h
 AUDIT		opt_global.h
 BOOTHOWTO	opt_global.h
 BOOTVERBOSE	opt_global.h
 CALLOUT_PROFILING
 CAPABILITIES	opt_capsicum.h
 CAPABILITY_MODE	opt_capsicum.h
 COMPAT_43	opt_compat.h
 COMPAT_43TTY	opt_compat.h
 COMPAT_FREEBSD4	opt_compat.h
 COMPAT_FREEBSD5	opt_compat.h
 COMPAT_FREEBSD6	opt_compat.h
 COMPAT_FREEBSD7	opt_compat.h
 COMPILING_LINT	opt_global.h
 COMPRESS_USER_CORES opt_core.h
 CY_PCI_FASTINTR
 DEADLKRES	opt_watchdog.h
 DIRECTIO
 FILEMON		opt_dontuse.h
 FFCLOCK
 FULL_PREEMPTION	opt_sched.h
 IMAGACT_BINMISC		opt_dontuse.h
 IPI_PREEMPTION	opt_sched.h
 GEOM_AES	opt_geom.h
 GEOM_BDE	opt_geom.h
 GEOM_BSD	opt_geom.h
 GEOM_CACHE	opt_geom.h
 GEOM_CONCAT	opt_geom.h
 GEOM_ELI	opt_geom.h
 GEOM_FOX	opt_geom.h
 GEOM_GATE	opt_geom.h
 GEOM_JOURNAL	opt_geom.h
 GEOM_LABEL	opt_geom.h
 GEOM_LABEL_GPT	opt_geom.h
 GEOM_LINUX_LVM	opt_geom.h
 GEOM_MBR	opt_geom.h
 GEOM_MIRROR	opt_geom.h
 GEOM_MULTIPATH	opt_geom.h
 GEOM_NOP	opt_geom.h
 GEOM_PART_APM	opt_geom.h
 GEOM_PART_BSD	opt_geom.h
 GEOM_PART_EBR	opt_geom.h
 GEOM_PART_EBR_COMPAT	opt_geom.h
 GEOM_PART_GPT	opt_geom.h
 GEOM_PART_LDM	opt_geom.h
 GEOM_PART_MBR	opt_geom.h
 GEOM_PART_PC98	opt_geom.h
 GEOM_PART_VTOC8	opt_geom.h
 GEOM_PC98	opt_geom.h
 GEOM_RAID	opt_geom.h
 GEOM_RAID3	opt_geom.h
 GEOM_SHSEC	opt_geom.h
 GEOM_STRIPE	opt_geom.h
 GEOM_SUNLABEL	opt_geom.h
 GEOM_UNCOMPRESS	opt_geom.h
 GEOM_UZIP	opt_geom.h
 GEOM_VINUM	opt_geom.h
 GEOM_VIRSTOR	opt_geom.h
 GEOM_VOL	opt_geom.h
 GEOM_ZERO	opt_geom.h
 KDTRACE_HOOKS	opt_global.h
 KDTRACE_FRAME	opt_kdtrace.h
 KN_HASHSIZE	opt_kqueue.h
 KSTACK_MAX_PAGES
 KSTACK_PAGES
 KTRACE
 KTRACE_REQUEST_POOL	opt_ktrace.h
 LIBICONV
 MAC		opt_global.h
 MAC_BIBA	opt_dontuse.h
 MAC_BSDEXTENDED	opt_dontuse.h
 MAC_IFOFF	opt_dontuse.h
 MAC_LOMAC	opt_dontuse.h
 MAC_MLS		opt_dontuse.h
 MAC_NONE	opt_dontuse.h
 MAC_PARTITION	opt_dontuse.h
 MAC_PORTACL	opt_dontuse.h
 MAC_SEEOTHERUIDS	opt_dontuse.h
 MAC_STATIC	opt_mac.h
 MAC_STUB	opt_dontuse.h
 MAC_TEST	opt_dontuse.h
 MD_ROOT		opt_md.h
 MD_ROOT_FSTYPE	opt_md.h
 MD_ROOT_SIZE	opt_md.h
 MFI_DEBUG	opt_mfi.h
 MFI_DECODE_LOG	opt_mfi.h
 MPROF_BUFFERS	opt_mprof.h
 MPROF_HASH_SIZE	opt_mprof.h
 NEW_PCIB	opt_global.h
 NO_ADAPTIVE_MUTEXES	opt_adaptive_mutexes.h
 NO_ADAPTIVE_RWLOCKS
 NO_ADAPTIVE_SX
 NO_EVENTTIMERS		opt_timer.h
 NO_SYSCTL_DESCR	opt_global.h
 NSWBUF_MIN	opt_swap.h
 MBUF_PACKET_ZONE_DISABLE	opt_global.h
 PANIC_REBOOT_WAIT_TIME	opt_panic.h
 PPC_DEBUG	opt_ppc.h
 PPC_PROBE_CHIPSET	opt_ppc.h
 PPS_SYNC	opt_ntp.h
 PREEMPTION	opt_sched.h
 QUOTA
 SCHED_4BSD	opt_sched.h
 SCHED_STATS	opt_sched.h
 SCHED_ULE	opt_sched.h
 SLEEPQUEUE_PROFILING
 SLHCI_DEBUG	opt_slhci.h
 SPX_HACK
 STACK		opt_stack.h
 SUIDDIR
 MSGMNB		opt_sysvipc.h
 MSGMNI		opt_sysvipc.h
 MSGSEG		opt_sysvipc.h
 MSGSSZ		opt_sysvipc.h
 MSGTQL		opt_sysvipc.h
 SEMMNI		opt_sysvipc.h
 SEMMNS		opt_sysvipc.h
 SEMMNU		opt_sysvipc.h
 SEMMSL		opt_sysvipc.h
 SEMOPM		opt_sysvipc.h
 SEMUME		opt_sysvipc.h
 SHMALL		opt_sysvipc.h
 SHMMAX		opt_sysvipc.h
 SHMMAXPGS	opt_sysvipc.h
 SHMMIN		opt_sysvipc.h
 SHMMNI		opt_sysvipc.h
 SHMSEG		opt_sysvipc.h
 SYSVMSG		opt_sysvipc.h
 SYSVSEM		opt_sysvipc.h
 SYSVSHM		opt_sysvipc.h
 SW_WATCHDOG	opt_watchdog.h
 TURNSTILE_PROFILING
 UMTX_PROFILING
 VFS_AIO
 VERBOSE_SYSINIT	opt_global.h
 WLCACHE		opt_wavelan.h
 WLDEBUG		opt_wavelan.h
 
 # POSIX kernel options
 P1003_1B_MQUEUE			opt_posix.h
 P1003_1B_SEMAPHORES		opt_posix.h
 _KPOSIX_PRIORITY_SCHEDULING	opt_posix.h
 
 # Do we want the config file compiled into the kernel?
 INCLUDE_CONFIG_FILE	opt_config.h
 
 # Options for static filesystems.  These should only be used at config
 # time, since the corresponding lkms cannot work if there are any static
 # dependencies.  Unusability is enforced by hiding the defines for the
 # options in a never-included header.
 CD9660		opt_dontuse.h
 EXT2FS		opt_dontuse.h
 FDESCFS		opt_dontuse.h
 FFS		opt_dontuse.h
 FUSE		opt_dontuse.h
 MSDOSFS		opt_dontuse.h
 NANDFS		opt_dontuse.h
 NULLFS		opt_dontuse.h
 PROCFS		opt_dontuse.h
 PSEUDOFS	opt_dontuse.h
 REISERFS	opt_dontuse.h
 SMBFS		opt_dontuse.h
 TMPFS		opt_dontuse.h
 UDF		opt_dontuse.h
 UNIONFS		opt_dontuse.h
 ZFS		opt_dontuse.h
 
 # Pseudofs debugging
 PSEUDOFS_TRACE	opt_pseudofs.h
 
 # In-kernel GSS-API
 KGSSAPI		opt_kgssapi.h
 KGSSAPI_DEBUG	opt_kgssapi.h
 
 # These static filesystems have one slightly bogus static dependency in
 # sys/i386/i386/autoconf.c.  If any of these filesystems are
 # statically compiled into the kernel, code for mounting them as root
 # filesystems will be enabled - but look below.
 NFSCLIENT	opt_nfs.h
 NFSSERVER	opt_nfs.h
 
 # Use these options to compile the experimental nfs client and/or
 # server that supports NFSv4 into a kernel.
 # NFSCL - client
 # NFSD - server
 NFSCL		opt_nfs.h
 NFSD		opt_nfs.h
 
 # filesystems and libiconv bridge
 CD9660_ICONV	opt_dontuse.h
 MSDOSFS_ICONV	opt_dontuse.h
 UDF_ICONV	opt_dontuse.h
 
 # If you are following the conditions in the copyright,
 # you can enable soft-updates which will speed up a lot of thigs
 # and make the system safer from crashes at the same time.
 # otherwise a STUB module will be compiled in.
 SOFTUPDATES	opt_ffs.h
 
 # On small, embedded systems, it can be useful to turn off support for
 # snapshots.  It saves about 30-40k for a feature that would be lightly
 # used, if it is used at all.
 NO_FFS_SNAPSHOT	opt_ffs.h
 
 # Enabling this option turns on support for Access Control Lists in UFS,
 # which can be used to support high security configurations.  Depends on
 # UFS_EXTATTR.
 UFS_ACL		opt_ufs.h
 
 # Enabling this option turns on support for extended attributes in UFS-based
 # filesystems, which can be used to support high security configurations
 # as well as new filesystem features.
 UFS_EXTATTR	opt_ufs.h
 UFS_EXTATTR_AUTOSTART	opt_ufs.h
 
 # Enable fast hash lookups for large directories on UFS-based filesystems.
 UFS_DIRHASH	opt_ufs.h
 
 # Enable gjournal-based UFS journal.
 UFS_GJOURNAL	opt_ufs.h
 
 # The below sentence is not in English, and neither is this one.
 # We plan to remove the static dependences above, with a
 # <filesystem>_ROOT option to control if it usable as root.  This list
 # allows these options to be present in config files already (though
 # they won't make any difference yet).
 NFS_ROOT	opt_nfsroot.h
 
 # SMB/CIFS requester
 NETSMB		opt_netsmb.h
 
 # Options used only in subr_param.c.
 HZ		opt_param.h
 MAXFILES	opt_param.h
 NBUF		opt_param.h
 NSFBUFS		opt_param.h
 VM_BCACHE_SIZE_MAX	opt_param.h
 VM_SWZONE_SIZE_MAX	opt_param.h
 MAXUSERS
 DFLDSIZ		opt_param.h
 MAXDSIZ		opt_param.h
 MAXSSIZ		opt_param.h
 
 # Generic SCSI options.
 CAM_MAX_HIGHPOWER	opt_cam.h
 CAMDEBUG		opt_cam.h
 CAM_DEBUG_COMPILE	opt_cam.h
 CAM_DEBUG_DELAY		opt_cam.h
 CAM_DEBUG_BUS		opt_cam.h
 CAM_DEBUG_TARGET	opt_cam.h
 CAM_DEBUG_LUN		opt_cam.h
 CAM_DEBUG_FLAGS		opt_cam.h
 CAM_BOOT_DELAY		opt_cam.h
 SCSI_DELAY		opt_scsi.h
 SCSI_NO_SENSE_STRINGS	opt_scsi.h
 SCSI_NO_OP_STRINGS	opt_scsi.h
 
 # Options used only in cam/ata/ata_da.c
 ADA_TEST_FAILURE	opt_ada.h
 ATA_STATIC_ID		opt_ada.h
 
 # Options used only in cam/scsi/scsi_cd.c
 CHANGER_MIN_BUSY_SECONDS	opt_cd.h
 CHANGER_MAX_BUSY_SECONDS	opt_cd.h
 
 # Options used only in cam/scsi/scsi_sa.c.
 SA_IO_TIMEOUT		opt_sa.h
 SA_SPACE_TIMEOUT	opt_sa.h
 SA_REWIND_TIMEOUT	opt_sa.h
 SA_ERASE_TIMEOUT	opt_sa.h
 SA_1FM_AT_EOD		opt_sa.h
 
 # Options used only in cam/scsi/scsi_pt.c
 SCSI_PT_DEFAULT_TIMEOUT	opt_pt.h
 
 # Options used only in cam/scsi/scsi_ses.c
 SES_ENABLE_PASSTHROUGH	opt_ses.h
 
 # Options used in dev/sym/ (Symbios SCSI driver).
 SYM_SETUP_LP_PROBE_MAP	opt_sym.h	#-Low Priority Probe Map (bits)
 					# Allows the ncr to take precedence
 					# 1 (1<<0) -> 810a, 860
 					# 2 (1<<1) -> 825a, 875, 885, 895
 					# 4 (1<<2) -> 895a, 896, 1510d 
 SYM_SETUP_SCSI_DIFF	opt_sym.h	#-HVD support for 825a, 875, 885
 					# disabled:0 (default), enabled:1
 SYM_SETUP_PCI_PARITY	opt_sym.h	#-PCI parity checking
 					# disabled:0, enabled:1 (default)
 SYM_SETUP_MAX_LUN	opt_sym.h	#-Number of LUNs supported
 					# default:8, range:[1..64]
 
 # Options used only in pci/ncr.c
 SCSI_NCR_DEBUG		opt_ncr.h
 SCSI_NCR_MAX_SYNC	opt_ncr.h
 SCSI_NCR_MAX_WIDE	opt_ncr.h
 SCSI_NCR_MYADDR		opt_ncr.h
 
 # Options used only in dev/isp/*
 ISP_TARGET_MODE		opt_isp.h
 ISP_FW_CRASH_DUMP	opt_isp.h
 ISP_DEFAULT_ROLES	opt_isp.h
 ISP_INTERNAL_TARGET	opt_isp.h
 
 # Options used only in dev/iscsi
 ISCSI_INITIATOR_DEBUG	opt_iscsi_initiator.h
 
 # Net stuff.
 ACCEPT_FILTER_DATA
 ACCEPT_FILTER_DNS
 ACCEPT_FILTER_HTTP
 ALTQ			opt_global.h
 ALTQ_CBQ		opt_altq.h
 ALTQ_CDNR		opt_altq.h
 ALTQ_DEBUG		opt_altq.h
 ALTQ_HFSC		opt_altq.h
 ALTQ_NOPCC		opt_altq.h
 ALTQ_PRIQ		opt_altq.h
 ALTQ_RED		opt_altq.h
 ALTQ_RIO		opt_altq.h
 BOOTP			opt_bootp.h
 BOOTP_BLOCKSIZE		opt_bootp.h
 BOOTP_COMPAT		opt_bootp.h
 BOOTP_NFSROOT		opt_bootp.h
 BOOTP_NFSV3		opt_bootp.h
 BOOTP_WIRED_TO		opt_bootp.h
 DEVICE_POLLING
 DEV_ENC			opt_enc.h
 DEV_PF			opt_pf.h
 DEV_PFLOG		opt_pf.h
 DEV_PFSYNC		opt_pf.h
 DEV_VLAN		opt_vlan.h
 DUMMYNET		opt_ipdn.h
 INET			opt_inet.h
 INET6			opt_inet6.h
 IPDIVERT
 IPFILTER		opt_ipfilter.h
 IPFILTER_DEFAULT_BLOCK	opt_ipfilter.h
 IPFILTER_LOG		opt_ipfilter.h
 IPFILTER_LOOKUP		opt_ipfilter.h
 IPFIREWALL		opt_ipfw.h
 IPFIREWALL_DEFAULT_TO_ACCEPT	opt_ipfw.h
 IPFIREWALL_NAT		opt_ipfw.h
 IPFIREWALL_VERBOSE	opt_ipfw.h
 IPFIREWALL_VERBOSE_LIMIT	opt_ipfw.h
 IPSEC			opt_ipsec.h
 IPSEC_DEBUG		opt_ipsec.h
 IPSEC_FILTERTUNNEL	opt_ipsec.h
 IPSEC_NAT_T		opt_ipsec.h
 IPSTEALTH
 KRPC
 LIBALIAS
 LIBMBPOOL
 LIBMCHAIN
 MBUF_PROFILING
 MBUF_STRESS_TEST
 MROUTING		opt_mrouting.h
 NFSLOCKD
 PCBGROUP		opt_pcbgroup.h
 PF_DEFAULT_TO_DROP	opt_pf.h
 RADIX_MPATH		opt_mpath.h
 ROUTETABLES		opt_route.h
 RSS			opt_rss.h
 SLIP_IFF_OPTS		opt_slip.h
 TCPDEBUG
 TCP_OFFLOAD		opt_inet.h # Enable code to dispatch TCP offloading
 TCP_SIGNATURE		opt_inet.h
 VLAN_ARRAY		opt_vlan.h
 XBONEHACK
 FLOWTABLE		opt_route.h
 FLOWTABLE_HASH_ALL	opt_route.h
 
 #
 # SCTP
 #
 SCTP			opt_sctp.h
 SCTP_DEBUG		opt_sctp.h # Enable debug printfs
 SCTP_WITH_NO_CSUM	opt_sctp.h # Use this at your peril
 SCTP_LOCK_LOGGING	opt_sctp.h # Log to KTR lock activity
 SCTP_MBUF_LOGGING	opt_sctp.h # Log to KTR general mbuf aloc/free
 SCTP_MBCNT_LOGGING	opt_sctp.h # Log to KTR mbcnt activity
 SCTP_PACKET_LOGGING	opt_sctp.h # Log to a packet buffer last N packets
 SCTP_LTRACE_CHUNKS	opt_sctp.h # Log to KTR chunks processed
 SCTP_LTRACE_ERRORS	opt_sctp.h # Log to KTR error returns.
-SCTP_USE_PERCPU_STAT    opt_sctp.h # Use per cpu stats.
-SCTP_MCORE_INPUT        opt_sctp.h # Have multiple input threads for input mbufs
+SCTP_USE_PERCPU_STAT	opt_sctp.h # Use per cpu stats.
+SCTP_MCORE_INPUT	opt_sctp.h # Have multiple input threads for input mbufs
+SCTP_LOCAL_TRACE_BUF	opt_sctp.h # Use tracebuffer exported via sysctl
 #
 #
 #
 
 # Netgraph(4). Use option NETGRAPH to enable the base netgraph code.
 # Each netgraph node type can be either be compiled into the kernel
 # or loaded dynamically. To get the former, include the corresponding
 # option below. Each type has its own man page, e.g. ng_async(4).
 NETGRAPH
 NETGRAPH_DEBUG		opt_netgraph.h
 NETGRAPH_ASYNC		opt_netgraph.h
 NETGRAPH_ATMLLC		opt_netgraph.h
 NETGRAPH_ATM_ATMPIF	opt_netgraph.h
 NETGRAPH_BLUETOOTH	opt_netgraph.h
 NETGRAPH_BLUETOOTH_BT3C	opt_netgraph.h
 NETGRAPH_BLUETOOTH_H4	opt_netgraph.h
 NETGRAPH_BLUETOOTH_HCI	opt_netgraph.h
 NETGRAPH_BLUETOOTH_L2CAP	opt_netgraph.h
 NETGRAPH_BLUETOOTH_SOCKET	opt_netgraph.h
 NETGRAPH_BLUETOOTH_UBT	opt_netgraph.h
 NETGRAPH_BLUETOOTH_UBTBCMFW	opt_netgraph.h
 NETGRAPH_BPF		opt_netgraph.h
 NETGRAPH_BRIDGE		opt_netgraph.h
 NETGRAPH_CAR		opt_netgraph.h
 NETGRAPH_CISCO		opt_netgraph.h
 NETGRAPH_DEFLATE	opt_netgraph.h
 NETGRAPH_DEVICE		opt_netgraph.h
 NETGRAPH_ECHO		opt_netgraph.h
 NETGRAPH_EIFACE		opt_netgraph.h
 NETGRAPH_ETHER		opt_netgraph.h
 NETGRAPH_ETHER_ECHO	opt_netgraph.h
 NETGRAPH_FEC		opt_netgraph.h
 NETGRAPH_FRAME_RELAY	opt_netgraph.h
 NETGRAPH_GIF		opt_netgraph.h
 NETGRAPH_GIF_DEMUX	opt_netgraph.h
 NETGRAPH_HOLE		opt_netgraph.h
 NETGRAPH_IFACE		opt_netgraph.h
 NETGRAPH_IP_INPUT	opt_netgraph.h
 NETGRAPH_IPFW		opt_netgraph.h
 NETGRAPH_KSOCKET	opt_netgraph.h
 NETGRAPH_L2TP		opt_netgraph.h
 NETGRAPH_LMI		opt_netgraph.h
 # MPPC compression requires proprietary files (not included)
 NETGRAPH_MPPC_COMPRESSION	opt_netgraph.h
 NETGRAPH_MPPC_ENCRYPTION	opt_netgraph.h
 NETGRAPH_NAT		opt_netgraph.h
 NETGRAPH_NETFLOW	opt_netgraph.h
 NETGRAPH_ONE2MANY	opt_netgraph.h
 NETGRAPH_PATCH		opt_netgraph.h
 NETGRAPH_PIPE		opt_netgraph.h
 NETGRAPH_PPP		opt_netgraph.h
 NETGRAPH_PPPOE		opt_netgraph.h
 NETGRAPH_PPTPGRE	opt_netgraph.h
 NETGRAPH_PRED1		opt_netgraph.h
 NETGRAPH_RFC1490	opt_netgraph.h
 NETGRAPH_SOCKET		opt_netgraph.h
 NETGRAPH_SPLIT		opt_netgraph.h
 NETGRAPH_SPPP		opt_netgraph.h
 NETGRAPH_TAG		opt_netgraph.h
 NETGRAPH_TCPMSS		opt_netgraph.h
 NETGRAPH_TEE		opt_netgraph.h
 NETGRAPH_TTY		opt_netgraph.h
 NETGRAPH_UI		opt_netgraph.h
 NETGRAPH_VJC		opt_netgraph.h
 NETGRAPH_VLAN		opt_netgraph.h
 
 # NgATM options
 NGATM_ATM		opt_netgraph.h
 NGATM_ATMBASE		opt_netgraph.h
 NGATM_SSCOP		opt_netgraph.h
 NGATM_SSCFU		opt_netgraph.h
 NGATM_UNI		opt_netgraph.h
 NGATM_CCATM		opt_netgraph.h
 
 # DRM options
 DRM_DEBUG		opt_drm.h
 
 TI_SF_BUF_JUMBO		opt_ti.h
 TI_JUMBO_HDRSPLIT	opt_ti.h
 
 # XXX Conflict: # of devices vs network protocol (Native ATM).
 # This makes "atm.h" unusable.
 NATM
 
 # DPT driver debug flags
 DPT_MEASURE_PERFORMANCE	opt_dpt.h
 DPT_RESET_HBA		opt_dpt.h
 
 # Misc debug flags.  Most of these should probably be replaced with
 # 'DEBUG', and then let people recompile just the interesting modules
 # with 'make CC="cc -DDEBUG"'.
 CLUSTERDEBUG		opt_debug_cluster.h
 DEBUG_1284		opt_ppb_1284.h
 VP0_DEBUG		opt_vpo.h
 LPT_DEBUG		opt_lpt.h
 PLIP_DEBUG		opt_plip.h
 LOCKF_DEBUG		opt_debug_lockf.h
 SI_DEBUG		opt_debug_si.h
 
 # Fb options
 FB_DEBUG		opt_fb.h
 FB_INSTALL_CDEV		opt_fb.h
 
 # ppbus related options
 PERIPH_1284		opt_ppb_1284.h
 DONTPROBE_1284		opt_ppb_1284.h
 
 # smbus related options
 ENABLE_ALART		opt_intpm.h
 
 # These cause changes all over the kernel
 BLKDEV_IOSIZE		opt_global.h
 BURN_BRIDGES		opt_global.h
 DEBUG			opt_global.h
 DEBUG_LOCKS		opt_global.h
 DEBUG_VFS_LOCKS		opt_global.h
 DFLTPHYS		opt_global.h
 DIAGNOSTIC		opt_global.h
 INVARIANT_SUPPORT	opt_global.h
 INVARIANTS		opt_global.h
 MAXCPU			opt_global.h
 MAXMEMDOM		opt_global.h
 MAXPHYS			opt_global.h
 MCLSHIFT		opt_global.h
 MUTEX_DEBUG		opt_global.h
 MUTEX_NOINLINE		opt_global.h
 LOCK_PROFILING		opt_global.h
 LOCK_PROFILING_FAST	opt_global.h
 MSIZE			opt_global.h
 REGRESSION		opt_global.h
 RWLOCK_NOINLINE		opt_global.h
 SX_NOINLINE		opt_global.h
 VFS_BIO_DEBUG		opt_global.h
 
 # These are VM related options
 VM_KMEM_SIZE		opt_vm.h
 VM_KMEM_SIZE_SCALE	opt_vm.h
 VM_KMEM_SIZE_MAX	opt_vm.h
 VM_NRESERVLEVEL		opt_vm.h
 VM_LEVEL_0_ORDER	opt_vm.h
 NO_SWAPPING		opt_vm.h
 MALLOC_MAKE_FAILURES	opt_vm.h
 MALLOC_PROFILE		opt_vm.h
 MALLOC_DEBUG_MAXZONES	opt_vm.h
 
 # The MemGuard replacement allocator used for tamper-after-free detection
 DEBUG_MEMGUARD		opt_vm.h
 
 # The RedZone malloc(9) protection
 DEBUG_REDZONE		opt_vm.h
 
 # Standard SMP options
 SMP			opt_global.h
 
 # Size of the kernel message buffer
 MSGBUF_SIZE		opt_msgbuf.h
 
 # NFS options
 NFS_MINATTRTIMO		opt_nfs.h
 NFS_MAXATTRTIMO		opt_nfs.h
 NFS_MINDIRATTRTIMO	opt_nfs.h
 NFS_MAXDIRATTRTIMO	opt_nfs.h
 NFS_GATHERDELAY		opt_nfs.h
 NFS_WDELAYHASHSIZ	opt_nfs.h
 NFS_DEBUG		opt_nfs.h
 
 # For the Bt848/Bt848A/Bt849/Bt878/Bt879 driver
 OVERRIDE_CARD			opt_bktr.h
 OVERRIDE_TUNER			opt_bktr.h
 OVERRIDE_DBX			opt_bktr.h
 OVERRIDE_MSP			opt_bktr.h
 BROOKTREE_SYSTEM_DEFAULT	opt_bktr.h
 BROOKTREE_ALLOC_PAGES		opt_bktr.h
 BKTR_OVERRIDE_CARD		opt_bktr.h
 BKTR_OVERRIDE_TUNER		opt_bktr.h
 BKTR_OVERRIDE_DBX		opt_bktr.h
 BKTR_OVERRIDE_MSP		opt_bktr.h
 BKTR_SYSTEM_DEFAULT		opt_bktr.h
 BKTR_ALLOC_PAGES		opt_bktr.h
 BKTR_USE_PLL			opt_bktr.h	
 BKTR_GPIO_ACCESS		opt_bktr.h
 BKTR_NO_MSP_RESET		opt_bktr.h
 BKTR_430_FX_MODE		opt_bktr.h
 BKTR_SIS_VIA_MODE		opt_bktr.h
 BKTR_USE_FREEBSD_SMBUS		opt_bktr.h
 BKTR_NEW_MSP34XX_DRIVER		opt_bktr.h
 
 # Options for uart(4)
 UART_PPS_ON_CTS		opt_uart.h
 UART_POLL_FREQ		opt_uart.h
 
 # options for bus/device framework
 BUS_DEBUG		opt_bus.h
 
 # options for USB support
 USB_DEBUG		opt_usb.h
 USB_HOST_ALIGN		opt_usb.h
 USB_REQ_DEBUG		opt_usb.h
 USB_TEMPLATE		opt_usb.h
 USB_VERBOSE		opt_usb.h
 USB_EHCI_BIG_ENDIAN_DESC	opt_usb.h
 U3G_DEBUG		opt_u3g.h
 UKBD_DFLT_KEYMAP	opt_ukbd.h
 UPLCOM_INTR_INTERVAL	opt_uplcom.h
 UVSCOM_DEFAULT_OPKTSIZE	opt_uvscom.h
 UVSCOM_INTR_INTERVAL	opt_uvscom.h
 
 # Embedded system options
 INIT_PATH
 
 ROOTDEVNAME
 
 FDC_DEBUG		opt_fdc.h
 PCFCLOCK_VERBOSE	opt_pcfclock.h
 PCFCLOCK_MAX_RETRIES	opt_pcfclock.h
 
 KTR			opt_global.h
 KTR_ALQ			opt_ktr.h
 KTR_MASK		opt_ktr.h
 KTR_CPUMASK		opt_ktr.h
 KTR_COMPILE		opt_global.h
 KTR_BOOT_ENTRIES	opt_global.h
 KTR_ENTRIES		opt_global.h
 KTR_VERBOSE		opt_ktr.h
 WITNESS			opt_global.h
 WITNESS_KDB		opt_witness.h
 WITNESS_NO_VNODE	opt_witness.h
 WITNESS_SKIPSPIN	opt_witness.h
 OPENSOLARIS_WITNESS	opt_global.h
 
 # options for ACPI support
 ACPI_DEBUG		opt_acpi.h
 ACPI_MAX_TASKS		opt_acpi.h
 ACPI_MAX_THREADS	opt_acpi.h
 ACPI_DMAR		opt_acpi.h
 
 # ISA support
 DEV_ISA			opt_isa.h
 ISAPNP			opt_isa.h
 
 # various 'device presence' options.
 DEV_BPF			opt_bpf.h
 DEV_NETMAP		opt_global.h
 DEV_MCA			opt_mca.h
 DEV_CARP		opt_carp.h
 DEV_SPLASH		opt_splash.h
 
 # EISA support
 DEV_EISA		opt_eisa.h
 EISA_SLOTS		opt_eisa.h
 
 # ed driver
 ED_HPP			opt_ed.h
 ED_3C503		opt_ed.h
 ED_SIC			opt_ed.h
 
 # bce driver
 BCE_DEBUG		opt_bce.h
 BCE_NVRAM_WRITE_SUPPORT	opt_bce.h
 
 SOCKBUF_DEBUG		opt_global.h
 
 
 # options for ubsec driver
 UBSEC_DEBUG		opt_ubsec.h
 UBSEC_RNDTEST		opt_ubsec.h
 UBSEC_NO_RNG		opt_ubsec.h
 
 # options for hifn driver
 HIFN_DEBUG		opt_hifn.h
 HIFN_RNDTEST		opt_hifn.h
 
 # options for safenet driver
 SAFE_DEBUG		opt_safe.h
 SAFE_NO_RNG		opt_safe.h
 SAFE_RNDTEST		opt_safe.h
 
 # syscons/vt options
 MAXCONS			opt_syscons.h
 VT_FB_DEFAULT_WIDTH	opt_syscons.h
 VT_FB_DEFAULT_HEIGHT	opt_syscons.h
 SC_ALT_MOUSE_IMAGE	opt_syscons.h
 SC_CUT_SPACES2TABS	opt_syscons.h
 SC_CUT_SEPCHARS		opt_syscons.h
 SC_DEBUG_LEVEL		opt_syscons.h
 SC_DFLT_FONT		opt_syscons.h
 SC_DISABLE_KDBKEY	opt_syscons.h
 SC_DISABLE_REBOOT	opt_syscons.h
 SC_HISTORY_SIZE		opt_syscons.h
 SC_KERNEL_CONS_ATTR	opt_syscons.h
 SC_KERNEL_CONS_REV_ATTR	opt_syscons.h
 SC_MOUSE_CHAR		opt_syscons.h
 SC_NO_CUTPASTE		opt_syscons.h
 SC_NO_FONT_LOADING	opt_syscons.h
 SC_NO_HISTORY		opt_syscons.h
 SC_NO_MODE_CHANGE	opt_syscons.h
 SC_NO_SUSPEND_VTYSWITCH	opt_syscons.h
 SC_NO_SYSMOUSE		opt_syscons.h
 SC_NORM_ATTR		opt_syscons.h
 SC_NORM_REV_ATTR	opt_syscons.h
 SC_PIXEL_MODE		opt_syscons.h
 SC_RENDER_DEBUG		opt_syscons.h
 SC_TWOBUTTON_MOUSE	opt_syscons.h
 DEV_SC			opt_syscons.h
 DEV_VT			opt_syscons.h
 
 
 # teken terminal emulator options
 TEKEN_CONS25		opt_teken.h
 TEKEN_UTF8		opt_teken.h
 
 # options for printf
 PRINTF_BUFR_SIZE	opt_printf.h
 
 # kbd options
 KBD_DISABLE_KEYMAP_LOAD	opt_kbd.h
 KBD_INSTALL_CDEV	opt_kbd.h
 KBD_MAXRETRY		opt_kbd.h
 KBD_MAXWAIT		opt_kbd.h
 KBD_RESETDELAY		opt_kbd.h
 KBDIO_DEBUG		opt_kbd.h
 
 # options for the Atheros driver
 ATH_DEBUG		opt_ath.h
 ATH_TXBUF		opt_ath.h
 ATH_RXBUF		opt_ath.h
 ATH_DIAGAPI		opt_ath.h
 ATH_TX99_DIAG		opt_ath.h
 ATH_ENABLE_11N		opt_ath.h
 ATH_ENABLE_DFS		opt_ath.h
 ATH_EEPROM_FIRMWARE	opt_ath.h
 ATH_ENABLE_RADIOTAP_VENDOR_EXT	opt_ath.h
 ATH_DEBUG_ALQ		opt_ath.h
 ATH_KTR_INTR_DEBUG	opt_ath.h
 
 # options for the Atheros hal
 AH_SUPPORT_AR5416	opt_ah.h
 # XXX For now, this breaks non-AR9130 chipsets, so only use it
 # XXX when actually targetting AR9130.
 AH_SUPPORT_AR9130	opt_ah.h
 
 # This is required for AR933x SoC support
 AH_SUPPORT_AR9330	opt_ah.h
 AH_SUPPORT_AR9340	opt_ah.h
 
 AH_DEBUG		opt_ah.h
 AH_ASSERT		opt_ah.h
 AH_DEBUG_ALQ		opt_ah.h
 AH_REGOPS_FUNC		opt_ah.h
 AH_WRITE_REGDOMAIN	opt_ah.h
 AH_DEBUG_COUNTRY	opt_ah.h
 AH_WRITE_EEPROM		opt_ah.h
 AH_PRIVATE_DIAG		opt_ah.h
 AH_NEED_DESC_SWAP	opt_ah.h
 AH_USE_INIPDGAIN	opt_ah.h
 AH_MAXCHAN		opt_ah.h
 AH_RXCFG_SDMAMW_4BYTES	opt_ah.h
 AH_INTERRUPT_DEBUGGING	opt_ah.h
 # AR5416 and later interrupt mitigation
 # XXX do not use this for AR9130
 AH_AR5416_INTERRUPT_MITIGATION	opt_ah.h
 
 # options for the Broadcom BCM43xx driver (bwi)
 BWI_DEBUG		opt_bwi.h
 BWI_DEBUG_VERBOSE	opt_bwi.h
 
 # options for the Marvell 8335 wireless driver
 MALO_DEBUG		opt_malo.h
 MALO_TXBUF		opt_malo.h
 MALO_RXBUF		opt_malo.h
 
 # options for the Marvell wireless driver
 MWL_DEBUG		opt_mwl.h
 MWL_TXBUF		opt_mwl.h
 MWL_RXBUF		opt_mwl.h
 MWL_DIAGAPI		opt_mwl.h
 MWL_AGGR_SIZE		opt_mwl.h
 MWL_TX_NODROP		opt_mwl.h
 
 # Options for the Intel 802.11n wireless driver
 IWN_DEBUG		opt_iwn.h
 
 # dcons options 
 DCONS_BUF_SIZE		opt_dcons.h
 DCONS_POLL_HZ		opt_dcons.h
 DCONS_FORCE_CONSOLE	opt_dcons.h
 DCONS_FORCE_GDB		opt_dcons.h
 
 # HWPMC options
 HWPMC_HOOKS
 HWPMC_MIPS_BACKTRACE 	opt_hwpmc_hooks.h
 
 # XBOX options for FreeBSD/i386, but some files are MI
 XBOX			opt_xbox.h
 
 # Interrupt filtering
 INTR_FILTER
 
 # 802.11 support layer
 IEEE80211_DEBUG		opt_wlan.h
 IEEE80211_DEBUG_REFCNT	opt_wlan.h
 IEEE80211_AMPDU_AGE	opt_wlan.h
 IEEE80211_SUPPORT_MESH	opt_wlan.h
 IEEE80211_SUPPORT_SUPERG	opt_wlan.h
 IEEE80211_SUPPORT_TDMA	opt_wlan.h
 IEEE80211_ALQ		opt_wlan.h
 IEEE80211_DFS_DEBUG	opt_wlan.h
 
 # 802.11 TDMA support
 TDMA_SLOTLEN_DEFAULT	opt_tdma.h
 TDMA_SLOTCNT_DEFAULT	opt_tdma.h
 TDMA_BINTVAL_DEFAULT	opt_tdma.h
 TDMA_TXRATE_11B_DEFAULT	opt_tdma.h
 TDMA_TXRATE_11G_DEFAULT	opt_tdma.h
 TDMA_TXRATE_11A_DEFAULT	opt_tdma.h
 TDMA_TXRATE_TURBO_DEFAULT	opt_tdma.h
 TDMA_TXRATE_HALF_DEFAULT	opt_tdma.h
 TDMA_TXRATE_QUARTER_DEFAULT	opt_tdma.h
 TDMA_TXRATE_11NA_DEFAULT	opt_tdma.h
 TDMA_TXRATE_11NG_DEFAULT	opt_tdma.h
 
 # Network stack virtualization options
 VIMAGE			opt_global.h
 VNET_DEBUG		opt_global.h
 
 # Common Flash Interface (CFI) options
 CFI_SUPPORT_STRATAFLASH	opt_cfi.h
 CFI_ARMEDANDDANGEROUS	opt_cfi.h
 
 # Sound options
 SND_DEBUG		opt_snd.h
 SND_DIAGNOSTIC		opt_snd.h
 SND_FEEDER_MULTIFORMAT	opt_snd.h
 SND_FEEDER_FULL_MULTIFORMAT	opt_snd.h
 SND_FEEDER_RATE_HP	opt_snd.h
 SND_PCM_64		opt_snd.h
 SND_OLDSTEREO		opt_snd.h
 
 X86BIOS
 
 # Flattened device tree options
 FDT		opt_platform.h
 FDT_DTB_STATIC	opt_platform.h
 
 # OFED Infiniband stack
 OFED		opt_ofed.h
 OFED_DEBUG_INIT	opt_ofed.h
 SDP		opt_ofed.h
 SDP_DEBUG	opt_ofed.h
 IPOIB		opt_ofed.h
 IPOIB_DEBUG	opt_ofed.h
 IPOIB_CM	opt_ofed.h
 
 # Resource Accounting
 RACCT		opt_global.h
 
 # Resource Limits
 RCTL		opt_global.h
 
 # Random number generator(s)
 RANDOM_YARROW	opt_random.h
 RANDOM_FORTUNA	opt_random.h
 RANDOM_DEBUG	opt_random.h
 RANDOM_RWFILE	opt_random.h
Index: projects/release-debugdist/sys/conf
===================================================================
--- projects/release-debugdist/sys/conf	(revision 267343)
+++ projects/release-debugdist/sys/conf	(revision 267344)

Property changes on: projects/release-debugdist/sys/conf
___________________________________________________________________
Modified: svn:mergeinfo
## -0,0 +0,1 ##
   Merged /head/sys/conf:r267325-267343
Index: projects/release-debugdist/sys/dev/advansys/adwcam.c
===================================================================
--- projects/release-debugdist/sys/dev/advansys/adwcam.c	(revision 267343)
+++ projects/release-debugdist/sys/dev/advansys/adwcam.c	(revision 267344)
@@ -1,1506 +1,1504 @@
 /*-
  * CAM SCSI interface for the Advanced Systems Inc.
  * Second Generation SCSI controllers.
  *
  * Product specific probe and attach routines can be found in:
  * 
  * adw_pci.c	ABP[3]940UW, ABP950UW, ABP3940U2W
  *
  * Copyright (c) 1998, 1999, 2000 Justin Gibbs.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions, and the following disclaimer,
  *    without modification.
  * 2. The name of the author may not be used to endorse or promote products
  *    derived from this software without specific prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 /*
  * Ported from:
  * advansys.c - Linux Host Driver for AdvanSys SCSI Adapters
  *     
  * Copyright (c) 1995-1998 Advanced System Products, Inc.
  * All Rights Reserved.
  *   
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that redistributions of source
  * code retain the above copyright notice and this comment without
  * modification.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/conf.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/malloc.h>
 #include <sys/lock.h>
 #include <sys/module.h>
 #include <sys/mutex.h>
 #include <sys/bus.h>
 
 #include <machine/bus.h>
 #include <machine/resource.h>
 
 #include <sys/rman.h>
 
 #include <cam/cam.h>
 #include <cam/cam_ccb.h>
 #include <cam/cam_sim.h>
 #include <cam/cam_xpt_sim.h>
 #include <cam/cam_debug.h>
 
 #include <cam/scsi/scsi_message.h>
 
 #include <dev/advansys/adwvar.h>
 
 /* Definitions for our use of the SIM private CCB area */
 #define ccb_acb_ptr spriv_ptr0
 #define ccb_adw_ptr spriv_ptr1
 
 static __inline struct acb*	adwgetacb(struct adw_softc *adw);
 static __inline void		adwfreeacb(struct adw_softc *adw,
 					   struct acb *acb);
 
 static void		adwmapmem(void *arg, bus_dma_segment_t *segs,
 				  int nseg, int error);
 static struct sg_map_node*
 			adwallocsgmap(struct adw_softc *adw);
 static int		adwallocacbs(struct adw_softc *adw);
 
 static void		adwexecuteacb(void *arg, bus_dma_segment_t *dm_segs,
 				      int nseg, int error);
 static void		adw_action(struct cam_sim *sim, union ccb *ccb);
 static void		adw_intr_locked(struct adw_softc *adw);
 static void		adw_poll(struct cam_sim *sim);
 static void		adw_async(void *callback_arg, u_int32_t code,
 				  struct cam_path *path, void *arg);
 static void		adwprocesserror(struct adw_softc *adw, struct acb *acb);
 static void		adwtimeout(void *arg);
 static void		adw_handle_device_reset(struct adw_softc *adw,
 						u_int target);
 static void		adw_handle_bus_reset(struct adw_softc *adw,
 					     int initiated);
 
 static __inline struct acb*
 adwgetacb(struct adw_softc *adw)
 {
 	struct	acb* acb;
 
 	if (!dumping)
 		mtx_assert(&adw->lock, MA_OWNED);
 	if ((acb = SLIST_FIRST(&adw->free_acb_list)) != NULL) {
 		SLIST_REMOVE_HEAD(&adw->free_acb_list, links);
 	} else if (adw->num_acbs < adw->max_acbs) {
 		adwallocacbs(adw);
 		acb = SLIST_FIRST(&adw->free_acb_list);
 		if (acb == NULL)
 			device_printf(adw->device, "Can't malloc ACB\n");
 		else {
 			SLIST_REMOVE_HEAD(&adw->free_acb_list, links);
 		}
 	}
 
 	return (acb);
 }
 
 static __inline void
 adwfreeacb(struct adw_softc *adw, struct acb *acb)
 {
 
 	if (!dumping)
 		mtx_assert(&adw->lock, MA_OWNED);
 	if ((acb->state & ACB_ACTIVE) != 0)
 		LIST_REMOVE(&acb->ccb->ccb_h, sim_links.le);
 	if ((acb->state & ACB_RELEASE_SIMQ) != 0)
 		acb->ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
 	else if ((adw->state & ADW_RESOURCE_SHORTAGE) != 0
 	      && (acb->ccb->ccb_h.status & CAM_RELEASE_SIMQ) == 0) {
 		acb->ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
 		adw->state &= ~ADW_RESOURCE_SHORTAGE;
 	}
 	acb->state = ACB_FREE;
 	SLIST_INSERT_HEAD(&adw->free_acb_list, acb, links);
 }
 
 static void
 adwmapmem(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 {
 	bus_addr_t *busaddrp;
 
 	busaddrp = (bus_addr_t *)arg;
 	*busaddrp = segs->ds_addr;
 }
 
 static struct sg_map_node *
 adwallocsgmap(struct adw_softc *adw)
 {
 	struct sg_map_node *sg_map;
 
 	sg_map = malloc(sizeof(*sg_map), M_DEVBUF, M_NOWAIT);
 
 	if (sg_map == NULL)
 		return (NULL);
 
 	/* Allocate S/G space for the next batch of ACBS */
 	if (bus_dmamem_alloc(adw->sg_dmat, (void **)&sg_map->sg_vaddr,
 			     BUS_DMA_NOWAIT, &sg_map->sg_dmamap) != 0) {
 		free(sg_map, M_DEVBUF);
 		return (NULL);
 	}
 
 	SLIST_INSERT_HEAD(&adw->sg_maps, sg_map, links);
 
 	bus_dmamap_load(adw->sg_dmat, sg_map->sg_dmamap, sg_map->sg_vaddr,
 			PAGE_SIZE, adwmapmem, &sg_map->sg_physaddr, /*flags*/0);
 
 	bzero(sg_map->sg_vaddr, PAGE_SIZE);
 	return (sg_map);
 }
 
 /*
  * Allocate another chunk of CCB's. Return count of entries added.
  */
 static int
 adwallocacbs(struct adw_softc *adw)
 {
 	struct acb *next_acb;
 	struct sg_map_node *sg_map;
 	bus_addr_t busaddr;
 	struct adw_sg_block *blocks;
 	int newcount;
 	int i;
 
 	next_acb = &adw->acbs[adw->num_acbs];
 	sg_map = adwallocsgmap(adw);
 
 	if (sg_map == NULL)
 		return (0);
 
 	blocks = sg_map->sg_vaddr;
 	busaddr = sg_map->sg_physaddr;
 
 	newcount = (PAGE_SIZE / (ADW_SG_BLOCKCNT * sizeof(*blocks)));
 	for (i = 0; adw->num_acbs < adw->max_acbs && i < newcount; i++) {
 		int error;
 
 		error = bus_dmamap_create(adw->buffer_dmat, /*flags*/0,
 					  &next_acb->dmamap);
 		if (error != 0)
 			break;
 		next_acb->queue.scsi_req_baddr = acbvtob(adw, next_acb);
 		next_acb->queue.scsi_req_bo = acbvtobo(adw, next_acb);
 		next_acb->queue.sense_baddr =
 		    acbvtob(adw, next_acb) + offsetof(struct acb, sense_data);
 		next_acb->sg_blocks = blocks;
 		next_acb->sg_busaddr = busaddr;
 		next_acb->state = ACB_FREE;
 		callout_init_mtx(&next_acb->timer, &adw->lock, 0);
 		SLIST_INSERT_HEAD(&adw->free_acb_list, next_acb, links);
 		blocks += ADW_SG_BLOCKCNT;
 		busaddr += ADW_SG_BLOCKCNT * sizeof(*blocks);
 		next_acb++;
 		adw->num_acbs++;
 	}
 	return (i);
 }
 
 static void
 adwexecuteacb(void *arg, bus_dma_segment_t *dm_segs, int nseg, int error)
 {
 	struct	 acb *acb;
 	union	 ccb *ccb;
 	struct	 adw_softc *adw;
 
 	acb = (struct acb *)arg;
 	ccb = acb->ccb;
 	adw = (struct adw_softc *)ccb->ccb_h.ccb_adw_ptr;
 
 	if (!dumping)
 		mtx_assert(&adw->lock, MA_OWNED);
 	if (error != 0) {
 		if (error != EFBIG)
 			device_printf(adw->device, "Unexepected error 0x%x "
 			    "returned from bus_dmamap_load\n", error);
 		if (ccb->ccb_h.status == CAM_REQ_INPROG) {
 			xpt_freeze_devq(ccb->ccb_h.path, /*count*/1);
 			ccb->ccb_h.status = CAM_REQ_TOO_BIG|CAM_DEV_QFRZN;
 		}
 		adwfreeacb(adw, acb);
 		xpt_done(ccb);
 		return;
 	}
 		
 	if (nseg != 0) {
 		bus_dmasync_op_t op;
 
 		acb->queue.data_addr = dm_segs[0].ds_addr;
 		acb->queue.data_cnt = ccb->csio.dxfer_len;
 		if (nseg > 1) {
 			struct adw_sg_block *sg_block;
 			struct adw_sg_elm *sg;
 			bus_addr_t sg_busaddr;
 			u_int sg_index;
 			bus_dma_segment_t *end_seg;
 
 			end_seg = dm_segs + nseg;
 
 			sg_busaddr = acb->sg_busaddr;
 			sg_index = 0;
 			/* Copy the segments into our SG list */
 			for (sg_block = acb->sg_blocks;; sg_block++) {
 				u_int i;
 
 				sg = sg_block->sg_list;
 				for (i = 0; i < ADW_NO_OF_SG_PER_BLOCK; i++) {
 					if (dm_segs >= end_seg)
 						break;
 				    
 					sg->sg_addr = dm_segs->ds_addr;
 					sg->sg_count = dm_segs->ds_len;
 					sg++;
 					dm_segs++;
 				}
 				sg_block->sg_cnt = i;
 				sg_index += i;
 				if (dm_segs == end_seg) {
 					sg_block->sg_busaddr_next = 0;
 					break;
 				} else {
 					sg_busaddr +=
 					    sizeof(struct adw_sg_block);
 					sg_block->sg_busaddr_next = sg_busaddr;
 				}
 			}
 			acb->queue.sg_real_addr = acb->sg_busaddr;
 		} else {
 			acb->queue.sg_real_addr = 0;
 		}
 
 		if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN)
 			op = BUS_DMASYNC_PREREAD;
 		else
 			op = BUS_DMASYNC_PREWRITE;
 
 		bus_dmamap_sync(adw->buffer_dmat, acb->dmamap, op);
 
 	} else {
 		acb->queue.data_addr = 0;
 		acb->queue.data_cnt = 0;
 		acb->queue.sg_real_addr = 0;
 	}
 
 	/*
 	 * Last time we need to check if this CCB needs to
 	 * be aborted.
 	 */
 	if (ccb->ccb_h.status != CAM_REQ_INPROG) {
 		if (nseg != 0)
 			bus_dmamap_unload(adw->buffer_dmat, acb->dmamap);
 		adwfreeacb(adw, acb);
 		xpt_done(ccb);
 		return;
 	}
 
 	acb->state |= ACB_ACTIVE;
 	ccb->ccb_h.status |= CAM_SIM_QUEUED;
 	LIST_INSERT_HEAD(&adw->pending_ccbs, &ccb->ccb_h, sim_links.le);
 	callout_reset(&acb->timer, (ccb->ccb_h.timeout * hz) / 1000,
 	    adwtimeout, acb);
 
 	adw_send_acb(adw, acb, acbvtob(adw, acb));
 }
 
 static void
 adw_action(struct cam_sim *sim, union ccb *ccb)
 {
 	struct	adw_softc *adw;
 
 	CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE, ("adw_action\n"));
 	
 	adw = (struct adw_softc *)cam_sim_softc(sim);
 	if (!dumping)
 		mtx_assert(&adw->lock, MA_OWNED);
 
 	switch (ccb->ccb_h.func_code) {
 	/* Common cases first */
 	case XPT_SCSI_IO:	/* Execute the requested I/O operation */
 	{
 		struct	ccb_scsiio *csio;
 		struct	acb *acb;
 		int error;
 
 		csio = &ccb->csio;
 
 		/* Max supported CDB length is 12 bytes */
 		if (csio->cdb_len > 12) { 
 			ccb->ccb_h.status = CAM_REQ_INVALID;
 			xpt_done(ccb);
 			return;
 		}
 
 		if ((acb = adwgetacb(adw)) == NULL) {
 			adw->state |= ADW_RESOURCE_SHORTAGE;
 			xpt_freeze_simq(sim, /*count*/1);
 			ccb->ccb_h.status = CAM_REQUEUE_REQ;
 			xpt_done(ccb);
 			return;
 		}
 
 		/* Link acb and ccb so we can find one from the other */
 		acb->ccb = ccb;
 		ccb->ccb_h.ccb_acb_ptr = acb;
 		ccb->ccb_h.ccb_adw_ptr = adw;
 
 		acb->queue.cntl = 0;
 		acb->queue.target_cmd = 0;
 		acb->queue.target_id = ccb->ccb_h.target_id;
 		acb->queue.target_lun = ccb->ccb_h.target_lun;
 
 		acb->queue.mflag = 0;
 		acb->queue.sense_len =
 			MIN(csio->sense_len, sizeof(acb->sense_data));
 		acb->queue.cdb_len = csio->cdb_len;
 		if ((ccb->ccb_h.flags & CAM_TAG_ACTION_VALID) != 0) {
 			switch (csio->tag_action) {
 			case MSG_SIMPLE_Q_TAG:
 				acb->queue.scsi_cntl = ADW_QSC_SIMPLE_Q_TAG;
 				break;
 			case MSG_HEAD_OF_Q_TAG:
 				acb->queue.scsi_cntl = ADW_QSC_HEAD_OF_Q_TAG;
 				break;
 			case MSG_ORDERED_Q_TAG:
 				acb->queue.scsi_cntl = ADW_QSC_ORDERED_Q_TAG;
 				break;
 			default:
 				acb->queue.scsi_cntl = ADW_QSC_NO_TAGMSG;
 				break;
 			}
 		} else
 			acb->queue.scsi_cntl = ADW_QSC_NO_TAGMSG;
 
 		if ((ccb->ccb_h.flags & CAM_DIS_DISCONNECT) != 0)
 			acb->queue.scsi_cntl |= ADW_QSC_NO_DISC;
 
 		acb->queue.done_status = 0;
 		acb->queue.scsi_status = 0;
 		acb->queue.host_status = 0;
 		acb->queue.sg_wk_ix = 0;
 		if ((ccb->ccb_h.flags & CAM_CDB_POINTER) != 0) {
 			if ((ccb->ccb_h.flags & CAM_CDB_PHYS) == 0) {
 				bcopy(csio->cdb_io.cdb_ptr,
 				      acb->queue.cdb, csio->cdb_len);
 			} else {
 				/* I guess I could map it in... */
 				ccb->ccb_h.status = CAM_REQ_INVALID;
 				adwfreeacb(adw, acb);
 				xpt_done(ccb);
 				return;
 			}
 		} else {
 			bcopy(csio->cdb_io.cdb_bytes,
 			      acb->queue.cdb, csio->cdb_len);
 		}
 
 		error = bus_dmamap_load_ccb(adw->buffer_dmat,
 					    acb->dmamap,
 					    ccb,
 					    adwexecuteacb,
 					    acb, /*flags*/0);
 		if (error == EINPROGRESS) {
 			/*
 			 * So as to maintain ordering, freeze the controller
 			 * queue until our mapping is returned.
 			 */
 			xpt_freeze_simq(sim, 1);
 			acb->state |= CAM_RELEASE_SIMQ;
 		}
 		break;
 	}
 	case XPT_RESET_DEV:	/* Bus Device Reset the specified SCSI device */
 	{
 		adw_idle_cmd_status_t status;
 
 		status = adw_idle_cmd_send(adw, ADW_IDLE_CMD_DEVICE_RESET,
 					   ccb->ccb_h.target_id);
 		if (status == ADW_IDLE_CMD_SUCCESS) {
 			ccb->ccb_h.status = CAM_REQ_CMP;
 			if (bootverbose) {
 				xpt_print_path(ccb->ccb_h.path);
 				printf("BDR Delivered\n");
 			}
 		} else
 			ccb->ccb_h.status = CAM_REQ_CMP_ERR;
 		xpt_done(ccb);
 		break;
 	}
 	case XPT_ABORT:			/* Abort the specified CCB */
 		/* XXX Implement */
 		ccb->ccb_h.status = CAM_REQ_INVALID;
 		xpt_done(ccb);
 		break;
 	case XPT_SET_TRAN_SETTINGS:
 	{
 		struct ccb_trans_settings_scsi *scsi;
 		struct ccb_trans_settings_spi *spi;
 		struct	  ccb_trans_settings *cts;
 		u_int	  target_mask;
 
 		cts = &ccb->cts;
 		target_mask = 0x01 << ccb->ccb_h.target_id;
 
 		scsi = &cts->proto_specific.scsi;
 		spi = &cts->xport_specific.spi;
 		if (cts->type == CTS_TYPE_CURRENT_SETTINGS) {
 			u_int sdtrdone;
 
 			sdtrdone = adw_lram_read_16(adw, ADW_MC_SDTR_DONE);
 			if ((spi->valid & CTS_SPI_VALID_DISC) != 0) {
 				u_int discenb;
 
 				discenb =
 				    adw_lram_read_16(adw, ADW_MC_DISC_ENABLE);
 
 				if ((spi->flags & CTS_SPI_FLAGS_DISC_ENB) != 0)
 					discenb |= target_mask;
 				else
 					discenb &= ~target_mask;
 
 				adw_lram_write_16(adw, ADW_MC_DISC_ENABLE,
 						  discenb);
 			}
 		
 			if ((scsi->valid & CTS_SCSI_VALID_TQ) != 0) {
 
 				if ((scsi->flags & CTS_SCSI_FLAGS_TAG_ENB) != 0)
 					adw->tagenb |= target_mask;
 				else
 					adw->tagenb &= ~target_mask;
 			}	
 
 			if ((spi->valid & CTS_SPI_VALID_BUS_WIDTH) != 0) {
 				u_int wdtrenb_orig;
 				u_int wdtrenb;
 				u_int wdtrdone;
 
 				wdtrenb_orig =
 				    adw_lram_read_16(adw, ADW_MC_WDTR_ABLE);
 				wdtrenb = wdtrenb_orig;
 				wdtrdone = adw_lram_read_16(adw,
 							    ADW_MC_WDTR_DONE);
 				switch (spi->bus_width) {
 				case MSG_EXT_WDTR_BUS_32_BIT:
 				case MSG_EXT_WDTR_BUS_16_BIT:
 					wdtrenb |= target_mask;
 					break;
 				case MSG_EXT_WDTR_BUS_8_BIT:
 				default:
 					wdtrenb &= ~target_mask;
 					break;
 				}
 				if (wdtrenb != wdtrenb_orig) {
 					adw_lram_write_16(adw,
 							  ADW_MC_WDTR_ABLE,
 							  wdtrenb);
 					wdtrdone &= ~target_mask;
 					adw_lram_write_16(adw,
 							  ADW_MC_WDTR_DONE,
 							  wdtrdone);
 					/* Wide negotiation forces async */
 					sdtrdone &= ~target_mask;
 					adw_lram_write_16(adw,
 							  ADW_MC_SDTR_DONE,
 							  sdtrdone);
 				}
 			}
 
 			if (((spi->valid & CTS_SPI_VALID_SYNC_RATE) != 0)
 			 || ((spi->valid & CTS_SPI_VALID_SYNC_OFFSET) != 0)) {
 				u_int sdtr_orig;
 				u_int sdtr;
 				u_int sdtrable_orig;
 				u_int sdtrable;
 
 				sdtr = adw_get_chip_sdtr(adw,
 							 ccb->ccb_h.target_id);
 				sdtr_orig = sdtr;
 				sdtrable = adw_lram_read_16(adw,
 							    ADW_MC_SDTR_ABLE);
 				sdtrable_orig = sdtrable;
 
 				if ((spi->valid
 				   & CTS_SPI_VALID_SYNC_RATE) != 0) {
 
 					sdtr =
 					    adw_find_sdtr(adw,
 							  spi->sync_period);
 				}
 					
 				if ((spi->valid
 				   & CTS_SPI_VALID_SYNC_OFFSET) != 0) {
 					if (spi->sync_offset == 0)
 						sdtr = ADW_MC_SDTR_ASYNC;
 				}
 
 				if (sdtr == ADW_MC_SDTR_ASYNC)
 					sdtrable &= ~target_mask;
 				else
 					sdtrable |= target_mask;
 				if (sdtr != sdtr_orig
 				 || sdtrable != sdtrable_orig) {
 					adw_set_chip_sdtr(adw,
 							  ccb->ccb_h.target_id,
 							  sdtr);
 					sdtrdone &= ~target_mask;
 					adw_lram_write_16(adw, ADW_MC_SDTR_ABLE,
 							  sdtrable);
 					adw_lram_write_16(adw, ADW_MC_SDTR_DONE,
 							  sdtrdone);
 					
 				}
 			} 
 		}
 		ccb->ccb_h.status = CAM_REQ_CMP;
 		xpt_done(ccb);
 		break;
 	}
 	case XPT_GET_TRAN_SETTINGS:
 	/* Get default/user set transfer settings for the target */
 	{
 		struct ccb_trans_settings_scsi *scsi;
 		struct ccb_trans_settings_spi *spi;
 		struct	ccb_trans_settings *cts;
 		u_int	target_mask;
  
 		cts = &ccb->cts;
 		target_mask = 0x01 << ccb->ccb_h.target_id;
 		cts->protocol = PROTO_SCSI;
 		cts->protocol_version = SCSI_REV_2;
 		cts->transport = XPORT_SPI;
 		cts->transport_version = 2;
 
 		scsi = &cts->proto_specific.scsi;
 		spi = &cts->xport_specific.spi;
 		if (cts->type == CTS_TYPE_CURRENT_SETTINGS) {
 			u_int mc_sdtr;
 
 			spi->flags = 0;
 			if ((adw->user_discenb & target_mask) != 0)
 				spi->flags |= CTS_SPI_FLAGS_DISC_ENB;
 
 			if ((adw->user_tagenb & target_mask) != 0)
 				scsi->flags |= CTS_SCSI_FLAGS_TAG_ENB;
 
 			if ((adw->user_wdtr & target_mask) != 0)
 				spi->bus_width = MSG_EXT_WDTR_BUS_16_BIT;
 			else
 				spi->bus_width = MSG_EXT_WDTR_BUS_8_BIT;
 
 			mc_sdtr = adw_get_user_sdtr(adw, ccb->ccb_h.target_id);
 			spi->sync_period = adw_find_period(adw, mc_sdtr);
 			if (spi->sync_period != 0)
 				spi->sync_offset = 15; /* XXX ??? */
 			else
 				spi->sync_offset = 0;
 
 
 		} else {
 			u_int targ_tinfo;
 
 			spi->flags = 0;
 			if ((adw_lram_read_16(adw, ADW_MC_DISC_ENABLE)
 			  & target_mask) != 0)
 				spi->flags |= CTS_SPI_FLAGS_DISC_ENB;
 
 			if ((adw->tagenb & target_mask) != 0)
 				scsi->flags |= CTS_SCSI_FLAGS_TAG_ENB;
 
 			targ_tinfo =
 			    adw_lram_read_16(adw,
 					     ADW_MC_DEVICE_HSHK_CFG_TABLE
 					     + (2 * ccb->ccb_h.target_id));
 
 			if ((targ_tinfo & ADW_HSHK_CFG_WIDE_XFR) != 0)
 				spi->bus_width = MSG_EXT_WDTR_BUS_16_BIT;
 			else
 				spi->bus_width = MSG_EXT_WDTR_BUS_8_BIT;
 
 			spi->sync_period =
 			    adw_hshk_cfg_period_factor(targ_tinfo);
 
 			spi->sync_offset = targ_tinfo & ADW_HSHK_CFG_OFFSET;
 			if (spi->sync_period == 0)
 				spi->sync_offset = 0;
 
 			if (spi->sync_offset == 0)
 				spi->sync_period = 0;
 		}
 
 		spi->valid = CTS_SPI_VALID_SYNC_RATE
 			   | CTS_SPI_VALID_SYNC_OFFSET
 			   | CTS_SPI_VALID_BUS_WIDTH
 			   | CTS_SPI_VALID_DISC;
 		scsi->valid = CTS_SCSI_VALID_TQ;
 		ccb->ccb_h.status = CAM_REQ_CMP;
 		xpt_done(ccb);
 		break;
 	}
 	case XPT_CALC_GEOMETRY:
 	{
 		/*
 		 * XXX Use Adaptec translation until I find out how to
 		 *     get this information from the card.
 		 */
 		cam_calc_geometry(&ccb->ccg, /*extended*/1); 
 		xpt_done(ccb);
 		break;
 	}
 	case XPT_RESET_BUS:		/* Reset the specified SCSI bus */
 	{
 		int failure;
 
 		failure = adw_reset_bus(adw);
 		if (failure != 0) {
 			ccb->ccb_h.status = CAM_REQ_CMP_ERR;
 		} else {
 			if (bootverbose) {
 				xpt_print_path(adw->path);
 				printf("Bus Reset Delivered\n");
 			}
 			ccb->ccb_h.status = CAM_REQ_CMP;
 		}
 		xpt_done(ccb);
 		break;
 	}
 	case XPT_TERM_IO:		/* Terminate the I/O process */
 		/* XXX Implement */
 		ccb->ccb_h.status = CAM_REQ_INVALID;
 		xpt_done(ccb);
 		break;
 	case XPT_PATH_INQ:		/* Path routing inquiry */
 	{
 		struct ccb_pathinq *cpi = &ccb->cpi;
 		
 		cpi->version_num = 1;
 		cpi->hba_inquiry = PI_WIDE_16|PI_SDTR_ABLE|PI_TAG_ABLE;
 		cpi->target_sprt = 0;
 		cpi->hba_misc = 0;
 		cpi->hba_eng_cnt = 0;
 		cpi->max_target = ADW_MAX_TID;
 		cpi->max_lun = ADW_MAX_LUN;
 		cpi->initiator_id = adw->initiator_id;
 		cpi->bus_id = cam_sim_bus(sim);
 		cpi->base_transfer_speed = 3300;
 		strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
 		strncpy(cpi->hba_vid, "AdvanSys", HBA_IDLEN);
 		strncpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
 		cpi->unit_number = cam_sim_unit(sim);
                 cpi->transport = XPORT_SPI;
                 cpi->transport_version = 2;
                 cpi->protocol = PROTO_SCSI;
                 cpi->protocol_version = SCSI_REV_2;
 		cpi->ccb_h.status = CAM_REQ_CMP;
 		xpt_done(ccb);
 		break;
 	}
 	default:
 		ccb->ccb_h.status = CAM_REQ_INVALID;
 		xpt_done(ccb);
 		break;
 	}
 }
 
 static void
 adw_poll(struct cam_sim *sim)
 {
 	adw_intr_locked(cam_sim_softc(sim));
 }
 
 static void
 adw_async(void *callback_arg, u_int32_t code, struct cam_path *path, void *arg)
 {
 }
 
 struct adw_softc *
 adw_alloc(device_t dev, struct resource *regs, int regs_type, int regs_id)
 {
 	struct	 adw_softc *adw;
 
 	adw = device_get_softc(dev);
 	LIST_INIT(&adw->pending_ccbs);
 	SLIST_INIT(&adw->sg_maps);
 	mtx_init(&adw->lock, "adw", NULL, MTX_DEF);
 	adw->device = dev;
 	adw->regs_res_type = regs_type;
 	adw->regs_res_id = regs_id;
 	adw->regs = regs;
 	return(adw);
 }
 
 void
 adw_free(struct adw_softc *adw)
 {
 	switch (adw->init_level) {
 	case 9:
 	{
 		struct sg_map_node *sg_map;
 
 		while ((sg_map = SLIST_FIRST(&adw->sg_maps)) != NULL) {
 			SLIST_REMOVE_HEAD(&adw->sg_maps, links);
 			bus_dmamap_unload(adw->sg_dmat,
 					  sg_map->sg_dmamap);
 			bus_dmamem_free(adw->sg_dmat, sg_map->sg_vaddr,
 					sg_map->sg_dmamap);
 			free(sg_map, M_DEVBUF);
 		}
 		bus_dma_tag_destroy(adw->sg_dmat);
 	}
 	case 8:
 		bus_dmamap_unload(adw->acb_dmat, adw->acb_dmamap);
 	case 7:
 		bus_dmamem_free(adw->acb_dmat, adw->acbs,
 				adw->acb_dmamap);
-		bus_dmamap_destroy(adw->acb_dmat, adw->acb_dmamap);
 	case 6:
 		bus_dma_tag_destroy(adw->acb_dmat);
 	case 5:
 		bus_dmamap_unload(adw->carrier_dmat, adw->carrier_dmamap);
 	case 4:
 		bus_dmamem_free(adw->carrier_dmat, adw->carriers,
 				adw->carrier_dmamap);
-		bus_dmamap_destroy(adw->carrier_dmat, adw->carrier_dmamap);
 	case 3:
 		bus_dma_tag_destroy(adw->carrier_dmat);
 	case 2:
 		bus_dma_tag_destroy(adw->buffer_dmat);
 	case 1:
 		bus_dma_tag_destroy(adw->parent_dmat);
 	case 0:
 		break;
 	}
 	
 	if (adw->regs != NULL)
 		bus_release_resource(adw->device,
 				     adw->regs_res_type,
 				     adw->regs_res_id,
 				     adw->regs);
 
 	if (adw->irq != NULL)
 		bus_release_resource(adw->device,
 				     adw->irq_res_type,
 				     0, adw->irq);
 
 	if (adw->sim != NULL) {
 		if (adw->path != NULL) {
 			xpt_async(AC_LOST_DEVICE, adw->path, NULL);
 			xpt_free_path(adw->path);
 		}
 		xpt_bus_deregister(cam_sim_path(adw->sim));
 		cam_sim_free(adw->sim, /*free_devq*/TRUE);
 	}
 	mtx_destroy(&adw->lock);
 }
 
 int
 adw_init(struct adw_softc *adw)
 {
 	struct	  adw_eeprom eep_config;
 	u_int	  tid;
 	u_int	  i;
 	u_int16_t checksum;
 	u_int16_t scsicfg1;
 
 	checksum = adw_eeprom_read(adw, &eep_config);
 	bcopy(eep_config.serial_number, adw->serial_number,
 	      sizeof(adw->serial_number));
 	if (checksum != eep_config.checksum) {
 		u_int16_t serial_number[3];
 
 		adw->flags |= ADW_EEPROM_FAILED;
 		device_printf(adw->device,
 		    "EEPROM checksum failed.  Restoring Defaults\n");
 
 	        /*
 		 * Restore the default EEPROM settings.
 		 * Assume the 6 byte board serial number that was read
 		 * from EEPROM is correct even if the EEPROM checksum
 		 * failed.
 		 */
 		bcopy(adw->default_eeprom, &eep_config, sizeof(eep_config));
 		bcopy(adw->serial_number, eep_config.serial_number,
 		      sizeof(serial_number));
 		adw_eeprom_write(adw, &eep_config);
 	}
 
 	/* Pull eeprom information into our softc. */
 	adw->bios_ctrl = eep_config.bios_ctrl;
 	adw->user_wdtr = eep_config.wdtr_able;
 	for (tid = 0; tid < ADW_MAX_TID; tid++) {
 		u_int	  mc_sdtr;
 		u_int16_t tid_mask;
 
 		tid_mask = 0x1 << tid;
 		if ((adw->features & ADW_ULTRA) != 0) {
 			/*
 			 * Ultra chips store sdtr and ultraenb
 			 * bits in their seeprom, so we must
 			 * construct valid mc_sdtr entries for
 			 * indirectly.
 			 */
 			if (eep_config.sync1.sync_enable & tid_mask) {
 				if (eep_config.sync2.ultra_enable & tid_mask)
 					mc_sdtr = ADW_MC_SDTR_20;
 				else
 					mc_sdtr = ADW_MC_SDTR_10;
 			} else
 				mc_sdtr = ADW_MC_SDTR_ASYNC;
 		} else {
 			switch (ADW_TARGET_GROUP(tid)) {
 			case 3:
 				mc_sdtr = eep_config.sync4.sdtr4;
 				break;
 			case 2:
 				mc_sdtr = eep_config.sync3.sdtr3;
 				break;
 			case 1:
 				mc_sdtr = eep_config.sync2.sdtr2;
 				break;
 			default: /* Shut up compiler */
 			case 0:
 				mc_sdtr = eep_config.sync1.sdtr1;
 				break;
 			}
 			mc_sdtr >>= ADW_TARGET_GROUP_SHIFT(tid);
 			mc_sdtr &= 0xFF;
 		}
 		adw_set_user_sdtr(adw, tid, mc_sdtr);
 	}
 	adw->user_tagenb = eep_config.tagqng_able;
 	adw->user_discenb = eep_config.disc_enable;
 	adw->max_acbs = eep_config.max_host_qng;
 	adw->initiator_id = (eep_config.adapter_scsi_id & ADW_MAX_TID);
 
 	/*
 	 * Sanity check the number of host openings.
 	 */
 	if (adw->max_acbs > ADW_DEF_MAX_HOST_QNG)
 		adw->max_acbs = ADW_DEF_MAX_HOST_QNG;
 	else if (adw->max_acbs < ADW_DEF_MIN_HOST_QNG) {
         	/* If the value is zero, assume it is uninitialized. */
 		if (adw->max_acbs == 0)
 			adw->max_acbs = ADW_DEF_MAX_HOST_QNG;
 		else
 			adw->max_acbs = ADW_DEF_MIN_HOST_QNG;
 	}
 	
 	scsicfg1 = 0;
 	if ((adw->features & ADW_ULTRA2) != 0) {
 		switch (eep_config.termination_lvd) {
 		default:
 			device_printf(adw->device,
 			    "Invalid EEPROM LVD Termination Settings.\n");
 			device_printf(adw->device,
 			    "Reverting to Automatic LVD Termination\n");
 			/* FALLTHROUGH */
 		case ADW_EEPROM_TERM_AUTO:
 			break;
 		case ADW_EEPROM_TERM_BOTH_ON:
 			scsicfg1 |= ADW2_SCSI_CFG1_TERM_LVD_LO;
 			/* FALLTHROUGH */
 		case ADW_EEPROM_TERM_HIGH_ON:
 			scsicfg1 |= ADW2_SCSI_CFG1_TERM_LVD_HI;
 			/* FALLTHROUGH */
 		case ADW_EEPROM_TERM_OFF:
 			scsicfg1 |= ADW2_SCSI_CFG1_DIS_TERM_DRV;
 			break;
 		}
 	}
 
 	switch (eep_config.termination_se) {
 	default:
 		device_printf(adw->device,
 		    "Invalid SE EEPROM Termination Settings.\n");
 		device_printf(adw->device,
 		    "Reverting to Automatic SE Termination\n");
 		/* FALLTHROUGH */
 	case ADW_EEPROM_TERM_AUTO:
 		break;
 	case ADW_EEPROM_TERM_BOTH_ON:
 		scsicfg1 |= ADW_SCSI_CFG1_TERM_CTL_L;
 		/* FALLTHROUGH */
 	case ADW_EEPROM_TERM_HIGH_ON:
 		scsicfg1 |= ADW_SCSI_CFG1_TERM_CTL_H;
 		/* FALLTHROUGH */
 	case ADW_EEPROM_TERM_OFF:
 		scsicfg1 |= ADW_SCSI_CFG1_TERM_CTL_MANUAL;
 		break;
 	}
 	device_printf(adw->device, "SCSI ID %d, ", adw->initiator_id);
 
 	/* DMA tag for mapping buffers into device visible space. */
 	if (bus_dma_tag_create(
 			/* parent	*/ adw->parent_dmat,
 			/* alignment	*/ 1,
 			/* boundary	*/ 0,
 			/* lowaddr	*/ BUS_SPACE_MAXADDR_32BIT,
 			/* highaddr	*/ BUS_SPACE_MAXADDR,
 			/* filter	*/ NULL,
 			/* filterarg	*/ NULL,
 			/* maxsize	*/ MAXBSIZE,
 			/* nsegments	*/ ADW_SGSIZE,
 			/* maxsegsz	*/ BUS_SPACE_MAXSIZE_32BIT,
 			/* flags	*/ BUS_DMA_ALLOCNOW,
 			/* lockfunc	*/ busdma_lock_mutex,
 			/* lockarg	*/ &adw->lock,
 			&adw->buffer_dmat) != 0) {
 		return (ENOMEM);
 	}
 
 	adw->init_level++;
 
 	/* DMA tag for our ccb carrier structures */
 	if (bus_dma_tag_create(
 			/* parent	*/ adw->parent_dmat,
 			/* alignment	*/ 0x10,
 			/* boundary	*/ 0,
 			/* lowaddr	*/ BUS_SPACE_MAXADDR_32BIT,
 			/* highaddr	*/ BUS_SPACE_MAXADDR,
 			/* filter	*/ NULL,
 			/* filterarg	*/ NULL,
 			/* maxsize	*/ (adw->max_acbs +
 					    ADW_NUM_CARRIER_QUEUES + 1) *
 					    sizeof(struct adw_carrier),
 			/* nsegments	*/ 1,
 			/* maxsegsz	*/ BUS_SPACE_MAXSIZE_32BIT,
 			/* flags	*/ 0,
 			/* lockfunc	*/ NULL,
 			/* lockarg	*/ NULL,
 			&adw->carrier_dmat) != 0) {
 		return (ENOMEM);
         }
 
 	adw->init_level++;
 
 	/* Allocation for our ccb carrier structures */
 	if (bus_dmamem_alloc(adw->carrier_dmat, (void **)&adw->carriers,
 			     BUS_DMA_NOWAIT, &adw->carrier_dmamap) != 0) {
 		return (ENOMEM);
 	}
 
 	adw->init_level++;
 
 	/* And permanently map them */
 	bus_dmamap_load(adw->carrier_dmat, adw->carrier_dmamap,
 			adw->carriers,
 			(adw->max_acbs + ADW_NUM_CARRIER_QUEUES + 1)
 			 * sizeof(struct adw_carrier),
 			adwmapmem, &adw->carrier_busbase, /*flags*/0);
 
 	/* Clear them out. */
 	bzero(adw->carriers, (adw->max_acbs + ADW_NUM_CARRIER_QUEUES + 1)
 			     * sizeof(struct adw_carrier));
 
 	/* Setup our free carrier list */
 	adw->free_carriers = adw->carriers;
 	for (i = 0; i < adw->max_acbs + ADW_NUM_CARRIER_QUEUES; i++) {
 		adw->carriers[i].carr_offset =
 			carriervtobo(adw, &adw->carriers[i]);
 		adw->carriers[i].carr_ba = 
 			carriervtob(adw, &adw->carriers[i]);
 		adw->carriers[i].areq_ba = 0;
 		adw->carriers[i].next_ba = 
 			carriervtobo(adw, &adw->carriers[i+1]);
 	}
 	/* Terminal carrier.  Never leaves the freelist */
 	adw->carriers[i].carr_offset =
 		carriervtobo(adw, &adw->carriers[i]);
 	adw->carriers[i].carr_ba = 
 		carriervtob(adw, &adw->carriers[i]);
 	adw->carriers[i].areq_ba = 0;
 	adw->carriers[i].next_ba = ~0;
 
 	adw->init_level++;
 
 	/* DMA tag for our acb structures */
 	if (bus_dma_tag_create(
 			/* parent	*/ adw->parent_dmat,
 			/* alignment	*/ 1,
 			/* boundary	*/ 0,
 			/* lowaddr	*/ BUS_SPACE_MAXADDR,
 			/* highaddr	*/ BUS_SPACE_MAXADDR,
 			/* filter	*/ NULL,
 			/* filterarg	*/ NULL,
 			/* maxsize	*/ adw->max_acbs * sizeof(struct acb),
 			/* nsegments	*/ 1,
 			/* maxsegsz	*/ BUS_SPACE_MAXSIZE_32BIT,
 			/* flags	*/ 0,
 			/* lockfunc	*/ NULL,
 			/* lockarg	*/ NULL,
 			&adw->acb_dmat) != 0) {
 		return (ENOMEM);
         }
 
 	adw->init_level++;
 
 	/* Allocation for our ccbs */
 	if (bus_dmamem_alloc(adw->acb_dmat, (void **)&adw->acbs,
 			     BUS_DMA_NOWAIT, &adw->acb_dmamap) != 0)
 		return (ENOMEM);
 
 	adw->init_level++;
 
 	/* And permanently map them */
 	bus_dmamap_load(adw->acb_dmat, adw->acb_dmamap,
 			adw->acbs,
 			adw->max_acbs * sizeof(struct acb),
 			adwmapmem, &adw->acb_busbase, /*flags*/0);
 
 	/* Clear them out. */
 	bzero(adw->acbs, adw->max_acbs * sizeof(struct acb)); 
 
 	/* DMA tag for our S/G structures.  We allocate in page sized chunks */
 	if (bus_dma_tag_create(
 			/* parent	*/ adw->parent_dmat,
 			/* alignment	*/ 1,
 			/* boundary	*/ 0,
 			/* lowaddr	*/ BUS_SPACE_MAXADDR,
 			/* highaddr	*/ BUS_SPACE_MAXADDR,
 			/* filter	*/ NULL,
 			/* filterarg	*/ NULL,
 			/* maxsize	*/ PAGE_SIZE,
 			/* nsegments	*/ 1,
 			/* maxsegsz	*/ BUS_SPACE_MAXSIZE_32BIT,
 			/* flags	*/ 0,
 			/* lockfunc	*/ NULL,
 			/* lockarg	*/ NULL,
 			&adw->sg_dmat) != 0) {
 		return (ENOMEM);
         }
 
 	adw->init_level++;
 
 	/* Allocate our first batch of ccbs */
 	mtx_lock(&adw->lock);
 	if (adwallocacbs(adw) == 0) {
 		mtx_unlock(&adw->lock);
 		return (ENOMEM);
 	}
 
 	if (adw_init_chip(adw, scsicfg1) != 0) {
 		mtx_unlock(&adw->lock);
 		return (ENXIO);
 	}
 
 	printf("Queue Depth %d\n", adw->max_acbs);
 	mtx_unlock(&adw->lock);
 
 	return (0);
 }
 
 /*
  * Attach all the sub-devices we can find
  */
 int
 adw_attach(struct adw_softc *adw)
 {
 	struct ccb_setasync csa;
 	struct cam_devq *devq;
 	int error;
 
 	/* Hook up our interrupt handler */
 	error = bus_setup_intr(adw->device, adw->irq,
 	    INTR_TYPE_CAM | INTR_ENTROPY | INTR_MPSAFE, NULL, adw_intr, adw,
 	    &adw->ih);
 	if (error != 0) {				    
 		device_printf(adw->device, "bus_setup_intr() failed: %d\n",
 			      error);
 		return (error);
 	}
 
 	/* Start the Risc processor now that we are fully configured. */
 	adw_outw(adw, ADW_RISC_CSR, ADW_RISC_CSR_RUN);
 
 	/*
 	 * Create the device queue for our SIM.
 	 */
 	devq = cam_simq_alloc(adw->max_acbs);
 	if (devq == NULL)
 		return (ENOMEM);
 
 	/*
 	 * Construct our SIM entry.
 	 */
 	adw->sim = cam_sim_alloc(adw_action, adw_poll, "adw", adw,
 	    device_get_unit(adw->device), &adw->lock, 1, adw->max_acbs, devq);
 	if (adw->sim == NULL)
 		return (ENOMEM);
 
 	/*
 	 * Register the bus.
 	 */
 	mtx_lock(&adw->lock);
 	if (xpt_bus_register(adw->sim, adw->device, 0) != CAM_SUCCESS) {
 		cam_sim_free(adw->sim, /*free devq*/TRUE);
 		error = ENOMEM;
 		goto fail;
 	}
 
 	if (xpt_create_path(&adw->path, /*periph*/NULL, cam_sim_path(adw->sim),
 			    CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD)
 	   == CAM_REQ_CMP) {
 		xpt_setup_ccb(&csa.ccb_h, adw->path, /*priority*/5);
 		csa.ccb_h.func_code = XPT_SASYNC_CB;
 		csa.event_enable = AC_LOST_DEVICE;
 		csa.callback = adw_async;
 		csa.callback_arg = adw;
 		xpt_action((union ccb *)&csa);
 	}
 
 fail:
 	mtx_unlock(&adw->lock);
 	return (error);
 }
 
 void
 adw_intr(void *arg)
 {
 	struct	adw_softc *adw;
 
 	adw = arg;
 	mtx_lock(&adw->lock);
 	adw_intr_locked(adw);
 	mtx_unlock(&adw->lock);
 }
 
 void
 adw_intr_locked(struct adw_softc *adw)
 {
 	u_int	int_stat;
 	
 	if ((adw_inw(adw, ADW_CTRL_REG) & ADW_CTRL_REG_HOST_INTR) == 0)
 		return;
 
 	/* Reading the register clears the interrupt. */
 	int_stat = adw_inb(adw, ADW_INTR_STATUS_REG);
 
 	if ((int_stat & ADW_INTR_STATUS_INTRB) != 0) {
 		u_int intrb_code;
 
 		/* Async Microcode Event */
 		intrb_code = adw_lram_read_8(adw, ADW_MC_INTRB_CODE);
 		switch (intrb_code) {
 		case ADW_ASYNC_CARRIER_READY_FAILURE:
 			/*
 			 * The RISC missed our update of
 			 * the commandq.
 			 */
 			if (LIST_FIRST(&adw->pending_ccbs) != NULL)
 				adw_tickle_risc(adw, ADW_TICKLE_A);
 			break;
     		case ADW_ASYNC_SCSI_BUS_RESET_DET:
 			/*
 			 * The firmware detected a SCSI Bus reset.
 			 */
 			device_printf(adw->device, "Someone Reset the Bus\n");
 			adw_handle_bus_reset(adw, /*initiated*/FALSE);
 			break;
 		case ADW_ASYNC_RDMA_FAILURE:
 			/*
 			 * Handle RDMA failure by resetting the
 			 * SCSI Bus and chip.
 			 */
 #if 0 /* XXX */
 			AdvResetChipAndSB(adv_dvc_varp);
 #endif
 			break;
 
 		case ADW_ASYNC_HOST_SCSI_BUS_RESET:
 			/*
 			 * Host generated SCSI bus reset occurred.
 			 */
 			adw_handle_bus_reset(adw, /*initiated*/TRUE);
         		break;
     		default:
 			printf("adw_intr: unknown async code 0x%x\n",
 			       intrb_code);
 			break;
 		}
 	}
 
 	/*
 	 * Run down the RequestQ.
 	 */
 	while ((adw->responseq->next_ba & ADW_RQ_DONE) != 0) {
 		struct adw_carrier *free_carrier;
 		struct acb *acb;
 		union ccb *ccb;
 
 #if 0
 		printf("0x%x, 0x%x, 0x%x, 0x%x\n",
 		       adw->responseq->carr_offset,
 		       adw->responseq->carr_ba,
 		       adw->responseq->areq_ba,
 		       adw->responseq->next_ba);
 #endif
 		/*
 		 * The firmware copies the adw_scsi_req_q.acb_baddr
 		 * field into the areq_ba field of the carrier.
 		 */
 		acb = acbbotov(adw, adw->responseq->areq_ba);
 
 		/*
 		 * The least significant four bits of the next_ba
 		 * field are used as flags.  Mask them out and then
 		 * advance through the list.
 		 */
 		free_carrier = adw->responseq;
 		adw->responseq =
 		    carrierbotov(adw, free_carrier->next_ba & ADW_NEXT_BA_MASK);
 		free_carrier->next_ba = adw->free_carriers->carr_offset;
 		adw->free_carriers = free_carrier;
 
 		/* Process CCB */
 		ccb = acb->ccb;
 		callout_stop(&acb->timer);
 		if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE) {
 			bus_dmasync_op_t op;
 
 			if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN)
 				op = BUS_DMASYNC_POSTREAD;
 			else
 				op = BUS_DMASYNC_POSTWRITE;
 			bus_dmamap_sync(adw->buffer_dmat, acb->dmamap, op);
 			bus_dmamap_unload(adw->buffer_dmat, acb->dmamap);
 			ccb->csio.resid = acb->queue.data_cnt;
 		} else 
 			ccb->csio.resid = 0;
 
 		/* Common Cases inline... */
 		if (acb->queue.host_status == QHSTA_NO_ERROR
 		 && (acb->queue.done_status == QD_NO_ERROR
 		  || acb->queue.done_status == QD_WITH_ERROR)) {
 			ccb->csio.scsi_status = acb->queue.scsi_status;
 			ccb->ccb_h.status = 0;
 			switch (ccb->csio.scsi_status) {
 			case SCSI_STATUS_OK:
 				ccb->ccb_h.status |= CAM_REQ_CMP;
 				break;
 			case SCSI_STATUS_CHECK_COND:
 			case SCSI_STATUS_CMD_TERMINATED:
 				bcopy(&acb->sense_data, &ccb->csio.sense_data,
 				      ccb->csio.sense_len);
 				ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
 				ccb->csio.sense_resid = acb->queue.sense_len;
 				/* FALLTHROUGH */
 			default:
 				ccb->ccb_h.status |= CAM_SCSI_STATUS_ERROR
 						  |  CAM_DEV_QFRZN;
 				xpt_freeze_devq(ccb->ccb_h.path, /*count*/1);
 				break;
 			}
 			adwfreeacb(adw, acb);
 			xpt_done(ccb);
 		} else {
 			adwprocesserror(adw, acb);
 		}
 	}
 }
 
 static void
 adwprocesserror(struct adw_softc *adw, struct acb *acb)
 {
 	union ccb *ccb;
 
 	ccb = acb->ccb;
 	if (acb->queue.done_status == QD_ABORTED_BY_HOST) {
 		ccb->ccb_h.status = CAM_REQ_ABORTED;
 	} else {
 
 		switch (acb->queue.host_status) {
 		case QHSTA_M_SEL_TIMEOUT:
 			ccb->ccb_h.status = CAM_SEL_TIMEOUT;
 			break;
 		case QHSTA_M_SXFR_OFF_UFLW:
 		case QHSTA_M_SXFR_OFF_OFLW:
 		case QHSTA_M_DATA_OVER_RUN:
 			ccb->ccb_h.status = CAM_DATA_RUN_ERR;
 			break;
 		case QHSTA_M_SXFR_DESELECTED:
 		case QHSTA_M_UNEXPECTED_BUS_FREE:
 			ccb->ccb_h.status = CAM_UNEXP_BUSFREE;
 			break;
 		case QHSTA_M_SCSI_BUS_RESET:
 		case QHSTA_M_SCSI_BUS_RESET_UNSOL:
 			ccb->ccb_h.status = CAM_SCSI_BUS_RESET;
 			break;
 		case QHSTA_M_BUS_DEVICE_RESET:
 			ccb->ccb_h.status = CAM_BDR_SENT;
 			break;
 		case QHSTA_M_QUEUE_ABORTED:
 			/* BDR or Bus Reset */
 			xpt_print_path(adw->path);
 			printf("Saw Queue Aborted\n");
 			ccb->ccb_h.status = adw->last_reset;
 			break;
 		case QHSTA_M_SXFR_SDMA_ERR:
 		case QHSTA_M_SXFR_SXFR_PERR:
 		case QHSTA_M_RDMA_PERR:
 			ccb->ccb_h.status = CAM_UNCOR_PARITY;
 			break;
 		case QHSTA_M_WTM_TIMEOUT:
 		case QHSTA_M_SXFR_WD_TMO:
 		{
 			/* The SCSI bus hung in a phase */
 			xpt_print_path(adw->path);
 			printf("Watch Dog timer expired.  Resetting bus\n");
 			adw_reset_bus(adw);
 			break;
 		}
 		case QHSTA_M_SXFR_XFR_PH_ERR:
 			ccb->ccb_h.status = CAM_SEQUENCE_FAIL;
 			break;
 		case QHSTA_M_SXFR_UNKNOWN_ERROR:
 			break;
 		case QHSTA_M_BAD_CMPL_STATUS_IN:
 			/* No command complete after a status message */
 			ccb->ccb_h.status = CAM_SEQUENCE_FAIL;
 			break;
 		case QHSTA_M_AUTO_REQ_SENSE_FAIL:
 			ccb->ccb_h.status = CAM_AUTOSENSE_FAIL;
 			break;
 		case QHSTA_M_INVALID_DEVICE:
 			ccb->ccb_h.status = CAM_PATH_INVALID;
 			break;
 		case QHSTA_M_NO_AUTO_REQ_SENSE:
 			/*
 			 * User didn't request sense, but we got a
 			 * check condition.
 			 */
 			ccb->csio.scsi_status = acb->queue.scsi_status;
 			ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR;
 			break;
 		default:
 			panic("%s: Unhandled Host status error %x",
 			    device_get_nameunit(adw->device),
 			    acb->queue.host_status);
 			/* NOTREACHED */
 		}
 	}
 	if ((acb->state & ACB_RECOVERY_ACB) != 0) {
 		if (ccb->ccb_h.status == CAM_SCSI_BUS_RESET
 		 || ccb->ccb_h.status == CAM_BDR_SENT)
 		 	ccb->ccb_h.status = CAM_CMD_TIMEOUT;
 	}
 	if (ccb->ccb_h.status != CAM_REQ_CMP) {
 		xpt_freeze_devq(ccb->ccb_h.path, /*count*/1);
 		ccb->ccb_h.status |= CAM_DEV_QFRZN;
 	}
 	adwfreeacb(adw, acb);
 	xpt_done(ccb);
 }
 
 static void
 adwtimeout(void *arg)
 {
 	struct acb	     *acb;
 	union  ccb	     *ccb;
 	struct adw_softc     *adw;
 	adw_idle_cmd_status_t status;
 	int		      target_id;
 
 	acb = (struct acb *)arg;
 	ccb = acb->ccb;
 	adw = (struct adw_softc *)ccb->ccb_h.ccb_adw_ptr;
 	xpt_print_path(ccb->ccb_h.path);
 	printf("ACB %p - timed out\n", (void *)acb);
 
 	mtx_assert(&adw->lock, MA_OWNED);
 
 	if ((acb->state & ACB_ACTIVE) == 0) {
 		xpt_print_path(ccb->ccb_h.path);
 		printf("ACB %p - timed out CCB already completed\n",
 		       (void *)acb);
 		return;
 	}
 
 	acb->state |= ACB_RECOVERY_ACB;
 	target_id = ccb->ccb_h.target_id;
 
 	/* Attempt a BDR first */
 	status = adw_idle_cmd_send(adw, ADW_IDLE_CMD_DEVICE_RESET,
 				   ccb->ccb_h.target_id);
 	if (status == ADW_IDLE_CMD_SUCCESS) {
 		device_printf(adw->device,
 		    "BDR Delivered.  No longer in timeout\n");
 		adw_handle_device_reset(adw, target_id);
 	} else {
 		adw_reset_bus(adw);
 		xpt_print_path(adw->path);
 		printf("Bus Reset Delivered.  No longer in timeout\n");
 	}
 }
 
 static void
 adw_handle_device_reset(struct adw_softc *adw, u_int target)
 {
 	struct cam_path *path;
 	cam_status error;
 
 	error = xpt_create_path(&path, /*periph*/NULL, cam_sim_path(adw->sim),
 				target, CAM_LUN_WILDCARD);
 
 	if (error == CAM_REQ_CMP) {
 		xpt_async(AC_SENT_BDR, path, NULL);
 		xpt_free_path(path);
 	}
 	adw->last_reset = CAM_BDR_SENT;
 }
 
 static void
 adw_handle_bus_reset(struct adw_softc *adw, int initiated)
 {
 	if (initiated) {
 		/*
 		 * The microcode currently sets the SCSI Bus Reset signal
 		 * while handling the AscSendIdleCmd() IDLE_CMD_SCSI_RESET
 		 * command above.  But the SCSI Bus Reset Hold Time in the
 		 * microcode is not deterministic (it may in fact be for less
 		 * than the SCSI Spec. minimum of 25 us).  Therefore on return
 		 * the Adv Library sets the SCSI Bus Reset signal for
 		 * ADW_SCSI_RESET_HOLD_TIME_US, which is defined to be greater
 		 * than 25 us.
 		 */
 		u_int scsi_ctrl;
 
 	    	scsi_ctrl = adw_inw(adw, ADW_SCSI_CTRL) & ~ADW_SCSI_CTRL_RSTOUT;
 		adw_outw(adw, ADW_SCSI_CTRL, scsi_ctrl | ADW_SCSI_CTRL_RSTOUT);
 		DELAY(ADW_SCSI_RESET_HOLD_TIME_US);
 		adw_outw(adw, ADW_SCSI_CTRL, scsi_ctrl);
 
 		/*
 		 * We will perform the async notification when the
 		 * SCSI Reset interrupt occurs.
 		 */
 	} else
 		xpt_async(AC_BUS_RESET, adw->path, NULL);
 	adw->last_reset = CAM_SCSI_BUS_RESET;
 }
 MODULE_DEPEND(adw, cam, 1, 1, 1);
 
Index: projects/release-debugdist/sys/dev/an/if_an.c
===================================================================
--- projects/release-debugdist/sys/dev/an/if_an.c	(revision 267343)
+++ projects/release-debugdist/sys/dev/an/if_an.c	(revision 267344)
@@ -1,3833 +1,3825 @@
 /*-
  * Copyright (c) 1997, 1998, 1999
  *	Bill Paul <wpaul@ctr.columbia.edu>.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  * 3. All advertising materials mentioning features or use of this software
  *    must display the following acknowledgement:
  *	This product includes software developed by Bill Paul.
  * 4. Neither the name of the author nor the names of any co-contributors
  *    may be used to endorse or promote products derived from this software
  *    without specific prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD
  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  * THE POSSIBILITY OF SUCH DAMAGE.
  */
 /*
  * Aironet 4500/4800 802.11 PCMCIA/ISA/PCI driver for FreeBSD.
  *
  * Written by Bill Paul <wpaul@ctr.columbia.edu>
  * Electrical Engineering Department
  * Columbia University, New York City
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 /*
  * The Aironet 4500/4800 series cards come in PCMCIA, ISA and PCI form.
  * This driver supports all three device types (PCI devices are supported
  * through an extra PCI shim: /sys/dev/an/if_an_pci.c). ISA devices can be
  * supported either using hard-coded IO port/IRQ settings or via Plug
  * and Play. The 4500 series devices support 1Mbps and 2Mbps data rates.
  * The 4800 devices support 1, 2, 5.5 and 11Mbps rates.
  *
  * Like the WaveLAN/IEEE cards, the Aironet NICs are all essentially
  * PCMCIA devices. The ISA and PCI cards are a combination of a PCMCIA
  * device and a PCMCIA to ISA or PCMCIA to PCI adapter card. There are
  * a couple of important differences though:
  *
  * - Lucent ISA card looks to the host like a PCMCIA controller with
  *   a PCMCIA WaveLAN card inserted. This means that even desktop
  *   machines need to be configured with PCMCIA support in order to
  *   use WaveLAN/IEEE ISA cards. The Aironet cards on the other hand
  *   actually look like normal ISA and PCI devices to the host, so
  *   no PCMCIA controller support is needed
  *
  * The latter point results in a small gotcha. The Aironet PCMCIA
  * cards can be configured for one of two operating modes depending
  * on how the Vpp1 and Vpp2 programming voltages are set when the
  * card is activated. In order to put the card in proper PCMCIA
  * operation (where the CIS table is visible and the interface is
  * programmed for PCMCIA operation), both Vpp1 and Vpp2 have to be
  * set to 5 volts. FreeBSD by default doesn't set the Vpp voltages,
  * which leaves the card in ISA/PCI mode, which prevents it from
  * being activated as an PCMCIA device.
  *
  * Note that some PCMCIA controller software packages for Windows NT
  * fail to set the voltages as well.
  *
  * The Aironet devices can operate in both station mode and access point
  * mode. Typically, when programmed for station mode, the card can be set
  * to automatically perform encapsulation/decapsulation of Ethernet II
  * and 802.3 frames within 802.11 frames so that the host doesn't have
  * to do it itself. This driver doesn't program the card that way: the
  * driver handles all of the encapsulation/decapsulation itself.
  */
 
 #include "opt_inet.h"
 
 #ifdef INET
 #define ANCACHE			/* enable signal strength cache */
 #endif
 
 #include <sys/param.h>
 #include <sys/ctype.h>
 #include <sys/systm.h>
 #include <sys/sockio.h>
 #include <sys/mbuf.h>
 #include <sys/priv.h>
 #include <sys/proc.h>
 #include <sys/kernel.h>
 #include <sys/socket.h>
 #ifdef ANCACHE
 #include <sys/syslog.h>
 #endif
 #include <sys/sysctl.h>
 
 #include <sys/module.h>
 #include <sys/bus.h>
 #include <machine/bus.h>
 #include <sys/rman.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <machine/resource.h>
 #include <sys/malloc.h>
 
 #include <net/if.h>
 #include <net/if_var.h>
 #include <net/if_arp.h>
 #include <net/if_dl.h>
 #include <net/ethernet.h>
 #include <net/if_types.h>
 #include <net/if_media.h>
 
 #include <net80211/ieee80211_var.h>
 #include <net80211/ieee80211_ioctl.h>
 
 #ifdef INET
 #include <netinet/in.h>
 #include <netinet/in_systm.h>
 #include <netinet/in_var.h>
 #include <netinet/ip.h>
 #endif
 
 #include <net/bpf.h>
 
 #include <machine/md_var.h>
 
 #include <dev/an/if_aironet_ieee.h>
 #include <dev/an/if_anreg.h>
 
 /* These are global because we need them in sys/pci/if_an_p.c. */
 static void an_reset(struct an_softc *);
 static int an_init_mpi350_desc(struct an_softc *);
 static int an_ioctl(struct ifnet *, u_long, caddr_t);
 static void an_init(void *);
 static void an_init_locked(struct an_softc *);
 static int an_init_tx_ring(struct an_softc *);
 static void an_start(struct ifnet *);
 static void an_start_locked(struct ifnet *);
 static void an_watchdog(struct an_softc *);
 static void an_rxeof(struct an_softc *);
 static void an_txeof(struct an_softc *, int);
 
 static void an_promisc(struct an_softc *, int);
 static int an_cmd(struct an_softc *, int, int);
 static int an_cmd_struct(struct an_softc *, struct an_command *,
     struct an_reply *);
 static int an_read_record(struct an_softc *, struct an_ltv_gen *);
 static int an_write_record(struct an_softc *, struct an_ltv_gen *);
 static int an_read_data(struct an_softc *, int, int, caddr_t, int);
 static int an_write_data(struct an_softc *, int, int, caddr_t, int);
 static int an_seek(struct an_softc *, int, int, int);
 static int an_alloc_nicmem(struct an_softc *, int, int *);
 static int an_dma_malloc(struct an_softc *, bus_size_t, struct an_dma_alloc *,
     int);
 static void an_dma_free(struct an_softc *, struct an_dma_alloc *);
 static void an_dma_malloc_cb(void *, bus_dma_segment_t *, int, int);
 static void an_stats_update(void *);
 static void an_setdef(struct an_softc *, struct an_req *);
 #ifdef ANCACHE
 static void an_cache_store(struct an_softc *, struct ether_header *,
     struct mbuf *, u_int8_t, u_int8_t);
 #endif
 
 /* function definitions for use with the Cisco's Linux configuration
    utilities
 */
 
 static int readrids(struct ifnet*, struct aironet_ioctl*);
 static int writerids(struct ifnet*, struct aironet_ioctl*);
 static int flashcard(struct ifnet*, struct aironet_ioctl*);
 
 static int cmdreset(struct ifnet *);
 static int setflashmode(struct ifnet *);
 static int flashgchar(struct ifnet *,int,int);
 static int flashpchar(struct ifnet *,int,int);
 static int flashputbuf(struct ifnet *);
 static int flashrestart(struct ifnet *);
 static int WaitBusy(struct ifnet *, int);
 static int unstickbusy(struct ifnet *);
 
 static void an_dump_record	(struct an_softc *,struct an_ltv_gen *,
 				    char *);
 
 static int an_media_change	(struct ifnet *);
 static void an_media_status	(struct ifnet *, struct ifmediareq *);
 
 static int	an_dump = 0;
 static int	an_cache_mode = 0;
 
 #define DBM 0
 #define PERCENT 1
 #define RAW 2
 
 static char an_conf[256];
 static char an_conf_cache[256];
 
 /* sysctl vars */
 
 static SYSCTL_NODE(_hw, OID_AUTO, an, CTLFLAG_RD, 0,
     "Wireless driver parameters");
 
 /* XXX violate ethernet/netgraph callback hooks */
 extern	void	(*ng_ether_attach_p)(struct ifnet *ifp);
 extern	void	(*ng_ether_detach_p)(struct ifnet *ifp);
 
 static int
 sysctl_an_dump(SYSCTL_HANDLER_ARGS)
 {
 	int	error, r, last;
 	char 	*s = an_conf;
 
 	last = an_dump;
 
 	switch (an_dump) {
 	case 0:
 		strcpy(an_conf, "off");
 		break;
 	case 1:
 		strcpy(an_conf, "type");
 		break;
 	case 2:
 		strcpy(an_conf, "dump");
 		break;
 	default:
 		snprintf(an_conf, 5, "%x", an_dump);
 		break;
 	}
 
 	error = sysctl_handle_string(oidp, an_conf, sizeof(an_conf), req);
 
 	if (strncmp(an_conf,"off", 3) == 0) {
 		an_dump = 0;
  	}
 	if (strncmp(an_conf,"dump", 4) == 0) {
 		an_dump = 1;
 	}
 	if (strncmp(an_conf,"type", 4) == 0) {
 		an_dump = 2;
 	}
 	if (*s == 'f') {
 		r = 0;
 		for (;;s++) {
 			if ((*s >= '0') && (*s <= '9')) {
 				r = r * 16 + (*s - '0');
 			} else if ((*s >= 'a') && (*s <= 'f')) {
 				r = r * 16 + (*s - 'a' + 10);
 			} else {
 				break;
 			}
 		}
 		an_dump = r;
 	}
 	if (an_dump != last)
 		printf("Sysctl changed for Aironet driver\n");
 
 	return error;
 }
 
 SYSCTL_PROC(_hw_an, OID_AUTO, an_dump, CTLTYPE_STRING | CTLFLAG_RW,
 	    0, sizeof(an_conf), sysctl_an_dump, "A", "");
 
 static int
 sysctl_an_cache_mode(SYSCTL_HANDLER_ARGS)
 {
 	int	error, last;
 
 	last = an_cache_mode;
 
 	switch (an_cache_mode) {
 	case 1:
 		strcpy(an_conf_cache, "per");
 		break;
 	case 2:
 		strcpy(an_conf_cache, "raw");
 		break;
 	default:
 		strcpy(an_conf_cache, "dbm");
 		break;
 	}
 
 	error = sysctl_handle_string(oidp, an_conf_cache,
 			sizeof(an_conf_cache), req);
 
 	if (strncmp(an_conf_cache,"dbm", 3) == 0) {
 		an_cache_mode = 0;
 	}
 	if (strncmp(an_conf_cache,"per", 3) == 0) {
 		an_cache_mode = 1;
  	}
 	if (strncmp(an_conf_cache,"raw", 3) == 0) {
 		an_cache_mode = 2;
 	}
 
 	return error;
 }
 
 SYSCTL_PROC(_hw_an, OID_AUTO, an_cache_mode, CTLTYPE_STRING | CTLFLAG_RW,
 	    0, sizeof(an_conf_cache), sysctl_an_cache_mode, "A", "");
 
 /*
  * Setup the lock for PCI attachment since it skips the an_probe
  * function.  We need to setup the lock in an_probe since some
  * operations need the lock.  So we might as well create the
  * lock in the probe.
  */
 int
 an_pci_probe(device_t dev)
 {
 	struct an_softc *sc = device_get_softc(dev);
 
 	mtx_init(&sc->an_mtx, device_get_nameunit(dev), MTX_NETWORK_LOCK,
 	    MTX_DEF);
 
 	return(0);
 }
 
 /*
  * We probe for an Aironet 4500/4800 card by attempting to
  * read the default SSID list. On reset, the first entry in
  * the SSID list will contain the name "tsunami." If we don't
  * find this, then there's no card present.
  */
 int
 an_probe(device_t dev)
 {
 	struct an_softc *sc = device_get_softc(dev);
 	struct an_ltv_ssidlist_new	ssid;
 	int	error;
 
 	bzero((char *)&ssid, sizeof(ssid));
 
 	error = an_alloc_port(dev, 0, AN_IOSIZ);
 	if (error != 0)
 		return (0);
 
 	/* can't do autoprobing */
 	if (rman_get_start(sc->port_res) == -1)
 		return(0);
 
 	/*
 	 * We need to fake up a softc structure long enough
 	 * to be able to issue commands and call some of the
 	 * other routines.
 	 */
 	ssid.an_len = sizeof(ssid);
 	ssid.an_type = AN_RID_SSIDLIST;
 
 	/* Make sure interrupts are disabled. */
 	sc->mpi350 = 0;
 	CSR_WRITE_2(sc, AN_INT_EN(sc->mpi350), 0);
 	CSR_WRITE_2(sc, AN_EVENT_ACK(sc->mpi350), 0xFFFF);
 
 	sc->an_dev = dev;
 	mtx_init(&sc->an_mtx, device_get_nameunit(dev), MTX_NETWORK_LOCK,
 	    MTX_DEF);
 	AN_LOCK(sc);
 	an_reset(sc);
 
 	if (an_cmd(sc, AN_CMD_READCFG, 0)) {
 		AN_UNLOCK(sc);
 		goto fail;
 	}
 
 	if (an_read_record(sc, (struct an_ltv_gen *)&ssid)) {
 		AN_UNLOCK(sc);
 		goto fail;
 	}
 
 	/* See if the ssid matches what we expect ... but doesn't have to */
 	if (strcmp(ssid.an_entry[0].an_ssid, AN_DEF_SSID)) {
 		AN_UNLOCK(sc);
 		goto fail;
 	}
 
 	AN_UNLOCK(sc);
 	return(AN_IOSIZ);
 fail:
 	mtx_destroy(&sc->an_mtx);
 	return(0);
 }
 
 /*
  * Allocate a port resource with the given resource id.
  */
 int
 an_alloc_port(device_t dev, int rid, int size)
 {
 	struct an_softc *sc = device_get_softc(dev);
 	struct resource *res;
 
 	res = bus_alloc_resource(dev, SYS_RES_IOPORT, &rid,
 				 0ul, ~0ul, size, RF_ACTIVE);
 	if (res) {
 		sc->port_rid = rid;
 		sc->port_res = res;
 		return (0);
 	} else {
 		return (ENOENT);
 	}
 }
 
 /*
  * Allocate a memory resource with the given resource id.
  */
 int an_alloc_memory(device_t dev, int rid, int size)
 {
 	struct an_softc *sc = device_get_softc(dev);
 	struct resource *res;
 
 	res = bus_alloc_resource(dev, SYS_RES_MEMORY, &rid,
 				 0ul, ~0ul, size, RF_ACTIVE);
 	if (res) {
 		sc->mem_rid = rid;
 		sc->mem_res = res;
 		sc->mem_used = size;
 		return (0);
 	} else {
 		return (ENOENT);
 	}
 }
 
 /*
  * Allocate a auxilary memory resource with the given resource id.
  */
 int an_alloc_aux_memory(device_t dev, int rid, int size)
 {
 	struct an_softc *sc = device_get_softc(dev);
 	struct resource *res;
 
 	res = bus_alloc_resource(dev, SYS_RES_MEMORY, &rid,
 				 0ul, ~0ul, size, RF_ACTIVE);
 	if (res) {
 		sc->mem_aux_rid = rid;
 		sc->mem_aux_res = res;
 		sc->mem_aux_used = size;
 		return (0);
 	} else {
 		return (ENOENT);
 	}
 }
 
 /*
  * Allocate an irq resource with the given resource id.
  */
 int
 an_alloc_irq(device_t dev, int rid, int flags)
 {
 	struct an_softc *sc = device_get_softc(dev);
 	struct resource *res;
 
 	res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
 				     (RF_ACTIVE | flags));
 	if (res) {
 		sc->irq_rid = rid;
 		sc->irq_res = res;
 		return (0);
 	} else {
 		return (ENOENT);
 	}
 }
 
 static void
 an_dma_malloc_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 {
 	bus_addr_t *paddr = (bus_addr_t*) arg;
 	*paddr = segs->ds_addr;
 }
 
 /*
  * Alloc DMA memory and set the pointer to it
  */
 static int
 an_dma_malloc(struct an_softc *sc, bus_size_t size, struct an_dma_alloc *dma,
     int mapflags)
 {
 	int r;
 
-	r = bus_dmamap_create(sc->an_dtag, BUS_DMA_NOWAIT, &dma->an_dma_map);
-	if (r != 0)
-		goto fail_0;
-
 	r = bus_dmamem_alloc(sc->an_dtag, (void**) &dma->an_dma_vaddr,
 			     BUS_DMA_NOWAIT, &dma->an_dma_map);
 	if (r != 0)
 		goto fail_1;
 
 	r = bus_dmamap_load(sc->an_dtag, dma->an_dma_map, dma->an_dma_vaddr,
 			    size,
 			    an_dma_malloc_cb,
 			    &dma->an_dma_paddr,
 			    mapflags | BUS_DMA_NOWAIT);
 	if (r != 0)
 		goto fail_2;
 
 	dma->an_dma_size = size;
 	return (0);
 
 fail_2:
 	bus_dmamap_unload(sc->an_dtag, dma->an_dma_map);
 fail_1:
 	bus_dmamem_free(sc->an_dtag, dma->an_dma_vaddr, dma->an_dma_map);
-fail_0:
-	bus_dmamap_destroy(sc->an_dtag, dma->an_dma_map);
-	dma->an_dma_map = NULL;
 	return (r);
 }
 
 static void
 an_dma_free(struct an_softc *sc, struct an_dma_alloc *dma)
 {
 	bus_dmamap_unload(sc->an_dtag, dma->an_dma_map);
 	bus_dmamem_free(sc->an_dtag, dma->an_dma_vaddr, dma->an_dma_map);
 	dma->an_dma_vaddr = 0;
-	bus_dmamap_destroy(sc->an_dtag, dma->an_dma_map);
 }
 
 /*
  * Release all resources
  */
 void
 an_release_resources(device_t dev)
 {
 	struct an_softc *sc = device_get_softc(dev);
 	int i;
 
 	if (sc->port_res) {
 		bus_release_resource(dev, SYS_RES_IOPORT,
 				     sc->port_rid, sc->port_res);
 		sc->port_res = 0;
 	}
 	if (sc->mem_res) {
 		bus_release_resource(dev, SYS_RES_MEMORY,
 				     sc->mem_rid, sc->mem_res);
 		sc->mem_res = 0;
 	}
 	if (sc->mem_aux_res) {
 		bus_release_resource(dev, SYS_RES_MEMORY,
 				     sc->mem_aux_rid, sc->mem_aux_res);
 		sc->mem_aux_res = 0;
 	}
 	if (sc->irq_res) {
 		bus_release_resource(dev, SYS_RES_IRQ,
 				     sc->irq_rid, sc->irq_res);
 		sc->irq_res = 0;
 	}
 	if (sc->an_rid_buffer.an_dma_paddr) {
 		an_dma_free(sc, &sc->an_rid_buffer);
 	}
 	for (i = 0; i < AN_MAX_RX_DESC; i++)
 		if (sc->an_rx_buffer[i].an_dma_paddr) {
 			an_dma_free(sc, &sc->an_rx_buffer[i]);
 		}
 	for (i = 0; i < AN_MAX_TX_DESC; i++)
 		if (sc->an_tx_buffer[i].an_dma_paddr) {
 			an_dma_free(sc, &sc->an_tx_buffer[i]);
 		}
 	if (sc->an_dtag) {
 		bus_dma_tag_destroy(sc->an_dtag);
 	}
 
 }
 
 int
 an_init_mpi350_desc(struct an_softc *sc)
 {
 	struct an_command	cmd_struct;
 	struct an_reply		reply;
 	struct an_card_rid_desc an_rid_desc;
 	struct an_card_rx_desc	an_rx_desc;
 	struct an_card_tx_desc	an_tx_desc;
 	int			i, desc;
 
 	AN_LOCK_ASSERT(sc);
 	if(!sc->an_rid_buffer.an_dma_paddr)
 		an_dma_malloc(sc, AN_RID_BUFFER_SIZE,
 				 &sc->an_rid_buffer, 0);
 	for (i = 0; i < AN_MAX_RX_DESC; i++)
 		if(!sc->an_rx_buffer[i].an_dma_paddr)
 			an_dma_malloc(sc, AN_RX_BUFFER_SIZE,
 				      &sc->an_rx_buffer[i], 0);
 	for (i = 0; i < AN_MAX_TX_DESC; i++)
 		if(!sc->an_tx_buffer[i].an_dma_paddr)
 			an_dma_malloc(sc, AN_TX_BUFFER_SIZE,
 				      &sc->an_tx_buffer[i], 0);
 
 	/*
 	 * Allocate RX descriptor
 	 */
 	bzero(&reply,sizeof(reply));
 	cmd_struct.an_cmd   = AN_CMD_ALLOC_DESC;
 	cmd_struct.an_parm0 = AN_DESCRIPTOR_RX;
 	cmd_struct.an_parm1 = AN_RX_DESC_OFFSET;
 	cmd_struct.an_parm2 = AN_MAX_RX_DESC;
 	if (an_cmd_struct(sc, &cmd_struct, &reply)) {
 		if_printf(sc->an_ifp, "failed to allocate RX descriptor\n");
 		return(EIO);
 	}
 
 	for (desc = 0; desc < AN_MAX_RX_DESC; desc++) {
 		bzero(&an_rx_desc, sizeof(an_rx_desc));
 		an_rx_desc.an_valid = 1;
 		an_rx_desc.an_len = AN_RX_BUFFER_SIZE;
 		an_rx_desc.an_done = 0;
 		an_rx_desc.an_phys = sc->an_rx_buffer[desc].an_dma_paddr;
 
 		for (i = 0; i < sizeof(an_rx_desc) / 4; i++)
 			CSR_MEM_AUX_WRITE_4(sc, AN_RX_DESC_OFFSET
 			    + (desc * sizeof(an_rx_desc))
 			    + (i * 4),
 			    ((u_int32_t *)(void *)&an_rx_desc)[i]);
 	}
 
 	/*
 	 * Allocate TX descriptor
 	 */
 
 	bzero(&reply,sizeof(reply));
 	cmd_struct.an_cmd   = AN_CMD_ALLOC_DESC;
 	cmd_struct.an_parm0 = AN_DESCRIPTOR_TX;
 	cmd_struct.an_parm1 = AN_TX_DESC_OFFSET;
 	cmd_struct.an_parm2 = AN_MAX_TX_DESC;
 	if (an_cmd_struct(sc, &cmd_struct, &reply)) {
 		if_printf(sc->an_ifp, "failed to allocate TX descriptor\n");
 		return(EIO);
 	}
 
 	for (desc = 0; desc < AN_MAX_TX_DESC; desc++) {
 		bzero(&an_tx_desc, sizeof(an_tx_desc));
 		an_tx_desc.an_offset = 0;
 		an_tx_desc.an_eoc = 0;
 		an_tx_desc.an_valid = 0;
 		an_tx_desc.an_len = 0;
 		an_tx_desc.an_phys = sc->an_tx_buffer[desc].an_dma_paddr;
 
 		for (i = 0; i < sizeof(an_tx_desc) / 4; i++)
 			CSR_MEM_AUX_WRITE_4(sc, AN_TX_DESC_OFFSET
 			    + (desc * sizeof(an_tx_desc))
 			    + (i * 4),
 			    ((u_int32_t *)(void *)&an_tx_desc)[i]);
 	}
 
 	/*
 	 * Allocate RID descriptor
 	 */
 
 	bzero(&reply,sizeof(reply));
 	cmd_struct.an_cmd   = AN_CMD_ALLOC_DESC;
 	cmd_struct.an_parm0 = AN_DESCRIPTOR_HOSTRW;
 	cmd_struct.an_parm1 = AN_HOST_DESC_OFFSET;
 	cmd_struct.an_parm2 = 1;
 	if (an_cmd_struct(sc, &cmd_struct, &reply)) {
 		if_printf(sc->an_ifp, "failed to allocate host descriptor\n");
 		return(EIO);
 	}
 
 	bzero(&an_rid_desc, sizeof(an_rid_desc));
 	an_rid_desc.an_valid = 1;
 	an_rid_desc.an_len = AN_RID_BUFFER_SIZE;
 	an_rid_desc.an_rid = 0;
 	an_rid_desc.an_phys = sc->an_rid_buffer.an_dma_paddr;
 
 	for (i = 0; i < sizeof(an_rid_desc) / 4; i++)
 		CSR_MEM_AUX_WRITE_4(sc, AN_HOST_DESC_OFFSET + i * 4,
 				    ((u_int32_t *)(void *)&an_rid_desc)[i]);
 
 	return(0);
 }
 
 int
 an_attach(struct an_softc *sc, int flags)
 {
 	struct ifnet		*ifp;
 	int			error = EIO;
 	int			i, nrate, mword;
 	u_int8_t		r;
 
 	ifp = sc->an_ifp = if_alloc(IFT_ETHER);
 	if (ifp == NULL) {
 		device_printf(sc->an_dev, "can not if_alloc()\n");
 		goto fail;
 	}
 	ifp->if_softc = sc;
 	if_initname(ifp, device_get_name(sc->an_dev),
 	    device_get_unit(sc->an_dev));
 
 	sc->an_gone = 0;
 	sc->an_associated = 0;
 	sc->an_monitor = 0;
 	sc->an_was_monitor = 0;
 	sc->an_flash_buffer = NULL;
 
 	/* Reset the NIC. */
 	AN_LOCK(sc);
 	an_reset(sc);
 	if (sc->mpi350) {
 		error = an_init_mpi350_desc(sc);
 		if (error)
 			goto fail;
 	}
 
 	/* Load factory config */
 	if (an_cmd(sc, AN_CMD_READCFG, 0)) {
 		device_printf(sc->an_dev, "failed to load config data\n");
 		goto fail;
 	}
 
 	/* Read the current configuration */
 	sc->an_config.an_type = AN_RID_GENCONFIG;
 	sc->an_config.an_len = sizeof(struct an_ltv_genconfig);
 	if (an_read_record(sc, (struct an_ltv_gen *)&sc->an_config)) {
 		device_printf(sc->an_dev, "read record failed\n");
 		goto fail;
 	}
 
 	/* Read the card capabilities */
 	sc->an_caps.an_type = AN_RID_CAPABILITIES;
 	sc->an_caps.an_len = sizeof(struct an_ltv_caps);
 	if (an_read_record(sc, (struct an_ltv_gen *)&sc->an_caps)) {
 		device_printf(sc->an_dev, "read record failed\n");
 		goto fail;
 	}
 
 	/* Read ssid list */
 	sc->an_ssidlist.an_type = AN_RID_SSIDLIST;
 	sc->an_ssidlist.an_len = sizeof(struct an_ltv_ssidlist_new);
 	if (an_read_record(sc, (struct an_ltv_gen *)&sc->an_ssidlist)) {
 		device_printf(sc->an_dev, "read record failed\n");
 		goto fail;
 	}
 
 	/* Read AP list */
 	sc->an_aplist.an_type = AN_RID_APLIST;
 	sc->an_aplist.an_len = sizeof(struct an_ltv_aplist);
 	if (an_read_record(sc, (struct an_ltv_gen *)&sc->an_aplist)) {
 		device_printf(sc->an_dev, "read record failed\n");
 		goto fail;
 	}
 
 #ifdef ANCACHE
 	/* Read the RSSI <-> dBm map */
 	sc->an_have_rssimap = 0;
 	if (sc->an_caps.an_softcaps & 8) {
 		sc->an_rssimap.an_type = AN_RID_RSSI_MAP;
 		sc->an_rssimap.an_len = sizeof(struct an_ltv_rssi_map);
 		if (an_read_record(sc, (struct an_ltv_gen *)&sc->an_rssimap)) {
 			device_printf(sc->an_dev,
 			    "unable to get RSSI <-> dBM map\n");
 		} else {
 			device_printf(sc->an_dev, "got RSSI <-> dBM map\n");
 			sc->an_have_rssimap = 1;
 		}
 	} else {
 		device_printf(sc->an_dev, "no RSSI <-> dBM map\n");
 	}
 #endif
 	AN_UNLOCK(sc);
 
 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
 	ifp->if_ioctl = an_ioctl;
 	ifp->if_start = an_start;
 	ifp->if_init = an_init;
 	ifp->if_baudrate = 10000000;
 	IFQ_SET_MAXLEN(&ifp->if_snd, ifqmaxlen);
 	ifp->if_snd.ifq_drv_maxlen = ifqmaxlen;
 	IFQ_SET_READY(&ifp->if_snd);
 
 	bzero(sc->an_config.an_nodename, sizeof(sc->an_config.an_nodename));
 	bcopy(AN_DEFAULT_NODENAME, sc->an_config.an_nodename,
 	    sizeof(AN_DEFAULT_NODENAME) - 1);
 
 	bzero(sc->an_ssidlist.an_entry[0].an_ssid,
 	      sizeof(sc->an_ssidlist.an_entry[0].an_ssid));
 	bcopy(AN_DEFAULT_NETNAME, sc->an_ssidlist.an_entry[0].an_ssid,
 	    sizeof(AN_DEFAULT_NETNAME) - 1);
 	sc->an_ssidlist.an_entry[0].an_len = strlen(AN_DEFAULT_NETNAME);
 
 	sc->an_config.an_opmode =
 	    AN_OPMODE_INFRASTRUCTURE_STATION;
 
 	sc->an_tx_rate = 0;
 	bzero((char *)&sc->an_stats, sizeof(sc->an_stats));
 
 	nrate = 8;
 
 	ifmedia_init(&sc->an_ifmedia, 0, an_media_change, an_media_status);
 	if_printf(ifp, "supported rates: ");
 #define	ADD(s, o)	ifmedia_add(&sc->an_ifmedia, \
 	IFM_MAKEWORD(IFM_IEEE80211, (s), (o), 0), 0, NULL)
 	ADD(IFM_AUTO, 0);
 	ADD(IFM_AUTO, IFM_IEEE80211_ADHOC);
 	for (i = 0; i < nrate; i++) {
 		r = sc->an_caps.an_rates[i];
 		mword = ieee80211_rate2media(NULL, r, IEEE80211_MODE_AUTO);
 		if (mword == 0)
 			continue;
 		printf("%s%d%sMbps", (i != 0 ? " " : ""),
 		    (r & IEEE80211_RATE_VAL) / 2, ((r & 0x1) != 0 ? ".5" : ""));
 		ADD(mword, 0);
 		ADD(mword, IFM_IEEE80211_ADHOC);
 	}
 	printf("\n");
 	ifmedia_set(&sc->an_ifmedia, IFM_MAKEWORD(IFM_IEEE80211,
 	    IFM_AUTO, 0, 0));
 #undef ADD
 
 	/*
 	 * Call MI attach routine.
 	 */
 
 	ether_ifattach(ifp, sc->an_caps.an_oemaddr);
 	callout_init_mtx(&sc->an_stat_ch, &sc->an_mtx, 0);
 
 	return(0);
 fail:
 	AN_UNLOCK(sc);
 	mtx_destroy(&sc->an_mtx);
 	if (ifp != NULL)
 		if_free(ifp);
 	return(error);
 }
 
 int
 an_detach(device_t dev)
 {
 	struct an_softc		*sc = device_get_softc(dev);
 	struct ifnet		*ifp = sc->an_ifp;
 
 	if (sc->an_gone) {
 		device_printf(dev,"already unloaded\n");
 		return(0);
 	}
 	AN_LOCK(sc);
 	an_stop(sc);
 	sc->an_gone = 1;
 	ifmedia_removeall(&sc->an_ifmedia);
 	ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
 	AN_UNLOCK(sc);
 	ether_ifdetach(ifp);
 	bus_teardown_intr(dev, sc->irq_res, sc->irq_handle);
 	callout_drain(&sc->an_stat_ch);
 	if_free(ifp);
 	an_release_resources(dev);
 	mtx_destroy(&sc->an_mtx);
 	return (0);
 }
 
 static void
 an_rxeof(struct an_softc *sc)
 {
 	struct ifnet   *ifp;
 	struct ether_header *eh;
 	struct ieee80211_frame *ih;
 	struct an_rxframe rx_frame;
 	struct an_rxframe_802_3 rx_frame_802_3;
 	struct mbuf    *m;
 	int		len, id, error = 0, i, count = 0;
 	int		ieee80211_header_len;
 	u_char		*bpf_buf;
 	u_short		fc1;
 	struct an_card_rx_desc an_rx_desc;
 	u_int8_t	*buf;
 
 	AN_LOCK_ASSERT(sc);
 
 	ifp = sc->an_ifp;
 
 	if (!sc->mpi350) {
 		id = CSR_READ_2(sc, AN_RX_FID);
 
 		if (sc->an_monitor && (ifp->if_flags & IFF_PROMISC)) {
 			/* read raw 802.11 packet */
 			bpf_buf = sc->buf_802_11;
 
 			/* read header */
 			if (an_read_data(sc, id, 0x0, (caddr_t)&rx_frame,
 					 sizeof(rx_frame))) {
 				ifp->if_ierrors++;
 				return;
 			}
 
 			/*
 			 * skip beacon by default since this increases the
 			 * system load a lot
 			 */
 
 			if (!(sc->an_monitor & AN_MONITOR_INCLUDE_BEACON) &&
 			    (rx_frame.an_frame_ctl & 
 			     IEEE80211_FC0_SUBTYPE_BEACON)) {
 				return;
 			}
 
 			if (sc->an_monitor & AN_MONITOR_AIRONET_HEADER) {
 				len = rx_frame.an_rx_payload_len
 					+ sizeof(rx_frame);
 				/* Check for insane frame length */
 				if (len > sizeof(sc->buf_802_11)) {
 					if_printf(ifp, "oversized packet "
 					       "received (%d, %d)\n",
 					       len, MCLBYTES);
 					ifp->if_ierrors++;
 					return;
 				}
 
 				bcopy((char *)&rx_frame,
 				      bpf_buf, sizeof(rx_frame));
 
 				error = an_read_data(sc, id, sizeof(rx_frame),
 					    (caddr_t)bpf_buf+sizeof(rx_frame),
 					    rx_frame.an_rx_payload_len);
 			} else {
 				fc1=rx_frame.an_frame_ctl >> 8;
 				ieee80211_header_len =
 					sizeof(struct ieee80211_frame);
 				if ((fc1 & IEEE80211_FC1_DIR_TODS) &&
 				    (fc1 & IEEE80211_FC1_DIR_FROMDS)) {
 					ieee80211_header_len += ETHER_ADDR_LEN;
 				}
 
 				len = rx_frame.an_rx_payload_len
 					+ ieee80211_header_len;
 				/* Check for insane frame length */
 				if (len > sizeof(sc->buf_802_11)) {
 					if_printf(ifp, "oversized packet "
 					       "received (%d, %d)\n",
 					       len, MCLBYTES);
 					ifp->if_ierrors++;
 					return;
 				}
 
 				ih = (struct ieee80211_frame *)bpf_buf;
 
 				bcopy((char *)&rx_frame.an_frame_ctl,
 				      (char *)ih, ieee80211_header_len);
 
 				error = an_read_data(sc, id, sizeof(rx_frame) +
 					    rx_frame.an_gaplen,
 					    (caddr_t)ih +ieee80211_header_len,
 					    rx_frame.an_rx_payload_len);
 			}
 			/* dump raw 802.11 packet to bpf and skip ip stack */
 			BPF_TAP(ifp, bpf_buf, len);
 		} else {
 			MGETHDR(m, M_NOWAIT, MT_DATA);
 			if (m == NULL) {
 				ifp->if_ierrors++;
 				return;
 			}
 			MCLGET(m, M_NOWAIT);
 			if (!(m->m_flags & M_EXT)) {
 				m_freem(m);
 				ifp->if_ierrors++;
 				return;
 			}
 			m->m_pkthdr.rcvif = ifp;
 			/* Read Ethernet encapsulated packet */
 
 #ifdef ANCACHE
 			/* Read NIC frame header */
 			if (an_read_data(sc, id, 0, (caddr_t)&rx_frame,
 					 sizeof(rx_frame))) {
 				m_freem(m);
 				ifp->if_ierrors++;
 				return;
 			}
 #endif
 			/* Read in the 802_3 frame header */
 			if (an_read_data(sc, id, 0x34,
 					 (caddr_t)&rx_frame_802_3,
 					 sizeof(rx_frame_802_3))) {
 				m_freem(m);
 				ifp->if_ierrors++;
 				return;
 			}
 			if (rx_frame_802_3.an_rx_802_3_status != 0) {
 				m_freem(m);
 				ifp->if_ierrors++;
 				return;
 			}
 			/* Check for insane frame length */
 			len = rx_frame_802_3.an_rx_802_3_payload_len;
 			if (len > sizeof(sc->buf_802_11)) {
 				m_freem(m);
 				if_printf(ifp, "oversized packet "
 				       "received (%d, %d)\n",
 				       len, MCLBYTES);
 				ifp->if_ierrors++;
 				return;
 			}
 			m->m_pkthdr.len = m->m_len =
 				rx_frame_802_3.an_rx_802_3_payload_len + 12;
 
 			eh = mtod(m, struct ether_header *);
 
 			bcopy((char *)&rx_frame_802_3.an_rx_dst_addr,
 			      (char *)&eh->ether_dhost, ETHER_ADDR_LEN);
 			bcopy((char *)&rx_frame_802_3.an_rx_src_addr,
 			      (char *)&eh->ether_shost, ETHER_ADDR_LEN);
 
 			/* in mbuf header type is just before payload */
 			error = an_read_data(sc, id, 0x44,
 				    (caddr_t)&(eh->ether_type),
 				    rx_frame_802_3.an_rx_802_3_payload_len);
 
 			if (error) {
 				m_freem(m);
 				ifp->if_ierrors++;
 				return;
 			}
 			ifp->if_ipackets++;
 
 			/* Receive packet. */
 #ifdef ANCACHE
 			an_cache_store(sc, eh, m,
 				rx_frame.an_rx_signal_strength,
 				rx_frame.an_rsvd0);
 #endif
 			AN_UNLOCK(sc);
 			(*ifp->if_input)(ifp, m);
 			AN_LOCK(sc);
 		}
 
 	} else { /* MPI-350 */
 		for (count = 0; count < AN_MAX_RX_DESC; count++){
 			for (i = 0; i < sizeof(an_rx_desc) / 4; i++)
 				((u_int32_t *)(void *)&an_rx_desc)[i]
 					= CSR_MEM_AUX_READ_4(sc,
 						AN_RX_DESC_OFFSET
 						+ (count * sizeof(an_rx_desc))
 						+ (i * 4));
 
 			if (an_rx_desc.an_done && !an_rx_desc.an_valid) {
 				buf = sc->an_rx_buffer[count].an_dma_vaddr;
 
 				MGETHDR(m, M_NOWAIT, MT_DATA);
 				if (m == NULL) {
 					ifp->if_ierrors++;
 					return;
 				}
 				MCLGET(m, M_NOWAIT);
 				if (!(m->m_flags & M_EXT)) {
 					m_freem(m);
 					ifp->if_ierrors++;
 					return;
 				}
 				m->m_pkthdr.rcvif = ifp;
 				/* Read Ethernet encapsulated packet */
 
 				/*
 				 * No ANCACHE support since we just get back
 				 * an Ethernet packet no 802.11 info
 				 */
 #if 0
 #ifdef ANCACHE
 				/* Read NIC frame header */
 				bcopy(buf, (caddr_t)&rx_frame,
 				      sizeof(rx_frame));
 #endif
 #endif
 				/* Check for insane frame length */
 				len = an_rx_desc.an_len + 12;
 				if (len > MCLBYTES) {
 					m_freem(m);
 					if_printf(ifp, "oversized packet "
 					       "received (%d, %d)\n",
 					       len, MCLBYTES);
 					ifp->if_ierrors++;
 					return;
 				}
 
 				m->m_pkthdr.len = m->m_len =
 					an_rx_desc.an_len + 12;
 
 				eh = mtod(m, struct ether_header *);
 
 				bcopy(buf, (char *)eh,
 				      m->m_pkthdr.len);
 
 				ifp->if_ipackets++;
 
 				/* Receive packet. */
 #if 0
 #ifdef ANCACHE
 				an_cache_store(sc, eh, m,
 					rx_frame.an_rx_signal_strength,
 					rx_frame.an_rsvd0);
 #endif
 #endif
 				AN_UNLOCK(sc);
 				(*ifp->if_input)(ifp, m);
 				AN_LOCK(sc);
 
 				an_rx_desc.an_valid = 1;
 				an_rx_desc.an_len = AN_RX_BUFFER_SIZE;
 				an_rx_desc.an_done = 0;
 				an_rx_desc.an_phys =
 					sc->an_rx_buffer[count].an_dma_paddr;
 
 				for (i = 0; i < sizeof(an_rx_desc) / 4; i++)
 					CSR_MEM_AUX_WRITE_4(sc,
 					    AN_RX_DESC_OFFSET
 					    + (count * sizeof(an_rx_desc))
 					    + (i * 4),
 					    ((u_int32_t *)(void *)&an_rx_desc)[i]);
 
 			} else {
 				if_printf(ifp, "Didn't get valid RX packet "
 				       "%x %x %d\n",
 				       an_rx_desc.an_done,
 				       an_rx_desc.an_valid, an_rx_desc.an_len);
 			}
 		}
 	}
 }
 
 static void
 an_txeof(struct an_softc *sc, int status)
 {
 	struct ifnet		*ifp;
 	int			id, i;
 
 	AN_LOCK_ASSERT(sc);
 	ifp = sc->an_ifp;
 
 	sc->an_timer = 0;
 	ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
 
 	if (!sc->mpi350) {
 		id = CSR_READ_2(sc, AN_TX_CMP_FID(sc->mpi350));
 
 		if (status & AN_EV_TX_EXC) {
 			ifp->if_oerrors++;
 		} else
 			ifp->if_opackets++;
 
 		for (i = 0; i < AN_TX_RING_CNT; i++) {
 			if (id == sc->an_rdata.an_tx_ring[i]) {
 				sc->an_rdata.an_tx_ring[i] = 0;
 				break;
 			}
 		}
 
 		AN_INC(sc->an_rdata.an_tx_cons, AN_TX_RING_CNT);
 	} else { /* MPI 350 */
 		id = CSR_READ_2(sc, AN_TX_CMP_FID(sc->mpi350));
 		if (!sc->an_rdata.an_tx_empty){
 			if (status & AN_EV_TX_EXC) {
 				ifp->if_oerrors++;
 			} else
 				ifp->if_opackets++;
 			AN_INC(sc->an_rdata.an_tx_cons, AN_MAX_TX_DESC);
 			if (sc->an_rdata.an_tx_prod ==
 			    sc->an_rdata.an_tx_cons)
 				sc->an_rdata.an_tx_empty = 1;
 		}
 	}
 
 	return;
 }
 
 /*
  * We abuse the stats updater to check the current NIC status. This
  * is important because we don't want to allow transmissions until
  * the NIC has synchronized to the current cell (either as the master
  * in an ad-hoc group, or as a station connected to an access point).
  *
  * Note that this function will be called via callout(9) with a lock held.
  */
 static void
 an_stats_update(void *xsc)
 {
 	struct an_softc		*sc;
 	struct ifnet		*ifp;
 
 	sc = xsc;
 	AN_LOCK_ASSERT(sc);
 	ifp = sc->an_ifp;
 	if (sc->an_timer > 0 && --sc->an_timer == 0)
 		an_watchdog(sc);
 
 	sc->an_status.an_type = AN_RID_STATUS;
 	sc->an_status.an_len = sizeof(struct an_ltv_status);
 	if (an_read_record(sc, (struct an_ltv_gen *)&sc->an_status))
 		return;
 
 	if (sc->an_status.an_opmode & AN_STATUS_OPMODE_IN_SYNC)
 		sc->an_associated = 1;
 	else
 		sc->an_associated = 0;
 
 	/* Don't do this while we're transmitting */
 	if (ifp->if_drv_flags & IFF_DRV_OACTIVE) {
 		callout_reset(&sc->an_stat_ch, hz, an_stats_update, sc);
 		return;
 	}
 
 	sc->an_stats.an_len = sizeof(struct an_ltv_stats);
 	sc->an_stats.an_type = AN_RID_32BITS_CUM;
 	if (an_read_record(sc, (struct an_ltv_gen *)&sc->an_stats.an_len))
 		return;
 
 	callout_reset(&sc->an_stat_ch, hz, an_stats_update, sc);
 
 	return;
 }
 
 void
 an_intr(void *xsc)
 {
 	struct an_softc		*sc;
 	struct ifnet		*ifp;
 	u_int16_t		status;
 
 	sc = (struct an_softc*)xsc;
 
 	AN_LOCK(sc);
 
 	if (sc->an_gone) {
 		AN_UNLOCK(sc);
 		return;
 	}
 
 	ifp = sc->an_ifp;
 
 	/* Disable interrupts. */
 	CSR_WRITE_2(sc, AN_INT_EN(sc->mpi350), 0);
 
 	status = CSR_READ_2(sc, AN_EVENT_STAT(sc->mpi350));
 	CSR_WRITE_2(sc, AN_EVENT_ACK(sc->mpi350), ~AN_INTRS(sc->mpi350));
 
 	if (status & AN_EV_MIC) {
 		CSR_WRITE_2(sc, AN_EVENT_ACK(sc->mpi350), AN_EV_MIC);
 	}
 
 	if (status & AN_EV_LINKSTAT) {
 		if (CSR_READ_2(sc, AN_LINKSTAT(sc->mpi350))
 		    == AN_LINKSTAT_ASSOCIATED)
 			sc->an_associated = 1;
 		else
 			sc->an_associated = 0;
 		CSR_WRITE_2(sc, AN_EVENT_ACK(sc->mpi350), AN_EV_LINKSTAT);
 	}
 
 	if (status & AN_EV_RX) {
 		an_rxeof(sc);
 		CSR_WRITE_2(sc, AN_EVENT_ACK(sc->mpi350), AN_EV_RX);
 	}
 
 	if (sc->mpi350 && status & AN_EV_TX_CPY) {
 		an_txeof(sc, status);
 		CSR_WRITE_2(sc, AN_EVENT_ACK(sc->mpi350), AN_EV_TX_CPY);
 	}
 
 	if (status & AN_EV_TX) {
 		an_txeof(sc, status);
 		CSR_WRITE_2(sc, AN_EVENT_ACK(sc->mpi350), AN_EV_TX);
 	}
 
 	if (status & AN_EV_TX_EXC) {
 		an_txeof(sc, status);
 		CSR_WRITE_2(sc, AN_EVENT_ACK(sc->mpi350), AN_EV_TX_EXC);
 	}
 
 	if (status & AN_EV_ALLOC)
 		CSR_WRITE_2(sc, AN_EVENT_ACK(sc->mpi350), AN_EV_ALLOC);
 
 	/* Re-enable interrupts. */
 	CSR_WRITE_2(sc, AN_INT_EN(sc->mpi350), AN_INTRS(sc->mpi350));
 
 	if ((ifp->if_flags & IFF_UP) && !IFQ_DRV_IS_EMPTY(&ifp->if_snd))
 		an_start_locked(ifp);
 
 	AN_UNLOCK(sc);
 
 	return;
 }
 
 
 static int
 an_cmd_struct(struct an_softc *sc, struct an_command *cmd,
     struct an_reply *reply)
 {
 	int			i;
 
 	AN_LOCK_ASSERT(sc);
 	for (i = 0; i != AN_TIMEOUT; i++) {
 		if (CSR_READ_2(sc, AN_COMMAND(sc->mpi350)) & AN_CMD_BUSY) {
 			DELAY(1000);
 		} else
 			break;
 	}
 
 	if( i == AN_TIMEOUT) {
 		printf("BUSY\n");
 		return(ETIMEDOUT);
 	}
 
 	CSR_WRITE_2(sc, AN_PARAM0(sc->mpi350), cmd->an_parm0);
 	CSR_WRITE_2(sc, AN_PARAM1(sc->mpi350), cmd->an_parm1);
 	CSR_WRITE_2(sc, AN_PARAM2(sc->mpi350), cmd->an_parm2);
 	CSR_WRITE_2(sc, AN_COMMAND(sc->mpi350), cmd->an_cmd);
 
 	for (i = 0; i < AN_TIMEOUT; i++) {
 		if (CSR_READ_2(sc, AN_EVENT_STAT(sc->mpi350)) & AN_EV_CMD)
 			break;
 		DELAY(1000);
 	}
 
 	reply->an_resp0 = CSR_READ_2(sc, AN_RESP0(sc->mpi350));
 	reply->an_resp1 = CSR_READ_2(sc, AN_RESP1(sc->mpi350));
 	reply->an_resp2 = CSR_READ_2(sc, AN_RESP2(sc->mpi350));
 	reply->an_status = CSR_READ_2(sc, AN_STATUS(sc->mpi350));
 
 	if (CSR_READ_2(sc, AN_COMMAND(sc->mpi350)) & AN_CMD_BUSY)
 		CSR_WRITE_2(sc, AN_EVENT_ACK(sc->mpi350),
 		    AN_EV_CLR_STUCK_BUSY);
 
 	/* Ack the command */
 	CSR_WRITE_2(sc, AN_EVENT_ACK(sc->mpi350), AN_EV_CMD);
 
 	if (i == AN_TIMEOUT)
 		return(ETIMEDOUT);
 
 	return(0);
 }
 
 static int
 an_cmd(struct an_softc *sc, int cmd, int val)
 {
 	int			i, s = 0;
 
 	AN_LOCK_ASSERT(sc);
 	CSR_WRITE_2(sc, AN_PARAM0(sc->mpi350), val);
 	CSR_WRITE_2(sc, AN_PARAM1(sc->mpi350), 0);
 	CSR_WRITE_2(sc, AN_PARAM2(sc->mpi350), 0);
 	CSR_WRITE_2(sc, AN_COMMAND(sc->mpi350), cmd);
 
 	for (i = 0; i < AN_TIMEOUT; i++) {
 		if (CSR_READ_2(sc, AN_EVENT_STAT(sc->mpi350)) & AN_EV_CMD)
 			break;
 		else {
 			if (CSR_READ_2(sc, AN_COMMAND(sc->mpi350)) == cmd)
 				CSR_WRITE_2(sc, AN_COMMAND(sc->mpi350), cmd);
 		}
 	}
 
 	for (i = 0; i < AN_TIMEOUT; i++) {
 		CSR_READ_2(sc, AN_RESP0(sc->mpi350));
 		CSR_READ_2(sc, AN_RESP1(sc->mpi350));
 		CSR_READ_2(sc, AN_RESP2(sc->mpi350));
 		s = CSR_READ_2(sc, AN_STATUS(sc->mpi350));
 		if ((s & AN_STAT_CMD_CODE) == (cmd & AN_STAT_CMD_CODE))
 			break;
 	}
 
 	/* Ack the command */
 	CSR_WRITE_2(sc, AN_EVENT_ACK(sc->mpi350), AN_EV_CMD);
 
 	if (CSR_READ_2(sc, AN_COMMAND(sc->mpi350)) & AN_CMD_BUSY)
 		CSR_WRITE_2(sc, AN_EVENT_ACK(sc->mpi350), AN_EV_CLR_STUCK_BUSY);
 
 	if (i == AN_TIMEOUT)
 		return(ETIMEDOUT);
 
 	return(0);
 }
 
 /*
  * This reset sequence may look a little strange, but this is the
  * most reliable method I've found to really kick the NIC in the
  * head and force it to reboot correctly.
  */
 static void
 an_reset(struct an_softc *sc)
 {
 	if (sc->an_gone)
 		return;
 
 	AN_LOCK_ASSERT(sc);
 	an_cmd(sc, AN_CMD_ENABLE, 0);
 	an_cmd(sc, AN_CMD_FW_RESTART, 0);
 	an_cmd(sc, AN_CMD_NOOP2, 0);
 
 	if (an_cmd(sc, AN_CMD_FORCE_SYNCLOSS, 0) == ETIMEDOUT)
 		device_printf(sc->an_dev, "reset failed\n");
 
 	an_cmd(sc, AN_CMD_DISABLE, 0);
 
 	return;
 }
 
 /*
  * Read an LTV record from the NIC.
  */
 static int
 an_read_record(struct an_softc *sc, struct an_ltv_gen *ltv)
 {
 	struct an_ltv_gen	*an_ltv;
 	struct an_card_rid_desc an_rid_desc;
 	struct an_command	cmd;
 	struct an_reply		reply;
 	struct ifnet		*ifp;
 	u_int16_t		*ptr;
 	u_int8_t		*ptr2;
 	int			i, len;
 
 	AN_LOCK_ASSERT(sc);
 	if (ltv->an_len < 4 || ltv->an_type == 0)
 		return(EINVAL);
 
 	ifp = sc->an_ifp;
 	if (!sc->mpi350){
 		/* Tell the NIC to enter record read mode. */
 		if (an_cmd(sc, AN_CMD_ACCESS|AN_ACCESS_READ, ltv->an_type)) {
 			if_printf(ifp, "RID access failed\n");
 			return(EIO);
 		}
 
 		/* Seek to the record. */
 		if (an_seek(sc, ltv->an_type, 0, AN_BAP1)) {
 			if_printf(ifp, "seek to record failed\n");
 			return(EIO);
 		}
 
 		/*
 		 * Read the length and record type and make sure they
 		 * match what we expect (this verifies that we have enough
 		 * room to hold all of the returned data).
 		 * Length includes type but not length.
 		 */
 		len = CSR_READ_2(sc, AN_DATA1);
 		if (len > (ltv->an_len - 2)) {
 			if_printf(ifp, "record length mismatch -- expected %d, "
 			       "got %d for Rid %x\n",
 			       ltv->an_len - 2, len, ltv->an_type);
 			len = ltv->an_len - 2;
 		} else {
 			ltv->an_len = len + 2;
 		}
 
 		/* Now read the data. */
 		len -= 2;	/* skip the type */
 		ptr = &ltv->an_val;
 		for (i = len; i > 1; i -= 2)
 			*ptr++ = CSR_READ_2(sc, AN_DATA1);
 		if (i) {
 			ptr2 = (u_int8_t *)ptr;
 			*ptr2 = CSR_READ_1(sc, AN_DATA1);
 		}
 	} else { /* MPI-350 */
 		if (!sc->an_rid_buffer.an_dma_vaddr)
 			return(EIO);
 		an_rid_desc.an_valid = 1;
 		an_rid_desc.an_len = AN_RID_BUFFER_SIZE;
 		an_rid_desc.an_rid = 0;
 		an_rid_desc.an_phys = sc->an_rid_buffer.an_dma_paddr;
 		bzero(sc->an_rid_buffer.an_dma_vaddr, AN_RID_BUFFER_SIZE);
 
 		bzero(&cmd, sizeof(cmd));
 		bzero(&reply, sizeof(reply));
 		cmd.an_cmd = AN_CMD_ACCESS|AN_ACCESS_READ;
 		cmd.an_parm0 = ltv->an_type;
 
 		for (i = 0; i < sizeof(an_rid_desc) / 4; i++)
 			CSR_MEM_AUX_WRITE_4(sc, AN_HOST_DESC_OFFSET + i * 4,
 			    ((u_int32_t *)(void *)&an_rid_desc)[i]);
 
 		if (an_cmd_struct(sc, &cmd, &reply)
 		    || reply.an_status & AN_CMD_QUAL_MASK) {
 			if_printf(ifp, "failed to read RID %x %x %x %x %x, %d\n",
 			       ltv->an_type,
 			       reply.an_status,
 			       reply.an_resp0,
 			       reply.an_resp1,
 			       reply.an_resp2,
 			       i);
 			return(EIO);
 		}
 
 		an_ltv = (struct an_ltv_gen *)sc->an_rid_buffer.an_dma_vaddr;
 		if (an_ltv->an_len + 2 < an_rid_desc.an_len) {
 			an_rid_desc.an_len = an_ltv->an_len;
 		}
 
 		len = an_rid_desc.an_len;
 		if (len > (ltv->an_len - 2)) {
 			if_printf(ifp, "record length mismatch -- expected %d, "
 			       "got %d for Rid %x\n",
 			       ltv->an_len - 2, len, ltv->an_type);
 			len = ltv->an_len - 2;
 		} else {
 			ltv->an_len = len + 2;
 		}
 		bcopy(&an_ltv->an_type,
 		    &ltv->an_val,
 		    len);
 	}
 
 	if (an_dump)
 		an_dump_record(sc, ltv, "Read");
 
 	return(0);
 }
 
 /*
  * Same as read, except we inject data instead of reading it.
  */
 static int
 an_write_record(struct an_softc *sc, struct an_ltv_gen *ltv)
 {
 	struct an_card_rid_desc an_rid_desc;
 	struct an_command	cmd;
 	struct an_reply		reply;
 	u_int16_t		*ptr;
 	u_int8_t		*ptr2;
 	int			i, len;
 
 	AN_LOCK_ASSERT(sc);
 	if (an_dump)
 		an_dump_record(sc, ltv, "Write");
 
 	if (!sc->mpi350){
 		if (an_cmd(sc, AN_CMD_ACCESS|AN_ACCESS_READ, ltv->an_type))
 			return(EIO);
 
 		if (an_seek(sc, ltv->an_type, 0, AN_BAP1))
 			return(EIO);
 
 		/*
 		 * Length includes type but not length.
 		 */
 		len = ltv->an_len - 2;
 		CSR_WRITE_2(sc, AN_DATA1, len);
 
 		len -= 2;	/* skip the type */
 		ptr = &ltv->an_val;
 		for (i = len; i > 1; i -= 2)
 			CSR_WRITE_2(sc, AN_DATA1, *ptr++);
 		if (i) {
 			ptr2 = (u_int8_t *)ptr;
 			CSR_WRITE_1(sc, AN_DATA0, *ptr2);
 		}
 
 		if (an_cmd(sc, AN_CMD_ACCESS|AN_ACCESS_WRITE, ltv->an_type))
 			return(EIO);
 	} else {
 		/* MPI-350 */
 
 		for (i = 0; i != AN_TIMEOUT; i++) {
 			if (CSR_READ_2(sc, AN_COMMAND(sc->mpi350))
 			    & AN_CMD_BUSY) {
 				DELAY(10);
 			} else
 				break;
 		}
 		if (i == AN_TIMEOUT) {
 			printf("BUSY\n");
 		}
 
 		an_rid_desc.an_valid = 1;
 		an_rid_desc.an_len = ltv->an_len - 2;
 		an_rid_desc.an_rid = ltv->an_type;
 		an_rid_desc.an_phys = sc->an_rid_buffer.an_dma_paddr;
 
 		bcopy(&ltv->an_type, sc->an_rid_buffer.an_dma_vaddr,
 		      an_rid_desc.an_len);
 
 		bzero(&cmd,sizeof(cmd));
 		bzero(&reply,sizeof(reply));
 		cmd.an_cmd = AN_CMD_ACCESS|AN_ACCESS_WRITE;
 		cmd.an_parm0 = ltv->an_type;
 
 		for (i = 0; i < sizeof(an_rid_desc) / 4; i++)
 			CSR_MEM_AUX_WRITE_4(sc, AN_HOST_DESC_OFFSET + i * 4,
 			    ((u_int32_t *)(void *)&an_rid_desc)[i]);
 
 		DELAY(100000);
 
 		if ((i = an_cmd_struct(sc, &cmd, &reply))) {
 			if_printf(sc->an_ifp,
 			    "failed to write RID 1 %x %x %x %x %x, %d\n",
 			    ltv->an_type,
 			    reply.an_status,
 			    reply.an_resp0,
 			    reply.an_resp1,
 			    reply.an_resp2,
 			    i);
 			return(EIO);
 		}
 
 
 		if (reply.an_status & AN_CMD_QUAL_MASK) {
 			if_printf(sc->an_ifp,
 			    "failed to write RID 2 %x %x %x %x %x, %d\n",
 			    ltv->an_type,
 			    reply.an_status,
 			    reply.an_resp0,
 			    reply.an_resp1,
 			    reply.an_resp2,
 			    i);
 			return(EIO);
 		}
 		DELAY(100000);
 	}
 
 	return(0);
 }
 
 static void
 an_dump_record(struct an_softc *sc, struct an_ltv_gen *ltv, char *string)
 {
 	u_int8_t		*ptr2;
 	int			len;
 	int			i;
 	int			count = 0;
 	char			buf[17], temp;
 
 	len = ltv->an_len - 4;
 	if_printf(sc->an_ifp, "RID %4x, Length %4d, Mode %s\n",
 		ltv->an_type, ltv->an_len - 4, string);
 
 	if (an_dump == 1 || (an_dump == ltv->an_type)) {
 		if_printf(sc->an_ifp, "\t");
 		bzero(buf,sizeof(buf));
 
 		ptr2 = (u_int8_t *)&ltv->an_val;
 		for (i = len; i > 0; i--) {
 			printf("%02x ", *ptr2);
 
 			temp = *ptr2++;
 			if (isprint(temp))
 				buf[count] = temp;
 			else
 				buf[count] = '.';
 			if (++count == 16) {
 				count = 0;
 				printf("%s\n",buf);
 				if_printf(sc->an_ifp, "\t");
 				bzero(buf,sizeof(buf));
 			}
 		}
 		for (; count != 16; count++) {
 			printf("   ");
 		}
 		printf(" %s\n",buf);
 	}
 }
 
 static int
 an_seek(struct an_softc *sc, int id, int off, int chan)
 {
 	int			i;
 	int			selreg, offreg;
 
 	switch (chan) {
 	case AN_BAP0:
 		selreg = AN_SEL0;
 		offreg = AN_OFF0;
 		break;
 	case AN_BAP1:
 		selreg = AN_SEL1;
 		offreg = AN_OFF1;
 		break;
 	default:
 		if_printf(sc->an_ifp, "invalid data path: %x\n", chan);
 		return(EIO);
 	}
 
 	CSR_WRITE_2(sc, selreg, id);
 	CSR_WRITE_2(sc, offreg, off);
 
 	for (i = 0; i < AN_TIMEOUT; i++) {
 		if (!(CSR_READ_2(sc, offreg) & (AN_OFF_BUSY|AN_OFF_ERR)))
 			break;
 	}
 
 	if (i == AN_TIMEOUT)
 		return(ETIMEDOUT);
 
 	return(0);
 }
 
 static int
 an_read_data(struct an_softc *sc, int id, int off, caddr_t buf, int len)
 {
 	int			i;
 	u_int16_t		*ptr;
 	u_int8_t		*ptr2;
 
 	if (off != -1) {
 		if (an_seek(sc, id, off, AN_BAP1))
 			return(EIO);
 	}
 
 	ptr = (u_int16_t *)buf;
 	for (i = len; i > 1; i -= 2)
 		*ptr++ = CSR_READ_2(sc, AN_DATA1);
 	if (i) {
 		ptr2 = (u_int8_t *)ptr;
 		*ptr2 = CSR_READ_1(sc, AN_DATA1);
 	}
 
 	return(0);
 }
 
 static int
 an_write_data(struct an_softc *sc, int id, int off, caddr_t buf, int len)
 {
 	int			i;
 	u_int16_t		*ptr;
 	u_int8_t		*ptr2;
 
 	if (off != -1) {
 		if (an_seek(sc, id, off, AN_BAP0))
 			return(EIO);
 	}
 
 	ptr = (u_int16_t *)buf;
 	for (i = len; i > 1; i -= 2)
 		CSR_WRITE_2(sc, AN_DATA0, *ptr++);
 	if (i) {
 		ptr2 = (u_int8_t *)ptr;
 		CSR_WRITE_1(sc, AN_DATA0, *ptr2);
 	}
 
 	return(0);
 }
 
 /*
  * Allocate a region of memory inside the NIC and zero
  * it out.
  */
 static int
 an_alloc_nicmem(struct an_softc *sc, int len, int *id)
 {
 	int			i;
 
 	if (an_cmd(sc, AN_CMD_ALLOC_MEM, len)) {
 		if_printf(sc->an_ifp, "failed to allocate %d bytes on NIC\n",
 		    len);
 		return(ENOMEM);
 	}
 
 	for (i = 0; i < AN_TIMEOUT; i++) {
 		if (CSR_READ_2(sc, AN_EVENT_STAT(sc->mpi350)) & AN_EV_ALLOC)
 			break;
 	}
 
 	if (i == AN_TIMEOUT)
 		return(ETIMEDOUT);
 
 	CSR_WRITE_2(sc, AN_EVENT_ACK(sc->mpi350), AN_EV_ALLOC);
 	*id = CSR_READ_2(sc, AN_ALLOC_FID);
 
 	if (an_seek(sc, *id, 0, AN_BAP0))
 		return(EIO);
 
 	for (i = 0; i < len / 2; i++)
 		CSR_WRITE_2(sc, AN_DATA0, 0);
 
 	return(0);
 }
 
 static void
 an_setdef(struct an_softc *sc, struct an_req *areq)
 {
 	struct ifnet		*ifp;
 	struct an_ltv_genconfig	*cfg;
 	struct an_ltv_ssidlist_new	*ssid;
 	struct an_ltv_aplist	*ap;
 	struct an_ltv_gen	*sp;
 
 	ifp = sc->an_ifp;
 
 	AN_LOCK_ASSERT(sc);
 	switch (areq->an_type) {
 	case AN_RID_GENCONFIG:
 		cfg = (struct an_ltv_genconfig *)areq;
 
 		bcopy((char *)&cfg->an_macaddr, IF_LLADDR(sc->an_ifp),
 		    ETHER_ADDR_LEN);
 
 		bcopy((char *)cfg, (char *)&sc->an_config,
 			sizeof(struct an_ltv_genconfig));
 		break;
 	case AN_RID_SSIDLIST:
 		ssid = (struct an_ltv_ssidlist_new *)areq;
 		bcopy((char *)ssid, (char *)&sc->an_ssidlist,
 			sizeof(struct an_ltv_ssidlist_new));
 		break;
 	case AN_RID_APLIST:
 		ap = (struct an_ltv_aplist *)areq;
 		bcopy((char *)ap, (char *)&sc->an_aplist,
 			sizeof(struct an_ltv_aplist));
 		break;
 	case AN_RID_TX_SPEED:
 		sp = (struct an_ltv_gen *)areq;
 		sc->an_tx_rate = sp->an_val;
 
 		/* Read the current configuration */
 		sc->an_config.an_type = AN_RID_GENCONFIG;
 		sc->an_config.an_len = sizeof(struct an_ltv_genconfig);
 		an_read_record(sc, (struct an_ltv_gen *)&sc->an_config);
 		cfg = &sc->an_config;
 
 		/* clear other rates and set the only one we want */
 		bzero(cfg->an_rates, sizeof(cfg->an_rates));
 		cfg->an_rates[0] = sc->an_tx_rate;
 
 		/* Save the new rate */
 		sc->an_config.an_type = AN_RID_GENCONFIG;
 		sc->an_config.an_len = sizeof(struct an_ltv_genconfig);
 		break;
 	case AN_RID_WEP_TEMP:
 		/* Cache the temp keys */
 		bcopy(areq,
 		    &sc->an_temp_keys[((struct an_ltv_key *)areq)->kindex],
 		    sizeof(struct an_ltv_key));
 	case AN_RID_WEP_PERM:
 	case AN_RID_LEAPUSERNAME:
 	case AN_RID_LEAPPASSWORD:
 		an_init_locked(sc);
 
 		/* Disable the MAC. */
 		an_cmd(sc, AN_CMD_DISABLE, 0);
 
 		/* Write the key */
 		an_write_record(sc, (struct an_ltv_gen *)areq);
 
 		/* Turn the MAC back on. */
 		an_cmd(sc, AN_CMD_ENABLE, 0);
 
 		break;
 	case AN_RID_MONITOR_MODE:
 		cfg = (struct an_ltv_genconfig *)areq;
 		bpfdetach(ifp);
 		if (ng_ether_detach_p != NULL)
 			(*ng_ether_detach_p) (ifp);
 		sc->an_monitor = cfg->an_len;
 
 		if (sc->an_monitor & AN_MONITOR) {
 			if (sc->an_monitor & AN_MONITOR_AIRONET_HEADER) {
 				bpfattach(ifp, DLT_AIRONET_HEADER,
 					sizeof(struct ether_header));
 			} else {
 				bpfattach(ifp, DLT_IEEE802_11,
 					sizeof(struct ether_header));
 			}
 		} else {
 			bpfattach(ifp, DLT_EN10MB,
 				  sizeof(struct ether_header));
 			if (ng_ether_attach_p != NULL)
 				(*ng_ether_attach_p) (ifp);
 		}
 		break;
 	default:
 		if_printf(ifp, "unknown RID: %x\n", areq->an_type);
 		return;
 	}
 
 
 	/* Reinitialize the card. */
 	if (ifp->if_flags)
 		an_init_locked(sc);
 
 	return;
 }
 
 /*
  * Derived from Linux driver to enable promiscious mode.
  */
 
 static void
 an_promisc(struct an_softc *sc, int promisc)
 {
 	AN_LOCK_ASSERT(sc);
 	if (sc->an_was_monitor) {
 		an_reset(sc);
 		if (sc->mpi350)
 			an_init_mpi350_desc(sc);
 	}
 	if (sc->an_monitor || sc->an_was_monitor)
 		an_init_locked(sc);
 
 	sc->an_was_monitor = sc->an_monitor;
 	an_cmd(sc, AN_CMD_SET_MODE, promisc ? 0xffff : 0);
 
 	return;
 }
 
 static int
 an_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
 {
 	int			error = 0;
 	int			len;
 	int			i, max;
 	struct an_softc		*sc;
 	struct ifreq		*ifr;
 	struct thread		*td = curthread;
 	struct ieee80211req	*ireq;
 	struct ieee80211_channel	ch;
 	u_int8_t		tmpstr[IEEE80211_NWID_LEN*2];
 	u_int8_t		*tmpptr;
 	struct an_ltv_genconfig	*config;
 	struct an_ltv_key	*key;
 	struct an_ltv_status	*status;
 	struct an_ltv_ssidlist_new	*ssids;
 	int			mode;
 	struct aironet_ioctl	l_ioctl;
 
 	sc = ifp->if_softc;
 	ifr = (struct ifreq *)data;
 	ireq = (struct ieee80211req *)data;
 
 	config = (struct an_ltv_genconfig *)&sc->areq;
 	key = (struct an_ltv_key *)&sc->areq;
 	status = (struct an_ltv_status *)&sc->areq;
 	ssids = (struct an_ltv_ssidlist_new *)&sc->areq;
 
 	if (sc->an_gone) {
 		error = ENODEV;
 		goto out;
 	}
 
 	switch (command) {
 	case SIOCSIFFLAGS:
 		AN_LOCK(sc);
 		if (ifp->if_flags & IFF_UP) {
 			if (ifp->if_drv_flags & IFF_DRV_RUNNING &&
 			    ifp->if_flags & IFF_PROMISC &&
 			    !(sc->an_if_flags & IFF_PROMISC)) {
 				an_promisc(sc, 1);
 			} else if (ifp->if_drv_flags & IFF_DRV_RUNNING &&
 			    !(ifp->if_flags & IFF_PROMISC) &&
 			    sc->an_if_flags & IFF_PROMISC) {
 				an_promisc(sc, 0);
 			} else
 				an_init_locked(sc);
 		} else {
 			if (ifp->if_drv_flags & IFF_DRV_RUNNING)
 				an_stop(sc);
 		}
 		sc->an_if_flags = ifp->if_flags;
 		AN_UNLOCK(sc);
 		error = 0;
 		break;
 	case SIOCSIFMEDIA:
 	case SIOCGIFMEDIA:
 		error = ifmedia_ioctl(ifp, ifr, &sc->an_ifmedia, command);
 		break;
 	case SIOCADDMULTI:
 	case SIOCDELMULTI:
 		/* The Aironet has no multicast filter. */
 		error = 0;
 		break;
 	case SIOCGAIRONET:
 		error = copyin(ifr->ifr_data, &sc->areq, sizeof(sc->areq));
 		if (error != 0)
 			break;
 		AN_LOCK(sc);
 #ifdef ANCACHE
 		if (sc->areq.an_type == AN_RID_ZERO_CACHE) {
 			error = priv_check(td, PRIV_DRIVER);
 			if (error)
 				break;
 			sc->an_sigitems = sc->an_nextitem = 0;
 			break;
 		} else if (sc->areq.an_type == AN_RID_READ_CACHE) {
 			char *pt = (char *)&sc->areq.an_val;
 			bcopy((char *)&sc->an_sigitems, (char *)pt,
 			    sizeof(int));
 			pt += sizeof(int);
 			sc->areq.an_len = sizeof(int) / 2;
 			bcopy((char *)&sc->an_sigcache, (char *)pt,
 			    sizeof(struct an_sigcache) * sc->an_sigitems);
 			sc->areq.an_len += ((sizeof(struct an_sigcache) *
 			    sc->an_sigitems) / 2) + 1;
 		} else
 #endif
 		if (an_read_record(sc, (struct an_ltv_gen *)&sc->areq)) {
 			AN_UNLOCK(sc);
 			error = EINVAL;
 			break;
 		}
 		AN_UNLOCK(sc);
 		error = copyout(&sc->areq, ifr->ifr_data, sizeof(sc->areq));
 		break;
 	case SIOCSAIRONET:
 		if ((error = priv_check(td, PRIV_DRIVER)))
 			goto out;
 		AN_LOCK(sc);
 		error = copyin(ifr->ifr_data, &sc->areq, sizeof(sc->areq));
 		if (error != 0)
 			break;
 		an_setdef(sc, &sc->areq);
 		AN_UNLOCK(sc);
 		break;
 	case SIOCGPRIVATE_0:		/* used by Cisco client utility */
 		if ((error = priv_check(td, PRIV_DRIVER)))
 			goto out;
 		error = copyin(ifr->ifr_data, &l_ioctl, sizeof(l_ioctl));
 		if (error)
 			goto out;
 		mode = l_ioctl.command;
 
 		AN_LOCK(sc);
 		if (mode >= AIROGCAP && mode <= AIROGSTATSD32) {
 			error = readrids(ifp, &l_ioctl);
 		} else if (mode >= AIROPCAP && mode <= AIROPLEAPUSR) {
 			error = writerids(ifp, &l_ioctl);
 		} else if (mode >= AIROFLSHRST && mode <= AIRORESTART) {
 			error = flashcard(ifp, &l_ioctl);
 		} else {
 			error =-1;
 		}
 		AN_UNLOCK(sc);
 		if (!error) {
 			/* copy out the updated command info */
 			error = copyout(&l_ioctl, ifr->ifr_data, sizeof(l_ioctl));
 		}
 		break;
 	case SIOCGPRIVATE_1:		/* used by Cisco client utility */
 		if ((error = priv_check(td, PRIV_DRIVER)))
 			goto out;
 		error = copyin(ifr->ifr_data, &l_ioctl, sizeof(l_ioctl));
 		if (error)
 			goto out;
 		l_ioctl.command = 0;
 		error = AIROMAGIC;
 		(void) copyout(&error, l_ioctl.data, sizeof(error));
 		error = 0;
 		break;
 	case SIOCG80211:
 		sc->areq.an_len = sizeof(sc->areq);
 		/* was that a good idea DJA we are doing a short-cut */
 		switch (ireq->i_type) {
 		case IEEE80211_IOC_SSID:
 			AN_LOCK(sc);
 			if (ireq->i_val == -1) {
 				sc->areq.an_type = AN_RID_STATUS;
 				if (an_read_record(sc,
 				    (struct an_ltv_gen *)&sc->areq)) {
 					error = EINVAL;
 					AN_UNLOCK(sc);
 					break;
 				}
 				len = status->an_ssidlen;
 				tmpptr = status->an_ssid;
 			} else if (ireq->i_val >= 0) {
 				sc->areq.an_type = AN_RID_SSIDLIST;
 				if (an_read_record(sc,
 				    (struct an_ltv_gen *)&sc->areq)) {
 					error = EINVAL;
 					AN_UNLOCK(sc);
 					break;
 				}
 				max = (sc->areq.an_len - 4)
 				    / sizeof(struct an_ltv_ssid_entry);
 				if ( max > MAX_SSIDS ) {
 					printf("To many SSIDs only using "
 					    "%d of %d\n",
 					    MAX_SSIDS, max);
 					max = MAX_SSIDS;
 				}
 				if (ireq->i_val > max) {
 					error = EINVAL;
 					AN_UNLOCK(sc);
 					break;
 				} else {
 					len = ssids->an_entry[ireq->i_val].an_len;
 					tmpptr = ssids->an_entry[ireq->i_val].an_ssid;
 				}
 			} else {
 				error = EINVAL;
 				AN_UNLOCK(sc);
 				break;
 			}
 			if (len > IEEE80211_NWID_LEN) {
 				error = EINVAL;
 				AN_UNLOCK(sc);
 				break;
 			}
 			AN_UNLOCK(sc);
 			ireq->i_len = len;
 			bzero(tmpstr, IEEE80211_NWID_LEN);
 			bcopy(tmpptr, tmpstr, len);
 			error = copyout(tmpstr, ireq->i_data,
 			    IEEE80211_NWID_LEN);
 			break;
 		case IEEE80211_IOC_NUMSSIDS:
 			AN_LOCK(sc);
 			sc->areq.an_len = sizeof(sc->areq);
 			sc->areq.an_type = AN_RID_SSIDLIST;
 			if (an_read_record(sc,
 			    (struct an_ltv_gen *)&sc->areq)) {
 				AN_UNLOCK(sc);
 				error = EINVAL;
 				break;
 			}
 			max = (sc->areq.an_len - 4)
 			    / sizeof(struct an_ltv_ssid_entry);
 			AN_UNLOCK(sc);
 			if ( max > MAX_SSIDS ) {
 				printf("To many SSIDs only using "
 				    "%d of %d\n",
 				    MAX_SSIDS, max);
 				max = MAX_SSIDS;
 			}
 			ireq->i_val = max;
 			break;
 		case IEEE80211_IOC_WEP:
 			AN_LOCK(sc);
 			sc->areq.an_type = AN_RID_ACTUALCFG;
 			if (an_read_record(sc,
 			    (struct an_ltv_gen *)&sc->areq)) {
 				error = EINVAL;
 				AN_UNLOCK(sc);
 				break;
 			}
 			AN_UNLOCK(sc);
 			if (config->an_authtype & AN_AUTHTYPE_PRIVACY_IN_USE) {
 				if (config->an_authtype &
 				    AN_AUTHTYPE_ALLOW_UNENCRYPTED)
 					ireq->i_val = IEEE80211_WEP_MIXED;
 				else
 					ireq->i_val = IEEE80211_WEP_ON;
 			} else {
 				ireq->i_val = IEEE80211_WEP_OFF;
 			}
 			break;
 		case IEEE80211_IOC_WEPKEY:
 			/*
 			 * XXX: I'm not entierly convinced this is
 			 * correct, but it's what is implemented in
 			 * ancontrol so it will have to do until we get
 			 * access to actual Cisco code.
 			 */
 			if (ireq->i_val < 0 || ireq->i_val > 8) {
 				error = EINVAL;
 				break;
 			}
 			len = 0;
 			if (ireq->i_val < 5) {
 				AN_LOCK(sc);
 				sc->areq.an_type = AN_RID_WEP_TEMP;
 				for (i = 0; i < 5; i++) {
 					if (an_read_record(sc,
 					    (struct an_ltv_gen *)&sc->areq)) {
 						error = EINVAL;
 						break;
 					}
 					if (key->kindex == 0xffff)
 						break;
 					if (key->kindex == ireq->i_val)
 						len = key->klen;
 					/* Required to get next entry */
 					sc->areq.an_type = AN_RID_WEP_PERM;
 				}
 				AN_UNLOCK(sc);
 				if (error != 0) {
 					break;
 				}
 			}
 			/* We aren't allowed to read the value of the
 			 * key from the card so we just output zeros
 			 * like we would if we could read the card, but
 			 * denied the user access.
 			 */
 			bzero(tmpstr, len);
 			ireq->i_len = len;
 			error = copyout(tmpstr, ireq->i_data, len);
 			break;
 		case IEEE80211_IOC_NUMWEPKEYS:
 			ireq->i_val = 9; /* include home key */
 			break;
 		case IEEE80211_IOC_WEPTXKEY:
 			/*
 			 * For some strange reason, you have to read all
 			 * keys before you can read the txkey.
 			 */
 			AN_LOCK(sc);
 			sc->areq.an_type = AN_RID_WEP_TEMP;
 			for (i = 0; i < 5; i++) {
 				if (an_read_record(sc,
 				    (struct an_ltv_gen *) &sc->areq)) {
 					error = EINVAL;
 					break;
 				}
 				if (key->kindex == 0xffff) {
 					break;
 				}
 				/* Required to get next entry */
 				sc->areq.an_type = AN_RID_WEP_PERM;
 			}
 			if (error != 0) {
 				AN_UNLOCK(sc);
 				break;
 			}
 
 			sc->areq.an_type = AN_RID_WEP_PERM;
 			key->kindex = 0xffff;
 			if (an_read_record(sc,
 			    (struct an_ltv_gen *)&sc->areq)) {
 				error = EINVAL;
 				AN_UNLOCK(sc);
 				break;
 			}
 			ireq->i_val = key->mac[0];
 			/*
 			 * Check for home mode.  Map home mode into
 			 * 5th key since that is how it is stored on
 			 * the card
 			 */
 			sc->areq.an_len  = sizeof(struct an_ltv_genconfig);
 			sc->areq.an_type = AN_RID_GENCONFIG;
 			if (an_read_record(sc,
 			    (struct an_ltv_gen *)&sc->areq)) {
 				error = EINVAL;
 				AN_UNLOCK(sc);
 				break;
 			}
 			if (config->an_home_product & AN_HOME_NETWORK)
 				ireq->i_val = 4;
 			AN_UNLOCK(sc);
 			break;
 		case IEEE80211_IOC_AUTHMODE:
 			AN_LOCK(sc);
 			sc->areq.an_type = AN_RID_ACTUALCFG;
 			if (an_read_record(sc,
 			    (struct an_ltv_gen *)&sc->areq)) {
 				error = EINVAL;
 				AN_UNLOCK(sc);
 				break;
 			}
 			AN_UNLOCK(sc);
 			if ((config->an_authtype & AN_AUTHTYPE_MASK) ==
 			    AN_AUTHTYPE_NONE) {
 			    ireq->i_val = IEEE80211_AUTH_NONE;
 			} else if ((config->an_authtype & AN_AUTHTYPE_MASK) ==
 			    AN_AUTHTYPE_OPEN) {
 			    ireq->i_val = IEEE80211_AUTH_OPEN;
 			} else if ((config->an_authtype & AN_AUTHTYPE_MASK) ==
 			    AN_AUTHTYPE_SHAREDKEY) {
 			    ireq->i_val = IEEE80211_AUTH_SHARED;
 			} else
 				error = EINVAL;
 			break;
 		case IEEE80211_IOC_STATIONNAME:
 			AN_LOCK(sc);
 			sc->areq.an_type = AN_RID_ACTUALCFG;
 			if (an_read_record(sc,
 			    (struct an_ltv_gen *)&sc->areq)) {
 				error = EINVAL;
 				AN_UNLOCK(sc);
 				break;
 			}
 			AN_UNLOCK(sc);
 			ireq->i_len = sizeof(config->an_nodename);
 			tmpptr = config->an_nodename;
 			bzero(tmpstr, IEEE80211_NWID_LEN);
 			bcopy(tmpptr, tmpstr, ireq->i_len);
 			error = copyout(tmpstr, ireq->i_data,
 			    IEEE80211_NWID_LEN);
 			break;
 		case IEEE80211_IOC_CHANNEL:
 			AN_LOCK(sc);
 			sc->areq.an_type = AN_RID_STATUS;
 			if (an_read_record(sc,
 			    (struct an_ltv_gen *)&sc->areq)) {
 				error = EINVAL;
 				AN_UNLOCK(sc);
 				break;
 			}
 			AN_UNLOCK(sc);
 			ireq->i_val = status->an_cur_channel;
 			break;
 		case IEEE80211_IOC_CURCHAN:
 			AN_LOCK(sc);
 			sc->areq.an_type = AN_RID_STATUS;
 			if (an_read_record(sc,
 			    (struct an_ltv_gen *)&sc->areq)) {
 				error = EINVAL;
 				AN_UNLOCK(sc);
 				break;
 			}
 			AN_UNLOCK(sc);
 			bzero(&ch, sizeof(ch));
 			ch.ic_freq = ieee80211_ieee2mhz(status->an_cur_channel,
 			    IEEE80211_CHAN_B);
 			ch.ic_flags = IEEE80211_CHAN_B;
 			ch.ic_ieee = status->an_cur_channel;
 			error = copyout(&ch, ireq->i_data, sizeof(ch));
 			break;
 		case IEEE80211_IOC_POWERSAVE:
 			AN_LOCK(sc);
 			sc->areq.an_type = AN_RID_ACTUALCFG;
 			if (an_read_record(sc,
 			    (struct an_ltv_gen *)&sc->areq)) {
 				error = EINVAL;
 				AN_UNLOCK(sc);
 				break;
 			}
 			AN_UNLOCK(sc);
 			if (config->an_psave_mode == AN_PSAVE_NONE) {
 				ireq->i_val = IEEE80211_POWERSAVE_OFF;
 			} else if (config->an_psave_mode == AN_PSAVE_CAM) {
 				ireq->i_val = IEEE80211_POWERSAVE_CAM;
 			} else if (config->an_psave_mode == AN_PSAVE_PSP) {
 				ireq->i_val = IEEE80211_POWERSAVE_PSP;
 			} else if (config->an_psave_mode == AN_PSAVE_PSP_CAM) {
 				ireq->i_val = IEEE80211_POWERSAVE_PSP_CAM;
 			} else
 				error = EINVAL;
 			break;
 		case IEEE80211_IOC_POWERSAVESLEEP:
 			AN_LOCK(sc);
 			sc->areq.an_type = AN_RID_ACTUALCFG;
 			if (an_read_record(sc,
 			    (struct an_ltv_gen *)&sc->areq)) {
 				error = EINVAL;
 				AN_UNLOCK(sc);
 				break;
 			}
 			AN_UNLOCK(sc);
 			ireq->i_val = config->an_listen_interval;
 			break;
 		}
 		break;
 	case SIOCS80211:
 		if ((error = priv_check(td, PRIV_NET80211_MANAGE)))
 			goto out;
 		AN_LOCK(sc);
 		sc->areq.an_len = sizeof(sc->areq);
 		/*
 		 * We need a config structure for everything but the WEP
 		 * key management and SSIDs so we get it now so avoid
 		 * duplicating this code every time.
 		 */
 		if (ireq->i_type != IEEE80211_IOC_SSID &&
 		    ireq->i_type != IEEE80211_IOC_WEPKEY &&
 		    ireq->i_type != IEEE80211_IOC_WEPTXKEY) {
 			sc->areq.an_type = AN_RID_GENCONFIG;
 			if (an_read_record(sc,
 			    (struct an_ltv_gen *)&sc->areq)) {
 				error = EINVAL;
 				AN_UNLOCK(sc);
 				break;
 			}
 		}
 		switch (ireq->i_type) {
 		case IEEE80211_IOC_SSID:
 			sc->areq.an_len = sizeof(sc->areq);
 			sc->areq.an_type = AN_RID_SSIDLIST;
 			if (an_read_record(sc,
 			    (struct an_ltv_gen *)&sc->areq)) {
 				error = EINVAL;
 				AN_UNLOCK(sc);
 				break;
 			}
 			if (ireq->i_len > IEEE80211_NWID_LEN) {
 				error = EINVAL;
 				AN_UNLOCK(sc);
 				break;
 			}
 			max = (sc->areq.an_len - 4)
 			    / sizeof(struct an_ltv_ssid_entry);
 			if ( max > MAX_SSIDS ) {
 				printf("To many SSIDs only using "
 				    "%d of %d\n",
 				    MAX_SSIDS, max);
 				max = MAX_SSIDS;
 			}
 			if (ireq->i_val > max) {
 				error = EINVAL;
 				AN_UNLOCK(sc);
 				break;
 			} else {
 				error = copyin(ireq->i_data,
 				    ssids->an_entry[ireq->i_val].an_ssid,
 				    ireq->i_len);
 				ssids->an_entry[ireq->i_val].an_len
 				    = ireq->i_len;
 				sc->areq.an_len = sizeof(sc->areq);
 				sc->areq.an_type = AN_RID_SSIDLIST;
 				an_setdef(sc, &sc->areq);
 				AN_UNLOCK(sc);
 				break;
 			}
 			break;
 		case IEEE80211_IOC_WEP:
 			switch (ireq->i_val) {
 			case IEEE80211_WEP_OFF:
 				config->an_authtype &=
 				    ~(AN_AUTHTYPE_PRIVACY_IN_USE |
 				    AN_AUTHTYPE_ALLOW_UNENCRYPTED);
 				break;
 			case IEEE80211_WEP_ON:
 				config->an_authtype |=
 				    AN_AUTHTYPE_PRIVACY_IN_USE;
 				config->an_authtype &=
 				    ~AN_AUTHTYPE_ALLOW_UNENCRYPTED;
 				break;
 			case IEEE80211_WEP_MIXED:
 				config->an_authtype |=
 				    AN_AUTHTYPE_PRIVACY_IN_USE |
 				    AN_AUTHTYPE_ALLOW_UNENCRYPTED;
 				break;
 			default:
 				error = EINVAL;
 				break;
 			}
 			if (error != EINVAL)
 				an_setdef(sc, &sc->areq);
 			AN_UNLOCK(sc);
 			break;
 		case IEEE80211_IOC_WEPKEY:
 			if (ireq->i_val < 0 || ireq->i_val > 8 ||
 			    ireq->i_len > 13) {
 				error = EINVAL;
 				AN_UNLOCK(sc);
 				break;
 			}
 			error = copyin(ireq->i_data, tmpstr, 13);
 			if (error != 0) {
 				AN_UNLOCK(sc);
 				break;
 			}
 			/*
 			 * Map the 9th key into the home mode
 			 * since that is how it is stored on
 			 * the card
 			 */
 			bzero(&sc->areq, sizeof(struct an_ltv_key));
 			sc->areq.an_len = sizeof(struct an_ltv_key);
 			key->mac[0] = 1;	/* The others are 0. */
 			if (ireq->i_val < 4) {
 				sc->areq.an_type = AN_RID_WEP_TEMP;
 				key->kindex = ireq->i_val;
 			} else {
 				sc->areq.an_type = AN_RID_WEP_PERM;
 				key->kindex = ireq->i_val - 4;
 			}
 			key->klen = ireq->i_len;
 			bcopy(tmpstr, key->key, key->klen);
 			an_setdef(sc, &sc->areq);
 			AN_UNLOCK(sc);
 			break;
 		case IEEE80211_IOC_WEPTXKEY:
 			if (ireq->i_val < 0 || ireq->i_val > 4) {
 				error = EINVAL;
 				AN_UNLOCK(sc);
 				break;
 			}
 
 			/*
 			 * Map the 5th key into the home mode
 			 * since that is how it is stored on
 			 * the card
 			 */
 			sc->areq.an_len  = sizeof(struct an_ltv_genconfig);
 			sc->areq.an_type = AN_RID_ACTUALCFG;
 			if (an_read_record(sc,
 			    (struct an_ltv_gen *)&sc->areq)) {
 				error = EINVAL;
 				AN_UNLOCK(sc);
 				break;
 			}
 			if (ireq->i_val ==  4) {
 				config->an_home_product |= AN_HOME_NETWORK;
 				ireq->i_val = 0;
 			} else {
 				config->an_home_product &= ~AN_HOME_NETWORK;
 			}
 
 			sc->an_config.an_home_product
 				= config->an_home_product;
 
 			/* update configuration */
 			an_init_locked(sc);
 
 			bzero(&sc->areq, sizeof(struct an_ltv_key));
 			sc->areq.an_len = sizeof(struct an_ltv_key);
 			sc->areq.an_type = AN_RID_WEP_PERM;
 			key->kindex = 0xffff;
 			key->mac[0] = ireq->i_val;
 			an_setdef(sc, &sc->areq);
 			AN_UNLOCK(sc);
 			break;
 		case IEEE80211_IOC_AUTHMODE:
 			switch (ireq->i_val) {
 			case IEEE80211_AUTH_NONE:
 				config->an_authtype = AN_AUTHTYPE_NONE |
 				    (config->an_authtype & ~AN_AUTHTYPE_MASK);
 				break;
 			case IEEE80211_AUTH_OPEN:
 				config->an_authtype = AN_AUTHTYPE_OPEN |
 				    (config->an_authtype & ~AN_AUTHTYPE_MASK);
 				break;
 			case IEEE80211_AUTH_SHARED:
 				config->an_authtype = AN_AUTHTYPE_SHAREDKEY |
 				    (config->an_authtype & ~AN_AUTHTYPE_MASK);
 				break;
 			default:
 				error = EINVAL;
 			}
 			if (error != EINVAL) {
 				an_setdef(sc, &sc->areq);
 			}
 			AN_UNLOCK(sc);
 			break;
 		case IEEE80211_IOC_STATIONNAME:
 			if (ireq->i_len > 16) {
 				error = EINVAL;
 				AN_UNLOCK(sc);
 				break;
 			}
 			bzero(config->an_nodename, 16);
 			error = copyin(ireq->i_data,
 			    config->an_nodename, ireq->i_len);
 			an_setdef(sc, &sc->areq);
 			AN_UNLOCK(sc);
 			break;
 		case IEEE80211_IOC_CHANNEL:
 			/*
 			 * The actual range is 1-14, but if you set it
 			 * to 0 you get the default so we let that work
 			 * too.
 			 */
 			if (ireq->i_val < 0 || ireq->i_val >14) {
 				error = EINVAL;
 				AN_UNLOCK(sc);
 				break;
 			}
 			config->an_ds_channel = ireq->i_val;
 			an_setdef(sc, &sc->areq);
 			AN_UNLOCK(sc);
 			break;
 		case IEEE80211_IOC_POWERSAVE:
 			switch (ireq->i_val) {
 			case IEEE80211_POWERSAVE_OFF:
 				config->an_psave_mode = AN_PSAVE_NONE;
 				break;
 			case IEEE80211_POWERSAVE_CAM:
 				config->an_psave_mode = AN_PSAVE_CAM;
 				break;
 			case IEEE80211_POWERSAVE_PSP:
 				config->an_psave_mode = AN_PSAVE_PSP;
 				break;
 			case IEEE80211_POWERSAVE_PSP_CAM:
 				config->an_psave_mode = AN_PSAVE_PSP_CAM;
 				break;
 			default:
 				error = EINVAL;
 				break;
 			}
 			an_setdef(sc, &sc->areq);
 			AN_UNLOCK(sc);
 			break;
 		case IEEE80211_IOC_POWERSAVESLEEP:
 			config->an_listen_interval = ireq->i_val;
 			an_setdef(sc, &sc->areq);
 			AN_UNLOCK(sc);
 			break;
 		default:
 			AN_UNLOCK(sc);
 			break;
 		}
 
 		/*
 		if (!error) {
 			AN_LOCK(sc);
 			an_setdef(sc, &sc->areq);
 			AN_UNLOCK(sc);
 		}
 		*/
 		break;
 	default:
 		error = ether_ioctl(ifp, command, data);
 		break;
 	}
 out:
 
 	return(error != 0);
 }
 
 static int
 an_init_tx_ring(struct an_softc *sc)
 {
 	int			i;
 	int			id;
 
 	if (sc->an_gone)
 		return (0);
 
 	if (!sc->mpi350) {
 		for (i = 0; i < AN_TX_RING_CNT; i++) {
 			if (an_alloc_nicmem(sc, 1518 +
 			    0x44, &id))
 				return(ENOMEM);
 			sc->an_rdata.an_tx_fids[i] = id;
 			sc->an_rdata.an_tx_ring[i] = 0;
 		}
 	}
 
 	sc->an_rdata.an_tx_prod = 0;
 	sc->an_rdata.an_tx_cons = 0;
 	sc->an_rdata.an_tx_empty = 1;
 
 	return(0);
 }
 
 static void
 an_init(void *xsc)
 {
 	struct an_softc		*sc = xsc;
 
 	AN_LOCK(sc);
 	an_init_locked(sc);
 	AN_UNLOCK(sc);
 }
 
 static void
 an_init_locked(struct an_softc *sc)
 {
 	struct ifnet *ifp;
 
 	AN_LOCK_ASSERT(sc);
 	ifp = sc->an_ifp;
 	if (sc->an_gone)
 		return;
 
 	if (ifp->if_drv_flags & IFF_DRV_RUNNING)
 		an_stop(sc);
 
 	sc->an_associated = 0;
 
 	/* Allocate the TX buffers */
 	if (an_init_tx_ring(sc)) {
 		an_reset(sc);
 		if (sc->mpi350)
 			an_init_mpi350_desc(sc);
 		if (an_init_tx_ring(sc)) {
 			if_printf(ifp, "tx buffer allocation failed\n");
 			return;
 		}
 	}
 
 	/* Set our MAC address. */
 	bcopy((char *)IF_LLADDR(sc->an_ifp),
 	    (char *)&sc->an_config.an_macaddr, ETHER_ADDR_LEN);
 
 	if (ifp->if_flags & IFF_BROADCAST)
 		sc->an_config.an_rxmode = AN_RXMODE_BC_ADDR;
 	else
 		sc->an_config.an_rxmode = AN_RXMODE_ADDR;
 
 	if (ifp->if_flags & IFF_MULTICAST)
 		sc->an_config.an_rxmode = AN_RXMODE_BC_MC_ADDR;
 
 	if (ifp->if_flags & IFF_PROMISC) {
 		if (sc->an_monitor & AN_MONITOR) {
 			if (sc->an_monitor & AN_MONITOR_ANY_BSS) {
 				sc->an_config.an_rxmode |=
 				    AN_RXMODE_80211_MONITOR_ANYBSS |
 				    AN_RXMODE_NO_8023_HEADER;
 			} else {
 				sc->an_config.an_rxmode |=
 				    AN_RXMODE_80211_MONITOR_CURBSS |
 				    AN_RXMODE_NO_8023_HEADER;
 			}
 		}
 	}
 
 #ifdef ANCACHE
 	if (sc->an_have_rssimap)
 		sc->an_config.an_rxmode |= AN_RXMODE_NORMALIZED_RSSI;
 #endif
 
 	/* Set the ssid list */
 	sc->an_ssidlist.an_type = AN_RID_SSIDLIST;
 	sc->an_ssidlist.an_len = sizeof(struct an_ltv_ssidlist_new);
 	if (an_write_record(sc, (struct an_ltv_gen *)&sc->an_ssidlist)) {
 		if_printf(ifp, "failed to set ssid list\n");
 		return;
 	}
 
 	/* Set the AP list */
 	sc->an_aplist.an_type = AN_RID_APLIST;
 	sc->an_aplist.an_len = sizeof(struct an_ltv_aplist);
 	if (an_write_record(sc, (struct an_ltv_gen *)&sc->an_aplist)) {
 		if_printf(ifp, "failed to set AP list\n");
 		return;
 	}
 
 	/* Set the configuration in the NIC */
 	sc->an_config.an_len = sizeof(struct an_ltv_genconfig);
 	sc->an_config.an_type = AN_RID_GENCONFIG;
 	if (an_write_record(sc, (struct an_ltv_gen *)&sc->an_config)) {
 		if_printf(ifp, "failed to set configuration\n");
 		return;
 	}
 
 	/* Enable the MAC */
 	if (an_cmd(sc, AN_CMD_ENABLE, 0)) {
 		if_printf(ifp, "failed to enable MAC\n");
 		return;
 	}
 
 	if (ifp->if_flags & IFF_PROMISC)
 		an_cmd(sc, AN_CMD_SET_MODE, 0xffff);
 
 	/* enable interrupts */
 	CSR_WRITE_2(sc, AN_INT_EN(sc->mpi350), AN_INTRS(sc->mpi350));
 
 	ifp->if_drv_flags |= IFF_DRV_RUNNING;
 	ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
 
 	callout_reset(&sc->an_stat_ch, hz, an_stats_update, sc);
 
 	return;
 }
 
 static void
 an_start(struct ifnet *ifp)
 {
 	struct an_softc		*sc;
 
 	sc = ifp->if_softc;
 	AN_LOCK(sc);
 	an_start_locked(ifp);
 	AN_UNLOCK(sc);
 }
 
 static void
 an_start_locked(struct ifnet *ifp)
 {
 	struct an_softc		*sc;
 	struct mbuf		*m0 = NULL;
 	struct an_txframe_802_3	tx_frame_802_3;
 	struct ether_header	*eh;
 	int			id, idx, i;
 	unsigned char		txcontrol;
 	struct an_card_tx_desc an_tx_desc;
 	u_int8_t		*buf;
 
 	sc = ifp->if_softc;
 
 	AN_LOCK_ASSERT(sc);
 	if (sc->an_gone)
 		return;
 
 	if (ifp->if_drv_flags & IFF_DRV_OACTIVE)
 		return;
 
 	if (!sc->an_associated)
 		return;
 
 	/* We can't send in monitor mode so toss any attempts. */
 	if (sc->an_monitor && (ifp->if_flags & IFF_PROMISC)) {
 		for (;;) {
 			IFQ_DRV_DEQUEUE(&ifp->if_snd, m0);
 			if (m0 == NULL)
 				break;
 			m_freem(m0);
 		}
 		return;
 	}
 
 	idx = sc->an_rdata.an_tx_prod;
 
 	if (!sc->mpi350) {
 		bzero((char *)&tx_frame_802_3, sizeof(tx_frame_802_3));
 
 		while (sc->an_rdata.an_tx_ring[idx] == 0) {
 			IFQ_DRV_DEQUEUE(&ifp->if_snd, m0);
 			if (m0 == NULL)
 				break;
 
 			id = sc->an_rdata.an_tx_fids[idx];
 			eh = mtod(m0, struct ether_header *);
 
 			bcopy((char *)&eh->ether_dhost,
 			      (char *)&tx_frame_802_3.an_tx_dst_addr,
 			      ETHER_ADDR_LEN);
 			bcopy((char *)&eh->ether_shost,
 			      (char *)&tx_frame_802_3.an_tx_src_addr,
 			      ETHER_ADDR_LEN);
 
 			/* minus src/dest mac & type */
 			tx_frame_802_3.an_tx_802_3_payload_len =
 				m0->m_pkthdr.len - 12;
 
 			m_copydata(m0, sizeof(struct ether_header) - 2 ,
 				   tx_frame_802_3.an_tx_802_3_payload_len,
 				   (caddr_t)&sc->an_txbuf);
 
 			txcontrol = AN_TXCTL_8023 | AN_TXCTL_HW(sc->mpi350);
 			/* write the txcontrol only */
 			an_write_data(sc, id, 0x08, (caddr_t)&txcontrol,
 				      sizeof(txcontrol));
 
 			/* 802_3 header */
 			an_write_data(sc, id, 0x34, (caddr_t)&tx_frame_802_3,
 				      sizeof(struct an_txframe_802_3));
 
 			/* in mbuf header type is just before payload */
 			an_write_data(sc, id, 0x44, (caddr_t)&sc->an_txbuf,
 				      tx_frame_802_3.an_tx_802_3_payload_len);
 
 			/*
 			 * If there's a BPF listner, bounce a copy of
 			 * this frame to him.
 			 */
 			BPF_MTAP(ifp, m0);
 
 			m_freem(m0);
 			m0 = NULL;
 
 			sc->an_rdata.an_tx_ring[idx] = id;
 			if (an_cmd(sc, AN_CMD_TX, id))
 				if_printf(ifp, "xmit failed\n");
 
 			AN_INC(idx, AN_TX_RING_CNT);
 
 			/*
 			 * Set a timeout in case the chip goes out to lunch.
 			 */
 			sc->an_timer = 5;
 		}
 	} else { /* MPI-350 */
 		/* Disable interrupts. */
 		CSR_WRITE_2(sc, AN_INT_EN(sc->mpi350), 0);
 
 		while (sc->an_rdata.an_tx_empty ||
 		    idx != sc->an_rdata.an_tx_cons) {
 			IFQ_DRV_DEQUEUE(&ifp->if_snd, m0);
 			if (m0 == NULL) {
 				break;
 			}
 			buf = sc->an_tx_buffer[idx].an_dma_vaddr;
 
 			eh = mtod(m0, struct ether_header *);
 
 			/* DJA optimize this to limit bcopy */
 			bcopy((char *)&eh->ether_dhost,
 			      (char *)&tx_frame_802_3.an_tx_dst_addr,
 			      ETHER_ADDR_LEN);
 			bcopy((char *)&eh->ether_shost,
 			      (char *)&tx_frame_802_3.an_tx_src_addr,
 			      ETHER_ADDR_LEN);
 
 			/* minus src/dest mac & type */
 			tx_frame_802_3.an_tx_802_3_payload_len =
 				m0->m_pkthdr.len - 12;
 
 			m_copydata(m0, sizeof(struct ether_header) - 2 ,
 				   tx_frame_802_3.an_tx_802_3_payload_len,
 				   (caddr_t)&sc->an_txbuf);
 
 			txcontrol = AN_TXCTL_8023 | AN_TXCTL_HW(sc->mpi350);
 			/* write the txcontrol only */
 			bcopy((caddr_t)&txcontrol, &buf[0x08],
 			      sizeof(txcontrol));
 
 			/* 802_3 header */
 			bcopy((caddr_t)&tx_frame_802_3, &buf[0x34],
 			      sizeof(struct an_txframe_802_3));
 
 			/* in mbuf header type is just before payload */
 			bcopy((caddr_t)&sc->an_txbuf, &buf[0x44],
 			      tx_frame_802_3.an_tx_802_3_payload_len);
 
 
 			bzero(&an_tx_desc, sizeof(an_tx_desc));
 			an_tx_desc.an_offset = 0;
 			an_tx_desc.an_eoc = 1;
 			an_tx_desc.an_valid = 1;
 			an_tx_desc.an_len =  0x44 +
 			    tx_frame_802_3.an_tx_802_3_payload_len;
 			an_tx_desc.an_phys
 			    = sc->an_tx_buffer[idx].an_dma_paddr;
 			for (i = sizeof(an_tx_desc) / 4 - 1; i >= 0; i--) {
 				CSR_MEM_AUX_WRITE_4(sc, AN_TX_DESC_OFFSET
 				    /* zero for now */
 				    + (0 * sizeof(an_tx_desc))
 				    + (i * 4),
 				    ((u_int32_t *)(void *)&an_tx_desc)[i]);
 			}
 
 			/*
 			 * If there's a BPF listner, bounce a copy of
 			 * this frame to him.
 			 */
 			BPF_MTAP(ifp, m0);
 
 			m_freem(m0);
 			m0 = NULL;
 			AN_INC(idx, AN_MAX_TX_DESC);
 			sc->an_rdata.an_tx_empty = 0;
 			CSR_WRITE_2(sc, AN_EVENT_ACK(sc->mpi350), AN_EV_ALLOC);
 
 			/*
 			 * Set a timeout in case the chip goes out to lunch.
 			 */
 			sc->an_timer = 5;
 		}
 
 		/* Re-enable interrupts. */
 		CSR_WRITE_2(sc, AN_INT_EN(sc->mpi350), AN_INTRS(sc->mpi350));
 	}
 
 	if (m0 != NULL)
 		ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 
 	sc->an_rdata.an_tx_prod = idx;
 
 	return;
 }
 
 void
 an_stop(struct an_softc *sc)
 {
 	struct ifnet		*ifp;
 	int			i;
 
 	AN_LOCK_ASSERT(sc);
 
 	if (sc->an_gone)
 		return;
 
 	ifp = sc->an_ifp;
 
 	an_cmd(sc, AN_CMD_FORCE_SYNCLOSS, 0);
 	CSR_WRITE_2(sc, AN_INT_EN(sc->mpi350), 0);
 	an_cmd(sc, AN_CMD_DISABLE, 0);
 
 	for (i = 0; i < AN_TX_RING_CNT; i++)
 		an_cmd(sc, AN_CMD_DEALLOC_MEM, sc->an_rdata.an_tx_fids[i]);
 
 	callout_stop(&sc->an_stat_ch);
 
 	ifp->if_drv_flags &= ~(IFF_DRV_RUNNING|IFF_DRV_OACTIVE);
 
 	if (sc->an_flash_buffer) {
 		free(sc->an_flash_buffer, M_DEVBUF);
 		sc->an_flash_buffer = NULL;
 	}
 }
 
 static void
 an_watchdog(struct an_softc *sc)
 {
 	struct ifnet *ifp;
 
 	AN_LOCK_ASSERT(sc);
 
 	if (sc->an_gone)
 		return;
 
 	ifp = sc->an_ifp;
 	if_printf(ifp, "device timeout\n");
 
 	an_reset(sc);
 	if (sc->mpi350)
 		an_init_mpi350_desc(sc);
 	an_init_locked(sc);
 
 	ifp->if_oerrors++;
 }
 
 int
 an_shutdown(device_t dev)
 {
 	struct an_softc		*sc;
 
 	sc = device_get_softc(dev);
 	AN_LOCK(sc);
 	an_stop(sc);
 	sc->an_gone = 1;
 	AN_UNLOCK(sc);
 
 	return (0);
 }
 
 void
 an_resume(device_t dev)
 {
 	struct an_softc		*sc;
 	struct ifnet		*ifp;
 	int			i;
 
 	sc = device_get_softc(dev);
 	AN_LOCK(sc);
 	ifp = sc->an_ifp;
 
 	sc->an_gone = 0;
 	an_reset(sc);
 	if (sc->mpi350)
 		an_init_mpi350_desc(sc);
 	an_init_locked(sc);
 
 	/* Recovery temporary keys */
 	for (i = 0; i < 4; i++) {
 		sc->areq.an_type = AN_RID_WEP_TEMP;
 		sc->areq.an_len = sizeof(struct an_ltv_key);
 		bcopy(&sc->an_temp_keys[i],
 		    &sc->areq, sizeof(struct an_ltv_key));
 		an_setdef(sc, &sc->areq);
 	}
 
 	if (ifp->if_flags & IFF_UP)
 		an_start_locked(ifp);
 	AN_UNLOCK(sc);
 
 	return;
 }
 
 #ifdef ANCACHE
 /* Aironet signal strength cache code.
  * store signal/noise/quality on per MAC src basis in
  * a small fixed cache.  The cache wraps if > MAX slots
  * used.  The cache may be zeroed out to start over.
  * Two simple filters exist to reduce computation:
  * 1. ip only (literally 0x800, ETHERTYPE_IP) which may be used
  * to ignore some packets.  It defaults to ip only.
  * it could be used to focus on broadcast, non-IP 802.11 beacons.
  * 2. multicast/broadcast only.  This may be used to
  * ignore unicast packets and only cache signal strength
  * for multicast/broadcast packets (beacons); e.g., Mobile-IP
  * beacons and not unicast traffic.
  *
  * The cache stores (MAC src(index), IP src (major clue), signal,
  *	quality, noise)
  *
  * No apologies for storing IP src here.  It's easy and saves much
  * trouble elsewhere.  The cache is assumed to be INET dependent,
  * although it need not be.
  *
  * Note: the Aironet only has a single byte of signal strength value
  * in the rx frame header, and it's not scaled to anything sensible.
  * This is kind of lame, but it's all we've got.
  */
 
 #ifdef documentation
 
 int an_sigitems;				/* number of cached entries */
 struct an_sigcache an_sigcache[MAXANCACHE];	/* array of cache entries */
 int an_nextitem;				/* index/# of entries */
 
 
 #endif
 
 /* control variables for cache filtering.  Basic idea is
  * to reduce cost (e.g., to only Mobile-IP agent beacons
  * which are broadcast or multicast).  Still you might
  * want to measure signal strength anth unicast ping packets
  * on a pt. to pt. ant. setup.
  */
 /* set true if you want to limit cache items to broadcast/mcast
  * only packets (not unicast).  Useful for mobile-ip beacons which
  * are broadcast/multicast at network layer.  Default is all packets
  * so ping/unicast anll work say anth pt. to pt. antennae setup.
  */
 static int an_cache_mcastonly = 0;
 SYSCTL_INT(_hw_an, OID_AUTO, an_cache_mcastonly, CTLFLAG_RW,
 	&an_cache_mcastonly, 0, "");
 
 /* set true if you want to limit cache items to IP packets only
 */
 static int an_cache_iponly = 1;
 SYSCTL_INT(_hw_an, OID_AUTO, an_cache_iponly, CTLFLAG_RW,
 	&an_cache_iponly, 0, "");
 
 /*
  * an_cache_store, per rx packet store signal
  * strength in MAC (src) indexed cache.
  */
 static void
 an_cache_store(struct an_softc *sc, struct ether_header *eh, struct mbuf *m,
     u_int8_t rx_rssi, u_int8_t rx_quality)
 {
 	struct ip *ip = 0;
 	int i;
 	static int cache_slot = 0; 	/* use this cache entry */
 	static int wrapindex = 0;	/* next "free" cache entry */
 	int type_ipv4 = 0;
 
 	/* filters:
 	 * 1. ip only
 	 * 2. configurable filter to throw out unicast packets,
 	 * keep multicast only.
 	 */
 
 	if ((ntohs(eh->ether_type) == ETHERTYPE_IP)) {
 		type_ipv4 = 1;
 	}
 
 	/* filter for ip packets only
 	*/
 	if ( an_cache_iponly && !type_ipv4) {
 		return;
 	}
 
 	/* filter for broadcast/multicast only
 	 */
 	if (an_cache_mcastonly && ((eh->ether_dhost[0] & 1) == 0)) {
 		return;
 	}
 
 #ifdef SIGDEBUG
 	if_printf(sc->an_ifp, "q value %x (MSB=0x%x, LSB=0x%x) \n",
 		rx_rssi & 0xffff, rx_rssi >> 8, rx_rssi & 0xff);
 #endif
 
 	/* find the ip header.  we want to store the ip_src
 	 * address.
 	 */
 	if (type_ipv4) {
 		ip = mtod(m, struct ip *);
 	}
 
 	/* do a linear search for a matching MAC address
 	 * in the cache table
 	 * . MAC address is 6 bytes,
 	 * . var w_nextitem holds total number of entries already cached
 	 */
 	for (i = 0; i < sc->an_nextitem; i++) {
 		if (! bcmp(eh->ether_shost , sc->an_sigcache[i].macsrc,  6 )) {
 			/* Match!,
 			 * so we already have this entry,
 			 * update the data
 			 */
 			break;
 		}
 	}
 
 	/* did we find a matching mac address?
 	 * if yes, then overwrite a previously existing cache entry
 	 */
 	if (i < sc->an_nextitem )   {
 		cache_slot = i;
 	}
 	/* else, have a new address entry,so
 	 * add this new entry,
 	 * if table full, then we need to replace LRU entry
 	 */
 	else    {
 
 		/* check for space in cache table
 		 * note: an_nextitem also holds number of entries
 		 * added in the cache table
 		 */
 		if ( sc->an_nextitem < MAXANCACHE ) {
 			cache_slot = sc->an_nextitem;
 			sc->an_nextitem++;
 			sc->an_sigitems = sc->an_nextitem;
 		}
 		/* no space found, so simply wrap anth wrap index
 		 * and "zap" the next entry
 		 */
 		else {
 			if (wrapindex == MAXANCACHE) {
 				wrapindex = 0;
 			}
 			cache_slot = wrapindex++;
 		}
 	}
 
 	/* invariant: cache_slot now points at some slot
 	 * in cache.
 	 */
 	if (cache_slot < 0 || cache_slot >= MAXANCACHE) {
 		log(LOG_ERR, "an_cache_store, bad index: %d of "
 		    "[0..%d], gross cache error\n",
 		    cache_slot, MAXANCACHE);
 		return;
 	}
 
 	/*  store items in cache
 	 *  .ip source address
 	 *  .mac src
 	 *  .signal, etc.
 	 */
 	if (type_ipv4) {
 		sc->an_sigcache[cache_slot].ipsrc = ip->ip_src.s_addr;
 	}
 	bcopy( eh->ether_shost, sc->an_sigcache[cache_slot].macsrc,  6);
 
 
 	switch (an_cache_mode) {
 	case DBM:
 		if (sc->an_have_rssimap) {
 			sc->an_sigcache[cache_slot].signal =
 				- sc->an_rssimap.an_entries[rx_rssi].an_rss_dbm;
 			sc->an_sigcache[cache_slot].quality =
 				- sc->an_rssimap.an_entries[rx_quality].an_rss_dbm;
 		} else {
 			sc->an_sigcache[cache_slot].signal = rx_rssi - 100;
 			sc->an_sigcache[cache_slot].quality = rx_quality - 100;
 		}
 		break;
 	case PERCENT:
 		if (sc->an_have_rssimap) {
 			sc->an_sigcache[cache_slot].signal =
 				sc->an_rssimap.an_entries[rx_rssi].an_rss_pct;
 			sc->an_sigcache[cache_slot].quality =
 				sc->an_rssimap.an_entries[rx_quality].an_rss_pct;
 		} else {
 			if (rx_rssi > 100)
 				rx_rssi = 100;
 			if (rx_quality > 100)
 				rx_quality = 100;
 			sc->an_sigcache[cache_slot].signal = rx_rssi;
 			sc->an_sigcache[cache_slot].quality = rx_quality;
 		}
 		break;
 	case RAW:
 		sc->an_sigcache[cache_slot].signal = rx_rssi;
 		sc->an_sigcache[cache_slot].quality = rx_quality;
 		break;
 	}
 
 	sc->an_sigcache[cache_slot].noise = 0;
 
 	return;
 }
 #endif
 
 static int
 an_media_change(struct ifnet *ifp)
 {
 	struct an_softc *sc = ifp->if_softc;
 	struct an_ltv_genconfig	*cfg;
 	int otype = sc->an_config.an_opmode;
 	int orate = sc->an_tx_rate;
 
 	AN_LOCK(sc);
 	sc->an_tx_rate = ieee80211_media2rate(
 		IFM_SUBTYPE(sc->an_ifmedia.ifm_cur->ifm_media));
 	if (sc->an_tx_rate < 0)
 		sc->an_tx_rate = 0;
 
 	if (orate != sc->an_tx_rate) {
 		/* Read the current configuration */
 		sc->an_config.an_type = AN_RID_GENCONFIG;
 		sc->an_config.an_len = sizeof(struct an_ltv_genconfig);
 		an_read_record(sc, (struct an_ltv_gen *)&sc->an_config);
 		cfg = &sc->an_config;
 
 		/* clear other rates and set the only one we want */
 		bzero(cfg->an_rates, sizeof(cfg->an_rates));
 		cfg->an_rates[0] = sc->an_tx_rate;
 
 		/* Save the new rate */
 		sc->an_config.an_type = AN_RID_GENCONFIG;
 		sc->an_config.an_len = sizeof(struct an_ltv_genconfig);
 	}
 
 	if ((sc->an_ifmedia.ifm_cur->ifm_media & IFM_IEEE80211_ADHOC) != 0)
 		sc->an_config.an_opmode &= ~AN_OPMODE_INFRASTRUCTURE_STATION;
 	else
 		sc->an_config.an_opmode |= AN_OPMODE_INFRASTRUCTURE_STATION;
 
 	if (otype != sc->an_config.an_opmode ||
 	    orate != sc->an_tx_rate)
 		an_init_locked(sc);
 	AN_UNLOCK(sc);
 
 	return(0);
 }
 
 static void
 an_media_status(struct ifnet *ifp, struct ifmediareq *imr)
 {
 	struct an_ltv_status	status;
 	struct an_softc		*sc = ifp->if_softc;
 
 	imr->ifm_active = IFM_IEEE80211;
 
 	AN_LOCK(sc);
 	status.an_len = sizeof(status);
 	status.an_type = AN_RID_STATUS;
 	if (an_read_record(sc, (struct an_ltv_gen *)&status)) {
 		/* If the status read fails, just lie. */
 		imr->ifm_active = sc->an_ifmedia.ifm_cur->ifm_media;
 		imr->ifm_status = IFM_AVALID|IFM_ACTIVE;
 	}
 
 	if (sc->an_tx_rate == 0) {
 		imr->ifm_active = IFM_IEEE80211|IFM_AUTO;
 	}
 
 	if (sc->an_config.an_opmode == AN_OPMODE_IBSS_ADHOC)
 		imr->ifm_active |= IFM_IEEE80211_ADHOC;
 	imr->ifm_active |= ieee80211_rate2media(NULL,
 		status.an_current_tx_rate, IEEE80211_MODE_AUTO);
 	imr->ifm_status = IFM_AVALID;
 	if (status.an_opmode & AN_STATUS_OPMODE_ASSOCIATED)
 		imr->ifm_status |= IFM_ACTIVE;
 	AN_UNLOCK(sc);
 }
 
 /********************** Cisco utility support routines *************/
 
 /*
  * ReadRids & WriteRids derived from Cisco driver additions to Ben Reed's
  * Linux driver
  */
 
 static int
 readrids(struct ifnet *ifp, struct aironet_ioctl *l_ioctl)
 {
 	unsigned short  rid;
 	struct an_softc *sc;
 	int error;
 
 	switch (l_ioctl->command) {
 	case AIROGCAP:
 		rid = AN_RID_CAPABILITIES;
 		break;
 	case AIROGCFG:
 		rid = AN_RID_GENCONFIG;
 		break;
 	case AIROGSLIST:
 		rid = AN_RID_SSIDLIST;
 		break;
 	case AIROGVLIST:
 		rid = AN_RID_APLIST;
 		break;
 	case AIROGDRVNAM:
 		rid = AN_RID_DRVNAME;
 		break;
 	case AIROGEHTENC:
 		rid = AN_RID_ENCAPPROTO;
 		break;
 	case AIROGWEPKTMP:
 		rid = AN_RID_WEP_TEMP;
 		break;
 	case AIROGWEPKNV:
 		rid = AN_RID_WEP_PERM;
 		break;
 	case AIROGSTAT:
 		rid = AN_RID_STATUS;
 		break;
 	case AIROGSTATSD32:
 		rid = AN_RID_32BITS_DELTA;
 		break;
 	case AIROGSTATSC32:
 		rid = AN_RID_32BITS_CUM;
 		break;
 	default:
 		rid = 999;
 		break;
 	}
 
 	if (rid == 999)	/* Is bad command */
 		return -EINVAL;
 
 	sc = ifp->if_softc;
 	sc->areq.an_len  = AN_MAX_DATALEN;
 	sc->areq.an_type = rid;
 
 	an_read_record(sc, (struct an_ltv_gen *)&sc->areq);
 
 	l_ioctl->len = sc->areq.an_len - 4;	/* just data */
 
 	AN_UNLOCK(sc);
 	/* the data contains the length at first */
 	if (copyout(&(sc->areq.an_len), l_ioctl->data,
 		    sizeof(sc->areq.an_len))) {
 		error = -EFAULT;
 		goto lock_exit;
 	}
 	/* Just copy the data back */
 	if (copyout(&(sc->areq.an_val), l_ioctl->data + 2,
 		    l_ioctl->len)) {
 		error = -EFAULT;
 		goto lock_exit;
 	}
 	error = 0;
 lock_exit:
 	AN_LOCK(sc);
 	return (error);
 }
 
 static int
 writerids(struct ifnet *ifp, struct aironet_ioctl *l_ioctl)
 {
 	struct an_softc *sc;
 	int		rid, command, error;
 
 	sc = ifp->if_softc;
 	AN_LOCK_ASSERT(sc);
 	rid = 0;
 	command = l_ioctl->command;
 
 	switch (command) {
 	case AIROPSIDS:
 		rid = AN_RID_SSIDLIST;
 		break;
 	case AIROPCAP:
 		rid = AN_RID_CAPABILITIES;
 		break;
 	case AIROPAPLIST:
 		rid = AN_RID_APLIST;
 		break;
 	case AIROPCFG:
 		rid = AN_RID_GENCONFIG;
 		break;
 	case AIROPMACON:
 		an_cmd(sc, AN_CMD_ENABLE, 0);
 		return 0;
 		break;
 	case AIROPMACOFF:
 		an_cmd(sc, AN_CMD_DISABLE, 0);
 		return 0;
 		break;
 	case AIROPSTCLR:
 		/*
 		 * This command merely clears the counts does not actually
 		 * store any data only reads rid. But as it changes the cards
 		 * state, I put it in the writerid routines.
 		 */
 
 		rid = AN_RID_32BITS_DELTACLR;
 		sc = ifp->if_softc;
 		sc->areq.an_len = AN_MAX_DATALEN;
 		sc->areq.an_type = rid;
 
 		an_read_record(sc, (struct an_ltv_gen *)&sc->areq);
 		l_ioctl->len = sc->areq.an_len - 4;	/* just data */
 
 		AN_UNLOCK(sc);
 		/* the data contains the length at first */
 		error = copyout(&(sc->areq.an_len), l_ioctl->data,
 			    sizeof(sc->areq.an_len));
 		if (error) {
 			AN_LOCK(sc);
 			return -EFAULT;
 		}
 		/* Just copy the data */
 		error = copyout(&(sc->areq.an_val), l_ioctl->data + 2,
 			    l_ioctl->len);
 		AN_LOCK(sc);
 		if (error)
 			return -EFAULT;
 		return 0;
 		break;
 	case AIROPWEPKEY:
 		rid = AN_RID_WEP_TEMP;
 		break;
 	case AIROPWEPKEYNV:
 		rid = AN_RID_WEP_PERM;
 		break;
 	case AIROPLEAPUSR:
 		rid = AN_RID_LEAPUSERNAME;
 		break;
 	case AIROPLEAPPWD:
 		rid = AN_RID_LEAPPASSWORD;
 		break;
 	default:
 		return -EOPNOTSUPP;
 	}
 
 	if (rid) {
 		if (l_ioctl->len > sizeof(sc->areq.an_val) + 4)
 			return -EINVAL;
 		sc->areq.an_len = l_ioctl->len + 4;	/* add type & length */
 		sc->areq.an_type = rid;
 
 		/* Just copy the data back */
 		AN_UNLOCK(sc);
 		error = copyin((l_ioctl->data) + 2, &sc->areq.an_val,
 		       l_ioctl->len);
 		AN_LOCK(sc);
 		if (error)
 			return -EFAULT;
 
 		an_cmd(sc, AN_CMD_DISABLE, 0);
 		an_write_record(sc, (struct an_ltv_gen *)&sc->areq);
 		an_cmd(sc, AN_CMD_ENABLE, 0);
 		return 0;
 	}
 	return -EOPNOTSUPP;
 }
 
 /*
  * General Flash utilities derived from Cisco driver additions to Ben Reed's
  * Linux driver
  */
 
 #define FLASH_DELAY(_sc, x)	msleep(ifp, &(_sc)->an_mtx, PZERO, \
 	"flash", ((x) / hz) + 1);
 #define FLASH_COMMAND	0x7e7e
 #define FLASH_SIZE	32 * 1024
 
 static int
 unstickbusy(struct ifnet *ifp)
 {
 	struct an_softc *sc = ifp->if_softc;
 
 	if (CSR_READ_2(sc, AN_COMMAND(sc->mpi350)) & AN_CMD_BUSY) {
 		CSR_WRITE_2(sc, AN_EVENT_ACK(sc->mpi350),
 			    AN_EV_CLR_STUCK_BUSY);
 		return 1;
 	}
 	return 0;
 }
 
 /*
  * Wait for busy completion from card wait for delay uSec's Return true for
  * success meaning command reg is clear
  */
 
 static int
 WaitBusy(struct ifnet *ifp, int uSec)
 {
 	int		statword = 0xffff;
 	int		delay = 0;
 	struct an_softc	*sc = ifp->if_softc;
 
 	while ((statword & AN_CMD_BUSY) && delay <= (1000 * 100)) {
 		FLASH_DELAY(sc, 10);
 		delay += 10;
 		statword = CSR_READ_2(sc, AN_COMMAND(sc->mpi350));
 
 		if ((AN_CMD_BUSY & statword) && (delay % 200)) {
 			unstickbusy(ifp);
 		}
 	}
 
 	return 0 == (AN_CMD_BUSY & statword);
 }
 
 /*
  * STEP 1) Disable MAC and do soft reset on card.
  */
 
 static int
 cmdreset(struct ifnet *ifp)
 {
 	int		status;
 	struct an_softc	*sc = ifp->if_softc;
 
 	AN_LOCK(sc);
 	an_stop(sc);
 
 	an_cmd(sc, AN_CMD_DISABLE, 0);
 
 	if (!(status = WaitBusy(ifp, AN_TIMEOUT))) {
 		if_printf(ifp, "Waitbusy hang b4 RESET =%d\n", status);
 		AN_UNLOCK(sc);
 		return -EBUSY;
 	}
 	CSR_WRITE_2(sc, AN_COMMAND(sc->mpi350), AN_CMD_FW_RESTART);
 
 	FLASH_DELAY(sc, 1000);	/* WAS 600 12/7/00 */
 
 
 	if (!(status = WaitBusy(ifp, 100))) {
 		if_printf(ifp, "Waitbusy hang AFTER RESET =%d\n", status);
 		AN_UNLOCK(sc);
 		return -EBUSY;
 	}
 	AN_UNLOCK(sc);
 	return 0;
 }
 
 /*
  * STEP 2) Put the card in legendary flash mode
  */
 
 static int
 setflashmode(struct ifnet *ifp)
 {
 	int		status;
 	struct an_softc	*sc = ifp->if_softc;
 
 	CSR_WRITE_2(sc, AN_SW0(sc->mpi350), FLASH_COMMAND);
 	CSR_WRITE_2(sc, AN_SW1(sc->mpi350), FLASH_COMMAND);
 	CSR_WRITE_2(sc, AN_SW0(sc->mpi350), FLASH_COMMAND);
 	CSR_WRITE_2(sc, AN_COMMAND(sc->mpi350), FLASH_COMMAND);
 
 	/*
 	 * mdelay(500); // 500ms delay
 	 */
 
 	FLASH_DELAY(sc, 500);
 
 	if (!(status = WaitBusy(ifp, AN_TIMEOUT))) {
 		printf("Waitbusy hang after setflash mode\n");
 		return -EIO;
 	}
 	return 0;
 }
 
 /*
  * Get a character from the card matching matchbyte Step 3)
  */
 
 static int
 flashgchar(struct ifnet *ifp, int matchbyte, int dwelltime)
 {
 	int		rchar;
 	unsigned char	rbyte = 0;
 	int		success = -1;
 	struct an_softc	*sc = ifp->if_softc;
 
 
 	do {
 		rchar = CSR_READ_2(sc, AN_SW1(sc->mpi350));
 
 		if (dwelltime && !(0x8000 & rchar)) {
 			dwelltime -= 10;
 			FLASH_DELAY(sc, 10);
 			continue;
 		}
 		rbyte = 0xff & rchar;
 
 		if ((rbyte == matchbyte) && (0x8000 & rchar)) {
 			CSR_WRITE_2(sc, AN_SW1(sc->mpi350), 0);
 			success = 1;
 			break;
 		}
 		if (rbyte == 0x81 || rbyte == 0x82 || rbyte == 0x83 || rbyte == 0x1a || 0xffff == rchar)
 			break;
 		CSR_WRITE_2(sc, AN_SW1(sc->mpi350), 0);
 
 	} while (dwelltime > 0);
 	return success;
 }
 
 /*
  * Put character to SWS0 wait for dwelltime x 50us for  echo .
  */
 
 static int
 flashpchar(struct ifnet *ifp, int byte, int dwelltime)
 {
 	int		echo;
 	int		pollbusy, waittime;
 	struct an_softc	*sc = ifp->if_softc;
 
 	byte |= 0x8000;
 
 	if (dwelltime == 0)
 		dwelltime = 200;
 
 	waittime = dwelltime;
 
 	/*
 	 * Wait for busy bit d15 to go false indicating buffer empty
 	 */
 	do {
 		pollbusy = CSR_READ_2(sc, AN_SW0(sc->mpi350));
 
 		if (pollbusy & 0x8000) {
 			FLASH_DELAY(sc, 50);
 			waittime -= 50;
 			continue;
 		} else
 			break;
 	}
 	while (waittime >= 0);
 
 	/* timeout for busy clear wait */
 
 	if (waittime <= 0) {
 		if_printf(ifp, "flash putchar busywait timeout!\n");
 		return -1;
 	}
 	/*
 	 * Port is clear now write byte and wait for it to echo back
 	 */
 	do {
 		CSR_WRITE_2(sc, AN_SW0(sc->mpi350), byte);
 		FLASH_DELAY(sc, 50);
 		dwelltime -= 50;
 		echo = CSR_READ_2(sc, AN_SW1(sc->mpi350));
 	} while (dwelltime >= 0 && echo != byte);
 
 
 	CSR_WRITE_2(sc, AN_SW1(sc->mpi350), 0);
 
 	return echo == byte;
 }
 
 /*
  * Transfer 32k of firmware data from user buffer to our buffer and send to
  * the card
  */
 
 static int
 flashputbuf(struct ifnet *ifp)
 {
 	unsigned short *bufp;
 	int		nwords;
 	struct an_softc	*sc = ifp->if_softc;
 
 	/* Write stuff */
 
 	bufp = sc->an_flash_buffer;
 
 	if (!sc->mpi350) {
 		CSR_WRITE_2(sc, AN_AUX_PAGE, 0x100);
 		CSR_WRITE_2(sc, AN_AUX_OFFSET, 0);
 
 		for (nwords = 0; nwords != FLASH_SIZE / 2; nwords++) {
 			CSR_WRITE_2(sc, AN_AUX_DATA, bufp[nwords] & 0xffff);
 		}
 	} else {
 		for (nwords = 0; nwords != FLASH_SIZE / 4; nwords++) {
 			CSR_MEM_AUX_WRITE_4(sc, 0x8000,
 				((u_int32_t *)bufp)[nwords] & 0xffff);
 		}
 	}
 
 	CSR_WRITE_2(sc, AN_SW0(sc->mpi350), 0x8000);
 
 	return 0;
 }
 
 /*
  * After flashing restart the card.
  */
 
 static int
 flashrestart(struct ifnet *ifp)
 {
 	int		status = 0;
 	struct an_softc	*sc = ifp->if_softc;
 
 	FLASH_DELAY(sc, 1024);		/* Added 12/7/00 */
 
 	an_init_locked(sc);
 
 	FLASH_DELAY(sc, 1024);		/* Added 12/7/00 */
 	return status;
 }
 
 /*
  * Entry point for flash ioclt.
  */
 
 static int
 flashcard(struct ifnet *ifp, struct aironet_ioctl *l_ioctl)
 {
 	int		z = 0, status;
 	struct an_softc	*sc;
 
 	sc = ifp->if_softc;
 	if (sc->mpi350) {
 		if_printf(ifp, "flashing not supported on MPI 350 yet\n");
 		return(-1);
 	}
 	status = l_ioctl->command;
 
 	switch (l_ioctl->command) {
 	case AIROFLSHRST:
 		return cmdreset(ifp);
 		break;
 	case AIROFLSHSTFL:
 		if (sc->an_flash_buffer) {
 			free(sc->an_flash_buffer, M_DEVBUF);
 			sc->an_flash_buffer = NULL;
 		}
 		sc->an_flash_buffer = malloc(FLASH_SIZE, M_DEVBUF, M_WAITOK);
 		if (sc->an_flash_buffer)
 			return setflashmode(ifp);
 		else
 			return ENOBUFS;
 		break;
 	case AIROFLSHGCHR:	/* Get char from aux */
 		AN_UNLOCK(sc);
 		status = copyin(l_ioctl->data, &sc->areq, l_ioctl->len);
 		AN_LOCK(sc);
 		if (status)
 			return status;
 		z = *(int *)&sc->areq;
 		if ((status = flashgchar(ifp, z, 8000)) == 1)
 			return 0;
 		else
 			return -1;
 	case AIROFLSHPCHR:	/* Send char to card. */
 		AN_UNLOCK(sc);
 		status = copyin(l_ioctl->data, &sc->areq, l_ioctl->len);
 		AN_LOCK(sc);
 		if (status)
 			return status;
 		z = *(int *)&sc->areq;
 		if ((status = flashpchar(ifp, z, 8000)) == -1)
 			return -EIO;
 		else
 			return 0;
 		break;
 	case AIROFLPUTBUF:	/* Send 32k to card */
 		if (l_ioctl->len > FLASH_SIZE) {
 			if_printf(ifp, "Buffer to big, %x %x\n",
 			       l_ioctl->len, FLASH_SIZE);
 			return -EINVAL;
 		}
 		AN_UNLOCK(sc);
 		status = copyin(l_ioctl->data, sc->an_flash_buffer, l_ioctl->len);
 		AN_LOCK(sc);
 		if (status)
 			return status;
 
 		if ((status = flashputbuf(ifp)) != 0)
 			return -EIO;
 		else
 			return 0;
 		break;
 	case AIRORESTART:
 		if ((status = flashrestart(ifp)) != 0) {
 			if_printf(ifp, "FLASHRESTART returned %d\n", status);
 			return -EIO;
 		} else
 			return 0;
 
 		break;
 	default:
 		return -EINVAL;
 	}
 
 	return -EINVAL;
 }
Index: projects/release-debugdist/sys/dev/ata/ata-dma.c
===================================================================
--- projects/release-debugdist/sys/dev/ata/ata-dma.c	(revision 267343)
+++ projects/release-debugdist/sys/dev/ata/ata-dma.c	(revision 267344)
@@ -1,351 +1,349 @@
 /*-
  * Copyright (c) 1998 - 2008 Søren Schmidt <sos@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer,
  *    without modification, immediately at the beginning of the file.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/ata.h>
 #include <sys/kernel.h>
 #include <sys/endian.h>
 #include <sys/malloc.h> 
 #include <sys/lock.h>
 #include <sys/sema.h>
 #include <sys/taskqueue.h>
 #include <vm/uma.h>
 #include <sys/bus.h>
 #include <machine/bus.h>
 #include <sys/rman.h>
 #include <dev/ata/ata-all.h>
 
 /* prototypes */
 static void ata_dmasetupc_cb(void *xsc, bus_dma_segment_t *segs, int nsegs, int error);
 static void ata_dmaalloc(device_t dev);
 static void ata_dmafree(device_t dev);
 static void ata_dmasetprd(void *xsc, bus_dma_segment_t *segs, int nsegs, int error);
 static int ata_dmaload(struct ata_request *request, void *addr, int *nsegs);
 static int ata_dmaunload(struct ata_request *request);
 
 /* local vars */
 static MALLOC_DEFINE(M_ATADMA, "ata_dma", "ATA driver DMA");
 
 /* misc defines */
 #define MAXTABSZ        PAGE_SIZE
 #define MAXWSPCSZ       PAGE_SIZE*2
 
 struct ata_dc_cb_args {
     bus_addr_t maddr;
     int error;
 };
 
 void 
 ata_dmainit(device_t dev)
 {
     struct ata_channel *ch = device_get_softc(dev);
     struct ata_dc_cb_args dcba;
 
     if (ch->dma.alloc == NULL)
 	ch->dma.alloc = ata_dmaalloc;
     if (ch->dma.free == NULL)
 	ch->dma.free = ata_dmafree;
     if (ch->dma.setprd == NULL)
 	ch->dma.setprd = ata_dmasetprd;
     if (ch->dma.load == NULL)
 	ch->dma.load = ata_dmaload;
     if (ch->dma.unload == NULL)
 	ch->dma.unload = ata_dmaunload;
     if (ch->dma.alignment == 0)
 	ch->dma.alignment = 2;
     if (ch->dma.boundary == 0)
 	ch->dma.boundary = 65536;
     if (ch->dma.segsize == 0)
 	ch->dma.segsize = 65536;
     if (ch->dma.max_iosize == 0)
 	ch->dma.max_iosize = MIN((ATA_DMA_ENTRIES - 1) * PAGE_SIZE, MAXPHYS);
     if (ch->dma.max_address == 0)
 	ch->dma.max_address = BUS_SPACE_MAXADDR_32BIT;
     if (ch->dma.dma_slots == 0)
 	ch->dma.dma_slots = 1;
 
     if (bus_dma_tag_create(bus_get_dma_tag(dev), ch->dma.alignment, 0,
 			   ch->dma.max_address, BUS_SPACE_MAXADDR,
 			   NULL, NULL, ch->dma.max_iosize,
 			   ATA_DMA_ENTRIES, ch->dma.segsize,
 			   0, NULL, NULL, &ch->dma.dmatag))
 	goto error;
 
     if (bus_dma_tag_create(ch->dma.dmatag, PAGE_SIZE, 64 * 1024,
 			   ch->dma.max_address, BUS_SPACE_MAXADDR,
 			   NULL, NULL, MAXWSPCSZ, 1, MAXWSPCSZ,
 			   0, NULL, NULL, &ch->dma.work_tag))
 	goto error;
 
     if (bus_dmamem_alloc(ch->dma.work_tag, (void **)&ch->dma.work,
 			 BUS_DMA_WAITOK | BUS_DMA_COHERENT,
 			 &ch->dma.work_map))
 	goto error;
 
     if (bus_dmamap_load(ch->dma.work_tag, ch->dma.work_map, ch->dma.work,
 			MAXWSPCSZ, ata_dmasetupc_cb, &dcba, 0) ||
 			dcba.error) {
 	bus_dmamem_free(ch->dma.work_tag, ch->dma.work, ch->dma.work_map);
 	goto error;
     }
     ch->dma.work_bus = dcba.maddr;
     return;
 
 error:
     device_printf(dev, "WARNING - DMA initialization failed, disabling DMA\n");
     ata_dmafini(dev);
 }
 
 void 
 ata_dmafini(device_t dev)
 {
     struct ata_channel *ch = device_get_softc(dev);
 
     if (ch->dma.work_bus) {
 	bus_dmamap_unload(ch->dma.work_tag, ch->dma.work_map);
 	bus_dmamem_free(ch->dma.work_tag, ch->dma.work, ch->dma.work_map);
 	ch->dma.work_bus = 0;
 	ch->dma.work_map = NULL;
 	ch->dma.work = NULL;
     }
     if (ch->dma.work_tag) {
 	bus_dma_tag_destroy(ch->dma.work_tag);
 	ch->dma.work_tag = NULL;
     }
     if (ch->dma.dmatag) {
 	bus_dma_tag_destroy(ch->dma.dmatag);
 	ch->dma.dmatag = NULL;
     }
 }
 
 static void
 ata_dmasetupc_cb(void *xsc, bus_dma_segment_t *segs, int nsegs, int error)
 {
     struct ata_dc_cb_args *dcba = (struct ata_dc_cb_args *)xsc;
 
     if (!(dcba->error = error))
 	dcba->maddr = segs[0].ds_addr;
 }
 
 static void
 ata_dmaalloc(device_t dev)
 {
     struct ata_channel *ch = device_get_softc(dev);
     struct ata_dc_cb_args dcba;
     int i;
 
     /* alloc and setup needed dma slots */
     bzero(ch->dma.slot, sizeof(struct ata_dmaslot) * ATA_DMA_SLOTS);
     for (i = 0; i < ch->dma.dma_slots; i++) {
 	struct ata_dmaslot *slot = &ch->dma.slot[i];
 
 	if (bus_dma_tag_create(ch->dma.dmatag, PAGE_SIZE, PAGE_SIZE,
 			       ch->dma.max_address, BUS_SPACE_MAXADDR,
 			       NULL, NULL, PAGE_SIZE, 1, PAGE_SIZE,
 			       0, NULL, NULL, &slot->sg_tag)) {
             device_printf(ch->dev, "FAILURE - create sg_tag\n");
             goto error;
 	}
 
 	if (bus_dmamem_alloc(slot->sg_tag, (void **)&slot->sg, BUS_DMA_WAITOK,
 			     &slot->sg_map)) {
 	    device_printf(ch->dev, "FAILURE - alloc sg_map\n");
 	    goto error;
         }
 
 	if (bus_dmamap_load(slot->sg_tag, slot->sg_map, slot->sg, MAXTABSZ,
 			    ata_dmasetupc_cb, &dcba, 0) || dcba.error) {
 	    device_printf(ch->dev, "FAILURE - load sg\n");
 	    goto error;
 	}
 	slot->sg_bus = dcba.maddr;
 
 	if (bus_dma_tag_create(ch->dma.dmatag,
 			       ch->dma.alignment, ch->dma.boundary,
                                ch->dma.max_address, BUS_SPACE_MAXADDR,
                                NULL, NULL, ch->dma.max_iosize,
                                ATA_DMA_ENTRIES, ch->dma.segsize,
                                BUS_DMA_ALLOCNOW, NULL, NULL, &slot->data_tag)) {
 	    device_printf(ch->dev, "FAILURE - create data_tag\n");
 	    goto error;
 	}
 
 	if (bus_dmamap_create(slot->data_tag, 0, &slot->data_map)) {
 	    device_printf(ch->dev, "FAILURE - create data_map\n");
 	    goto error;
         }
     }
 
     return;
 
 error:
     device_printf(dev, "WARNING - DMA allocation failed, disabling DMA\n");
     ata_dmafree(dev);
 }
 
 static void
 ata_dmafree(device_t dev)
 {
     struct ata_channel *ch = device_get_softc(dev);
     int i;
 
     /* free all dma slots */
     for (i = 0; i < ATA_DMA_SLOTS; i++) {
 	struct ata_dmaslot *slot = &ch->dma.slot[i];
 
 	if (slot->sg_bus) {
             bus_dmamap_unload(slot->sg_tag, slot->sg_map);
             slot->sg_bus = 0;
 	}
-	if (slot->sg_map) {
+	if (slot->sg) {
             bus_dmamem_free(slot->sg_tag, slot->sg, slot->sg_map);
-            bus_dmamap_destroy(slot->sg_tag, slot->sg_map);
             slot->sg = NULL;
-            slot->sg_map = NULL;
 	}
 	if (slot->data_map) {
             bus_dmamap_destroy(slot->data_tag, slot->data_map);
             slot->data_map = NULL;
 	}
 	if (slot->sg_tag) {
             bus_dma_tag_destroy(slot->sg_tag);
             slot->sg_tag = NULL;
 	}
 	if (slot->data_tag) {
             bus_dma_tag_destroy(slot->data_tag);
             slot->data_tag = NULL;
 	}
     }
 }
 
 static void
 ata_dmasetprd(void *xsc, bus_dma_segment_t *segs, int nsegs, int error)
 {
     struct ata_dmasetprd_args *args = xsc;
     struct ata_dma_prdentry *prd = args->dmatab;
     int i;
 
     if ((args->error = error))
 	return;
 
     for (i = 0; i < nsegs; i++) {
 	prd[i].addr = htole32(segs[i].ds_addr);
 	prd[i].count = htole32(segs[i].ds_len);
     }
     prd[i - 1].count |= htole32(ATA_DMA_EOT);
     KASSERT(nsegs <= ATA_DMA_ENTRIES, ("too many DMA segment entries\n"));
     args->nsegs = nsegs;
 }
 
 static int
 ata_dmaload(struct ata_request *request, void *addr, int *entries)
 {
     struct ata_channel *ch = device_get_softc(request->parent);
     struct ata_dmasetprd_args dspa;
     int error;
 
     ATA_DEBUG_RQ(request, "dmaload");
 
     if (request->dma) {
 	device_printf(request->parent,
 		      "FAILURE - already active DMA on this device\n");
 	return EIO;
     }
     if (!request->bytecount) {
 	device_printf(request->parent,
 		      "FAILURE - zero length DMA transfer attempted\n");
 	return EIO;
     }
     if (request->bytecount & (ch->dma.alignment - 1)) {
 	device_printf(request->parent,
 		      "FAILURE - odd-sized DMA transfer attempt %d %% %d\n",
 		      request->bytecount, ch->dma.alignment);
 	return EIO;
     }
     if (request->bytecount > ch->dma.max_iosize) {
 	device_printf(request->parent,
 		      "FAILURE - oversized DMA transfer attempt %d > %d\n",
 		      request->bytecount, ch->dma.max_iosize);
 	return EIO;
     }
 
     /* set our slot. XXX SOS NCQ will change that */
     request->dma = &ch->dma.slot[0];
 
     if (addr)
 	dspa.dmatab = addr;
     else
 	dspa.dmatab = request->dma->sg;
 
     if (request->flags & ATA_R_DATA_IN_CCB)
         error = bus_dmamap_load_ccb(request->dma->data_tag,
 				request->dma->data_map, request->ccb,
 				ch->dma.setprd, &dspa, BUS_DMA_NOWAIT);
     else
         error = bus_dmamap_load(request->dma->data_tag, request->dma->data_map,
 				request->data, request->bytecount,
 				ch->dma.setprd, &dspa, BUS_DMA_NOWAIT);
     if (error || (error = dspa.error)) {
 	device_printf(request->parent, "FAILURE - load data\n");
 	goto error;
     }
 
     if (entries)
 	*entries = dspa.nsegs;
 
     bus_dmamap_sync(request->dma->sg_tag, request->dma->sg_map,
 		    BUS_DMASYNC_PREWRITE);
     bus_dmamap_sync(request->dma->data_tag, request->dma->data_map,
 		    (request->flags & ATA_R_READ) ?
 		    BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE);
     return 0;
 
 error:
     ata_dmaunload(request);
     return EIO;
 }
 
 int
 ata_dmaunload(struct ata_request *request)
 {
     ATA_DEBUG_RQ(request, "dmaunload");
 
     if (request->dma) {
 	bus_dmamap_sync(request->dma->sg_tag, request->dma->sg_map,
 			BUS_DMASYNC_POSTWRITE);
 	bus_dmamap_sync(request->dma->data_tag, request->dma->data_map,
 			(request->flags & ATA_R_READ) ?
 			BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
 
 	bus_dmamap_unload(request->dma->data_tag, request->dma->data_map);
         request->dma = NULL;
     }
     return 0;
 }
Index: projects/release-debugdist/sys/dev/buslogic/bt.c
===================================================================
--- projects/release-debugdist/sys/dev/buslogic/bt.c	(revision 267343)
+++ projects/release-debugdist/sys/dev/buslogic/bt.c	(revision 267344)
@@ -1,2395 +1,2393 @@
 /*-
  * Generic driver for the BusLogic MultiMaster SCSI host adapters
  * Product specific probe and attach routines can be found in:
  * sys/dev/buslogic/bt_isa.c	BT-54X, BT-445 cards
  * sys/dev/buslogic/bt_mca.c	BT-64X, SDC3211B, SDC3211F
  * sys/dev/buslogic/bt_eisa.c	BT-74X, BT-75x cards, SDC3222F
  * sys/dev/buslogic/bt_pci.c	BT-946, BT-948, BT-956, BT-958 cards
  *
  * Copyright (c) 1998, 1999 Justin T. Gibbs.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions, and the following disclaimer,
  *    without modification, immediately at the beginning of the file.
  * 2. The name of the author may not be used to endorse or promote products
  *    derived from this software without specific prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
  /*
   * Special thanks to Leonard N. Zubkoff for writing such a complete and
   * well documented Mylex/BusLogic MultiMaster driver for Linux.  Support
   * in this driver for the wide range of MultiMaster controllers and
   * firmware revisions, with their otherwise undocumented quirks, would not
   * have been possible without his efforts.
   */
 
 #include <sys/param.h>
 #include <sys/conf.h>
 #include <sys/systm.h> 
 #include <sys/malloc.h>
 #include <sys/kernel.h>
 #include <sys/lock.h>
 #include <sys/module.h>
 #include <sys/mutex.h>
 #include <sys/sysctl.h>
 #include <sys/bus.h>
  
 #include <machine/bus.h>
 #include <sys/rman.h>
 
 #include <cam/cam.h>
 #include <cam/cam_ccb.h>
 #include <cam/cam_sim.h>
 #include <cam/cam_xpt_sim.h>
 #include <cam/cam_debug.h>
 
 #include <cam/scsi/scsi_message.h>
 
 #include <vm/vm.h>
 #include <vm/pmap.h>
  
 #include <dev/buslogic/btreg.h>
 
 /* MailBox Management functions */
 static __inline void	btnextinbox(struct bt_softc *bt);
 static __inline void	btnextoutbox(struct bt_softc *bt);
 
 static __inline void
 btnextinbox(struct bt_softc *bt)
 {
 	if (bt->cur_inbox == bt->last_inbox)
 		bt->cur_inbox = bt->in_boxes;
 	else
 		bt->cur_inbox++;
 }
 
 static __inline void
 btnextoutbox(struct bt_softc *bt)
 {
 	if (bt->cur_outbox == bt->last_outbox)
 		bt->cur_outbox = bt->out_boxes;
 	else
 		bt->cur_outbox++;
 }
 
 /* CCB Mangement functions */
 static __inline u_int32_t		btccbvtop(struct bt_softc *bt,
 						  struct bt_ccb *bccb);
 static __inline struct bt_ccb*		btccbptov(struct bt_softc *bt,
 						  u_int32_t ccb_addr);
 static __inline u_int32_t		btsensepaddr(struct bt_softc *bt,
 						     struct bt_ccb *bccb);
 static __inline struct scsi_sense_data* btsensevaddr(struct bt_softc *bt,
 						     struct bt_ccb *bccb);
 
 static __inline u_int32_t
 btccbvtop(struct bt_softc *bt, struct bt_ccb *bccb)
 {
 	return (bt->bt_ccb_physbase
 	      + (u_int32_t)((caddr_t)bccb - (caddr_t)bt->bt_ccb_array));
 }
 
 static __inline struct bt_ccb *
 btccbptov(struct bt_softc *bt, u_int32_t ccb_addr)
 {
 	return (bt->bt_ccb_array +
 	        ((struct bt_ccb*)(uintptr_t)ccb_addr - (struct bt_ccb*)(uintptr_t)bt->bt_ccb_physbase));
 }
 
 static __inline u_int32_t
 btsensepaddr(struct bt_softc *bt, struct bt_ccb *bccb)
 {
 	u_int index;
 
 	index = (u_int)(bccb - bt->bt_ccb_array);
 	return (bt->sense_buffers_physbase
 		+ (index * sizeof(struct scsi_sense_data)));
 }
 
 static __inline struct scsi_sense_data *
 btsensevaddr(struct bt_softc *bt, struct bt_ccb *bccb)
 {
 	u_int index;
 
 	index = (u_int)(bccb - bt->bt_ccb_array);
 	return (bt->sense_buffers + index);
 }
 
 static __inline struct bt_ccb*	btgetccb(struct bt_softc *bt);
 static __inline void		btfreeccb(struct bt_softc *bt,
 					  struct bt_ccb *bccb);
 static void		btallocccbs(struct bt_softc *bt);
 static bus_dmamap_callback_t btexecuteccb;
 static void		btdone(struct bt_softc *bt, struct bt_ccb *bccb,
 			       bt_mbi_comp_code_t comp_code);
 static void		bt_intr_locked(struct bt_softc *bt);
 
 /* Host adapter command functions */
 static int	btreset(struct bt_softc* bt, int hard_reset);
 
 /* Initialization functions */
 static int			btinitmboxes(struct bt_softc *bt);
 static bus_dmamap_callback_t	btmapmboxes;
 static bus_dmamap_callback_t	btmapccbs;
 static bus_dmamap_callback_t	btmapsgs;
 
 /* Transfer Negotiation Functions */
 static void btfetchtransinfo(struct bt_softc *bt,
 			     struct ccb_trans_settings *cts);
 
 /* CAM SIM entry points */
 #define ccb_bccb_ptr spriv_ptr0
 #define ccb_bt_ptr spriv_ptr1
 static void	btaction(struct cam_sim *sim, union ccb *ccb);
 static void	btpoll(struct cam_sim *sim);
 
 /* Our timeout handler */
 static void	bttimeout(void *arg);
 
 /*
  * XXX
  * Do our own re-probe protection until a configuration
  * manager can do it for us.  This ensures that we don't
  * reprobe a card already found by the EISA or PCI probes.
  */
 struct bt_isa_port bt_isa_ports[] =
 {
 	{ 0x130, 0, 4 },
 	{ 0x134, 0, 5 },
 	{ 0x230, 0, 2 },
 	{ 0x234, 0, 3 },
 	{ 0x330, 0, 0 },
 	{ 0x334, 0, 1 }
 };
 
 /*
  * I/O ports listed in the order enumerated by the
  * card for certain op codes.
  */
 u_int16_t bt_board_ports[] =
 {
 	0x330,
 	0x334,
 	0x230,
 	0x234,
 	0x130,
 	0x134
 };
 
 /* Exported functions */
 void
 bt_init_softc(device_t dev, struct resource *port,
 	      struct resource *irq, struct resource *drq)
 {
 	struct bt_softc *bt = device_get_softc(dev);
 
 	SLIST_INIT(&bt->free_bt_ccbs);
 	LIST_INIT(&bt->pending_ccbs);
 	SLIST_INIT(&bt->sg_maps);
 	bt->dev = dev;
 	bt->port = port;
 	bt->irq = irq;
 	bt->drq = drq;
 	mtx_init(&bt->lock, "bt", NULL, MTX_DEF);
 }
 
 void
 bt_free_softc(device_t dev)
 {
 	struct bt_softc *bt = device_get_softc(dev);
 
 	switch (bt->init_level) {
 	default:
 	case 11:
 		bus_dmamap_unload(bt->sense_dmat, bt->sense_dmamap);
 	case 10:
 		bus_dmamem_free(bt->sense_dmat, bt->sense_buffers,
 				bt->sense_dmamap);
 	case 9:
 		bus_dma_tag_destroy(bt->sense_dmat);
 	case 8:
 	{
 		struct sg_map_node *sg_map;
 
 		while ((sg_map = SLIST_FIRST(&bt->sg_maps))!= NULL) {
 			SLIST_REMOVE_HEAD(&bt->sg_maps, links);
 			bus_dmamap_unload(bt->sg_dmat,
 					  sg_map->sg_dmamap);
 			bus_dmamem_free(bt->sg_dmat, sg_map->sg_vaddr,
 					sg_map->sg_dmamap);
 			free(sg_map, M_DEVBUF);
 		}
 		bus_dma_tag_destroy(bt->sg_dmat);
 	}
 	case 7:
 		bus_dmamap_unload(bt->ccb_dmat, bt->ccb_dmamap);
 		/* FALLTHROUGH */
 	case 6:
 		bus_dmamem_free(bt->ccb_dmat, bt->bt_ccb_array,
 				bt->ccb_dmamap);
-		bus_dmamap_destroy(bt->ccb_dmat, bt->ccb_dmamap);
 		/* FALLTHROUGH */
 	case 5:
 		bus_dma_tag_destroy(bt->ccb_dmat);
 		/* FALLTHROUGH */
 	case 4:
 		bus_dmamap_unload(bt->mailbox_dmat, bt->mailbox_dmamap);
 		/* FALLTHROUGH */
 	case 3:
 		bus_dmamem_free(bt->mailbox_dmat, bt->in_boxes,
 				bt->mailbox_dmamap);
-		bus_dmamap_destroy(bt->mailbox_dmat, bt->mailbox_dmamap);
 		/* FALLTHROUGH */
 	case 2:
 		bus_dma_tag_destroy(bt->buffer_dmat);
 		/* FALLTHROUGH */
 	case 1:
 		bus_dma_tag_destroy(bt->mailbox_dmat);
 		/* FALLTHROUGH */
 	case 0:
 		break;
 	}
 	mtx_destroy(&bt->lock);
 }
 
 int
 bt_port_probe(device_t dev, struct bt_probe_info *info)
 {
 	struct bt_softc *bt = device_get_softc(dev);
 	config_data_t config_data;
 	int error;
 
 	/* See if there is really a card present */
 	if (bt_probe(dev) || bt_fetch_adapter_info(dev))
 		return(1);
 
 	/*
 	 * Determine our IRQ, and DMA settings and
 	 * export them to the configuration system.
 	 */
 	mtx_lock(&bt->lock);
 	error = bt_cmd(bt, BOP_INQUIRE_CONFIG, NULL, /*parmlen*/0,
 		       (u_int8_t*)&config_data, sizeof(config_data),
 		       DEFAULT_CMD_TIMEOUT);
 	mtx_unlock(&bt->lock);
 	if (error != 0) {
 		printf("bt_port_probe: Could not determine IRQ or DMA "
 		       "settings for adapter.\n");
 		return (1);
 	}
 
 	if (bt->model[0] == '5') {
 		/* DMA settings only make sense for ISA cards */
 		switch (config_data.dma_chan) {
 		case DMA_CHAN_5:
 			info->drq = 5;
 			break;
 		case DMA_CHAN_6:
 			info->drq = 6;
 			break;
 		case DMA_CHAN_7:
 			info->drq = 7;
 			break;
 		default:
 			printf("bt_port_probe: Invalid DMA setting "
 			       "detected for adapter.\n");
 			return (1);
 		}
 	} else {
 		/* VL/EISA/PCI DMA */
 		info->drq = -1;
 	}
 	switch (config_data.irq) {
 	case IRQ_9:
 	case IRQ_10:
 	case IRQ_11:
 	case IRQ_12:
 	case IRQ_14:
 	case IRQ_15:
 		info->irq = ffs(config_data.irq) + 8;
 		break;
 	default:
 		printf("bt_port_probe: Invalid IRQ setting %x"
 		       "detected for adapter.\n", config_data.irq);
 		return (1);
 	}
 	return (0);
 }
 
 /*
  * Probe the adapter and verify that the card is a BusLogic.
  */
 int
 bt_probe(device_t dev)
 {
 	struct bt_softc *bt = device_get_softc(dev);
 	esetup_info_data_t esetup_info;
 	u_int	 status;
 	u_int	 intstat;
 	u_int	 geometry;
 	int	 error;
 	u_int8_t param;
 
 	/*
 	 * See if the three I/O ports look reasonable.
 	 * Touch the minimal number of registers in the
 	 * failure case.
 	 */
 	status = bt_inb(bt, STATUS_REG);
 	if ((status == 0)
 	 || (status & (DIAG_ACTIVE|CMD_REG_BUSY|
 		       STATUS_REG_RSVD|CMD_INVALID)) != 0) {
 		if (bootverbose)
 			device_printf(dev, "Failed Status Reg Test - %x\n",
 			       status);
 		return (ENXIO);
 	}
 
 	intstat = bt_inb(bt, INTSTAT_REG);
 	if ((intstat & INTSTAT_REG_RSVD) != 0) {
 		device_printf(dev, "Failed Intstat Reg Test\n");
 		return (ENXIO);
 	}
 
 	geometry = bt_inb(bt, GEOMETRY_REG);
 	if (geometry == 0xFF) {
 		if (bootverbose)
 			device_printf(dev, "Failed Geometry Reg Test\n");
 		return (ENXIO);
 	}
 
 	/*
 	 * Looking good so far.  Final test is to reset the
 	 * adapter and attempt to fetch the extended setup
 	 * information.  This should filter out all 1542 cards.
 	 */
 	mtx_lock(&bt->lock);
 	if ((error = btreset(bt, /*hard_reset*/TRUE)) != 0) {
 		mtx_unlock(&bt->lock);
 		if (bootverbose)
 			device_printf(dev, "Failed Reset\n");
 		return (ENXIO);
 	}
 	
 	param = sizeof(esetup_info);
 	error = bt_cmd(bt, BOP_INQUIRE_ESETUP_INFO, &param, /*parmlen*/1,
 		       (u_int8_t*)&esetup_info, sizeof(esetup_info),
 		       DEFAULT_CMD_TIMEOUT);
 	mtx_unlock(&bt->lock);
 	if (error != 0) {
 		return (ENXIO);
 	}
 
 	return (0);
 }
 
 /*
  * Pull the boards setup information and record it in our softc.
  */
 int
 bt_fetch_adapter_info(device_t dev)
 {
 	struct bt_softc *bt = device_get_softc(dev);
 	board_id_data_t	board_id;
 	esetup_info_data_t esetup_info;
 	config_data_t config_data;
 	int	 error;
 	u_int8_t length_param;
 
 	/* First record the firmware version */
 	mtx_lock(&bt->lock);
 	error = bt_cmd(bt, BOP_INQUIRE_BOARD_ID, NULL, /*parmlen*/0,
 		       (u_int8_t*)&board_id, sizeof(board_id),
 		       DEFAULT_CMD_TIMEOUT);
 	if (error != 0) {
 		mtx_unlock(&bt->lock);
 		device_printf(dev, "bt_fetch_adapter_info - Failed Get Board Info\n");
 		return (error);
 	}
 	bt->firmware_ver[0] = board_id.firmware_rev_major;
 	bt->firmware_ver[1] = '.';
 	bt->firmware_ver[2] = board_id.firmware_rev_minor;
 	bt->firmware_ver[3] = '\0';
 		
 	/*
 	 * Depending on the firmware major and minor version,
 	 * we may be able to fetch additional minor version info.
 	 */
 	if (bt->firmware_ver[0] > '0') {
 		
 		error = bt_cmd(bt, BOP_INQUIRE_FW_VER_3DIG, NULL, /*parmlen*/0,
 			       (u_int8_t*)&bt->firmware_ver[3], 1,
 			       DEFAULT_CMD_TIMEOUT);
 		if (error != 0) {
 			mtx_unlock(&bt->lock);
 			device_printf(dev,
 				      "bt_fetch_adapter_info - Failed Get "
 				      "Firmware 3rd Digit\n");
 			return (error);
 		}
 		if (bt->firmware_ver[3] == ' ')
 			bt->firmware_ver[3] = '\0';
 		bt->firmware_ver[4] = '\0';
 	}
 
 	if (strcmp(bt->firmware_ver, "3.3") >= 0) {
 
 		error = bt_cmd(bt, BOP_INQUIRE_FW_VER_4DIG, NULL, /*parmlen*/0,
 			       (u_int8_t*)&bt->firmware_ver[4], 1,
 			       DEFAULT_CMD_TIMEOUT);
 		if (error != 0) {
 			mtx_unlock(&bt->lock);
 			device_printf(dev,
 				      "bt_fetch_adapter_info - Failed Get "
 				      "Firmware 4th Digit\n");
 			return (error);
 		}
 		if (bt->firmware_ver[4] == ' ')
 			bt->firmware_ver[4] = '\0';
 		bt->firmware_ver[5] = '\0';
 	}
 
 	/*
 	 * Some boards do not handle the "recently documented"
 	 * Inquire Board Model Number command correctly or do not give
 	 * exact information.  Use the Firmware and Extended Setup
 	 * information in these cases to come up with the right answer.
 	 * The major firmware revision number indicates:
 	 *
 	 * 	5.xx	BusLogic "W" Series Host Adapters:
 	 *		BT-948/958/958D
 	 *	4.xx	BusLogic "C" Series Host Adapters:
 	 *		BT-946C/956C/956CD/747C/757C/757CD/445C/545C/540CF
 	 *	3.xx	BusLogic "S" Series Host Adapters:
 	 *		BT-747S/747D/757S/757D/445S/545S/542D
 	 *		BT-542B/742A (revision H)
 	 *	2.xx	BusLogic "A" Series Host Adapters:
 	 *		BT-542B/742A (revision G and below)
 	 *	0.xx	AMI FastDisk VLB/EISA BusLogic Clone Host Adapter
 	 */
 	length_param = sizeof(esetup_info);
 	error = bt_cmd(bt, BOP_INQUIRE_ESETUP_INFO, &length_param, /*parmlen*/1,
 		       (u_int8_t*)&esetup_info, sizeof(esetup_info),
 		       DEFAULT_CMD_TIMEOUT);
 	if (error != 0) {
 		mtx_unlock(&bt->lock);
 		return (error);
 	}
 	
   	bt->bios_addr = esetup_info.bios_addr << 12;
 
 	bt->mailbox_addrlimit = BUS_SPACE_MAXADDR;
 	if (esetup_info.bus_type == 'A'
 	 && bt->firmware_ver[0] == '2') {
 		snprintf(bt->model, sizeof(bt->model), "542B");
 	} else if (esetup_info.bus_type == 'E'
 	 	&& bt->firmware_ver[0] == '2') {
 
 		/*
 		 * The 742A seems to object if its mailboxes are
 		 * allocated above the 16MB mark.
 		 */
 		bt->mailbox_addrlimit = BUS_SPACE_MAXADDR_24BIT;
 		snprintf(bt->model, sizeof(bt->model), "742A");
 	} else if (esetup_info.bus_type == 'E'
 		&& bt->firmware_ver[0] == '0') {
 		/* AMI FastDisk EISA Series 441 0.x */
 		snprintf(bt->model, sizeof(bt->model), "747A");
 	} else {
 		ha_model_data_t model_data;
 		int i;
 
 		length_param = sizeof(model_data);
 		error = bt_cmd(bt, BOP_INQUIRE_MODEL, &length_param, 1,
 			       (u_int8_t*)&model_data, sizeof(model_data),
 			       DEFAULT_CMD_TIMEOUT);
 		if (error != 0) {
 			mtx_unlock(&bt->lock);
 			device_printf(dev,
 				      "bt_fetch_adapter_info - Failed Inquire "
 				      "Model Number\n");
 			return (error);
 		}
 		for (i = 0; i < sizeof(model_data.ascii_model); i++) {
 			bt->model[i] = model_data.ascii_model[i];
 			if (bt->model[i] == ' ')
 				break;
 		}
 		bt->model[i] = '\0';
 	}
 
 	bt->level_trigger_ints = esetup_info.level_trigger_ints ? 1 : 0;
 
 	/* SG element limits */
 	bt->max_sg = esetup_info.max_sg;
 
 	/* Set feature flags */
 	bt->wide_bus = esetup_info.wide_bus;
 	bt->diff_bus = esetup_info.diff_bus;
 	bt->ultra_scsi = esetup_info.ultra_scsi;
 
 	if ((bt->firmware_ver[0] == '5')
 	 || (bt->firmware_ver[0] == '4' && bt->wide_bus))
 		bt->extended_lun = TRUE;
 
 	bt->strict_rr = (strcmp(bt->firmware_ver, "3.31") >= 0);
 
 	bt->extended_trans =
 	    ((bt_inb(bt, GEOMETRY_REG) & EXTENDED_TRANSLATION) != 0);
 
 	/*
 	 * Determine max CCB count and whether tagged queuing is
 	 * available based on controller type. Tagged queuing
 	 * only works on 'W' series adapters, 'C' series adapters
 	 * with firmware of rev 4.42 and higher, and 'S' series
 	 * adapters with firmware of rev 3.35 and higher.  The
 	 * maximum CCB counts are as follows:
 	 *
 	 *	192	BT-948/958/958D
 	 *	100	BT-946C/956C/956CD/747C/757C/757CD/445C
 	 * 	50	BT-545C/540CF
 	 * 	30	BT-747S/747D/757S/757D/445S/545S/542D/542B/742A
 	 */
 	if (bt->firmware_ver[0] == '5') {
 		bt->max_ccbs = 192;
 		bt->tag_capable = TRUE;
 	} else if (bt->firmware_ver[0] == '4') {
 		if (bt->model[0] == '5')
 			bt->max_ccbs = 50;
 		else
 			bt->max_ccbs = 100;
 		bt->tag_capable = (strcmp(bt->firmware_ver, "4.22") >= 0);
 	} else {
 		bt->max_ccbs = 30;
 		if (bt->firmware_ver[0] == '3'
 		 && (strcmp(bt->firmware_ver, "3.35") >= 0))
 			bt->tag_capable = TRUE;
 		else
 			bt->tag_capable = FALSE;
 	}
 
 	if (bt->tag_capable != FALSE)
 		bt->tags_permitted = ALL_TARGETS;
 
 	/* Determine Sync/Wide/Disc settings */
 	if (bt->firmware_ver[0] >= '4') {
 		auto_scsi_data_t auto_scsi_data;
 		fetch_lram_params_t fetch_lram_params;
 		int error;
 		
 		/*
 		 * These settings are stored in the
 		 * AutoSCSI data in LRAM of 'W' and 'C'
 		 * adapters.
 		 */
 		fetch_lram_params.offset = AUTO_SCSI_BYTE_OFFSET;
 		fetch_lram_params.response_len = sizeof(auto_scsi_data);
 		error = bt_cmd(bt, BOP_FETCH_LRAM,
 			       (u_int8_t*)&fetch_lram_params,
 			       sizeof(fetch_lram_params),
 			       (u_int8_t*)&auto_scsi_data,
 			       sizeof(auto_scsi_data), DEFAULT_CMD_TIMEOUT);
 
 		if (error != 0) {
 			mtx_unlock(&bt->lock);
 			device_printf(dev,
 				      "bt_fetch_adapter_info - Failed "
 				      "Get Auto SCSI Info\n");
 			return (error);
 		}
 
 		bt->disc_permitted = auto_scsi_data.low_disc_permitted
 				   | (auto_scsi_data.high_disc_permitted << 8);
 		bt->sync_permitted = auto_scsi_data.low_sync_permitted
 				   | (auto_scsi_data.high_sync_permitted << 8);
 		bt->fast_permitted = auto_scsi_data.low_fast_permitted
 				   | (auto_scsi_data.high_fast_permitted << 8);
 		bt->ultra_permitted = auto_scsi_data.low_ultra_permitted
 				   | (auto_scsi_data.high_ultra_permitted << 8);
 		bt->wide_permitted = auto_scsi_data.low_wide_permitted
 				   | (auto_scsi_data.high_wide_permitted << 8);
 
 		if (bt->ultra_scsi == FALSE)
 			bt->ultra_permitted = 0;
 
 		if (bt->wide_bus == FALSE)
 			bt->wide_permitted = 0;
 	} else {
 		/*
 		 * 'S' and 'A' series have this information in the setup
 		 * information structure.
 		 */
 		setup_data_t	setup_info;
 
 		length_param = sizeof(setup_info);
 		error = bt_cmd(bt, BOP_INQUIRE_SETUP_INFO, &length_param,
 			       /*paramlen*/1, (u_int8_t*)&setup_info,
 			       sizeof(setup_info), DEFAULT_CMD_TIMEOUT);
 
 		if (error != 0) {
 			mtx_unlock(&bt->lock);
 			device_printf(dev,
 				      "bt_fetch_adapter_info - Failed "
 				      "Get Setup Info\n");
 			return (error);
 		}
 
 		if (setup_info.initiate_sync != 0) {
 			bt->sync_permitted = ALL_TARGETS;
 
 			if (bt->model[0] == '7') {
 				if (esetup_info.sync_neg10MB != 0)
 					bt->fast_permitted = ALL_TARGETS;
 				if (strcmp(bt->model, "757") == 0)
 					bt->wide_permitted = ALL_TARGETS;
 			}
 		}
 		bt->disc_permitted = ALL_TARGETS;
 	}
 
 	/* We need as many mailboxes as we can have ccbs */
 	bt->num_boxes = bt->max_ccbs;
 
 	/* Determine our SCSI ID */
 	
 	error = bt_cmd(bt, BOP_INQUIRE_CONFIG, NULL, /*parmlen*/0,
 		       (u_int8_t*)&config_data, sizeof(config_data),
 		       DEFAULT_CMD_TIMEOUT);
 	mtx_unlock(&bt->lock);
 	if (error != 0) {
 		device_printf(dev,
 			      "bt_fetch_adapter_info - Failed Get Config\n");
 		return (error);
 	}
 	bt->scsi_id = config_data.scsi_id;
 
 	return (0);
 }
 
 /*
  * Start the board, ready for normal operation
  */
 int
 bt_init(device_t dev)
 {
 	struct bt_softc *bt = device_get_softc(dev);
 
 	/* Announce the Adapter */
 	device_printf(dev, "BT-%s FW Rev. %s ", bt->model, bt->firmware_ver);
 
 	if (bt->ultra_scsi != 0)
 		printf("Ultra ");
 
 	if (bt->wide_bus != 0)
 		printf("Wide ");
 	else
 		printf("Narrow ");
 
 	if (bt->diff_bus != 0)
 		printf("Diff ");
 
 	printf("SCSI Host Adapter, SCSI ID %d, %d CCBs\n", bt->scsi_id,
 	       bt->max_ccbs);
 
 	/*
 	 * Create our DMA tags.  These tags define the kinds of device
 	 * accessible memory allocations and memory mappings we will 
 	 * need to perform during normal operation.
 	 *
 	 * Unless we need to further restrict the allocation, we rely
 	 * on the restrictions of the parent dmat, hence the common
 	 * use of MAXADDR and MAXSIZE.
 	 */
 
 	/* DMA tag for mapping buffers into device visible space. */
 	if (bus_dma_tag_create( /* parent	*/ bt->parent_dmat,
 				/* alignment	*/ 1,
 				/* boundary	*/ 0,
 				/* lowaddr	*/ BUS_SPACE_MAXADDR,
 				/* highaddr	*/ BUS_SPACE_MAXADDR,
 				/* filter	*/ NULL,
 				/* filterarg	*/ NULL,
 				/* maxsize	*/ MAXBSIZE,
 				/* nsegments	*/ BT_NSEG,
 				/* maxsegsz	*/ BUS_SPACE_MAXSIZE_32BIT,
 				/* flags	*/ BUS_DMA_ALLOCNOW,
 				/* lockfunc	*/ busdma_lock_mutex,
 				/* lockarg	*/ &bt->lock,
 				&bt->buffer_dmat) != 0) {
 		goto error_exit;
 	}
 
 	bt->init_level++;
 	/* DMA tag for our mailboxes */
 	if (bus_dma_tag_create(	/* parent	*/ bt->parent_dmat,
 				/* alignment	*/ 1,
 				/* boundary	*/ 0,
 				/* lowaddr	*/ bt->mailbox_addrlimit,
 				/* highaddr	*/ BUS_SPACE_MAXADDR,
 				/* filter	*/ NULL,
 				/* filterarg	*/ NULL,
 				/* maxsize	*/ bt->num_boxes *
 						   (sizeof(bt_mbox_in_t) +
 						    sizeof(bt_mbox_out_t)),
 				/* nsegments	*/ 1,
 				/* maxsegsz	*/ BUS_SPACE_MAXSIZE_32BIT,
 				/* flags	*/ 0,
 				/* lockfunc	*/ NULL,
 				/* lockarg	*/ NULL,
 				&bt->mailbox_dmat) != 0) {
 		goto error_exit;
         }
 
 	bt->init_level++;
 
 	/* Allocation for our mailboxes */
 	if (bus_dmamem_alloc(bt->mailbox_dmat, (void **)&bt->out_boxes,
 			     BUS_DMA_NOWAIT, &bt->mailbox_dmamap) != 0) {
 		goto error_exit;
 	}
 
 	bt->init_level++;
 
 	/* And permanently map them */
 	bus_dmamap_load(bt->mailbox_dmat, bt->mailbox_dmamap,
        			bt->out_boxes,
 			bt->num_boxes * (sizeof(bt_mbox_in_t)
 				       + sizeof(bt_mbox_out_t)),
 			btmapmboxes, bt, /*flags*/0);
 
 	bt->init_level++;
 
 	bt->in_boxes = (bt_mbox_in_t *)&bt->out_boxes[bt->num_boxes];
 
 	mtx_lock(&bt->lock);
 	btinitmboxes(bt);
 	mtx_unlock(&bt->lock);
 
 	/* DMA tag for our ccb structures */
 	if (bus_dma_tag_create(	/* parent	*/ bt->parent_dmat,
 				/* alignment	*/ 1,
 				/* boundary	*/ 0,
 				/* lowaddr	*/ BUS_SPACE_MAXADDR,
 				/* highaddr	*/ BUS_SPACE_MAXADDR,
 				/* filter	*/ NULL,
 				/* filterarg	*/ NULL,
 				/* maxsize	*/ bt->max_ccbs *
 						   sizeof(struct bt_ccb),
 				/* nsegments	*/ 1,
 				/* maxsegsz	*/ BUS_SPACE_MAXSIZE_32BIT,
 				/* flags	*/ 0,
 				/* lockfunc	*/ NULL,
 				/* lockarg	*/ NULL,
 				&bt->ccb_dmat) != 0) {
 		goto error_exit;
         }
 
 	bt->init_level++;
 
 	/* Allocation for our ccbs */
 	if (bus_dmamem_alloc(bt->ccb_dmat, (void **)&bt->bt_ccb_array,
 			     BUS_DMA_NOWAIT, &bt->ccb_dmamap) != 0) {
 		goto error_exit;
 	}
 
 	bt->init_level++;
 
 	/* And permanently map them */
 	bus_dmamap_load(bt->ccb_dmat, bt->ccb_dmamap,
        			bt->bt_ccb_array,
 			bt->max_ccbs * sizeof(struct bt_ccb),
 			btmapccbs, bt, /*flags*/0);
 
 	bt->init_level++;
 
 	/* DMA tag for our S/G structures.  We allocate in page sized chunks */
 	if (bus_dma_tag_create(	/* parent	*/ bt->parent_dmat,
 				/* alignment	*/ 1,
 				/* boundary	*/ 0,
 				/* lowaddr	*/ BUS_SPACE_MAXADDR,
 				/* highaddr	*/ BUS_SPACE_MAXADDR,
 				/* filter	*/ NULL,
 				/* filterarg	*/ NULL,
 				/* maxsize	*/ PAGE_SIZE,
 				/* nsegments	*/ 1,
 				/* maxsegsz	*/ BUS_SPACE_MAXSIZE_32BIT,
 				/* flags	*/ 0,
 				/* lockfunc	*/ NULL,
 				/* lockarg	*/ NULL,
 				&bt->sg_dmat) != 0) {
 		goto error_exit;
         }
 
 	bt->init_level++;
 
 	/* Perform initial CCB allocation */
 	bzero(bt->bt_ccb_array, bt->max_ccbs * sizeof(struct bt_ccb));
 	btallocccbs(bt);
 
 	if (bt->num_ccbs == 0) {
 		device_printf(dev,
 			      "bt_init - Unable to allocate initial ccbs\n");
 		goto error_exit;
 	}
 
 	/*
 	 * Note that we are going and return (to attach)
 	 */
 	return 0;
 
 error_exit:
 
 	return (ENXIO);
 }
 
 int
 bt_attach(device_t dev)
 {
 	struct bt_softc *bt = device_get_softc(dev);
 	int tagged_dev_openings;
 	struct cam_devq *devq;
 	int error;
 
 	/*
 	 * We reserve 1 ccb for error recovery, so don't
 	 * tell the XPT about it.
 	 */
 	if (bt->tag_capable != 0)
 		tagged_dev_openings = bt->max_ccbs - 1;
 	else
 		tagged_dev_openings = 0;
 
 	/*
 	 * Create the device queue for our SIM.
 	 */
 	devq = cam_simq_alloc(bt->max_ccbs - 1);
 	if (devq == NULL)
 		return (ENOMEM);
 
 	/*
 	 * Construct our SIM entry
 	 */
 	bt->sim = cam_sim_alloc(btaction, btpoll, "bt", bt,
 	    device_get_unit(bt->dev), &bt->lock, 2, tagged_dev_openings, devq);
 	if (bt->sim == NULL) {
 		cam_simq_free(devq);
 		return (ENOMEM);
 	}
 
 	mtx_lock(&bt->lock);
 	if (xpt_bus_register(bt->sim, dev, 0) != CAM_SUCCESS) {
 		cam_sim_free(bt->sim, /*free_devq*/TRUE);
 		mtx_unlock(&bt->lock);
 		return (ENXIO);
 	}
 	
 	if (xpt_create_path(&bt->path, /*periph*/NULL,
 			    cam_sim_path(bt->sim), CAM_TARGET_WILDCARD,
 			    CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
 		xpt_bus_deregister(cam_sim_path(bt->sim));
 		cam_sim_free(bt->sim, /*free_devq*/TRUE);
 		mtx_unlock(&bt->lock);
 		return (ENXIO);
 	}
 	mtx_unlock(&bt->lock);
 		
 	/*
 	 * Setup interrupt.
 	 */
 	error = bus_setup_intr(dev, bt->irq, INTR_TYPE_CAM | INTR_ENTROPY |
 	    INTR_MPSAFE, NULL, bt_intr, bt, &bt->ih);
 	if (error) {
 		device_printf(dev, "bus_setup_intr() failed: %d\n", error);
 		return (error);
 	}
 
 	return (0);
 }
 
 int
 bt_check_probed_iop(u_int ioport)
 {
 	u_int i;
 
 	for (i = 0; i < BT_NUM_ISAPORTS; i++) {
 		if (bt_isa_ports[i].addr == ioport) {
 			if (bt_isa_ports[i].probed != 0)
 				return (1);
 			else {
 				return (0);
 			}
 		}
 	}
 	return (1);
 }
 
 void
 bt_mark_probed_bio(isa_compat_io_t port)
 {
 	if (port < BIO_DISABLED)
 		bt_mark_probed_iop(bt_board_ports[port]);
 }
 
 void
 bt_mark_probed_iop(u_int ioport)
 {
 	u_int i;
 
 	for (i = 0; i < BT_NUM_ISAPORTS; i++) {
 		if (ioport == bt_isa_ports[i].addr) {
 			bt_isa_ports[i].probed = 1;
 			break;
 		}
 	}
 }
 
 void
 bt_find_probe_range(int ioport, int *port_index, int *max_port_index)
 {
 	if (ioport > 0) {
 		int i;
 
 		for (i = 0;i < BT_NUM_ISAPORTS; i++)
 			if (ioport <= bt_isa_ports[i].addr)
 				break;
 		if ((i >= BT_NUM_ISAPORTS)
 		 || (ioport != bt_isa_ports[i].addr)) {
 			printf(
 "bt_find_probe_range: Invalid baseport of 0x%x specified.\n"
 "bt_find_probe_range: Nearest valid baseport is 0x%x.\n"
 "bt_find_probe_range: Failing probe.\n",
 			       ioport,
 			       (i < BT_NUM_ISAPORTS)
 				    ? bt_isa_ports[i].addr
 				    : bt_isa_ports[BT_NUM_ISAPORTS - 1].addr);
 			*port_index = *max_port_index = -1;
 			return;
 		}
 		*port_index = *max_port_index = bt_isa_ports[i].bio;
 	} else {
 		*port_index = 0;
 		*max_port_index = BT_NUM_ISAPORTS - 1;
 	}
 }
 
 int
 bt_iop_from_bio(isa_compat_io_t bio_index)
 {
 	if (bio_index < BT_NUM_ISAPORTS)
 		return (bt_board_ports[bio_index]);
 	return (-1);
 }
 
 
 static void
 btallocccbs(struct bt_softc *bt)
 {
 	struct bt_ccb *next_ccb;
 	struct sg_map_node *sg_map;
 	bus_addr_t physaddr;
 	bt_sg_t *segs;
 	int newcount;
 	int i;
 
 	if (bt->num_ccbs >= bt->max_ccbs)
 		/* Can't allocate any more */
 		return;
 
 	next_ccb = &bt->bt_ccb_array[bt->num_ccbs];
 
 	sg_map = malloc(sizeof(*sg_map), M_DEVBUF, M_NOWAIT);
 
 	if (sg_map == NULL)
 		goto error_exit;
 
 	/* Allocate S/G space for the next batch of CCBS */
 	if (bus_dmamem_alloc(bt->sg_dmat, (void **)&sg_map->sg_vaddr,
 			     BUS_DMA_NOWAIT, &sg_map->sg_dmamap) != 0) {
 		free(sg_map, M_DEVBUF);
 		goto error_exit;
 	}
 
 	SLIST_INSERT_HEAD(&bt->sg_maps, sg_map, links);
 
 	bus_dmamap_load(bt->sg_dmat, sg_map->sg_dmamap, sg_map->sg_vaddr,
 			PAGE_SIZE, btmapsgs, bt, /*flags*/0);
 	
 	segs = sg_map->sg_vaddr;
 	physaddr = sg_map->sg_physaddr;
 
 	newcount = (PAGE_SIZE / (BT_NSEG * sizeof(bt_sg_t)));
 	for (i = 0; bt->num_ccbs < bt->max_ccbs && i < newcount; i++) {
 		int error;
 
 		next_ccb->sg_list = segs;
 		next_ccb->sg_list_phys = physaddr;
 		next_ccb->flags = BCCB_FREE;
 		callout_init_mtx(&next_ccb->timer, &bt->lock, 0);
 		error = bus_dmamap_create(bt->buffer_dmat, /*flags*/0,
 					  &next_ccb->dmamap);
 		if (error != 0)
 			break;
 		SLIST_INSERT_HEAD(&bt->free_bt_ccbs, next_ccb, links);
 		segs += BT_NSEG;
 		physaddr += (BT_NSEG * sizeof(bt_sg_t));
 		next_ccb++;
 		bt->num_ccbs++;
 	}
 
 	/* Reserve a CCB for error recovery */
 	if (bt->recovery_bccb == NULL) {
 		bt->recovery_bccb = SLIST_FIRST(&bt->free_bt_ccbs);
 		SLIST_REMOVE_HEAD(&bt->free_bt_ccbs, links);
 	}
 
 	if (SLIST_FIRST(&bt->free_bt_ccbs) != NULL)
 		return;
 
 error_exit:
 	device_printf(bt->dev, "Can't malloc BCCBs\n");
 }
 
 static __inline void
 btfreeccb(struct bt_softc *bt, struct bt_ccb *bccb)
 {
 
 	if (!dumping)
 		mtx_assert(&bt->lock, MA_OWNED);
 	if ((bccb->flags & BCCB_ACTIVE) != 0)
 		LIST_REMOVE(&bccb->ccb->ccb_h, sim_links.le);
 	if (bt->resource_shortage != 0
 	 && (bccb->ccb->ccb_h.status & CAM_RELEASE_SIMQ) == 0) {
 		bccb->ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
 		bt->resource_shortage = FALSE;
 	}
 	bccb->flags = BCCB_FREE;
 	SLIST_INSERT_HEAD(&bt->free_bt_ccbs, bccb, links);
 	bt->active_ccbs--;
 }
 
 static __inline struct bt_ccb*
 btgetccb(struct bt_softc *bt)
 {
 	struct	bt_ccb* bccb;
 
 	if (!dumping)
 		mtx_assert(&bt->lock, MA_OWNED);
 	if ((bccb = SLIST_FIRST(&bt->free_bt_ccbs)) != NULL) {
 		SLIST_REMOVE_HEAD(&bt->free_bt_ccbs, links);
 		bt->active_ccbs++;
 	} else {
 		btallocccbs(bt);
 		bccb = SLIST_FIRST(&bt->free_bt_ccbs);
 		if (bccb != NULL) {
 			SLIST_REMOVE_HEAD(&bt->free_bt_ccbs, links);
 			bt->active_ccbs++;
 		}
 	}
 
 	return (bccb);
 }
 
 static void
 btaction(struct cam_sim *sim, union ccb *ccb)
 {
 	struct	bt_softc *bt;
 
 	CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE, ("btaction\n"));
 	
 	bt = (struct bt_softc *)cam_sim_softc(sim);
 	mtx_assert(&bt->lock, MA_OWNED);
 	
 	switch (ccb->ccb_h.func_code) {
 	/* Common cases first */
 	case XPT_SCSI_IO:	/* Execute the requested I/O operation */
 	case XPT_RESET_DEV:	/* Bus Device Reset the specified SCSI device */
 	{
 		struct	bt_ccb	*bccb;
 		struct	bt_hccb *hccb;
 
 		/*
 		 * get a bccb to use.
 		 */
 		if ((bccb = btgetccb(bt)) == NULL) {
 
 			bt->resource_shortage = TRUE;
 			xpt_freeze_simq(bt->sim, /*count*/1);
 			ccb->ccb_h.status = CAM_REQUEUE_REQ;
 			xpt_done(ccb);
 			return;
 		}
 		
 		hccb = &bccb->hccb;
 
 		/*
 		 * So we can find the BCCB when an abort is requested
 		 */
 		bccb->ccb = ccb;
 		ccb->ccb_h.ccb_bccb_ptr = bccb;
 		ccb->ccb_h.ccb_bt_ptr = bt;
 
 		/*
 		 * Put all the arguments for the xfer in the bccb
 		 */
 		hccb->target_id = ccb->ccb_h.target_id;
 		hccb->target_lun = ccb->ccb_h.target_lun;
 		hccb->btstat = 0;
 		hccb->sdstat = 0;
 
 		if (ccb->ccb_h.func_code == XPT_SCSI_IO) {
 			struct ccb_scsiio *csio;
 			struct ccb_hdr *ccbh;
 			int error;
 
 			csio = &ccb->csio;
 			ccbh = &csio->ccb_h;
 			hccb->opcode = INITIATOR_CCB_WRESID;
 			hccb->datain = (ccb->ccb_h.flags & CAM_DIR_IN) ? 1 : 0;
 			hccb->dataout =(ccb->ccb_h.flags & CAM_DIR_OUT) ? 1 : 0;
 			hccb->cmd_len = csio->cdb_len;
 			if (hccb->cmd_len > sizeof(hccb->scsi_cdb)) {
 				ccb->ccb_h.status = CAM_REQ_INVALID;
 				btfreeccb(bt, bccb);
 				xpt_done(ccb);
 				return;
 			}
 			hccb->sense_len = csio->sense_len;
 			if ((ccbh->flags & CAM_TAG_ACTION_VALID) != 0
 			 && ccb->csio.tag_action != CAM_TAG_ACTION_NONE) {
 				hccb->tag_enable = TRUE;
 				hccb->tag_type = (ccb->csio.tag_action & 0x3);
 			} else {
 				hccb->tag_enable = FALSE;
 				hccb->tag_type = 0;
 			}
 			if ((ccbh->flags & CAM_CDB_POINTER) != 0) {
 				if ((ccbh->flags & CAM_CDB_PHYS) == 0) {
 					bcopy(csio->cdb_io.cdb_ptr,
 					      hccb->scsi_cdb, hccb->cmd_len);
 				} else {
 					/* I guess I could map it in... */
 					ccbh->status = CAM_REQ_INVALID;
 					btfreeccb(bt, bccb);
 					xpt_done(ccb);
 					return;
 				}
 			} else {
 				bcopy(csio->cdb_io.cdb_bytes,
 				      hccb->scsi_cdb, hccb->cmd_len);
 			}
 			/* If need be, bounce our sense buffer */
 			if (bt->sense_buffers != NULL) {
 				hccb->sense_addr = btsensepaddr(bt, bccb);
 			} else {
 				hccb->sense_addr = vtophys(&csio->sense_data);
 			}
 			/*
 			 * If we have any data to send with this command,
 			 * map it into bus space.
 			 */
 			error = bus_dmamap_load_ccb(
 			    bt->buffer_dmat,
 			    bccb->dmamap,
 			    ccb,
 			    btexecuteccb,
 			    bccb,
 			    /*flags*/0);
 			if (error == EINPROGRESS) {
 				/*
 				 * So as to maintain ordering, freeze the
 				 * controller queue until our mapping is
 				 * returned.
 				 */
 				xpt_freeze_simq(bt->sim, 1);
 				csio->ccb_h.status |= CAM_RELEASE_SIMQ;
 			}
 		} else {
 			hccb->opcode = INITIATOR_BUS_DEV_RESET;
 			/* No data transfer */
 			hccb->datain = TRUE;
 			hccb->dataout = TRUE;
 			hccb->cmd_len = 0;
 			hccb->sense_len = 0;
 			hccb->tag_enable = FALSE;
 			hccb->tag_type = 0;
 			btexecuteccb(bccb, NULL, 0, 0);
 		}
 		break;
 	}
 	case XPT_EN_LUN:		/* Enable LUN as a target */
 	case XPT_TARGET_IO:		/* Execute target I/O request */
 	case XPT_ACCEPT_TARGET_IO:	/* Accept Host Target Mode CDB */
 	case XPT_CONT_TARGET_IO:	/* Continue Host Target I/O Connection*/
 	case XPT_ABORT:			/* Abort the specified CCB */
 		/* XXX Implement */
 		ccb->ccb_h.status = CAM_REQ_INVALID;
 		xpt_done(ccb);
 		break;
 	case XPT_SET_TRAN_SETTINGS:
 	{
 		/* XXX Implement */
 		ccb->ccb_h.status = CAM_PROVIDE_FAIL;
 		xpt_done(ccb);
 		break;
 	}
 	case XPT_GET_TRAN_SETTINGS:
 	/* Get default/user set transfer settings for the target */
 	{
 		struct	ccb_trans_settings *cts;
 		u_int	target_mask;
 
 		cts = &ccb->cts;
 		target_mask = 0x01 << ccb->ccb_h.target_id;
 		if (cts->type == CTS_TYPE_CURRENT_SETTINGS) {
 			struct ccb_trans_settings_scsi *scsi =
 			    &cts->proto_specific.scsi;
 			struct ccb_trans_settings_spi *spi =
 			    &cts->xport_specific.spi;
 			cts->protocol = PROTO_SCSI;
 			cts->protocol_version = SCSI_REV_2;
 			cts->transport = XPORT_SPI;
 			cts->transport_version = 2;
 
 			scsi->flags &= ~CTS_SCSI_FLAGS_TAG_ENB;
 			spi->flags &= ~CTS_SPI_FLAGS_DISC_ENB;
 
 			if ((bt->disc_permitted & target_mask) != 0)
 				spi->flags |= CTS_SPI_FLAGS_DISC_ENB;
 			if ((bt->tags_permitted & target_mask) != 0)
 				scsi->flags |= CTS_SCSI_FLAGS_TAG_ENB;
 
 			if ((bt->ultra_permitted & target_mask) != 0)
 				spi->sync_period = 12;
 			else if ((bt->fast_permitted & target_mask) != 0)
 				spi->sync_period = 25;
 			else if ((bt->sync_permitted & target_mask) != 0)
 				spi->sync_period = 50;
 			else
 				spi->sync_period = 0;
 
 			if (spi->sync_period != 0)
 				spi->sync_offset = 15;
 
 			spi->valid |= CTS_SPI_VALID_SYNC_RATE;
 			spi->valid |= CTS_SPI_VALID_SYNC_OFFSET;
 
 			spi->valid |= CTS_SPI_VALID_BUS_WIDTH;
 			if ((bt->wide_permitted & target_mask) != 0)
 				spi->bus_width = MSG_EXT_WDTR_BUS_16_BIT;
 			else
 				spi->bus_width = MSG_EXT_WDTR_BUS_8_BIT;
 
 			if (cts->ccb_h.target_lun != CAM_LUN_WILDCARD) {
 				scsi->valid = CTS_SCSI_VALID_TQ;
 				spi->valid |= CTS_SPI_VALID_DISC;
 			} else
 				scsi->valid = 0;
 		} else {
 			btfetchtransinfo(bt, cts);
 		}
 
 		ccb->ccb_h.status = CAM_REQ_CMP;
 		xpt_done(ccb);
 		break;
 	}
 	case XPT_CALC_GEOMETRY:
 	{
 		struct	  ccb_calc_geometry *ccg;
 		u_int32_t size_mb;
 		u_int32_t secs_per_cylinder;
 
 		ccg = &ccb->ccg;
 		size_mb = ccg->volume_size
 			/ ((1024L * 1024L) / ccg->block_size);
 		
 		if (size_mb >= 1024 && (bt->extended_trans != 0)) {
 			if (size_mb >= 2048) {
 				ccg->heads = 255;
 				ccg->secs_per_track = 63;
 			} else {
 				ccg->heads = 128;
 				ccg->secs_per_track = 32;
 			}
 		} else {
 			ccg->heads = 64;
 			ccg->secs_per_track = 32;
 		}
 		secs_per_cylinder = ccg->heads * ccg->secs_per_track;
 		ccg->cylinders = ccg->volume_size / secs_per_cylinder;
 		ccb->ccb_h.status = CAM_REQ_CMP;
 		xpt_done(ccb);
 		break;
 	}
 	case XPT_RESET_BUS:		/* Reset the specified SCSI bus */
 	{
 		btreset(bt, /*hardreset*/TRUE);
 		ccb->ccb_h.status = CAM_REQ_CMP;
 		xpt_done(ccb);
 		break;
 	}
 	case XPT_TERM_IO:		/* Terminate the I/O process */
 		/* XXX Implement */
 		ccb->ccb_h.status = CAM_REQ_INVALID;
 		xpt_done(ccb);
 		break;
 	case XPT_PATH_INQ:		/* Path routing inquiry */
 	{
 		struct ccb_pathinq *cpi = &ccb->cpi;
 		
 		cpi->version_num = 1; /* XXX??? */
 		cpi->hba_inquiry = PI_SDTR_ABLE;
 		if (bt->tag_capable != 0)
 			cpi->hba_inquiry |= PI_TAG_ABLE;
 		if (bt->wide_bus != 0)
 			cpi->hba_inquiry |= PI_WIDE_16;
 		cpi->target_sprt = 0;
 		cpi->hba_misc = 0;
 		cpi->hba_eng_cnt = 0;
 		cpi->max_target = bt->wide_bus ? 15 : 7;
 		cpi->max_lun = 7;
 		cpi->initiator_id = bt->scsi_id;
 		cpi->bus_id = cam_sim_bus(sim);
 		cpi->base_transfer_speed = 3300;
 		strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
 		strncpy(cpi->hba_vid, "BusLogic", HBA_IDLEN);
 		strncpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
 		cpi->unit_number = cam_sim_unit(sim);
 		cpi->ccb_h.status = CAM_REQ_CMP;
 		cpi->transport = XPORT_SPI;
 		cpi->transport_version = 2;
 		cpi->protocol = PROTO_SCSI;
 		cpi->protocol_version = SCSI_REV_2;
 		xpt_done(ccb);
 		break;
 	}
 	default:
 		ccb->ccb_h.status = CAM_REQ_INVALID;
 		xpt_done(ccb);
 		break;
 	}
 }
 
 static void
 btexecuteccb(void *arg, bus_dma_segment_t *dm_segs, int nseg, int error)
 {
 	struct	 bt_ccb *bccb;
 	union	 ccb *ccb;
 	struct	 bt_softc *bt;
 
 	bccb = (struct bt_ccb *)arg;
 	ccb = bccb->ccb;
 	bt = (struct bt_softc *)ccb->ccb_h.ccb_bt_ptr;
 
 	if (error != 0) {
 		if (error != EFBIG)
 			device_printf(bt->dev,
 				      "Unexepected error 0x%x returned from "
 				      "bus_dmamap_load\n", error);
 		if (ccb->ccb_h.status == CAM_REQ_INPROG) {
 			xpt_freeze_devq(ccb->ccb_h.path, /*count*/1);
 			ccb->ccb_h.status = CAM_REQ_TOO_BIG|CAM_DEV_QFRZN;
 		}
 		btfreeccb(bt, bccb);
 		xpt_done(ccb);
 		return;
 	}
 		
 	if (nseg != 0) {
 		bt_sg_t *sg;
 		bus_dma_segment_t *end_seg;
 		bus_dmasync_op_t op;
 
 		end_seg = dm_segs + nseg;
 
 		/* Copy the segments into our SG list */
 		sg = bccb->sg_list;
 		while (dm_segs < end_seg) {
 			sg->len = dm_segs->ds_len;
 			sg->addr = dm_segs->ds_addr;
 			sg++;
 			dm_segs++;
 		}
 
 		if (nseg > 1) {
 			bccb->hccb.opcode = INITIATOR_SG_CCB_WRESID;
 			bccb->hccb.data_len = sizeof(bt_sg_t) * nseg;
 			bccb->hccb.data_addr = bccb->sg_list_phys;
 		} else {
 			bccb->hccb.data_len = bccb->sg_list->len;
 			bccb->hccb.data_addr = bccb->sg_list->addr;
 		}
 
 		if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN)
 			op = BUS_DMASYNC_PREREAD;
 		else
 			op = BUS_DMASYNC_PREWRITE;
 
 		bus_dmamap_sync(bt->buffer_dmat, bccb->dmamap, op);
 
 	} else {
 		bccb->hccb.opcode = INITIATOR_CCB;
 		bccb->hccb.data_len = 0;
 		bccb->hccb.data_addr = 0;
 	}
 
 	/*
 	 * Last time we need to check if this CCB needs to
 	 * be aborted.
 	 */
 	if (ccb->ccb_h.status != CAM_REQ_INPROG) {
 		if (nseg != 0)
 			bus_dmamap_unload(bt->buffer_dmat, bccb->dmamap);
 		btfreeccb(bt, bccb);
 		xpt_done(ccb);
 		return;
 	}
 		
 	bccb->flags = BCCB_ACTIVE;
 	ccb->ccb_h.status |= CAM_SIM_QUEUED;
 	LIST_INSERT_HEAD(&bt->pending_ccbs, &ccb->ccb_h, sim_links.le);
 
 	callout_reset(&bccb->timer, (ccb->ccb_h.timeout * hz) / 1000,
 	    bttimeout, bccb);
 
 	/* Tell the adapter about this command */
 	bt->cur_outbox->ccb_addr = btccbvtop(bt, bccb);
 	if (bt->cur_outbox->action_code != BMBO_FREE) {
 		/*
 		 * We should never encounter a busy mailbox.
 		 * If we do, warn the user, and treat it as
 		 * a resource shortage.  If the controller is
 		 * hung, one of the pending transactions will
 		 * timeout causing us to start recovery operations.
 		 */
 		device_printf(bt->dev,
 			      "Encountered busy mailbox with %d out of %d "
 			      "commands active!!!\n", bt->active_ccbs,
 			      bt->max_ccbs);
 		callout_stop(&bccb->timer);
 		if (nseg != 0)
 			bus_dmamap_unload(bt->buffer_dmat, bccb->dmamap);
 		btfreeccb(bt, bccb);
 		bt->resource_shortage = TRUE;
 		xpt_freeze_simq(bt->sim, /*count*/1);
 		ccb->ccb_h.status = CAM_REQUEUE_REQ;
 		xpt_done(ccb);
 		return;
 	}
 	bt->cur_outbox->action_code = BMBO_START;	
 	bt_outb(bt, COMMAND_REG, BOP_START_MBOX);
 	btnextoutbox(bt);
 }
 
 void
 bt_intr(void *arg)
 {
 	struct	bt_softc *bt;
 
 	bt = arg;
 	mtx_lock(&bt->lock);
 	bt_intr_locked(bt);
 	mtx_unlock(&bt->lock);
 }
 
 void
 bt_intr_locked(struct bt_softc *bt)
 {
 	u_int	intstat;
 
 	while (((intstat = bt_inb(bt, INTSTAT_REG)) & INTR_PENDING) != 0) {
 
 		if ((intstat & CMD_COMPLETE) != 0) {
 			bt->latched_status = bt_inb(bt, STATUS_REG);
 			bt->command_cmp = TRUE;
 		}
 
 		bt_outb(bt, CONTROL_REG, RESET_INTR);
 
 		if ((intstat & IMB_LOADED) != 0) {
 			while (bt->cur_inbox->comp_code != BMBI_FREE) {
 				btdone(bt,
 				       btccbptov(bt, bt->cur_inbox->ccb_addr),
 				       bt->cur_inbox->comp_code);
 				bt->cur_inbox->comp_code = BMBI_FREE;
 				btnextinbox(bt);
 			}
 		}
 
 		if ((intstat & SCSI_BUS_RESET) != 0) {
 			btreset(bt, /*hardreset*/FALSE);
 		}
 	}
 }
 
 static void
 btdone(struct bt_softc *bt, struct bt_ccb *bccb, bt_mbi_comp_code_t comp_code)
 {
 	union  ccb	  *ccb;
 	struct ccb_scsiio *csio;
 
 	ccb = bccb->ccb;
 	csio = &bccb->ccb->csio;
 
 	if ((bccb->flags & BCCB_ACTIVE) == 0) {
 		device_printf(bt->dev,
 			      "btdone - Attempt to free non-active BCCB %p\n",
 			      (void *)bccb);
 		return;
 	}
 
 	if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE) {
 		bus_dmasync_op_t op;
 
 		if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN)
 			op = BUS_DMASYNC_POSTREAD;
 		else
 			op = BUS_DMASYNC_POSTWRITE;
 		bus_dmamap_sync(bt->buffer_dmat, bccb->dmamap, op);
 		bus_dmamap_unload(bt->buffer_dmat, bccb->dmamap);
 	}
 
 	if (bccb == bt->recovery_bccb) {
 		/*
 		 * The recovery BCCB does not have a CCB associated
 		 * with it, so short circuit the normal error handling.
 		 * We now traverse our list of pending CCBs and process
 		 * any that were terminated by the recovery CCBs action.
 		 * We also reinstate timeouts for all remaining, pending,
 		 * CCBs.
 		 */
 		struct cam_path *path;
 		struct ccb_hdr *ccb_h;
 		cam_status error;
 
 		/* Notify all clients that a BDR occured */
 		error = xpt_create_path(&path, /*periph*/NULL,
 					cam_sim_path(bt->sim),
 					bccb->hccb.target_id,
 					CAM_LUN_WILDCARD);
 		
 		if (error == CAM_REQ_CMP) {
 			xpt_async(AC_SENT_BDR, path, NULL);
 			xpt_free_path(path);
 		}
 
 		ccb_h = LIST_FIRST(&bt->pending_ccbs);
 		while (ccb_h != NULL) {
 			struct bt_ccb *pending_bccb;
 
 			pending_bccb = (struct bt_ccb *)ccb_h->ccb_bccb_ptr;
 			if (pending_bccb->hccb.target_id
 			 == bccb->hccb.target_id) {
 				pending_bccb->hccb.btstat = BTSTAT_HA_BDR;
 				ccb_h = LIST_NEXT(ccb_h, sim_links.le);
 				btdone(bt, pending_bccb, BMBI_ERROR);
 			} else {
 				callout_reset(&pending_bccb->timer,
 				    (ccb_h->timeout * hz) / 1000,
 				    bttimeout, pending_bccb);
 				ccb_h = LIST_NEXT(ccb_h, sim_links.le);
 			}
 		}
 		device_printf(bt->dev, "No longer in timeout\n");
 		return;
 	}
 
 	callout_stop(&bccb->timer);
 
 	switch (comp_code) {
 	case BMBI_FREE:
 		device_printf(bt->dev,
 			      "btdone - CCB completed with free status!\n");
 		break;
 	case BMBI_NOT_FOUND:
 		device_printf(bt->dev,
 			      "btdone - CCB Abort failed to find CCB\n");
 		break;
 	case BMBI_ABORT:
 	case BMBI_ERROR:
 		if (bootverbose) {
 			printf("bt: ccb %p - error %x occured.  "
 			       "btstat = %x, sdstat = %x\n",
 			       (void *)bccb, comp_code, bccb->hccb.btstat,
 			       bccb->hccb.sdstat);
 		}
 		/* An error occured */
 		switch(bccb->hccb.btstat) {
 		case BTSTAT_DATARUN_ERROR:
 			if (bccb->hccb.data_len == 0) {
 				/*
 				 * At least firmware 4.22, does this
 				 * for a QUEUE FULL condition.
 				 */
 				bccb->hccb.sdstat = SCSI_STATUS_QUEUE_FULL;
 			} else if (bccb->hccb.data_len < 0) {
 				csio->ccb_h.status = CAM_DATA_RUN_ERR;
 				break;
 			}
 			/* FALLTHROUGH */
 		case BTSTAT_NOERROR:
 		case BTSTAT_LINKED_CMD_COMPLETE:
 		case BTSTAT_LINKED_CMD_FLAG_COMPLETE:
 		case BTSTAT_DATAUNDERUN_ERROR:
 
 			csio->scsi_status = bccb->hccb.sdstat;
 			csio->ccb_h.status |= CAM_SCSI_STATUS_ERROR;
 			switch(csio->scsi_status) {
 			case SCSI_STATUS_CHECK_COND:
 			case SCSI_STATUS_CMD_TERMINATED:
 				csio->ccb_h.status |= CAM_AUTOSNS_VALID;
 				/* Bounce sense back if necessary */
 				if (bt->sense_buffers != NULL) {
 					csio->sense_data =
 					    *btsensevaddr(bt, bccb);
 				}
 				break;
 			default:
 				break;
 			case SCSI_STATUS_OK:
 				csio->ccb_h.status = CAM_REQ_CMP;
 				break;
 			}
 			csio->resid = bccb->hccb.data_len;
 			break;
 		case BTSTAT_SELTIMEOUT:
 			csio->ccb_h.status = CAM_SEL_TIMEOUT;
 			break;
 		case BTSTAT_UNEXPECTED_BUSFREE:
 			csio->ccb_h.status = CAM_UNEXP_BUSFREE;
 			break;
 		case BTSTAT_INVALID_PHASE:
 			csio->ccb_h.status = CAM_SEQUENCE_FAIL;
 			break;
 		case BTSTAT_INVALID_ACTION_CODE:
 			panic("%s: Inavlid Action code", bt_name(bt));
 			break;
 		case BTSTAT_INVALID_OPCODE:
 			panic("%s: Inavlid CCB Opcode code", bt_name(bt));
 			break;
 		case BTSTAT_LINKED_CCB_LUN_MISMATCH:
 			/* We don't even support linked commands... */
 			panic("%s: Linked CCB Lun Mismatch", bt_name(bt));
 			break;
 		case BTSTAT_INVALID_CCB_OR_SG_PARAM:
 			panic("%s: Invalid CCB or SG list", bt_name(bt));
 			break;
 		case BTSTAT_AUTOSENSE_FAILED:
 			csio->ccb_h.status = CAM_AUTOSENSE_FAIL;
 			break;
 		case BTSTAT_TAGGED_MSG_REJECTED:
 		{
 			struct ccb_trans_settings neg; 
 			struct ccb_trans_settings_scsi *scsi =
 			    &neg.proto_specific.scsi;
 
 			neg.protocol = PROTO_SCSI;
 			neg.protocol_version = SCSI_REV_2;
 			neg.transport = XPORT_SPI;
 			neg.transport_version = 2;
 			scsi->valid = CTS_SCSI_VALID_TQ;
 			scsi->flags = 0;
 			xpt_print_path(csio->ccb_h.path);
 			printf("refuses tagged commands.  Performing "
 			       "non-tagged I/O\n");
 			xpt_setup_ccb(&neg.ccb_h, csio->ccb_h.path,
 				      /*priority*/1); 
 			xpt_async(AC_TRANSFER_NEG, csio->ccb_h.path, &neg);
 			bt->tags_permitted &= ~(0x01 << csio->ccb_h.target_id);
 			csio->ccb_h.status = CAM_MSG_REJECT_REC;
 			break;
 		}
 		case BTSTAT_UNSUPPORTED_MSG_RECEIVED:
 			/*
 			 * XXX You would think that this is
 			 *     a recoverable error... Hmmm.
 			 */
 			csio->ccb_h.status = CAM_REQ_CMP_ERR;
 			break;
 		case BTSTAT_HA_SOFTWARE_ERROR:
 		case BTSTAT_HA_WATCHDOG_ERROR:
 		case BTSTAT_HARDWARE_FAILURE:
 			/* Hardware reset ??? Can we recover ??? */
 			csio->ccb_h.status = CAM_NO_HBA;
 			break;
 		case BTSTAT_TARGET_IGNORED_ATN:
 		case BTSTAT_OTHER_SCSI_BUS_RESET:
 		case BTSTAT_HA_SCSI_BUS_RESET:
 			if ((csio->ccb_h.status & CAM_STATUS_MASK)
 			 != CAM_CMD_TIMEOUT)
 				csio->ccb_h.status = CAM_SCSI_BUS_RESET;
 			break;
 		case BTSTAT_HA_BDR:
 			if ((bccb->flags & BCCB_DEVICE_RESET) == 0)
 				csio->ccb_h.status = CAM_BDR_SENT;
 			else
 				csio->ccb_h.status = CAM_CMD_TIMEOUT;
 			break;
 		case BTSTAT_INVALID_RECONNECT:
 		case BTSTAT_ABORT_QUEUE_GENERATED:
 			csio->ccb_h.status = CAM_REQ_TERMIO;
 			break;
 		case BTSTAT_SCSI_PERROR_DETECTED:
 			csio->ccb_h.status = CAM_UNCOR_PARITY;
 			break;
 		}
 		if (csio->ccb_h.status != CAM_REQ_CMP) {
 			xpt_freeze_devq(csio->ccb_h.path, /*count*/1);
 			csio->ccb_h.status |= CAM_DEV_QFRZN;
 		}
 		if ((bccb->flags & BCCB_RELEASE_SIMQ) != 0)
 			ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
 		btfreeccb(bt, bccb);
 		xpt_done(ccb);
 		break;
 	case BMBI_OK:
 		/* All completed without incident */
 		ccb->ccb_h.status |= CAM_REQ_CMP;
 		if ((bccb->flags & BCCB_RELEASE_SIMQ) != 0)
 			ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
 		btfreeccb(bt, bccb);
 		xpt_done(ccb);
 		break;
 	}
 }
 
 static int
 btreset(struct bt_softc* bt, int hard_reset)
 {
 	struct	 ccb_hdr *ccb_h;
 	u_int	 status;
 	u_int	 timeout;
 	u_int8_t reset_type;
 
 	if (hard_reset != 0)
 		reset_type = HARD_RESET;
 	else
 		reset_type = SOFT_RESET;
 	bt_outb(bt, CONTROL_REG, reset_type);
 
 	/* Wait 5sec. for Diagnostic start */
 	timeout = 5 * 10000;
 	while (--timeout) {
 		status = bt_inb(bt, STATUS_REG);
 		if ((status & DIAG_ACTIVE) != 0)
 			break;
 		DELAY(100);
 	}
 	if (timeout == 0) {
 		if (bootverbose)
 			device_printf(bt->dev,
 			    "btreset - Diagnostic Active failed to "
 			    "assert. status = 0x%x\n", status);
 		return (ETIMEDOUT);
 	}
 
 	/* Wait 10sec. for Diagnostic end */
 	timeout = 10 * 10000;
 	while (--timeout) {
 		status = bt_inb(bt, STATUS_REG);
 		if ((status & DIAG_ACTIVE) == 0)
 			break;
 		DELAY(100);
 	}
 	if (timeout == 0) {
 		panic("%s: btreset - Diagnostic Active failed to drop. "
 		       "status = 0x%x\n", bt_name(bt), status);
 		return (ETIMEDOUT);
 	}
 
 	/* Wait for the host adapter to become ready or report a failure */
 	timeout = 10000;
 	while (--timeout) {
 		status = bt_inb(bt, STATUS_REG);
 		if ((status & (DIAG_FAIL|HA_READY|DATAIN_REG_READY)) != 0)
 			break;
 		DELAY(100);
 	}
 	if (timeout == 0) {
 		device_printf(bt->dev,
 		    "btreset - Host adapter failed to come ready. "
 		    "status = 0x%x\n", status);
 		return (ETIMEDOUT);
 	}
 
 	/* If the diagnostics failed, tell the user */
 	if ((status & DIAG_FAIL) != 0
 	 || (status & HA_READY) == 0) {
 		device_printf(bt->dev,
 		    "btreset - Adapter failed diagnostics\n");
 
 		if ((status & DATAIN_REG_READY) != 0)
 			device_printf(bt->dev,
 			    "btreset - Host Adapter Error code = 0x%x\n",
 			    bt_inb(bt, DATAIN_REG));
 		return (ENXIO);
 	}
 
 	/* If we've allocated mailboxes, initialize them */
 	if (bt->init_level > 4)
 		btinitmboxes(bt);
 
 	/* If we've attached to the XPT, tell it about the event */
 	if (bt->path != NULL)
 		xpt_async(AC_BUS_RESET, bt->path, NULL);
 
 	/*
 	 * Perform completion processing for all outstanding CCBs.
 	 */
 	while ((ccb_h = LIST_FIRST(&bt->pending_ccbs)) != NULL) {
 		struct bt_ccb *pending_bccb;
 
 		pending_bccb = (struct bt_ccb *)ccb_h->ccb_bccb_ptr;
 		pending_bccb->hccb.btstat = BTSTAT_HA_SCSI_BUS_RESET;
 		btdone(bt, pending_bccb, BMBI_ERROR);
 	}
 
 	return (0);
 }
 
 /*
  * Send a command to the adapter.
  */
 int
 bt_cmd(struct bt_softc *bt, bt_op_t opcode, u_int8_t *params, u_int param_len,
       u_int8_t *reply_data, u_int reply_len, u_int cmd_timeout)
 {
 	u_int	timeout;
 	u_int	status;
 	u_int	saved_status;
 	u_int	intstat;
 	u_int	reply_buf_size;
 	int	cmd_complete;
 	int	error;
 
 	/* No data returned to start */
 	reply_buf_size = reply_len;
 	reply_len = 0;
 	intstat = 0;
 	cmd_complete = 0;
 	saved_status = 0;
 	error = 0;
 
 	bt->command_cmp = 0;
 	/*
 	 * Wait up to 10 sec. for the adapter to become
 	 * ready to accept commands.
 	 */
 	timeout = 100000;
 	while (--timeout) {
 		status = bt_inb(bt, STATUS_REG);
 		if ((status & HA_READY) != 0
 		 && (status & CMD_REG_BUSY) == 0)
 			break;
 		/*
 		 * Throw away any pending data which may be
 		 * left over from earlier commands that we
 		 * timedout on.
 		 */
 		if ((status & DATAIN_REG_READY) != 0)
 			(void)bt_inb(bt, DATAIN_REG);
 		DELAY(100);
 	}
 	if (timeout == 0) {
 		device_printf(bt->dev,
 		    "bt_cmd: Timeout waiting for adapter ready, "
 		    "status = 0x%x\n", status);
 		return (ETIMEDOUT);
 	}
 
 	/*
 	 * Send the opcode followed by any necessary parameter bytes.
 	 */
 	bt_outb(bt, COMMAND_REG, opcode);
 
 	/*
 	 * Wait for up to 1sec for each byte of the
 	 * parameter list sent to be sent.
 	 */
 	timeout = 10000;
 	while (param_len && --timeout) {
 		DELAY(100);
 		status = bt_inb(bt, STATUS_REG);
 		intstat = bt_inb(bt, INTSTAT_REG);
 	
 		if ((intstat & (INTR_PENDING|CMD_COMPLETE))
 		 == (INTR_PENDING|CMD_COMPLETE)) {
 			saved_status = status;
 			cmd_complete = 1;
 			break;
 		}
 		if (bt->command_cmp != 0) {
 			saved_status = bt->latched_status;
 			cmd_complete = 1;
 			break;
 		}
 		if ((status & DATAIN_REG_READY) != 0)
 			break;
 		if ((status & CMD_REG_BUSY) == 0) {
 			bt_outb(bt, COMMAND_REG, *params++);
 			param_len--;
 			timeout = 10000;
 		}
 	}
 	if (timeout == 0) {
 		device_printf(bt->dev, "bt_cmd: Timeout sending parameters, "
 		    "status = 0x%x\n", status);
 		cmd_complete = 1;
 		saved_status = status;
 		error = ETIMEDOUT;
 	}
 
 	/*
 	 * Wait for the command to complete.
 	 */
 	while (cmd_complete == 0 && --cmd_timeout) {
 
 		status = bt_inb(bt, STATUS_REG);
 		intstat = bt_inb(bt, INTSTAT_REG);
 		/*
 		 * It may be that this command was issued with
 		 * controller interrupts disabled.  We'll never
 		 * get to our command if an incoming mailbox
 		 * interrupt is pending, so take care of completed
 		 * mailbox commands by calling our interrupt handler.
 		 */
 		if ((intstat & (INTR_PENDING|IMB_LOADED))
 		 == (INTR_PENDING|IMB_LOADED))
 			bt_intr_locked(bt);
 
 		if (bt->command_cmp != 0) {
  			/*
 			 * Our interrupt handler saw CMD_COMPLETE
 			 * status before we did.
 			 */
 			cmd_complete = 1;
 			saved_status = bt->latched_status;
 		} else if ((intstat & (INTR_PENDING|CMD_COMPLETE))
 			== (INTR_PENDING|CMD_COMPLETE)) {
 			/*
 			 * Our poll (in case interrupts are blocked)
 			 * saw the CMD_COMPLETE interrupt.
 			 */
 			cmd_complete = 1;
 			saved_status = status;
 		} else if (opcode == BOP_MODIFY_IO_ADDR
 			&& (status & CMD_REG_BUSY) == 0) {
 			/*
 			 * The BOP_MODIFY_IO_ADDR does not issue a CMD_COMPLETE,
 			 * but it should update the status register.  So, we
 			 * consider this command complete when the CMD_REG_BUSY
 			 * status clears.
 			 */
 			saved_status = status;
 			cmd_complete = 1;
 		} else if ((status & DATAIN_REG_READY) != 0) {
 			u_int8_t data;
 
 			data = bt_inb(bt, DATAIN_REG);
 			if (reply_len < reply_buf_size) {
 				*reply_data++ = data;
 			} else {
 				device_printf(bt->dev,
 				    "bt_cmd - Discarded reply data byte "
 				    "for opcode 0x%x\n", opcode);
 			}
 			/*
 			 * Reset timeout to ensure at least a second
 			 * between response bytes.
 			 */
 			cmd_timeout = MAX(cmd_timeout, 10000);
 			reply_len++;
 
 		} else if ((opcode == BOP_FETCH_LRAM)
 			&& (status & HA_READY) != 0) {
 				saved_status = status;
 				cmd_complete = 1;
 		}
 		DELAY(100);
 	}
 	if (cmd_timeout == 0) {
 		device_printf(bt->dev,
 		    "bt_cmd: Timeout waiting for command (%x) "
 		    "to complete.\n", opcode);
 		device_printf(bt->dev, "status = 0x%x, intstat = 0x%x, "
 		    "rlen %d\n", status, intstat, reply_len);
 		error = (ETIMEDOUT);
 	}
 
 	/*
 	 * Clear any pending interrupts.
 	 */
 	bt_intr_locked(bt);
 	
 	if (error != 0)
 		return (error);
 
 	/*
 	 * If the command was rejected by the controller, tell the caller.
 	 */
 	if ((saved_status & CMD_INVALID) != 0) {
 		/*
 		 * Some early adapters may not recover properly from
 		 * an invalid command.  If it appears that the controller
 		 * has wedged (i.e. status was not cleared by our interrupt
 		 * reset above), perform a soft reset.
       		 */
 		if (bootverbose)
 			device_printf(bt->dev, "Invalid Command 0x%x\n",
 				opcode);
 		DELAY(1000);
 		status = bt_inb(bt, STATUS_REG);
 		if ((status & (CMD_INVALID|STATUS_REG_RSVD|DATAIN_REG_READY|
 			      CMD_REG_BUSY|DIAG_FAIL|DIAG_ACTIVE)) != 0
 		 || (status & (HA_READY|INIT_REQUIRED))
 		  != (HA_READY|INIT_REQUIRED)) {
 			btreset(bt, /*hard_reset*/FALSE);
 		}
 		return (EINVAL);
 	}
 
 	if (param_len > 0) {
 		/* The controller did not accept the full argument list */
 	 	return (E2BIG);
 	}
 
 	if (reply_len != reply_buf_size) {
 		/* Too much or too little data received */
 		return (EMSGSIZE);
 	}
 
 	/* We were successful */
 	return (0);
 }
 
 static int
 btinitmboxes(struct bt_softc *bt) {
 	init_32b_mbox_params_t init_mbox;
 	int error;
 
 	bzero(bt->in_boxes, sizeof(bt_mbox_in_t) * bt->num_boxes);
 	bzero(bt->out_boxes, sizeof(bt_mbox_out_t) * bt->num_boxes);
 	bt->cur_inbox = bt->in_boxes;
 	bt->last_inbox = bt->in_boxes + bt->num_boxes - 1;
 	bt->cur_outbox = bt->out_boxes;
 	bt->last_outbox = bt->out_boxes + bt->num_boxes - 1;
 
 	/* Tell the adapter about them */
 	init_mbox.num_boxes = bt->num_boxes;
 	init_mbox.base_addr[0] = bt->mailbox_physbase & 0xFF;
 	init_mbox.base_addr[1] = (bt->mailbox_physbase >> 8) & 0xFF;
 	init_mbox.base_addr[2] = (bt->mailbox_physbase >> 16) & 0xFF;
 	init_mbox.base_addr[3] = (bt->mailbox_physbase >> 24) & 0xFF;
 	error = bt_cmd(bt, BOP_INITIALIZE_32BMBOX, (u_int8_t *)&init_mbox,
 		       /*parmlen*/sizeof(init_mbox), /*reply_buf*/NULL,
 		       /*reply_len*/0, DEFAULT_CMD_TIMEOUT);
 
 	if (error != 0)
 		printf("btinitmboxes: Initialization command failed\n");
 	else if (bt->strict_rr != 0) {
 		/*
 		 * If the controller supports
 		 * strict round robin mode,
 		 * enable it
 		 */
 		u_int8_t param;
 
 		param = 0;
 		error = bt_cmd(bt, BOP_ENABLE_STRICT_RR, &param, 1,
 			       /*reply_buf*/NULL, /*reply_len*/0,
 			       DEFAULT_CMD_TIMEOUT);
 
 		if (error != 0) {
 			printf("btinitmboxes: Unable to enable strict RR\n");
 			error = 0;
 		} else if (bootverbose) {
 			device_printf(bt->dev,
 			    "Using Strict Round Robin Mailbox Mode\n");
 		}
 	}
 	
 	return (error);
 }
 
 /*
  * Update the XPT's idea of the negotiated transfer
  * parameters for a particular target.
  */
 static void
 btfetchtransinfo(struct bt_softc *bt, struct ccb_trans_settings *cts)
 {
 	setup_data_t	setup_info;
 	u_int		target;
 	u_int		targ_offset;
 	u_int		targ_mask;
 	u_int		sync_period;
 	u_int		sync_offset;
 	u_int		bus_width;
 	int		error;
 	u_int8_t	param;
 	targ_syncinfo_t	sync_info;
 	struct ccb_trans_settings_scsi *scsi =
 	    &cts->proto_specific.scsi;
 	struct ccb_trans_settings_spi *spi =
 	    &cts->xport_specific.spi;
 
 	spi->valid = 0;
 	scsi->valid = 0;
 
 	target = cts->ccb_h.target_id;
 	targ_offset = (target & 0x7);
 	targ_mask = (0x01 << targ_offset);
 
 	/*
 	 * Inquire Setup Information.  This command retreives the
 	 * Wide negotiation status for recent adapters as well as
 	 * the sync info for older models.
 	 */
 	param = sizeof(setup_info);
 	error = bt_cmd(bt, BOP_INQUIRE_SETUP_INFO, &param, /*paramlen*/1,
 		       (u_int8_t*)&setup_info, sizeof(setup_info),
 		       DEFAULT_CMD_TIMEOUT);
 
 	if (error != 0) {
 		device_printf(bt->dev,
 		    "btfetchtransinfo - Inquire Setup Info Failed %x\n",
 		    error);
 		return;
 	}
 
 	sync_info = (target < 8) ? setup_info.low_syncinfo[targ_offset]
 				 : setup_info.high_syncinfo[targ_offset];
 
 	if (sync_info.sync == 0)
 		sync_offset = 0;
 	else
 		sync_offset = sync_info.offset;
 
 
 	bus_width = MSG_EXT_WDTR_BUS_8_BIT;
 	if (strcmp(bt->firmware_ver, "5.06L") >= 0) {
 		u_int wide_active;
 
 		wide_active =
 		    (target < 8) ? (setup_info.low_wide_active & targ_mask)
 		    		 : (setup_info.high_wide_active & targ_mask);
 
 		if (wide_active)
 			bus_width = MSG_EXT_WDTR_BUS_16_BIT;
 	} else if ((bt->wide_permitted & targ_mask) != 0) {
 		struct ccb_getdev cgd;
 
 		/*
 		 * Prior to rev 5.06L, wide status isn't provided,
 		 * so we "guess" that wide transfers are in effect
 		 * if the user settings allow for wide and the inquiry
 		 * data for the device indicates that it can handle
 		 * wide transfers.
 		 */
 		xpt_setup_ccb(&cgd.ccb_h, cts->ccb_h.path, /*priority*/1);
 		cgd.ccb_h.func_code = XPT_GDEV_TYPE;
 		xpt_action((union ccb *)&cgd);
 		if ((cgd.ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP
 		 && (cgd.inq_data.flags & SID_WBus16) != 0)
 			bus_width = MSG_EXT_WDTR_BUS_16_BIT;
 	}
 
 	if (bt->firmware_ver[0] >= '3') {
 		/*
 		 * For adapters that can do fast or ultra speeds,
 		 * use the more exact Target Sync Information command.
 		 */
 		target_sync_info_data_t sync_info;
 
 		param = sizeof(sync_info);
 		error = bt_cmd(bt, BOP_TARG_SYNC_INFO, &param, /*paramlen*/1,
 			       (u_int8_t*)&sync_info, sizeof(sync_info),
 			       DEFAULT_CMD_TIMEOUT);
 		
 		if (error != 0) {
 			device_printf(bt->dev,
 			    "btfetchtransinfo - Inquire Sync "
 			    "Info Failed 0x%x\n", error);
 			return;
 		}
 		sync_period = sync_info.sync_rate[target] * 100;
 	} else {
 		sync_period = 2000 + (500 * sync_info.period);
 	}
 
 	cts->protocol = PROTO_SCSI;
 	cts->protocol_version = SCSI_REV_2;
 	cts->transport = XPORT_SPI;
 	cts->transport_version = 2;
 
 	spi->sync_period = sync_period;
 	spi->valid |= CTS_SPI_VALID_SYNC_RATE;
 	spi->sync_offset = sync_offset;
 	spi->valid |= CTS_SPI_VALID_SYNC_OFFSET;
 
 	spi->valid |= CTS_SPI_VALID_BUS_WIDTH;
 	spi->bus_width = bus_width;
 
 	if (cts->ccb_h.target_lun != CAM_LUN_WILDCARD) {
 		scsi->valid = CTS_SCSI_VALID_TQ;
 		spi->valid |= CTS_SPI_VALID_DISC;
 	} else
 		scsi->valid = 0;
 	
         xpt_async(AC_TRANSFER_NEG, cts->ccb_h.path, cts);
 }
 
 static void
 btmapmboxes(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 {
 	struct bt_softc* bt;
 
 	bt = (struct bt_softc*)arg;
 	bt->mailbox_physbase = segs->ds_addr;
 }
 
 static void
 btmapccbs(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 {
 	struct bt_softc* bt;
 
 	bt = (struct bt_softc*)arg;
 	bt->bt_ccb_physbase = segs->ds_addr;
 }
 
 static void
 btmapsgs(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 {
 
 	struct bt_softc* bt;
 
 	bt = (struct bt_softc*)arg;
 	SLIST_FIRST(&bt->sg_maps)->sg_physaddr = segs->ds_addr;
 }
 
 static void
 btpoll(struct cam_sim *sim)
 {
 	bt_intr_locked(cam_sim_softc(sim));
 }
 
 void
 bttimeout(void *arg)
 {
 	struct bt_ccb	*bccb;
 	union  ccb	*ccb;
 	struct bt_softc *bt;
 
 	bccb = (struct bt_ccb *)arg;
 	ccb = bccb->ccb;
 	bt = (struct bt_softc *)ccb->ccb_h.ccb_bt_ptr;
 	mtx_assert(&bt->lock, MA_OWNED);
 	xpt_print_path(ccb->ccb_h.path);
 	printf("CCB %p - timed out\n", (void *)bccb);
 
 	if ((bccb->flags & BCCB_ACTIVE) == 0) {
 		xpt_print_path(ccb->ccb_h.path);
 		printf("CCB %p - timed out CCB already completed\n",
 		       (void *)bccb);
 		return;
 	}
 
 	/*
 	 * In order to simplify the recovery process, we ask the XPT
 	 * layer to halt the queue of new transactions and we traverse
 	 * the list of pending CCBs and remove their timeouts. This
 	 * means that the driver attempts to clear only one error
 	 * condition at a time.  In general, timeouts that occur
 	 * close together are related anyway, so there is no benefit
 	 * in attempting to handle errors in parrallel.  Timeouts will
 	 * be reinstated when the recovery process ends.
 	 */
 	if ((bccb->flags & BCCB_DEVICE_RESET) == 0) {
 		struct ccb_hdr *ccb_h;
 
 		if ((bccb->flags & BCCB_RELEASE_SIMQ) == 0) {
 			xpt_freeze_simq(bt->sim, /*count*/1);
 			bccb->flags |= BCCB_RELEASE_SIMQ;
 		}
 
 		ccb_h = LIST_FIRST(&bt->pending_ccbs);
 		while (ccb_h != NULL) {
 			struct bt_ccb *pending_bccb;
 
 			pending_bccb = (struct bt_ccb *)ccb_h->ccb_bccb_ptr;
 			callout_stop(&pending_bccb->timer);
 			ccb_h = LIST_NEXT(ccb_h, sim_links.le);
 		}
 	}
 
 	if ((bccb->flags & BCCB_DEVICE_RESET) != 0
 	 || bt->cur_outbox->action_code != BMBO_FREE
 	 || ((bccb->hccb.tag_enable == TRUE)
 	  && (bt->firmware_ver[0] < '5'))) {
 		/*
 		 * Try a full host adapter/SCSI bus reset.
 		 * We do this only if we have already attempted
 		 * to clear the condition with a BDR, or we cannot
 		 * attempt a BDR for lack of mailbox resources
 		 * or because of faulty firmware.  It turns out
 		 * that firmware versions prior to 5.xx treat BDRs
 		 * as untagged commands that cannot be sent until
 		 * all outstanding tagged commands have been processed.
 		 * This makes it somewhat difficult to use a BDR to
 		 * clear up a problem with an uncompleted tagged command.
 		 */
 		ccb->ccb_h.status = CAM_CMD_TIMEOUT;
 		btreset(bt, /*hardreset*/TRUE);
 		device_printf(bt->dev, "No longer in timeout\n");
 	} else {
 		/*    
 		 * Send a Bus Device Reset message:
 		 * The target that is holding up the bus may not
 		 * be the same as the one that triggered this timeout
 		 * (different commands have different timeout lengths),
 		 * but we have no way of determining this from our
 		 * timeout handler.  Our strategy here is to queue a
 		 * BDR message to the target of the timed out command.
 		 * If this fails, we'll get another timeout 2 seconds
 		 * later which will attempt a bus reset.
 		 */
 		bccb->flags |= BCCB_DEVICE_RESET;
 		callout_reset(&bccb->timer, 2 * hz, bttimeout, bccb);
 
 		bt->recovery_bccb->hccb.opcode = INITIATOR_BUS_DEV_RESET;
 
 		/* No Data Transfer */
 		bt->recovery_bccb->hccb.datain = TRUE;
 		bt->recovery_bccb->hccb.dataout = TRUE;
 		bt->recovery_bccb->hccb.btstat = 0;
 		bt->recovery_bccb->hccb.sdstat = 0;
 		bt->recovery_bccb->hccb.target_id = ccb->ccb_h.target_id;
 
 		/* Tell the adapter about this command */
 		bt->cur_outbox->ccb_addr = btccbvtop(bt, bt->recovery_bccb);
 		bt->cur_outbox->action_code = BMBO_START;
 		bt_outb(bt, COMMAND_REG, BOP_START_MBOX);
 		btnextoutbox(bt);
 	}
 }
 
 MODULE_VERSION(bt, 1);
 MODULE_DEPEND(bt, cam, 1, 1, 1);
Index: projects/release-debugdist/sys/dev/dpt/dpt_scsi.c
===================================================================
--- projects/release-debugdist/sys/dev/dpt/dpt_scsi.c	(revision 267343)
+++ projects/release-debugdist/sys/dev/dpt/dpt_scsi.c	(revision 267344)
@@ -1,2524 +1,2523 @@
 /*-
  *       Copyright (c) 1997 by Simon Shapiro
  *       All Rights Reserved
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions, and the following disclaimer,
  *    without modification, immediately at the beginning of the file.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  * 3. The name of the author may not be used to endorse or promote products
  *    derived from this software without specific prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 /*
  * dpt_scsi.c: SCSI dependant code for the DPT driver
  *
  * credits:	Assisted by Mike Neuffer in the early low level DPT code
  *		Thanx to Mark Salyzyn of DPT for his assistance.
  *		Special thanx to Justin Gibbs for invaluable help in
  *		making this driver look and work like a FreeBSD component.
  *		Last but not least, many thanx to UCB and the FreeBSD
  *		team for creating and maintaining such a wonderful O/S.
  *
  * TODO:     * Add ISA probe code.
  *	     * Add driver-level RAID-0. This will allow interoperability with
  *	       NiceTry, M$-Doze, Win-Dog, Slowlaris, etc., in recognizing RAID
  *	       arrays that span controllers (Wow!).
  */
 
 #define _DPT_C_
 
 #include "opt_dpt.h"
 #include "opt_eisa.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/conf.h>
 #include <sys/eventhandler.h>
 #include <sys/malloc.h>
 #include <sys/kernel.h>
 
 #include <sys/bus.h>
 
 #include <machine/bus.h>
 
 #include <machine/resource.h>
 #include <sys/rman.h>
 
 
 #include <cam/cam.h>
 #include <cam/cam_ccb.h>
 #include <cam/cam_sim.h>
 #include <cam/cam_xpt_sim.h>
 #include <cam/cam_debug.h>
 #include <cam/scsi/scsi_all.h>
 #include <cam/scsi/scsi_message.h>
 
 #include <vm/vm.h>
 #include <vm/pmap.h>
 
 #include <dev/dpt/dpt.h>
 
 /* dpt_isa.c, dpt_eisa.c, and dpt_pci.c need this in a central place */
 devclass_t	dpt_devclass;
 
 #define microtime_now dpt_time_now()
 
 #define dpt_inl(dpt, port)				\
 	bus_read_4((dpt)->io_res, (dpt)->io_offset + port)
 #define dpt_inb(dpt, port)				\
 	bus_read_1((dpt)->io_res, (dpt)->io_offset + port)
 #define dpt_outl(dpt, port, value)			\
 	bus_write_4((dpt)->io_res, (dpt)->io_offset + port, value)
 #define dpt_outb(dpt, port, value)			\
 	bus_write_1((dpt)->io_res, (dpt)->io_offset + port, value)
 
 /*
  * These will have to be setup by parameters passed at boot/load time. For
  * perfromance reasons, we make them constants for the time being.
  */
 #define	dpt_min_segs	DPT_MAX_SEGS
 #define	dpt_max_segs	DPT_MAX_SEGS
 
 /* Definitions for our use of the SIM private CCB area */
 #define ccb_dccb_ptr spriv_ptr0
 #define ccb_dpt_ptr spriv_ptr1
 
 /* ================= Private Inline Function declarations ===================*/
 static __inline int		dpt_just_reset(dpt_softc_t * dpt);
 static __inline int		dpt_raid_busy(dpt_softc_t * dpt);
 #ifdef DEV_EISA
 static __inline int		dpt_pio_wait (u_int32_t, u_int, u_int, u_int);
 #endif
 static __inline int		dpt_wait(dpt_softc_t *dpt, u_int bits,
 					 u_int state);
 static __inline struct dpt_ccb* dptgetccb(struct dpt_softc *dpt);
 static __inline void		dptfreeccb(struct dpt_softc *dpt,
 					   struct dpt_ccb *dccb);
 static __inline bus_addr_t	dptccbvtop(struct dpt_softc *dpt,
 					   struct dpt_ccb *dccb);
 
 static __inline int		dpt_send_immediate(dpt_softc_t *dpt,
 						   eata_ccb_t *cmd_block,
 						   u_int32_t cmd_busaddr,  
 						   u_int retries,
 						   u_int ifc, u_int code,
 						   u_int code2);
 
 /* ==================== Private Function declarations =======================*/
 static void		dptmapmem(void *arg, bus_dma_segment_t *segs,
 				  int nseg, int error);
 
 static struct sg_map_node*
 			dptallocsgmap(struct dpt_softc *dpt);
 
 static int		dptallocccbs(dpt_softc_t *dpt);
 
 static int		dpt_get_conf(dpt_softc_t *dpt, dpt_ccb_t *dccb,
 				     u_int32_t dccb_busaddr, u_int size,
 				     u_int page, u_int target, int extent);
 static void		dpt_detect_cache(dpt_softc_t *dpt, dpt_ccb_t *dccb,
 					 u_int32_t dccb_busaddr,
 					 u_int8_t *buff);
 
 static void		dpt_poll(struct cam_sim *sim);
 static void		dpt_intr_locked(dpt_softc_t *dpt);
 
 static void		dptexecuteccb(void *arg, bus_dma_segment_t *dm_segs,
 				      int nseg, int error);
 
 static void		dpt_action(struct cam_sim *sim, union ccb *ccb);
 
 static int		dpt_send_eata_command(dpt_softc_t *dpt, eata_ccb_t *cmd,
 					      u_int32_t cmd_busaddr,
 					      u_int command, u_int retries,
 					      u_int ifc, u_int code,
 					      u_int code2);
 static void		dptprocesserror(dpt_softc_t *dpt, dpt_ccb_t *dccb,
 					union ccb *ccb, u_int hba_stat,
 					u_int scsi_stat, u_int32_t resid);
 
 static void		dpttimeout(void *arg);
 static void		dptshutdown(void *arg, int howto);
 
 /* ================= Private Inline Function definitions ====================*/
 static __inline int
 dpt_just_reset(dpt_softc_t * dpt)
 {
 	if ((dpt_inb(dpt, 2) == 'D')
 	 && (dpt_inb(dpt, 3) == 'P')
 	 && (dpt_inb(dpt, 4) == 'T')
 	 && (dpt_inb(dpt, 5) == 'H'))
 		return (1);
 	else
 		return (0);
 }
 
 static __inline int
 dpt_raid_busy(dpt_softc_t * dpt)
 {
 	if ((dpt_inb(dpt, 0) == 'D')
 	 && (dpt_inb(dpt, 1) == 'P')
 	 && (dpt_inb(dpt, 2) == 'T'))
 		return (1);
 	else
 		return (0);
 }
 
 #ifdef DEV_EISA
 static __inline int
 dpt_pio_wait (u_int32_t base, u_int reg, u_int bits, u_int state)
 {
 	int   i;
 	u_int c;
 
 	for (i = 0; i < 20000; i++) {	/* wait 20ms for not busy */
 		c = inb(base + reg) & bits;
 		if (!(c == state))
 			return (0);
 		else
 			DELAY(50);
 	}
 	return (-1);
 }
 #endif
 
 static __inline int
 dpt_wait(dpt_softc_t *dpt, u_int bits, u_int state)
 {
 	int   i;
 	u_int c;
 
 	for (i = 0; i < 20000; i++) {	/* wait 20ms for not busy */
 		c = dpt_inb(dpt, HA_RSTATUS) & bits;
 		if (c == state)
 			return (0);
 		else
 			DELAY(50);
 	}
 	return (-1);
 }
 
 static __inline struct dpt_ccb*
 dptgetccb(struct dpt_softc *dpt)
 {
 	struct	dpt_ccb* dccb;
 
 	if (!dumping)
 		mtx_assert(&dpt->lock, MA_OWNED);
 	if ((dccb = SLIST_FIRST(&dpt->free_dccb_list)) != NULL) {
 		SLIST_REMOVE_HEAD(&dpt->free_dccb_list, links);
 		dpt->free_dccbs--;
 	} else if (dpt->total_dccbs < dpt->max_dccbs) {
 		dptallocccbs(dpt);
 		dccb = SLIST_FIRST(&dpt->free_dccb_list);
 		if (dccb == NULL)
 			device_printf(dpt->dev, "Can't malloc DCCB\n");
 		else {
 			SLIST_REMOVE_HEAD(&dpt->free_dccb_list, links);
 			dpt->free_dccbs--;
 		}
 	}
 
 	return (dccb);
 }
 
 static __inline void
 dptfreeccb(struct dpt_softc *dpt, struct dpt_ccb *dccb)
 {
 
 	if (!dumping)
 		mtx_assert(&dpt->lock, MA_OWNED);
 	if ((dccb->state & DCCB_ACTIVE) != 0)
 		LIST_REMOVE(&dccb->ccb->ccb_h, sim_links.le);
 	if ((dccb->state & DCCB_RELEASE_SIMQ) != 0)
 		dccb->ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
 	else if (dpt->resource_shortage != 0
 	 && (dccb->ccb->ccb_h.status & CAM_RELEASE_SIMQ) == 0) {
 		dccb->ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
 		dpt->resource_shortage = FALSE;
 	}
 	dccb->state = DCCB_FREE;
 	SLIST_INSERT_HEAD(&dpt->free_dccb_list, dccb, links);
 	++dpt->free_dccbs;
 }
 
 static __inline bus_addr_t
 dptccbvtop(struct dpt_softc *dpt, struct dpt_ccb *dccb)
 {
 	return (dpt->dpt_ccb_busbase
 	      + (u_int32_t)((caddr_t)dccb - (caddr_t)dpt->dpt_dccbs));
 }
 
 static __inline struct dpt_ccb *
 dptccbptov(struct dpt_softc *dpt, bus_addr_t busaddr)
 {
 	return (dpt->dpt_dccbs
 	     +  ((struct dpt_ccb *)busaddr
 	       - (struct dpt_ccb *)dpt->dpt_ccb_busbase));
 }
 
 /*
  * Send a command for immediate execution by the DPT
  * See above function for IMPORTANT notes.
  */
 static __inline int
 dpt_send_immediate(dpt_softc_t *dpt, eata_ccb_t *cmd_block,
 		   u_int32_t cmd_busaddr, u_int retries,
 		   u_int ifc, u_int code, u_int code2)
 {
 	return (dpt_send_eata_command(dpt, cmd_block, cmd_busaddr,
 				      EATA_CMD_IMMEDIATE, retries, ifc,
 				      code, code2));
 }
 
 
 /* ===================== Private Function definitions =======================*/
 static void
 dptmapmem(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 {
 	bus_addr_t *busaddrp;
 
 	busaddrp = (bus_addr_t *)arg;
 	*busaddrp = segs->ds_addr;
 }
 
 static struct sg_map_node *
 dptallocsgmap(struct dpt_softc *dpt)
 {
 	struct sg_map_node *sg_map;
 
 	sg_map = malloc(sizeof(*sg_map), M_DEVBUF, M_NOWAIT);
 
 	if (sg_map == NULL)
 		return (NULL);
 
 	/* Allocate S/G space for the next batch of CCBS */
 	if (bus_dmamem_alloc(dpt->sg_dmat, (void **)&sg_map->sg_vaddr,
 			     BUS_DMA_NOWAIT, &sg_map->sg_dmamap) != 0) {
 		free(sg_map, M_DEVBUF);
 		return (NULL);
 	}
 
 	(void)bus_dmamap_load(dpt->sg_dmat, sg_map->sg_dmamap, sg_map->sg_vaddr,
 			      PAGE_SIZE, dptmapmem, &sg_map->sg_physaddr,
 			      /*flags*/0);
 
 	SLIST_INSERT_HEAD(&dpt->sg_maps, sg_map, links);
 
 	return (sg_map);
 }
 
 /*
  * Allocate another chunk of CCB's. Return count of entries added.
  */
 static int
 dptallocccbs(dpt_softc_t *dpt)
 {
 	struct dpt_ccb *next_ccb;
 	struct sg_map_node *sg_map;
 	bus_addr_t physaddr;
 	dpt_sg_t *segs;
 	int newcount;
 	int i;
 
 	if (!dumping)
 		mtx_assert(&dpt->lock, MA_OWNED);
 	next_ccb = &dpt->dpt_dccbs[dpt->total_dccbs];
 
 	if (next_ccb == dpt->dpt_dccbs) {
 		/*
 		 * First time through.  Re-use the S/G
 		 * space we allocated for initialization
 		 * CCBS.
 		 */
 		sg_map = SLIST_FIRST(&dpt->sg_maps);
 	} else {
 		sg_map = dptallocsgmap(dpt);
 	}
 
 	if (sg_map == NULL)
 		return (0);
 
 	segs = sg_map->sg_vaddr;
 	physaddr = sg_map->sg_physaddr;
 
 	newcount = (PAGE_SIZE / (dpt->sgsize * sizeof(dpt_sg_t)));
 	for (i = 0; dpt->total_dccbs < dpt->max_dccbs && i < newcount; i++) {
 		int error;
 
 		error = bus_dmamap_create(dpt->buffer_dmat, /*flags*/0,
 					  &next_ccb->dmamap);
 		if (error != 0)
 			break;
 		callout_init_mtx(&next_ccb->timer, &dpt->lock, 0);
 		next_ccb->sg_list = segs;
 		next_ccb->sg_busaddr = htonl(physaddr);
 		next_ccb->eata_ccb.cp_dataDMA = htonl(physaddr);
 		next_ccb->eata_ccb.cp_statDMA = htonl(dpt->sp_physaddr);
 		next_ccb->eata_ccb.cp_reqDMA =
 		    htonl(dptccbvtop(dpt, next_ccb)
 			+ offsetof(struct dpt_ccb, sense_data));
 		next_ccb->eata_ccb.cp_busaddr = dpt->dpt_ccb_busend;
 		next_ccb->state = DCCB_FREE;
 		next_ccb->tag = dpt->total_dccbs;
 		SLIST_INSERT_HEAD(&dpt->free_dccb_list, next_ccb, links);
 		segs += dpt->sgsize;
 		physaddr += (dpt->sgsize * sizeof(dpt_sg_t));
 		dpt->dpt_ccb_busend += sizeof(*next_ccb);
 		next_ccb++;
 		dpt->total_dccbs++;
 	}
 	return (i);
 }
 
 #ifdef DEV_EISA
 dpt_conf_t *
 dpt_pio_get_conf (u_int32_t base)
 {
 	static dpt_conf_t *	conf;
 	u_int16_t *		p;
 	int			i;
 
 	/*
 	 * Allocate a dpt_conf_t
 	 */
 	if (!conf) {
 		conf = (dpt_conf_t *)malloc(sizeof(dpt_conf_t),
 						 M_DEVBUF, M_NOWAIT | M_ZERO);
 	}
 	
 	/*
 	 * If we didn't get one then we probably won't ever get one.
 	 */
 	if (!conf) {
 		printf("dpt: unable to allocate dpt_conf_t\n");
 		return (NULL);
 	}
 
 	/*
 	 * Reset the controller.
 	 */
 	outb((base + HA_WCOMMAND), EATA_CMD_RESET);
 
 	/*
 	 * Wait for the controller to become ready.
 	 * For some reason there can be -no- delays after calling reset
 	 * before we wait on ready status.
 	 */
 	if (dpt_pio_wait(base, HA_RSTATUS, HA_SBUSY, 0)) {
 		printf("dpt: timeout waiting for controller to become ready\n");
 		return (NULL);
 	}
 
 	if (dpt_pio_wait(base, HA_RAUXSTAT, HA_ABUSY, 0)) {
 		printf("dpt: timetout waiting for adapter ready.\n");
 		return (NULL);
 	}
 
 	/*
 	 * Send the PIO_READ_CONFIG command.
 	 */
 	outb((base + HA_WCOMMAND), EATA_CMD_PIO_READ_CONFIG);
 
 	/*
 	 * Read the data into the struct.
 	 */
 	p = (u_int16_t *)conf;
 	for (i = 0; i < (sizeof(dpt_conf_t) / 2); i++) {
 
 		if (dpt_pio_wait(base, HA_RSTATUS, HA_SDRQ, 0)) {
 			if (bootverbose)
 				printf("dpt: timeout in data read.\n");
 			return (NULL);
 		}
 
 		(*p) = inw(base + HA_RDATA);
 		p++;
 	}
 
 	if (inb(base + HA_RSTATUS) & HA_SERROR) {
 		if (bootverbose)
 			printf("dpt: error reading configuration data.\n");
 		return (NULL);
 	}
 
 #define BE_EATA_SIGNATURE	0x45415441
 #define LE_EATA_SIGNATURE	0x41544145
 
 	/*
 	 * Test to see if we have a valid card.
 	 */
 	if ((conf->signature == BE_EATA_SIGNATURE) ||
 	    (conf->signature == LE_EATA_SIGNATURE)) {
 
 		while (inb(base + HA_RSTATUS) & HA_SDRQ) {
  			inw(base + HA_RDATA);
 		}
 
 		return (conf);
 	}
 	return (NULL);
 }
 #endif
 
 /*
  * Read a configuration page into the supplied dpt_cont_t buffer.
  */
 static int
 dpt_get_conf(dpt_softc_t *dpt, dpt_ccb_t *dccb, u_int32_t dccb_busaddr,
 	     u_int size, u_int page, u_int target, int extent)
 {
 	eata_ccb_t *cp;
 
 	u_int8_t   status;
 
 	int	   ndx;
 	int	   result;
 
 	mtx_assert(&dpt->lock, MA_OWNED);
 	cp = &dccb->eata_ccb;
 	bzero((void *)(uintptr_t)(volatile void *)dpt->sp, sizeof(*dpt->sp));
 
 	cp->Interpret = 1;
 	cp->DataIn = 1;
 	cp->Auto_Req_Sen = 1;
 	cp->reqlen = sizeof(struct scsi_sense_data);
 
 	cp->cp_id = target;
 	cp->cp_LUN = 0;		/* In the EATA packet */
 	cp->cp_lun = 0;		/* In the SCSI command */
 
 	cp->cp_scsi_cmd = INQUIRY;
 	cp->cp_len = size;
 
 	cp->cp_extent = extent;
 
 	cp->cp_page = page;
 	cp->cp_channel = 0;	/* DNC, Interpret mode is set */
 	cp->cp_identify = 1;
 	cp->cp_datalen = htonl(size);
 
 	/*
 	 * This could be a simple for loop, but we suspected the compiler To
 	 * have optimized it a bit too much. Wait for the controller to
 	 * become ready
 	 */
 	while (((status = dpt_inb(dpt, HA_RSTATUS)) != (HA_SREADY | HA_SSC)
 	     && (status != (HA_SREADY | HA_SSC | HA_SERROR))
 	     && (status != (HA_SDRDY | HA_SERROR | HA_SDRQ)))
 	    || (dpt_wait(dpt, HA_SBUSY, 0))) {
 
 		/*
 		 * RAID Drives still Spinning up? (This should only occur if
 		 * the DPT controller is in a NON PC (PCI?) platform).
 		 */
 		if (dpt_raid_busy(dpt)) {
 			device_printf(dpt->dev,
 			    "WARNING: Get_conf() RSUS failed.\n");
 			return (0);
 		}
 	}
 
 	DptStat_Reset_BUSY(dpt->sp);
 
 	/*
 	 * XXXX We might want to do something more clever than aborting at
 	 * this point, like resetting (rebooting) the controller and trying
 	 * again.
 	 */
 	if ((result = dpt_send_eata_command(dpt, cp, dccb_busaddr,
 					    EATA_CMD_DMA_SEND_CP,
 					    10000, 0, 0, 0)) != 0) {
 		device_printf(dpt->dev,
 		       "WARNING: Get_conf() failed (%d) to send "
 		       "EATA_CMD_DMA_READ_CONFIG\n",
 		       result);
 		return (0);
 	}
 	/* Wait for two seconds for a response.  This can be slow  */
 	for (ndx = 0;
 	     (ndx < 20000)
 	     && !((status = dpt_inb(dpt, HA_RAUXSTAT)) & HA_AIRQ);
 	     ndx++) {
 		DELAY(50);
 	}
 
 	/* Grab the status and clear interrupts */
 	status = dpt_inb(dpt, HA_RSTATUS);
 
 	/*
 	 * Check the status carefully.  Return only if the
 	 * command was successful.
 	 */
 	if (((status & HA_SERROR) == 0)
 	 && (dpt->sp->hba_stat == 0)
 	 && (dpt->sp->scsi_stat == 0)
 	 && (dpt->sp->residue_len == 0))
 		return (0);
 
 	if (dpt->sp->scsi_stat == SCSI_STATUS_CHECK_COND)
 		return (0);
 
 	return (1);
 }
 
 /* Detect Cache parameters and size */
 static void
 dpt_detect_cache(dpt_softc_t *dpt, dpt_ccb_t *dccb, u_int32_t dccb_busaddr,
 		 u_int8_t *buff)
 {
 	eata_ccb_t *cp;
 	u_int8_t   *param;
 	int	    bytes;
 	int	    result;
 	int	    ndx;
 	u_int8_t    status;
 
 	mtx_assert(&dpt->lock, MA_OWNED);
 
 	/*
 	 * Default setting, for best perfromance..
 	 * This is what virtually all cards default to..
 	 */
 	dpt->cache_type = DPT_CACHE_WRITEBACK;
 	dpt->cache_size = 0;
 
 	cp = &dccb->eata_ccb;
 	bzero((void *)(uintptr_t)(volatile void *)dpt->sp, sizeof(dpt->sp));
 	bzero(buff, 512);
 
 	/* Setup the command structure */
 	cp->Interpret = 1;
 	cp->DataIn = 1;
 	cp->Auto_Req_Sen = 1;
 	cp->reqlen = sizeof(struct scsi_sense_data);
 
 	cp->cp_id = 0;		/* who cares?  The HBA will interpret.. */
 	cp->cp_LUN = 0;		/* In the EATA packet */
 	cp->cp_lun = 0;		/* In the SCSI command */
 	cp->cp_channel = 0;
 
 	cp->cp_scsi_cmd = EATA_CMD_DMA_SEND_CP;
 	cp->cp_len = 56;
 
 	cp->cp_extent = 0;
 	cp->cp_page = 0;
 	cp->cp_identify = 1;
 	cp->cp_dispri = 1;
 
 	/*
 	 * Build the EATA Command Packet structure
 	 * for a Log Sense Command.
 	 */
 	cp->cp_cdb[0] = 0x4d;
 	cp->cp_cdb[1] = 0x0;
 	cp->cp_cdb[2] = 0x40 | 0x33;
 	cp->cp_cdb[7] = 1;
 
 	cp->cp_datalen = htonl(512);
 
 	result = dpt_send_eata_command(dpt, cp, dccb_busaddr,
 				       EATA_CMD_DMA_SEND_CP,
 				       10000, 0, 0, 0);
 	if (result != 0) {
 		device_printf(dpt->dev,
 		       "WARNING: detect_cache() failed (%d) to send "
 		       "EATA_CMD_DMA_SEND_CP\n", result);
 		return;
 	}
 	/* Wait for two seconds for a response.  This can be slow... */
 	for (ndx = 0;
 	     (ndx < 20000) &&
 	     !((status = dpt_inb(dpt, HA_RAUXSTAT)) & HA_AIRQ);
 	     ndx++) {
 		DELAY(50);
 	}
 
 	/* Grab the status and clear interrupts */
 	status = dpt_inb(dpt, HA_RSTATUS);
 
 	/*
 	 * Sanity check
 	 */
 	if (buff[0] != 0x33) {
 		return;
 	}
 	bytes = DPT_HCP_LENGTH(buff);
 	param = DPT_HCP_FIRST(buff);
 
 	if (DPT_HCP_CODE(param) != 1) {
 		/*
 		 * DPT Log Page layout error
 		 */
 		device_printf(dpt->dev, "NOTICE: Log Page (1) layout error\n");
 		return;
 	}
 	if (!(param[4] & 0x4)) {
 		dpt->cache_type = DPT_NO_CACHE;
 		return;
 	}
 	while (DPT_HCP_CODE(param) != 6) {
 		param = DPT_HCP_NEXT(param);
 		if ((param < buff)
 		 || (param >= &buff[bytes])) {
 			return;
 		}
 	}
 
 	if (param[4] & 0x2) {
 		/*
 		 * Cache disabled
 		 */
 		dpt->cache_type = DPT_NO_CACHE;
 		return;
 	}
 
 	if (param[4] & 0x4) {
 		dpt->cache_type = DPT_CACHE_WRITETHROUGH;
 	}
 
 	/* XXX This isn't correct.  This log parameter only has two bytes.... */
 #if 0
 	dpt->cache_size = param[5]
 			| (param[6] << 8)
 			| (param[7] << 16)
 			| (param[8] << 24);
 #endif
 }
 
 static void
 dpt_poll(struct cam_sim *sim)
 {
 	dpt_intr_locked(cam_sim_softc(sim));
 }
 
 static void
 dptexecuteccb(void *arg, bus_dma_segment_t *dm_segs, int nseg, int error)
 {
 	struct	 dpt_ccb *dccb;
 	union	 ccb *ccb;
 	struct	 dpt_softc *dpt;
 
 	dccb = (struct dpt_ccb *)arg;
 	ccb = dccb->ccb;
 	dpt = (struct dpt_softc *)ccb->ccb_h.ccb_dpt_ptr;
 	if (!dumping)
 		mtx_assert(&dpt->lock, MA_OWNED);
 
 	if (error != 0) {
 		if (error != EFBIG)
 			device_printf(dpt->dev,
 			       "Unexepected error 0x%x returned from "
 			       "bus_dmamap_load\n", error);
 		if (ccb->ccb_h.status == CAM_REQ_INPROG) {
 			xpt_freeze_devq(ccb->ccb_h.path, /*count*/1);
 			ccb->ccb_h.status = CAM_REQ_TOO_BIG|CAM_DEV_QFRZN;
 		}
 		dptfreeccb(dpt, dccb);
 		xpt_done(ccb);
 		return;
 	}
 		
 	if (nseg != 0) {
 		dpt_sg_t *sg;
 		bus_dma_segment_t *end_seg;
 		bus_dmasync_op_t op;
 
 		end_seg = dm_segs + nseg;
 
 		/* Copy the segments into our SG list */
 		sg = dccb->sg_list;
 		while (dm_segs < end_seg) {
 			sg->seg_len = htonl(dm_segs->ds_len);
 			sg->seg_addr = htonl(dm_segs->ds_addr);
 			sg++;
 			dm_segs++;
 		}
 
 		if (nseg > 1) {
 			dccb->eata_ccb.scatter = 1;
 			dccb->eata_ccb.cp_dataDMA = dccb->sg_busaddr;
 			dccb->eata_ccb.cp_datalen =
 			    htonl(nseg * sizeof(dpt_sg_t));
 		} else {
 			dccb->eata_ccb.cp_dataDMA = dccb->sg_list[0].seg_addr;
 			dccb->eata_ccb.cp_datalen = dccb->sg_list[0].seg_len;
 		}
 
 		if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN)
 			op = BUS_DMASYNC_PREREAD;
 		else
 			op = BUS_DMASYNC_PREWRITE;
 
 		bus_dmamap_sync(dpt->buffer_dmat, dccb->dmamap, op);
 
 	} else {
 		dccb->eata_ccb.cp_dataDMA = 0;
 		dccb->eata_ccb.cp_datalen = 0;
 	}
 
 	/*
 	 * Last time we need to check if this CCB needs to
 	 * be aborted.
 	 */
 	if (ccb->ccb_h.status != CAM_REQ_INPROG) {
 		if (nseg != 0)
 			bus_dmamap_unload(dpt->buffer_dmat, dccb->dmamap);
 		dptfreeccb(dpt, dccb);
 		xpt_done(ccb);
 		return;
 	}
 		
 	dccb->state |= DCCB_ACTIVE;
 	ccb->ccb_h.status |= CAM_SIM_QUEUED;
 	LIST_INSERT_HEAD(&dpt->pending_ccb_list, &ccb->ccb_h, sim_links.le);
 	callout_reset(&dccb->timer, (ccb->ccb_h.timeout * hz) / 1000,
 	    dpttimeout, dccb);
 	if (dpt_send_eata_command(dpt, &dccb->eata_ccb,
 				  dccb->eata_ccb.cp_busaddr,
 				  EATA_CMD_DMA_SEND_CP, 0, 0, 0, 0) != 0) {
 		ccb->ccb_h.status = CAM_NO_HBA; /* HBA dead or just busy?? */
 		if (nseg != 0)
 			bus_dmamap_unload(dpt->buffer_dmat, dccb->dmamap);
 		dptfreeccb(dpt, dccb);
 		xpt_done(ccb);
 	}
 }
 
 static void
 dpt_action(struct cam_sim *sim, union ccb *ccb)
 {
 	struct	  dpt_softc *dpt;
 
 	CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE, ("dpt_action\n"));
 	
 	dpt = (struct dpt_softc *)cam_sim_softc(sim);
 	mtx_assert(&dpt->lock, MA_OWNED);
 
 	if ((dpt->state & DPT_HA_SHUTDOWN_ACTIVE) != 0) {
 		xpt_print_path(ccb->ccb_h.path);
 		printf("controller is shutdown. Aborting CCB.\n");
 		ccb->ccb_h.status = CAM_NO_HBA;
 		xpt_done(ccb);
 		return;
 	}
 
 	switch (ccb->ccb_h.func_code) {
 	/* Common cases first */
 	case XPT_SCSI_IO:	/* Execute the requested I/O operation */
 	{
 		struct	ccb_scsiio *csio;
 		struct	ccb_hdr *ccbh;
 		struct	dpt_ccb *dccb;
 		struct	eata_ccb *eccb;
 
 		csio = &ccb->csio;
 		ccbh = &ccb->ccb_h;
 		/* Max CDB length is 12 bytes */
 		if (csio->cdb_len > 12) { 
 			ccb->ccb_h.status = CAM_REQ_INVALID;
 			xpt_done(ccb);
 			return;
 		}
 		if ((dccb = dptgetccb(dpt)) == NULL) {
 			dpt->resource_shortage = 1;
 			xpt_freeze_simq(sim, /*count*/1);
 			ccb->ccb_h.status = CAM_REQUEUE_REQ;
 			xpt_done(ccb);
 			return;
 		}
 		eccb = &dccb->eata_ccb;
 
 		/* Link dccb and ccb so we can find one from the other */
 		dccb->ccb = ccb;
 		ccb->ccb_h.ccb_dccb_ptr = dccb;
 		ccb->ccb_h.ccb_dpt_ptr = dpt;
 
 		/*
 		 * Explicitly set all flags so that the compiler can
 		 * be smart about setting them.
 		 */
 		eccb->SCSI_Reset = 0;
 		eccb->HBA_Init = 0;
 		eccb->Auto_Req_Sen = (ccb->ccb_h.flags & CAM_DIS_AUTOSENSE)
 				   ? 0 : 1;
 		eccb->scatter = 0;
 		eccb->Quick = 0;
 		eccb->Interpret =
 		    ccb->ccb_h.target_id == dpt->hostid[cam_sim_bus(sim)]
 		    ? 1 : 0;
 		eccb->DataOut = (ccb->ccb_h.flags & CAM_DIR_OUT) ? 1 : 0;
 		eccb->DataIn = (ccb->ccb_h.flags & CAM_DIR_IN) ? 1 : 0;
 		eccb->reqlen = csio->sense_len;
 		eccb->cp_id = ccb->ccb_h.target_id;
 		eccb->cp_channel = cam_sim_bus(sim);
 		eccb->cp_LUN = ccb->ccb_h.target_lun;
 		eccb->cp_luntar = 0;
 		eccb->cp_dispri = (ccb->ccb_h.flags & CAM_DIS_DISCONNECT)
 				? 0 : 1;
 		eccb->cp_identify = 1;
 
 		if ((ccb->ccb_h.flags & CAM_TAG_ACTION_VALID) != 0
 		 && csio->tag_action != CAM_TAG_ACTION_NONE) {
 			eccb->cp_msg[0] = csio->tag_action;
 			eccb->cp_msg[1] = dccb->tag;
 		} else {
 			eccb->cp_msg[0] = 0;
 			eccb->cp_msg[1] = 0;
 		}
 		eccb->cp_msg[2] = 0;
 
 		if ((ccb->ccb_h.flags & CAM_CDB_POINTER) != 0) {
 			if ((ccb->ccb_h.flags & CAM_CDB_PHYS) == 0) {
 				bcopy(csio->cdb_io.cdb_ptr,
 				      eccb->cp_cdb, csio->cdb_len);
 			} else {
 				/* I guess I could map it in... */
 				ccb->ccb_h.status = CAM_REQ_INVALID;
 				dptfreeccb(dpt, dccb);
 				xpt_done(ccb);
 				return;
 			}
 		} else {
 			bcopy(csio->cdb_io.cdb_bytes,
 			      eccb->cp_cdb, csio->cdb_len);
 		}
 		/*
 		 * If we have any data to send with this command,
 		 * map it into bus space.
 		 */
 	        /* Only use S/G if there is a transfer */
 		if ((ccbh->flags & CAM_DIR_MASK) != CAM_DIR_NONE) {
 			int error;
 
 			error = bus_dmamap_load_ccb(dpt->buffer_dmat,
 						    dccb->dmamap,
 						    ccb,
 						    dptexecuteccb,
 						    dccb, /*flags*/0);
 			if (error == EINPROGRESS) {
 				/*
 				 * So as to maintain ordering,
 				 * freeze the controller queue
 				 * until our mapping is
 				 * returned.
 				 */
 				xpt_freeze_simq(sim, 1);
 				dccb->state |= CAM_RELEASE_SIMQ;
 			}
 		} else {
 			/*
 			 * XXX JGibbs.
 			 * Does it want them both on or both off?
 			 * CAM_DIR_NONE is both on, so this code can
 			 * be removed if this is also what the DPT
 			 * exptects.
 			 */
 			eccb->DataOut = 0;
 			eccb->DataIn = 0;
 			dptexecuteccb(dccb, NULL, 0, 0);
 		}
 		break;
 	}
 	case XPT_RESET_DEV:	/* Bus Device Reset the specified SCSI device */
 	case XPT_ABORT:			/* Abort the specified CCB */
 		/* XXX Implement */
 		ccb->ccb_h.status = CAM_REQ_INVALID;
 		xpt_done(ccb);
 		break;
 	case XPT_SET_TRAN_SETTINGS:
 	{
 		ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
 		xpt_done(ccb);  
 		break;
 	}
 	case XPT_GET_TRAN_SETTINGS:
 	/* Get default/user set transfer settings for the target */
 	{
 		struct	ccb_trans_settings *cts = &ccb->cts;
 		struct ccb_trans_settings_scsi *scsi =
 		    &cts->proto_specific.scsi;
 		struct ccb_trans_settings_spi *spi =
 		    &cts->xport_specific.spi;
 
 		cts->protocol = PROTO_SCSI;
 		cts->protocol_version = SCSI_REV_2;
 		cts->transport = XPORT_SPI;
 		cts->transport_version = 2;
  
 		if (cts->type == CTS_TYPE_USER_SETTINGS) {
 			spi->flags = CTS_SPI_FLAGS_DISC_ENB;
 			spi->bus_width = (dpt->max_id > 7)
 				       ? MSG_EXT_WDTR_BUS_8_BIT
 				       : MSG_EXT_WDTR_BUS_16_BIT;
 			spi->sync_period = 25; /* 10MHz */
 			if (spi->sync_period != 0)
 				spi->sync_offset = 15;
 			scsi->flags = CTS_SCSI_FLAGS_TAG_ENB;
 
 			spi->valid = CTS_SPI_VALID_SYNC_RATE
 				| CTS_SPI_VALID_SYNC_OFFSET
 				| CTS_SPI_VALID_BUS_WIDTH
 				| CTS_SPI_VALID_DISC;
 			scsi->valid = CTS_SCSI_VALID_TQ;
 			ccb->ccb_h.status = CAM_REQ_CMP;
 		} else {
 			ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
 		}
 		xpt_done(ccb);
 		break;
 	}
 	case XPT_CALC_GEOMETRY:
 	{
 		/*
 		 * XXX Use Adaptec translation until I find out how to
 		 *     get this information from the card.
 		 */
 		cam_calc_geometry(&ccb->ccg, /*extended*/1);
 		xpt_done(ccb);
 		break;
 	}
 	case XPT_RESET_BUS:		/* Reset the specified SCSI bus */
 	{
 		/* XXX Implement */
 		ccb->ccb_h.status = CAM_REQ_CMP;
 		xpt_done(ccb);
 		break;
 	}
 	case XPT_TERM_IO:		/* Terminate the I/O process */
 		/* XXX Implement */
 		ccb->ccb_h.status = CAM_REQ_INVALID;
 		xpt_done(ccb);
 		break;
 	case XPT_PATH_INQ:		/* Path routing inquiry */
 	{
 		struct ccb_pathinq *cpi = &ccb->cpi;
 		
 		cpi->version_num = 1;
 		cpi->hba_inquiry = PI_SDTR_ABLE|PI_TAG_ABLE;
 		if (dpt->max_id > 7)
 			cpi->hba_inquiry |= PI_WIDE_16;
 		cpi->target_sprt = 0;
 		cpi->hba_misc = 0;
 		cpi->hba_eng_cnt = 0;
 		cpi->max_target = dpt->max_id;
 		cpi->max_lun = dpt->max_lun;
 		cpi->initiator_id = dpt->hostid[cam_sim_bus(sim)];
 		cpi->bus_id = cam_sim_bus(sim);
 		cpi->base_transfer_speed = 3300;
 		strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
 		strncpy(cpi->hba_vid, "DPT", HBA_IDLEN);
 		strncpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
 		cpi->unit_number = cam_sim_unit(sim);
                 cpi->transport = XPORT_SPI;
                 cpi->transport_version = 2;
                 cpi->protocol = PROTO_SCSI;
                 cpi->protocol_version = SCSI_REV_2;
 		cpi->ccb_h.status = CAM_REQ_CMP;
 		xpt_done(ccb);
 		break;
 	}
 	default:
 		ccb->ccb_h.status = CAM_REQ_INVALID;
 		xpt_done(ccb);
 		break;
 	}
 }
 
 /*
  * This routine will try to send an EATA command to the DPT HBA.
  * It will, by default, try 20,000 times, waiting 50us between tries.
  * It returns 0 on success and 1 on failure.
  */
 static int
 dpt_send_eata_command(dpt_softc_t *dpt, eata_ccb_t *cmd_block,
 		      u_int32_t cmd_busaddr, u_int command, u_int retries,
 		      u_int ifc, u_int code, u_int code2)
 {
 	u_int	loop;
 	
 	if (!retries)
 		retries = 20000;
 
 	/*
 	 * I hate this polling nonsense. Wish there was a way to tell the DPT
 	 * to go get commands at its own pace,  or to interrupt when ready.
 	 * In the mean time we will measure how many itterations it really
 	 * takes.
 	 */
 	for (loop = 0; loop < retries; loop++) {
 		if ((dpt_inb(dpt, HA_RAUXSTAT) & HA_ABUSY) == 0)
 			break;
 		else
 			DELAY(50);
 	}
 
 	if (loop < retries) {
 #ifdef DPT_MEASURE_PERFORMANCE
 		if (loop > dpt->performance.max_eata_tries)
 			dpt->performance.max_eata_tries = loop;
 
 		if (loop < dpt->performance.min_eata_tries)
 			dpt->performance.min_eata_tries = loop;
 #endif
 	} else {
 #ifdef DPT_MEASURE_PERFORMANCE
 		++dpt->performance.command_too_busy;
 #endif
 		return (1);
 	}
 
 	/* The controller is alive, advance the wedge timer */
 #ifdef DPT_RESET_HBA
 	dpt->last_contact = microtime_now;
 #endif
 
 	if (cmd_block == NULL)
 		cmd_busaddr = 0;
 #if (BYTE_ORDER == BIG_ENDIAN)
 	else {
 		cmd_busaddr = ((cmd_busaddr >> 24) & 0xFF)
 			    | ((cmd_busaddr >> 16) & 0xFF)
 			    | ((cmd_busaddr >> 8) & 0xFF)
 			    | (cmd_busaddr & 0xFF);
 	}
 #endif
 	/* And now the address */
 	dpt_outl(dpt, HA_WDMAADDR, cmd_busaddr);
 
 	if (command == EATA_CMD_IMMEDIATE) {
 		if (cmd_block == NULL) {
 			dpt_outb(dpt, HA_WCODE2, code2);
 			dpt_outb(dpt, HA_WCODE, code);
 		}
 		dpt_outb(dpt, HA_WIFC, ifc);
 	}
 	dpt_outb(dpt, HA_WCOMMAND, command);
 
 	return (0);
 }
 
 
 /* ==================== Exported Function definitions =======================*/
 void
 dpt_alloc(device_t dev)
 {
 	dpt_softc_t	*dpt = device_get_softc(dev);
 	int    i;
 
 	mtx_init(&dpt->lock, "dpt", NULL, MTX_DEF);
 	SLIST_INIT(&dpt->free_dccb_list);
 	LIST_INIT(&dpt->pending_ccb_list);
 	for (i = 0; i < MAX_CHANNELS; i++)
 		dpt->resetlevel[i] = DPT_HA_OK;
 
 #ifdef DPT_MEASURE_PERFORMANCE
 	dpt_reset_performance(dpt);
 #endif /* DPT_MEASURE_PERFORMANCE */
 	return;
 }
 
 void
 dpt_free(struct dpt_softc *dpt)
 {
 	switch (dpt->init_level) {
 	default:
 	case 5:
 		bus_dmamap_unload(dpt->dccb_dmat, dpt->dccb_dmamap);
 	case 4:
 		bus_dmamem_free(dpt->dccb_dmat, dpt->dpt_dccbs,
 				dpt->dccb_dmamap);
-		bus_dmamap_destroy(dpt->dccb_dmat, dpt->dccb_dmamap);
 	case 3:
 		bus_dma_tag_destroy(dpt->dccb_dmat);
 	case 2:
 		bus_dma_tag_destroy(dpt->buffer_dmat);
 	case 1:
 	{
 		struct sg_map_node *sg_map;
 
 		while ((sg_map = SLIST_FIRST(&dpt->sg_maps)) != NULL) {
 			SLIST_REMOVE_HEAD(&dpt->sg_maps, links);
 			bus_dmamap_unload(dpt->sg_dmat,
 					  sg_map->sg_dmamap);
 			bus_dmamem_free(dpt->sg_dmat, sg_map->sg_vaddr,
 					sg_map->sg_dmamap);
 			free(sg_map, M_DEVBUF);
 		}
 		bus_dma_tag_destroy(dpt->sg_dmat);
 	}
 	case 0:
 		break;
 	}
 	mtx_destroy(&dpt->lock);
 }
 
 int
 dpt_alloc_resources (device_t dev)
 {
 	dpt_softc_t *	dpt;
 	int		error;
 
 	dpt = device_get_softc(dev);
 
 	dpt->io_res = bus_alloc_resource_any(dev, dpt->io_type, &dpt->io_rid,
 					     RF_ACTIVE);
 	if (dpt->io_res == NULL) {
 		device_printf(dev, "No I/O space?!\n");
 		error = ENOMEM;
 		goto bad;
 	}
 
 	dpt->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &dpt->irq_rid,
 					      RF_ACTIVE);
 	if (dpt->irq_res == NULL) {
 		device_printf(dev, "No IRQ!\n");
 		error = ENOMEM;
 		goto bad;
 	}
 
 	return (0);
 bad:
 	return(error);
 }
 
 
 void
 dpt_release_resources (device_t dev)
 {
 	struct dpt_softc *	dpt;
 
 	dpt = device_get_softc(dev);
 
 	if (dpt->ih)
 		bus_teardown_intr(dev, dpt->irq_res, dpt->ih);
         if (dpt->io_res)
                 bus_release_resource(dev, dpt->io_type, dpt->io_rid, dpt->io_res);
         if (dpt->irq_res)
                 bus_release_resource(dev, SYS_RES_IRQ, dpt->irq_rid, dpt->irq_res);
         if (dpt->drq_res)
                 bus_release_resource(dev, SYS_RES_DRQ, dpt->drq_rid, dpt->drq_res);
 
 	return;
 }
 
 static u_int8_t string_sizes[] =
 {
 	sizeof(((dpt_inq_t*)NULL)->vendor),
 	sizeof(((dpt_inq_t*)NULL)->modelNum),
 	sizeof(((dpt_inq_t*)NULL)->firmware),
 	sizeof(((dpt_inq_t*)NULL)->protocol),
 };
 
 int
 dpt_init(struct dpt_softc *dpt)
 {
 	dpt_conf_t  conf;
 	struct	    sg_map_node *sg_map;
 	dpt_ccb_t  *dccb;
 	u_int8_t   *strp;
 	int	    index;
 	int	    i;
 	int	    retval;
 
 	dpt->init_level = 0;
 	SLIST_INIT(&dpt->sg_maps);
 	mtx_lock(&dpt->lock);
 
 #ifdef DPT_RESET_BOARD
 	device_printf(dpt->dev, "resetting HBA\n");
 	dpt_outb(dpt, HA_WCOMMAND, EATA_CMD_RESET);
 	DELAY(750000);
 	/* XXX Shouldn't we poll a status register or something??? */
 #endif
 	/* DMA tag for our S/G structures.  We allocate in page sized chunks */
 	if (bus_dma_tag_create(	/* parent	*/ dpt->parent_dmat,
 				/* alignment	*/ 1,
 				/* boundary	*/ 0,
 				/* lowaddr	*/ BUS_SPACE_MAXADDR,
 				/* highaddr	*/ BUS_SPACE_MAXADDR,
 				/* filter	*/ NULL,
 				/* filterarg	*/ NULL,
 				/* maxsize	*/ PAGE_SIZE,
 				/* nsegments	*/ 1,
 				/* maxsegsz	*/ BUS_SPACE_MAXSIZE_32BIT,
 				/* flags	*/ 0,
 				/* lockfunc	*/ NULL,
 				/* lockarg	*/ NULL,
 				&dpt->sg_dmat) != 0) {
 		goto error_exit;
         }
 
 	dpt->init_level++;
 
 	/*
 	 * We allocate our DPT ccbs as a contiguous array of bus dma'able
 	 * memory.  To get the allocation size, we need to know how many
 	 * ccbs the card supports.  This requires a ccb.  We solve this
 	 * chicken and egg problem by allocating some re-usable S/G space
 	 * up front, and treating it as our status packet, CCB, and target
 	 * memory space for these commands.
 	 */
 	sg_map = dptallocsgmap(dpt);
 	if (sg_map == NULL)
 		goto error_exit;
 
 	dpt->sp = (volatile dpt_sp_t *)sg_map->sg_vaddr;
 	dccb = (struct dpt_ccb *)(uintptr_t)(volatile void *)&dpt->sp[1];
 	bzero(dccb, sizeof(*dccb));
 	dpt->sp_physaddr = sg_map->sg_physaddr;
 	dccb->eata_ccb.cp_dataDMA =
 	    htonl(sg_map->sg_physaddr + sizeof(dpt_sp_t) + sizeof(*dccb));
 	dccb->eata_ccb.cp_busaddr = ~0;
 	dccb->eata_ccb.cp_statDMA = htonl(dpt->sp_physaddr);
 	dccb->eata_ccb.cp_reqDMA = htonl(dpt->sp_physaddr + sizeof(*dccb)
 				       + offsetof(struct dpt_ccb, sense_data));
 
 	/* Okay.  Fetch our config */
 	bzero(&dccb[1], sizeof(conf)); /* data area */
 	retval = dpt_get_conf(dpt, dccb, sg_map->sg_physaddr + sizeof(dpt_sp_t),
 			      sizeof(conf), 0xc1, 7, 1);
 
 	if (retval != 0) {
 		device_printf(dpt->dev, "Failed to get board configuration\n");
 		goto error_exit;
 	}
 	bcopy(&dccb[1], &conf, sizeof(conf));
 
 	bzero(&dccb[1], sizeof(dpt->board_data));
 	retval = dpt_get_conf(dpt, dccb, sg_map->sg_physaddr + sizeof(dpt_sp_t),
 			      sizeof(dpt->board_data), 0, conf.scsi_id0, 0);
 	if (retval != 0) {
 		device_printf(dpt->dev, "Failed to get inquiry information\n");
 		goto error_exit;
 	}
 	bcopy(&dccb[1], &dpt->board_data, sizeof(dpt->board_data));
 
 	dpt_detect_cache(dpt, dccb, sg_map->sg_physaddr + sizeof(dpt_sp_t),
 			 (u_int8_t *)&dccb[1]);
 
 	switch (ntohl(conf.splen)) {
 	case DPT_EATA_REVA:
 		dpt->EATA_revision = 'a';
 		break;
 	case DPT_EATA_REVB:
 		dpt->EATA_revision = 'b';
 		break;
 	case DPT_EATA_REVC:
 		dpt->EATA_revision = 'c';
 		break;
 	case DPT_EATA_REVZ:
 		dpt->EATA_revision = 'z';
 		break;
 	default:
 		dpt->EATA_revision = '?';
 	}
 
 	dpt->max_id	 = conf.MAX_ID;
 	dpt->max_lun	 = conf.MAX_LUN;
 	dpt->irq	 = conf.IRQ;
 	dpt->dma_channel = (8 - conf.DMA_channel) & 7;
 	dpt->channels	 = conf.MAX_CHAN + 1;
 	dpt->state	|= DPT_HA_OK;
 	if (conf.SECOND)
 		dpt->primary = FALSE;
 	else
 		dpt->primary = TRUE;
 
 	dpt->more_support = conf.MORE_support;
 
 	if (strncmp(dpt->board_data.firmware, "07G0", 4) >= 0)
 		dpt->immediate_support = 1;
 	else
 		dpt->immediate_support = 0;
 
 	dpt->broken_INQUIRY = FALSE;
 
 	dpt->cplen = ntohl(conf.cplen);
 	dpt->cppadlen = ntohs(conf.cppadlen);
 	dpt->max_dccbs = ntohs(conf.queuesiz);
 
 	if (dpt->max_dccbs > 256) {
 		device_printf(dpt->dev, "Max CCBs reduced from %d to "
 		       "256 due to tag algorithm\n", dpt->max_dccbs);
 		dpt->max_dccbs = 256;
 	}
 
 	dpt->hostid[0] = conf.scsi_id0;
 	dpt->hostid[1] = conf.scsi_id1;
 	dpt->hostid[2] = conf.scsi_id2;
 
 	if (conf.SG_64K)
 		dpt->sgsize = 8192;
 	else
 		dpt->sgsize = ntohs(conf.SGsiz);
 
 	/* We can only get 64k buffers, so don't bother to waste space. */
 	if (dpt->sgsize < 17 || dpt->sgsize > 32)
 		dpt->sgsize = 32; 
 
 	if (dpt->sgsize > dpt_max_segs)
 		dpt->sgsize = dpt_max_segs;
 	
 	/* DMA tag for mapping buffers into device visible space. */
 	if (bus_dma_tag_create(	/* parent	*/ dpt->parent_dmat,
 				/* alignment	*/ 1,
 				/* boundary	*/ 0,
 				/* lowaddr	*/ BUS_SPACE_MAXADDR,
 				/* highaddr	*/ BUS_SPACE_MAXADDR,
 				/* filter	*/ NULL,
 				/* filterarg	*/ NULL,
 				/* maxsize	*/ MAXBSIZE,
 				/* nsegments	*/ dpt->sgsize,
 				/* maxsegsz	*/ BUS_SPACE_MAXSIZE_32BIT,
 				/* flags	*/ BUS_DMA_ALLOCNOW,
 				/* lockfunc	*/ busdma_lock_mutex,
 				/* lockarg	*/ &dpt->lock,
 				&dpt->buffer_dmat) != 0) {
 		device_printf(dpt->dev,
 		    "bus_dma_tag_create(...,dpt->buffer_dmat) failed\n");
 		goto error_exit;
 	}
 
 	dpt->init_level++;
 
 	/* DMA tag for our ccb structures and interrupt status packet */
 	if (bus_dma_tag_create(	/* parent	*/ dpt->parent_dmat,
 				/* alignment	*/ 1,
 				/* boundary	*/ 0,
 				/* lowaddr	*/ BUS_SPACE_MAXADDR,
 				/* highaddr	*/ BUS_SPACE_MAXADDR,
 				/* filter	*/ NULL,
 				/* filterarg	*/ NULL,
 				/* maxsize	*/ (dpt->max_dccbs *
 						    sizeof(struct dpt_ccb)) +
 						    sizeof(dpt_sp_t),
 				/* nsegments	*/ 1,
 				/* maxsegsz	*/ BUS_SPACE_MAXSIZE_32BIT,
 				/* flags	*/ 0,
 				/* lockfunc	*/ NULL,
 				/* lockarg	*/ NULL,
 				&dpt->dccb_dmat) != 0) {
 		device_printf(dpt->dev,
 		    "bus_dma_tag_create(...,dpt->dccb_dmat) failed\n");
 		goto error_exit;
         }
 
 	dpt->init_level++;
 
 	/* Allocation for our ccbs and interrupt status packet */
 	if (bus_dmamem_alloc(dpt->dccb_dmat, (void **)&dpt->dpt_dccbs,
 			     BUS_DMA_NOWAIT, &dpt->dccb_dmamap) != 0) {
 		device_printf(dpt->dev,
 		    "bus_dmamem_alloc(dpt->dccb_dmat,...) failed\n");
 		goto error_exit;
 	}
 
 	dpt->init_level++;
 
 	/* And permanently map them */
 	bus_dmamap_load(dpt->dccb_dmat, dpt->dccb_dmamap,
        			dpt->dpt_dccbs,
 			(dpt->max_dccbs * sizeof(struct dpt_ccb))
 			+ sizeof(dpt_sp_t),
 			dptmapmem, &dpt->dpt_ccb_busbase, /*flags*/0);
 
 	/* Clear them out. */
 	bzero(dpt->dpt_dccbs,
 	      (dpt->max_dccbs * sizeof(struct dpt_ccb)) + sizeof(dpt_sp_t));
 
 	dpt->dpt_ccb_busend = dpt->dpt_ccb_busbase;
 
 	dpt->sp = (dpt_sp_t*)&dpt->dpt_dccbs[dpt->max_dccbs];
 	dpt->sp_physaddr = dpt->dpt_ccb_busbase
 			 + (dpt->max_dccbs * sizeof(dpt_ccb_t));
 	dpt->init_level++;
 
 	/* Allocate our first batch of ccbs */
 	if (dptallocccbs(dpt) == 0) {
 		device_printf(dpt->dev, "dptallocccbs(dpt) == 0\n");
 		mtx_unlock(&dpt->lock);
 		return (2);
 	}
 
 	/* Prepare for Target Mode */
 	dpt->target_mode_enabled = 1;
 
 	/* Nuke excess spaces from inquiry information */
 	strp = dpt->board_data.vendor;
 	for (i = 0; i < sizeof(string_sizes); i++) {
 		index = string_sizes[i] - 1;	
 		while (index && (strp[index] == ' '))
 			strp[index--] = '\0';
 		strp += string_sizes[i];
 	}
 
 	device_printf(dpt->dev, "%.8s %.16s FW Rev. %.4s, ",
 	       dpt->board_data.vendor,
 	       dpt->board_data.modelNum, dpt->board_data.firmware);
 
 	printf("%d channel%s, ", dpt->channels, dpt->channels > 1 ? "s" : "");
 
 	if (dpt->cache_type != DPT_NO_CACHE
 	 && dpt->cache_size != 0) {
 		printf("%s Cache, ",
 		       dpt->cache_type == DPT_CACHE_WRITETHROUGH
 		     ? "Write-Through" : "Write-Back");
 	}
 
 	printf("%d CCBs\n", dpt->max_dccbs);
 	mtx_unlock(&dpt->lock);
 	return (0);
 		
 error_exit:
 	mtx_unlock(&dpt->lock);
 	return (1);
 }
 
 int
 dpt_attach(dpt_softc_t *dpt)
 {
 	struct cam_devq *devq;
 	int i;
 
 	/*
 	 * Create the device queue for our SIM.
 	 */
 	devq = cam_simq_alloc(dpt->max_dccbs);
 	if (devq == NULL)
 		return (0);
 
 	mtx_lock(&dpt->lock);
 	for (i = 0; i < dpt->channels; i++) {
 		/*
 		 * Construct our SIM entry
 		 */
 		dpt->sims[i] = cam_sim_alloc(dpt_action, dpt_poll, "dpt",
 		    dpt, device_get_unit(dpt->dev), &dpt->lock,
 					     /*untagged*/2,
 					     /*tagged*/dpt->max_dccbs, devq);
 		if (dpt->sims[i] == NULL) {
 			if (i == 0)
 				cam_simq_free(devq);
 			else
 				printf(	"%s(): Unable to attach bus %d "
 					"due to resource shortage\n",
 					__func__, i);
 			break;
 		}
 
 		if (xpt_bus_register(dpt->sims[i], dpt->dev, i) != CAM_SUCCESS){
 			cam_sim_free(dpt->sims[i], /*free_devq*/i == 0);
 			dpt->sims[i] = NULL;
 			break;
 		}
 
 		if (xpt_create_path(&dpt->paths[i], /*periph*/NULL,
 				    cam_sim_path(dpt->sims[i]),
 				    CAM_TARGET_WILDCARD,
 				    CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
 			xpt_bus_deregister(cam_sim_path(dpt->sims[i]));
 			cam_sim_free(dpt->sims[i], /*free_devq*/i == 0);
 			dpt->sims[i] = NULL;
 			break;
 		}
 
 	}
 	mtx_unlock(&dpt->lock);
 	if (i > 0)
 		EVENTHANDLER_REGISTER(shutdown_final, dptshutdown,
 				      dpt, SHUTDOWN_PRI_DEFAULT);
 	return (i);
 }
 
 int
 dpt_detach (device_t dev)
 {
 	struct dpt_softc *	dpt;
 	int			i;
 
 	dpt = device_get_softc(dev);
 
 	mtx_lock(&dpt->lock);
 	for (i = 0; i < dpt->channels; i++) {
 #if 0
 	        xpt_async(AC_LOST_DEVICE, dpt->paths[i], NULL);
 #endif
         	xpt_free_path(dpt->paths[i]);
         	xpt_bus_deregister(cam_sim_path(dpt->sims[i]));
         	cam_sim_free(dpt->sims[i], /*free_devq*/TRUE);
 	}
 	mtx_unlock(&dpt->lock);
 
 	dptshutdown((void *)dpt, SHUTDOWN_PRI_DEFAULT);
 
 	dpt_release_resources(dev);
 
 	dpt_free(dpt);
 
 	return (0);
 }
 
 /*
  * This is the interrupt handler for the DPT driver.
  */
 void
 dpt_intr(void *arg)
 {
 	dpt_softc_t    *dpt;
 
 	dpt = arg;
 	mtx_lock(&dpt->lock);
 	dpt_intr_locked(dpt);
 	mtx_unlock(&dpt->lock);
 }
 
 void
 dpt_intr_locked(dpt_softc_t *dpt)
 {
 	dpt_ccb_t      *dccb;
 	union ccb      *ccb;
 	u_int		status;
 	u_int		aux_status;
 	u_int		hba_stat;
 	u_int		scsi_stat;
 	u_int32_t	residue_len;	/* Number of bytes not transferred */
 
 	/* First order of business is to check if this interrupt is for us */
 	while (((aux_status = dpt_inb(dpt, HA_RAUXSTAT)) & HA_AIRQ) != 0) {
 
 		/*
 		 * What we want to do now, is to capture the status, all of it,
 		 * move it where it belongs, wake up whoever sleeps waiting to
 		 * process this result, and get out of here.
 		 */
 		if (dpt->sp->ccb_busaddr < dpt->dpt_ccb_busbase
 		 || dpt->sp->ccb_busaddr >= dpt->dpt_ccb_busend) {
 			device_printf(dpt->dev,
 			    "Encountered bogus status packet\n");
 			status = dpt_inb(dpt, HA_RSTATUS);
 			return;
 		}
 
 		dccb = dptccbptov(dpt, dpt->sp->ccb_busaddr);
 
 		dpt->sp->ccb_busaddr = ~0;
 
 		/* Ignore status packets with EOC not set */
 		if (dpt->sp->EOC == 0) {
 			device_printf(dpt->dev,
 			       "ERROR: Request %d received with "
 			       "clear EOC.\n     Marking as LOST.\n",
 			       dccb->transaction_id);
 
 			/* This CLEARS the interrupt! */
 			status = dpt_inb(dpt, HA_RSTATUS);
 			continue;
 		}
 		dpt->sp->EOC = 0;
 
 		/*
 		 * Double buffer the status information so the hardware can
 		 * work on updating the status packet while we decifer the
 		 * one we were just interrupted for.
 		 * According to Mark Salyzyn, we only need few pieces of it.
 		 */
 		hba_stat = dpt->sp->hba_stat;
 		scsi_stat = dpt->sp->scsi_stat;
 		residue_len = dpt->sp->residue_len;
 
 		/* Clear interrupts, check for error */
 		if ((status = dpt_inb(dpt, HA_RSTATUS)) & HA_SERROR) {
 			/*
 			 * Error Condition. Check for magic cookie. Exit
 			 * this test on earliest sign of non-reset condition
 			 */
 
 			/* Check that this is not a board reset interrupt */
 			if (dpt_just_reset(dpt)) {
 				device_printf(dpt->dev, "HBA rebooted.\n"
 				       "      All transactions should be "
 				       "resubmitted\n");
 
 				device_printf(dpt->dev,
 				       ">>---->>  This is incomplete, "
 				       "fix me....  <<----<<");
 				panic("DPT Rebooted");
 
 			}
 		}
 		/* Process CCB */
 		ccb = dccb->ccb;
 		callout_stop(&dccb->timer);
 		if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE) {
 			bus_dmasync_op_t op;
 
 			if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN)
 				op = BUS_DMASYNC_POSTREAD;
 			else
 				op = BUS_DMASYNC_POSTWRITE;
 			bus_dmamap_sync(dpt->buffer_dmat, dccb->dmamap, op);
 			bus_dmamap_unload(dpt->buffer_dmat, dccb->dmamap);
 		}
 
 		/* Common Case inline... */
 		if (hba_stat == HA_NO_ERROR) {
 			ccb->csio.scsi_status = scsi_stat;
 			ccb->ccb_h.status = 0;
 			switch (scsi_stat) {
 			case SCSI_STATUS_OK:
 				ccb->ccb_h.status |= CAM_REQ_CMP;
 				break;
 			case SCSI_STATUS_CHECK_COND:
 			case SCSI_STATUS_CMD_TERMINATED:
 				bcopy(&dccb->sense_data, &ccb->csio.sense_data,
 				      ccb->csio.sense_len);
 				ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
 				/* FALLTHROUGH */
 			default:
 				ccb->ccb_h.status |= CAM_SCSI_STATUS_ERROR;
 				/* XXX Freeze DevQ */
 				break;
 			}
 			ccb->csio.resid = residue_len;
 			dptfreeccb(dpt, dccb);
 			xpt_done(ccb);
 		} else {
 			dptprocesserror(dpt, dccb, ccb, hba_stat, scsi_stat,
 					residue_len);
 		}
 	}
 }
 
 static void
 dptprocesserror(dpt_softc_t *dpt, dpt_ccb_t *dccb, union ccb *ccb,
 		u_int hba_stat, u_int scsi_stat, u_int32_t resid)
 {
 	ccb->csio.resid = resid;
 	switch (hba_stat) {
 	case HA_ERR_SEL_TO:
 		ccb->ccb_h.status = CAM_SEL_TIMEOUT;
 		break;
 	case HA_ERR_CMD_TO:
 		ccb->ccb_h.status = CAM_CMD_TIMEOUT;
 		break;
 	case HA_SCSIBUS_RESET:
 	case HA_HBA_POWER_UP:	/* Similar effect to a bus reset??? */
 		ccb->ccb_h.status = CAM_SCSI_BUS_RESET;
 		break;
 	case HA_CP_ABORTED:
 	case HA_CP_RESET:	/* XXX ??? */
 	case HA_CP_ABORT_NA:	/* XXX ??? */
 	case HA_CP_RESET_NA:	/* XXX ??? */
 		if ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_INPROG)
 			ccb->ccb_h.status = CAM_REQ_ABORTED;
 		break;
 	case HA_PCI_PARITY:
 	case HA_PCI_MABORT:
 	case HA_PCI_TABORT:
 	case HA_PCI_STABORT:
 	case HA_BUS_PARITY:
 	case HA_PARITY_ERR:
 	case HA_ECC_ERR:
 		ccb->ccb_h.status = CAM_UNCOR_PARITY;
 		break;
 	case HA_UNX_MSGRJCT:
 		ccb->ccb_h.status = CAM_MSG_REJECT_REC;
 		break;
 	case HA_UNX_BUSPHASE:
 		ccb->ccb_h.status = CAM_SEQUENCE_FAIL;
 		break;
 	case HA_UNX_BUS_FREE:
 		ccb->ccb_h.status = CAM_UNEXP_BUSFREE;
 		break;
 	case HA_SCSI_HUNG:
 	case HA_RESET_STUCK:
 		/*
 		 * Dead???  Can the controller get unstuck
 		 * from these conditions
 		 */
 		ccb->ccb_h.status = CAM_NO_HBA;
 		break;
 	case HA_RSENSE_FAIL:
 		ccb->ccb_h.status = CAM_AUTOSENSE_FAIL;
 		break;
 	default:
 		device_printf(dpt->dev, "Undocumented Error %x\n", hba_stat);
 		printf("Please mail this message to shimon@simon-shapiro.org\n");
 		ccb->ccb_h.status = CAM_REQ_CMP_ERR;
 		break;
 	}
 	dptfreeccb(dpt, dccb);
 	xpt_done(ccb);
 }
 
 static void
 dpttimeout(void *arg)
 {
 	struct dpt_ccb	 *dccb;
 	union  ccb	 *ccb;
 	struct dpt_softc *dpt;
 
 	dccb = (struct dpt_ccb *)arg;
 	ccb = dccb->ccb;
 	dpt = (struct dpt_softc *)ccb->ccb_h.ccb_dpt_ptr;
 	mtx_assert(&dpt->lock, MA_OWNED);
 	xpt_print_path(ccb->ccb_h.path);
 	printf("CCB %p - timed out\n", (void *)dccb);
 
 	/*
 	 * Try to clear any pending jobs.  FreeBSD will lose interrupts,
 	 * leaving the controller suspended, and commands timed-out.
 	 * By calling the interrupt handler, any command thus stuck will be
 	 * completed.
 	 */
 	dpt_intr_locked(dpt);
 	
 	if ((dccb->state & DCCB_ACTIVE) == 0) {
 		xpt_print_path(ccb->ccb_h.path);
 		printf("CCB %p - timed out CCB already completed\n",
 		       (void *)dccb);
 		return;
 	}
 
 	/* Abort this particular command.  Leave all others running */
 	dpt_send_immediate(dpt, &dccb->eata_ccb, dccb->eata_ccb.cp_busaddr,
 			   /*retries*/20000, EATA_SPECIFIC_ABORT, 0, 0);
 	ccb->ccb_h.status = CAM_CMD_TIMEOUT;
 }
 
 /*
  * Shutdown the controller and ensure that the cache is completely flushed.
  * Called from the shutdown_final event after all disk access has completed.
  */
 static void
 dptshutdown(void *arg, int howto)
 {
 	dpt_softc_t *dpt;
 
 	dpt = (dpt_softc_t *)arg;
 
 	device_printf(dpt->dev,
 	    "Shutting down (mode %x) HBA.	Please wait...\n", howto);
 
 	/*
 	 * What we do for a shutdown, is give the DPT early power loss warning
 	 */
 	mtx_lock(&dpt->lock);
 	dpt_send_immediate(dpt, NULL, 0, EATA_POWER_OFF_WARN, 0, 0, 0);
 	mtx_unlock(&dpt->lock);
 	DELAY(1000 * 1000 * 5);
 	device_printf(dpt->dev, "Controller was warned of shutdown and is now "
 	       "disabled\n");
 }
 
 /*============================================================================*/
 
 #if 0
 #ifdef DPT_RESET_HBA
 
 /*
 **	Function name : dpt_reset_hba
 **
 **	Description : Reset the HBA and properly discard all pending work
 **	Input :       Softc
 **	Output :      Nothing
 */
 static void
 dpt_reset_hba(dpt_softc_t *dpt)
 {
 	eata_ccb_t       *ccb;
 	dpt_ccb_t         dccb, *dccbp;
 	int               result;
 	struct scsi_xfer *xs;
 
 	mtx_assert(&dpt->lock, MA_OWNED);
 
 	/* Prepare a control block.  The SCSI command part is immaterial */
 	dccb.xs = NULL;
 	dccb.flags = 0;
 	dccb.state = DPT_CCB_STATE_NEW;
 	dccb.std_callback = NULL;
 	dccb.wrbuff_callback = NULL;
 
 	ccb = &dccb.eata_ccb;
 	ccb->CP_OpCode = EATA_CMD_RESET;
 	ccb->SCSI_Reset = 0;
 	ccb->HBA_Init = 1;
 	ccb->Auto_Req_Sen = 1;
 	ccb->cp_id = 0; /* Should be ignored */
 	ccb->DataIn = 1;
 	ccb->DataOut = 0;
 	ccb->Interpret = 1;
 	ccb->reqlen = htonl(sizeof(struct scsi_sense_data));
 	ccb->cp_statDMA = htonl(vtophys(&ccb->cp_statDMA));
 	ccb->cp_reqDMA = htonl(vtophys(&ccb->cp_reqDMA));
 	ccb->cp_viraddr = (u_int32_t) & ccb;
 
 	ccb->cp_msg[0] = HA_IDENTIFY_MSG | HA_DISCO_RECO;
 	ccb->cp_scsi_cmd = 0;  /* Should be ignored */
 
 	/* Lock up the submitted queue.  We are very persistant here */
 	while (dpt->queue_status & DPT_SUBMITTED_QUEUE_ACTIVE) {
 		DELAY(100);
 	}
 	
 	dpt->queue_status |= DPT_SUBMITTED_QUEUE_ACTIVE;
 
 	/* Send the RESET message */
 	if ((result = dpt_send_eata_command(dpt, &dccb.eata_ccb,
 					    EATA_CMD_RESET, 0, 0, 0, 0)) != 0) {
 		device_printf(dpt->dev, "Failed to send the RESET message.\n"
 		       "     Trying cold boot (ouch!)\n");
 	
 	
 		if ((result = dpt_send_eata_command(dpt, &dccb.eata_ccb,
 						    EATA_COLD_BOOT, 0, 0,
 						    0, 0)) != 0) {
 			panic("%s:  Faild to cold boot the HBA\n",
 			    device_get_nameunit(dpt->dev));
 		}
 #ifdef DPT_MEASURE_PERFORMANCE
 		dpt->performance.cold_boots++;
 #endif /* DPT_MEASURE_PERFORMANCE */
 	}
 	
 #ifdef DPT_MEASURE_PERFORMANCE
 	dpt->performance.warm_starts++;
 #endif /* DPT_MEASURE_PERFORMANCE */
 	
 	device_printf(dpt->dev,
 	    "Aborting pending requests.  O/S should re-submit\n");
 
 	while ((dccbp = TAILQ_FIRST(&dpt->completed_ccbs)) != NULL) {
 		struct scsi_xfer *xs = dccbp->xs;
 	    
 		/* Not all transactions have xs structs */
 		if (xs != NULL) {
 			/* Tell the kernel proper this did not complete well */
 			xs->error |= XS_SELTIMEOUT;
 			xs->flags |= SCSI_ITSDONE;
 			scsi_done(xs);
 		}
 	    
 		dpt_Qremove_submitted(dpt, dccbp);
 	
 		/* Remember, Callbacks are NOT in the standard queue */
 		if (dccbp->std_callback != NULL) {
 			(dccbp->std_callback)(dpt, dccbp->eata_ccb.cp_channel,
 					       dccbp);
 		} else {
 			dpt_Qpush_free(dpt, dccbp);
 		}
 	}
 
 	device_printf(dpt->dev, "reset done aborting all pending commands\n");
 	dpt->queue_status &= ~DPT_SUBMITTED_QUEUE_ACTIVE;
 }
 
 #endif /* DPT_RESET_HBA */ 
 
 /*
  * Build a Command Block for target mode READ/WRITE BUFFER,
  * with the ``sync'' bit ON.
  *
  * Although the length and offset are 24 bit fields in the command, they cannot
  * exceed 8192 bytes, so we take them as short integers andcheck their range.
  * If they are sensless, we round them to zero offset, maximum length and
  * complain.
  */
 
 static void
 dpt_target_ccb(dpt_softc_t * dpt, int bus, u_int8_t target, u_int8_t lun,
 	       dpt_ccb_t * ccb, int mode, u_int8_t command,
 	       u_int16_t length, u_int16_t offset)
 {
 	eata_ccb_t     *cp;
 
 	mtx_assert(&dpt->lock, MA_OWNED);
 	if ((length + offset) > DPT_MAX_TARGET_MODE_BUFFER_SIZE) {
 		device_printf(dpt->dev,
 		    "Length of %d, and offset of %d are wrong\n",
 		    length, offset);
 		length = DPT_MAX_TARGET_MODE_BUFFER_SIZE;
 		offset = 0;
 	}
 	ccb->xs = NULL;
 	ccb->flags = 0;
 	ccb->state = DPT_CCB_STATE_NEW;
 	ccb->std_callback = (ccb_callback) dpt_target_done;
 	ccb->wrbuff_callback = NULL;
 
 	cp = &ccb->eata_ccb;
 	cp->CP_OpCode = EATA_CMD_DMA_SEND_CP;
 	cp->SCSI_Reset = 0;
 	cp->HBA_Init = 0;
 	cp->Auto_Req_Sen = 1;
 	cp->cp_id = target;
 	cp->DataIn = 1;
 	cp->DataOut = 0;
 	cp->Interpret = 0;
 	cp->reqlen = htonl(sizeof(struct scsi_sense_data));
 	cp->cp_statDMA = htonl(vtophys(&cp->cp_statDMA));
 	cp->cp_reqDMA = htonl(vtophys(&cp->cp_reqDMA));
 	cp->cp_viraddr = (u_int32_t) & ccb;
 
 	cp->cp_msg[0] = HA_IDENTIFY_MSG | HA_DISCO_RECO;
 
 	cp->cp_scsi_cmd = command;
 	cp->cp_cdb[1] = (u_int8_t) (mode & SCSI_TM_MODE_MASK);
 	cp->cp_lun = lun;	/* Order is important here! */
 	cp->cp_cdb[2] = 0x00;	/* Buffer Id, only 1 :-( */
 	cp->cp_cdb[3] = (length >> 16) & 0xFF;	/* Buffer offset MSB */
 	cp->cp_cdb[4] = (length >> 8) & 0xFF;
 	cp->cp_cdb[5] = length & 0xFF;
 	cp->cp_cdb[6] = (length >> 16) & 0xFF;	/* Length MSB */
 	cp->cp_cdb[7] = (length >> 8) & 0xFF;
 	cp->cp_cdb[8] = length & 0xFF;	/* Length LSB */
 	cp->cp_cdb[9] = 0;	/* No sync, no match bits */
 
 	/*
 	 * This could be optimized to live in dpt_register_buffer.
 	 * We keep it here, just in case the kernel decides to reallocate pages
 	 */
 	if (dpt_scatter_gather(dpt, ccb, DPT_RW_BUFFER_SIZE,
 			       dpt->rw_buffer[bus][target][lun])) {
 		device_printf(dpt->dev, "Failed to setup Scatter/Gather for "
 		       "Target-Mode buffer\n");
 	}
 }
 
 /* Setup a target mode READ command */
 
 static void
 dpt_set_target(int redo, dpt_softc_t * dpt,
 	       u_int8_t bus, u_int8_t target, u_int8_t lun, int mode,
 	       u_int16_t length, u_int16_t offset, dpt_ccb_t * ccb)
 {
 
 	mtx_assert(&dpt->lock, MA_OWNED);
 	if (dpt->target_mode_enabled) {
 		if (!redo)
 			dpt_target_ccb(dpt, bus, target, lun, ccb, mode,
 				       SCSI_TM_READ_BUFFER, length, offset);
 
 		ccb->transaction_id = ++dpt->commands_processed;
 
 #ifdef DPT_MEASURE_PERFORMANCE
 		dpt->performance.command_count[ccb->eata_ccb.cp_scsi_cmd]++;
 		ccb->command_started = microtime_now;
 #endif
 		dpt_Qadd_waiting(dpt, ccb);
 		dpt_sched_queue(dpt);
 	} else {
 		device_printf(dpt->dev,
 		    "Target Mode Request, but Target Mode is OFF\n");
 	}
 }
 
 /*
  * Schedule a buffer to be sent to another target.
  * The work will be scheduled and the callback provided will be called when
  * the work is actually done.
  *
  * Please NOTE:  ``Anyone'' can send a buffer, but only registered clients
  * get notified of receipt of buffers.
  */
 
 int
 dpt_send_buffer(int unit, u_int8_t channel, u_int8_t target, u_int8_t lun,
 		u_int8_t mode, u_int16_t length, u_int16_t offset, void *data,
 		buff_wr_done callback)
 {
 	dpt_softc_t    *dpt;
 	dpt_ccb_t      *ccb = NULL;
 
 	/* This is an external call.  Be a bit paranoid */
 	dpt = devclass_get_device(dpt_devclass, unit);
 	if (dpt == NULL)
 		return (INVALID_UNIT);
 
 	mtx_lock(&dpt->lock);
 	if (dpt->target_mode_enabled) {
 		if ((channel >= dpt->channels) || (target > dpt->max_id) ||
 		    (lun > dpt->max_lun)) {
 			mtx_unlock(&dpt->lock);
 			return (INVALID_SENDER);
 		}
 		if ((dpt->rw_buffer[channel][target][lun] == NULL) ||
 		    (dpt->buffer_receiver[channel][target][lun] == NULL)) {
 			mtx_unlock(&dpt->lock);
 			return (NOT_REGISTERED);
 		}
 
 		/* Process the free list */
 		if ((TAILQ_EMPTY(&dpt->free_ccbs)) && dpt_alloc_freelist(dpt)) {
 			device_printf(dpt->dev,
 			    "ERROR: Cannot allocate any more free CCB's.\n"
 			    "             Please try later\n");
 			mtx_unlock(&dpt->lock);
 			return (NO_RESOURCES);
 		}
 		/* Now grab the newest CCB */
 		if ((ccb = dpt_Qpop_free(dpt)) == NULL) {
 			mtx_unlock(&dpt->lock);
 			panic("%s: Got a NULL CCB from pop_free()\n",
 			    device_get_nameunit(dpt->dev));
 		}
 
 		bcopy(dpt->rw_buffer[channel][target][lun] + offset, data, length);
 		dpt_target_ccb(dpt, channel, target, lun, ccb, mode, 
 					   SCSI_TM_WRITE_BUFFER,
 					   length, offset);
 		ccb->std_callback = (ccb_callback) callback; /* Potential trouble */
 
 		ccb->transaction_id = ++dpt->commands_processed;
 
 #ifdef DPT_MEASURE_PERFORMANCE
 		dpt->performance.command_count[ccb->eata_ccb.cp_scsi_cmd]++;
 		ccb->command_started = microtime_now;
 #endif
 		dpt_Qadd_waiting(dpt, ccb);
 		dpt_sched_queue(dpt);
 
 		mtx_unlock(&dpt->lock);
 		return (0);
 	}
 	mtx_unlock(&dpt->lock);
 	return (DRIVER_DOWN);
 }
 
 static void
 dpt_target_done(dpt_softc_t * dpt, int bus, dpt_ccb_t * ccb)
 {
 	eata_ccb_t     *cp;
 
 	cp = &ccb->eata_ccb;
 
 	/*
 	 * Remove the CCB from the waiting queue.
 	 *  We do NOT put it back on the free, etc., queues as it is a special
 	 * ccb, owned by the dpt_softc of this unit.
 	 */
 	dpt_Qremove_completed(dpt, ccb);
 
 #define br_channel           (ccb->eata_ccb.cp_channel)
 #define br_target            (ccb->eata_ccb.cp_id)
 #define br_lun               (ccb->eata_ccb.cp_LUN)
 #define br_index	     [br_channel][br_target][br_lun]
 #define read_buffer_callback (dpt->buffer_receiver br_index )
 #define	read_buffer	     (dpt->rw_buffer[br_channel][br_target][br_lun])
 #define cb(offset)           (ccb->eata_ccb.cp_cdb[offset])
 #define br_offset            ((cb(3) << 16) | (cb(4) << 8) | cb(5))
 #define br_length            ((cb(6) << 16) | (cb(7) << 8) | cb(8))
 
 	/* Different reasons for being here, you know... */
 	switch (ccb->eata_ccb.cp_scsi_cmd) {
 	case SCSI_TM_READ_BUFFER:
 		if (read_buffer_callback != NULL) {
 			/* This is a buffer generated by a kernel process */
 			read_buffer_callback(device_get_unit(dpt->dev),
 					     br_channel, br_target, br_lun,
 					     read_buffer,
 					     br_offset, br_length);
 		} else {
 			/*
 			 * This is a buffer waited for by a user (sleeping)
 			 * command
 			 */
 			wakeup(ccb);
 		}
 
 		/* We ALWAYS re-issue the same command; args are don't-care  */
 		dpt_set_target(1, 0, 0, 0, 0, 0, 0, 0, 0);
 		break;
 
 	case SCSI_TM_WRITE_BUFFER:
 		(ccb->wrbuff_callback) (device_get_unit(dpt->dev), br_channel,
 					br_target, br_offset, br_length,
 					br_lun, ccb->status_packet.hba_stat);
 		break;
 	default:
 		device_printf(dpt->dev,
 		    "%s is an unsupported command for target mode\n",
 		    scsi_cmd_name(ccb->eata_ccb.cp_scsi_cmd));
 	}
 	dpt->target_ccb[br_channel][br_target][br_lun] = NULL;
 	dpt_Qpush_free(dpt, ccb);
 }
 
 
 /*
  * Use this function to register a client for a buffer read target operation.
  * The function you register will be called every time a buffer is received
  * by the target mode code.
  */
 dpt_rb_t
 dpt_register_buffer(int unit, u_int8_t channel, u_int8_t target, u_int8_t lun,
 		    u_int8_t mode, u_int16_t length, u_int16_t offset,
 		    dpt_rec_buff callback, dpt_rb_op_t op)
 {
 	dpt_softc_t    *dpt;
 	dpt_ccb_t      *ccb = NULL;
 	int             ospl;
 
 	dpt = devclass_get_device(dpt_devclass, unit);
 	if (dpt == NULL)
 		return (INVALID_UNIT);
 	mtx_lock(&dpt->lock);
 
 	if (dpt->state & DPT_HA_SHUTDOWN_ACTIVE) {
 		mtx_unlock(&dpt->lock);
 		return (DRIVER_DOWN);
 	}
 
 	if ((channel > (dpt->channels - 1)) || (target > (dpt->max_id - 1)) ||
 	    (lun > (dpt->max_lun - 1))) {
 		mtx_unlock(&dpt->lock);
 		return (INVALID_SENDER);
 	}
 
 	if (dpt->buffer_receiver[channel][target][lun] == NULL) {
 		if (op == REGISTER_BUFFER) {
 			/* Assign the requested callback */
 			dpt->buffer_receiver[channel][target][lun] = callback;
 			/* Get a CCB */
 
 			/* Process the free list */
 			if ((TAILQ_EMPTY(&dpt->free_ccbs)) && dpt_alloc_freelist(dpt)) {
 				device_printf(dpt->dev,
 				    "ERROR: Cannot allocate any more free CCB's.\n"
 				    "             Please try later\n");
 				mtx_unlock(&dpt->lock);
 				return (NO_RESOURCES);
 			}
 			/* Now grab the newest CCB */
 			if ((ccb = dpt_Qpop_free(dpt)) == NULL) {
 				mtx_unlock(&dpt->lock);
 				panic("%s: Got a NULL CCB from pop_free()\n",
 				    device_get_nameunit(dpt->dev));
 			}
 
 			/* Clean up the leftover of the previous tenant */
 			ccb->status = DPT_CCB_STATE_NEW;
 			dpt->target_ccb[channel][target][lun] = ccb;
 
 			dpt->rw_buffer[channel][target][lun] =
 				malloc(DPT_RW_BUFFER_SIZE, M_DEVBUF, M_NOWAIT);
 			if (dpt->rw_buffer[channel][target][lun] == NULL) {
 				device_printf(dpt->dev, "Failed to allocate "
 				       "Target-Mode buffer\n");
 				dpt_Qpush_free(dpt, ccb);
 				mtx_unlock(&dpt->lock);
 				return (NO_RESOURCES);
 			}
 			dpt_set_target(0, dpt, channel, target, lun, mode,
 				       length, offset, ccb);
 			mtx_unlock(&dpt->lock);
 			return (SUCCESSFULLY_REGISTERED);
 		} else {
 			mtx_unlock(&dpt->lock);
 			return (NOT_REGISTERED);
 		}
 	} else {
 		if (op == REGISTER_BUFFER) {
 			if (dpt->buffer_receiver[channel][target][lun] == callback) {
 				mtx_unlock(&dpt->lock);
 				return (ALREADY_REGISTERED);
 			} else {
 				mtx_unlock(&dpt->lock);
 				return (REGISTERED_TO_ANOTHER);
 			}
 		} else {
 			if (dpt->buffer_receiver[channel][target][lun] == callback) {
 				dpt->buffer_receiver[channel][target][lun] = NULL;
 				dpt_Qpush_free(dpt, ccb);
 				free(dpt->rw_buffer[channel][target][lun], M_DEVBUF);
 				mtx_unlock(&dpt->lock);
 				return (SUCCESSFULLY_REGISTERED);
 			} else {
 				mtx_unlock(&dpt->lock);
 				return (INVALID_CALLBACK);
 			}
 		}
 
 	}
 	mtx_unlock(&dpt->lock);
 }
 
 /* Return the state of the blinking DPT LED's */
 u_int8_t
 dpt_blinking_led(dpt_softc_t * dpt)
 {
 	int             ndx;
 	u_int32_t       state;
 	u_int32_t       previous;
 	u_int8_t        result;
 
 	mtx_assert(&dpt->lock, MA_OWNED);
 	result = 0;
 
 	for (ndx = 0, state = 0, previous = 0;
 	     (ndx < 10) && (state != previous);
 	     ndx++) {
 		previous = state;
 		state = dpt_inl(dpt, 1);
 	}
 
 	if ((state == previous) && (state == DPT_BLINK_INDICATOR))
 		result = dpt_inb(dpt, 5);
 
 	return (result);
 }
 
 /*
  * Execute a command which did not come from the kernel's SCSI layer.
  * The only way to map user commands to bus and target is to comply with the
  * standard DPT wire-down scheme:
  */
 int
 dpt_user_cmd(dpt_softc_t * dpt, eata_pt_t * user_cmd,
 	     caddr_t cmdarg, int minor_no)
 {
 	dpt_ccb_t *ccb;
 	void	  *data;
 	int	   channel, target, lun;
 	int	   huh;
 	int	   result;
 	int	   submitted;
 
 	mtx_assert(&dpt->lock, MA_OWNED);
 	data = NULL;
 	channel = minor2hba(minor_no);
 	target = minor2target(minor_no);
 	lun = minor2lun(minor_no);
 
 	if ((channel > (dpt->channels - 1))
 	 || (target > dpt->max_id)
 	 || (lun > dpt->max_lun))
 		return (ENXIO);
 
 	if (target == dpt->sc_scsi_link[channel].adapter_targ) {
 		/* This one is for the controller itself */
 		if ((user_cmd->eataID[0] != 'E')
 		 || (user_cmd->eataID[1] != 'A')
 		 || (user_cmd->eataID[2] != 'T')
 		 || (user_cmd->eataID[3] != 'A')) {
 			return (ENXIO);
 		}
 	}
 	/* Get a DPT CCB, so we can prepare a command */
 
 	/* Process the free list */
 	if ((TAILQ_EMPTY(&dpt->free_ccbs)) && dpt_alloc_freelist(dpt)) {
 		device_printf(dpt->dev,
 		    "ERROR: Cannot allocate any more free CCB's.\n"
 		    "             Please try later\n");
 		return (EFAULT);
 	}
 	/* Now grab the newest CCB */
 	if ((ccb = dpt_Qpop_free(dpt)) == NULL) {
 		panic("%s: Got a NULL CCB from pop_free()\n",
 		    device_get_nameunit(dpt->dev));
 	} else {
 		/* Clean up the leftover of the previous tenant */
 		ccb->status = DPT_CCB_STATE_NEW;
 	}
 
 	bcopy((caddr_t) & user_cmd->command_packet, (caddr_t) & ccb->eata_ccb,
 	      sizeof(eata_ccb_t));
 
 	/* We do not want to do user specified scatter/gather.  Why?? */
 	if (ccb->eata_ccb.scatter == 1)
 		return (EINVAL);
 
 	ccb->eata_ccb.Auto_Req_Sen = 1;
 	ccb->eata_ccb.reqlen = htonl(sizeof(struct scsi_sense_data));
 	ccb->eata_ccb.cp_datalen = htonl(sizeof(ccb->eata_ccb.cp_datalen));
 	ccb->eata_ccb.cp_dataDMA = htonl(vtophys(ccb->eata_ccb.cp_dataDMA));
 	ccb->eata_ccb.cp_statDMA = htonl(vtophys(&ccb->eata_ccb.cp_statDMA));
 	ccb->eata_ccb.cp_reqDMA = htonl(vtophys(&ccb->eata_ccb.cp_reqDMA));
 	ccb->eata_ccb.cp_viraddr = (u_int32_t) & ccb;
 
 	if (ccb->eata_ccb.DataIn || ccb->eata_ccb.DataOut) {
 		/* Data I/O is involved in this command.  Alocate buffer */
 		if (ccb->eata_ccb.cp_datalen > PAGE_SIZE) {
 			data = contigmalloc(ccb->eata_ccb.cp_datalen,
 					    M_TEMP, M_WAITOK, 0, ~0,
 					    ccb->eata_ccb.cp_datalen,
 					    0x10000);
 		} else {
 			data = malloc(ccb->eata_ccb.cp_datalen, M_TEMP,
 				      M_WAITOK);
 		}
 
 		if (data == NULL) {
 			device_printf(dpt->dev, "Cannot allocate %d bytes "
 			       "for EATA command\n",
 			       ccb->eata_ccb.cp_datalen);
 			return (EFAULT);
 		}
 #define usr_cmd_DMA (caddr_t)user_cmd->command_packet.cp_dataDMA
 		if (ccb->eata_ccb.DataIn == 1) {
 			if (copyin(usr_cmd_DMA,
 				   data, ccb->eata_ccb.cp_datalen) == -1)
 				return (EFAULT);
 		}
 	} else {
 		/* No data I/O involved here.  Make sure the DPT knows that */
 		ccb->eata_ccb.cp_datalen = 0;
 		data = NULL;
 	}
 
 	if (ccb->eata_ccb.FWNEST == 1)
 		ccb->eata_ccb.FWNEST = 0;
 
 	if (ccb->eata_ccb.cp_datalen != 0) {
 		if (dpt_scatter_gather(dpt, ccb, ccb->eata_ccb.cp_datalen,
 				       data) != 0) {
 			if (data != NULL)
 				free(data, M_TEMP);
 			return (EFAULT);
 		}
 	}
 	/**
 	 * We are required to quiet a SCSI bus.
 	 * since we do not queue comands on a bus basis,
 	 * we wait for ALL commands on a controller to complete.
 	 * In the mean time, sched_queue() will not schedule new commands.
 	 */
 	if ((ccb->eata_ccb.cp_cdb[0] == MULTIFUNCTION_CMD)
 	    && (ccb->eata_ccb.cp_cdb[2] == BUS_QUIET)) {
 		/* We wait for ALL traffic for this HBa to subside */
 		dpt->state |= DPT_HA_QUIET;
 
 		while ((submitted = dpt->submitted_ccbs_count) != 0) {
 			huh = mtx_sleep((void *) dpt, &dpt->lock,
 			    PCATCH | PRIBIO, "dptqt", 100 * hz);
 			switch (huh) {
 			case 0:
 				/* Wakeup call received */
 				break;
 			case EWOULDBLOCK:
 				/* Timer Expired */
 				break;
 			default:
 				/* anything else */
 				break;
 			}
 		}
 	}
 	/* Resume normal operation */
 	if ((ccb->eata_ccb.cp_cdb[0] == MULTIFUNCTION_CMD)
 	    && (ccb->eata_ccb.cp_cdb[2] == BUS_UNQUIET)) {
 		dpt->state &= ~DPT_HA_QUIET;
 	}
 	/**
 	 * Schedule the command and submit it.
 	 * We bypass dpt_sched_queue, as it will block on DPT_HA_QUIET
 	 */
 	ccb->xs = NULL;
 	ccb->flags = 0;
 	ccb->eata_ccb.Auto_Req_Sen = 1;	/* We always want this feature */
 
 	ccb->transaction_id = ++dpt->commands_processed;
 	ccb->std_callback = (ccb_callback) dpt_user_cmd_done;
 	ccb->result = (u_int32_t) & cmdarg;
 	ccb->data = data;
 
 #ifdef DPT_MEASURE_PERFORMANCE
 	++dpt->performance.command_count[ccb->eata_ccb.cp_scsi_cmd];
 	ccb->command_started = microtime_now;
 #endif
 	dpt_Qadd_waiting(dpt, ccb);
 
 	dpt_sched_queue(dpt);
 
 	/* Wait for the command to complete */
 	(void) mtx_sleep((void *) ccb, &dpt->lock, PCATCH | PRIBIO, "dptucw",
 	    100 * hz);
 
 	/* Free allocated memory */
 	if (data != NULL)
 		free(data, M_TEMP);
 
 	return (0);
 }
 
 static void
 dpt_user_cmd_done(dpt_softc_t * dpt, int bus, dpt_ccb_t * ccb)
 {
 	u_int32_t       result;
 	caddr_t         cmd_arg;
 
 	mtx_unlock(&dpt->lock);
 
 	/**
 	 * If Auto Request Sense is on, copyout the sense struct
 	 */
 #define usr_pckt_DMA 	(caddr_t)(intptr_t)ntohl(ccb->eata_ccb.cp_reqDMA)
 #define usr_pckt_len	ntohl(ccb->eata_ccb.cp_datalen)
 	if (ccb->eata_ccb.Auto_Req_Sen == 1) {
 		if (copyout((caddr_t) & ccb->sense_data, usr_pckt_DMA,
 			    sizeof(struct scsi_sense_data))) {
 			mtx_lock(&dpt->lock);
 			ccb->result = EFAULT;
 			dpt_Qpush_free(dpt, ccb);
 			wakeup(ccb);
 			return;
 		}
 	}
 	/* If DataIn is on, copyout the data */
 	if ((ccb->eata_ccb.DataIn == 1)
 	    && (ccb->status_packet.hba_stat == HA_NO_ERROR)) {
 		if (copyout(ccb->data, usr_pckt_DMA, usr_pckt_len)) {
 			mtx_lock(&dpt->lock);
 			dpt_Qpush_free(dpt, ccb);
 			ccb->result = EFAULT;
 
 			wakeup(ccb);
 			return;
 		}
 	}
 	/* Copyout the status */
 	result = ccb->status_packet.hba_stat;
 	cmd_arg = (caddr_t) ccb->result;
 
 	if (copyout((caddr_t) & result, cmd_arg, sizeof(result))) {
 		mtx_lock(&dpt->lock);
 		dpt_Qpush_free(dpt, ccb);
 		ccb->result = EFAULT;
 		wakeup(ccb);
 		return;
 	}
 	mtx_lock(&dpt->lock);
 	/* Put the CCB back in the freelist */
 	ccb->state |= DPT_CCB_STATE_COMPLETED;
 	dpt_Qpush_free(dpt, ccb);
 
 	/* Free allocated memory */
 	return;
 }
 
 #endif
Index: projects/release-debugdist/sys/dev/hifn/hifn7751.c
===================================================================
--- projects/release-debugdist/sys/dev/hifn/hifn7751.c	(revision 267343)
+++ projects/release-debugdist/sys/dev/hifn/hifn7751.c	(revision 267344)
@@ -1,2934 +1,2931 @@
 /*	$OpenBSD: hifn7751.c,v 1.120 2002/05/17 00:33:34 deraadt Exp $	*/
 
 /*-
  * Invertex AEON / Hifn 7751 driver
  * Copyright (c) 1999 Invertex Inc. All rights reserved.
  * Copyright (c) 1999 Theo de Raadt
  * Copyright (c) 2000-2001 Network Security Technologies, Inc.
  *			http://www.netsec.net
  * Copyright (c) 2003 Hifn Inc.
  *
  * This driver is based on a previous driver by Invertex, for which they
  * requested:  Please send any comments, feedback, bug-fixes, or feature
  * requests to software@invertex.com.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  *
  * 1. Redistributions of source code must retain the above copyright
  *   notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *   notice, this list of conditions and the following disclaimer in the
  *   documentation and/or other materials provided with the distribution.
  * 3. The name of the author may not be used to endorse or promote products
  *   derived from this software without specific prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  * Effort sponsored in part by the Defense Advanced Research Projects
  * Agency (DARPA) and Air Force Research Laboratory, Air Force
  * Materiel Command, USAF, under agreement number F30602-01-2-0537.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 /*
  * Driver for various Hifn encryption processors.
  */
 #include "opt_hifn.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/proc.h>
 #include <sys/errno.h>
 #include <sys/malloc.h>
 #include <sys/kernel.h>
 #include <sys/module.h>
 #include <sys/mbuf.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <sys/sysctl.h>
 
 #include <vm/vm.h>
 #include <vm/pmap.h>
 
 #include <machine/bus.h>
 #include <machine/resource.h>
 #include <sys/bus.h>
 #include <sys/rman.h>
 
 #include <opencrypto/cryptodev.h>
 #include <sys/random.h>
 #include <sys/kobj.h>
 
 #include "cryptodev_if.h"
 
 #include <dev/pci/pcivar.h>
 #include <dev/pci/pcireg.h>
 
 #ifdef HIFN_RNDTEST
 #include <dev/rndtest/rndtest.h>
 #endif
 #include <dev/hifn/hifn7751reg.h>
 #include <dev/hifn/hifn7751var.h>
 
 #ifdef HIFN_VULCANDEV
 #include <sys/conf.h>
 #include <sys/uio.h>
 
 static struct cdevsw vulcanpk_cdevsw; /* forward declaration */
 #endif
 
 /*
  * Prototypes and count for the pci_device structure
  */
 static	int hifn_probe(device_t);
 static	int hifn_attach(device_t);
 static	int hifn_detach(device_t);
 static	int hifn_suspend(device_t);
 static	int hifn_resume(device_t);
 static	int hifn_shutdown(device_t);
 
 static	int hifn_newsession(device_t, u_int32_t *, struct cryptoini *);
 static	int hifn_freesession(device_t, u_int64_t);
 static	int hifn_process(device_t, struct cryptop *, int);
 
 static device_method_t hifn_methods[] = {
 	/* Device interface */
 	DEVMETHOD(device_probe,		hifn_probe),
 	DEVMETHOD(device_attach,	hifn_attach),
 	DEVMETHOD(device_detach,	hifn_detach),
 	DEVMETHOD(device_suspend,	hifn_suspend),
 	DEVMETHOD(device_resume,	hifn_resume),
 	DEVMETHOD(device_shutdown,	hifn_shutdown),
 
 	/* crypto device methods */
 	DEVMETHOD(cryptodev_newsession,	hifn_newsession),
 	DEVMETHOD(cryptodev_freesession,hifn_freesession),
 	DEVMETHOD(cryptodev_process,	hifn_process),
 
 	DEVMETHOD_END
 };
 static driver_t hifn_driver = {
 	"hifn",
 	hifn_methods,
 	sizeof (struct hifn_softc)
 };
 static devclass_t hifn_devclass;
 
 DRIVER_MODULE(hifn, pci, hifn_driver, hifn_devclass, 0, 0);
 MODULE_DEPEND(hifn, crypto, 1, 1, 1);
 #ifdef HIFN_RNDTEST
 MODULE_DEPEND(hifn, rndtest, 1, 1, 1);
 #endif
 
 static	void hifn_reset_board(struct hifn_softc *, int);
 static	void hifn_reset_puc(struct hifn_softc *);
 static	void hifn_puc_wait(struct hifn_softc *);
 static	int hifn_enable_crypto(struct hifn_softc *);
 static	void hifn_set_retry(struct hifn_softc *sc);
 static	void hifn_init_dma(struct hifn_softc *);
 static	void hifn_init_pci_registers(struct hifn_softc *);
 static	int hifn_sramsize(struct hifn_softc *);
 static	int hifn_dramsize(struct hifn_softc *);
 static	int hifn_ramtype(struct hifn_softc *);
 static	void hifn_sessions(struct hifn_softc *);
 static	void hifn_intr(void *);
 static	u_int hifn_write_command(struct hifn_command *, u_int8_t *);
 static	u_int32_t hifn_next_signature(u_int32_t a, u_int cnt);
 static	void hifn_callback(struct hifn_softc *, struct hifn_command *, u_int8_t *);
 static	int hifn_crypto(struct hifn_softc *, struct hifn_command *, struct cryptop *, int);
 static	int hifn_readramaddr(struct hifn_softc *, int, u_int8_t *);
 static	int hifn_writeramaddr(struct hifn_softc *, int, u_int8_t *);
 static	int hifn_dmamap_load_src(struct hifn_softc *, struct hifn_command *);
 static	int hifn_dmamap_load_dst(struct hifn_softc *, struct hifn_command *);
 static	int hifn_init_pubrng(struct hifn_softc *);
 static	void hifn_rng(void *);
 static	void hifn_tick(void *);
 static	void hifn_abort(struct hifn_softc *);
 static	void hifn_alloc_slot(struct hifn_softc *, int *, int *, int *, int *);
 
 static	void hifn_write_reg_0(struct hifn_softc *, bus_size_t, u_int32_t);
 static	void hifn_write_reg_1(struct hifn_softc *, bus_size_t, u_int32_t);
 
 static __inline u_int32_t
 READ_REG_0(struct hifn_softc *sc, bus_size_t reg)
 {
     u_int32_t v = bus_space_read_4(sc->sc_st0, sc->sc_sh0, reg);
     sc->sc_bar0_lastreg = (bus_size_t) -1;
     return (v);
 }
 #define	WRITE_REG_0(sc, reg, val)	hifn_write_reg_0(sc, reg, val)
 
 static __inline u_int32_t
 READ_REG_1(struct hifn_softc *sc, bus_size_t reg)
 {
     u_int32_t v = bus_space_read_4(sc->sc_st1, sc->sc_sh1, reg);
     sc->sc_bar1_lastreg = (bus_size_t) -1;
     return (v);
 }
 #define	WRITE_REG_1(sc, reg, val)	hifn_write_reg_1(sc, reg, val)
 
 static SYSCTL_NODE(_hw, OID_AUTO, hifn, CTLFLAG_RD, 0,
 	    "Hifn driver parameters");
 
 #ifdef HIFN_DEBUG
 static	int hifn_debug = 0;
 SYSCTL_INT(_hw_hifn, OID_AUTO, debug, CTLFLAG_RW, &hifn_debug,
 	    0, "control debugging msgs");
 #endif
 
 static	struct hifn_stats hifnstats;
 SYSCTL_STRUCT(_hw_hifn, OID_AUTO, stats, CTLFLAG_RD, &hifnstats,
 	    hifn_stats, "driver statistics");
 static	int hifn_maxbatch = 1;
 SYSCTL_INT(_hw_hifn, OID_AUTO, maxbatch, CTLFLAG_RW, &hifn_maxbatch,
 	    0, "max ops to batch w/o interrupt");
 
 /*
  * Probe for a supported device.  The PCI vendor and device
  * IDs are used to detect devices we know how to handle.
  */
 static int
 hifn_probe(device_t dev)
 {
 	if (pci_get_vendor(dev) == PCI_VENDOR_INVERTEX &&
 	    pci_get_device(dev) == PCI_PRODUCT_INVERTEX_AEON)
 		return (BUS_PROBE_DEFAULT);
 	if (pci_get_vendor(dev) == PCI_VENDOR_HIFN &&
 	    (pci_get_device(dev) == PCI_PRODUCT_HIFN_7751 ||
 	     pci_get_device(dev) == PCI_PRODUCT_HIFN_7951 ||
 	     pci_get_device(dev) == PCI_PRODUCT_HIFN_7955 ||
 	     pci_get_device(dev) == PCI_PRODUCT_HIFN_7956 ||
 	     pci_get_device(dev) == PCI_PRODUCT_HIFN_7811))
 		return (BUS_PROBE_DEFAULT);
 	if (pci_get_vendor(dev) == PCI_VENDOR_NETSEC &&
 	    pci_get_device(dev) == PCI_PRODUCT_NETSEC_7751)
 		return (BUS_PROBE_DEFAULT);
 	return (ENXIO);
 }
 
 static void
 hifn_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 {
 	bus_addr_t *paddr = (bus_addr_t*) arg;
 	*paddr = segs->ds_addr;
 }
 
 static const char*
 hifn_partname(struct hifn_softc *sc)
 {
 	/* XXX sprintf numbers when not decoded */
 	switch (pci_get_vendor(sc->sc_dev)) {
 	case PCI_VENDOR_HIFN:
 		switch (pci_get_device(sc->sc_dev)) {
 		case PCI_PRODUCT_HIFN_6500:	return "Hifn 6500";
 		case PCI_PRODUCT_HIFN_7751:	return "Hifn 7751";
 		case PCI_PRODUCT_HIFN_7811:	return "Hifn 7811";
 		case PCI_PRODUCT_HIFN_7951:	return "Hifn 7951";
 		case PCI_PRODUCT_HIFN_7955:	return "Hifn 7955";
 		case PCI_PRODUCT_HIFN_7956:	return "Hifn 7956";
 		}
 		return "Hifn unknown-part";
 	case PCI_VENDOR_INVERTEX:
 		switch (pci_get_device(sc->sc_dev)) {
 		case PCI_PRODUCT_INVERTEX_AEON:	return "Invertex AEON";
 		}
 		return "Invertex unknown-part";
 	case PCI_VENDOR_NETSEC:
 		switch (pci_get_device(sc->sc_dev)) {
 		case PCI_PRODUCT_NETSEC_7751:	return "NetSec 7751";
 		}
 		return "NetSec unknown-part";
 	}
 	return "Unknown-vendor unknown-part";
 }
 
 static void
 default_harvest(struct rndtest_state *rsp, void *buf, u_int count)
 {
 	random_harvest(buf, count, count*NBBY/2, RANDOM_PURE_HIFN);
 }
 
 static u_int
 checkmaxmin(device_t dev, const char *what, u_int v, u_int min, u_int max)
 {
 	if (v > max) {
 		device_printf(dev, "Warning, %s %u out of range, "
 			"using max %u\n", what, v, max);
 		v = max;
 	} else if (v < min) {
 		device_printf(dev, "Warning, %s %u out of range, "
 			"using min %u\n", what, v, min);
 		v = min;
 	}
 	return v;
 }
 
 /*
  * Select PLL configuration for 795x parts.  This is complicated in
  * that we cannot determine the optimal parameters without user input.
  * The reference clock is derived from an external clock through a
  * multiplier.  The external clock is either the host bus (i.e. PCI)
  * or an external clock generator.  When using the PCI bus we assume
  * the clock is either 33 or 66 MHz; for an external source we cannot
  * tell the speed.
  *
  * PLL configuration is done with a string: "pci" for PCI bus, or "ext"
  * for an external source, followed by the frequency.  We calculate
  * the appropriate multiplier and PLL register contents accordingly.
  * When no configuration is given we default to "pci66" since that
  * always will allow the card to work.  If a card is using the PCI
  * bus clock and in a 33MHz slot then it will be operating at half
  * speed until the correct information is provided.
  *
  * We use a default setting of "ext66" because according to Mike Ham
  * of HiFn, almost every board in existence has an external crystal
  * populated at 66Mhz. Using PCI can be a problem on modern motherboards,
  * because PCI33 can have clocks from 0 to 33Mhz, and some have
  * non-PCI-compliant spread-spectrum clocks, which can confuse the pll.
  */
 static void
 hifn_getpllconfig(device_t dev, u_int *pll)
 {
 	const char *pllspec;
 	u_int freq, mul, fl, fh;
 	u_int32_t pllconfig;
 	char *nxt;
 
 	if (resource_string_value("hifn", device_get_unit(dev),
 	    "pllconfig", &pllspec))
 		pllspec = "ext66";
 	fl = 33, fh = 66;
 	pllconfig = 0;
 	if (strncmp(pllspec, "ext", 3) == 0) {
 		pllspec += 3;
 		pllconfig |= HIFN_PLL_REF_SEL;
 		switch (pci_get_device(dev)) {
 		case PCI_PRODUCT_HIFN_7955:
 		case PCI_PRODUCT_HIFN_7956:
 			fl = 20, fh = 100;
 			break;
 #ifdef notyet
 		case PCI_PRODUCT_HIFN_7954:
 			fl = 20, fh = 66;
 			break;
 #endif
 		}
 	} else if (strncmp(pllspec, "pci", 3) == 0)
 		pllspec += 3;
 	freq = strtoul(pllspec, &nxt, 10);
 	if (nxt == pllspec)
 		freq = 66;
 	else
 		freq = checkmaxmin(dev, "frequency", freq, fl, fh);
 	/*
 	 * Calculate multiplier.  We target a Fck of 266 MHz,
 	 * allowing only even values, possibly rounded down.
 	 * Multipliers > 8 must set the charge pump current.
 	 */
 	mul = checkmaxmin(dev, "PLL divisor", (266 / freq) &~ 1, 2, 12);
 	pllconfig |= (mul / 2 - 1) << HIFN_PLL_ND_SHIFT;
 	if (mul > 8)
 		pllconfig |= HIFN_PLL_IS;
 	*pll = pllconfig;
 }
 
 /*
  * Attach an interface that successfully probed.
  */
 static int 
 hifn_attach(device_t dev)
 {
 	struct hifn_softc *sc = device_get_softc(dev);
 	caddr_t kva;
 	int rseg, rid;
 	char rbase;
 	u_int16_t ena, rev;
 
 	sc->sc_dev = dev;
 
 	mtx_init(&sc->sc_mtx, device_get_nameunit(dev), "hifn driver", MTX_DEF);
 
 	/* XXX handle power management */
 
 	/*
 	 * The 7951 and 795x have a random number generator and
 	 * public key support; note this.
 	 */
 	if (pci_get_vendor(dev) == PCI_VENDOR_HIFN &&
 	    (pci_get_device(dev) == PCI_PRODUCT_HIFN_7951 ||
 	     pci_get_device(dev) == PCI_PRODUCT_HIFN_7955 ||
 	     pci_get_device(dev) == PCI_PRODUCT_HIFN_7956))
 		sc->sc_flags = HIFN_HAS_RNG | HIFN_HAS_PUBLIC;
 	/*
 	 * The 7811 has a random number generator and
 	 * we also note it's identity 'cuz of some quirks.
 	 */
 	if (pci_get_vendor(dev) == PCI_VENDOR_HIFN &&
 	    pci_get_device(dev) == PCI_PRODUCT_HIFN_7811)
 		sc->sc_flags |= HIFN_IS_7811 | HIFN_HAS_RNG;
 
 	/*
 	 * The 795x parts support AES.
 	 */
 	if (pci_get_vendor(dev) == PCI_VENDOR_HIFN &&
 	    (pci_get_device(dev) == PCI_PRODUCT_HIFN_7955 ||
 	     pci_get_device(dev) == PCI_PRODUCT_HIFN_7956)) {
 		sc->sc_flags |= HIFN_IS_7956 | HIFN_HAS_AES;
 		/*
 		 * Select PLL configuration.  This depends on the
 		 * bus and board design and must be manually configured
 		 * if the default setting is unacceptable.
 		 */
 		hifn_getpllconfig(dev, &sc->sc_pllconfig);
 	}
 
 	/*
 	 * Setup PCI resources. Note that we record the bus
 	 * tag and handle for each register mapping, this is
 	 * used by the READ_REG_0, WRITE_REG_0, READ_REG_1,
 	 * and WRITE_REG_1 macros throughout the driver.
 	 */
 	pci_enable_busmaster(dev);
 
 	rid = HIFN_BAR0;
 	sc->sc_bar0res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 			 			RF_ACTIVE);
 	if (sc->sc_bar0res == NULL) {
 		device_printf(dev, "cannot map bar%d register space\n", 0);
 		goto fail_pci;
 	}
 	sc->sc_st0 = rman_get_bustag(sc->sc_bar0res);
 	sc->sc_sh0 = rman_get_bushandle(sc->sc_bar0res);
 	sc->sc_bar0_lastreg = (bus_size_t) -1;
 
 	rid = HIFN_BAR1;
 	sc->sc_bar1res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 						RF_ACTIVE);
 	if (sc->sc_bar1res == NULL) {
 		device_printf(dev, "cannot map bar%d register space\n", 1);
 		goto fail_io0;
 	}
 	sc->sc_st1 = rman_get_bustag(sc->sc_bar1res);
 	sc->sc_sh1 = rman_get_bushandle(sc->sc_bar1res);
 	sc->sc_bar1_lastreg = (bus_size_t) -1;
 
 	hifn_set_retry(sc);
 
 	/*
 	 * Setup the area where the Hifn DMA's descriptors
 	 * and associated data structures.
 	 */
 	if (bus_dma_tag_create(bus_get_dma_tag(dev),	/* PCI parent */
 			       1, 0,			/* alignment,boundary */
 			       BUS_SPACE_MAXADDR_32BIT,	/* lowaddr */
 			       BUS_SPACE_MAXADDR,	/* highaddr */
 			       NULL, NULL,		/* filter, filterarg */
 			       HIFN_MAX_DMALEN,		/* maxsize */
 			       MAX_SCATTER,		/* nsegments */
 			       HIFN_MAX_SEGLEN,		/* maxsegsize */
 			       BUS_DMA_ALLOCNOW,	/* flags */
 			       NULL,			/* lockfunc */
 			       NULL,			/* lockarg */
 			       &sc->sc_dmat)) {
 		device_printf(dev, "cannot allocate DMA tag\n");
 		goto fail_io1;
 	}
 	if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &sc->sc_dmamap)) {
 		device_printf(dev, "cannot create dma map\n");
 		bus_dma_tag_destroy(sc->sc_dmat);
 		goto fail_io1;
 	}
 	if (bus_dmamem_alloc(sc->sc_dmat, (void**) &kva, BUS_DMA_NOWAIT, &sc->sc_dmamap)) {
 		device_printf(dev, "cannot alloc dma buffer\n");
 		bus_dmamap_destroy(sc->sc_dmat, sc->sc_dmamap);
 		bus_dma_tag_destroy(sc->sc_dmat);
 		goto fail_io1;
 	}
 	if (bus_dmamap_load(sc->sc_dmat, sc->sc_dmamap, kva,
 			     sizeof (*sc->sc_dma),
 			     hifn_dmamap_cb, &sc->sc_dma_physaddr,
 			     BUS_DMA_NOWAIT)) {
 		device_printf(dev, "cannot load dma map\n");
 		bus_dmamem_free(sc->sc_dmat, kva, sc->sc_dmamap);
-		bus_dmamap_destroy(sc->sc_dmat, sc->sc_dmamap);
 		bus_dma_tag_destroy(sc->sc_dmat);
 		goto fail_io1;
 	}
 	sc->sc_dma = (struct hifn_dma *)kva;
 	bzero(sc->sc_dma, sizeof(*sc->sc_dma));
 
 	KASSERT(sc->sc_st0 != 0, ("hifn_attach: null bar0 tag!"));
 	KASSERT(sc->sc_sh0 != 0, ("hifn_attach: null bar0 handle!"));
 	KASSERT(sc->sc_st1 != 0, ("hifn_attach: null bar1 tag!"));
 	KASSERT(sc->sc_sh1 != 0, ("hifn_attach: null bar1 handle!"));
 
 	/*
 	 * Reset the board and do the ``secret handshake''
 	 * to enable the crypto support.  Then complete the
 	 * initialization procedure by setting up the interrupt
 	 * and hooking in to the system crypto support so we'll
 	 * get used for system services like the crypto device,
 	 * IPsec, RNG device, etc.
 	 */
 	hifn_reset_board(sc, 0);
 
 	if (hifn_enable_crypto(sc) != 0) {
 		device_printf(dev, "crypto enabling failed\n");
 		goto fail_mem;
 	}
 	hifn_reset_puc(sc);
 
 	hifn_init_dma(sc);
 	hifn_init_pci_registers(sc);
 
 	/* XXX can't dynamically determine ram type for 795x; force dram */
 	if (sc->sc_flags & HIFN_IS_7956)
 		sc->sc_drammodel = 1;
 	else if (hifn_ramtype(sc))
 		goto fail_mem;
 
 	if (sc->sc_drammodel == 0)
 		hifn_sramsize(sc);
 	else
 		hifn_dramsize(sc);
 
 	/*
 	 * Workaround for NetSec 7751 rev A: half ram size because two
 	 * of the address lines were left floating
 	 */
 	if (pci_get_vendor(dev) == PCI_VENDOR_NETSEC &&
 	    pci_get_device(dev) == PCI_PRODUCT_NETSEC_7751 &&
 	    pci_get_revid(dev) == 0x61)	/*XXX???*/
 		sc->sc_ramsize >>= 1;
 
 	/*
 	 * Arrange the interrupt line.
 	 */
 	rid = 0;
 	sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
 					    RF_SHAREABLE|RF_ACTIVE);
 	if (sc->sc_irq == NULL) {
 		device_printf(dev, "could not map interrupt\n");
 		goto fail_mem;
 	}
 	/*
 	 * NB: Network code assumes we are blocked with splimp()
 	 *     so make sure the IRQ is marked appropriately.
 	 */
 	if (bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET | INTR_MPSAFE,
 			   NULL, hifn_intr, sc, &sc->sc_intrhand)) {
 		device_printf(dev, "could not setup interrupt\n");
 		goto fail_intr2;
 	}
 
 	hifn_sessions(sc);
 
 	/*
 	 * NB: Keep only the low 16 bits; this masks the chip id
 	 *     from the 7951.
 	 */
 	rev = READ_REG_1(sc, HIFN_1_REVID) & 0xffff;
 
 	rseg = sc->sc_ramsize / 1024;
 	rbase = 'K';
 	if (sc->sc_ramsize >= (1024 * 1024)) {
 		rbase = 'M';
 		rseg /= 1024;
 	}
 	device_printf(sc->sc_dev, "%s, rev %u, %d%cB %cram",
 		hifn_partname(sc), rev,
 		rseg, rbase, sc->sc_drammodel ? 'd' : 's');
 	if (sc->sc_flags & HIFN_IS_7956)
 		printf(", pll=0x%x<%s clk, %ux mult>",
 			sc->sc_pllconfig,
 			sc->sc_pllconfig & HIFN_PLL_REF_SEL ? "ext" : "pci",
 			2 + 2*((sc->sc_pllconfig & HIFN_PLL_ND) >> 11));
 	printf("\n");
 
 	sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE);
 	if (sc->sc_cid < 0) {
 		device_printf(dev, "could not get crypto driver id\n");
 		goto fail_intr;
 	}
 
 	WRITE_REG_0(sc, HIFN_0_PUCNFG,
 	    READ_REG_0(sc, HIFN_0_PUCNFG) | HIFN_PUCNFG_CHIPID);
 	ena = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA;
 
 	switch (ena) {
 	case HIFN_PUSTAT_ENA_2:
 		crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
 		crypto_register(sc->sc_cid, CRYPTO_ARC4, 0, 0);
 		if (sc->sc_flags & HIFN_HAS_AES)
 			crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
 		/*FALLTHROUGH*/
 	case HIFN_PUSTAT_ENA_1:
 		crypto_register(sc->sc_cid, CRYPTO_MD5, 0, 0);
 		crypto_register(sc->sc_cid, CRYPTO_SHA1, 0, 0);
 		crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
 		crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
 		crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
 		break;
 	}
 
 	bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 
 	if (sc->sc_flags & (HIFN_HAS_PUBLIC | HIFN_HAS_RNG))
 		hifn_init_pubrng(sc);
 
 	callout_init(&sc->sc_tickto, CALLOUT_MPSAFE);
 	callout_reset(&sc->sc_tickto, hz, hifn_tick, sc);
 
 	return (0);
 
 fail_intr:
 	bus_teardown_intr(dev, sc->sc_irq, sc->sc_intrhand);
 fail_intr2:
 	/* XXX don't store rid */
 	bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
 fail_mem:
 	bus_dmamap_unload(sc->sc_dmat, sc->sc_dmamap);
 	bus_dmamem_free(sc->sc_dmat, sc->sc_dma, sc->sc_dmamap);
-	bus_dmamap_destroy(sc->sc_dmat, sc->sc_dmamap);
 	bus_dma_tag_destroy(sc->sc_dmat);
 
 	/* Turn off DMA polling */
 	WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
 	    HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
 fail_io1:
 	bus_release_resource(dev, SYS_RES_MEMORY, HIFN_BAR1, sc->sc_bar1res);
 fail_io0:
 	bus_release_resource(dev, SYS_RES_MEMORY, HIFN_BAR0, sc->sc_bar0res);
 fail_pci:
 	mtx_destroy(&sc->sc_mtx);
 	return (ENXIO);
 }
 
 /*
  * Detach an interface that successfully probed.
  */
 static int 
 hifn_detach(device_t dev)
 {
 	struct hifn_softc *sc = device_get_softc(dev);
 
 	KASSERT(sc != NULL, ("hifn_detach: null software carrier!"));
 
 	/* disable interrupts */
 	WRITE_REG_1(sc, HIFN_1_DMA_IER, 0);
 
 	/*XXX other resources */
 	callout_stop(&sc->sc_tickto);
 	callout_stop(&sc->sc_rngto);
 #ifdef HIFN_RNDTEST
 	if (sc->sc_rndtest)
 		rndtest_detach(sc->sc_rndtest);
 #endif
 
 	/* Turn off DMA polling */
 	WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
 	    HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
 
 	crypto_unregister_all(sc->sc_cid);
 
 	bus_generic_detach(dev);	/*XXX should be no children, right? */
 
 	bus_teardown_intr(dev, sc->sc_irq, sc->sc_intrhand);
 	/* XXX don't store rid */
 	bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
 
 	bus_dmamap_unload(sc->sc_dmat, sc->sc_dmamap);
 	bus_dmamem_free(sc->sc_dmat, sc->sc_dma, sc->sc_dmamap);
-	bus_dmamap_destroy(sc->sc_dmat, sc->sc_dmamap);
 	bus_dma_tag_destroy(sc->sc_dmat);
 
 	bus_release_resource(dev, SYS_RES_MEMORY, HIFN_BAR1, sc->sc_bar1res);
 	bus_release_resource(dev, SYS_RES_MEMORY, HIFN_BAR0, sc->sc_bar0res);
 
 	mtx_destroy(&sc->sc_mtx);
 
 	return (0);
 }
 
 /*
  * Stop all chip I/O so that the kernel's probe routines don't
  * get confused by errant DMAs when rebooting.
  */
 static int
 hifn_shutdown(device_t dev)
 {
 #ifdef notyet
 	hifn_stop(device_get_softc(dev));
 #endif
 	return (0);
 }
 
 /*
  * Device suspend routine.  Stop the interface and save some PCI
  * settings in case the BIOS doesn't restore them properly on
  * resume.
  */
 static int
 hifn_suspend(device_t dev)
 {
 	struct hifn_softc *sc = device_get_softc(dev);
 #ifdef notyet
 	hifn_stop(sc);
 #endif
 	sc->sc_suspended = 1;
 
 	return (0);
 }
 
 /*
  * Device resume routine.  Restore some PCI settings in case the BIOS
  * doesn't, re-enable busmastering, and restart the interface if
  * appropriate.
  */
 static int
 hifn_resume(device_t dev)
 {
 	struct hifn_softc *sc = device_get_softc(dev);
 #ifdef notyet
         /* reinitialize interface if necessary */
         if (ifp->if_flags & IFF_UP)
                 rl_init(sc);
 #endif
 	sc->sc_suspended = 0;
 
 	return (0);
 }
 
 static int
 hifn_init_pubrng(struct hifn_softc *sc)
 {
 	u_int32_t r;
 	int i;
 
 #ifdef HIFN_RNDTEST
 	sc->sc_rndtest = rndtest_attach(sc->sc_dev);
 	if (sc->sc_rndtest)
 		sc->sc_harvest = rndtest_harvest;
 	else
 		sc->sc_harvest = default_harvest;
 #else
 	sc->sc_harvest = default_harvest;
 #endif
 	if ((sc->sc_flags & HIFN_IS_7811) == 0) {
 		/* Reset 7951 public key/rng engine */
 		WRITE_REG_1(sc, HIFN_1_PUB_RESET,
 		    READ_REG_1(sc, HIFN_1_PUB_RESET) | HIFN_PUBRST_RESET);
 
 		for (i = 0; i < 100; i++) {
 			DELAY(1000);
 			if ((READ_REG_1(sc, HIFN_1_PUB_RESET) &
 			    HIFN_PUBRST_RESET) == 0)
 				break;
 		}
 
 		if (i == 100) {
 			device_printf(sc->sc_dev, "public key init failed\n");
 			return (1);
 		}
 	}
 
 	/* Enable the rng, if available */
 	if (sc->sc_flags & HIFN_HAS_RNG) {
 		if (sc->sc_flags & HIFN_IS_7811) {
 			r = READ_REG_1(sc, HIFN_1_7811_RNGENA);
 			if (r & HIFN_7811_RNGENA_ENA) {
 				r &= ~HIFN_7811_RNGENA_ENA;
 				WRITE_REG_1(sc, HIFN_1_7811_RNGENA, r);
 			}
 			WRITE_REG_1(sc, HIFN_1_7811_RNGCFG,
 			    HIFN_7811_RNGCFG_DEFL);
 			r |= HIFN_7811_RNGENA_ENA;
 			WRITE_REG_1(sc, HIFN_1_7811_RNGENA, r);
 		} else
 			WRITE_REG_1(sc, HIFN_1_RNG_CONFIG,
 			    READ_REG_1(sc, HIFN_1_RNG_CONFIG) |
 			    HIFN_RNGCFG_ENA);
 
 		sc->sc_rngfirst = 1;
 		if (hz >= 100)
 			sc->sc_rnghz = hz / 100;
 		else
 			sc->sc_rnghz = 1;
 		callout_init(&sc->sc_rngto, CALLOUT_MPSAFE);
 		callout_reset(&sc->sc_rngto, sc->sc_rnghz, hifn_rng, sc);
 	}
 
 	/* Enable public key engine, if available */
 	if (sc->sc_flags & HIFN_HAS_PUBLIC) {
 		WRITE_REG_1(sc, HIFN_1_PUB_IEN, HIFN_PUBIEN_DONE);
 		sc->sc_dmaier |= HIFN_DMAIER_PUBDONE;
 		WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
 #ifdef HIFN_VULCANDEV
 		sc->sc_pkdev = make_dev(&vulcanpk_cdevsw, 0, 
 					UID_ROOT, GID_WHEEL, 0666,
 					"vulcanpk");
 		sc->sc_pkdev->si_drv1 = sc;
 #endif
 	}
 
 	return (0);
 }
 
 static void
 hifn_rng(void *vsc)
 {
 #define	RANDOM_BITS(n)	(n)*sizeof (u_int32_t), (n)*sizeof (u_int32_t)*NBBY, 0
 	struct hifn_softc *sc = vsc;
 	u_int32_t sts, num[2];
 	int i;
 
 	if (sc->sc_flags & HIFN_IS_7811) {
 		/* ONLY VALID ON 7811!!!! */
 		for (i = 0; i < 5; i++) {
 			sts = READ_REG_1(sc, HIFN_1_7811_RNGSTS);
 			if (sts & HIFN_7811_RNGSTS_UFL) {
 				device_printf(sc->sc_dev,
 					      "RNG underflow: disabling\n");
 				return;
 			}
 			if ((sts & HIFN_7811_RNGSTS_RDY) == 0)
 				break;
 
 			/*
 			 * There are at least two words in the RNG FIFO
 			 * at this point.
 			 */
 			num[0] = READ_REG_1(sc, HIFN_1_7811_RNGDAT);
 			num[1] = READ_REG_1(sc, HIFN_1_7811_RNGDAT);
 			/* NB: discard first data read */
 			if (sc->sc_rngfirst)
 				sc->sc_rngfirst = 0;
 			else
 				(*sc->sc_harvest)(sc->sc_rndtest,
 					num, sizeof (num));
 		}
 	} else {
 		num[0] = READ_REG_1(sc, HIFN_1_RNG_DATA);
 
 		/* NB: discard first data read */
 		if (sc->sc_rngfirst)
 			sc->sc_rngfirst = 0;
 		else
 			(*sc->sc_harvest)(sc->sc_rndtest,
 				num, sizeof (num[0]));
 	}
 
 	callout_reset(&sc->sc_rngto, sc->sc_rnghz, hifn_rng, sc);
 #undef RANDOM_BITS
 }
 
 static void
 hifn_puc_wait(struct hifn_softc *sc)
 {
 	int i;
 	int reg = HIFN_0_PUCTRL;
 
 	if (sc->sc_flags & HIFN_IS_7956) {
 		reg = HIFN_0_PUCTRL2;
 	}
 
 	for (i = 5000; i > 0; i--) {
 		DELAY(1);
 		if (!(READ_REG_0(sc, reg) & HIFN_PUCTRL_RESET))
 			break;
 	}
 	if (!i)
 		device_printf(sc->sc_dev, "proc unit did not reset\n");
 }
 
 /*
  * Reset the processing unit.
  */
 static void
 hifn_reset_puc(struct hifn_softc *sc)
 {
 	/* Reset processing unit */
 	int reg = HIFN_0_PUCTRL;
 
 	if (sc->sc_flags & HIFN_IS_7956) {
 		reg = HIFN_0_PUCTRL2;
 	}
 	WRITE_REG_0(sc, reg, HIFN_PUCTRL_DMAENA);
 
 	hifn_puc_wait(sc);
 }
 
 /*
  * Set the Retry and TRDY registers; note that we set them to
  * zero because the 7811 locks up when forced to retry (section
  * 3.6 of "Specification Update SU-0014-04".  Not clear if we
  * should do this for all Hifn parts, but it doesn't seem to hurt.
  */
 static void
 hifn_set_retry(struct hifn_softc *sc)
 {
 	/* NB: RETRY only responds to 8-bit reads/writes */
 	pci_write_config(sc->sc_dev, HIFN_RETRY_TIMEOUT, 0, 1);
 	pci_write_config(sc->sc_dev, HIFN_TRDY_TIMEOUT, 0, 1);
 }
 
 /*
  * Resets the board.  Values in the regesters are left as is
  * from the reset (i.e. initial values are assigned elsewhere).
  */
 static void
 hifn_reset_board(struct hifn_softc *sc, int full)
 {
 	u_int32_t reg;
 
 	/*
 	 * Set polling in the DMA configuration register to zero.  0x7 avoids
 	 * resetting the board and zeros out the other fields.
 	 */
 	WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
 	    HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
 
 	/*
 	 * Now that polling has been disabled, we have to wait 1 ms
 	 * before resetting the board.
 	 */
 	DELAY(1000);
 
 	/* Reset the DMA unit */
 	if (full) {
 		WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE);
 		DELAY(1000);
 	} else {
 		WRITE_REG_1(sc, HIFN_1_DMA_CNFG,
 		    HIFN_DMACNFG_MODE | HIFN_DMACNFG_MSTRESET);
 		hifn_reset_puc(sc);
 	}
 
 	KASSERT(sc->sc_dma != NULL, ("hifn_reset_board: null DMA tag!"));
 	bzero(sc->sc_dma, sizeof(*sc->sc_dma));
 
 	/* Bring dma unit out of reset */
 	WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
 	    HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
 
 	hifn_puc_wait(sc);
 	hifn_set_retry(sc);
 
 	if (sc->sc_flags & HIFN_IS_7811) {
 		for (reg = 0; reg < 1000; reg++) {
 			if (READ_REG_1(sc, HIFN_1_7811_MIPSRST) &
 			    HIFN_MIPSRST_CRAMINIT)
 				break;
 			DELAY(1000);
 		}
 		if (reg == 1000)
 			printf(": cram init timeout\n");
 	} else {
 	  /* set up DMA configuration register #2 */
 	  /* turn off all PK and BAR0 swaps */
 	  WRITE_REG_1(sc, HIFN_1_DMA_CNFG2,
 		      (3 << HIFN_DMACNFG2_INIT_WRITE_BURST_SHIFT)|
 		      (3 << HIFN_DMACNFG2_INIT_READ_BURST_SHIFT)|
 		      (2 << HIFN_DMACNFG2_TGT_WRITE_BURST_SHIFT)|
 		      (2 << HIFN_DMACNFG2_TGT_READ_BURST_SHIFT));
 	}
 		      
 }
 
 static u_int32_t
 hifn_next_signature(u_int32_t a, u_int cnt)
 {
 	int i;
 	u_int32_t v;
 
 	for (i = 0; i < cnt; i++) {
 
 		/* get the parity */
 		v = a & 0x80080125;
 		v ^= v >> 16;
 		v ^= v >> 8;
 		v ^= v >> 4;
 		v ^= v >> 2;
 		v ^= v >> 1;
 
 		a = (v & 1) ^ (a << 1);
 	}
 
 	return a;
 }
 
 struct pci2id {
 	u_short		pci_vendor;
 	u_short		pci_prod;
 	char		card_id[13];
 };
 static struct pci2id pci2id[] = {
 	{
 		PCI_VENDOR_HIFN,
 		PCI_PRODUCT_HIFN_7951,
 		{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 		  0x00, 0x00, 0x00, 0x00, 0x00 }
 	}, {
 		PCI_VENDOR_HIFN,
 		PCI_PRODUCT_HIFN_7955,
 		{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 		  0x00, 0x00, 0x00, 0x00, 0x00 }
 	}, {
 		PCI_VENDOR_HIFN,
 		PCI_PRODUCT_HIFN_7956,
 		{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 		  0x00, 0x00, 0x00, 0x00, 0x00 }
 	}, {
 		PCI_VENDOR_NETSEC,
 		PCI_PRODUCT_NETSEC_7751,
 		{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 		  0x00, 0x00, 0x00, 0x00, 0x00 }
 	}, {
 		PCI_VENDOR_INVERTEX,
 		PCI_PRODUCT_INVERTEX_AEON,
 		{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 		  0x00, 0x00, 0x00, 0x00, 0x00 }
 	}, {
 		PCI_VENDOR_HIFN,
 		PCI_PRODUCT_HIFN_7811,
 		{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 		  0x00, 0x00, 0x00, 0x00, 0x00 }
 	}, {
 		/*
 		 * Other vendors share this PCI ID as well, such as
 		 * http://www.powercrypt.com, and obviously they also
 		 * use the same key.
 		 */
 		PCI_VENDOR_HIFN,
 		PCI_PRODUCT_HIFN_7751,
 		{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
 		  0x00, 0x00, 0x00, 0x00, 0x00 }
 	},
 };
 
 /*
  * Checks to see if crypto is already enabled.  If crypto isn't enable,
  * "hifn_enable_crypto" is called to enable it.  The check is important,
  * as enabling crypto twice will lock the board.
  */
 static int 
 hifn_enable_crypto(struct hifn_softc *sc)
 {
 	u_int32_t dmacfg, ramcfg, encl, addr, i;
 	char *offtbl = NULL;
 
 	for (i = 0; i < sizeof(pci2id)/sizeof(pci2id[0]); i++) {
 		if (pci2id[i].pci_vendor == pci_get_vendor(sc->sc_dev) &&
 		    pci2id[i].pci_prod == pci_get_device(sc->sc_dev)) {
 			offtbl = pci2id[i].card_id;
 			break;
 		}
 	}
 	if (offtbl == NULL) {
 		device_printf(sc->sc_dev, "Unknown card!\n");
 		return (1);
 	}
 
 	ramcfg = READ_REG_0(sc, HIFN_0_PUCNFG);
 	dmacfg = READ_REG_1(sc, HIFN_1_DMA_CNFG);
 
 	/*
 	 * The RAM config register's encrypt level bit needs to be set before
 	 * every read performed on the encryption level register.
 	 */
 	WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg | HIFN_PUCNFG_CHIPID);
 
 	encl = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA;
 
 	/*
 	 * Make sure we don't re-unlock.  Two unlocks kills chip until the
 	 * next reboot.
 	 */
 	if (encl == HIFN_PUSTAT_ENA_1 || encl == HIFN_PUSTAT_ENA_2) {
 #ifdef HIFN_DEBUG
 		if (hifn_debug)
 			device_printf(sc->sc_dev,
 			    "Strong crypto already enabled!\n");
 #endif
 		goto report;
 	}
 
 	if (encl != 0 && encl != HIFN_PUSTAT_ENA_0) {
 #ifdef HIFN_DEBUG
 		if (hifn_debug)
 			device_printf(sc->sc_dev,
 			      "Unknown encryption level 0x%x\n", encl);
 #endif
 		return 1;
 	}
 
 	WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_UNLOCK |
 	    HIFN_DMACNFG_MSTRESET | HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
 	DELAY(1000);
 	addr = READ_REG_1(sc, HIFN_UNLOCK_SECRET1);
 	DELAY(1000);
 	WRITE_REG_1(sc, HIFN_UNLOCK_SECRET2, 0);
 	DELAY(1000);
 
 	for (i = 0; i <= 12; i++) {
 		addr = hifn_next_signature(addr, offtbl[i] + 0x101);
 		WRITE_REG_1(sc, HIFN_UNLOCK_SECRET2, addr);
 
 		DELAY(1000);
 	}
 
 	WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg | HIFN_PUCNFG_CHIPID);
 	encl = READ_REG_0(sc, HIFN_0_PUSTAT) & HIFN_PUSTAT_CHIPENA;
 
 #ifdef HIFN_DEBUG
 	if (hifn_debug) {
 		if (encl != HIFN_PUSTAT_ENA_1 && encl != HIFN_PUSTAT_ENA_2)
 			device_printf(sc->sc_dev, "Engine is permanently "
 				"locked until next system reset!\n");
 		else
 			device_printf(sc->sc_dev, "Engine enabled "
 				"successfully!\n");
 	}
 #endif
 
 report:
 	WRITE_REG_0(sc, HIFN_0_PUCNFG, ramcfg);
 	WRITE_REG_1(sc, HIFN_1_DMA_CNFG, dmacfg);
 
 	switch (encl) {
 	case HIFN_PUSTAT_ENA_1:
 	case HIFN_PUSTAT_ENA_2:
 		break;
 	case HIFN_PUSTAT_ENA_0:
 	default:
 		device_printf(sc->sc_dev, "disabled");
 		break;
 	}
 
 	return 0;
 }
 
 /*
  * Give initial values to the registers listed in the "Register Space"
  * section of the HIFN Software Development reference manual.
  */
 static void 
 hifn_init_pci_registers(struct hifn_softc *sc)
 {
 	/* write fixed values needed by the Initialization registers */
 	WRITE_REG_0(sc, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA);
 	WRITE_REG_0(sc, HIFN_0_FIFOCNFG, HIFN_FIFOCNFG_THRESHOLD);
 	WRITE_REG_0(sc, HIFN_0_PUIER, HIFN_PUIER_DSTOVER);
 
 	/* write all 4 ring address registers */
 	WRITE_REG_1(sc, HIFN_1_DMA_CRAR, sc->sc_dma_physaddr +
 	    offsetof(struct hifn_dma, cmdr[0]));
 	WRITE_REG_1(sc, HIFN_1_DMA_SRAR, sc->sc_dma_physaddr +
 	    offsetof(struct hifn_dma, srcr[0]));
 	WRITE_REG_1(sc, HIFN_1_DMA_DRAR, sc->sc_dma_physaddr +
 	    offsetof(struct hifn_dma, dstr[0]));
 	WRITE_REG_1(sc, HIFN_1_DMA_RRAR, sc->sc_dma_physaddr +
 	    offsetof(struct hifn_dma, resr[0]));
 
 	DELAY(2000);
 
 	/* write status register */
 	WRITE_REG_1(sc, HIFN_1_DMA_CSR,
 	    HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS |
 	    HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS |
 	    HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST |
 	    HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER |
 	    HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST |
 	    HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER |
 	    HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST |
 	    HIFN_DMACSR_S_WAIT |
 	    HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST |
 	    HIFN_DMACSR_C_WAIT |
 	    HIFN_DMACSR_ENGINE |
 	    ((sc->sc_flags & HIFN_HAS_PUBLIC) ?
 		HIFN_DMACSR_PUBDONE : 0) |
 	    ((sc->sc_flags & HIFN_IS_7811) ?
 		HIFN_DMACSR_ILLW | HIFN_DMACSR_ILLR : 0));
 
 	sc->sc_d_busy = sc->sc_r_busy = sc->sc_s_busy = sc->sc_c_busy = 0;
 	sc->sc_dmaier |= HIFN_DMAIER_R_DONE | HIFN_DMAIER_C_ABORT |
 	    HIFN_DMAIER_D_OVER | HIFN_DMAIER_R_OVER |
 	    HIFN_DMAIER_S_ABORT | HIFN_DMAIER_D_ABORT | HIFN_DMAIER_R_ABORT |
 	    ((sc->sc_flags & HIFN_IS_7811) ?
 		HIFN_DMAIER_ILLW | HIFN_DMAIER_ILLR : 0);
 	sc->sc_dmaier &= ~HIFN_DMAIER_C_WAIT;
 	WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
 
 
 	if (sc->sc_flags & HIFN_IS_7956) {
 		u_int32_t pll;
 
 		WRITE_REG_0(sc, HIFN_0_PUCNFG, HIFN_PUCNFG_COMPSING |
 		    HIFN_PUCNFG_TCALLPHASES |
 		    HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32);
 
 		/* turn off the clocks and insure bypass is set */
 		pll = READ_REG_1(sc, HIFN_1_PLL);
 		pll = (pll &~ (HIFN_PLL_PK_CLK_SEL | HIFN_PLL_PE_CLK_SEL))
 		  | HIFN_PLL_BP | HIFN_PLL_MBSET;
 		WRITE_REG_1(sc, HIFN_1_PLL, pll);
 		DELAY(10*1000);		/* 10ms */
 
 		/* change configuration */
 		pll = (pll &~ HIFN_PLL_CONFIG) | sc->sc_pllconfig;
 		WRITE_REG_1(sc, HIFN_1_PLL, pll);
 		DELAY(10*1000);		/* 10ms */
 
 		/* disable bypass */
 		pll &= ~HIFN_PLL_BP;
 		WRITE_REG_1(sc, HIFN_1_PLL, pll);
 		/* enable clocks with new configuration */
 		pll |= HIFN_PLL_PK_CLK_SEL | HIFN_PLL_PE_CLK_SEL;
 		WRITE_REG_1(sc, HIFN_1_PLL, pll);
 	} else {
 		WRITE_REG_0(sc, HIFN_0_PUCNFG, HIFN_PUCNFG_COMPSING |
 		    HIFN_PUCNFG_DRFR_128 | HIFN_PUCNFG_TCALLPHASES |
 		    HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32 |
 		    (sc->sc_drammodel ? HIFN_PUCNFG_DRAM : HIFN_PUCNFG_SRAM));
 	}
 
 	WRITE_REG_0(sc, HIFN_0_PUISR, HIFN_PUISR_DSTOVER);
 	WRITE_REG_1(sc, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
 	    HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE | HIFN_DMACNFG_LAST |
 	    ((HIFN_POLL_FREQUENCY << 16 ) & HIFN_DMACNFG_POLLFREQ) |
 	    ((HIFN_POLL_SCALAR << 8) & HIFN_DMACNFG_POLLINVAL));
 }
 
 /*
  * The maximum number of sessions supported by the card
  * is dependent on the amount of context ram, which
  * encryption algorithms are enabled, and how compression
  * is configured.  This should be configured before this
  * routine is called.
  */
 static void
 hifn_sessions(struct hifn_softc *sc)
 {
 	u_int32_t pucnfg;
 	int ctxsize;
 
 	pucnfg = READ_REG_0(sc, HIFN_0_PUCNFG);
 
 	if (pucnfg & HIFN_PUCNFG_COMPSING) {
 		if (pucnfg & HIFN_PUCNFG_ENCCNFG)
 			ctxsize = 128;
 		else
 			ctxsize = 512;
 		/*
 		 * 7955/7956 has internal context memory of 32K
 		 */
 		if (sc->sc_flags & HIFN_IS_7956)
 			sc->sc_maxses = 32768 / ctxsize;
 		else
 			sc->sc_maxses = 1 +
 			    ((sc->sc_ramsize - 32768) / ctxsize);
 	} else
 		sc->sc_maxses = sc->sc_ramsize / 16384;
 
 	if (sc->sc_maxses > 2048)
 		sc->sc_maxses = 2048;
 }
 
 /*
  * Determine ram type (sram or dram).  Board should be just out of a reset
  * state when this is called.
  */
 static int
 hifn_ramtype(struct hifn_softc *sc)
 {
 	u_int8_t data[8], dataexpect[8];
 	int i;
 
 	for (i = 0; i < sizeof(data); i++)
 		data[i] = dataexpect[i] = 0x55;
 	if (hifn_writeramaddr(sc, 0, data))
 		return (-1);
 	if (hifn_readramaddr(sc, 0, data))
 		return (-1);
 	if (bcmp(data, dataexpect, sizeof(data)) != 0) {
 		sc->sc_drammodel = 1;
 		return (0);
 	}
 
 	for (i = 0; i < sizeof(data); i++)
 		data[i] = dataexpect[i] = 0xaa;
 	if (hifn_writeramaddr(sc, 0, data))
 		return (-1);
 	if (hifn_readramaddr(sc, 0, data))
 		return (-1);
 	if (bcmp(data, dataexpect, sizeof(data)) != 0) {
 		sc->sc_drammodel = 1;
 		return (0);
 	}
 
 	return (0);
 }
 
 #define	HIFN_SRAM_MAX		(32 << 20)
 #define	HIFN_SRAM_STEP_SIZE	16384
 #define	HIFN_SRAM_GRANULARITY	(HIFN_SRAM_MAX / HIFN_SRAM_STEP_SIZE)
 
 static int
 hifn_sramsize(struct hifn_softc *sc)
 {
 	u_int32_t a;
 	u_int8_t data[8];
 	u_int8_t dataexpect[sizeof(data)];
 	int32_t i;
 
 	for (i = 0; i < sizeof(data); i++)
 		data[i] = dataexpect[i] = i ^ 0x5a;
 
 	for (i = HIFN_SRAM_GRANULARITY - 1; i >= 0; i--) {
 		a = i * HIFN_SRAM_STEP_SIZE;
 		bcopy(&i, data, sizeof(i));
 		hifn_writeramaddr(sc, a, data);
 	}
 
 	for (i = 0; i < HIFN_SRAM_GRANULARITY; i++) {
 		a = i * HIFN_SRAM_STEP_SIZE;
 		bcopy(&i, dataexpect, sizeof(i));
 		if (hifn_readramaddr(sc, a, data) < 0)
 			return (0);
 		if (bcmp(data, dataexpect, sizeof(data)) != 0)
 			return (0);
 		sc->sc_ramsize = a + HIFN_SRAM_STEP_SIZE;
 	}
 
 	return (0);
 }
 
 /*
  * XXX For dram boards, one should really try all of the
  * HIFN_PUCNFG_DSZ_*'s.  This just assumes that PUCNFG
  * is already set up correctly.
  */
 static int
 hifn_dramsize(struct hifn_softc *sc)
 {
 	u_int32_t cnfg;
 
 	if (sc->sc_flags & HIFN_IS_7956) {
 		/*
 		 * 7955/7956 have a fixed internal ram of only 32K.
 		 */
 		sc->sc_ramsize = 32768;
 	} else {
 		cnfg = READ_REG_0(sc, HIFN_0_PUCNFG) &
 		    HIFN_PUCNFG_DRAMMASK;
 		sc->sc_ramsize = 1 << ((cnfg >> 13) + 18);
 	}
 	return (0);
 }
 
 static void
 hifn_alloc_slot(struct hifn_softc *sc, int *cmdp, int *srcp, int *dstp, int *resp)
 {
 	struct hifn_dma *dma = sc->sc_dma;
 
 	if (sc->sc_cmdi == HIFN_D_CMD_RSIZE) {
 		sc->sc_cmdi = 0;
 		dma->cmdr[HIFN_D_CMD_RSIZE].l = htole32(HIFN_D_VALID |
 		    HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
 		HIFN_CMDR_SYNC(sc, HIFN_D_CMD_RSIZE,
 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
 	}
 	*cmdp = sc->sc_cmdi++;
 	sc->sc_cmdk = sc->sc_cmdi;
 
 	if (sc->sc_srci == HIFN_D_SRC_RSIZE) {
 		sc->sc_srci = 0;
 		dma->srcr[HIFN_D_SRC_RSIZE].l = htole32(HIFN_D_VALID |
 		    HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
 		HIFN_SRCR_SYNC(sc, HIFN_D_SRC_RSIZE,
 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
 	}
 	*srcp = sc->sc_srci++;
 	sc->sc_srck = sc->sc_srci;
 
 	if (sc->sc_dsti == HIFN_D_DST_RSIZE) {
 		sc->sc_dsti = 0;
 		dma->dstr[HIFN_D_DST_RSIZE].l = htole32(HIFN_D_VALID |
 		    HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
 		HIFN_DSTR_SYNC(sc, HIFN_D_DST_RSIZE,
 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
 	}
 	*dstp = sc->sc_dsti++;
 	sc->sc_dstk = sc->sc_dsti;
 
 	if (sc->sc_resi == HIFN_D_RES_RSIZE) {
 		sc->sc_resi = 0;
 		dma->resr[HIFN_D_RES_RSIZE].l = htole32(HIFN_D_VALID |
 		    HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
 		HIFN_RESR_SYNC(sc, HIFN_D_RES_RSIZE,
 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
 	}
 	*resp = sc->sc_resi++;
 	sc->sc_resk = sc->sc_resi;
 }
 
 static int
 hifn_writeramaddr(struct hifn_softc *sc, int addr, u_int8_t *data)
 {
 	struct hifn_dma *dma = sc->sc_dma;
 	hifn_base_command_t wc;
 	const u_int32_t masks = HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ;
 	int r, cmdi, resi, srci, dsti;
 
 	wc.masks = htole16(3 << 13);
 	wc.session_num = htole16(addr >> 14);
 	wc.total_source_count = htole16(8);
 	wc.total_dest_count = htole16(addr & 0x3fff);
 
 	hifn_alloc_slot(sc, &cmdi, &srci, &dsti, &resi);
 
 	WRITE_REG_1(sc, HIFN_1_DMA_CSR,
 	    HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA |
 	    HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA);
 
 	/* build write command */
 	bzero(dma->command_bufs[cmdi], HIFN_MAX_COMMAND);
 	*(hifn_base_command_t *)dma->command_bufs[cmdi] = wc;
 	bcopy(data, &dma->test_src, sizeof(dma->test_src));
 
 	dma->srcr[srci].p = htole32(sc->sc_dma_physaddr
 	    + offsetof(struct hifn_dma, test_src));
 	dma->dstr[dsti].p = htole32(sc->sc_dma_physaddr
 	    + offsetof(struct hifn_dma, test_dst));
 
 	dma->cmdr[cmdi].l = htole32(16 | masks);
 	dma->srcr[srci].l = htole32(8 | masks);
 	dma->dstr[dsti].l = htole32(4 | masks);
 	dma->resr[resi].l = htole32(4 | masks);
 
 	bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 
 	for (r = 10000; r >= 0; r--) {
 		DELAY(10);
 		bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
 		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 		if ((dma->resr[resi].l & htole32(HIFN_D_VALID)) == 0)
 			break;
 		bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
 		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 	}
 	if (r == 0) {
 		device_printf(sc->sc_dev, "writeramaddr -- "
 		    "result[%d](addr %d) still valid\n", resi, addr);
 		r = -1;
 		return (-1);
 	} else
 		r = 0;
 
 	WRITE_REG_1(sc, HIFN_1_DMA_CSR,
 	    HIFN_DMACSR_C_CTRL_DIS | HIFN_DMACSR_S_CTRL_DIS |
 	    HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS);
 
 	return (r);
 }
 
 static int
 hifn_readramaddr(struct hifn_softc *sc, int addr, u_int8_t *data)
 {
 	struct hifn_dma *dma = sc->sc_dma;
 	hifn_base_command_t rc;
 	const u_int32_t masks = HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ;
 	int r, cmdi, srci, dsti, resi;
 
 	rc.masks = htole16(2 << 13);
 	rc.session_num = htole16(addr >> 14);
 	rc.total_source_count = htole16(addr & 0x3fff);
 	rc.total_dest_count = htole16(8);
 
 	hifn_alloc_slot(sc, &cmdi, &srci, &dsti, &resi);
 
 	WRITE_REG_1(sc, HIFN_1_DMA_CSR,
 	    HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA |
 	    HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA);
 
 	bzero(dma->command_bufs[cmdi], HIFN_MAX_COMMAND);
 	*(hifn_base_command_t *)dma->command_bufs[cmdi] = rc;
 
 	dma->srcr[srci].p = htole32(sc->sc_dma_physaddr +
 	    offsetof(struct hifn_dma, test_src));
 	dma->test_src = 0;
 	dma->dstr[dsti].p =  htole32(sc->sc_dma_physaddr +
 	    offsetof(struct hifn_dma, test_dst));
 	dma->test_dst = 0;
 	dma->cmdr[cmdi].l = htole32(8 | masks);
 	dma->srcr[srci].l = htole32(8 | masks);
 	dma->dstr[dsti].l = htole32(8 | masks);
 	dma->resr[resi].l = htole32(HIFN_MAX_RESULT | masks);
 
 	bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 
 	for (r = 10000; r >= 0; r--) {
 		DELAY(10);
 		bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
 		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 		if ((dma->resr[resi].l & htole32(HIFN_D_VALID)) == 0)
 			break;
 		bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
 		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 	}
 	if (r == 0) {
 		device_printf(sc->sc_dev, "readramaddr -- "
 		    "result[%d](addr %d) still valid\n", resi, addr);
 		r = -1;
 	} else {
 		r = 0;
 		bcopy(&dma->test_dst, data, sizeof(dma->test_dst));
 	}
 
 	WRITE_REG_1(sc, HIFN_1_DMA_CSR,
 	    HIFN_DMACSR_C_CTRL_DIS | HIFN_DMACSR_S_CTRL_DIS |
 	    HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS);
 
 	return (r);
 }
 
 /*
  * Initialize the descriptor rings.
  */
 static void 
 hifn_init_dma(struct hifn_softc *sc)
 {
 	struct hifn_dma *dma = sc->sc_dma;
 	int i;
 
 	hifn_set_retry(sc);
 
 	/* initialize static pointer values */
 	for (i = 0; i < HIFN_D_CMD_RSIZE; i++)
 		dma->cmdr[i].p = htole32(sc->sc_dma_physaddr +
 		    offsetof(struct hifn_dma, command_bufs[i][0]));
 	for (i = 0; i < HIFN_D_RES_RSIZE; i++)
 		dma->resr[i].p = htole32(sc->sc_dma_physaddr +
 		    offsetof(struct hifn_dma, result_bufs[i][0]));
 
 	dma->cmdr[HIFN_D_CMD_RSIZE].p =
 	    htole32(sc->sc_dma_physaddr + offsetof(struct hifn_dma, cmdr[0]));
 	dma->srcr[HIFN_D_SRC_RSIZE].p =
 	    htole32(sc->sc_dma_physaddr + offsetof(struct hifn_dma, srcr[0]));
 	dma->dstr[HIFN_D_DST_RSIZE].p =
 	    htole32(sc->sc_dma_physaddr + offsetof(struct hifn_dma, dstr[0]));
 	dma->resr[HIFN_D_RES_RSIZE].p =
 	    htole32(sc->sc_dma_physaddr + offsetof(struct hifn_dma, resr[0]));
 
 	sc->sc_cmdu = sc->sc_srcu = sc->sc_dstu = sc->sc_resu = 0;
 	sc->sc_cmdi = sc->sc_srci = sc->sc_dsti = sc->sc_resi = 0;
 	sc->sc_cmdk = sc->sc_srck = sc->sc_dstk = sc->sc_resk = 0;
 }
 
 /*
  * Writes out the raw command buffer space.  Returns the
  * command buffer size.
  */
 static u_int
 hifn_write_command(struct hifn_command *cmd, u_int8_t *buf)
 {
 	u_int8_t *buf_pos;
 	hifn_base_command_t *base_cmd;
 	hifn_mac_command_t *mac_cmd;
 	hifn_crypt_command_t *cry_cmd;
 	int using_mac, using_crypt, len, ivlen;
 	u_int32_t dlen, slen;
 
 	buf_pos = buf;
 	using_mac = cmd->base_masks & HIFN_BASE_CMD_MAC;
 	using_crypt = cmd->base_masks & HIFN_BASE_CMD_CRYPT;
 
 	base_cmd = (hifn_base_command_t *)buf_pos;
 	base_cmd->masks = htole16(cmd->base_masks);
 	slen = cmd->src_mapsize;
 	if (cmd->sloplen)
 		dlen = cmd->dst_mapsize - cmd->sloplen + sizeof(u_int32_t);
 	else
 		dlen = cmd->dst_mapsize;
 	base_cmd->total_source_count = htole16(slen & HIFN_BASE_CMD_LENMASK_LO);
 	base_cmd->total_dest_count = htole16(dlen & HIFN_BASE_CMD_LENMASK_LO);
 	dlen >>= 16;
 	slen >>= 16;
 	base_cmd->session_num = htole16(
 	    ((slen << HIFN_BASE_CMD_SRCLEN_S) & HIFN_BASE_CMD_SRCLEN_M) |
 	    ((dlen << HIFN_BASE_CMD_DSTLEN_S) & HIFN_BASE_CMD_DSTLEN_M));
 	buf_pos += sizeof(hifn_base_command_t);
 
 	if (using_mac) {
 		mac_cmd = (hifn_mac_command_t *)buf_pos;
 		dlen = cmd->maccrd->crd_len;
 		mac_cmd->source_count = htole16(dlen & 0xffff);
 		dlen >>= 16;
 		mac_cmd->masks = htole16(cmd->mac_masks |
 		    ((dlen << HIFN_MAC_CMD_SRCLEN_S) & HIFN_MAC_CMD_SRCLEN_M));
 		mac_cmd->header_skip = htole16(cmd->maccrd->crd_skip);
 		mac_cmd->reserved = 0;
 		buf_pos += sizeof(hifn_mac_command_t);
 	}
 
 	if (using_crypt) {
 		cry_cmd = (hifn_crypt_command_t *)buf_pos;
 		dlen = cmd->enccrd->crd_len;
 		cry_cmd->source_count = htole16(dlen & 0xffff);
 		dlen >>= 16;
 		cry_cmd->masks = htole16(cmd->cry_masks |
 		    ((dlen << HIFN_CRYPT_CMD_SRCLEN_S) & HIFN_CRYPT_CMD_SRCLEN_M));
 		cry_cmd->header_skip = htole16(cmd->enccrd->crd_skip);
 		cry_cmd->reserved = 0;
 		buf_pos += sizeof(hifn_crypt_command_t);
 	}
 
 	if (using_mac && cmd->mac_masks & HIFN_MAC_CMD_NEW_KEY) {
 		bcopy(cmd->mac, buf_pos, HIFN_MAC_KEY_LENGTH);
 		buf_pos += HIFN_MAC_KEY_LENGTH;
 	}
 
 	if (using_crypt && cmd->cry_masks & HIFN_CRYPT_CMD_NEW_KEY) {
 		switch (cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) {
 		case HIFN_CRYPT_CMD_ALG_3DES:
 			bcopy(cmd->ck, buf_pos, HIFN_3DES_KEY_LENGTH);
 			buf_pos += HIFN_3DES_KEY_LENGTH;
 			break;
 		case HIFN_CRYPT_CMD_ALG_DES:
 			bcopy(cmd->ck, buf_pos, HIFN_DES_KEY_LENGTH);
 			buf_pos += HIFN_DES_KEY_LENGTH;
 			break;
 		case HIFN_CRYPT_CMD_ALG_RC4:
 			len = 256;
 			do {
 				int clen;
 
 				clen = MIN(cmd->cklen, len);
 				bcopy(cmd->ck, buf_pos, clen);
 				len -= clen;
 				buf_pos += clen;
 			} while (len > 0);
 			bzero(buf_pos, 4);
 			buf_pos += 4;
 			break;
 		case HIFN_CRYPT_CMD_ALG_AES:
 			/*
 			 * AES keys are variable 128, 192 and
 			 * 256 bits (16, 24 and 32 bytes).
 			 */
 			bcopy(cmd->ck, buf_pos, cmd->cklen);
 			buf_pos += cmd->cklen;
 			break;
 		}
 	}
 
 	if (using_crypt && cmd->cry_masks & HIFN_CRYPT_CMD_NEW_IV) {
 		switch (cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) {
 		case HIFN_CRYPT_CMD_ALG_AES:
 			ivlen = HIFN_AES_IV_LENGTH;
 			break;
 		default:
 			ivlen = HIFN_IV_LENGTH;
 			break;
 		}
 		bcopy(cmd->iv, buf_pos, ivlen);
 		buf_pos += ivlen;
 	}
 
 	if ((cmd->base_masks & (HIFN_BASE_CMD_MAC|HIFN_BASE_CMD_CRYPT)) == 0) {
 		bzero(buf_pos, 8);
 		buf_pos += 8;
 	}
 
 	return (buf_pos - buf);
 }
 
 static int
 hifn_dmamap_aligned(struct hifn_operand *op)
 {
 	int i;
 
 	for (i = 0; i < op->nsegs; i++) {
 		if (op->segs[i].ds_addr & 3)
 			return (0);
 		if ((i != (op->nsegs - 1)) && (op->segs[i].ds_len & 3))
 			return (0);
 	}
 	return (1);
 }
 
 static __inline int
 hifn_dmamap_dstwrap(struct hifn_softc *sc, int idx)
 {
 	struct hifn_dma *dma = sc->sc_dma;
 
 	if (++idx == HIFN_D_DST_RSIZE) {
 		dma->dstr[idx].l = htole32(HIFN_D_VALID | HIFN_D_JUMP |
 		    HIFN_D_MASKDONEIRQ);
 		HIFN_DSTR_SYNC(sc, idx,
 		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 		idx = 0;
 	}
 	return (idx);
 }
 
 static int
 hifn_dmamap_load_dst(struct hifn_softc *sc, struct hifn_command *cmd)
 {
 	struct hifn_dma *dma = sc->sc_dma;
 	struct hifn_operand *dst = &cmd->dst;
 	u_int32_t p, l;
 	int idx, used = 0, i;
 
 	idx = sc->sc_dsti;
 	for (i = 0; i < dst->nsegs - 1; i++) {
 		dma->dstr[idx].p = htole32(dst->segs[i].ds_addr);
 		dma->dstr[idx].l = htole32(HIFN_D_VALID |
 		    HIFN_D_MASKDONEIRQ | dst->segs[i].ds_len);
 		HIFN_DSTR_SYNC(sc, idx,
 		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 		used++;
 
 		idx = hifn_dmamap_dstwrap(sc, idx);
 	}
 
 	if (cmd->sloplen == 0) {
 		p = dst->segs[i].ds_addr;
 		l = HIFN_D_VALID | HIFN_D_MASKDONEIRQ | HIFN_D_LAST |
 		    dst->segs[i].ds_len;
 	} else {
 		p = sc->sc_dma_physaddr +
 		    offsetof(struct hifn_dma, slop[cmd->slopidx]);
 		l = HIFN_D_VALID | HIFN_D_MASKDONEIRQ | HIFN_D_LAST |
 		    sizeof(u_int32_t);
 
 		if ((dst->segs[i].ds_len - cmd->sloplen) != 0) {
 			dma->dstr[idx].p = htole32(dst->segs[i].ds_addr);
 			dma->dstr[idx].l = htole32(HIFN_D_VALID |
 			    HIFN_D_MASKDONEIRQ |
 			    (dst->segs[i].ds_len - cmd->sloplen));
 			HIFN_DSTR_SYNC(sc, idx,
 			    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 			used++;
 
 			idx = hifn_dmamap_dstwrap(sc, idx);
 		}
 	}
 	dma->dstr[idx].p = htole32(p);
 	dma->dstr[idx].l = htole32(l);
 	HIFN_DSTR_SYNC(sc, idx, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 	used++;
 
 	idx = hifn_dmamap_dstwrap(sc, idx);
 
 	sc->sc_dsti = idx;
 	sc->sc_dstu += used;
 	return (idx);
 }
 
 static __inline int
 hifn_dmamap_srcwrap(struct hifn_softc *sc, int idx)
 {
 	struct hifn_dma *dma = sc->sc_dma;
 
 	if (++idx == HIFN_D_SRC_RSIZE) {
 		dma->srcr[idx].l = htole32(HIFN_D_VALID |
 		    HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
 		HIFN_SRCR_SYNC(sc, HIFN_D_SRC_RSIZE,
 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
 		idx = 0;
 	}
 	return (idx);
 }
 
 static int
 hifn_dmamap_load_src(struct hifn_softc *sc, struct hifn_command *cmd)
 {
 	struct hifn_dma *dma = sc->sc_dma;
 	struct hifn_operand *src = &cmd->src;
 	int idx, i;
 	u_int32_t last = 0;
 
 	idx = sc->sc_srci;
 	for (i = 0; i < src->nsegs; i++) {
 		if (i == src->nsegs - 1)
 			last = HIFN_D_LAST;
 
 		dma->srcr[idx].p = htole32(src->segs[i].ds_addr);
 		dma->srcr[idx].l = htole32(src->segs[i].ds_len |
 		    HIFN_D_VALID | HIFN_D_MASKDONEIRQ | last);
 		HIFN_SRCR_SYNC(sc, idx,
 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
 
 		idx = hifn_dmamap_srcwrap(sc, idx);
 	}
 	sc->sc_srci = idx;
 	sc->sc_srcu += src->nsegs;
 	return (idx);
 } 
 
 static void
 hifn_op_cb(void* arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error)
 {
 	struct hifn_operand *op = arg;
 
 	KASSERT(nsegs <= MAX_SCATTER,
 		("hifn_op_cb: too many DMA segments (%u > %u) "
 		 "returned when mapping operand", nsegs, MAX_SCATTER));
 	op->mapsize = mapsize;
 	op->nsegs = nsegs;
 	bcopy(seg, op->segs, nsegs * sizeof (seg[0]));
 }
 
 static int 
 hifn_crypto(
 	struct hifn_softc *sc,
 	struct hifn_command *cmd,
 	struct cryptop *crp,
 	int hint)
 {
 	struct	hifn_dma *dma = sc->sc_dma;
 	u_int32_t cmdlen, csr;
 	int cmdi, resi, err = 0;
 
 	/*
 	 * need 1 cmd, and 1 res
 	 *
 	 * NB: check this first since it's easy.
 	 */
 	HIFN_LOCK(sc);
 	if ((sc->sc_cmdu + 1) > HIFN_D_CMD_RSIZE ||
 	    (sc->sc_resu + 1) > HIFN_D_RES_RSIZE) {
 #ifdef HIFN_DEBUG
 		if (hifn_debug) {
 			device_printf(sc->sc_dev,
 				"cmd/result exhaustion, cmdu %u resu %u\n",
 				sc->sc_cmdu, sc->sc_resu);
 		}
 #endif
 		hifnstats.hst_nomem_cr++;
 		HIFN_UNLOCK(sc);
 		return (ERESTART);
 	}
 
 	if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &cmd->src_map)) {
 		hifnstats.hst_nomem_map++;
 		HIFN_UNLOCK(sc);
 		return (ENOMEM);
 	}
 
 	if (crp->crp_flags & CRYPTO_F_IMBUF) {
 		if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->src_map,
 		    cmd->src_m, hifn_op_cb, &cmd->src, BUS_DMA_NOWAIT)) {
 			hifnstats.hst_nomem_load++;
 			err = ENOMEM;
 			goto err_srcmap1;
 		}
 	} else if (crp->crp_flags & CRYPTO_F_IOV) {
 		if (bus_dmamap_load_uio(sc->sc_dmat, cmd->src_map,
 		    cmd->src_io, hifn_op_cb, &cmd->src, BUS_DMA_NOWAIT)) {
 			hifnstats.hst_nomem_load++;
 			err = ENOMEM;
 			goto err_srcmap1;
 		}
 	} else {
 		err = EINVAL;
 		goto err_srcmap1;
 	}
 
 	if (hifn_dmamap_aligned(&cmd->src)) {
 		cmd->sloplen = cmd->src_mapsize & 3;
 		cmd->dst = cmd->src;
 	} else {
 		if (crp->crp_flags & CRYPTO_F_IOV) {
 			err = EINVAL;
 			goto err_srcmap;
 		} else if (crp->crp_flags & CRYPTO_F_IMBUF) {
 			int totlen, len;
 			struct mbuf *m, *m0, *mlast;
 
 			KASSERT(cmd->dst_m == cmd->src_m,
 				("hifn_crypto: dst_m initialized improperly"));
 			hifnstats.hst_unaligned++;
 			/*
 			 * Source is not aligned on a longword boundary.
 			 * Copy the data to insure alignment.  If we fail
 			 * to allocate mbufs or clusters while doing this
 			 * we return ERESTART so the operation is requeued
 			 * at the crypto later, but only if there are
 			 * ops already posted to the hardware; otherwise we
 			 * have no guarantee that we'll be re-entered.
 			 */
 			totlen = cmd->src_mapsize;
 			if (cmd->src_m->m_flags & M_PKTHDR) {
 				len = MHLEN;
 				MGETHDR(m0, M_NOWAIT, MT_DATA);
 				if (m0 && !m_dup_pkthdr(m0, cmd->src_m, M_NOWAIT)) {
 					m_free(m0);
 					m0 = NULL;
 				}
 			} else {
 				len = MLEN;
 				MGET(m0, M_NOWAIT, MT_DATA);
 			}
 			if (m0 == NULL) {
 				hifnstats.hst_nomem_mbuf++;
 				err = sc->sc_cmdu ? ERESTART : ENOMEM;
 				goto err_srcmap;
 			}
 			if (totlen >= MINCLSIZE) {
 				MCLGET(m0, M_NOWAIT);
 				if ((m0->m_flags & M_EXT) == 0) {
 					hifnstats.hst_nomem_mcl++;
 					err = sc->sc_cmdu ? ERESTART : ENOMEM;
 					m_freem(m0);
 					goto err_srcmap;
 				}
 				len = MCLBYTES;
 			}
 			totlen -= len;
 			m0->m_pkthdr.len = m0->m_len = len;
 			mlast = m0;
 
 			while (totlen > 0) {
 				MGET(m, M_NOWAIT, MT_DATA);
 				if (m == NULL) {
 					hifnstats.hst_nomem_mbuf++;
 					err = sc->sc_cmdu ? ERESTART : ENOMEM;
 					m_freem(m0);
 					goto err_srcmap;
 				}
 				len = MLEN;
 				if (totlen >= MINCLSIZE) {
 					MCLGET(m, M_NOWAIT);
 					if ((m->m_flags & M_EXT) == 0) {
 						hifnstats.hst_nomem_mcl++;
 						err = sc->sc_cmdu ? ERESTART : ENOMEM;
 						mlast->m_next = m;
 						m_freem(m0);
 						goto err_srcmap;
 					}
 					len = MCLBYTES;
 				}
 
 				m->m_len = len;
 				m0->m_pkthdr.len += len;
 				totlen -= len;
 
 				mlast->m_next = m;
 				mlast = m;
 			}
 			cmd->dst_m = m0;
 		}
 	}
 
 	if (cmd->dst_map == NULL) {
 		if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &cmd->dst_map)) {
 			hifnstats.hst_nomem_map++;
 			err = ENOMEM;
 			goto err_srcmap;
 		}
 		if (crp->crp_flags & CRYPTO_F_IMBUF) {
 			if (bus_dmamap_load_mbuf(sc->sc_dmat, cmd->dst_map,
 			    cmd->dst_m, hifn_op_cb, &cmd->dst, BUS_DMA_NOWAIT)) {
 				hifnstats.hst_nomem_map++;
 				err = ENOMEM;
 				goto err_dstmap1;
 			}
 		} else if (crp->crp_flags & CRYPTO_F_IOV) {
 			if (bus_dmamap_load_uio(sc->sc_dmat, cmd->dst_map,
 			    cmd->dst_io, hifn_op_cb, &cmd->dst, BUS_DMA_NOWAIT)) {
 				hifnstats.hst_nomem_load++;
 				err = ENOMEM;
 				goto err_dstmap1;
 			}
 		}
 	}
 
 #ifdef HIFN_DEBUG
 	if (hifn_debug) {
 		device_printf(sc->sc_dev,
 		    "Entering cmd: stat %8x ien %8x u %d/%d/%d/%d n %d/%d\n",
 		    READ_REG_1(sc, HIFN_1_DMA_CSR),
 		    READ_REG_1(sc, HIFN_1_DMA_IER),
 		    sc->sc_cmdu, sc->sc_srcu, sc->sc_dstu, sc->sc_resu,
 		    cmd->src_nsegs, cmd->dst_nsegs);
 	}
 #endif
 
 	if (cmd->src_map == cmd->dst_map) {
 		bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
 		    BUS_DMASYNC_PREWRITE|BUS_DMASYNC_PREREAD);
 	} else {
 		bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
 		    BUS_DMASYNC_PREWRITE);
 		bus_dmamap_sync(sc->sc_dmat, cmd->dst_map,
 		    BUS_DMASYNC_PREREAD);
 	}
 
 	/*
 	 * need N src, and N dst
 	 */
 	if ((sc->sc_srcu + cmd->src_nsegs) > HIFN_D_SRC_RSIZE ||
 	    (sc->sc_dstu + cmd->dst_nsegs + 1) > HIFN_D_DST_RSIZE) {
 #ifdef HIFN_DEBUG
 		if (hifn_debug) {
 			device_printf(sc->sc_dev,
 				"src/dst exhaustion, srcu %u+%u dstu %u+%u\n",
 				sc->sc_srcu, cmd->src_nsegs,
 				sc->sc_dstu, cmd->dst_nsegs);
 		}
 #endif
 		hifnstats.hst_nomem_sd++;
 		err = ERESTART;
 		goto err_dstmap;
 	}
 
 	if (sc->sc_cmdi == HIFN_D_CMD_RSIZE) {
 		sc->sc_cmdi = 0;
 		dma->cmdr[HIFN_D_CMD_RSIZE].l = htole32(HIFN_D_VALID |
 		    HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
 		HIFN_CMDR_SYNC(sc, HIFN_D_CMD_RSIZE,
 		    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
 	}
 	cmdi = sc->sc_cmdi++;
 	cmdlen = hifn_write_command(cmd, dma->command_bufs[cmdi]);
 	HIFN_CMD_SYNC(sc, cmdi, BUS_DMASYNC_PREWRITE);
 
 	/* .p for command/result already set */
 	dma->cmdr[cmdi].l = htole32(cmdlen | HIFN_D_VALID | HIFN_D_LAST |
 	    HIFN_D_MASKDONEIRQ);
 	HIFN_CMDR_SYNC(sc, cmdi,
 	    BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD);
 	sc->sc_cmdu++;
 
 	/*
 	 * We don't worry about missing an interrupt (which a "command wait"
 	 * interrupt salvages us from), unless there is more than one command
 	 * in the queue.
 	 */
 	if (sc->sc_cmdu > 1) {
 		sc->sc_dmaier |= HIFN_DMAIER_C_WAIT;
 		WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
 	}
 
 	hifnstats.hst_ipackets++;
 	hifnstats.hst_ibytes += cmd->src_mapsize;
 
 	hifn_dmamap_load_src(sc, cmd);
 
 	/*
 	 * Unlike other descriptors, we don't mask done interrupt from
 	 * result descriptor.
 	 */
 #ifdef HIFN_DEBUG
 	if (hifn_debug)
 		printf("load res\n");
 #endif
 	if (sc->sc_resi == HIFN_D_RES_RSIZE) {
 		sc->sc_resi = 0;
 		dma->resr[HIFN_D_RES_RSIZE].l = htole32(HIFN_D_VALID |
 		    HIFN_D_JUMP | HIFN_D_MASKDONEIRQ);
 		HIFN_RESR_SYNC(sc, HIFN_D_RES_RSIZE,
 		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 	}
 	resi = sc->sc_resi++;
 	KASSERT(sc->sc_hifn_commands[resi] == NULL,
 		("hifn_crypto: command slot %u busy", resi));
 	sc->sc_hifn_commands[resi] = cmd;
 	HIFN_RES_SYNC(sc, resi, BUS_DMASYNC_PREREAD);
 	if ((hint & CRYPTO_HINT_MORE) && sc->sc_curbatch < hifn_maxbatch) {
 		dma->resr[resi].l = htole32(HIFN_MAX_RESULT |
 		    HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ);
 		sc->sc_curbatch++;
 		if (sc->sc_curbatch > hifnstats.hst_maxbatch)
 			hifnstats.hst_maxbatch = sc->sc_curbatch;
 		hifnstats.hst_totbatch++;
 	} else {
 		dma->resr[resi].l = htole32(HIFN_MAX_RESULT |
 		    HIFN_D_VALID | HIFN_D_LAST);
 		sc->sc_curbatch = 0;
 	}
 	HIFN_RESR_SYNC(sc, resi,
 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 	sc->sc_resu++;
 
 	if (cmd->sloplen)
 		cmd->slopidx = resi;
 
 	hifn_dmamap_load_dst(sc, cmd);
 
 	csr = 0;
 	if (sc->sc_c_busy == 0) {
 		csr |= HIFN_DMACSR_C_CTRL_ENA;
 		sc->sc_c_busy = 1;
 	}
 	if (sc->sc_s_busy == 0) {
 		csr |= HIFN_DMACSR_S_CTRL_ENA;
 		sc->sc_s_busy = 1;
 	}
 	if (sc->sc_r_busy == 0) {
 		csr |= HIFN_DMACSR_R_CTRL_ENA;
 		sc->sc_r_busy = 1;
 	}
 	if (sc->sc_d_busy == 0) {
 		csr |= HIFN_DMACSR_D_CTRL_ENA;
 		sc->sc_d_busy = 1;
 	}
 	if (csr)
 		WRITE_REG_1(sc, HIFN_1_DMA_CSR, csr);
 
 #ifdef HIFN_DEBUG
 	if (hifn_debug) {
 		device_printf(sc->sc_dev, "command: stat %8x ier %8x\n",
 		    READ_REG_1(sc, HIFN_1_DMA_CSR),
 		    READ_REG_1(sc, HIFN_1_DMA_IER));
 	}
 #endif
 
 	sc->sc_active = 5;
 	HIFN_UNLOCK(sc);
 	KASSERT(err == 0, ("hifn_crypto: success with error %u", err));
 	return (err);		/* success */
 
 err_dstmap:
 	if (cmd->src_map != cmd->dst_map)
 		bus_dmamap_unload(sc->sc_dmat, cmd->dst_map);
 err_dstmap1:
 	if (cmd->src_map != cmd->dst_map)
 		bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map);
 err_srcmap:
 	if (crp->crp_flags & CRYPTO_F_IMBUF) {
 		if (cmd->src_m != cmd->dst_m)
 			m_freem(cmd->dst_m);
 	}
 	bus_dmamap_unload(sc->sc_dmat, cmd->src_map);
 err_srcmap1:
 	bus_dmamap_destroy(sc->sc_dmat, cmd->src_map);
 	HIFN_UNLOCK(sc);
 	return (err);
 }
 
 static void
 hifn_tick(void* vsc)
 {
 	struct hifn_softc *sc = vsc;
 
 	HIFN_LOCK(sc);
 	if (sc->sc_active == 0) {
 		u_int32_t r = 0;
 
 		if (sc->sc_cmdu == 0 && sc->sc_c_busy) {
 			sc->sc_c_busy = 0;
 			r |= HIFN_DMACSR_C_CTRL_DIS;
 		}
 		if (sc->sc_srcu == 0 && sc->sc_s_busy) {
 			sc->sc_s_busy = 0;
 			r |= HIFN_DMACSR_S_CTRL_DIS;
 		}
 		if (sc->sc_dstu == 0 && sc->sc_d_busy) {
 			sc->sc_d_busy = 0;
 			r |= HIFN_DMACSR_D_CTRL_DIS;
 		}
 		if (sc->sc_resu == 0 && sc->sc_r_busy) {
 			sc->sc_r_busy = 0;
 			r |= HIFN_DMACSR_R_CTRL_DIS;
 		}
 		if (r)
 			WRITE_REG_1(sc, HIFN_1_DMA_CSR, r);
 	} else
 		sc->sc_active--;
 	HIFN_UNLOCK(sc);
 	callout_reset(&sc->sc_tickto, hz, hifn_tick, sc);
 }
 
 static void 
 hifn_intr(void *arg)
 {
 	struct hifn_softc *sc = arg;
 	struct hifn_dma *dma;
 	u_int32_t dmacsr, restart;
 	int i, u;
 
 	dmacsr = READ_REG_1(sc, HIFN_1_DMA_CSR);
 
 	/* Nothing in the DMA unit interrupted */
 	if ((dmacsr & sc->sc_dmaier) == 0)
 		return;
 
 	HIFN_LOCK(sc);
 
 	dma = sc->sc_dma;
 
 #ifdef HIFN_DEBUG
 	if (hifn_debug) {
 		device_printf(sc->sc_dev,
 		    "irq: stat %08x ien %08x damier %08x i %d/%d/%d/%d k %d/%d/%d/%d u %d/%d/%d/%d\n",
 		    dmacsr, READ_REG_1(sc, HIFN_1_DMA_IER), sc->sc_dmaier,
 		    sc->sc_cmdi, sc->sc_srci, sc->sc_dsti, sc->sc_resi,
 		    sc->sc_cmdk, sc->sc_srck, sc->sc_dstk, sc->sc_resk,
 		    sc->sc_cmdu, sc->sc_srcu, sc->sc_dstu, sc->sc_resu);
 	}
 #endif
 
 	WRITE_REG_1(sc, HIFN_1_DMA_CSR, dmacsr & sc->sc_dmaier);
 
 	if ((sc->sc_flags & HIFN_HAS_PUBLIC) &&
 	    (dmacsr & HIFN_DMACSR_PUBDONE))
 		WRITE_REG_1(sc, HIFN_1_PUB_STATUS,
 		    READ_REG_1(sc, HIFN_1_PUB_STATUS) | HIFN_PUBSTS_DONE);
 
 	restart = dmacsr & (HIFN_DMACSR_D_OVER | HIFN_DMACSR_R_OVER);
 	if (restart)
 		device_printf(sc->sc_dev, "overrun %x\n", dmacsr);
 
 	if (sc->sc_flags & HIFN_IS_7811) {
 		if (dmacsr & HIFN_DMACSR_ILLR)
 			device_printf(sc->sc_dev, "illegal read\n");
 		if (dmacsr & HIFN_DMACSR_ILLW)
 			device_printf(sc->sc_dev, "illegal write\n");
 	}
 
 	restart = dmacsr & (HIFN_DMACSR_C_ABORT | HIFN_DMACSR_S_ABORT |
 	    HIFN_DMACSR_D_ABORT | HIFN_DMACSR_R_ABORT);
 	if (restart) {
 		device_printf(sc->sc_dev, "abort, resetting.\n");
 		hifnstats.hst_abort++;
 		hifn_abort(sc);
 		HIFN_UNLOCK(sc);
 		return;
 	}
 
 	if ((dmacsr & HIFN_DMACSR_C_WAIT) && (sc->sc_cmdu == 0)) {
 		/*
 		 * If no slots to process and we receive a "waiting on
 		 * command" interrupt, we disable the "waiting on command"
 		 * (by clearing it).
 		 */
 		sc->sc_dmaier &= ~HIFN_DMAIER_C_WAIT;
 		WRITE_REG_1(sc, HIFN_1_DMA_IER, sc->sc_dmaier);
 	}
 
 	/* clear the rings */
 	i = sc->sc_resk; u = sc->sc_resu;
 	while (u != 0) {
 		HIFN_RESR_SYNC(sc, i,
 		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 		if (dma->resr[i].l & htole32(HIFN_D_VALID)) {
 			HIFN_RESR_SYNC(sc, i,
 			    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 			break;
 		}
 
 		if (i != HIFN_D_RES_RSIZE) {
 			struct hifn_command *cmd;
 			u_int8_t *macbuf = NULL;
 
 			HIFN_RES_SYNC(sc, i, BUS_DMASYNC_POSTREAD);
 			cmd = sc->sc_hifn_commands[i];
 			KASSERT(cmd != NULL,
 				("hifn_intr: null command slot %u", i));
 			sc->sc_hifn_commands[i] = NULL;
 
 			if (cmd->base_masks & HIFN_BASE_CMD_MAC) {
 				macbuf = dma->result_bufs[i];
 				macbuf += 12;
 			}
 
 			hifn_callback(sc, cmd, macbuf);
 			hifnstats.hst_opackets++;
 			u--;
 		}
 
 		if (++i == (HIFN_D_RES_RSIZE + 1))
 			i = 0;
 	}
 	sc->sc_resk = i; sc->sc_resu = u;
 
 	i = sc->sc_srck; u = sc->sc_srcu;
 	while (u != 0) {
 		if (i == HIFN_D_SRC_RSIZE)
 			i = 0;
 		HIFN_SRCR_SYNC(sc, i,
 		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 		if (dma->srcr[i].l & htole32(HIFN_D_VALID)) {
 			HIFN_SRCR_SYNC(sc, i,
 			    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 			break;
 		}
 		i++, u--;
 	}
 	sc->sc_srck = i; sc->sc_srcu = u;
 
 	i = sc->sc_cmdk; u = sc->sc_cmdu;
 	while (u != 0) {
 		HIFN_CMDR_SYNC(sc, i,
 		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 		if (dma->cmdr[i].l & htole32(HIFN_D_VALID)) {
 			HIFN_CMDR_SYNC(sc, i,
 			    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 			break;
 		}
 		if (i != HIFN_D_CMD_RSIZE) {
 			u--;
 			HIFN_CMD_SYNC(sc, i, BUS_DMASYNC_POSTWRITE);
 		}
 		if (++i == (HIFN_D_CMD_RSIZE + 1))
 			i = 0;
 	}
 	sc->sc_cmdk = i; sc->sc_cmdu = u;
 
 	HIFN_UNLOCK(sc);
 
 	if (sc->sc_needwakeup) {		/* XXX check high watermark */
 		int wakeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ);
 #ifdef HIFN_DEBUG
 		if (hifn_debug)
 			device_printf(sc->sc_dev,
 				"wakeup crypto (%x) u %d/%d/%d/%d\n",
 				sc->sc_needwakeup,
 				sc->sc_cmdu, sc->sc_srcu, sc->sc_dstu, sc->sc_resu);
 #endif
 		sc->sc_needwakeup &= ~wakeup;
 		crypto_unblock(sc->sc_cid, wakeup);
 	}
 }
 
 /*
  * Allocate a new 'session' and return an encoded session id.  'sidp'
  * contains our registration id, and should contain an encoded session
  * id on successful allocation.
  */
 static int
 hifn_newsession(device_t dev, u_int32_t *sidp, struct cryptoini *cri)
 {
 	struct hifn_softc *sc = device_get_softc(dev);
 	struct cryptoini *c;
 	int mac = 0, cry = 0, sesn;
 	struct hifn_session *ses = NULL;
 
 	KASSERT(sc != NULL, ("hifn_newsession: null softc"));
 	if (sidp == NULL || cri == NULL || sc == NULL)
 		return (EINVAL);
 
 	HIFN_LOCK(sc);
 	if (sc->sc_sessions == NULL) {
 		ses = sc->sc_sessions = (struct hifn_session *)malloc(
 		    sizeof(*ses), M_DEVBUF, M_NOWAIT);
 		if (ses == NULL) {
 			HIFN_UNLOCK(sc);
 			return (ENOMEM);
 		}
 		sesn = 0;
 		sc->sc_nsessions = 1;
 	} else {
 		for (sesn = 0; sesn < sc->sc_nsessions; sesn++) {
 			if (!sc->sc_sessions[sesn].hs_used) {
 				ses = &sc->sc_sessions[sesn];
 				break;
 			}
 		}
 
 		if (ses == NULL) {
 			sesn = sc->sc_nsessions;
 			ses = (struct hifn_session *)malloc((sesn + 1) *
 			    sizeof(*ses), M_DEVBUF, M_NOWAIT);
 			if (ses == NULL) {
 				HIFN_UNLOCK(sc);
 				return (ENOMEM);
 			}
 			bcopy(sc->sc_sessions, ses, sesn * sizeof(*ses));
 			bzero(sc->sc_sessions, sesn * sizeof(*ses));
 			free(sc->sc_sessions, M_DEVBUF);
 			sc->sc_sessions = ses;
 			ses = &sc->sc_sessions[sesn];
 			sc->sc_nsessions++;
 		}
 	}
 	HIFN_UNLOCK(sc);
 
 	bzero(ses, sizeof(*ses));
 	ses->hs_used = 1;
 
 	for (c = cri; c != NULL; c = c->cri_next) {
 		switch (c->cri_alg) {
 		case CRYPTO_MD5:
 		case CRYPTO_SHA1:
 		case CRYPTO_MD5_HMAC:
 		case CRYPTO_SHA1_HMAC:
 			if (mac)
 				return (EINVAL);
 			mac = 1;
 			ses->hs_mlen = c->cri_mlen;
 			if (ses->hs_mlen == 0) {
 				switch (c->cri_alg) {
 				case CRYPTO_MD5:
 				case CRYPTO_MD5_HMAC:
 					ses->hs_mlen = 16;
 					break;
 				case CRYPTO_SHA1:
 				case CRYPTO_SHA1_HMAC:
 					ses->hs_mlen = 20;
 					break;
 				}
 			}
 			break;
 		case CRYPTO_DES_CBC:
 		case CRYPTO_3DES_CBC:
 		case CRYPTO_AES_CBC:
 			/* XXX this may read fewer, does it matter? */
 			read_random(ses->hs_iv,
 				c->cri_alg == CRYPTO_AES_CBC ?
 					HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH);
 			/*FALLTHROUGH*/
 		case CRYPTO_ARC4:
 			if (cry)
 				return (EINVAL);
 			cry = 1;
 			break;
 		default:
 			return (EINVAL);
 		}
 	}
 	if (mac == 0 && cry == 0)
 		return (EINVAL);
 
 	*sidp = HIFN_SID(device_get_unit(sc->sc_dev), sesn);
 
 	return (0);
 }
 
 /*
  * Deallocate a session.
  * XXX this routine should run a zero'd mac/encrypt key into context ram.
  * XXX to blow away any keys already stored there.
  */
 static int
 hifn_freesession(device_t dev, u_int64_t tid)
 {
 	struct hifn_softc *sc = device_get_softc(dev);
 	int session, error;
 	u_int32_t sid = CRYPTO_SESID2LID(tid);
 
 	KASSERT(sc != NULL, ("hifn_freesession: null softc"));
 	if (sc == NULL)
 		return (EINVAL);
 
 	HIFN_LOCK(sc);
 	session = HIFN_SESSION(sid);
 	if (session < sc->sc_nsessions) {
 		bzero(&sc->sc_sessions[session], sizeof(struct hifn_session));
 		error = 0;
 	} else
 		error = EINVAL;
 	HIFN_UNLOCK(sc);
 
 	return (error);
 }
 
 static int
 hifn_process(device_t dev, struct cryptop *crp, int hint)
 {
 	struct hifn_softc *sc = device_get_softc(dev);
 	struct hifn_command *cmd = NULL;
 	int session, err, ivlen;
 	struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
 
 	if (crp == NULL || crp->crp_callback == NULL) {
 		hifnstats.hst_invalid++;
 		return (EINVAL);
 	}
 	session = HIFN_SESSION(crp->crp_sid);
 
 	if (sc == NULL || session >= sc->sc_nsessions) {
 		err = EINVAL;
 		goto errout;
 	}
 
 	cmd = malloc(sizeof(struct hifn_command), M_DEVBUF, M_NOWAIT | M_ZERO);
 	if (cmd == NULL) {
 		hifnstats.hst_nomem++;
 		err = ENOMEM;
 		goto errout;
 	}
 
 	if (crp->crp_flags & CRYPTO_F_IMBUF) {
 		cmd->src_m = (struct mbuf *)crp->crp_buf;
 		cmd->dst_m = (struct mbuf *)crp->crp_buf;
 	} else if (crp->crp_flags & CRYPTO_F_IOV) {
 		cmd->src_io = (struct uio *)crp->crp_buf;
 		cmd->dst_io = (struct uio *)crp->crp_buf;
 	} else {
 		err = EINVAL;
 		goto errout;	/* XXX we don't handle contiguous buffers! */
 	}
 
 	crd1 = crp->crp_desc;
 	if (crd1 == NULL) {
 		err = EINVAL;
 		goto errout;
 	}
 	crd2 = crd1->crd_next;
 
 	if (crd2 == NULL) {
 		if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
 		    crd1->crd_alg == CRYPTO_SHA1_HMAC ||
 		    crd1->crd_alg == CRYPTO_SHA1 ||
 		    crd1->crd_alg == CRYPTO_MD5) {
 			maccrd = crd1;
 			enccrd = NULL;
 		} else if (crd1->crd_alg == CRYPTO_DES_CBC ||
 		    crd1->crd_alg == CRYPTO_3DES_CBC ||
 		    crd1->crd_alg == CRYPTO_AES_CBC ||
 		    crd1->crd_alg == CRYPTO_ARC4) {
 			if ((crd1->crd_flags & CRD_F_ENCRYPT) == 0)
 				cmd->base_masks |= HIFN_BASE_CMD_DECODE;
 			maccrd = NULL;
 			enccrd = crd1;
 		} else {
 			err = EINVAL;
 			goto errout;
 		}
 	} else {
 		if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
                      crd1->crd_alg == CRYPTO_SHA1_HMAC ||
                      crd1->crd_alg == CRYPTO_MD5 ||
                      crd1->crd_alg == CRYPTO_SHA1) &&
 		    (crd2->crd_alg == CRYPTO_DES_CBC ||
 		     crd2->crd_alg == CRYPTO_3DES_CBC ||
 		     crd2->crd_alg == CRYPTO_AES_CBC ||
 		     crd2->crd_alg == CRYPTO_ARC4) &&
 		    ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
 			cmd->base_masks = HIFN_BASE_CMD_DECODE;
 			maccrd = crd1;
 			enccrd = crd2;
 		} else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
 		     crd1->crd_alg == CRYPTO_ARC4 ||
 		     crd1->crd_alg == CRYPTO_3DES_CBC ||
 		     crd1->crd_alg == CRYPTO_AES_CBC) &&
 		    (crd2->crd_alg == CRYPTO_MD5_HMAC ||
                      crd2->crd_alg == CRYPTO_SHA1_HMAC ||
                      crd2->crd_alg == CRYPTO_MD5 ||
                      crd2->crd_alg == CRYPTO_SHA1) &&
 		    (crd1->crd_flags & CRD_F_ENCRYPT)) {
 			enccrd = crd1;
 			maccrd = crd2;
 		} else {
 			/*
 			 * We cannot order the 7751 as requested
 			 */
 			err = EINVAL;
 			goto errout;
 		}
 	}
 
 	if (enccrd) {
 		cmd->enccrd = enccrd;
 		cmd->base_masks |= HIFN_BASE_CMD_CRYPT;
 		switch (enccrd->crd_alg) {
 		case CRYPTO_ARC4:
 			cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_RC4;
 			break;
 		case CRYPTO_DES_CBC:
 			cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_DES |
 			    HIFN_CRYPT_CMD_MODE_CBC |
 			    HIFN_CRYPT_CMD_NEW_IV;
 			break;
 		case CRYPTO_3DES_CBC:
 			cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_3DES |
 			    HIFN_CRYPT_CMD_MODE_CBC |
 			    HIFN_CRYPT_CMD_NEW_IV;
 			break;
 		case CRYPTO_AES_CBC:
 			cmd->cry_masks |= HIFN_CRYPT_CMD_ALG_AES |
 			    HIFN_CRYPT_CMD_MODE_CBC |
 			    HIFN_CRYPT_CMD_NEW_IV;
 			break;
 		default:
 			err = EINVAL;
 			goto errout;
 		}
 		if (enccrd->crd_alg != CRYPTO_ARC4) {
 			ivlen = ((enccrd->crd_alg == CRYPTO_AES_CBC) ?
 				HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH);
 			if (enccrd->crd_flags & CRD_F_ENCRYPT) {
 				if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
 					bcopy(enccrd->crd_iv, cmd->iv, ivlen);
 				else
 					bcopy(sc->sc_sessions[session].hs_iv,
 					    cmd->iv, ivlen);
 
 				if ((enccrd->crd_flags & CRD_F_IV_PRESENT)
 				    == 0) {
 					crypto_copyback(crp->crp_flags,
 					    crp->crp_buf, enccrd->crd_inject,
 					    ivlen, cmd->iv);
 				}
 			} else {
 				if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
 					bcopy(enccrd->crd_iv, cmd->iv, ivlen);
 				else {
 					crypto_copydata(crp->crp_flags,
 					    crp->crp_buf, enccrd->crd_inject,
 					    ivlen, cmd->iv);
 				}
 			}
 		}
 
 		if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT)
 			cmd->cry_masks |= HIFN_CRYPT_CMD_NEW_KEY;
 		cmd->ck = enccrd->crd_key;
 		cmd->cklen = enccrd->crd_klen >> 3;
 		cmd->cry_masks |= HIFN_CRYPT_CMD_NEW_KEY;
 
 		/* 
 		 * Need to specify the size for the AES key in the masks.
 		 */
 		if ((cmd->cry_masks & HIFN_CRYPT_CMD_ALG_MASK) ==
 		    HIFN_CRYPT_CMD_ALG_AES) {
 			switch (cmd->cklen) {
 			case 16:
 				cmd->cry_masks |= HIFN_CRYPT_CMD_KSZ_128;
 				break;
 			case 24:
 				cmd->cry_masks |= HIFN_CRYPT_CMD_KSZ_192;
 				break;
 			case 32:
 				cmd->cry_masks |= HIFN_CRYPT_CMD_KSZ_256;
 				break;
 			default:
 				err = EINVAL;
 				goto errout;
 			}
 		}
 	}
 
 	if (maccrd) {
 		cmd->maccrd = maccrd;
 		cmd->base_masks |= HIFN_BASE_CMD_MAC;
 
 		switch (maccrd->crd_alg) {
 		case CRYPTO_MD5:
 			cmd->mac_masks |= HIFN_MAC_CMD_ALG_MD5 |
 			    HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HASH |
 			    HIFN_MAC_CMD_POS_IPSEC;
                        break;
 		case CRYPTO_MD5_HMAC:
 			cmd->mac_masks |= HIFN_MAC_CMD_ALG_MD5 |
 			    HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HMAC |
 			    HIFN_MAC_CMD_POS_IPSEC | HIFN_MAC_CMD_TRUNC;
 			break;
 		case CRYPTO_SHA1:
 			cmd->mac_masks |= HIFN_MAC_CMD_ALG_SHA1 |
 			    HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HASH |
 			    HIFN_MAC_CMD_POS_IPSEC;
 			break;
 		case CRYPTO_SHA1_HMAC:
 			cmd->mac_masks |= HIFN_MAC_CMD_ALG_SHA1 |
 			    HIFN_MAC_CMD_RESULT | HIFN_MAC_CMD_MODE_HMAC |
 			    HIFN_MAC_CMD_POS_IPSEC | HIFN_MAC_CMD_TRUNC;
 			break;
 		}
 
 		if (maccrd->crd_alg == CRYPTO_SHA1_HMAC ||
 		     maccrd->crd_alg == CRYPTO_MD5_HMAC) {
 			cmd->mac_masks |= HIFN_MAC_CMD_NEW_KEY;
 			bcopy(maccrd->crd_key, cmd->mac, maccrd->crd_klen >> 3);
 			bzero(cmd->mac + (maccrd->crd_klen >> 3),
 			    HIFN_MAC_KEY_LENGTH - (maccrd->crd_klen >> 3));
 		}
 	}
 
 	cmd->crp = crp;
 	cmd->session_num = session;
 	cmd->softc = sc;
 
 	err = hifn_crypto(sc, cmd, crp, hint);
 	if (!err) {
 		return 0;
 	} else if (err == ERESTART) {
 		/*
 		 * There weren't enough resources to dispatch the request
 		 * to the part.  Notify the caller so they'll requeue this
 		 * request and resubmit it again soon.
 		 */
 #ifdef HIFN_DEBUG
 		if (hifn_debug)
 			device_printf(sc->sc_dev, "requeue request\n");
 #endif
 		free(cmd, M_DEVBUF);
 		sc->sc_needwakeup |= CRYPTO_SYMQ;
 		return (err);
 	}
 
 errout:
 	if (cmd != NULL)
 		free(cmd, M_DEVBUF);
 	if (err == EINVAL)
 		hifnstats.hst_invalid++;
 	else
 		hifnstats.hst_nomem++;
 	crp->crp_etype = err;
 	crypto_done(crp);
 	return (err);
 }
 
 static void
 hifn_abort(struct hifn_softc *sc)
 {
 	struct hifn_dma *dma = sc->sc_dma;
 	struct hifn_command *cmd;
 	struct cryptop *crp;
 	int i, u;
 
 	i = sc->sc_resk; u = sc->sc_resu;
 	while (u != 0) {
 		cmd = sc->sc_hifn_commands[i];
 		KASSERT(cmd != NULL, ("hifn_abort: null command slot %u", i));
 		sc->sc_hifn_commands[i] = NULL;
 		crp = cmd->crp;
 
 		if ((dma->resr[i].l & htole32(HIFN_D_VALID)) == 0) {
 			/* Salvage what we can. */
 			u_int8_t *macbuf;
 
 			if (cmd->base_masks & HIFN_BASE_CMD_MAC) {
 				macbuf = dma->result_bufs[i];
 				macbuf += 12;
 			} else
 				macbuf = NULL;
 			hifnstats.hst_opackets++;
 			hifn_callback(sc, cmd, macbuf);
 		} else {
 			if (cmd->src_map == cmd->dst_map) {
 				bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
 				    BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
 			} else {
 				bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
 				    BUS_DMASYNC_POSTWRITE);
 				bus_dmamap_sync(sc->sc_dmat, cmd->dst_map,
 				    BUS_DMASYNC_POSTREAD);
 			}
 
 			if (cmd->src_m != cmd->dst_m) {
 				m_freem(cmd->src_m);
 				crp->crp_buf = (caddr_t)cmd->dst_m;
 			}
 
 			/* non-shared buffers cannot be restarted */
 			if (cmd->src_map != cmd->dst_map) {
 				/*
 				 * XXX should be EAGAIN, delayed until
 				 * after the reset.
 				 */
 				crp->crp_etype = ENOMEM;
 				bus_dmamap_unload(sc->sc_dmat, cmd->dst_map);
 				bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map);
 			} else
 				crp->crp_etype = ENOMEM;
 
 			bus_dmamap_unload(sc->sc_dmat, cmd->src_map);
 			bus_dmamap_destroy(sc->sc_dmat, cmd->src_map);
 
 			free(cmd, M_DEVBUF);
 			if (crp->crp_etype != EAGAIN)
 				crypto_done(crp);
 		}
 
 		if (++i == HIFN_D_RES_RSIZE)
 			i = 0;
 		u--;
 	}
 	sc->sc_resk = i; sc->sc_resu = u;
 
 	hifn_reset_board(sc, 1);
 	hifn_init_dma(sc);
 	hifn_init_pci_registers(sc);
 }
 
 static void
 hifn_callback(struct hifn_softc *sc, struct hifn_command *cmd, u_int8_t *macbuf)
 {
 	struct hifn_dma *dma = sc->sc_dma;
 	struct cryptop *crp = cmd->crp;
 	struct cryptodesc *crd;
 	struct mbuf *m;
 	int totlen, i, u, ivlen;
 
 	if (cmd->src_map == cmd->dst_map) {
 		bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
 		    BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD);
 	} else {
 		bus_dmamap_sync(sc->sc_dmat, cmd->src_map,
 		    BUS_DMASYNC_POSTWRITE);
 		bus_dmamap_sync(sc->sc_dmat, cmd->dst_map,
 		    BUS_DMASYNC_POSTREAD);
 	}
 
 	if (crp->crp_flags & CRYPTO_F_IMBUF) {
 		if (cmd->src_m != cmd->dst_m) {
 			crp->crp_buf = (caddr_t)cmd->dst_m;
 			totlen = cmd->src_mapsize;
 			for (m = cmd->dst_m; m != NULL; m = m->m_next) {
 				if (totlen < m->m_len) {
 					m->m_len = totlen;
 					totlen = 0;
 				} else
 					totlen -= m->m_len;
 			}
 			cmd->dst_m->m_pkthdr.len = cmd->src_m->m_pkthdr.len;
 			m_freem(cmd->src_m);
 		}
 	}
 
 	if (cmd->sloplen != 0) {
 		crypto_copyback(crp->crp_flags, crp->crp_buf,
 		    cmd->src_mapsize - cmd->sloplen, cmd->sloplen,
 		    (caddr_t)&dma->slop[cmd->slopidx]);
 	}
 
 	i = sc->sc_dstk; u = sc->sc_dstu;
 	while (u != 0) {
 		if (i == HIFN_D_DST_RSIZE)
 			i = 0;
 		bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
 		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 		if (dma->dstr[i].l & htole32(HIFN_D_VALID)) {
 			bus_dmamap_sync(sc->sc_dmat, sc->sc_dmamap,
 			    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 			break;
 		}
 		i++, u--;
 	}
 	sc->sc_dstk = i; sc->sc_dstu = u;
 
 	hifnstats.hst_obytes += cmd->dst_mapsize;
 
 	if ((cmd->base_masks & (HIFN_BASE_CMD_CRYPT | HIFN_BASE_CMD_DECODE)) ==
 	    HIFN_BASE_CMD_CRYPT) {
 		for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
 			if (crd->crd_alg != CRYPTO_DES_CBC &&
 			    crd->crd_alg != CRYPTO_3DES_CBC &&
 			    crd->crd_alg != CRYPTO_AES_CBC)
 				continue;
 			ivlen = ((crd->crd_alg == CRYPTO_AES_CBC) ?
 				HIFN_AES_IV_LENGTH : HIFN_IV_LENGTH);
 			crypto_copydata(crp->crp_flags, crp->crp_buf,
 			    crd->crd_skip + crd->crd_len - ivlen, ivlen,
 			    cmd->softc->sc_sessions[cmd->session_num].hs_iv);
 			break;
 		}
 	}
 
 	if (macbuf != NULL) {
 		for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
                         int len;
 
 			if (crd->crd_alg != CRYPTO_MD5 &&
 			    crd->crd_alg != CRYPTO_SHA1 &&
 			    crd->crd_alg != CRYPTO_MD5_HMAC &&
 			    crd->crd_alg != CRYPTO_SHA1_HMAC) {
 				continue;
 			}
 			len = cmd->softc->sc_sessions[cmd->session_num].hs_mlen;
 			crypto_copyback(crp->crp_flags, crp->crp_buf,
 			    crd->crd_inject, len, macbuf);
 			break;
 		}
 	}
 
 	if (cmd->src_map != cmd->dst_map) {
 		bus_dmamap_unload(sc->sc_dmat, cmd->dst_map);
 		bus_dmamap_destroy(sc->sc_dmat, cmd->dst_map);
 	}
 	bus_dmamap_unload(sc->sc_dmat, cmd->src_map);
 	bus_dmamap_destroy(sc->sc_dmat, cmd->src_map);
 	free(cmd, M_DEVBUF);
 	crypto_done(crp);
 }
 
 /*
  * 7811 PB3 rev/2 parts lock-up on burst writes to Group 0
  * and Group 1 registers; avoid conditions that could create
  * burst writes by doing a read in between the writes.
  *
  * NB: The read we interpose is always to the same register;
  *     we do this because reading from an arbitrary (e.g. last)
  *     register may not always work.
  */
 static void
 hifn_write_reg_0(struct hifn_softc *sc, bus_size_t reg, u_int32_t val)
 {
 	if (sc->sc_flags & HIFN_IS_7811) {
 		if (sc->sc_bar0_lastreg == reg - 4)
 			bus_space_read_4(sc->sc_st0, sc->sc_sh0, HIFN_0_PUCNFG);
 		sc->sc_bar0_lastreg = reg;
 	}
 	bus_space_write_4(sc->sc_st0, sc->sc_sh0, reg, val);
 }
 
 static void
 hifn_write_reg_1(struct hifn_softc *sc, bus_size_t reg, u_int32_t val)
 {
 	if (sc->sc_flags & HIFN_IS_7811) {
 		if (sc->sc_bar1_lastreg == reg - 4)
 			bus_space_read_4(sc->sc_st1, sc->sc_sh1, HIFN_1_REVID);
 		sc->sc_bar1_lastreg = reg;
 	}
 	bus_space_write_4(sc->sc_st1, sc->sc_sh1, reg, val);
 }
 
 #ifdef HIFN_VULCANDEV
 /*
  * this code provides support for mapping the PK engine's register
  * into a userspace program.
  *
  */
 static int
 vulcanpk_mmap(struct cdev *dev, vm_ooffset_t offset,
 	      vm_paddr_t *paddr, int nprot, vm_memattr_t *memattr)
 {
 	struct hifn_softc *sc;
 	vm_paddr_t pd;
 	void *b;
 
 	sc = dev->si_drv1;
 
 	pd = rman_get_start(sc->sc_bar1res);
 	b = rman_get_virtual(sc->sc_bar1res);
 
 #if 0
 	printf("vpk mmap: %p(%016llx) offset=%lld\n", b,
 	    (unsigned long long)pd, offset);
 	hexdump(b, HIFN_1_PUB_MEMEND, "vpk", 0);
 #endif
 
 	if (offset == 0) {
 		*paddr = pd;
 		return (0);
 	}
 	return (-1);
 }
 
 static struct cdevsw vulcanpk_cdevsw = {
 	.d_version =	D_VERSION,
 	.d_mmap =	vulcanpk_mmap,
 	.d_name =	"vulcanpk",
 };
 #endif /* HIFN_VULCANDEV */
Index: projects/release-debugdist/sys/dev/malo/if_malo.c
===================================================================
--- projects/release-debugdist/sys/dev/malo/if_malo.c	(revision 267343)
+++ projects/release-debugdist/sys/dev/malo/if_malo.c	(revision 267344)
@@ -1,2292 +1,2282 @@
 /*-
  * Copyright (c) 2008 Weongyo Jeong <weongyo@freebsd.org>
  * Copyright (c) 2007 Marvell Semiconductor, Inc.
  * Copyright (c) 2007 Sam Leffler, Errno Consulting
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer,
  *    without modification.
  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  *    similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any
  *    redistribution must be conditioned upon including a substantially
  *    similar Disclaimer requirement for further binary redistribution.
  *
  * NO WARRANTY
  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY
  * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
  * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY,
  * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
  * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  * THE POSSIBILITY OF SUCH DAMAGES.
  */
 
 #include <sys/cdefs.h>
 #ifdef __FreeBSD__
 __FBSDID("$FreeBSD$");
 #endif
 
 #include "opt_malo.h"
 
 #include <sys/param.h>
 #include <sys/endian.h>
 #include <sys/kernel.h>
 #include <sys/socket.h>
 #include <sys/sockio.h>
 #include <sys/sysctl.h>
 #include <sys/taskqueue.h>
 
 #include <machine/bus.h>
 #include <sys/bus.h>
 
 #include <net/if.h>
 #include <net/if_var.h>
 #include <net/if_dl.h>
 #include <net/if_media.h>
 #include <net/if_types.h>
 #include <net/ethernet.h>
 
 #include <net80211/ieee80211_var.h>
 #include <net80211/ieee80211_regdomain.h>
 
 #include <net/bpf.h>
 
 #include <dev/malo/if_malo.h>
 
 SYSCTL_NODE(_hw, OID_AUTO, malo, CTLFLAG_RD, 0,
     "Marvell 88w8335 driver parameters");
 
 static	int malo_txcoalesce = 8;	/* # tx pkts to q before poking f/w*/
 SYSCTL_INT(_hw_malo, OID_AUTO, txcoalesce, CTLFLAG_RW, &malo_txcoalesce,
 	    0, "tx buffers to send at once");
 TUNABLE_INT("hw.malo.txcoalesce", &malo_txcoalesce);
 static	int malo_rxbuf = MALO_RXBUF;		/* # rx buffers to allocate */
 SYSCTL_INT(_hw_malo, OID_AUTO, rxbuf, CTLFLAG_RW, &malo_rxbuf,
 	    0, "rx buffers allocated");
 TUNABLE_INT("hw.malo.rxbuf", &malo_rxbuf);
 static	int malo_rxquota = MALO_RXBUF;		/* # max buffers to process */
 SYSCTL_INT(_hw_malo, OID_AUTO, rxquota, CTLFLAG_RW, &malo_rxquota,
 	    0, "max rx buffers to process per interrupt");
 TUNABLE_INT("hw.malo.rxquota", &malo_rxquota);
 static	int malo_txbuf = MALO_TXBUF;		/* # tx buffers to allocate */
 SYSCTL_INT(_hw_malo, OID_AUTO, txbuf, CTLFLAG_RW, &malo_txbuf,
 	    0, "tx buffers allocated");
 TUNABLE_INT("hw.malo.txbuf", &malo_txbuf);
 
 #ifdef MALO_DEBUG
 static	int malo_debug = 0;
 SYSCTL_INT(_hw_malo, OID_AUTO, debug, CTLFLAG_RW, &malo_debug,
 	    0, "control debugging printfs");
 TUNABLE_INT("hw.malo.debug", &malo_debug);
 enum {
 	MALO_DEBUG_XMIT		= 0x00000001,	/* basic xmit operation */
 	MALO_DEBUG_XMIT_DESC	= 0x00000002,	/* xmit descriptors */
 	MALO_DEBUG_RECV		= 0x00000004,	/* basic recv operation */
 	MALO_DEBUG_RECV_DESC	= 0x00000008,	/* recv descriptors */
 	MALO_DEBUG_RESET	= 0x00000010,	/* reset processing */
 	MALO_DEBUG_INTR		= 0x00000040,	/* ISR */
 	MALO_DEBUG_TX_PROC	= 0x00000080,	/* tx ISR proc */
 	MALO_DEBUG_RX_PROC	= 0x00000100,	/* rx ISR proc */
 	MALO_DEBUG_STATE	= 0x00000400,	/* 802.11 state transitions */
 	MALO_DEBUG_NODE		= 0x00000800,	/* node management */
 	MALO_DEBUG_RECV_ALL	= 0x00001000,	/* trace all frames (beacons) */
 	MALO_DEBUG_FW		= 0x00008000,	/* firmware */
 	MALO_DEBUG_ANY		= 0xffffffff
 };
 #define	IS_BEACON(wh)							\
 	((wh->i_fc[0] & (IEEE80211_FC0_TYPE_MASK |			\
 		IEEE80211_FC0_SUBTYPE_MASK)) ==				\
 	 (IEEE80211_FC0_TYPE_MGT|IEEE80211_FC0_SUBTYPE_BEACON))
 #define	IFF_DUMPPKTS_RECV(sc, wh)					\
 	(((sc->malo_debug & MALO_DEBUG_RECV) &&				\
 	  ((sc->malo_debug & MALO_DEBUG_RECV_ALL) || !IS_BEACON(wh))) || \
 	 (sc->malo_ifp->if_flags & (IFF_DEBUG|IFF_LINK2)) ==		\
 	  (IFF_DEBUG|IFF_LINK2))
 #define	IFF_DUMPPKTS_XMIT(sc)						\
 	((sc->malo_debug & MALO_DEBUG_XMIT) ||				\
 	 (sc->malo_ifp->if_flags & (IFF_DEBUG | IFF_LINK2)) ==		\
 	     (IFF_DEBUG | IFF_LINK2))
 #define	DPRINTF(sc, m, fmt, ...) do {				\
 	if (sc->malo_debug & (m))				\
 		printf(fmt, __VA_ARGS__);			\
 } while (0)
 #else
 #define	DPRINTF(sc, m, fmt, ...) do {				\
 	(void) sc;						\
 } while (0)
 #endif
 
 static MALLOC_DEFINE(M_MALODEV, "malodev", "malo driver dma buffers");
 
 static struct ieee80211vap *malo_vap_create(struct ieee80211com *,
 		    const char [IFNAMSIZ], int, enum ieee80211_opmode, int,
 		    const uint8_t [IEEE80211_ADDR_LEN],
 		    const uint8_t [IEEE80211_ADDR_LEN]);
 static  void	malo_vap_delete(struct ieee80211vap *);
 static	int	malo_dma_setup(struct malo_softc *);
 static	int	malo_setup_hwdma(struct malo_softc *);
 static	void	malo_txq_init(struct malo_softc *, struct malo_txq *, int);
 static	void	malo_tx_cleanupq(struct malo_softc *, struct malo_txq *);
 static	void	malo_start(struct ifnet *);
 static	void	malo_watchdog(void *);
 static	int	malo_ioctl(struct ifnet *, u_long, caddr_t);
 static	void	malo_updateslot(struct ifnet *);
 static	int	malo_newstate(struct ieee80211vap *, enum ieee80211_state, int);
 static	void	malo_scan_start(struct ieee80211com *);
 static	void	malo_scan_end(struct ieee80211com *);
 static	void	malo_set_channel(struct ieee80211com *);
 static	int	malo_raw_xmit(struct ieee80211_node *, struct mbuf *,
 		    const struct ieee80211_bpf_params *);
 static	void	malo_sysctlattach(struct malo_softc *);
 static	void	malo_announce(struct malo_softc *);
 static	void	malo_dma_cleanup(struct malo_softc *);
 static	void	malo_stop_locked(struct ifnet *, int);
 static	int	malo_chan_set(struct malo_softc *, struct ieee80211_channel *);
 static	int	malo_mode_init(struct malo_softc *);
 static	void	malo_tx_proc(void *, int);
 static	void	malo_rx_proc(void *, int);
 static	void	malo_init(void *);
 
 /*
  * Read/Write shorthands for accesses to BAR 0.  Note that all BAR 1
  * operations are done in the "hal" except getting H/W MAC address at
  * malo_attach and there should be no reference to them here.
  */
 static uint32_t
 malo_bar0_read4(struct malo_softc *sc, bus_size_t off)
 {
 	return bus_space_read_4(sc->malo_io0t, sc->malo_io0h, off);
 }
 
 static void
 malo_bar0_write4(struct malo_softc *sc, bus_size_t off, uint32_t val)
 {
 	DPRINTF(sc, MALO_DEBUG_FW, "%s: off 0x%jx val 0x%x\n",
 	    __func__, (intmax_t)off, val);
 
 	bus_space_write_4(sc->malo_io0t, sc->malo_io0h, off, val);
 }
 
 int
 malo_attach(uint16_t devid, struct malo_softc *sc)
 {
 	int error;
 	struct ieee80211com *ic;
 	struct ifnet *ifp;
 	struct malo_hal *mh;
 	uint8_t bands;
 
 	ifp = sc->malo_ifp = if_alloc(IFT_IEEE80211);
 	if (ifp == NULL) {
 		device_printf(sc->malo_dev, "can not if_alloc()\n");
 		return ENOSPC;
 	}
 	ic = ifp->if_l2com;
 
 	MALO_LOCK_INIT(sc);
 	callout_init_mtx(&sc->malo_watchdog_timer, &sc->malo_mtx, 0);
 
 	/* set these up early for if_printf use */
 	if_initname(ifp, device_get_name(sc->malo_dev),
 	    device_get_unit(sc->malo_dev));
 
 	mh = malo_hal_attach(sc->malo_dev, devid,
 	    sc->malo_io1h, sc->malo_io1t, sc->malo_dmat);
 	if (mh == NULL) {
 		if_printf(ifp, "unable to attach HAL\n");
 		error = EIO;
 		goto bad;
 	}
 	sc->malo_mh = mh;
 
 	/*
 	 * Load firmware so we can get setup.  We arbitrarily pick station
 	 * firmware; we'll re-load firmware as needed so setting up
 	 * the wrong mode isn't a big deal.
 	 */
 	error = malo_hal_fwload(mh, "malo8335-h", "malo8335-m");
 	if (error != 0) {
 		if_printf(ifp, "unable to setup firmware\n");
 		goto bad1;
 	}
 	/* XXX gethwspecs() extracts correct informations?  not maybe!  */
 	error = malo_hal_gethwspecs(mh, &sc->malo_hwspecs);
 	if (error != 0) {
 		if_printf(ifp, "unable to fetch h/w specs\n");
 		goto bad1;
 	}
 
 	DPRINTF(sc, MALO_DEBUG_FW,
 	    "malo_hal_gethwspecs: hwversion 0x%x hostif 0x%x"
 	    "maxnum_wcb 0x%x maxnum_mcaddr 0x%x maxnum_tx_wcb 0x%x"
 	    "regioncode 0x%x num_antenna 0x%x fw_releasenum 0x%x"
 	    "wcbbase0 0x%x rxdesc_read 0x%x rxdesc_write 0x%x"
 	    "ul_fw_awakecookie 0x%x w[4] = %x %x %x %x",
 	    sc->malo_hwspecs.hwversion,
 	    sc->malo_hwspecs.hostinterface, sc->malo_hwspecs.maxnum_wcb,
 	    sc->malo_hwspecs.maxnum_mcaddr, sc->malo_hwspecs.maxnum_tx_wcb,
 	    sc->malo_hwspecs.regioncode, sc->malo_hwspecs.num_antenna,
 	    sc->malo_hwspecs.fw_releasenum, sc->malo_hwspecs.wcbbase0,
 	    sc->malo_hwspecs.rxdesc_read, sc->malo_hwspecs.rxdesc_write,
 	    sc->malo_hwspecs.ul_fw_awakecookie,
 	    sc->malo_hwspecs.wcbbase[0], sc->malo_hwspecs.wcbbase[1],
 	    sc->malo_hwspecs.wcbbase[2], sc->malo_hwspecs.wcbbase[3]);
 
 	/* NB: firmware looks that it does not export regdomain info API.  */
 	bands = 0;
 	setbit(&bands, IEEE80211_MODE_11B);
 	setbit(&bands, IEEE80211_MODE_11G);
 	ieee80211_init_channels(ic, NULL, &bands);
 
 	sc->malo_txantenna = 0x2;	/* h/w default */
 	sc->malo_rxantenna = 0xffff;	/* h/w default */
 
 	/*
 	 * Allocate tx + rx descriptors and populate the lists.
 	 * We immediately push the information to the firmware
 	 * as otherwise it gets upset.
 	 */
 	error = malo_dma_setup(sc);
 	if (error != 0) {
 		if_printf(ifp, "failed to setup descriptors: %d\n", error);
 		goto bad1;
 	}
 	error = malo_setup_hwdma(sc);	/* push to firmware */
 	if (error != 0)			/* NB: malo_setupdma prints msg */
 		goto bad2;
 
 	sc->malo_tq = taskqueue_create_fast("malo_taskq", M_NOWAIT,
 		taskqueue_thread_enqueue, &sc->malo_tq);
 	taskqueue_start_threads(&sc->malo_tq, 1, PI_NET,
 		"%s taskq", ifp->if_xname);
 
 	TASK_INIT(&sc->malo_rxtask, 0, malo_rx_proc, sc);
 	TASK_INIT(&sc->malo_txtask, 0, malo_tx_proc, sc);
 
 	ifp->if_softc = sc;
 	ifp->if_flags = IFF_SIMPLEX | IFF_BROADCAST | IFF_MULTICAST;
 	ifp->if_start = malo_start;
 	ifp->if_ioctl = malo_ioctl;
 	ifp->if_init = malo_init;
 	IFQ_SET_MAXLEN(&ifp->if_snd, ifqmaxlen);
 	ifp->if_snd.ifq_drv_maxlen = ifqmaxlen;
 	IFQ_SET_READY(&ifp->if_snd);
 
 	ic->ic_ifp = ifp;
 	/* XXX not right but it's not used anywhere important */
 	ic->ic_phytype = IEEE80211_T_OFDM;
 	ic->ic_opmode = IEEE80211_M_STA;
 	ic->ic_caps =
 	      IEEE80211_C_STA			/* station mode supported */
 	    | IEEE80211_C_BGSCAN		/* capable of bg scanning */
 	    | IEEE80211_C_MONITOR		/* monitor mode */
 	    | IEEE80211_C_SHPREAMBLE		/* short preamble supported */
 	    | IEEE80211_C_SHSLOT		/* short slot time supported */
 	    | IEEE80211_C_TXPMGT		/* capable of txpow mgt */
 	    | IEEE80211_C_WPA			/* capable of WPA1+WPA2 */
 	    ;
 
 	/*
 	 * Transmit requires space in the packet for a special format transmit
 	 * record and optional padding between this record and the payload.
 	 * Ask the net80211 layer to arrange this when encapsulating
 	 * packets so we can add it efficiently. 
 	 */
 	ic->ic_headroom = sizeof(struct malo_txrec) -
 		sizeof(struct ieee80211_frame);
 
 	/* call MI attach routine. */
 	ieee80211_ifattach(ic, sc->malo_hwspecs.macaddr);
 	/* override default methods */
 	ic->ic_vap_create = malo_vap_create;
 	ic->ic_vap_delete = malo_vap_delete;
 	ic->ic_raw_xmit = malo_raw_xmit;
 	ic->ic_updateslot = malo_updateslot;
 
 	ic->ic_scan_start = malo_scan_start;
 	ic->ic_scan_end = malo_scan_end;
 	ic->ic_set_channel = malo_set_channel;
 
 	sc->malo_invalid = 0;		/* ready to go, enable int handling */
 
 	ieee80211_radiotap_attach(ic,
 	    &sc->malo_tx_th.wt_ihdr, sizeof(sc->malo_tx_th),
 		MALO_TX_RADIOTAP_PRESENT,
 	    &sc->malo_rx_th.wr_ihdr, sizeof(sc->malo_rx_th),
 		MALO_RX_RADIOTAP_PRESENT);
 
 	/*
 	 * Setup dynamic sysctl's.
 	 */
 	malo_sysctlattach(sc);
 
 	if (bootverbose)
 		ieee80211_announce(ic);
 	malo_announce(sc);
 
 	return 0;
 bad2:
 	malo_dma_cleanup(sc);
 bad1:
 	malo_hal_detach(mh);
 bad:
 	if_free(ifp);
 	sc->malo_invalid = 1;
 
 	return error;
 }
 
 static struct ieee80211vap *
 malo_vap_create(struct ieee80211com *ic, const char name[IFNAMSIZ], int unit,
     enum ieee80211_opmode opmode, int flags,
     const uint8_t bssid[IEEE80211_ADDR_LEN],
     const uint8_t mac[IEEE80211_ADDR_LEN])
 {
 	struct ifnet *ifp = ic->ic_ifp;
 	struct malo_vap *mvp;
 	struct ieee80211vap *vap;
 
 	if (!TAILQ_EMPTY(&ic->ic_vaps)) {
 		if_printf(ifp, "multiple vaps not supported\n");
 		return NULL;
 	}
 	switch (opmode) {
 	case IEEE80211_M_STA:
 		if (opmode == IEEE80211_M_STA)
 			flags |= IEEE80211_CLONE_NOBEACONS;
 		/* fall thru... */
 	case IEEE80211_M_MONITOR:
 		break;
 	default:
 		if_printf(ifp, "%s mode not supported\n",
 		    ieee80211_opmode_name[opmode]);
 		return NULL;		/* unsupported */
 	}
 	mvp = (struct malo_vap *) malloc(sizeof(struct malo_vap),
 	    M_80211_VAP, M_NOWAIT | M_ZERO);
 	if (mvp == NULL) {
 		if_printf(ifp, "cannot allocate vap state block\n");
 		return NULL;
 	}
 	vap = &mvp->malo_vap;
 	ieee80211_vap_setup(ic, vap, name, unit, opmode, flags, bssid, mac);
 
 	/* override state transition machine */
 	mvp->malo_newstate = vap->iv_newstate;
 	vap->iv_newstate = malo_newstate;
 
 	/* complete setup */
 	ieee80211_vap_attach(vap,
 	    ieee80211_media_change, ieee80211_media_status);
 	ic->ic_opmode = opmode;
 	return vap;
 }
 
 static void
 malo_vap_delete(struct ieee80211vap *vap)
 {
 	struct malo_vap *mvp = MALO_VAP(vap);
 
 	ieee80211_vap_detach(vap);
 	free(mvp, M_80211_VAP);
 }
 
 int
 malo_intr(void *arg)
 {
 	struct malo_softc *sc = arg;
 	struct malo_hal *mh = sc->malo_mh;
 	uint32_t status;
 
 	if (sc->malo_invalid) {
 		/*
 		 * The hardware is not ready/present, don't touch anything.
 		 * Note this can happen early on if the IRQ is shared.
 		 */
 		DPRINTF(sc, MALO_DEBUG_ANY, "%s: invalid; ignored\n", __func__);
 		return (FILTER_STRAY);
 	}
 
 	/*
 	 * Figure out the reason(s) for the interrupt.
 	 */
 	malo_hal_getisr(mh, &status);		/* NB: clears ISR too */
 	if (status == 0)			/* must be a shared irq */
 		return (FILTER_STRAY);
 
 	DPRINTF(sc, MALO_DEBUG_INTR, "%s: status 0x%x imask 0x%x\n",
 	    __func__, status, sc->malo_imask);
 
 	if (status & MALO_A2HRIC_BIT_RX_RDY)
 		taskqueue_enqueue_fast(sc->malo_tq, &sc->malo_rxtask);
 	if (status & MALO_A2HRIC_BIT_TX_DONE)
 		taskqueue_enqueue_fast(sc->malo_tq, &sc->malo_txtask);
 	if (status & MALO_A2HRIC_BIT_OPC_DONE)
 		malo_hal_cmddone(mh);
 	if (status & MALO_A2HRIC_BIT_MAC_EVENT)
 		;
 	if (status & MALO_A2HRIC_BIT_RX_PROBLEM)
 		;
 	if (status & MALO_A2HRIC_BIT_ICV_ERROR) {
 		/* TKIP ICV error */
 		sc->malo_stats.mst_rx_badtkipicv++;
 	}
 #ifdef MALO_DEBUG
 	if (((status | sc->malo_imask) ^ sc->malo_imask) != 0)
 		DPRINTF(sc, MALO_DEBUG_INTR,
 		    "%s: can't handle interrupt status 0x%x\n",
 		    __func__, status);
 #endif
 	return (FILTER_HANDLED);
 }
 
 static void
 malo_load_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
 {
 	bus_addr_t *paddr = (bus_addr_t*) arg;
 
 	KASSERT(error == 0, ("error %u on bus_dma callback", error));
 
 	*paddr = segs->ds_addr;
 }
 
 static int
 malo_desc_setup(struct malo_softc *sc, const char *name,
     struct malo_descdma *dd,
     int nbuf, size_t bufsize, int ndesc, size_t descsize)
 {
 	int error;
 	struct ifnet *ifp = sc->malo_ifp;
 	uint8_t *ds;
 
 	DPRINTF(sc, MALO_DEBUG_RESET,
 	    "%s: %s DMA: %u bufs (%ju) %u desc/buf (%ju)\n",
 	    __func__, name, nbuf, (uintmax_t) bufsize,
 	    ndesc, (uintmax_t) descsize);
 	
 	dd->dd_name = name;
 	dd->dd_desc_len = nbuf * ndesc * descsize;
 
 	/*
 	 * Setup DMA descriptor area.
 	 */
 	error = bus_dma_tag_create(bus_get_dma_tag(sc->malo_dev),/* parent */
 		       PAGE_SIZE, 0,		/* alignment, bounds */
 		       BUS_SPACE_MAXADDR_32BIT,	/* lowaddr */
 		       BUS_SPACE_MAXADDR,	/* highaddr */
 		       NULL, NULL,		/* filter, filterarg */
 		       dd->dd_desc_len,		/* maxsize */
 		       1,			/* nsegments */
 		       dd->dd_desc_len,		/* maxsegsize */
 		       BUS_DMA_ALLOCNOW,	/* flags */
 		       NULL,			/* lockfunc */
 		       NULL,			/* lockarg */
 		       &dd->dd_dmat);
 	if (error != 0) {
 		if_printf(ifp, "cannot allocate %s DMA tag\n", dd->dd_name);
 		return error;
 	}
 	
 	/* allocate descriptors */
-	error = bus_dmamap_create(dd->dd_dmat, BUS_DMA_NOWAIT, &dd->dd_dmamap);
-	if (error != 0) {
-		if_printf(ifp, "unable to create dmamap for %s descriptors, "
-		    "error %u\n", dd->dd_name, error);
-		goto fail0;
-	}
-	
 	error = bus_dmamem_alloc(dd->dd_dmat, (void**) &dd->dd_desc,
 	    BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &dd->dd_dmamap);
 	if (error != 0) {
 		if_printf(ifp, "unable to alloc memory for %u %s descriptors, "
 		    "error %u\n", nbuf * ndesc, dd->dd_name, error);
 		goto fail1;
 	}
 
 	error = bus_dmamap_load(dd->dd_dmat, dd->dd_dmamap,
 	    dd->dd_desc, dd->dd_desc_len,
 	    malo_load_cb, &dd->dd_desc_paddr, BUS_DMA_NOWAIT);
 	if (error != 0) {
 		if_printf(ifp, "unable to map %s descriptors, error %u\n",
 		    dd->dd_name, error);
 		goto fail2;
 	}
 	
 	ds = dd->dd_desc;
 	memset(ds, 0, dd->dd_desc_len);
 	DPRINTF(sc, MALO_DEBUG_RESET, "%s: %s DMA map: %p (%lu) -> %p (%lu)\n",
 	    __func__, dd->dd_name, ds, (u_long) dd->dd_desc_len,
 	    (caddr_t) dd->dd_desc_paddr, /*XXX*/ (u_long) dd->dd_desc_len);
 
 	return 0;
 fail2:
 	bus_dmamem_free(dd->dd_dmat, dd->dd_desc, dd->dd_dmamap);
 fail1:
-	bus_dmamap_destroy(dd->dd_dmat, dd->dd_dmamap);
-fail0:
 	bus_dma_tag_destroy(dd->dd_dmat);
 	memset(dd, 0, sizeof(*dd));
 	return error;
 }
 
 #define	DS2PHYS(_dd, _ds) \
 	((_dd)->dd_desc_paddr + ((caddr_t)(_ds) - (caddr_t)(_dd)->dd_desc))
 
 static int
 malo_rxdma_setup(struct malo_softc *sc)
 {
 	struct ifnet *ifp = sc->malo_ifp;
 	int error, bsize, i;
 	struct malo_rxbuf *bf;
 	struct malo_rxdesc *ds;
 
 	error = malo_desc_setup(sc, "rx", &sc->malo_rxdma,
 	    malo_rxbuf, sizeof(struct malo_rxbuf),
 	    1, sizeof(struct malo_rxdesc));
 	if (error != 0)
 		return error;
 
 	/*
 	 * Allocate rx buffers and set them up.
 	 */
 	bsize = malo_rxbuf * sizeof(struct malo_rxbuf);
 	bf = malloc(bsize, M_MALODEV, M_NOWAIT | M_ZERO);
 	if (bf == NULL) {
 		if_printf(ifp, "malloc of %u rx buffers failed\n", bsize);
 		return error;
 	}
 	sc->malo_rxdma.dd_bufptr = bf;
 	
 	STAILQ_INIT(&sc->malo_rxbuf);
 	ds = sc->malo_rxdma.dd_desc;
 	for (i = 0; i < malo_rxbuf; i++, bf++, ds++) {
 		bf->bf_desc = ds;
 		bf->bf_daddr = DS2PHYS(&sc->malo_rxdma, ds);
 		error = bus_dmamap_create(sc->malo_dmat, BUS_DMA_NOWAIT,
 		    &bf->bf_dmamap);
 		if (error != 0) {
 			if_printf(ifp, "%s: unable to dmamap for rx buffer, "
 			    "error %d\n", __func__, error);
 			return error;
 		}
 		/* NB: tail is intentional to preserve descriptor order */
 		STAILQ_INSERT_TAIL(&sc->malo_rxbuf, bf, bf_list);
 	}
 	return 0;
 }
 
 static int
 malo_txdma_setup(struct malo_softc *sc, struct malo_txq *txq)
 {
 	struct ifnet *ifp = sc->malo_ifp;
 	int error, bsize, i;
 	struct malo_txbuf *bf;
 	struct malo_txdesc *ds;
 
 	error = malo_desc_setup(sc, "tx", &txq->dma,
 	    malo_txbuf, sizeof(struct malo_txbuf),
 	    MALO_TXDESC, sizeof(struct malo_txdesc));
 	if (error != 0)
 		return error;
 	
 	/* allocate and setup tx buffers */
 	bsize = malo_txbuf * sizeof(struct malo_txbuf);
 	bf = malloc(bsize, M_MALODEV, M_NOWAIT | M_ZERO);
 	if (bf == NULL) {
 		if_printf(ifp, "malloc of %u tx buffers failed\n",
 		    malo_txbuf);
 		return ENOMEM;
 	}
 	txq->dma.dd_bufptr = bf;
 	
 	STAILQ_INIT(&txq->free);
 	txq->nfree = 0;
 	ds = txq->dma.dd_desc;
 	for (i = 0; i < malo_txbuf; i++, bf++, ds += MALO_TXDESC) {
 		bf->bf_desc = ds;
 		bf->bf_daddr = DS2PHYS(&txq->dma, ds);
 		error = bus_dmamap_create(sc->malo_dmat, BUS_DMA_NOWAIT,
 		    &bf->bf_dmamap);
 		if (error != 0) {
 			if_printf(ifp, "unable to create dmamap for tx "
 			    "buffer %u, error %u\n", i, error);
 			return error;
 		}
 		STAILQ_INSERT_TAIL(&txq->free, bf, bf_list);
 		txq->nfree++;
 	}
 
 	return 0;
 }
 
 static void
 malo_desc_cleanup(struct malo_softc *sc, struct malo_descdma *dd)
 {
 	bus_dmamap_unload(dd->dd_dmat, dd->dd_dmamap);
 	bus_dmamem_free(dd->dd_dmat, dd->dd_desc, dd->dd_dmamap);
-	bus_dmamap_destroy(dd->dd_dmat, dd->dd_dmamap);
 	bus_dma_tag_destroy(dd->dd_dmat);
 
 	memset(dd, 0, sizeof(*dd));
 }
 
 static void
 malo_rxdma_cleanup(struct malo_softc *sc)
 {
 	struct malo_rxbuf *bf;
 
 	STAILQ_FOREACH(bf, &sc->malo_rxbuf, bf_list) {
 		if (bf->bf_m != NULL) {
 			m_freem(bf->bf_m);
 			bf->bf_m = NULL;
 		}
 		if (bf->bf_dmamap != NULL) {
 			bus_dmamap_destroy(sc->malo_dmat, bf->bf_dmamap);
 			bf->bf_dmamap = NULL;
 		}
 	}
 	STAILQ_INIT(&sc->malo_rxbuf);
 	if (sc->malo_rxdma.dd_bufptr != NULL) {
 		free(sc->malo_rxdma.dd_bufptr, M_MALODEV);
 		sc->malo_rxdma.dd_bufptr = NULL;
 	}
 	if (sc->malo_rxdma.dd_desc_len != 0)
 		malo_desc_cleanup(sc, &sc->malo_rxdma);
 }
 
 static void
 malo_txdma_cleanup(struct malo_softc *sc, struct malo_txq *txq)
 {
 	struct malo_txbuf *bf;
 	struct ieee80211_node *ni;
 
 	STAILQ_FOREACH(bf, &txq->free, bf_list) {
 		if (bf->bf_m != NULL) {
 			m_freem(bf->bf_m);
 			bf->bf_m = NULL;
 		}
 		ni = bf->bf_node;
 		bf->bf_node = NULL;
 		if (ni != NULL) {
 			/*
 			 * Reclaim node reference.
 			 */
 			ieee80211_free_node(ni);
 		}
 		if (bf->bf_dmamap != NULL) {
 			bus_dmamap_destroy(sc->malo_dmat, bf->bf_dmamap);
 			bf->bf_dmamap = NULL;
 		}
 	}
 	STAILQ_INIT(&txq->free);
 	txq->nfree = 0;
 	if (txq->dma.dd_bufptr != NULL) {
 		free(txq->dma.dd_bufptr, M_MALODEV);
 		txq->dma.dd_bufptr = NULL;
 	}
 	if (txq->dma.dd_desc_len != 0)
 		malo_desc_cleanup(sc, &txq->dma);
 }
 
 static void
 malo_dma_cleanup(struct malo_softc *sc)
 {
 	int i;
 
 	for (i = 0; i < MALO_NUM_TX_QUEUES; i++)
 		malo_txdma_cleanup(sc, &sc->malo_txq[i]);
 
 	malo_rxdma_cleanup(sc);
 }
 
 static int
 malo_dma_setup(struct malo_softc *sc)
 {
 	int error, i;
 
 	/* rxdma initializing.  */
 	error = malo_rxdma_setup(sc);
 	if (error != 0)
 		return error;
 
 	/* NB: we just have 1 tx queue now.  */
 	for (i = 0; i < MALO_NUM_TX_QUEUES; i++) {
 		error = malo_txdma_setup(sc, &sc->malo_txq[i]);
 		if (error != 0) {
 			malo_dma_cleanup(sc);
 
 			return error;
 		}
 
 		malo_txq_init(sc, &sc->malo_txq[i], i);
 	}
 
 	return 0;
 }
 
 static void
 malo_hal_set_rxtxdma(struct malo_softc *sc)
 {
 	int i;
 
 	malo_bar0_write4(sc, sc->malo_hwspecs.rxdesc_read,
 	    sc->malo_hwdma.rxdesc_read);
 	malo_bar0_write4(sc, sc->malo_hwspecs.rxdesc_write,
 	    sc->malo_hwdma.rxdesc_read);
 
 	for (i = 0; i < MALO_NUM_TX_QUEUES; i++) {
 		malo_bar0_write4(sc,
 		    sc->malo_hwspecs.wcbbase[i], sc->malo_hwdma.wcbbase[i]);
 	}
 }
 
 /*
  * Inform firmware of our tx/rx dma setup.  The BAR 0 writes below are
  * for compatibility with older firmware.  For current firmware we send
  * this information with a cmd block via malo_hal_sethwdma.
  */
 static int
 malo_setup_hwdma(struct malo_softc *sc)
 {
 	int i;
 	struct malo_txq *txq;
 
 	sc->malo_hwdma.rxdesc_read = sc->malo_rxdma.dd_desc_paddr;
 
 	for (i = 0; i < MALO_NUM_TX_QUEUES; i++) {
 		txq = &sc->malo_txq[i];
 		sc->malo_hwdma.wcbbase[i] = txq->dma.dd_desc_paddr;
 	}
 	sc->malo_hwdma.maxnum_txwcb = malo_txbuf;
 	sc->malo_hwdma.maxnum_wcb = MALO_NUM_TX_QUEUES;
 
 	malo_hal_set_rxtxdma(sc);
 
 	return 0;
 }
 
 static void
 malo_txq_init(struct malo_softc *sc, struct malo_txq *txq, int qnum)
 {
 	struct malo_txbuf *bf, *bn;
 	struct malo_txdesc *ds;
 
 	MALO_TXQ_LOCK_INIT(sc, txq);
 	txq->qnum = qnum;
 	txq->txpri = 0;	/* XXX */
 
 	STAILQ_FOREACH(bf, &txq->free, bf_list) {
 		bf->bf_txq = txq;
 
 		ds = bf->bf_desc;
 		bn = STAILQ_NEXT(bf, bf_list);
 		if (bn == NULL)
 			bn = STAILQ_FIRST(&txq->free);
 		ds->physnext = htole32(bn->bf_daddr);
 	}
 	STAILQ_INIT(&txq->active);
 }
 
 /*
  * Reclaim resources for a setup queue.
  */
 static void
 malo_tx_cleanupq(struct malo_softc *sc, struct malo_txq *txq)
 {
 	/* XXX hal work? */
 	MALO_TXQ_LOCK_DESTROY(txq);
 }
 
 /*
  * Allocate a tx buffer for sending a frame.
  */
 static struct malo_txbuf *
 malo_getbuf(struct malo_softc *sc, struct malo_txq *txq)
 {
 	struct malo_txbuf *bf;
 
 	MALO_TXQ_LOCK(txq);
 	bf = STAILQ_FIRST(&txq->free);
 	if (bf != NULL) {
 		STAILQ_REMOVE_HEAD(&txq->free, bf_list);
 		txq->nfree--;
 	}
 	MALO_TXQ_UNLOCK(txq);
 	if (bf == NULL) {
 		DPRINTF(sc, MALO_DEBUG_XMIT,
 		    "%s: out of xmit buffers on q %d\n", __func__, txq->qnum);
 		sc->malo_stats.mst_tx_qstop++;
 	}
 	return bf;
 }
 
 static int
 malo_tx_dmasetup(struct malo_softc *sc, struct malo_txbuf *bf, struct mbuf *m0)
 {
 	struct mbuf *m;
 	int error;
 
 	/*
 	 * Load the DMA map so any coalescing is done.  This also calculates
 	 * the number of descriptors we need.
 	 */
 	error = bus_dmamap_load_mbuf_sg(sc->malo_dmat, bf->bf_dmamap, m0,
 				     bf->bf_segs, &bf->bf_nseg,
 				     BUS_DMA_NOWAIT);
 	if (error == EFBIG) {
 		/* XXX packet requires too many descriptors */
 		bf->bf_nseg = MALO_TXDESC + 1;
 	} else if (error != 0) {
 		sc->malo_stats.mst_tx_busdma++;
 		m_freem(m0);
 		return error;
 	}
 	/*
 	 * Discard null packets and check for packets that require too many
 	 * TX descriptors.  We try to convert the latter to a cluster.
 	 */
 	if (error == EFBIG) {		/* too many desc's, linearize */
 		sc->malo_stats.mst_tx_linear++;
 		m = m_defrag(m0, M_NOWAIT);
 		if (m == NULL) {
 			m_freem(m0);
 			sc->malo_stats.mst_tx_nombuf++;
 			return ENOMEM;
 		}
 		m0 = m;
 		error = bus_dmamap_load_mbuf_sg(sc->malo_dmat, bf->bf_dmamap, m0,
 					     bf->bf_segs, &bf->bf_nseg,
 					     BUS_DMA_NOWAIT);
 		if (error != 0) {
 			sc->malo_stats.mst_tx_busdma++;
 			m_freem(m0);
 			return error;
 		}
 		KASSERT(bf->bf_nseg <= MALO_TXDESC,
 		    ("too many segments after defrag; nseg %u", bf->bf_nseg));
 	} else if (bf->bf_nseg == 0) {		/* null packet, discard */
 		sc->malo_stats.mst_tx_nodata++;
 		m_freem(m0);
 		return EIO;
 	}
 	DPRINTF(sc, MALO_DEBUG_XMIT, "%s: m %p len %u\n",
 		__func__, m0, m0->m_pkthdr.len);
 	bus_dmamap_sync(sc->malo_dmat, bf->bf_dmamap, BUS_DMASYNC_PREWRITE);
 	bf->bf_m = m0;
 
 	return 0;
 }
 
 #ifdef MALO_DEBUG
 static void
 malo_printrxbuf(const struct malo_rxbuf *bf, u_int ix)
 {
 	const struct malo_rxdesc *ds = bf->bf_desc;
 	uint32_t status = le32toh(ds->status);
 	
 	printf("R[%2u] (DS.V:%p DS.P:%p) NEXT:%08x DATA:%08x RC:%02x%s\n"
 	    "      STAT:%02x LEN:%04x SNR:%02x NF:%02x CHAN:%02x"
 	    " RATE:%02x QOS:%04x\n",
 	    ix, ds, (const struct malo_desc *)bf->bf_daddr,
 	    le32toh(ds->physnext), le32toh(ds->physbuffdata),
 	    ds->rxcontrol, 
 	    ds->rxcontrol != MALO_RXD_CTRL_DRIVER_OWN ?
 	        "" : (status & MALO_RXD_STATUS_OK) ? " *" : " !",
 	    ds->status, le16toh(ds->pktlen), ds->snr, ds->nf, ds->channel,
 	    ds->rate, le16toh(ds->qosctrl));
 }
 
 static void
 malo_printtxbuf(const struct malo_txbuf *bf, u_int qnum, u_int ix)
 {
 	const struct malo_txdesc *ds = bf->bf_desc;
 	uint32_t status = le32toh(ds->status);
 	
 	printf("Q%u[%3u]", qnum, ix);
 	printf(" (DS.V:%p DS.P:%p)\n",
 	    ds, (const struct malo_txdesc *)bf->bf_daddr);
 	printf("    NEXT:%08x DATA:%08x LEN:%04x STAT:%08x%s\n",
 	    le32toh(ds->physnext),
 	    le32toh(ds->pktptr), le16toh(ds->pktlen), status,
 	    status & MALO_TXD_STATUS_USED ?
 	    "" : (status & 3) != 0 ? " *" : " !");
 	printf("    RATE:%02x PRI:%x QOS:%04x SAP:%08x FORMAT:%04x\n",
 	    ds->datarate, ds->txpriority, le16toh(ds->qosctrl),
 	    le32toh(ds->sap_pktinfo), le16toh(ds->format));
 #if 0
 	{
 		const uint8_t *cp = (const uint8_t *) ds;
 		int i;
 		for (i = 0; i < sizeof(struct malo_txdesc); i++) {
 			printf("%02x ", cp[i]);
 			if (((i+1) % 16) == 0)
 				printf("\n");
 		}
 		printf("\n");
 	}
 #endif
 }
 #endif /* MALO_DEBUG */
 
 static __inline void
 malo_updatetxrate(struct ieee80211_node *ni, int rix)
 {
 #define	N(x)	(sizeof(x)/sizeof(x[0]))
 	static const int ieeerates[] =
 	    { 2, 4, 11, 22, 44, 12, 18, 24, 36, 48, 96, 108 };
 	if (rix < N(ieeerates))
 		ni->ni_txrate = ieeerates[rix];
 #undef N
 }
 
 static int
 malo_fix2rate(int fix_rate)
 {
 #define	N(x)	(sizeof(x)/sizeof(x[0]))
 	static const int rates[] =
 	    { 2, 4, 11, 22, 12, 18, 24, 36, 48, 96, 108 };
 	return (fix_rate < N(rates) ? rates[fix_rate] : 0);
 #undef N
 }
 
 /* idiomatic shorthands: MS = mask+shift, SM = shift+mask */
 #define	MS(v,x)			(((v) & x) >> x##_S)
 #define	SM(v,x)			(((v) << x##_S) & x)
 
 /*
  * Process completed xmit descriptors from the specified queue.
  */
 static int
 malo_tx_processq(struct malo_softc *sc, struct malo_txq *txq)
 {
 	struct malo_txbuf *bf;
 	struct malo_txdesc *ds;
 	struct ieee80211_node *ni;
 	int nreaped;
 	uint32_t status;
 
 	DPRINTF(sc, MALO_DEBUG_TX_PROC, "%s: tx queue %u\n",
 	    __func__, txq->qnum);
 	for (nreaped = 0;; nreaped++) {
 		MALO_TXQ_LOCK(txq);
 		bf = STAILQ_FIRST(&txq->active);
 		if (bf == NULL) {
 			MALO_TXQ_UNLOCK(txq);
 			break;
 		}
 		ds = bf->bf_desc;
 		MALO_TXDESC_SYNC(txq, ds,
 		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 		if (ds->status & htole32(MALO_TXD_STATUS_FW_OWNED)) {
 			MALO_TXQ_UNLOCK(txq);
 			break;
 		}
 		STAILQ_REMOVE_HEAD(&txq->active, bf_list);
 		MALO_TXQ_UNLOCK(txq);
 
 #ifdef MALO_DEBUG
 		if (sc->malo_debug & MALO_DEBUG_XMIT_DESC)
 			malo_printtxbuf(bf, txq->qnum, nreaped);
 #endif
 		ni = bf->bf_node;
 		if (ni != NULL) {
 			status = le32toh(ds->status);
 			if (status & MALO_TXD_STATUS_OK) {
 				uint16_t format = le16toh(ds->format);
 				uint8_t txant = MS(format, MALO_TXD_ANTENNA);
 
 				sc->malo_stats.mst_ant_tx[txant]++;
 				if (status & MALO_TXD_STATUS_OK_RETRY)
 					sc->malo_stats.mst_tx_retries++;
 				if (status & MALO_TXD_STATUS_OK_MORE_RETRY)
 					sc->malo_stats.mst_tx_mretries++;
 				malo_updatetxrate(ni, ds->datarate);
 				sc->malo_stats.mst_tx_rate = ds->datarate;
 			} else {
 				if (status & MALO_TXD_STATUS_FAILED_LINK_ERROR)
 					sc->malo_stats.mst_tx_linkerror++;
 				if (status & MALO_TXD_STATUS_FAILED_XRETRY)
 					sc->malo_stats.mst_tx_xretries++;
 				if (status & MALO_TXD_STATUS_FAILED_AGING)
 					sc->malo_stats.mst_tx_aging++;
 			}
 			/*
 			 * Do any tx complete callback.  Note this must
 			 * be done before releasing the node reference.
 			 * XXX no way to figure out if frame was ACK'd
 			 */
 			if (bf->bf_m->m_flags & M_TXCB) {
 				/* XXX strip fw len in case header inspected */
 				m_adj(bf->bf_m, sizeof(uint16_t));
 				ieee80211_process_callback(ni, bf->bf_m,
 					(status & MALO_TXD_STATUS_OK) == 0);
 			}
 			/*
 			 * Reclaim reference to node.
 			 *
 			 * NB: the node may be reclaimed here if, for example
 			 *     this is a DEAUTH message that was sent and the
 			 *     node was timed out due to inactivity.
 			 */
 			ieee80211_free_node(ni);
 		}
 		ds->status = htole32(MALO_TXD_STATUS_IDLE);
 		ds->pktlen = htole32(0);
 
 		bus_dmamap_sync(sc->malo_dmat, bf->bf_dmamap,
 		    BUS_DMASYNC_POSTWRITE);
 		bus_dmamap_unload(sc->malo_dmat, bf->bf_dmamap);
 		m_freem(bf->bf_m);
 		bf->bf_m = NULL;
 		bf->bf_node = NULL;
 
 		MALO_TXQ_LOCK(txq);
 		STAILQ_INSERT_TAIL(&txq->free, bf, bf_list);
 		txq->nfree++;
 		MALO_TXQ_UNLOCK(txq);
 	}
 	return nreaped;
 }
 
 /*
  * Deferred processing of transmit interrupt.
  */
 static void
 malo_tx_proc(void *arg, int npending)
 {
 	struct malo_softc *sc = arg;
 	struct ifnet *ifp = sc->malo_ifp;
 	int i, nreaped;
 
 	/*
 	 * Process each active queue.
 	 */
 	nreaped = 0;
 	for (i = 0; i < MALO_NUM_TX_QUEUES; i++) {
 		if (!STAILQ_EMPTY(&sc->malo_txq[i].active))
 			nreaped += malo_tx_processq(sc, &sc->malo_txq[i]);
 	}
 
 	if (nreaped != 0) {
 		ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
 		sc->malo_timer = 0;
 		malo_start(ifp);
 	}
 }
 
 static int
 malo_tx_start(struct malo_softc *sc, struct ieee80211_node *ni,
     struct malo_txbuf *bf, struct mbuf *m0)
 {
 #define	IEEE80211_DIR_DSTODS(wh) \
 	((wh->i_fc[1] & IEEE80211_FC1_DIR_MASK) == IEEE80211_FC1_DIR_DSTODS)
 #define	IS_DATA_FRAME(wh)						\
 	((wh->i_fc[0] & (IEEE80211_FC0_TYPE_MASK)) == IEEE80211_FC0_TYPE_DATA)
 	int error, ismcast, iswep;
 	int copyhdrlen, hdrlen, pktlen;
 	struct ieee80211_frame *wh;
 	struct ifnet *ifp = sc->malo_ifp;
 	struct ieee80211com *ic = ifp->if_l2com;
 	struct ieee80211vap *vap = ni->ni_vap;
 	struct malo_txdesc *ds;
 	struct malo_txrec *tr;
 	struct malo_txq *txq;
 	uint16_t qos;
 
 	wh = mtod(m0, struct ieee80211_frame *);
 	iswep = wh->i_fc[1] & IEEE80211_FC1_PROTECTED;
 	ismcast = IEEE80211_IS_MULTICAST(wh->i_addr1);
 	copyhdrlen = hdrlen = ieee80211_anyhdrsize(wh);
 	pktlen = m0->m_pkthdr.len;
 	if (IEEE80211_QOS_HAS_SEQ(wh)) {
 		if (IEEE80211_DIR_DSTODS(wh)) {
 			qos = *(uint16_t *)
 			    (((struct ieee80211_qosframe_addr4 *) wh)->i_qos);
 			copyhdrlen -= sizeof(qos);
 		} else
 			qos = *(uint16_t *)
 			    (((struct ieee80211_qosframe *) wh)->i_qos);
 	} else
 		qos = 0;
 
 	if (iswep) {
 		struct ieee80211_key *k;
 
 		/*
 		 * Construct the 802.11 header+trailer for an encrypted
 		 * frame. The only reason this can fail is because of an
 		 * unknown or unsupported cipher/key type.
 		 *
 		 * NB: we do this even though the firmware will ignore
 		 *     what we've done for WEP and TKIP as we need the
 		 *     ExtIV filled in for CCMP and this also adjusts
 		 *     the headers which simplifies our work below.
 		 */
 		k = ieee80211_crypto_encap(ni, m0);
 		if (k == NULL) {
 			/*
 			 * This can happen when the key is yanked after the
 			 * frame was queued.  Just discard the frame; the
 			 * 802.11 layer counts failures and provides
 			 * debugging/diagnostics.
 			 */
 			m_freem(m0);
 			return EIO;
 		}
 
 		/*
 		 * Adjust the packet length for the crypto additions
 		 * done during encap and any other bits that the f/w
 		 * will add later on.
 		 */
 		pktlen = m0->m_pkthdr.len;
 
 		/* packet header may have moved, reset our local pointer */
 		wh = mtod(m0, struct ieee80211_frame *);
 	}
 
 	if (ieee80211_radiotap_active_vap(vap)) {
 		sc->malo_tx_th.wt_flags = 0;	/* XXX */
 		if (iswep)
 			sc->malo_tx_th.wt_flags |= IEEE80211_RADIOTAP_F_WEP;
 		sc->malo_tx_th.wt_txpower = ni->ni_txpower;
 		sc->malo_tx_th.wt_antenna = sc->malo_txantenna;
 
 		ieee80211_radiotap_tx(vap, m0);
 	}
 
 	/*
 	 * Copy up/down the 802.11 header; the firmware requires
 	 * we present a 2-byte payload length followed by a
 	 * 4-address header (w/o QoS), followed (optionally) by
 	 * any WEP/ExtIV header (but only filled in for CCMP).
 	 * We are assured the mbuf has sufficient headroom to
 	 * prepend in-place by the setup of ic_headroom in
 	 * malo_attach.
 	 */
 	if (hdrlen < sizeof(struct malo_txrec)) {
 		const int space = sizeof(struct malo_txrec) - hdrlen;
 		if (M_LEADINGSPACE(m0) < space) {
 			/* NB: should never happen */
 			device_printf(sc->malo_dev,
 			    "not enough headroom, need %d found %zd, "
 			    "m_flags 0x%x m_len %d\n",
 			    space, M_LEADINGSPACE(m0), m0->m_flags, m0->m_len);
 			ieee80211_dump_pkt(ic,
 			    mtod(m0, const uint8_t *), m0->m_len, 0, -1);
 			m_freem(m0);
 			/* XXX stat */
 			return EIO;
 		}
 		M_PREPEND(m0, space, M_NOWAIT);
 	}
 	tr = mtod(m0, struct malo_txrec *);
 	if (wh != (struct ieee80211_frame *) &tr->wh)
 		ovbcopy(wh, &tr->wh, hdrlen);
 	/*
 	 * Note: the "firmware length" is actually the length of the fully
 	 * formed "802.11 payload".  That is, it's everything except for
 	 * the 802.11 header.  In particular this includes all crypto
 	 * material including the MIC!
 	 */
 	tr->fwlen = htole16(pktlen - hdrlen);
 
 	/*
 	 * Load the DMA map so any coalescing is done.  This
 	 * also calculates the number of descriptors we need.
 	 */
 	error = malo_tx_dmasetup(sc, bf, m0);
 	if (error != 0)
 		return error;
 	bf->bf_node = ni;			/* NB: held reference */
 	m0 = bf->bf_m;				/* NB: may have changed */
 	tr = mtod(m0, struct malo_txrec *);
 	wh = (struct ieee80211_frame *)&tr->wh;
 
 	/*
 	 * Formulate tx descriptor.
 	 */
 	ds = bf->bf_desc;
 	txq = bf->bf_txq;
 
 	ds->qosctrl = qos;			/* NB: already little-endian */
 	ds->pktptr = htole32(bf->bf_segs[0].ds_addr);
 	ds->pktlen = htole16(bf->bf_segs[0].ds_len);
 	/* NB: pPhysNext setup once, don't touch */
 	ds->datarate = IS_DATA_FRAME(wh) ? 1 : 0;
 	ds->sap_pktinfo = 0;
 	ds->format = 0;
 
 	/*
 	 * Select transmit rate.
 	 */
 	switch (wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK) {
 	case IEEE80211_FC0_TYPE_MGT:
 		sc->malo_stats.mst_tx_mgmt++;
 		/* fall thru... */
 	case IEEE80211_FC0_TYPE_CTL:
 		ds->txpriority = 1;
 		break;
 	case IEEE80211_FC0_TYPE_DATA:
 		ds->txpriority = txq->qnum;
 		break;
 	default:
 		if_printf(ifp, "bogus frame type 0x%x (%s)\n",
 			wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK, __func__);
 		/* XXX statistic */
 		m_freem(m0);
 		return EIO;
 	}
 
 #ifdef MALO_DEBUG
 	if (IFF_DUMPPKTS_XMIT(sc))
 		ieee80211_dump_pkt(ic,
 		    mtod(m0, const uint8_t *)+sizeof(uint16_t),
 		    m0->m_len - sizeof(uint16_t), ds->datarate, -1);
 #endif
 
 	MALO_TXQ_LOCK(txq);
 	if (!IS_DATA_FRAME(wh))
 		ds->status |= htole32(1);
 	ds->status |= htole32(MALO_TXD_STATUS_FW_OWNED);
 	STAILQ_INSERT_TAIL(&txq->active, bf, bf_list);
 	MALO_TXDESC_SYNC(txq, ds, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 
 	ifp->if_opackets++;
 	sc->malo_timer = 5;
 	MALO_TXQ_UNLOCK(txq);
 	return 0;
 #undef IEEE80211_DIR_DSTODS
 }
 
 static void
 malo_start(struct ifnet *ifp)
 {
 	struct malo_softc *sc = ifp->if_softc;
 	struct ieee80211_node *ni;
 	struct malo_txq *txq = &sc->malo_txq[0];
 	struct malo_txbuf *bf = NULL;
 	struct mbuf *m;
 	int nqueued = 0;
 
 	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || sc->malo_invalid)
 		return;
 
 	for (;;) {
 		IFQ_DRV_DEQUEUE(&ifp->if_snd, m);
 		if (m == NULL)
 			break;
 		ni = (struct ieee80211_node *) m->m_pkthdr.rcvif;
 		bf = malo_getbuf(sc, txq);
 		if (bf == NULL) {
 			IFQ_DRV_PREPEND(&ifp->if_snd, m);
 
 			/* XXX blocks other traffic */
 			ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 			sc->malo_stats.mst_tx_qstop++;
 			break;
 		}
 		/*
 		 * Pass the frame to the h/w for transmission.
 		 */
 		if (malo_tx_start(sc, ni, bf, m)) {
 			ifp->if_oerrors++;
 			if (bf != NULL) {
 				bf->bf_m = NULL;
 				bf->bf_node = NULL;
 				MALO_TXQ_LOCK(txq);
 				STAILQ_INSERT_HEAD(&txq->free, bf, bf_list);
 				MALO_TXQ_UNLOCK(txq);
 			}
 			ieee80211_free_node(ni);
 			continue;
 		}
 		nqueued++;
 
 		if (nqueued >= malo_txcoalesce) {
 			/*
 			 * Poke the firmware to process queued frames;
 			 * see below about (lack of) locking.
 			 */
 			nqueued = 0;
 			malo_hal_txstart(sc->malo_mh, 0/*XXX*/);
 		}
 	}
 
 	if (nqueued) {
 		/*
 		 * NB: We don't need to lock against tx done because
 		 * this just prods the firmware to check the transmit
 		 * descriptors.  The firmware will also start fetching
 		 * descriptors by itself if it notices new ones are
 		 * present when it goes to deliver a tx done interrupt
 		 * to the host. So if we race with tx done processing
 		 * it's ok.  Delivering the kick here rather than in
 		 * malo_tx_start is an optimization to avoid poking the
 		 * firmware for each packet.
 		 *
 		 * NB: the queue id isn't used so 0 is ok.
 		 */
 		malo_hal_txstart(sc->malo_mh, 0/*XXX*/);
 	}
 }
 
 static void
 malo_watchdog(void *arg)
 {
 	struct malo_softc *sc;
 	struct ifnet *ifp;
 
 	sc = arg;
 	callout_reset(&sc->malo_watchdog_timer, hz, malo_watchdog, sc);
 	if (sc->malo_timer == 0 || --sc->malo_timer > 0)
 		return;
 
 	ifp = sc->malo_ifp;
 	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) && !sc->malo_invalid) {
 		if_printf(ifp, "watchdog timeout\n");
 
 		/* XXX no way to reset h/w. now  */
 
 		ifp->if_oerrors++;
 		sc->malo_stats.mst_watchdog++;
 	}
 }
 
 static int
 malo_hal_reset(struct malo_softc *sc)
 {
 	static int first = 0;
 	struct ifnet *ifp = sc->malo_ifp;
 	struct ieee80211com *ic = ifp->if_l2com;
 	struct malo_hal *mh = sc->malo_mh;
 
 	if (first == 0) {
 		/*
 		 * NB: when the device firstly is initialized, sometimes
 		 * firmware could override rx/tx dma registers so we re-set
 		 * these values once.
 		 */
 		malo_hal_set_rxtxdma(sc);
 		first = 1;
 	}
 
 	malo_hal_setantenna(mh, MHA_ANTENNATYPE_RX, sc->malo_rxantenna);
 	malo_hal_setantenna(mh, MHA_ANTENNATYPE_TX, sc->malo_txantenna);
 	malo_hal_setradio(mh, 1, MHP_AUTO_PREAMBLE);
 	malo_chan_set(sc, ic->ic_curchan);
 
 	/* XXX needs other stuffs?  */
 
 	return 1;
 }
 
 static __inline struct mbuf *
 malo_getrxmbuf(struct malo_softc *sc, struct malo_rxbuf *bf)
 {
 	struct mbuf *m;
 	bus_addr_t paddr;
 	int error;
 
 	/* XXX don't need mbuf, just dma buffer */
 	m = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, MJUMPAGESIZE);
 	if (m == NULL) {
 		sc->malo_stats.mst_rx_nombuf++;	/* XXX */
 		return NULL;
 	}
 	error = bus_dmamap_load(sc->malo_dmat, bf->bf_dmamap,
 	    mtod(m, caddr_t), MJUMPAGESIZE,
 	    malo_load_cb, &paddr, BUS_DMA_NOWAIT);
 	if (error != 0) {
 		if_printf(sc->malo_ifp,
 		    "%s: bus_dmamap_load failed, error %d\n", __func__, error);
 		m_freem(m);
 		return NULL;
 	}
 	bf->bf_data = paddr;
 	bus_dmamap_sync(sc->malo_dmat, bf->bf_dmamap, BUS_DMASYNC_PREWRITE);
 
 	return m;
 }
 
 static int
 malo_rxbuf_init(struct malo_softc *sc, struct malo_rxbuf *bf)
 {
 	struct malo_rxdesc *ds;
 
 	ds = bf->bf_desc;
 	if (bf->bf_m == NULL) {
 		bf->bf_m = malo_getrxmbuf(sc, bf);
 		if (bf->bf_m == NULL) {
 			/* mark descriptor to be skipped */
 			ds->rxcontrol = MALO_RXD_CTRL_OS_OWN;
 			/* NB: don't need PREREAD */
 			MALO_RXDESC_SYNC(sc, ds, BUS_DMASYNC_PREWRITE);
 			return ENOMEM;
 		}
 	}
 
 	/*
 	 * Setup descriptor.
 	 */
 	ds->qosctrl = 0;
 	ds->snr = 0;
 	ds->status = MALO_RXD_STATUS_IDLE;
 	ds->channel = 0;
 	ds->pktlen = htole16(MALO_RXSIZE);
 	ds->nf = 0;
 	ds->physbuffdata = htole32(bf->bf_data);
 	/* NB: don't touch pPhysNext, set once */
 	ds->rxcontrol = MALO_RXD_CTRL_DRIVER_OWN;
 	MALO_RXDESC_SYNC(sc, ds, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 
 	return 0;
 }
 
 /*
  * Setup the rx data structures.  This should only be done once or we may get
  * out of sync with the firmware.
  */
 static int
 malo_startrecv(struct malo_softc *sc)
 {
 	struct malo_rxbuf *bf, *prev;
 	struct malo_rxdesc *ds;
 	
 	if (sc->malo_recvsetup == 1) {
 		malo_mode_init(sc);		/* set filters, etc. */
 		return 0;
 	}
 	
 	prev = NULL;
 	STAILQ_FOREACH(bf, &sc->malo_rxbuf, bf_list) {
 		int error = malo_rxbuf_init(sc, bf);
 		if (error != 0) {
 			DPRINTF(sc, MALO_DEBUG_RECV,
 			    "%s: malo_rxbuf_init failed %d\n",
 			    __func__, error);
 			return error;
 		}
 		if (prev != NULL) {
 			ds = prev->bf_desc;
 			ds->physnext = htole32(bf->bf_daddr);
 		}
 		prev = bf;
 	}
 	if (prev != NULL) {
 		ds = prev->bf_desc;
 		ds->physnext =
 		    htole32(STAILQ_FIRST(&sc->malo_rxbuf)->bf_daddr);
 	}
 
 	sc->malo_recvsetup = 1;
 
 	malo_mode_init(sc);		/* set filters, etc. */
 	
 	return 0;
 }
 
 static void
 malo_init_locked(struct malo_softc *sc)
 {
 	struct ifnet *ifp = sc->malo_ifp;
 	struct malo_hal *mh = sc->malo_mh;
 	int error;
 	
 	DPRINTF(sc, MALO_DEBUG_ANY, "%s: if_flags 0x%x\n",
 	    __func__, ifp->if_flags);
 
 	MALO_LOCK_ASSERT(sc);
 	
 	/*
 	 * Stop anything previously setup.  This is safe whether this is
 	 * the first time through or not.
 	 */
 	malo_stop_locked(ifp, 0);
 
 	/*
 	 * Push state to the firmware.
 	 */
 	if (!malo_hal_reset(sc)) {
 		if_printf(ifp, "%s: unable to reset hardware\n", __func__);
 		return;
 	}
 
 	/*
 	 * Setup recv (once); transmit is already good to go.
 	 */
 	error = malo_startrecv(sc);
 	if (error != 0) {
 		if_printf(ifp, "%s: unable to start recv logic, error %d\n",
 		    __func__, error);
 		return;
 	}
 
 	/*
 	 * Enable interrupts.
 	 */
 	sc->malo_imask = MALO_A2HRIC_BIT_RX_RDY
 	    | MALO_A2HRIC_BIT_TX_DONE
 	    | MALO_A2HRIC_BIT_OPC_DONE
 	    | MALO_A2HRIC_BIT_MAC_EVENT
 	    | MALO_A2HRIC_BIT_RX_PROBLEM
 	    | MALO_A2HRIC_BIT_ICV_ERROR
 	    | MALO_A2HRIC_BIT_RADAR_DETECT
 	    | MALO_A2HRIC_BIT_CHAN_SWITCH;
 
 	ifp->if_drv_flags |= IFF_DRV_RUNNING;
 	malo_hal_intrset(mh, sc->malo_imask);
 	callout_reset(&sc->malo_watchdog_timer, hz, malo_watchdog, sc);
 }
 
 static void
 malo_init(void *arg)
 {
 	struct malo_softc *sc = (struct malo_softc *) arg;
 	struct ifnet *ifp = sc->malo_ifp;
 	struct ieee80211com *ic = ifp->if_l2com;
 	
 	DPRINTF(sc, MALO_DEBUG_ANY, "%s: if_flags 0x%x\n",
 	    __func__, ifp->if_flags);
 
 	MALO_LOCK(sc);
 	malo_init_locked(sc);
 
 	MALO_UNLOCK(sc);
 
 	if (ifp->if_drv_flags & IFF_DRV_RUNNING)
 		ieee80211_start_all(ic);	/* start all vap's */
 }
 
 /*
  * Set the multicast filter contents into the hardware.
  */
 static void
 malo_setmcastfilter(struct malo_softc *sc)
 {
 	struct ifnet *ifp = sc->malo_ifp;
 	struct ieee80211com *ic = ifp->if_l2com;
 	struct ifmultiaddr *ifma;
 	uint8_t macs[IEEE80211_ADDR_LEN * MALO_HAL_MCAST_MAX];
 	uint8_t *mp;
 	int nmc;
 
 	mp = macs;
 	nmc = 0;
 
 	if (ic->ic_opmode == IEEE80211_M_MONITOR ||
 	    (ifp->if_flags & (IFF_ALLMULTI | IFF_PROMISC)))
 		goto all;
 	
 	if_maddr_rlock(ifp);
 	TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
 		if (ifma->ifma_addr->sa_family != AF_LINK)
 			continue;
 
 		if (nmc == MALO_HAL_MCAST_MAX) {
 			ifp->if_flags |= IFF_ALLMULTI;
 			if_maddr_runlock(ifp);
 			goto all;
 		}
 		IEEE80211_ADDR_COPY(mp,
 		    LLADDR((struct sockaddr_dl *)ifma->ifma_addr));
 
 		mp += IEEE80211_ADDR_LEN, nmc++;
 	}
 	if_maddr_runlock(ifp);
 
 	malo_hal_setmcast(sc->malo_mh, nmc, macs);
 
 all:
 	/*
 	 * XXX we don't know how to set the f/w for supporting
 	 * IFF_ALLMULTI | IFF_PROMISC cases
 	 */
 	return;
 }
 
 static int
 malo_mode_init(struct malo_softc *sc)
 {
 	struct ifnet *ifp = sc->malo_ifp;
 	struct ieee80211com *ic = ifp->if_l2com;
 	struct malo_hal *mh = sc->malo_mh;
 
 	/*
 	 * NB: Ignore promisc in hostap mode; it's set by the
 	 * bridge.  This is wrong but we have no way to
 	 * identify internal requests (from the bridge)
 	 * versus external requests such as for tcpdump.
 	 */
 	malo_hal_setpromisc(mh, (ifp->if_flags & IFF_PROMISC) &&
 	    ic->ic_opmode != IEEE80211_M_HOSTAP);
 	malo_setmcastfilter(sc);
 
 	return ENXIO;
 }
 
 static void
 malo_tx_draintxq(struct malo_softc *sc, struct malo_txq *txq)
 {
 	struct ieee80211_node *ni;
 	struct malo_txbuf *bf;
 	u_int ix;
 	
 	/*
 	 * NB: this assumes output has been stopped and
 	 *     we do not need to block malo_tx_tasklet
 	 */
 	for (ix = 0;; ix++) {
 		MALO_TXQ_LOCK(txq);
 		bf = STAILQ_FIRST(&txq->active);
 		if (bf == NULL) {
 			MALO_TXQ_UNLOCK(txq);
 			break;
 		}
 		STAILQ_REMOVE_HEAD(&txq->active, bf_list);
 		MALO_TXQ_UNLOCK(txq);
 #ifdef MALO_DEBUG
 		if (sc->malo_debug & MALO_DEBUG_RESET) {
 			struct ifnet *ifp = sc->malo_ifp;
 			struct ieee80211com *ic = ifp->if_l2com;
 			const struct malo_txrec *tr =
 			    mtod(bf->bf_m, const struct malo_txrec *);
 			malo_printtxbuf(bf, txq->qnum, ix);
 			ieee80211_dump_pkt(ic, (const uint8_t *)&tr->wh,
 			    bf->bf_m->m_len - sizeof(tr->fwlen), 0, -1);
 		}
 #endif /* MALO_DEBUG */
 		bus_dmamap_unload(sc->malo_dmat, bf->bf_dmamap);
 		ni = bf->bf_node;
 		bf->bf_node = NULL;
 		if (ni != NULL) {
 			/*
 			 * Reclaim node reference.
 			 */
 			ieee80211_free_node(ni);
 		}
 		m_freem(bf->bf_m);
 		bf->bf_m = NULL;
 		
 		MALO_TXQ_LOCK(txq);
 		STAILQ_INSERT_TAIL(&txq->free, bf, bf_list);
 		txq->nfree++;
 		MALO_TXQ_UNLOCK(txq);
 	}
 }
 
 static void
 malo_stop_locked(struct ifnet *ifp, int disable)
 {
 	struct malo_softc *sc = ifp->if_softc;
 	struct malo_hal *mh = sc->malo_mh;
 	int i;
 
 	DPRINTF(sc, MALO_DEBUG_ANY, "%s: invalid %u if_flags 0x%x\n",
 	    __func__, sc->malo_invalid, ifp->if_flags);
 
 	MALO_LOCK_ASSERT(sc);
 
 	if (!(ifp->if_drv_flags & IFF_DRV_RUNNING))
 		return;
 
 	/*
 	 * Shutdown the hardware and driver:
 	 *    disable interrupts
 	 *    turn off the radio
 	 *    drain and release tx queues
 	 *
 	 * Note that some of this work is not possible if the hardware
 	 * is gone (invalid).
 	 */
 	ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
 	callout_stop(&sc->malo_watchdog_timer);
 	sc->malo_timer = 0;
 	/* diable interrupt.  */
 	malo_hal_intrset(mh, 0);
 	/* turn off the radio.  */
 	malo_hal_setradio(mh, 0, MHP_AUTO_PREAMBLE);
 
 	/* drain and release tx queues.  */
 	for (i = 0; i < MALO_NUM_TX_QUEUES; i++)
 		malo_tx_draintxq(sc, &sc->malo_txq[i]);
 }
 
 static int
 malo_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 {
 #define	MALO_IS_RUNNING(ifp) \
 	((ifp->if_flags & IFF_UP) && (ifp->if_drv_flags & IFF_DRV_RUNNING))
 	struct malo_softc *sc = ifp->if_softc;
 	struct ieee80211com *ic = ifp->if_l2com;
 	struct ifreq *ifr = (struct ifreq *) data;
 	int error = 0, startall = 0;
 
 	MALO_LOCK(sc);
 	switch (cmd) {
 	case SIOCSIFFLAGS:
 		if (MALO_IS_RUNNING(ifp)) {
 			/*
 			 * To avoid rescanning another access point,
 			 * do not call malo_init() here.  Instead,
 			 * only reflect promisc mode settings.
 			 */
 			malo_mode_init(sc);
 		} else if (ifp->if_flags & IFF_UP) {
 			/*
 			 * Beware of being called during attach/detach
 			 * to reset promiscuous mode.  In that case we
 			 * will still be marked UP but not RUNNING.
 			 * However trying to re-init the interface
 			 * is the wrong thing to do as we've already
 			 * torn down much of our state.  There's
 			 * probably a better way to deal with this.
 			 */
 			if (!sc->malo_invalid) {
 				malo_init_locked(sc);
 				startall = 1;
 			}
 		} else
 			malo_stop_locked(ifp, 1);
 		break;
 	case SIOCGIFMEDIA:
 	case SIOCSIFMEDIA:
 		error = ifmedia_ioctl(ifp, ifr, &ic->ic_media, cmd);
 		break;
 	default:
 		error = ether_ioctl(ifp, cmd, data);
 		break;
 	}
 	MALO_UNLOCK(sc);
 
 	if (startall)
 		ieee80211_start_all(ic);
 	return error;
 #undef MALO_IS_RUNNING
 }
 
 /*
  * Callback from the 802.11 layer to update the slot time
  * based on the current setting.  We use it to notify the
  * firmware of ERP changes and the f/w takes care of things
  * like slot time and preamble.
  */
 static void
 malo_updateslot(struct ifnet *ifp)
 {
 	struct malo_softc *sc = ifp->if_softc;
 	struct ieee80211com *ic = ifp->if_l2com;
 	struct malo_hal *mh = sc->malo_mh;
 	int error;
 	
 	/* NB: can be called early; suppress needless cmds */
 	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 		return;
 
 	DPRINTF(sc, MALO_DEBUG_RESET,
 	    "%s: chan %u MHz/flags 0x%x %s slot, (ic_flags 0x%x)\n",
 	    __func__, ic->ic_curchan->ic_freq, ic->ic_curchan->ic_flags,
 	    ic->ic_flags & IEEE80211_F_SHSLOT ? "short" : "long", ic->ic_flags);
 
 	if (ic->ic_flags & IEEE80211_F_SHSLOT)
 		error = malo_hal_set_slot(mh, 1);
 	else
 		error = malo_hal_set_slot(mh, 0);
 
 	if (error != 0)
 		device_printf(sc->malo_dev, "setting %s slot failed\n",
 			ic->ic_flags & IEEE80211_F_SHSLOT ? "short" : "long");
 }
 
 static int
 malo_newstate(struct ieee80211vap *vap, enum ieee80211_state nstate, int arg)
 {
 	struct ieee80211com *ic = vap->iv_ic;
 	struct malo_softc *sc = ic->ic_ifp->if_softc;
 	struct malo_hal *mh = sc->malo_mh;
 	int error;
 
 	DPRINTF(sc, MALO_DEBUG_STATE, "%s: %s -> %s\n", __func__,
 	    ieee80211_state_name[vap->iv_state],
 	    ieee80211_state_name[nstate]);
 
 	/*
 	 * Invoke the net80211 layer first so iv_bss is setup.
 	 */
 	error = MALO_VAP(vap)->malo_newstate(vap, nstate, arg);
 	if (error != 0)
 		return error;
 
 	if (nstate == IEEE80211_S_RUN && vap->iv_state != IEEE80211_S_RUN) {
 		struct ieee80211_node *ni = vap->iv_bss;
 		enum ieee80211_phymode mode = ieee80211_chan2mode(ni->ni_chan);
 		const struct ieee80211_txparam *tp = &vap->iv_txparms[mode];
 
 		DPRINTF(sc, MALO_DEBUG_STATE,
 		    "%s: %s(RUN): iv_flags 0x%08x bintvl %d bssid %s "
 		    "capinfo 0x%04x chan %d associd 0x%x mode %d rate %d\n",
 		    vap->iv_ifp->if_xname, __func__, vap->iv_flags,
 		    ni->ni_intval, ether_sprintf(ni->ni_bssid), ni->ni_capinfo,
 		    ieee80211_chan2ieee(ic, ic->ic_curchan),
 		    ni->ni_associd, mode, tp->ucastrate);
 
 		malo_hal_setradio(mh, 1,
 		    (ic->ic_flags & IEEE80211_F_SHPREAMBLE) ?
 			MHP_SHORT_PREAMBLE : MHP_LONG_PREAMBLE);
 		malo_hal_setassocid(sc->malo_mh, ni->ni_bssid, ni->ni_associd);
 		malo_hal_set_rate(mh, mode, 
 		   tp->ucastrate == IEEE80211_FIXED_RATE_NONE ?
 		       0 : malo_fix2rate(tp->ucastrate));
 	}
 	return 0;
 }
 
 static int
 malo_raw_xmit(struct ieee80211_node *ni, struct mbuf *m,
 	const struct ieee80211_bpf_params *params)
 {
 	struct ieee80211com *ic = ni->ni_ic;
 	struct ifnet *ifp = ic->ic_ifp;
 	struct malo_softc *sc = ifp->if_softc;
 	struct malo_txbuf *bf;
 	struct malo_txq *txq;
 
 	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || sc->malo_invalid) {
 		ieee80211_free_node(ni);
 		m_freem(m);
 		return ENETDOWN;
 	}
 
 	/*
 	 * Grab a TX buffer and associated resources.  Note that we depend
 	 * on the classification by the 802.11 layer to get to the right h/w
 	 * queue.  Management frames must ALWAYS go on queue 1 but we
 	 * cannot just force that here because we may receive non-mgt frames.
 	 */
 	txq = &sc->malo_txq[0];
 	bf = malo_getbuf(sc, txq);
 	if (bf == NULL) {
 		/* XXX blocks other traffic */
 		ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 		ieee80211_free_node(ni);
 		m_freem(m);
 		return ENOBUFS;
 	}
 
 	/*
 	 * Pass the frame to the h/w for transmission.
 	 */
 	if (malo_tx_start(sc, ni, bf, m) != 0) {
 		ifp->if_oerrors++;
 		bf->bf_m = NULL;
 		bf->bf_node = NULL;
 		MALO_TXQ_LOCK(txq);
 		STAILQ_INSERT_HEAD(&txq->free, bf, bf_list);
 		txq->nfree++;
 		MALO_TXQ_UNLOCK(txq);
 
 		ieee80211_free_node(ni);
 		return EIO;		/* XXX */
 	}
 
 	/*
 	 * NB: We don't need to lock against tx done because this just
 	 * prods the firmware to check the transmit descriptors.  The firmware
 	 * will also start fetching descriptors by itself if it notices
 	 * new ones are present when it goes to deliver a tx done interrupt
 	 * to the host. So if we race with tx done processing it's ok.
 	 * Delivering the kick here rather than in malo_tx_start is
 	 * an optimization to avoid poking the firmware for each packet.
 	 *
 	 * NB: the queue id isn't used so 0 is ok.
 	 */
 	malo_hal_txstart(sc->malo_mh, 0/*XXX*/);
 
 	return 0;
 }
 
 static void
 malo_sysctlattach(struct malo_softc *sc)
 {
 #ifdef	MALO_DEBUG
 	struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->malo_dev);
 	struct sysctl_oid *tree = device_get_sysctl_tree(sc->malo_dev);
 
 	sc->malo_debug = malo_debug;
 	SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
 		"debug", CTLFLAG_RW, &sc->malo_debug, 0,
 		"control debugging printfs");
 #endif
 }
 
 static void
 malo_announce(struct malo_softc *sc)
 {
 	struct ifnet *ifp = sc->malo_ifp;
 
 	if_printf(ifp, "versions [hw %d fw %d.%d.%d.%d] (regioncode %d)\n",
 		sc->malo_hwspecs.hwversion,
 		(sc->malo_hwspecs.fw_releasenum >> 24) & 0xff,
 		(sc->malo_hwspecs.fw_releasenum >> 16) & 0xff,
 		(sc->malo_hwspecs.fw_releasenum >> 8) & 0xff,
 		(sc->malo_hwspecs.fw_releasenum >> 0) & 0xff,
 		sc->malo_hwspecs.regioncode);
 
 	if (bootverbose || malo_rxbuf != MALO_RXBUF)
 		if_printf(ifp, "using %u rx buffers\n", malo_rxbuf);
 	if (bootverbose || malo_txbuf != MALO_TXBUF)
 		if_printf(ifp, "using %u tx buffers\n", malo_txbuf);
 }
 
 /*
  * Convert net80211 channel to a HAL channel.
  */
 static void
 malo_mapchan(struct malo_hal_channel *hc, const struct ieee80211_channel *chan)
 {
 	hc->channel = chan->ic_ieee;
 
 	*(uint32_t *)&hc->flags = 0;
 	if (IEEE80211_IS_CHAN_2GHZ(chan))
 		hc->flags.freqband = MALO_FREQ_BAND_2DOT4GHZ;
 }
 
 /*
  * Set/change channels.  If the channel is really being changed,
  * it's done by reseting the chip.  To accomplish this we must
  * first cleanup any pending DMA, then restart stuff after a la
  * malo_init.
  */
 static int
 malo_chan_set(struct malo_softc *sc, struct ieee80211_channel *chan)
 {
 	struct malo_hal *mh = sc->malo_mh;
 	struct malo_hal_channel hchan;
 
 	DPRINTF(sc, MALO_DEBUG_RESET, "%s: chan %u MHz/flags 0x%x\n",
 	    __func__, chan->ic_freq, chan->ic_flags);
 
 	/*
 	 * Convert to a HAL channel description with the flags constrained
 	 * to reflect the current operating mode.
 	 */
 	malo_mapchan(&hchan, chan);
 	malo_hal_intrset(mh, 0);		/* disable interrupts */
 	malo_hal_setchannel(mh, &hchan);
 	malo_hal_settxpower(mh, &hchan);
 
 	/*
 	 * Update internal state.
 	 */
 	sc->malo_tx_th.wt_chan_freq = htole16(chan->ic_freq);
 	sc->malo_rx_th.wr_chan_freq = htole16(chan->ic_freq);
 	if (IEEE80211_IS_CHAN_ANYG(chan)) {
 		sc->malo_tx_th.wt_chan_flags = htole16(IEEE80211_CHAN_G);
 		sc->malo_rx_th.wr_chan_flags = htole16(IEEE80211_CHAN_G);
 	} else {
 		sc->malo_tx_th.wt_chan_flags = htole16(IEEE80211_CHAN_B);
 		sc->malo_rx_th.wr_chan_flags = htole16(IEEE80211_CHAN_B);
 	}
 	sc->malo_curchan = hchan;
 	malo_hal_intrset(mh, sc->malo_imask);
 
 	return 0;
 }
 
 static void
 malo_scan_start(struct ieee80211com *ic)
 {
 	struct ifnet *ifp = ic->ic_ifp;
 	struct malo_softc *sc = ifp->if_softc;
 
 	DPRINTF(sc, MALO_DEBUG_STATE, "%s\n", __func__);
 }
 
 static void
 malo_scan_end(struct ieee80211com *ic)
 {
 	struct ifnet *ifp = ic->ic_ifp;
 	struct malo_softc *sc = ifp->if_softc;
 
 	DPRINTF(sc, MALO_DEBUG_STATE, "%s\n", __func__);
 }
 
 static void
 malo_set_channel(struct ieee80211com *ic)
 {
 	struct ifnet *ifp = ic->ic_ifp;
 	struct malo_softc *sc = ifp->if_softc;
 
 	(void) malo_chan_set(sc, ic->ic_curchan);
 }
 
 static void
 malo_rx_proc(void *arg, int npending)
 {
 #define	IEEE80211_DIR_DSTODS(wh)					\
 	((((const struct ieee80211_frame *)wh)->i_fc[1] &		\
 	    IEEE80211_FC1_DIR_MASK) == IEEE80211_FC1_DIR_DSTODS)
 	struct malo_softc *sc = arg;
 	struct ifnet *ifp = sc->malo_ifp;
 	struct ieee80211com *ic = ifp->if_l2com;
 	struct malo_rxbuf *bf;
 	struct malo_rxdesc *ds;
 	struct mbuf *m, *mnew;
 	struct ieee80211_qosframe *wh;
 	struct ieee80211_qosframe_addr4 *wh4;
 	struct ieee80211_node *ni;
 	int off, len, hdrlen, pktlen, rssi, ntodo;
 	uint8_t *data, status;
 	uint32_t readptr, writeptr;
 
 	DPRINTF(sc, MALO_DEBUG_RX_PROC,
 	    "%s: pending %u rdptr(0x%x) 0x%x wrptr(0x%x) 0x%x\n",
 	    __func__, npending,
 	    sc->malo_hwspecs.rxdesc_read,
 	    malo_bar0_read4(sc, sc->malo_hwspecs.rxdesc_read),
 	    sc->malo_hwspecs.rxdesc_write,
 	    malo_bar0_read4(sc, sc->malo_hwspecs.rxdesc_write));
 
 	readptr = malo_bar0_read4(sc, sc->malo_hwspecs.rxdesc_read);
 	writeptr = malo_bar0_read4(sc, sc->malo_hwspecs.rxdesc_write);
 	if (readptr == writeptr)
 		return;
 
 	bf = sc->malo_rxnext;
 	for (ntodo = malo_rxquota; ntodo > 0 && readptr != writeptr; ntodo--) {
 		if (bf == NULL) {
 			bf = STAILQ_FIRST(&sc->malo_rxbuf);
 			break;
 		}
 		ds = bf->bf_desc;
 		if (bf->bf_m == NULL) {
 			/*
 			 * If data allocation failed previously there
 			 * will be no buffer; try again to re-populate it.
 			 * Note the firmware will not advance to the next
 			 * descriptor with a dma buffer so we must mimic
 			 * this or we'll get out of sync.
 			 */ 
 			DPRINTF(sc, MALO_DEBUG_ANY,
 			    "%s: rx buf w/o dma memory\n", __func__);
 			(void)malo_rxbuf_init(sc, bf);
 			break;
 		}
 		MALO_RXDESC_SYNC(sc, ds,
 		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 		if (ds->rxcontrol != MALO_RXD_CTRL_DMA_OWN)
 			break;
 
 		readptr = le32toh(ds->physnext);
 
 #ifdef MALO_DEBUG
 		if (sc->malo_debug & MALO_DEBUG_RECV_DESC)
 			malo_printrxbuf(bf, 0);
 #endif
 		status = ds->status;
 		if (status & MALO_RXD_STATUS_DECRYPT_ERR_MASK) {
 			ifp->if_ierrors++;
 			goto rx_next;
 		}
 		/*
 		 * Sync the data buffer.
 		 */
 		len = le16toh(ds->pktlen);
 		bus_dmamap_sync(sc->malo_dmat, bf->bf_dmamap,
 		    BUS_DMASYNC_POSTREAD);
 		/*
 		 * The 802.11 header is provided all or in part at the front;
 		 * use it to calculate the true size of the header that we'll
 		 * construct below.  We use this to figure out where to copy
 		 * payload prior to constructing the header.
 		 */
 		m = bf->bf_m;
 		data = mtod(m, uint8_t *);
 		hdrlen = ieee80211_anyhdrsize(data + sizeof(uint16_t));
 		off = sizeof(uint16_t) + sizeof(struct ieee80211_frame_addr4);
 
 		/*
 		 * Calculate RSSI. XXX wrong
 		 */
 		rssi = 2 * ((int) ds->snr - ds->nf);	/* NB: .5 dBm  */
 		if (rssi > 100)
 			rssi = 100;
 
 		pktlen = hdrlen + (len - off);
 		/*
 		 * NB: we know our frame is at least as large as
 		 * IEEE80211_MIN_LEN because there is a 4-address frame at
 		 * the front.  Hence there's no need to vet the packet length.
 		 * If the frame in fact is too small it should be discarded
 		 * at the net80211 layer.
 		 */
 
 		/* XXX don't need mbuf, just dma buffer */
 		mnew = malo_getrxmbuf(sc, bf);
 		if (mnew == NULL) {
 			ifp->if_ierrors++;
 			goto rx_next;
 		}
 		/*
 		 * Attach the dma buffer to the mbuf; malo_rxbuf_init will
 		 * re-setup the rx descriptor using the replacement dma
 		 * buffer we just installed above.
 		 */
 		bf->bf_m = mnew;
 		m->m_data += off - hdrlen;
 		m->m_pkthdr.len = m->m_len = pktlen;
 		m->m_pkthdr.rcvif = ifp;
 
 		/*
 		 * Piece 802.11 header together.
 		 */
 		wh = mtod(m, struct ieee80211_qosframe *);
 		/* NB: don't need to do this sometimes but ... */
 		/* XXX special case so we can memcpy after m_devget? */
 		ovbcopy(data + sizeof(uint16_t), wh, hdrlen);
 		if (IEEE80211_QOS_HAS_SEQ(wh)) {
 			if (IEEE80211_DIR_DSTODS(wh)) {
 				wh4 = mtod(m,
 				    struct ieee80211_qosframe_addr4*);
 				*(uint16_t *)wh4->i_qos = ds->qosctrl;
 			} else {
 				*(uint16_t *)wh->i_qos = ds->qosctrl;
 			}
 		}
 		if (ieee80211_radiotap_active(ic)) {
 			sc->malo_rx_th.wr_flags = 0;
 			sc->malo_rx_th.wr_rate = ds->rate;
 			sc->malo_rx_th.wr_antsignal = rssi;
 			sc->malo_rx_th.wr_antnoise = ds->nf;
 		}
 #ifdef MALO_DEBUG
 		if (IFF_DUMPPKTS_RECV(sc, wh)) {
 			ieee80211_dump_pkt(ic, mtod(m, caddr_t),
 			    len, ds->rate, rssi);
 		}
 #endif
 		ifp->if_ipackets++;
 		
 		/* dispatch */
 		ni = ieee80211_find_rxnode(ic,
 		    (struct ieee80211_frame_min *)wh);
 		if (ni != NULL) {
 			(void) ieee80211_input(ni, m, rssi, ds->nf);
 			ieee80211_free_node(ni);
 		} else
 			(void) ieee80211_input_all(ic, m, rssi, ds->nf);
 rx_next:
 		/* NB: ignore ENOMEM so we process more descriptors */
 		(void) malo_rxbuf_init(sc, bf);
 		bf = STAILQ_NEXT(bf, bf_list);
 	}
 	
 	malo_bar0_write4(sc, sc->malo_hwspecs.rxdesc_read, readptr);
 	sc->malo_rxnext = bf;
 
 	if ((ifp->if_drv_flags & IFF_DRV_OACTIVE) == 0 &&
 	    !IFQ_IS_EMPTY(&ifp->if_snd))
 		malo_start(ifp);
 #undef IEEE80211_DIR_DSTODS
 }
 
 static void
 malo_stop(struct ifnet *ifp, int disable)
 {
 	struct malo_softc *sc = ifp->if_softc;
 
 	MALO_LOCK(sc);
 	malo_stop_locked(ifp, disable);
 	MALO_UNLOCK(sc);
 }
 
 /*
  * Reclaim all tx queue resources.
  */
 static void
 malo_tx_cleanup(struct malo_softc *sc)
 {
 	int i;
 
 	for (i = 0; i < MALO_NUM_TX_QUEUES; i++)
 		malo_tx_cleanupq(sc, &sc->malo_txq[i]);
 }
 
 int
 malo_detach(struct malo_softc *sc)
 {
 	struct ifnet *ifp = sc->malo_ifp;
 	struct ieee80211com *ic = ifp->if_l2com;
 
 	DPRINTF(sc, MALO_DEBUG_ANY, "%s: if_flags %x\n",
 		__func__, ifp->if_flags);
 
 	malo_stop(ifp, 1);
 
 	if (sc->malo_tq != NULL) {
 		taskqueue_drain(sc->malo_tq, &sc->malo_rxtask);
 		taskqueue_drain(sc->malo_tq, &sc->malo_txtask);
 		taskqueue_free(sc->malo_tq);
 		sc->malo_tq = NULL;
 	}
 
 	/*
 	 * NB: the order of these is important:
 	 * o call the 802.11 layer before detaching the hal to
 	 *   insure callbacks into the driver to delete global
 	 *   key cache entries can be handled
 	 * o reclaim the tx queue data structures after calling
 	 *   the 802.11 layer as we'll get called back to reclaim
 	 *   node state and potentially want to use them
 	 * o to cleanup the tx queues the hal is called, so detach
 	 *   it last
 	 * Other than that, it's straightforward...
 	 */
 	ieee80211_ifdetach(ic);
 	callout_drain(&sc->malo_watchdog_timer);
 	malo_dma_cleanup(sc);
 	malo_tx_cleanup(sc);
 	malo_hal_detach(sc->malo_mh);
 	if_free(ifp);
 
 	MALO_LOCK_DESTROY(sc);
 
 	return 0;
 }
 
 void
 malo_shutdown(struct malo_softc *sc)
 {
 	malo_stop(sc->malo_ifp, 1);
 }
 
 void
 malo_suspend(struct malo_softc *sc)
 {
 	struct ifnet *ifp = sc->malo_ifp;
 
 	DPRINTF(sc, MALO_DEBUG_ANY, "%s: if_flags %x\n",
 		__func__, ifp->if_flags);
 
 	malo_stop(ifp, 1);
 }
 
 void
 malo_resume(struct malo_softc *sc)
 {
 	struct ifnet *ifp = sc->malo_ifp;
 
 	DPRINTF(sc, MALO_DEBUG_ANY, "%s: if_flags %x\n",
 		__func__, ifp->if_flags);
 
 	if (ifp->if_flags & IFF_UP)
 		malo_init(sc);
 }
Index: projects/release-debugdist/sys/dev/malo/if_malohal.c
===================================================================
--- projects/release-debugdist/sys/dev/malo/if_malohal.c	(revision 267343)
+++ projects/release-debugdist/sys/dev/malo/if_malohal.c	(revision 267344)
@@ -1,920 +1,908 @@
 /*-
  * Copyright (c) 2007 Marvell Semiconductor, Inc.
  * Copyright (c) 2007 Sam Leffler, Errno Consulting
  * Copyright (c) 2008 Weongyo Jeong <weongyo@freebsd.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer,
  *    without modification.
  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  *    similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any
  *    redistribution must be conditioned upon including a substantially
  *    similar Disclaimer requirement for further binary redistribution.
  *
  * NO WARRANTY
  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY
  * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
  * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY,
  * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
  * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  * THE POSSIBILITY OF SUCH DAMAGES.
  */
 
 #include <sys/cdefs.h>
 #ifdef __FreeBSD__
 __FBSDID("$FreeBSD$");
 #endif
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/endian.h>
 #include <sys/kernel.h>
 #include <sys/malloc.h>
 #include <sys/firmware.h>
 #include <sys/socket.h>
 
 #include <machine/bus.h>
 #include <sys/bus.h>
 
 #include <net/if.h>
 #include <net/if_var.h>
 #include <net/if_dl.h>
 #include <net/if_media.h>
 #include <net/ethernet.h>
 
 #include <net80211/ieee80211_var.h>
 
 #include <dev/malo/if_malo.h>
 
 #define MALO_WAITOK				1
 #define MALO_NOWAIT				0
 
 #define	_CMD_SETUP(pCmd, _type, _cmd) do {				\
 	pCmd = (_type *)&mh->mh_cmdbuf[0];				\
 	memset(pCmd, 0, sizeof(_type));					\
 	pCmd->cmdhdr.cmd = htole16(_cmd);				\
 	pCmd->cmdhdr.length = htole16(sizeof(_type));			\
 } while (0)
 
 static __inline uint32_t
 malo_hal_read4(struct malo_hal *mh, bus_size_t off)
 {
 	return bus_space_read_4(mh->mh_iot, mh->mh_ioh, off);
 }
 
 static __inline void
 malo_hal_write4(struct malo_hal *mh, bus_size_t off, uint32_t val)
 {
 	bus_space_write_4(mh->mh_iot, mh->mh_ioh, off, val);
 }
 
 static void
 malo_hal_load_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
 {
 	bus_addr_t *paddr = (bus_addr_t*) arg;
 
 	KASSERT(error == 0, ("error %u on bus_dma callback", error));
 	*paddr = segs->ds_addr;
 }
 
 /*
  * Setup for communication with the device.  We allocate
  * a command buffer and map it for bus dma use.  The pci
  * device id is used to identify whether the device has
  * SRAM on it (in which case f/w download must include a
  * memory controller reset).  All bus i/o operations happen
  * in BAR 1; the driver passes in the tag and handle we need.
  */
 struct malo_hal *
 malo_hal_attach(device_t dev, uint16_t devid,
     bus_space_handle_t ioh, bus_space_tag_t iot, bus_dma_tag_t tag)
 {
 	int error;
 	struct malo_hal *mh;
 
 	mh = malloc(sizeof(struct malo_hal), M_DEVBUF, M_NOWAIT | M_ZERO);
 	if (mh == NULL)
 		return NULL;
 
 	mh->mh_dev = dev;
 	mh->mh_ioh = ioh;
 	mh->mh_iot = iot;
 
 	snprintf(mh->mh_mtxname, sizeof(mh->mh_mtxname),
 	    "%s_hal", device_get_nameunit(dev));
 	mtx_init(&mh->mh_mtx, mh->mh_mtxname, NULL, MTX_DEF);
 
 	/*
 	 * Allocate the command buffer and map into the address
 	 * space of the h/w.  We request "coherent" memory which
 	 * will be uncached on some architectures.
 	 */
 	error = bus_dma_tag_create(tag,		/* parent */
 		       PAGE_SIZE, 0,		/* alignment, bounds */
 		       BUS_SPACE_MAXADDR_32BIT,	/* lowaddr */
 		       BUS_SPACE_MAXADDR,	/* highaddr */
 		       NULL, NULL,		/* filter, filterarg */
 		       MALO_CMDBUF_SIZE,	/* maxsize */
 		       1,			/* nsegments */
 		       MALO_CMDBUF_SIZE,	/* maxsegsize */
 		       BUS_DMA_ALLOCNOW,	/* flags */
 		       NULL,			/* lockfunc */
 		       NULL,			/* lockarg */
 		       &mh->mh_dmat);
 	if (error != 0) {
 		device_printf(dev, "unable to allocate memory for cmd tag, "
 			"error %u\n", error);
 		goto fail;
 	}
 
 	/* allocate descriptors */
-	error = bus_dmamap_create(mh->mh_dmat, BUS_DMA_NOWAIT, &mh->mh_dmamap);
-	if (error != 0) {
-		device_printf(dev, "unable to create dmamap for cmd buffers, "
-			"error %u\n", error);
-		goto fail;
-	}
-
 	error = bus_dmamem_alloc(mh->mh_dmat, (void**) &mh->mh_cmdbuf,
 				 BUS_DMA_NOWAIT | BUS_DMA_COHERENT, 
 				 &mh->mh_dmamap);
 	if (error != 0) {
 		device_printf(dev, "unable to allocate memory for cmd buffer, "
 			"error %u\n", error);
 		goto fail;
 	}
 
 	error = bus_dmamap_load(mh->mh_dmat, mh->mh_dmamap,
 				mh->mh_cmdbuf, MALO_CMDBUF_SIZE,
 				malo_hal_load_cb, &mh->mh_cmdaddr,
 				BUS_DMA_NOWAIT);
 	if (error != 0) {
 		device_printf(dev, "unable to load cmd buffer, error %u\n",
 			error);
 		goto fail;
 	}
 
 	return (mh);
 
 fail:
-	if (mh->mh_dmamap != NULL) {
-		bus_dmamap_unload(mh->mh_dmat, mh->mh_dmamap);
-		if (mh->mh_cmdbuf != NULL)
-			bus_dmamem_free(mh->mh_dmat, mh->mh_cmdbuf,
-			    mh->mh_dmamap);
-		bus_dmamap_destroy(mh->mh_dmat, mh->mh_dmamap);
-	}
+	if (mh->mh_cmdbuf != NULL)
+		bus_dmamem_free(mh->mh_dmat, mh->mh_cmdbuf,
+		    mh->mh_dmamap);
 	if (mh->mh_dmat)
 		bus_dma_tag_destroy(mh->mh_dmat);
 	free(mh, M_DEVBUF);
 
 	return (NULL);
 }
 
 /*
  * Low level firmware cmd block handshake support.
  */
 
 static void
 malo_hal_send_cmd(struct malo_hal *mh)
 {
 	uint32_t dummy;
 
 	bus_dmamap_sync(mh->mh_dmat, mh->mh_dmamap,
 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 
 	malo_hal_write4(mh, MALO_REG_GEN_PTR, mh->mh_cmdaddr);
 	dummy = malo_hal_read4(mh, MALO_REG_INT_CODE);
 
 	malo_hal_write4(mh, MALO_REG_H2A_INTERRUPT_EVENTS,
 	    MALO_H2ARIC_BIT_DOOR_BELL);
 }
 
 static int
 malo_hal_waitforcmd(struct malo_hal *mh, uint16_t cmd)
 {
 #define MAX_WAIT_FW_COMPLETE_ITERATIONS 10000
 	int i;
 
 	for (i = 0; i < MAX_WAIT_FW_COMPLETE_ITERATIONS; i++) {
 		if (mh->mh_cmdbuf[0] == le16toh(cmd))
 			return 1;
 
 		DELAY(1 * 1000);
 	}
 
 	return 0;
 #undef MAX_WAIT_FW_COMPLETE_ITERATIONS
 }
 
 static int
 malo_hal_execute_cmd(struct malo_hal *mh, unsigned short cmd)
 {
 	MALO_HAL_LOCK_ASSERT(mh);
 
 	if ((mh->mh_flags & MHF_FWHANG) &&
 	    (mh->mh_debug & MALO_HAL_DEBUG_IGNHANG) == 0) {
 		device_printf(mh->mh_dev, "firmware hung, skipping cmd 0x%x\n",
 			cmd);
 		return ENXIO;
 	}
 
 	if (malo_hal_read4(mh, MALO_REG_INT_CODE) == 0xffffffff) {
 		device_printf(mh->mh_dev, "%s: device not present!\n",
 		    __func__);
 		return EIO;
 	}
 
 	malo_hal_send_cmd(mh);
 	if (!malo_hal_waitforcmd(mh, cmd | 0x8000)) {
 		device_printf(mh->mh_dev,
 		    "timeout waiting for f/w cmd 0x%x\n", cmd);
 		mh->mh_flags |= MHF_FWHANG;
 		return ETIMEDOUT;
 	}
 
 	bus_dmamap_sync(mh->mh_dmat, mh->mh_dmamap,
 	    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 
 	return 0;
 }
 
 static int
 malo_hal_get_cal_table(struct malo_hal *mh, uint8_t annex, uint8_t index)
 {
 	struct malo_cmd_caltable *cmd;
 	int ret;
 
 	MALO_HAL_LOCK_ASSERT(mh);
 
 	_CMD_SETUP(cmd, struct malo_cmd_caltable, MALO_HOSTCMD_GET_CALTABLE);
 	cmd->annex = annex;
 	cmd->index = index;
 
 	ret = malo_hal_execute_cmd(mh, MALO_HOSTCMD_GET_CALTABLE);
 	if (ret == 0 && cmd->caltbl[0] != annex && annex != 0 && annex != 255)
 		ret = EIO;
 	return ret;
 }							  
 
 static int
 malo_hal_get_pwrcal_table(struct malo_hal *mh, struct malo_hal_caldata *cal)
 {
 	const uint8_t *data;
 	int len;
 
 	MALO_HAL_LOCK(mh);
 	/* NB: we hold the lock so it's ok to use cmdbuf */
 	data = ((const struct malo_cmd_caltable *) mh->mh_cmdbuf)->caltbl;
 	if (malo_hal_get_cal_table(mh, 33, 0) == 0) {
 		len = (data[2] | (data[3] << 8)) - 12;
 		/* XXX validate len */
 		memcpy(cal->pt_ratetable_20m, &data[12], len);	
 	}
 	mh->mh_flags |= MHF_CALDATA;
 	MALO_HAL_UNLOCK(mh);
 
 	return 0;
 }
 
 /*
  * Reset internal state after a firmware download.
  */
 static int
 malo_hal_resetstate(struct malo_hal *mh)
 {
 	/*
 	 * Fetch cal data for later use.
 	 * XXX may want to fetch other stuff too.
 	 */
 	if ((mh->mh_flags & MHF_CALDATA) == 0)
 		malo_hal_get_pwrcal_table(mh, &mh->mh_caldata);
 	return 0;
 }
 
 static void
 malo_hal_fw_reset(struct malo_hal *mh)
 {
 
 	if (malo_hal_read4(mh,  MALO_REG_INT_CODE) == 0xffffffff) {
 		device_printf(mh->mh_dev, "%s: device not present!\n",
 		    __func__);
 		return;
 	}
 
 	malo_hal_write4(mh, MALO_REG_H2A_INTERRUPT_EVENTS, MALO_ISR_RESET);
 	mh->mh_flags &= ~MHF_FWHANG;
 }
 
 static void
 malo_hal_trigger_pcicmd(struct malo_hal *mh)
 {
 	uint32_t dummy;
 
 	bus_dmamap_sync(mh->mh_dmat, mh->mh_dmamap, BUS_DMASYNC_PREWRITE);
 
 	malo_hal_write4(mh, MALO_REG_GEN_PTR, mh->mh_cmdaddr);
 	dummy = malo_hal_read4(mh, MALO_REG_INT_CODE);
 
 	malo_hal_write4(mh, MALO_REG_INT_CODE, 0x00);
 	dummy = malo_hal_read4(mh, MALO_REG_INT_CODE);
 
 	malo_hal_write4(mh, MALO_REG_H2A_INTERRUPT_EVENTS,
 	    MALO_H2ARIC_BIT_DOOR_BELL);
 	dummy = malo_hal_read4(mh, MALO_REG_INT_CODE);
 }
 
 static int
 malo_hal_waitfor(struct malo_hal *mh, uint32_t val)
 {
 	int i;
 
 	for (i = 0; i < MALO_FW_MAX_NUM_CHECKS; i++) {
 		DELAY(MALO_FW_CHECK_USECS);
 		if (malo_hal_read4(mh, MALO_REG_INT_CODE) == val)
 			return 0;
 	}
 
 	return -1;
 }
 
 /*
  * Firmware block xmit when talking to the boot-rom.
  */
 static int
 malo_hal_send_helper(struct malo_hal *mh, int bsize,
     const void *data, size_t dsize, int waitfor)
 {
 	mh->mh_cmdbuf[0] = htole16(MALO_HOSTCMD_CODE_DNLD);
 	mh->mh_cmdbuf[1] = htole16(bsize);
 	memcpy(&mh->mh_cmdbuf[4], data , dsize);
 
 	malo_hal_trigger_pcicmd(mh);
 
 	if (waitfor == MALO_NOWAIT)
 		goto pass;
 
 	/* XXX 2000 vs 200 */
 	if (malo_hal_waitfor(mh, MALO_INT_CODE_CMD_FINISHED) != 0) {
 		device_printf(mh->mh_dev,
 		    "%s: timeout waiting for CMD_FINISHED, INT_CODE 0x%x\n",
 		    __func__, malo_hal_read4(mh, MALO_REG_INT_CODE));
 		
 		return ETIMEDOUT;
 	}
 
 pass:
 	malo_hal_write4(mh, MALO_REG_INT_CODE, 0);
 
 	return (0);
 }
 
 static int
 malo_hal_fwload_helper(struct malo_hal *mh, char *helper)
 {
 	const struct firmware *fw;
 	int error;
 
 	fw = firmware_get(helper);
 	if (fw == NULL) {
 		device_printf(mh->mh_dev, "could not read microcode %s!\n",
 		    helper);
 		return (EIO);
 	}
 
 	device_printf(mh->mh_dev, "load %s firmware image (%zu bytes)\n",
 	    helper, fw->datasize);
 
 	error = malo_hal_send_helper(mh, fw->datasize, fw->data, fw->datasize,
 		MALO_WAITOK);
 	if (error != 0)
 		goto fail;
 
 	/* tell the card we're done and... */
 	error = malo_hal_send_helper(mh, 0, NULL, 0, MALO_NOWAIT);
 
 fail:
 	firmware_put(fw, FIRMWARE_UNLOAD);
 
 	return (error);
 }
 
 /*
  * Firmware block xmit when talking to the 1st-stage loader.
  */
 static int
 malo_hal_send_main(struct malo_hal *mh, const void *data, size_t dsize,
     uint16_t seqnum, int waitfor)
 {
 	mh->mh_cmdbuf[0] = htole16(MALO_HOSTCMD_CODE_DNLD);
 	mh->mh_cmdbuf[1] = htole16(dsize);
 	mh->mh_cmdbuf[2] = htole16(seqnum);
 	mh->mh_cmdbuf[3] = 0;
 	memcpy(&mh->mh_cmdbuf[4], data, dsize);
 
 	malo_hal_trigger_pcicmd(mh);
 
 	if (waitfor == MALO_NOWAIT)
 		goto pass;
 
 	if (malo_hal_waitfor(mh, MALO_INT_CODE_CMD_FINISHED) != 0) {
 		device_printf(mh->mh_dev,
 		    "%s: timeout waiting for CMD_FINISHED, INT_CODE 0x%x\n",
 		    __func__, malo_hal_read4(mh, MALO_REG_INT_CODE));
 
 		return ETIMEDOUT;
 	}
 
 pass:
 	malo_hal_write4(mh, MALO_REG_INT_CODE, 0);
 
 	return 0;
 }
 
 static int
 malo_hal_fwload_main(struct malo_hal *mh, char *firmware)
 {
 	const struct firmware *fw;
 	const uint8_t *fp;
 	int error;
 	size_t count;
 	uint16_t seqnum;
 	uint32_t blocksize;
 
 	error = 0;
 
 	fw = firmware_get(firmware);
 	if (fw == NULL) {
 		device_printf(mh->mh_dev, "could not read firmware %s!\n",
 		    firmware);
 		return (EIO);
 	}
 
 	device_printf(mh->mh_dev, "load %s firmware image (%zu bytes)\n",
 	    firmware, fw->datasize);
 
 	seqnum = 1;
 	for (count = 0; count < fw->datasize; count += blocksize) {
 		blocksize = MIN(256, fw->datasize - count);
 		fp = (const uint8_t *)fw->data + count;
 
 		error = malo_hal_send_main(mh, fp, blocksize, seqnum++,
 		    MALO_NOWAIT);
 		if (error != 0)
 			goto fail;
 		DELAY(500);
 	}
 	
 	/*
 	 * send a command with size 0 to tell that the firmware has been
 	 * uploaded
 	 */
 	error = malo_hal_send_main(mh, NULL, 0, seqnum++, MALO_NOWAIT);
 	DELAY(100);
 
 fail:
 	firmware_put(fw, FIRMWARE_UNLOAD);
 
 	return (error);
 }
 
 int
 malo_hal_fwload(struct malo_hal *mh, char *helper, char *firmware)
 {
 	int error, i;
 	uint32_t fwreadysig, opmode;
 
 	/*
 	 * NB: now malo(4) supports only STA mode.  It will be better if it
 	 * supports AP mode.
 	 */
 	fwreadysig = MALO_HOSTCMD_STA_FWRDY_SIGNATURE;
 	opmode = MALO_HOSTCMD_STA_MODE;
 
 	malo_hal_fw_reset(mh);
 
 	malo_hal_write4(mh, MALO_REG_A2H_INTERRUPT_CLEAR_SEL,
 	    MALO_A2HRIC_BIT_MASK);
 	malo_hal_write4(mh, MALO_REG_A2H_INTERRUPT_CAUSE, 0x00);
 	malo_hal_write4(mh, MALO_REG_A2H_INTERRUPT_MASK, 0x00);
 	malo_hal_write4(mh, MALO_REG_A2H_INTERRUPT_STATUS_MASK,
 	    MALO_A2HRIC_BIT_MASK);
 
 	error = malo_hal_fwload_helper(mh, helper);
 	if (error != 0) {
 		device_printf(mh->mh_dev, "failed to load bootrom loader.\n");
 		goto fail;
 	}
 
 	DELAY(200 * MALO_FW_CHECK_USECS);
 
 	error = malo_hal_fwload_main(mh, firmware);
 	if (error != 0) {
 		device_printf(mh->mh_dev, "failed to load firmware.\n");
 		goto fail;
 	}
 
 	/*
 	 * Wait for firmware to startup; we monitor the INT_CODE register
 	 * waiting for a signature to written back indicating it's ready to go.
 	 */
 	mh->mh_cmdbuf[1] = 0;
 
 	if (opmode != MALO_HOSTCMD_STA_MODE)
 		malo_hal_trigger_pcicmd(mh);
 	
 	for (i = 0; i < MALO_FW_MAX_NUM_CHECKS; i++) {
 		malo_hal_write4(mh, MALO_REG_GEN_PTR, opmode);
 		DELAY(MALO_FW_CHECK_USECS);
 		if (malo_hal_read4(mh, MALO_REG_INT_CODE) == fwreadysig) {
 			malo_hal_write4(mh, MALO_REG_INT_CODE, 0x00);
 			return malo_hal_resetstate(mh);
 		}
 	}
 
 	return ETIMEDOUT;
 fail:
 	malo_hal_fw_reset(mh);
 
 	return (error);
 }
 
 /*
  * Return "hw specs".  Note this must be the first cmd MUST be done after
  * a firmware download or the f/w will lockup.
  */
 int
 malo_hal_gethwspecs(struct malo_hal *mh, struct malo_hal_hwspec *hw)
 {
 	struct malo_cmd_get_hwspec *cmd;
 	int ret;
 
 	MALO_HAL_LOCK(mh);
 
 	_CMD_SETUP(cmd, struct malo_cmd_get_hwspec, MALO_HOSTCMD_GET_HW_SPEC);
 	memset(&cmd->permaddr[0], 0xff, IEEE80211_ADDR_LEN);
 	cmd->ul_fw_awakecookie = htole32((unsigned int)mh->mh_cmdaddr + 2048);
 
 	ret = malo_hal_execute_cmd(mh, MALO_HOSTCMD_GET_HW_SPEC);
 	if (ret == 0) {
 		IEEE80211_ADDR_COPY(hw->macaddr, cmd->permaddr);
 		hw->wcbbase[0] = le32toh(cmd->wcbbase0) & 0x0000ffff;
 		hw->wcbbase[1] = le32toh(cmd->wcbbase1) & 0x0000ffff;
 		hw->wcbbase[2] = le32toh(cmd->wcbbase2) & 0x0000ffff;
 		hw->wcbbase[3] = le32toh(cmd->wcbbase3) & 0x0000ffff;
 		hw->rxdesc_read = le32toh(cmd->rxpdrd_ptr)& 0x0000ffff;
 		hw->rxdesc_write = le32toh(cmd->rxpdwr_ptr)& 0x0000ffff;
 		hw->regioncode = le16toh(cmd->regioncode) & 0x00ff;
 		hw->fw_releasenum = le32toh(cmd->fw_releasenum);
 		hw->maxnum_wcb = le16toh(cmd->num_wcb);
 		hw->maxnum_mcaddr = le16toh(cmd->num_mcastaddr);
 		hw->num_antenna = le16toh(cmd->num_antenna);
 		hw->hwversion = cmd->version;
 		hw->hostinterface = cmd->hostif;
 	}
 
 	MALO_HAL_UNLOCK(mh);
 
 	return ret;
 }
 
 void
 malo_hal_detach(struct malo_hal *mh)
 {
 
 	bus_dmamem_free(mh->mh_dmat, mh->mh_cmdbuf, mh->mh_dmamap);
-	bus_dmamap_destroy(mh->mh_dmat, mh->mh_dmamap);
 	bus_dma_tag_destroy(mh->mh_dmat);
 	mtx_destroy(&mh->mh_mtx);
 	free(mh, M_DEVBUF);
 }
 
 /*
  * Configure antenna use.  Takes effect immediately.
  *
  * XXX tx antenna setting ignored
  * XXX rx antenna setting should always be 3 (for now)
  */
 int
 malo_hal_setantenna(struct malo_hal *mh, enum malo_hal_antenna dirset, int ant)
 {
 	struct malo_cmd_rf_antenna *cmd;
 	int ret;
 
 	if (!(dirset == MHA_ANTENNATYPE_RX || dirset == MHA_ANTENNATYPE_TX))
 		return EINVAL;
 
 	MALO_HAL_LOCK(mh);
 
 	_CMD_SETUP(cmd, struct malo_cmd_rf_antenna,
 	    MALO_HOSTCMD_802_11_RF_ANTENNA);
 	cmd->action = htole16(dirset);
 	if (ant == 0) {			/* default to all/both antennae */
 		/* XXX never reach now.  */
 		ant = 3;
 	}
 	cmd->mode = htole16(ant);
 
 	ret = malo_hal_execute_cmd(mh, MALO_HOSTCMD_802_11_RF_ANTENNA);
 
 	MALO_HAL_UNLOCK(mh);
 
 	return ret;
 }
 
 /*
  * Configure radio.  Takes effect immediately.
  *
  * XXX preamble installed after set fixed rate cmd
  */
 int
 malo_hal_setradio(struct malo_hal *mh, int onoff,
     enum malo_hal_preamble preamble)
 {
 	struct malo_cmd_radio_control *cmd;
 	int ret;
 
 	MALO_HAL_LOCK(mh);
 
 	_CMD_SETUP(cmd, struct malo_cmd_radio_control,
 	    MALO_HOSTCMD_802_11_RADIO_CONTROL);
 	cmd->action = htole16(MALO_HOSTCMD_ACT_GEN_SET);
 	if (onoff == 0)
 		cmd->control = 0;
 	else
 		cmd->control = htole16(preamble);
 	cmd->radio_on = htole16(onoff);
 
 	ret = malo_hal_execute_cmd(mh, MALO_HOSTCMD_802_11_RADIO_CONTROL);
 
 	MALO_HAL_UNLOCK(mh);
 
 	return ret;
 }
 
 /*
  * Set the interrupt mask.
  */
 void
 malo_hal_intrset(struct malo_hal *mh, uint32_t mask)
 {
 
 	malo_hal_write4(mh, MALO_REG_A2H_INTERRUPT_MASK, 0);
 	(void)malo_hal_read4(mh, MALO_REG_INT_CODE);
 
 	mh->mh_imask = mask;
 	malo_hal_write4(mh, MALO_REG_A2H_INTERRUPT_MASK, mask);
 	(void)malo_hal_read4(mh, MALO_REG_INT_CODE);
 }
 
 int
 malo_hal_setchannel(struct malo_hal *mh, const struct malo_hal_channel *chan)
 {
 	struct malo_cmd_fw_set_rf_channel *cmd;
 	int ret;
 
 	MALO_HAL_LOCK(mh);
 
 	_CMD_SETUP(cmd, struct malo_cmd_fw_set_rf_channel,
 	    MALO_HOSTCMD_SET_RF_CHANNEL);
 	cmd->action = htole16(MALO_HOSTCMD_ACT_GEN_SET);
 	cmd->cur_channel = chan->channel;
 
 	ret = malo_hal_execute_cmd(mh, MALO_HOSTCMD_SET_RF_CHANNEL);
 
 	MALO_HAL_UNLOCK(mh);
 
 	return ret;
 }
 
 int
 malo_hal_settxpower(struct malo_hal *mh, const struct malo_hal_channel *c)
 {
 	struct malo_cmd_rf_tx_power *cmd;
 	const struct malo_hal_caldata *cal = &mh->mh_caldata;
 	uint8_t chan = c->channel;
 	uint16_t pow;
 	int i, idx, ret;
 	
 	MALO_HAL_LOCK(mh);
 
 	_CMD_SETUP(cmd, struct malo_cmd_rf_tx_power,
 	    MALO_HOSTCMD_802_11_RF_TX_POWER);
 	cmd->action = htole16(MALO_HOSTCMD_ACT_GEN_SET_LIST);
 	for (i = 0; i < 4; i++) {
 		idx = (chan - 1) * 4 + i;
 		pow = cal->pt_ratetable_20m[idx];
 		cmd->power_levellist[i] = htole16(pow);
 	}
 	ret = malo_hal_execute_cmd(mh, MALO_HOSTCMD_802_11_RF_TX_POWER);
 
 	MALO_HAL_UNLOCK(mh);
 
 	return ret;
 }
 
 int
 malo_hal_setpromisc(struct malo_hal *mh, int enable)
 {
 	/* XXX need host cmd */
 	return 0;
 }
 
 int
 malo_hal_setassocid(struct malo_hal *mh,
     const uint8_t bssid[IEEE80211_ADDR_LEN], uint16_t associd)
 {
 	struct malo_cmd_fw_set_aid *cmd;
 	int ret;
 
 	MALO_HAL_LOCK(mh);
 
 	_CMD_SETUP(cmd, struct malo_cmd_fw_set_aid,
 	    MALO_HOSTCMD_SET_AID);
 	cmd->cmdhdr.seqnum = 1;
 	cmd->associd = htole16(associd);
 	IEEE80211_ADDR_COPY(&cmd->macaddr[0], bssid);
 	
 	ret = malo_hal_execute_cmd(mh, MALO_HOSTCMD_SET_AID);
 	MALO_HAL_UNLOCK(mh);
 	return ret;
 }
 
 /*
  * Kick the firmware to tell it there are new tx descriptors
  * for processing.  The driver says what h/w q has work in
  * case the f/w ever gets smarter.
  */
 void
 malo_hal_txstart(struct malo_hal *mh, int qnum)
 {
 	bus_space_write_4(mh->mh_iot, mh->mh_ioh,
 	    MALO_REG_H2A_INTERRUPT_EVENTS, MALO_H2ARIC_BIT_PPA_READY);
 	(void) bus_space_read_4(mh->mh_iot, mh->mh_ioh, MALO_REG_INT_CODE);
 }
 
 /*
  * Return the current ISR setting and clear the cause.
  */
 void
 malo_hal_getisr(struct malo_hal *mh, uint32_t *status)
 {
 	uint32_t cause;
 
 	cause = bus_space_read_4(mh->mh_iot, mh->mh_ioh,
 	    MALO_REG_A2H_INTERRUPT_CAUSE);
 	if (cause == 0xffffffff) {	/* card removed */
 		cause = 0;
 	} else if (cause != 0) {
 		/* clear cause bits */
 		bus_space_write_4(mh->mh_iot, mh->mh_ioh,
 		    MALO_REG_A2H_INTERRUPT_CAUSE, cause &~ mh->mh_imask);
 		(void) bus_space_read_4(mh->mh_iot, mh->mh_ioh,
 		    MALO_REG_INT_CODE);
 		cause &= mh->mh_imask;
 	}
 
 	*status = cause;
 }
 
 /*
  * Callback from the driver on a cmd done interrupt.  Nothing to do right
  * now as we spin waiting for cmd completion.
  */
 void
 malo_hal_cmddone(struct malo_hal *mh)
 {
 	/* NB : do nothing.  */
 }
 
 int
 malo_hal_prescan(struct malo_hal *mh)
 {
 	struct malo_cmd_prescan *cmd;
 	int ret;
 
 	MALO_HAL_LOCK(mh);
 
 	_CMD_SETUP(cmd, struct malo_cmd_prescan, MALO_HOSTCMD_SET_PRE_SCAN);
 	cmd->cmdhdr.seqnum = 1;
 	
 	ret = malo_hal_execute_cmd(mh, MALO_HOSTCMD_SET_PRE_SCAN);
 
 	MALO_HAL_UNLOCK(mh);
 
 	return ret;
 }
 
 int
 malo_hal_postscan(struct malo_hal *mh, uint8_t *macaddr, uint8_t ibsson)
 {
 	struct malo_cmd_postscan *cmd;
 	int ret;
 
 	MALO_HAL_LOCK(mh);
 
 	_CMD_SETUP(cmd, struct malo_cmd_postscan, MALO_HOSTCMD_SET_POST_SCAN);
 	cmd->cmdhdr.seqnum = 1;
 	cmd->isibss = htole32(ibsson);
 	IEEE80211_ADDR_COPY(&cmd->bssid[0], macaddr);
 
 	ret = malo_hal_execute_cmd(mh, MALO_HOSTCMD_SET_POST_SCAN);
 
 	MALO_HAL_UNLOCK(mh);
 
 	return ret;
 }
 
 int
 malo_hal_set_slot(struct malo_hal *mh, int is_short)
 {
 	int ret;
 	struct malo_cmd_fw_setslot *cmd;
 
 	MALO_HAL_LOCK(mh);
 
 	_CMD_SETUP(cmd, struct malo_cmd_fw_setslot, MALO_HOSTCMD_SET_SLOT);
 	cmd->action = htole16(MALO_HOSTCMD_ACT_GEN_SET);
 	cmd->slot = (is_short == 1 ? 1 : 0);
 
 	ret = malo_hal_execute_cmd(mh, MALO_HOSTCMD_SET_SLOT);
 
 	MALO_HAL_UNLOCK(mh);
 
 	return ret;
 }
 
 int
 malo_hal_set_rate(struct malo_hal *mh, uint16_t curmode, uint8_t rate)
 {
 	int i, ret;
 	struct malo_cmd_set_rate *cmd;
 
 	MALO_HAL_LOCK(mh);
 
 	_CMD_SETUP(cmd, struct malo_cmd_set_rate, MALO_HOSTCMD_SET_RATE);
 	cmd->aprates[0] = 2;
 	cmd->aprates[1] = 4;
 	cmd->aprates[2] = 11;
 	cmd->aprates[3] = 22;
 	if (curmode == IEEE80211_MODE_11G) {
 		cmd->aprates[4] = 0;		/* XXX reserved?  */
 		cmd->aprates[5] = 12;
 		cmd->aprates[6] = 18;
 		cmd->aprates[7] = 24;
 		cmd->aprates[8] = 36;
 		cmd->aprates[9] = 48;
 		cmd->aprates[10] = 72;
 		cmd->aprates[11] = 96;
 		cmd->aprates[12] = 108;
 	}
 
 	if (rate != 0) {
 		/* fixed rate */
 		for (i = 0; i < 13; i++) {
 			if (cmd->aprates[i] == rate) {
 				cmd->rateindex = i;
 				cmd->dataratetype = 1;
 				break;
 			}
 		}
 	}
 
 	ret = malo_hal_execute_cmd(mh, MALO_HOSTCMD_SET_RATE);
 
 	MALO_HAL_UNLOCK(mh);
 
 	return ret;
 }
 
 int
 malo_hal_setmcast(struct malo_hal *mh, int nmc, const uint8_t macs[])
 {
 	struct malo_cmd_mcast *cmd;
 	int ret;
 
 	if (nmc > MALO_HAL_MCAST_MAX)
 		return EINVAL;
 
 	MALO_HAL_LOCK(mh);
 
 	_CMD_SETUP(cmd, struct malo_cmd_mcast, MALO_HOSTCMD_MAC_MULTICAST_ADR);
 	memcpy(cmd->maclist, macs, nmc * IEEE80211_ADDR_LEN);
 	cmd->numaddr = htole16(nmc);
 	cmd->action = htole16(0xffff);
 
 	ret = malo_hal_execute_cmd(mh, MALO_HOSTCMD_MAC_MULTICAST_ADR);
 
 	MALO_HAL_UNLOCK(mh);
 
 	return ret;
 }
Index: projects/release-debugdist/sys/dev/mwl/if_mwl.c
===================================================================
--- projects/release-debugdist/sys/dev/mwl/if_mwl.c	(revision 267343)
+++ projects/release-debugdist/sys/dev/mwl/if_mwl.c	(revision 267344)
@@ -1,5054 +1,5041 @@
 /*-
  * Copyright (c) 2007-2009 Sam Leffler, Errno Consulting
  * Copyright (c) 2007-2008 Marvell Semiconductor, Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer,
  *    without modification.
  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  *    similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any
  *    redistribution must be conditioned upon including a substantially
  *    similar Disclaimer requirement for further binary redistribution.
  *
  * NO WARRANTY
  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY
  * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
  * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY,
  * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
  * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  * THE POSSIBILITY OF SUCH DAMAGES.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 /*
  * Driver for the Marvell 88W8363 Wireless LAN controller.
  */
 
 #include "opt_inet.h"
 #include "opt_mwl.h"
 #include "opt_wlan.h"
 
 #include <sys/param.h>
 #include <sys/systm.h> 
 #include <sys/sysctl.h>
 #include <sys/mbuf.h>   
 #include <sys/malloc.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <sys/kernel.h>
 #include <sys/socket.h>
 #include <sys/sockio.h>
 #include <sys/errno.h>
 #include <sys/callout.h>
 #include <sys/bus.h>
 #include <sys/endian.h>
 #include <sys/kthread.h>
 #include <sys/taskqueue.h>
 
 #include <machine/bus.h>
  
 #include <net/if.h>
 #include <net/if_var.h>
 #include <net/if_dl.h>
 #include <net/if_media.h>
 #include <net/if_types.h>
 #include <net/if_arp.h>
 #include <net/ethernet.h>
 #include <net/if_llc.h>
 
 #include <net/bpf.h>
 
 #include <net80211/ieee80211_var.h>
 #include <net80211/ieee80211_regdomain.h>
 
 #ifdef INET
 #include <netinet/in.h>
 #include <netinet/if_ether.h>
 #endif /* INET */
 
 #include <dev/mwl/if_mwlvar.h>
 #include <dev/mwl/mwldiag.h>
 
 /* idiomatic shorthands: MS = mask+shift, SM = shift+mask */
 #define	MS(v,x)	(((v) & x) >> x##_S)
 #define	SM(v,x)	(((v) << x##_S) & x)
 
 static struct ieee80211vap *mwl_vap_create(struct ieee80211com *,
 		    const char [IFNAMSIZ], int, enum ieee80211_opmode, int,
 		    const uint8_t [IEEE80211_ADDR_LEN],
 		    const uint8_t [IEEE80211_ADDR_LEN]);
 static void	mwl_vap_delete(struct ieee80211vap *);
 static int	mwl_setupdma(struct mwl_softc *);
 static int	mwl_hal_reset(struct mwl_softc *sc);
 static int	mwl_init_locked(struct mwl_softc *);
 static void	mwl_init(void *);
 static void	mwl_stop_locked(struct ifnet *, int);
 static int	mwl_reset(struct ieee80211vap *, u_long);
 static void	mwl_stop(struct ifnet *, int);
 static void	mwl_start(struct ifnet *);
 static int	mwl_raw_xmit(struct ieee80211_node *, struct mbuf *,
 			const struct ieee80211_bpf_params *);
 static int	mwl_media_change(struct ifnet *);
 static void	mwl_watchdog(void *);
 static int	mwl_ioctl(struct ifnet *, u_long, caddr_t);
 static void	mwl_radar_proc(void *, int);
 static void	mwl_chanswitch_proc(void *, int);
 static void	mwl_bawatchdog_proc(void *, int);
 static int	mwl_key_alloc(struct ieee80211vap *,
 			struct ieee80211_key *,
 			ieee80211_keyix *, ieee80211_keyix *);
 static int	mwl_key_delete(struct ieee80211vap *,
 			const struct ieee80211_key *);
 static int	mwl_key_set(struct ieee80211vap *, const struct ieee80211_key *,
 			const uint8_t mac[IEEE80211_ADDR_LEN]);
 static int	mwl_mode_init(struct mwl_softc *);
 static void	mwl_update_mcast(struct ifnet *);
 static void	mwl_update_promisc(struct ifnet *);
 static void	mwl_updateslot(struct ifnet *);
 static int	mwl_beacon_setup(struct ieee80211vap *);
 static void	mwl_beacon_update(struct ieee80211vap *, int);
 #ifdef MWL_HOST_PS_SUPPORT
 static void	mwl_update_ps(struct ieee80211vap *, int);
 static int	mwl_set_tim(struct ieee80211_node *, int);
 #endif
 static int	mwl_dma_setup(struct mwl_softc *);
 static void	mwl_dma_cleanup(struct mwl_softc *);
 static struct ieee80211_node *mwl_node_alloc(struct ieee80211vap *,
 		    const uint8_t [IEEE80211_ADDR_LEN]);
 static void	mwl_node_cleanup(struct ieee80211_node *);
 static void	mwl_node_drain(struct ieee80211_node *);
 static void	mwl_node_getsignal(const struct ieee80211_node *,
 			int8_t *, int8_t *);
 static void	mwl_node_getmimoinfo(const struct ieee80211_node *,
 			struct ieee80211_mimo_info *);
 static int	mwl_rxbuf_init(struct mwl_softc *, struct mwl_rxbuf *);
 static void	mwl_rx_proc(void *, int);
 static void	mwl_txq_init(struct mwl_softc *sc, struct mwl_txq *, int);
 static int	mwl_tx_setup(struct mwl_softc *, int, int);
 static int	mwl_wme_update(struct ieee80211com *);
 static void	mwl_tx_cleanupq(struct mwl_softc *, struct mwl_txq *);
 static void	mwl_tx_cleanup(struct mwl_softc *);
 static uint16_t	mwl_calcformat(uint8_t rate, const struct ieee80211_node *);
 static int	mwl_tx_start(struct mwl_softc *, struct ieee80211_node *,
 			     struct mwl_txbuf *, struct mbuf *);
 static void	mwl_tx_proc(void *, int);
 static int	mwl_chan_set(struct mwl_softc *, struct ieee80211_channel *);
 static void	mwl_draintxq(struct mwl_softc *);
 static void	mwl_cleartxq(struct mwl_softc *, struct ieee80211vap *);
 static int	mwl_recv_action(struct ieee80211_node *,
 			const struct ieee80211_frame *,
 			const uint8_t *, const uint8_t *);
 static int	mwl_addba_request(struct ieee80211_node *,
 			struct ieee80211_tx_ampdu *, int dialogtoken,
 			int baparamset, int batimeout);
 static int	mwl_addba_response(struct ieee80211_node *,
 			struct ieee80211_tx_ampdu *, int status,
 			int baparamset, int batimeout);
 static void	mwl_addba_stop(struct ieee80211_node *,
 			struct ieee80211_tx_ampdu *);
 static int	mwl_startrecv(struct mwl_softc *);
 static MWL_HAL_APMODE mwl_getapmode(const struct ieee80211vap *,
 			struct ieee80211_channel *);
 static int	mwl_setapmode(struct ieee80211vap *, struct ieee80211_channel*);
 static void	mwl_scan_start(struct ieee80211com *);
 static void	mwl_scan_end(struct ieee80211com *);
 static void	mwl_set_channel(struct ieee80211com *);
 static int	mwl_peerstadb(struct ieee80211_node *,
 			int aid, int staid, MWL_HAL_PEERINFO *pi);
 static int	mwl_localstadb(struct ieee80211vap *);
 static int	mwl_newstate(struct ieee80211vap *, enum ieee80211_state, int);
 static int	allocstaid(struct mwl_softc *sc, int aid);
 static void	delstaid(struct mwl_softc *sc, int staid);
 static void	mwl_newassoc(struct ieee80211_node *, int);
 static void	mwl_agestations(void *);
 static int	mwl_setregdomain(struct ieee80211com *,
 			struct ieee80211_regdomain *, int,
 			struct ieee80211_channel []);
 static void	mwl_getradiocaps(struct ieee80211com *, int, int *,
 			struct ieee80211_channel []);
 static int	mwl_getchannels(struct mwl_softc *);
 
 static void	mwl_sysctlattach(struct mwl_softc *);
 static void	mwl_announce(struct mwl_softc *);
 
 SYSCTL_NODE(_hw, OID_AUTO, mwl, CTLFLAG_RD, 0, "Marvell driver parameters");
 
 static	int mwl_rxdesc = MWL_RXDESC;		/* # rx desc's to allocate */
 SYSCTL_INT(_hw_mwl, OID_AUTO, rxdesc, CTLFLAG_RW, &mwl_rxdesc,
 	    0, "rx descriptors allocated");
 static	int mwl_rxbuf = MWL_RXBUF;		/* # rx buffers to allocate */
 SYSCTL_INT(_hw_mwl, OID_AUTO, rxbuf, CTLFLAG_RW, &mwl_rxbuf,
 	    0, "rx buffers allocated");
 TUNABLE_INT("hw.mwl.rxbuf", &mwl_rxbuf);
 static	int mwl_txbuf = MWL_TXBUF;		/* # tx buffers to allocate */
 SYSCTL_INT(_hw_mwl, OID_AUTO, txbuf, CTLFLAG_RW, &mwl_txbuf,
 	    0, "tx buffers allocated");
 TUNABLE_INT("hw.mwl.txbuf", &mwl_txbuf);
 static	int mwl_txcoalesce = 8;		/* # tx packets to q before poking f/w*/
 SYSCTL_INT(_hw_mwl, OID_AUTO, txcoalesce, CTLFLAG_RW, &mwl_txcoalesce,
 	    0, "tx buffers to send at once");
 TUNABLE_INT("hw.mwl.txcoalesce", &mwl_txcoalesce);
 static	int mwl_rxquota = MWL_RXBUF;		/* # max buffers to process */
 SYSCTL_INT(_hw_mwl, OID_AUTO, rxquota, CTLFLAG_RW, &mwl_rxquota,
 	    0, "max rx buffers to process per interrupt");
 TUNABLE_INT("hw.mwl.rxquota", &mwl_rxquota);
 static	int mwl_rxdmalow = 3;			/* # min buffers for wakeup */
 SYSCTL_INT(_hw_mwl, OID_AUTO, rxdmalow, CTLFLAG_RW, &mwl_rxdmalow,
 	    0, "min free rx buffers before restarting traffic");
 TUNABLE_INT("hw.mwl.rxdmalow", &mwl_rxdmalow);
 
 #ifdef MWL_DEBUG
 static	int mwl_debug = 0;
 SYSCTL_INT(_hw_mwl, OID_AUTO, debug, CTLFLAG_RW, &mwl_debug,
 	    0, "control debugging printfs");
 TUNABLE_INT("hw.mwl.debug", &mwl_debug);
 enum {
 	MWL_DEBUG_XMIT		= 0x00000001,	/* basic xmit operation */
 	MWL_DEBUG_XMIT_DESC	= 0x00000002,	/* xmit descriptors */
 	MWL_DEBUG_RECV		= 0x00000004,	/* basic recv operation */
 	MWL_DEBUG_RECV_DESC	= 0x00000008,	/* recv descriptors */
 	MWL_DEBUG_RESET		= 0x00000010,	/* reset processing */
 	MWL_DEBUG_BEACON 	= 0x00000020,	/* beacon handling */
 	MWL_DEBUG_INTR		= 0x00000040,	/* ISR */
 	MWL_DEBUG_TX_PROC	= 0x00000080,	/* tx ISR proc */
 	MWL_DEBUG_RX_PROC	= 0x00000100,	/* rx ISR proc */
 	MWL_DEBUG_KEYCACHE	= 0x00000200,	/* key cache management */
 	MWL_DEBUG_STATE		= 0x00000400,	/* 802.11 state transitions */
 	MWL_DEBUG_NODE		= 0x00000800,	/* node management */
 	MWL_DEBUG_RECV_ALL	= 0x00001000,	/* trace all frames (beacons) */
 	MWL_DEBUG_TSO		= 0x00002000,	/* TSO processing */
 	MWL_DEBUG_AMPDU		= 0x00004000,	/* BA stream handling */
 	MWL_DEBUG_ANY		= 0xffffffff
 };
 #define	IS_BEACON(wh) \
     ((wh->i_fc[0] & (IEEE80211_FC0_TYPE_MASK|IEEE80211_FC0_SUBTYPE_MASK)) == \
 	 (IEEE80211_FC0_TYPE_MGT|IEEE80211_FC0_SUBTYPE_BEACON))
 #define	IFF_DUMPPKTS_RECV(sc, wh) \
     (((sc->sc_debug & MWL_DEBUG_RECV) && \
       ((sc->sc_debug & MWL_DEBUG_RECV_ALL) || !IS_BEACON(wh))) || \
      (sc->sc_ifp->if_flags & (IFF_DEBUG|IFF_LINK2)) == (IFF_DEBUG|IFF_LINK2))
 #define	IFF_DUMPPKTS_XMIT(sc) \
 	((sc->sc_debug & MWL_DEBUG_XMIT) || \
 	 (sc->sc_ifp->if_flags & (IFF_DEBUG|IFF_LINK2)) == (IFF_DEBUG|IFF_LINK2))
 #define	DPRINTF(sc, m, fmt, ...) do {				\
 	if (sc->sc_debug & (m))					\
 		printf(fmt, __VA_ARGS__);			\
 } while (0)
 #define	KEYPRINTF(sc, hk, mac) do {				\
 	if (sc->sc_debug & MWL_DEBUG_KEYCACHE)			\
 		mwl_keyprint(sc, __func__, hk, mac);		\
 } while (0)
 static	void mwl_printrxbuf(const struct mwl_rxbuf *bf, u_int ix);
 static	void mwl_printtxbuf(const struct mwl_txbuf *bf, u_int qnum, u_int ix);
 #else
 #define	IFF_DUMPPKTS_RECV(sc, wh) \
 	((sc->sc_ifp->if_flags & (IFF_DEBUG|IFF_LINK2)) == (IFF_DEBUG|IFF_LINK2))
 #define	IFF_DUMPPKTS_XMIT(sc) \
 	((sc->sc_ifp->if_flags & (IFF_DEBUG|IFF_LINK2)) == (IFF_DEBUG|IFF_LINK2))
 #define	DPRINTF(sc, m, fmt, ...) do {				\
 	(void) sc;						\
 } while (0)
 #define	KEYPRINTF(sc, k, mac) do {				\
 	(void) sc;						\
 } while (0)
 #endif
 
 static MALLOC_DEFINE(M_MWLDEV, "mwldev", "mwl driver dma buffers");
 
 /*
  * Each packet has fixed front matter: a 2-byte length
  * of the payload, followed by a 4-address 802.11 header
  * (regardless of the actual header and always w/o any
  * QoS header).  The payload then follows.
  */
 struct mwltxrec {
 	uint16_t fwlen;
 	struct ieee80211_frame_addr4 wh;
 } __packed;
 
 /*
  * Read/Write shorthands for accesses to BAR 0.  Note
  * that all BAR 1 operations are done in the "hal" and
  * there should be no reference to them here.
  */
 #ifdef MWL_DEBUG
 static __inline uint32_t
 RD4(struct mwl_softc *sc, bus_size_t off)
 {
 	return bus_space_read_4(sc->sc_io0t, sc->sc_io0h, off);
 }
 #endif
 
 static __inline void
 WR4(struct mwl_softc *sc, bus_size_t off, uint32_t val)
 {
 	bus_space_write_4(sc->sc_io0t, sc->sc_io0h, off, val);
 }
 
 int
 mwl_attach(uint16_t devid, struct mwl_softc *sc)
 {
 	struct ifnet *ifp;
 	struct ieee80211com *ic;
 	struct mwl_hal *mh;
 	int error = 0;
 
 	DPRINTF(sc, MWL_DEBUG_ANY, "%s: devid 0x%x\n", __func__, devid);
 
 	ifp = sc->sc_ifp = if_alloc(IFT_IEEE80211);
 	if (ifp == NULL) {
 		device_printf(sc->sc_dev, "cannot if_alloc()\n");
 		return ENOSPC;
 	}
 	ic = ifp->if_l2com;
 
 	/*
 	 * Setup the RX free list lock early, so it can be consistently
 	 * removed.
 	 */
 	MWL_RXFREE_INIT(sc);
 
 	/* set these up early for if_printf use */
 	if_initname(ifp, device_get_name(sc->sc_dev),
 		device_get_unit(sc->sc_dev));
 
 	mh = mwl_hal_attach(sc->sc_dev, devid,
 	    sc->sc_io1h, sc->sc_io1t, sc->sc_dmat);
 	if (mh == NULL) {
 		if_printf(ifp, "unable to attach HAL\n");
 		error = EIO;
 		goto bad;
 	}
 	sc->sc_mh = mh;
 	/*
 	 * Load firmware so we can get setup.  We arbitrarily
 	 * pick station firmware; we'll re-load firmware as
 	 * needed so setting up the wrong mode isn't a big deal.
 	 */
 	if (mwl_hal_fwload(mh, NULL) != 0) {
 		if_printf(ifp, "unable to setup builtin firmware\n");
 		error = EIO;
 		goto bad1;
 	}
 	if (mwl_hal_gethwspecs(mh, &sc->sc_hwspecs) != 0) {
 		if_printf(ifp, "unable to fetch h/w specs\n");
 		error = EIO;
 		goto bad1;
 	}
 	error = mwl_getchannels(sc);
 	if (error != 0)
 		goto bad1;
 
 	sc->sc_txantenna = 0;		/* h/w default */
 	sc->sc_rxantenna = 0;		/* h/w default */
 	sc->sc_invalid = 0;		/* ready to go, enable int handling */
 	sc->sc_ageinterval = MWL_AGEINTERVAL;
 
 	/*
 	 * Allocate tx+rx descriptors and populate the lists.
 	 * We immediately push the information to the firmware
 	 * as otherwise it gets upset.
 	 */
 	error = mwl_dma_setup(sc);
 	if (error != 0) {
 		if_printf(ifp, "failed to setup descriptors: %d\n", error);
 		goto bad1;
 	}
 	error = mwl_setupdma(sc);	/* push to firmware */
 	if (error != 0)			/* NB: mwl_setupdma prints msg */
 		goto bad1;
 
 	callout_init(&sc->sc_timer, CALLOUT_MPSAFE);
 	callout_init_mtx(&sc->sc_watchdog, &sc->sc_mtx, 0);
 
 	sc->sc_tq = taskqueue_create("mwl_taskq", M_NOWAIT,
 		taskqueue_thread_enqueue, &sc->sc_tq);
 	taskqueue_start_threads(&sc->sc_tq, 1, PI_NET,
 		"%s taskq", ifp->if_xname);
 
 	TASK_INIT(&sc->sc_rxtask, 0, mwl_rx_proc, sc);
 	TASK_INIT(&sc->sc_radartask, 0, mwl_radar_proc, sc);
 	TASK_INIT(&sc->sc_chanswitchtask, 0, mwl_chanswitch_proc, sc);
 	TASK_INIT(&sc->sc_bawatchdogtask, 0, mwl_bawatchdog_proc, sc);
 
 	/* NB: insure BK queue is the lowest priority h/w queue */
 	if (!mwl_tx_setup(sc, WME_AC_BK, MWL_WME_AC_BK)) {
 		if_printf(ifp, "unable to setup xmit queue for %s traffic!\n",
 			ieee80211_wme_acnames[WME_AC_BK]);
 		error = EIO;
 		goto bad2;
 	}
 	if (!mwl_tx_setup(sc, WME_AC_BE, MWL_WME_AC_BE) ||
 	    !mwl_tx_setup(sc, WME_AC_VI, MWL_WME_AC_VI) ||
 	    !mwl_tx_setup(sc, WME_AC_VO, MWL_WME_AC_VO)) {
 		/*
 		 * Not enough hardware tx queues to properly do WME;
 		 * just punt and assign them all to the same h/w queue.
 		 * We could do a better job of this if, for example,
 		 * we allocate queues when we switch from station to
 		 * AP mode.
 		 */
 		if (sc->sc_ac2q[WME_AC_VI] != NULL)
 			mwl_tx_cleanupq(sc, sc->sc_ac2q[WME_AC_VI]);
 		if (sc->sc_ac2q[WME_AC_BE] != NULL)
 			mwl_tx_cleanupq(sc, sc->sc_ac2q[WME_AC_BE]);
 		sc->sc_ac2q[WME_AC_BE] = sc->sc_ac2q[WME_AC_BK];
 		sc->sc_ac2q[WME_AC_VI] = sc->sc_ac2q[WME_AC_BK];
 		sc->sc_ac2q[WME_AC_VO] = sc->sc_ac2q[WME_AC_BK];
 	}
 	TASK_INIT(&sc->sc_txtask, 0, mwl_tx_proc, sc);
 
 	ifp->if_softc = sc;
 	ifp->if_flags = IFF_SIMPLEX | IFF_BROADCAST | IFF_MULTICAST;
 	ifp->if_start = mwl_start;
 	ifp->if_ioctl = mwl_ioctl;
 	ifp->if_init = mwl_init;
 	IFQ_SET_MAXLEN(&ifp->if_snd, ifqmaxlen);
 	ifp->if_snd.ifq_drv_maxlen = ifqmaxlen;
 	IFQ_SET_READY(&ifp->if_snd);
 
 	ic->ic_ifp = ifp;
 	/* XXX not right but it's not used anywhere important */
 	ic->ic_phytype = IEEE80211_T_OFDM;
 	ic->ic_opmode = IEEE80211_M_STA;
 	ic->ic_caps =
 		  IEEE80211_C_STA		/* station mode supported */
 		| IEEE80211_C_HOSTAP		/* hostap mode */
 		| IEEE80211_C_MONITOR		/* monitor mode */
 #if 0
 		| IEEE80211_C_IBSS		/* ibss, nee adhoc, mode */
 		| IEEE80211_C_AHDEMO		/* adhoc demo mode */
 #endif
 		| IEEE80211_C_MBSS		/* mesh point link mode */
 		| IEEE80211_C_WDS		/* WDS supported */
 		| IEEE80211_C_SHPREAMBLE	/* short preamble supported */
 		| IEEE80211_C_SHSLOT		/* short slot time supported */
 		| IEEE80211_C_WME		/* WME/WMM supported */
 		| IEEE80211_C_BURST		/* xmit bursting supported */
 		| IEEE80211_C_WPA		/* capable of WPA1+WPA2 */
 		| IEEE80211_C_BGSCAN		/* capable of bg scanning */
 		| IEEE80211_C_TXFRAG		/* handle tx frags */
 		| IEEE80211_C_TXPMGT		/* capable of txpow mgt */
 		| IEEE80211_C_DFS		/* DFS supported */
 		;
 
 	ic->ic_htcaps =
 		  IEEE80211_HTCAP_SMPS_ENA	/* SM PS mode enabled */
 		| IEEE80211_HTCAP_CHWIDTH40	/* 40MHz channel width */
 		| IEEE80211_HTCAP_SHORTGI20	/* short GI in 20MHz */
 		| IEEE80211_HTCAP_SHORTGI40	/* short GI in 40MHz */
 		| IEEE80211_HTCAP_RXSTBC_2STREAM/* 1-2 spatial streams */
 #if MWL_AGGR_SIZE == 7935
 		| IEEE80211_HTCAP_MAXAMSDU_7935	/* max A-MSDU length */
 #else
 		| IEEE80211_HTCAP_MAXAMSDU_3839	/* max A-MSDU length */
 #endif
 #if 0
 		| IEEE80211_HTCAP_PSMP		/* PSMP supported */
 		| IEEE80211_HTCAP_40INTOLERANT	/* 40MHz intolerant */
 #endif
 		/* s/w capabilities */
 		| IEEE80211_HTC_HT		/* HT operation */
 		| IEEE80211_HTC_AMPDU		/* tx A-MPDU */
 		| IEEE80211_HTC_AMSDU		/* tx A-MSDU */
 		| IEEE80211_HTC_SMPS		/* SMPS available */
 		;
 
 	/*
 	 * Mark h/w crypto support.
 	 * XXX no way to query h/w support.
 	 */
 	ic->ic_cryptocaps |= IEEE80211_CRYPTO_WEP
 			  |  IEEE80211_CRYPTO_AES_CCM
 			  |  IEEE80211_CRYPTO_TKIP
 			  |  IEEE80211_CRYPTO_TKIPMIC
 			  ;
 	/*
 	 * Transmit requires space in the packet for a special
 	 * format transmit record and optional padding between
 	 * this record and the payload.  Ask the net80211 layer
 	 * to arrange this when encapsulating packets so we can
 	 * add it efficiently. 
 	 */
 	ic->ic_headroom = sizeof(struct mwltxrec) -
 		sizeof(struct ieee80211_frame);
 
 	/* call MI attach routine. */
 	ieee80211_ifattach(ic, sc->sc_hwspecs.macAddr);
 	ic->ic_setregdomain = mwl_setregdomain;
 	ic->ic_getradiocaps = mwl_getradiocaps;
 	/* override default methods */
 	ic->ic_raw_xmit = mwl_raw_xmit;
 	ic->ic_newassoc = mwl_newassoc;
 	ic->ic_updateslot = mwl_updateslot;
 	ic->ic_update_mcast = mwl_update_mcast;
 	ic->ic_update_promisc = mwl_update_promisc;
 	ic->ic_wme.wme_update = mwl_wme_update;
 
 	ic->ic_node_alloc = mwl_node_alloc;
 	sc->sc_node_cleanup = ic->ic_node_cleanup;
 	ic->ic_node_cleanup = mwl_node_cleanup;
 	sc->sc_node_drain = ic->ic_node_drain;
 	ic->ic_node_drain = mwl_node_drain;
 	ic->ic_node_getsignal = mwl_node_getsignal;
 	ic->ic_node_getmimoinfo = mwl_node_getmimoinfo;
 
 	ic->ic_scan_start = mwl_scan_start;
 	ic->ic_scan_end = mwl_scan_end;
 	ic->ic_set_channel = mwl_set_channel;
 
 	sc->sc_recv_action = ic->ic_recv_action;
 	ic->ic_recv_action = mwl_recv_action;
 	sc->sc_addba_request = ic->ic_addba_request;
 	ic->ic_addba_request = mwl_addba_request;
 	sc->sc_addba_response = ic->ic_addba_response;
 	ic->ic_addba_response = mwl_addba_response;
 	sc->sc_addba_stop = ic->ic_addba_stop;
 	ic->ic_addba_stop = mwl_addba_stop;
 
 	ic->ic_vap_create = mwl_vap_create;
 	ic->ic_vap_delete = mwl_vap_delete;
 
 	ieee80211_radiotap_attach(ic,
 	    &sc->sc_tx_th.wt_ihdr, sizeof(sc->sc_tx_th),
 		MWL_TX_RADIOTAP_PRESENT,
 	    &sc->sc_rx_th.wr_ihdr, sizeof(sc->sc_rx_th),
 		MWL_RX_RADIOTAP_PRESENT);
 	/*
 	 * Setup dynamic sysctl's now that country code and
 	 * regdomain are available from the hal.
 	 */
 	mwl_sysctlattach(sc);
 
 	if (bootverbose)
 		ieee80211_announce(ic);
 	mwl_announce(sc);
 	return 0;
 bad2:
 	mwl_dma_cleanup(sc);
 bad1:
 	mwl_hal_detach(mh);
 bad:
 	MWL_RXFREE_DESTROY(sc);
 	if_free(ifp);
 	sc->sc_invalid = 1;
 	return error;
 }
 
 int
 mwl_detach(struct mwl_softc *sc)
 {
 	struct ifnet *ifp = sc->sc_ifp;
 	struct ieee80211com *ic = ifp->if_l2com;
 
 	DPRINTF(sc, MWL_DEBUG_ANY, "%s: if_flags %x\n",
 		__func__, ifp->if_flags);
 
 	mwl_stop(ifp, 1);
 	/*
 	 * NB: the order of these is important:
 	 * o call the 802.11 layer before detaching the hal to
 	 *   insure callbacks into the driver to delete global
 	 *   key cache entries can be handled
 	 * o reclaim the tx queue data structures after calling
 	 *   the 802.11 layer as we'll get called back to reclaim
 	 *   node state and potentially want to use them
 	 * o to cleanup the tx queues the hal is called, so detach
 	 *   it last
 	 * Other than that, it's straightforward...
 	 */
 	ieee80211_ifdetach(ic);
 	callout_drain(&sc->sc_watchdog);
 	mwl_dma_cleanup(sc);
 	MWL_RXFREE_DESTROY(sc);
 	mwl_tx_cleanup(sc);
 	mwl_hal_detach(sc->sc_mh);
 	if_free(ifp);
 
 	return 0;
 }
 
 /*
  * MAC address handling for multiple BSS on the same radio.
  * The first vap uses the MAC address from the EEPROM.  For
  * subsequent vap's we set the U/L bit (bit 1) in the MAC
  * address and use the next six bits as an index.
  */
 static void
 assign_address(struct mwl_softc *sc, uint8_t mac[IEEE80211_ADDR_LEN], int clone)
 {
 	int i;
 
 	if (clone && mwl_hal_ismbsscapable(sc->sc_mh)) {
 		/* NB: we only do this if h/w supports multiple bssid */
 		for (i = 0; i < 32; i++)
 			if ((sc->sc_bssidmask & (1<<i)) == 0)
 				break;
 		if (i != 0)
 			mac[0] |= (i << 2)|0x2;
 	} else
 		i = 0;
 	sc->sc_bssidmask |= 1<<i;
 	if (i == 0)
 		sc->sc_nbssid0++;
 }
 
 static void
 reclaim_address(struct mwl_softc *sc, uint8_t mac[IEEE80211_ADDR_LEN])
 {
 	int i = mac[0] >> 2;
 	if (i != 0 || --sc->sc_nbssid0 == 0)
 		sc->sc_bssidmask &= ~(1<<i);
 }
 
 static struct ieee80211vap *
 mwl_vap_create(struct ieee80211com *ic, const char name[IFNAMSIZ], int unit,
     enum ieee80211_opmode opmode, int flags,
     const uint8_t bssid[IEEE80211_ADDR_LEN],
     const uint8_t mac0[IEEE80211_ADDR_LEN])
 {
 	struct ifnet *ifp = ic->ic_ifp;
 	struct mwl_softc *sc = ifp->if_softc;
 	struct mwl_hal *mh = sc->sc_mh;
 	struct ieee80211vap *vap, *apvap;
 	struct mwl_hal_vap *hvap;
 	struct mwl_vap *mvp;
 	uint8_t mac[IEEE80211_ADDR_LEN];
 
 	IEEE80211_ADDR_COPY(mac, mac0);
 	switch (opmode) {
 	case IEEE80211_M_HOSTAP:
 	case IEEE80211_M_MBSS:
 		if ((flags & IEEE80211_CLONE_MACADDR) == 0)
 			assign_address(sc, mac, flags & IEEE80211_CLONE_BSSID);
 		hvap = mwl_hal_newvap(mh, MWL_HAL_AP, mac);
 		if (hvap == NULL) {
 			if ((flags & IEEE80211_CLONE_MACADDR) == 0)
 				reclaim_address(sc, mac);
 			return NULL;
 		}
 		break;
 	case IEEE80211_M_STA:
 		if ((flags & IEEE80211_CLONE_MACADDR) == 0)
 			assign_address(sc, mac, flags & IEEE80211_CLONE_BSSID);
 		hvap = mwl_hal_newvap(mh, MWL_HAL_STA, mac);
 		if (hvap == NULL) {
 			if ((flags & IEEE80211_CLONE_MACADDR) == 0)
 				reclaim_address(sc, mac);
 			return NULL;
 		}
 		/* no h/w beacon miss support; always use s/w */
 		flags |= IEEE80211_CLONE_NOBEACONS;
 		break;
 	case IEEE80211_M_WDS:
 		hvap = NULL;		/* NB: we use associated AP vap */
 		if (sc->sc_napvaps == 0)
 			return NULL;	/* no existing AP vap */
 		break;
 	case IEEE80211_M_MONITOR:
 		hvap = NULL;
 		break;
 	case IEEE80211_M_IBSS:
 	case IEEE80211_M_AHDEMO:
 	default:
 		return NULL;
 	}
 
 	mvp = (struct mwl_vap *) malloc(sizeof(struct mwl_vap),
 	    M_80211_VAP, M_NOWAIT | M_ZERO);
 	if (mvp == NULL) {
 		if (hvap != NULL) {
 			mwl_hal_delvap(hvap);
 			if ((flags & IEEE80211_CLONE_MACADDR) == 0)
 				reclaim_address(sc, mac);
 		}
 		/* XXX msg */
 		return NULL;
 	}
 	mvp->mv_hvap = hvap;
 	if (opmode == IEEE80211_M_WDS) {
 		/*
 		 * WDS vaps must have an associated AP vap; find one.
 		 * XXX not right.
 		 */
 		TAILQ_FOREACH(apvap, &ic->ic_vaps, iv_next)
 			if (apvap->iv_opmode == IEEE80211_M_HOSTAP) {
 				mvp->mv_ap_hvap = MWL_VAP(apvap)->mv_hvap;
 				break;
 			}
 		KASSERT(mvp->mv_ap_hvap != NULL, ("no ap vap"));
 	}
 	vap = &mvp->mv_vap;
 	ieee80211_vap_setup(ic, vap, name, unit, opmode, flags, bssid, mac);
 	if (hvap != NULL)
 		IEEE80211_ADDR_COPY(vap->iv_myaddr, mac);
 	/* override with driver methods */
 	mvp->mv_newstate = vap->iv_newstate;
 	vap->iv_newstate = mwl_newstate;
 	vap->iv_max_keyix = 0;	/* XXX */
 	vap->iv_key_alloc = mwl_key_alloc;
 	vap->iv_key_delete = mwl_key_delete;
 	vap->iv_key_set = mwl_key_set;
 #ifdef MWL_HOST_PS_SUPPORT
 	if (opmode == IEEE80211_M_HOSTAP || opmode == IEEE80211_M_MBSS) {
 		vap->iv_update_ps = mwl_update_ps;
 		mvp->mv_set_tim = vap->iv_set_tim;
 		vap->iv_set_tim = mwl_set_tim;
 	}
 #endif
 	vap->iv_reset = mwl_reset;
 	vap->iv_update_beacon = mwl_beacon_update;
 
 	/* override max aid so sta's cannot assoc when we're out of sta id's */
 	vap->iv_max_aid = MWL_MAXSTAID;
 	/* override default A-MPDU rx parameters */
 	vap->iv_ampdu_rxmax = IEEE80211_HTCAP_MAXRXAMPDU_64K;
 	vap->iv_ampdu_density = IEEE80211_HTCAP_MPDUDENSITY_4;
 
 	/* complete setup */
 	ieee80211_vap_attach(vap, mwl_media_change, ieee80211_media_status);
 
 	switch (vap->iv_opmode) {
 	case IEEE80211_M_HOSTAP:
 	case IEEE80211_M_MBSS:
 	case IEEE80211_M_STA:
 		/*
 		 * Setup sta db entry for local address.
 		 */
 		mwl_localstadb(vap);
 		if (vap->iv_opmode == IEEE80211_M_HOSTAP ||
 		    vap->iv_opmode == IEEE80211_M_MBSS)
 			sc->sc_napvaps++;
 		else
 			sc->sc_nstavaps++;
 		break;
 	case IEEE80211_M_WDS:
 		sc->sc_nwdsvaps++;
 		break;
 	default:
 		break;
 	}
 	/*
 	 * Setup overall operating mode.
 	 */
 	if (sc->sc_napvaps)
 		ic->ic_opmode = IEEE80211_M_HOSTAP;
 	else if (sc->sc_nstavaps)
 		ic->ic_opmode = IEEE80211_M_STA;
 	else
 		ic->ic_opmode = opmode;
 
 	return vap;
 }
 
 static void
 mwl_vap_delete(struct ieee80211vap *vap)
 {
 	struct mwl_vap *mvp = MWL_VAP(vap);
 	struct ifnet *parent = vap->iv_ic->ic_ifp;
 	struct mwl_softc *sc = parent->if_softc;
 	struct mwl_hal *mh = sc->sc_mh;
 	struct mwl_hal_vap *hvap = mvp->mv_hvap;
 	enum ieee80211_opmode opmode = vap->iv_opmode;
 
 	/* XXX disallow ap vap delete if WDS still present */
 	if (parent->if_drv_flags & IFF_DRV_RUNNING) {
 		/* quiesce h/w while we remove the vap */
 		mwl_hal_intrset(mh, 0);		/* disable interrupts */
 	}
 	ieee80211_vap_detach(vap);
 	switch (opmode) {
 	case IEEE80211_M_HOSTAP:
 	case IEEE80211_M_MBSS:
 	case IEEE80211_M_STA:
 		KASSERT(hvap != NULL, ("no hal vap handle"));
 		(void) mwl_hal_delstation(hvap, vap->iv_myaddr);
 		mwl_hal_delvap(hvap);
 		if (opmode == IEEE80211_M_HOSTAP || opmode == IEEE80211_M_MBSS)
 			sc->sc_napvaps--;
 		else
 			sc->sc_nstavaps--;
 		/* XXX don't do it for IEEE80211_CLONE_MACADDR */
 		reclaim_address(sc, vap->iv_myaddr);
 		break;
 	case IEEE80211_M_WDS:
 		sc->sc_nwdsvaps--;
 		break;
 	default:
 		break;
 	}
 	mwl_cleartxq(sc, vap);
 	free(mvp, M_80211_VAP);
 	if (parent->if_drv_flags & IFF_DRV_RUNNING)
 		mwl_hal_intrset(mh, sc->sc_imask);
 }
 
 void
 mwl_suspend(struct mwl_softc *sc)
 {
 	struct ifnet *ifp = sc->sc_ifp;
 
 	DPRINTF(sc, MWL_DEBUG_ANY, "%s: if_flags %x\n",
 		__func__, ifp->if_flags);
 
 	mwl_stop(ifp, 1);
 }
 
 void
 mwl_resume(struct mwl_softc *sc)
 {
 	struct ifnet *ifp = sc->sc_ifp;
 
 	DPRINTF(sc, MWL_DEBUG_ANY, "%s: if_flags %x\n",
 		__func__, ifp->if_flags);
 
 	if (ifp->if_flags & IFF_UP)
 		mwl_init(sc);
 }
 
 void
 mwl_shutdown(void *arg)
 {
 	struct mwl_softc *sc = arg;
 
 	mwl_stop(sc->sc_ifp, 1);
 }
 
 /*
  * Interrupt handler.  Most of the actual processing is deferred.
  */
 void
 mwl_intr(void *arg)
 {
 	struct mwl_softc *sc = arg;
 	struct mwl_hal *mh = sc->sc_mh;
 	uint32_t status;
 
 	if (sc->sc_invalid) {
 		/*
 		 * The hardware is not ready/present, don't touch anything.
 		 * Note this can happen early on if the IRQ is shared.
 		 */
 		DPRINTF(sc, MWL_DEBUG_ANY, "%s: invalid; ignored\n", __func__);
 		return;
 	}
 	/*
 	 * Figure out the reason(s) for the interrupt.
 	 */
 	mwl_hal_getisr(mh, &status);		/* NB: clears ISR too */
 	if (status == 0)			/* must be a shared irq */
 		return;
 
 	DPRINTF(sc, MWL_DEBUG_INTR, "%s: status 0x%x imask 0x%x\n",
 	    __func__, status, sc->sc_imask);
 	if (status & MACREG_A2HRIC_BIT_RX_RDY)
 		taskqueue_enqueue(sc->sc_tq, &sc->sc_rxtask);
 	if (status & MACREG_A2HRIC_BIT_TX_DONE)
 		taskqueue_enqueue(sc->sc_tq, &sc->sc_txtask);
 	if (status & MACREG_A2HRIC_BIT_BA_WATCHDOG)
 		taskqueue_enqueue(sc->sc_tq, &sc->sc_bawatchdogtask);
 	if (status & MACREG_A2HRIC_BIT_OPC_DONE)
 		mwl_hal_cmddone(mh);
 	if (status & MACREG_A2HRIC_BIT_MAC_EVENT) {
 		;
 	}
 	if (status & MACREG_A2HRIC_BIT_ICV_ERROR) {
 		/* TKIP ICV error */
 		sc->sc_stats.mst_rx_badtkipicv++;
 	}
 	if (status & MACREG_A2HRIC_BIT_QUEUE_EMPTY) {
 		/* 11n aggregation queue is empty, re-fill */
 		;
 	}
 	if (status & MACREG_A2HRIC_BIT_QUEUE_FULL) {
 		;
 	}
 	if (status & MACREG_A2HRIC_BIT_RADAR_DETECT) {
 		/* radar detected, process event */
 		taskqueue_enqueue(sc->sc_tq, &sc->sc_radartask);
 	}
 	if (status & MACREG_A2HRIC_BIT_CHAN_SWITCH) {
 		/* DFS channel switch */
 		taskqueue_enqueue(sc->sc_tq, &sc->sc_chanswitchtask);
 	}
 }
 
 static void
 mwl_radar_proc(void *arg, int pending)
 {
 	struct mwl_softc *sc = arg;
 	struct ifnet *ifp = sc->sc_ifp;
 	struct ieee80211com *ic = ifp->if_l2com;
 
 	DPRINTF(sc, MWL_DEBUG_ANY, "%s: radar detected, pending %u\n",
 	    __func__, pending);
 
 	sc->sc_stats.mst_radardetect++;
 	/* XXX stop h/w BA streams? */
 
 	IEEE80211_LOCK(ic);
 	ieee80211_dfs_notify_radar(ic, ic->ic_curchan);
 	IEEE80211_UNLOCK(ic);
 }
 
 static void
 mwl_chanswitch_proc(void *arg, int pending)
 {
 	struct mwl_softc *sc = arg;
 	struct ifnet *ifp = sc->sc_ifp;
 	struct ieee80211com *ic = ifp->if_l2com;
 
 	DPRINTF(sc, MWL_DEBUG_ANY, "%s: channel switch notice, pending %u\n",
 	    __func__, pending);
 
 	IEEE80211_LOCK(ic);
 	sc->sc_csapending = 0;
 	ieee80211_csa_completeswitch(ic);
 	IEEE80211_UNLOCK(ic);
 }
 
 static void
 mwl_bawatchdog(const MWL_HAL_BASTREAM *sp)
 {
 	struct ieee80211_node *ni = sp->data[0];
 
 	/* send DELBA and drop the stream */
 	ieee80211_ampdu_stop(ni, sp->data[1], IEEE80211_REASON_UNSPECIFIED);
 }
 
 static void
 mwl_bawatchdog_proc(void *arg, int pending)
 {
 	struct mwl_softc *sc = arg;
 	struct mwl_hal *mh = sc->sc_mh;
 	const MWL_HAL_BASTREAM *sp;
 	uint8_t bitmap, n;
 
 	sc->sc_stats.mst_bawatchdog++;
 
 	if (mwl_hal_getwatchdogbitmap(mh, &bitmap) != 0) {
 		DPRINTF(sc, MWL_DEBUG_AMPDU,
 		    "%s: could not get bitmap\n", __func__);
 		sc->sc_stats.mst_bawatchdog_failed++;
 		return;
 	}
 	DPRINTF(sc, MWL_DEBUG_AMPDU, "%s: bitmap 0x%x\n", __func__, bitmap);
 	if (bitmap == 0xff) {
 		n = 0;
 		/* disable all ba streams */
 		for (bitmap = 0; bitmap < 8; bitmap++) {
 			sp = mwl_hal_bastream_lookup(mh, bitmap);
 			if (sp != NULL) {
 				mwl_bawatchdog(sp);
 				n++;
 			}
 		}
 		if (n == 0) {
 			DPRINTF(sc, MWL_DEBUG_AMPDU,
 			    "%s: no BA streams found\n", __func__);
 			sc->sc_stats.mst_bawatchdog_empty++;
 		}
 	} else if (bitmap != 0xaa) {
 		/* disable a single ba stream */
 		sp = mwl_hal_bastream_lookup(mh, bitmap);
 		if (sp != NULL) {
 			mwl_bawatchdog(sp);
 		} else {
 			DPRINTF(sc, MWL_DEBUG_AMPDU,
 			    "%s: no BA stream %d\n", __func__, bitmap);
 			sc->sc_stats.mst_bawatchdog_notfound++;
 		}
 	}
 }
 
 /*
  * Convert net80211 channel to a HAL channel.
  */
 static void
 mwl_mapchan(MWL_HAL_CHANNEL *hc, const struct ieee80211_channel *chan)
 {
 	hc->channel = chan->ic_ieee;
 
 	*(uint32_t *)&hc->channelFlags = 0;
 	if (IEEE80211_IS_CHAN_2GHZ(chan))
 		hc->channelFlags.FreqBand = MWL_FREQ_BAND_2DOT4GHZ;
 	else if (IEEE80211_IS_CHAN_5GHZ(chan))
 		hc->channelFlags.FreqBand = MWL_FREQ_BAND_5GHZ;
 	if (IEEE80211_IS_CHAN_HT40(chan)) {
 		hc->channelFlags.ChnlWidth = MWL_CH_40_MHz_WIDTH;
 		if (IEEE80211_IS_CHAN_HT40U(chan))
 			hc->channelFlags.ExtChnlOffset = MWL_EXT_CH_ABOVE_CTRL_CH;
 		else
 			hc->channelFlags.ExtChnlOffset = MWL_EXT_CH_BELOW_CTRL_CH;
 	} else
 		hc->channelFlags.ChnlWidth = MWL_CH_20_MHz_WIDTH;
 	/* XXX 10MHz channels */
 }
 
 /*
  * Inform firmware of our tx/rx dma setup.  The BAR 0
  * writes below are for compatibility with older firmware.
  * For current firmware we send this information with a
  * cmd block via mwl_hal_sethwdma.
  */
 static int
 mwl_setupdma(struct mwl_softc *sc)
 {
 	int error, i;
 
 	sc->sc_hwdma.rxDescRead = sc->sc_rxdma.dd_desc_paddr;
 	WR4(sc, sc->sc_hwspecs.rxDescRead, sc->sc_hwdma.rxDescRead);
 	WR4(sc, sc->sc_hwspecs.rxDescWrite, sc->sc_hwdma.rxDescRead);
 
 	for (i = 0; i < MWL_NUM_TX_QUEUES-MWL_NUM_ACK_QUEUES; i++) {
 		struct mwl_txq *txq = &sc->sc_txq[i];
 		sc->sc_hwdma.wcbBase[i] = txq->dma.dd_desc_paddr;
 		WR4(sc, sc->sc_hwspecs.wcbBase[i], sc->sc_hwdma.wcbBase[i]);
 	}
 	sc->sc_hwdma.maxNumTxWcb = mwl_txbuf;
 	sc->sc_hwdma.maxNumWCB = MWL_NUM_TX_QUEUES-MWL_NUM_ACK_QUEUES;
 
 	error = mwl_hal_sethwdma(sc->sc_mh, &sc->sc_hwdma);
 	if (error != 0) {
 		device_printf(sc->sc_dev,
 		    "unable to setup tx/rx dma; hal status %u\n", error);
 		/* XXX */
 	}
 	return error;
 }
 
 /*
  * Inform firmware of tx rate parameters.
  * Called after a channel change.
  */
 static int
 mwl_setcurchanrates(struct mwl_softc *sc)
 {
 	struct ifnet *ifp = sc->sc_ifp;
 	struct ieee80211com *ic = ifp->if_l2com;
 	const struct ieee80211_rateset *rs;
 	MWL_HAL_TXRATE rates;
 
 	memset(&rates, 0, sizeof(rates));
 	rs = ieee80211_get_suprates(ic, ic->ic_curchan);
 	/* rate used to send management frames */
 	rates.MgtRate = rs->rs_rates[0] & IEEE80211_RATE_VAL;
 	/* rate used to send multicast frames */
 	rates.McastRate = rates.MgtRate;
 
 	return mwl_hal_settxrate_auto(sc->sc_mh, &rates);
 }
 
 /*
  * Inform firmware of tx rate parameters.  Called whenever
  * user-settable params change and after a channel change.
  */
 static int
 mwl_setrates(struct ieee80211vap *vap)
 {
 	struct mwl_vap *mvp = MWL_VAP(vap);
 	struct ieee80211_node *ni = vap->iv_bss;
 	const struct ieee80211_txparam *tp = ni->ni_txparms;
 	MWL_HAL_TXRATE rates;
 
 	KASSERT(vap->iv_state == IEEE80211_S_RUN, ("state %d", vap->iv_state));
 
 	/*
 	 * Update the h/w rate map.
 	 * NB: 0x80 for MCS is passed through unchanged
 	 */
 	memset(&rates, 0, sizeof(rates));
 	/* rate used to send management frames */
 	rates.MgtRate = tp->mgmtrate;
 	/* rate used to send multicast frames */
 	rates.McastRate = tp->mcastrate;
 
 	/* while here calculate EAPOL fixed rate cookie */
 	mvp->mv_eapolformat = htole16(mwl_calcformat(rates.MgtRate, ni));
 
 	return mwl_hal_settxrate(mvp->mv_hvap,
 	    tp->ucastrate != IEEE80211_FIXED_RATE_NONE ?
 		RATE_FIXED : RATE_AUTO, &rates);
 }
 
 /*
  * Setup a fixed xmit rate cookie for EAPOL frames.
  */
 static void
 mwl_seteapolformat(struct ieee80211vap *vap)
 {
 	struct mwl_vap *mvp = MWL_VAP(vap);
 	struct ieee80211_node *ni = vap->iv_bss;
 	enum ieee80211_phymode mode;
 	uint8_t rate;
 
 	KASSERT(vap->iv_state == IEEE80211_S_RUN, ("state %d", vap->iv_state));
 
 	mode = ieee80211_chan2mode(ni->ni_chan);
 	/*
 	 * Use legacy rates when operating a mixed HT+non-HT bss.
 	 * NB: this may violate POLA for sta and wds vap's.
 	 */
 	if (mode == IEEE80211_MODE_11NA &&
 	    (vap->iv_flags_ht & IEEE80211_FHT_PUREN) == 0)
 		rate = vap->iv_txparms[IEEE80211_MODE_11A].mgmtrate;
 	else if (mode == IEEE80211_MODE_11NG &&
 	    (vap->iv_flags_ht & IEEE80211_FHT_PUREN) == 0)
 		rate = vap->iv_txparms[IEEE80211_MODE_11G].mgmtrate;
 	else
 		rate = vap->iv_txparms[mode].mgmtrate;
 
 	mvp->mv_eapolformat = htole16(mwl_calcformat(rate, ni));
 }
 
 /*
  * Map SKU+country code to region code for radar bin'ing.
  */
 static int
 mwl_map2regioncode(const struct ieee80211_regdomain *rd)
 {
 	switch (rd->regdomain) {
 	case SKU_FCC:
 	case SKU_FCC3:
 		return DOMAIN_CODE_FCC;
 	case SKU_CA:
 		return DOMAIN_CODE_IC;
 	case SKU_ETSI:
 	case SKU_ETSI2:
 	case SKU_ETSI3:
 		if (rd->country == CTRY_SPAIN)
 			return DOMAIN_CODE_SPAIN;
 		if (rd->country == CTRY_FRANCE || rd->country == CTRY_FRANCE2)
 			return DOMAIN_CODE_FRANCE;
 		/* XXX force 1.3.1 radar type */
 		return DOMAIN_CODE_ETSI_131;
 	case SKU_JAPAN:
 		return DOMAIN_CODE_MKK;
 	case SKU_ROW:
 		return DOMAIN_CODE_DGT;	/* Taiwan */
 	case SKU_APAC:
 	case SKU_APAC2:
 	case SKU_APAC3:
 		return DOMAIN_CODE_AUS;	/* Australia */
 	}
 	/* XXX KOREA? */
 	return DOMAIN_CODE_FCC;			/* XXX? */
 }
 
 static int
 mwl_hal_reset(struct mwl_softc *sc)
 {
 	struct ifnet *ifp = sc->sc_ifp;
 	struct ieee80211com *ic = ifp->if_l2com;
 	struct mwl_hal *mh = sc->sc_mh;
 
 	mwl_hal_setantenna(mh, WL_ANTENNATYPE_RX, sc->sc_rxantenna);
 	mwl_hal_setantenna(mh, WL_ANTENNATYPE_TX, sc->sc_txantenna);
 	mwl_hal_setradio(mh, 1, WL_AUTO_PREAMBLE);
 	mwl_hal_setwmm(sc->sc_mh, (ic->ic_flags & IEEE80211_F_WME) != 0);
 	mwl_chan_set(sc, ic->ic_curchan);
 	/* NB: RF/RA performance tuned for indoor mode */
 	mwl_hal_setrateadaptmode(mh, 0);
 	mwl_hal_setoptimizationlevel(mh,
 	    (ic->ic_flags & IEEE80211_F_BURST) != 0);
 
 	mwl_hal_setregioncode(mh, mwl_map2regioncode(&ic->ic_regdomain));
 
 	mwl_hal_setaggampduratemode(mh, 1, 80);		/* XXX */
 	mwl_hal_setcfend(mh, 0);			/* XXX */
 
 	return 1;
 }
 
 static int
 mwl_init_locked(struct mwl_softc *sc)
 {
 	struct ifnet *ifp = sc->sc_ifp;
 	struct mwl_hal *mh = sc->sc_mh;
 	int error = 0;
 
 	DPRINTF(sc, MWL_DEBUG_ANY, "%s: if_flags 0x%x\n",
 		__func__, ifp->if_flags);
 
 	MWL_LOCK_ASSERT(sc);
 
 	/*
 	 * Stop anything previously setup.  This is safe
 	 * whether this is the first time through or not.
 	 */
 	mwl_stop_locked(ifp, 0);
 
 	/*
 	 * Push vap-independent state to the firmware.
 	 */
 	if (!mwl_hal_reset(sc)) {
 		if_printf(ifp, "unable to reset hardware\n");
 		return EIO;
 	}
 
 	/*
 	 * Setup recv (once); transmit is already good to go.
 	 */
 	error = mwl_startrecv(sc);
 	if (error != 0) {
 		if_printf(ifp, "unable to start recv logic\n");
 		return error;
 	}
 
 	/*
 	 * Enable interrupts.
 	 */
 	sc->sc_imask = MACREG_A2HRIC_BIT_RX_RDY
 		     | MACREG_A2HRIC_BIT_TX_DONE
 		     | MACREG_A2HRIC_BIT_OPC_DONE
 #if 0
 		     | MACREG_A2HRIC_BIT_MAC_EVENT
 #endif
 		     | MACREG_A2HRIC_BIT_ICV_ERROR
 		     | MACREG_A2HRIC_BIT_RADAR_DETECT
 		     | MACREG_A2HRIC_BIT_CHAN_SWITCH
 #if 0
 		     | MACREG_A2HRIC_BIT_QUEUE_EMPTY
 #endif
 		     | MACREG_A2HRIC_BIT_BA_WATCHDOG
 		     | MACREQ_A2HRIC_BIT_TX_ACK
 		     ;
 
 	ifp->if_drv_flags |= IFF_DRV_RUNNING;
 	mwl_hal_intrset(mh, sc->sc_imask);
 	callout_reset(&sc->sc_watchdog, hz, mwl_watchdog, sc);
 
 	return 0;
 }
 
 static void
 mwl_init(void *arg)
 {
 	struct mwl_softc *sc = arg;
 	struct ifnet *ifp = sc->sc_ifp;
 	struct ieee80211com *ic = ifp->if_l2com;
 	int error = 0;
 
 	DPRINTF(sc, MWL_DEBUG_ANY, "%s: if_flags 0x%x\n",
 		__func__, ifp->if_flags);
 
 	MWL_LOCK(sc);
 	error = mwl_init_locked(sc);
 	MWL_UNLOCK(sc);
 
 	if (error == 0)
 		ieee80211_start_all(ic);	/* start all vap's */
 }
 
 static void
 mwl_stop_locked(struct ifnet *ifp, int disable)
 {
 	struct mwl_softc *sc = ifp->if_softc;
 
 	DPRINTF(sc, MWL_DEBUG_ANY, "%s: invalid %u if_flags 0x%x\n",
 		__func__, sc->sc_invalid, ifp->if_flags);
 
 	MWL_LOCK_ASSERT(sc);
 	if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 		/*
 		 * Shutdown the hardware and driver.
 		 */
 		ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
 		callout_stop(&sc->sc_watchdog);
 		sc->sc_tx_timer = 0;
 		mwl_draintxq(sc);
 	}
 }
 
 static void
 mwl_stop(struct ifnet *ifp, int disable)
 {
 	struct mwl_softc *sc = ifp->if_softc;
 
 	MWL_LOCK(sc);
 	mwl_stop_locked(ifp, disable);
 	MWL_UNLOCK(sc);
 }
 
 static int
 mwl_reset_vap(struct ieee80211vap *vap, int state)
 {
 	struct mwl_hal_vap *hvap = MWL_VAP(vap)->mv_hvap;
 	struct ieee80211com *ic = vap->iv_ic;
 
 	if (state == IEEE80211_S_RUN)
 		mwl_setrates(vap);
 	/* XXX off by 1? */
 	mwl_hal_setrtsthreshold(hvap, vap->iv_rtsthreshold);
 	/* XXX auto? 20/40 split? */
 	mwl_hal_sethtgi(hvap, (vap->iv_flags_ht &
 	    (IEEE80211_FHT_SHORTGI20|IEEE80211_FHT_SHORTGI40)) ? 1 : 0);
 	mwl_hal_setnprot(hvap, ic->ic_htprotmode == IEEE80211_PROT_NONE ?
 	    HTPROTECT_NONE : HTPROTECT_AUTO);
 	/* XXX txpower cap */
 
 	/* re-setup beacons */
 	if (state == IEEE80211_S_RUN &&
 	    (vap->iv_opmode == IEEE80211_M_HOSTAP ||
 	     vap->iv_opmode == IEEE80211_M_MBSS ||
 	     vap->iv_opmode == IEEE80211_M_IBSS)) {
 		mwl_setapmode(vap, vap->iv_bss->ni_chan);
 		mwl_hal_setnprotmode(hvap,
 		    MS(ic->ic_curhtprotmode, IEEE80211_HTINFO_OPMODE));
 		return mwl_beacon_setup(vap);
 	}
 	return 0;
 }
 
 /*
  * Reset the hardware w/o losing operational state.
  * Used to to reset or reload hardware state for a vap.
  */
 static int
 mwl_reset(struct ieee80211vap *vap, u_long cmd)
 {
 	struct mwl_hal_vap *hvap = MWL_VAP(vap)->mv_hvap;
 	int error = 0;
 
 	if (hvap != NULL) {			/* WDS, MONITOR, etc. */
 		struct ieee80211com *ic = vap->iv_ic;
 		struct ifnet *ifp = ic->ic_ifp;
 		struct mwl_softc *sc = ifp->if_softc;
 		struct mwl_hal *mh = sc->sc_mh;
 
 		/* XXX handle DWDS sta vap change */
 		/* XXX do we need to disable interrupts? */
 		mwl_hal_intrset(mh, 0);		/* disable interrupts */
 		error = mwl_reset_vap(vap, vap->iv_state);
 		mwl_hal_intrset(mh, sc->sc_imask);
 	}
 	return error;
 }
 
 /*
  * Allocate a tx buffer for sending a frame.  The
  * packet is assumed to have the WME AC stored so
  * we can use it to select the appropriate h/w queue.
  */
 static struct mwl_txbuf *
 mwl_gettxbuf(struct mwl_softc *sc, struct mwl_txq *txq)
 {
 	struct mwl_txbuf *bf;
 
 	/*
 	 * Grab a TX buffer and associated resources.
 	 */
 	MWL_TXQ_LOCK(txq);
 	bf = STAILQ_FIRST(&txq->free);
 	if (bf != NULL) {
 		STAILQ_REMOVE_HEAD(&txq->free, bf_list);
 		txq->nfree--;
 	}
 	MWL_TXQ_UNLOCK(txq);
 	if (bf == NULL)
 		DPRINTF(sc, MWL_DEBUG_XMIT,
 		    "%s: out of xmit buffers on q %d\n", __func__, txq->qnum);
 	return bf;
 }
 
 /*
  * Return a tx buffer to the queue it came from.  Note there
  * are two cases because we must preserve the order of buffers
  * as it reflects the fixed order of descriptors in memory
  * (the firmware pre-fetches descriptors so we cannot reorder).
  */
 static void
 mwl_puttxbuf_head(struct mwl_txq *txq, struct mwl_txbuf *bf)
 {
 	bf->bf_m = NULL;
 	bf->bf_node = NULL;
 	MWL_TXQ_LOCK(txq);
 	STAILQ_INSERT_HEAD(&txq->free, bf, bf_list);
 	txq->nfree++;
 	MWL_TXQ_UNLOCK(txq);
 }
 
 static void
 mwl_puttxbuf_tail(struct mwl_txq *txq, struct mwl_txbuf *bf)
 {
 	bf->bf_m = NULL;
 	bf->bf_node = NULL;
 	MWL_TXQ_LOCK(txq);
 	STAILQ_INSERT_TAIL(&txq->free, bf, bf_list);
 	txq->nfree++;
 	MWL_TXQ_UNLOCK(txq);
 }
 
 static void
 mwl_start(struct ifnet *ifp)
 {
 	struct mwl_softc *sc = ifp->if_softc;
 	struct ieee80211_node *ni;
 	struct mwl_txbuf *bf;
 	struct mbuf *m;
 	struct mwl_txq *txq = NULL;	/* XXX silence gcc */
 	int nqueued;
 
 	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || sc->sc_invalid)
 		return;
 	nqueued = 0;
 	for (;;) {
 		bf = NULL;
 		IFQ_DEQUEUE(&ifp->if_snd, m);
 		if (m == NULL)
 			break;
 		/*
 		 * Grab the node for the destination.
 		 */
 		ni = (struct ieee80211_node *) m->m_pkthdr.rcvif;
 		KASSERT(ni != NULL, ("no node"));
 		m->m_pkthdr.rcvif = NULL;	/* committed, clear ref */
 		/*
 		 * Grab a TX buffer and associated resources.
 		 * We honor the classification by the 802.11 layer.
 		 */
 		txq = sc->sc_ac2q[M_WME_GETAC(m)];
 		bf = mwl_gettxbuf(sc, txq);
 		if (bf == NULL) {
 			m_freem(m);
 			ieee80211_free_node(ni);
 #ifdef MWL_TX_NODROP
 			sc->sc_stats.mst_tx_qstop++;
 			/* XXX blocks other traffic */
 			ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 			break;
 #else
 			DPRINTF(sc, MWL_DEBUG_XMIT,
 			    "%s: tail drop on q %d\n", __func__, txq->qnum);
 			sc->sc_stats.mst_tx_qdrop++;
 			continue;
 #endif /* MWL_TX_NODROP */
 		}
 
 		/*
 		 * Pass the frame to the h/w for transmission.
 		 */
 		if (mwl_tx_start(sc, ni, bf, m)) {
 			ifp->if_oerrors++;
 			mwl_puttxbuf_head(txq, bf);
 			ieee80211_free_node(ni);
 			continue;
 		}
 		nqueued++;
 		if (nqueued >= mwl_txcoalesce) {
 			/*
 			 * Poke the firmware to process queued frames;
 			 * see below about (lack of) locking.
 			 */
 			nqueued = 0;
 			mwl_hal_txstart(sc->sc_mh, 0/*XXX*/);
 		}
 	}
 	if (nqueued) {
 		/*
 		 * NB: We don't need to lock against tx done because
 		 * this just prods the firmware to check the transmit
 		 * descriptors.  The firmware will also start fetching
 		 * descriptors by itself if it notices new ones are
 		 * present when it goes to deliver a tx done interrupt
 		 * to the host. So if we race with tx done processing
 		 * it's ok.  Delivering the kick here rather than in
 		 * mwl_tx_start is an optimization to avoid poking the
 		 * firmware for each packet.
 		 *
 		 * NB: the queue id isn't used so 0 is ok.
 		 */
 		mwl_hal_txstart(sc->sc_mh, 0/*XXX*/);
 	}
 }
 
 static int
 mwl_raw_xmit(struct ieee80211_node *ni, struct mbuf *m,
 	const struct ieee80211_bpf_params *params)
 {
 	struct ieee80211com *ic = ni->ni_ic;
 	struct ifnet *ifp = ic->ic_ifp;
 	struct mwl_softc *sc = ifp->if_softc;
 	struct mwl_txbuf *bf;
 	struct mwl_txq *txq;
 
 	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0 || sc->sc_invalid) {
 		ieee80211_free_node(ni);
 		m_freem(m);
 		return ENETDOWN;
 	}
 	/*
 	 * Grab a TX buffer and associated resources.
 	 * Note that we depend on the classification
 	 * by the 802.11 layer to get to the right h/w
 	 * queue.  Management frames must ALWAYS go on
 	 * queue 1 but we cannot just force that here
 	 * because we may receive non-mgt frames.
 	 */
 	txq = sc->sc_ac2q[M_WME_GETAC(m)];
 	bf = mwl_gettxbuf(sc, txq);
 	if (bf == NULL) {
 		sc->sc_stats.mst_tx_qstop++;
 		/* XXX blocks other traffic */
 		ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 		ieee80211_free_node(ni);
 		m_freem(m);
 		return ENOBUFS;
 	}
 	/*
 	 * Pass the frame to the h/w for transmission.
 	 */
 	if (mwl_tx_start(sc, ni, bf, m)) {
 		ifp->if_oerrors++;
 		mwl_puttxbuf_head(txq, bf);
 
 		ieee80211_free_node(ni);
 		return EIO;		/* XXX */
 	}
 	/*
 	 * NB: We don't need to lock against tx done because
 	 * this just prods the firmware to check the transmit
 	 * descriptors.  The firmware will also start fetching
 	 * descriptors by itself if it notices new ones are
 	 * present when it goes to deliver a tx done interrupt
 	 * to the host. So if we race with tx done processing
 	 * it's ok.  Delivering the kick here rather than in
 	 * mwl_tx_start is an optimization to avoid poking the
 	 * firmware for each packet.
 	 *
 	 * NB: the queue id isn't used so 0 is ok.
 	 */
 	mwl_hal_txstart(sc->sc_mh, 0/*XXX*/);
 	return 0;
 }
 
 static int
 mwl_media_change(struct ifnet *ifp)
 {
 	struct ieee80211vap *vap = ifp->if_softc;
 	int error;
 
 	error = ieee80211_media_change(ifp);
 	/* NB: only the fixed rate can change and that doesn't need a reset */
 	if (error == ENETRESET) {
 		mwl_setrates(vap);
 		error = 0;
 	}
 	return error;
 }
 
 #ifdef MWL_DEBUG
 static void
 mwl_keyprint(struct mwl_softc *sc, const char *tag,
 	const MWL_HAL_KEYVAL *hk, const uint8_t mac[IEEE80211_ADDR_LEN])
 {
 	static const char *ciphers[] = {
 		"WEP",
 		"TKIP",
 		"AES-CCM",
 	};
 	int i, n;
 
 	printf("%s: [%u] %-7s", tag, hk->keyIndex, ciphers[hk->keyTypeId]);
 	for (i = 0, n = hk->keyLen; i < n; i++)
 		printf(" %02x", hk->key.aes[i]);
 	printf(" mac %s", ether_sprintf(mac));
 	if (hk->keyTypeId == KEY_TYPE_ID_TKIP) {
 		printf(" %s", "rxmic");
 		for (i = 0; i < sizeof(hk->key.tkip.rxMic); i++)
 			printf(" %02x", hk->key.tkip.rxMic[i]);
 		printf(" txmic");
 		for (i = 0; i < sizeof(hk->key.tkip.txMic); i++)
 			printf(" %02x", hk->key.tkip.txMic[i]);
 	}
 	printf(" flags 0x%x\n", hk->keyFlags);
 }
 #endif
 
 /*
  * Allocate a key cache slot for a unicast key.  The
  * firmware handles key allocation and every station is
  * guaranteed key space so we are always successful.
  */
 static int
 mwl_key_alloc(struct ieee80211vap *vap, struct ieee80211_key *k,
 	ieee80211_keyix *keyix, ieee80211_keyix *rxkeyix)
 {
 	struct mwl_softc *sc = vap->iv_ic->ic_ifp->if_softc;
 
 	if (k->wk_keyix != IEEE80211_KEYIX_NONE ||
 	    (k->wk_flags & IEEE80211_KEY_GROUP)) {
 		if (!(&vap->iv_nw_keys[0] <= k &&
 		      k < &vap->iv_nw_keys[IEEE80211_WEP_NKID])) {
 			/* should not happen */
 			DPRINTF(sc, MWL_DEBUG_KEYCACHE,
 				"%s: bogus group key\n", __func__);
 			return 0;
 		}
 		/* give the caller what they requested */
 		*keyix = *rxkeyix = k - vap->iv_nw_keys;
 	} else {
 		/*
 		 * Firmware handles key allocation.
 		 */
 		*keyix = *rxkeyix = 0;
 	}
 	return 1;
 }
 
 /*
  * Delete a key entry allocated by mwl_key_alloc.
  */
 static int
 mwl_key_delete(struct ieee80211vap *vap, const struct ieee80211_key *k)
 {
 	struct mwl_softc *sc = vap->iv_ic->ic_ifp->if_softc;
 	struct mwl_hal_vap *hvap = MWL_VAP(vap)->mv_hvap;
 	MWL_HAL_KEYVAL hk;
 	const uint8_t bcastaddr[IEEE80211_ADDR_LEN] =
 	    { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff };
 
 	if (hvap == NULL) {
 		if (vap->iv_opmode != IEEE80211_M_WDS) {
 			/* XXX monitor mode? */
 			DPRINTF(sc, MWL_DEBUG_KEYCACHE,
 			    "%s: no hvap for opmode %d\n", __func__,
 			    vap->iv_opmode);
 			return 0;
 		}
 		hvap = MWL_VAP(vap)->mv_ap_hvap;
 	}
 
 	DPRINTF(sc, MWL_DEBUG_KEYCACHE, "%s: delete key %u\n",
 	    __func__, k->wk_keyix);
 
 	memset(&hk, 0, sizeof(hk));
 	hk.keyIndex = k->wk_keyix;
 	switch (k->wk_cipher->ic_cipher) {
 	case IEEE80211_CIPHER_WEP:
 		hk.keyTypeId = KEY_TYPE_ID_WEP;
 		break;
 	case IEEE80211_CIPHER_TKIP:
 		hk.keyTypeId = KEY_TYPE_ID_TKIP;
 		break;
 	case IEEE80211_CIPHER_AES_CCM:
 		hk.keyTypeId = KEY_TYPE_ID_AES;
 		break;
 	default:
 		/* XXX should not happen */
 		DPRINTF(sc, MWL_DEBUG_KEYCACHE, "%s: unknown cipher %d\n",
 		    __func__, k->wk_cipher->ic_cipher);
 		return 0;
 	}
 	return (mwl_hal_keyreset(hvap, &hk, bcastaddr) == 0);	/*XXX*/
 }
 
 static __inline int
 addgroupflags(MWL_HAL_KEYVAL *hk, const struct ieee80211_key *k)
 {
 	if (k->wk_flags & IEEE80211_KEY_GROUP) {
 		if (k->wk_flags & IEEE80211_KEY_XMIT)
 			hk->keyFlags |= KEY_FLAG_TXGROUPKEY;
 		if (k->wk_flags & IEEE80211_KEY_RECV)
 			hk->keyFlags |= KEY_FLAG_RXGROUPKEY;
 		return 1;
 	} else
 		return 0;
 }
 
 /*
  * Set the key cache contents for the specified key.  Key cache
  * slot(s) must already have been allocated by mwl_key_alloc.
  */
 static int
 mwl_key_set(struct ieee80211vap *vap, const struct ieee80211_key *k,
 	const uint8_t mac[IEEE80211_ADDR_LEN])
 {
 #define	GRPXMIT	(IEEE80211_KEY_XMIT | IEEE80211_KEY_GROUP)
 /* NB: static wep keys are marked GROUP+tx/rx; GTK will be tx or rx */
 #define	IEEE80211_IS_STATICKEY(k) \
 	(((k)->wk_flags & (GRPXMIT|IEEE80211_KEY_RECV)) == \
 	 (GRPXMIT|IEEE80211_KEY_RECV))
 	struct mwl_softc *sc = vap->iv_ic->ic_ifp->if_softc;
 	struct mwl_hal_vap *hvap = MWL_VAP(vap)->mv_hvap;
 	const struct ieee80211_cipher *cip = k->wk_cipher;
 	const uint8_t *macaddr;
 	MWL_HAL_KEYVAL hk;
 
 	KASSERT((k->wk_flags & IEEE80211_KEY_SWCRYPT) == 0,
 		("s/w crypto set?"));
 
 	if (hvap == NULL) {
 		if (vap->iv_opmode != IEEE80211_M_WDS) {
 			/* XXX monitor mode? */
 			DPRINTF(sc, MWL_DEBUG_KEYCACHE,
 			    "%s: no hvap for opmode %d\n", __func__,
 			    vap->iv_opmode);
 			return 0;
 		}
 		hvap = MWL_VAP(vap)->mv_ap_hvap;
 	}
 	memset(&hk, 0, sizeof(hk));
 	hk.keyIndex = k->wk_keyix;
 	switch (cip->ic_cipher) {
 	case IEEE80211_CIPHER_WEP:
 		hk.keyTypeId = KEY_TYPE_ID_WEP;
 		hk.keyLen = k->wk_keylen;
 		if (k->wk_keyix == vap->iv_def_txkey)
 			hk.keyFlags = KEY_FLAG_WEP_TXKEY;
 		if (!IEEE80211_IS_STATICKEY(k)) {
 			/* NB: WEP is never used for the PTK */
 			(void) addgroupflags(&hk, k);
 		}
 		break;
 	case IEEE80211_CIPHER_TKIP:
 		hk.keyTypeId = KEY_TYPE_ID_TKIP;
 		hk.key.tkip.tsc.high = (uint32_t)(k->wk_keytsc >> 16);
 		hk.key.tkip.tsc.low = (uint16_t)k->wk_keytsc;
 		hk.keyFlags = KEY_FLAG_TSC_VALID | KEY_FLAG_MICKEY_VALID;
 		hk.keyLen = k->wk_keylen + IEEE80211_MICBUF_SIZE;
 		if (!addgroupflags(&hk, k))
 			hk.keyFlags |= KEY_FLAG_PAIRWISE;
 		break;
 	case IEEE80211_CIPHER_AES_CCM:
 		hk.keyTypeId = KEY_TYPE_ID_AES;
 		hk.keyLen = k->wk_keylen;
 		if (!addgroupflags(&hk, k))
 			hk.keyFlags |= KEY_FLAG_PAIRWISE;
 		break;
 	default:
 		/* XXX should not happen */
 		DPRINTF(sc, MWL_DEBUG_KEYCACHE, "%s: unknown cipher %d\n",
 		    __func__, k->wk_cipher->ic_cipher);
 		return 0;
 	}
 	/*
 	 * NB: tkip mic keys get copied here too; the layout
 	 *     just happens to match that in ieee80211_key.
 	 */
 	memcpy(hk.key.aes, k->wk_key, hk.keyLen);
 
 	/*
 	 * Locate address of sta db entry for writing key;
 	 * the convention unfortunately is somewhat different
 	 * than how net80211, hostapd, and wpa_supplicant think.
 	 */
 	if (vap->iv_opmode == IEEE80211_M_STA) {
 		/*
 		 * NB: keys plumbed before the sta reaches AUTH state
 		 * will be discarded or written to the wrong sta db
 		 * entry because iv_bss is meaningless.  This is ok
 		 * (right now) because we handle deferred plumbing of
 		 * WEP keys when the sta reaches AUTH state.
 		 */
 		macaddr = vap->iv_bss->ni_bssid;
 		if ((k->wk_flags & IEEE80211_KEY_GROUP) == 0) {
 			/* XXX plumb to local sta db too for static key wep */
 			mwl_hal_keyset(hvap, &hk, vap->iv_myaddr);
 		}
 	} else if (vap->iv_opmode == IEEE80211_M_WDS &&
 	    vap->iv_state != IEEE80211_S_RUN) {
 		/*
 		 * Prior to RUN state a WDS vap will not it's BSS node
 		 * setup so we will plumb the key to the wrong mac
 		 * address (it'll be our local address).  Workaround
 		 * this for the moment by grabbing the correct address.
 		 */
 		macaddr = vap->iv_des_bssid;
 	} else if ((k->wk_flags & GRPXMIT) == GRPXMIT)
 		macaddr = vap->iv_myaddr;
 	else
 		macaddr = mac;
 	KEYPRINTF(sc, &hk, macaddr);
 	return (mwl_hal_keyset(hvap, &hk, macaddr) == 0);
 #undef IEEE80211_IS_STATICKEY
 #undef GRPXMIT
 }
 
 /* unaligned little endian access */
 #define LE_READ_2(p)				\
 	((uint16_t)				\
 	 ((((const uint8_t *)(p))[0]      ) |	\
 	  (((const uint8_t *)(p))[1] <<  8)))
 #define LE_READ_4(p)				\
 	((uint32_t)				\
 	 ((((const uint8_t *)(p))[0]      ) |	\
 	  (((const uint8_t *)(p))[1] <<  8) |	\
 	  (((const uint8_t *)(p))[2] << 16) |	\
 	  (((const uint8_t *)(p))[3] << 24)))
 
 /*
  * Set the multicast filter contents into the hardware.
  * XXX f/w has no support; just defer to the os.
  */
 static void
 mwl_setmcastfilter(struct mwl_softc *sc)
 {
 	struct ifnet *ifp = sc->sc_ifp;
 #if 0
 	struct ether_multi *enm;
 	struct ether_multistep estep;
 	uint8_t macs[IEEE80211_ADDR_LEN*MWL_HAL_MCAST_MAX];/* XXX stack use */
 	uint8_t *mp;
 	int nmc;
 
 	mp = macs;
 	nmc = 0;
 	ETHER_FIRST_MULTI(estep, &sc->sc_ec, enm);
 	while (enm != NULL) {
 		/* XXX Punt on ranges. */
 		if (nmc == MWL_HAL_MCAST_MAX ||
 		    !IEEE80211_ADDR_EQ(enm->enm_addrlo, enm->enm_addrhi)) {
 			ifp->if_flags |= IFF_ALLMULTI;
 			return;
 		}
 		IEEE80211_ADDR_COPY(mp, enm->enm_addrlo);
 		mp += IEEE80211_ADDR_LEN, nmc++;
 		ETHER_NEXT_MULTI(estep, enm);
 	}
 	ifp->if_flags &= ~IFF_ALLMULTI;
 	mwl_hal_setmcast(sc->sc_mh, nmc, macs);
 #else
 	/* XXX no mcast filter support; we get everything */
 	ifp->if_flags |= IFF_ALLMULTI;
 #endif
 }
 
 static int
 mwl_mode_init(struct mwl_softc *sc)
 {
 	struct ifnet *ifp = sc->sc_ifp;
 	struct ieee80211com *ic = ifp->if_l2com;
 	struct mwl_hal *mh = sc->sc_mh;
 
 	/*
 	 * NB: Ignore promisc in hostap mode; it's set by the
 	 * bridge.  This is wrong but we have no way to
 	 * identify internal requests (from the bridge)
 	 * versus external requests such as for tcpdump.
 	 */
 	mwl_hal_setpromisc(mh, (ifp->if_flags & IFF_PROMISC) &&
 	    ic->ic_opmode != IEEE80211_M_HOSTAP);
 	mwl_setmcastfilter(sc);
 
 	return 0;
 }
 
 /*
  * Callback from the 802.11 layer after a multicast state change.
  */
 static void
 mwl_update_mcast(struct ifnet *ifp)
 {
 	struct mwl_softc *sc = ifp->if_softc;
 
 	mwl_setmcastfilter(sc);
 }
 
 /*
  * Callback from the 802.11 layer after a promiscuous mode change.
  * Note this interface does not check the operating mode as this
  * is an internal callback and we are expected to honor the current
  * state (e.g. this is used for setting the interface in promiscuous
  * mode when operating in hostap mode to do ACS).
  */
 static void
 mwl_update_promisc(struct ifnet *ifp)
 {
 	struct mwl_softc *sc = ifp->if_softc;
 
 	mwl_hal_setpromisc(sc->sc_mh, (ifp->if_flags & IFF_PROMISC) != 0);
 }
 
 /*
  * Callback from the 802.11 layer to update the slot time
  * based on the current setting.  We use it to notify the
  * firmware of ERP changes and the f/w takes care of things
  * like slot time and preamble.
  */
 static void
 mwl_updateslot(struct ifnet *ifp)
 {
 	struct mwl_softc *sc = ifp->if_softc;
 	struct ieee80211com *ic = ifp->if_l2com;
 	struct mwl_hal *mh = sc->sc_mh;
 	int prot;
 
 	/* NB: can be called early; suppress needless cmds */
 	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0)
 		return;
 
 	/*
 	 * Calculate the ERP flags.  The firwmare will use
 	 * this to carry out the appropriate measures.
 	 */
 	prot = 0;
 	if (IEEE80211_IS_CHAN_ANYG(ic->ic_curchan)) {
 		if ((ic->ic_flags & IEEE80211_F_SHSLOT) == 0)
 			prot |= IEEE80211_ERP_NON_ERP_PRESENT;
 		if (ic->ic_flags & IEEE80211_F_USEPROT)
 			prot |= IEEE80211_ERP_USE_PROTECTION;
 		if (ic->ic_flags & IEEE80211_F_USEBARKER)
 			prot |= IEEE80211_ERP_LONG_PREAMBLE;
 	}
 
 	DPRINTF(sc, MWL_DEBUG_RESET,
 	    "%s: chan %u MHz/flags 0x%x %s slot, (prot 0x%x ic_flags 0x%x)\n",
 	    __func__, ic->ic_curchan->ic_freq, ic->ic_curchan->ic_flags,
 	    ic->ic_flags & IEEE80211_F_SHSLOT ? "short" : "long", prot,
 	    ic->ic_flags);
 
 	mwl_hal_setgprot(mh, prot);
 }
 
 /*
  * Setup the beacon frame.
  */
 static int
 mwl_beacon_setup(struct ieee80211vap *vap)
 {
 	struct mwl_hal_vap *hvap = MWL_VAP(vap)->mv_hvap;
 	struct ieee80211_node *ni = vap->iv_bss;
 	struct ieee80211_beacon_offsets bo;
 	struct mbuf *m;
 
 	m = ieee80211_beacon_alloc(ni, &bo);
 	if (m == NULL)
 		return ENOBUFS;
 	mwl_hal_setbeacon(hvap, mtod(m, const void *), m->m_len);
 	m_free(m);
 
 	return 0;
 }
 
 /*
  * Update the beacon frame in response to a change.
  */
 static void
 mwl_beacon_update(struct ieee80211vap *vap, int item)
 {
 	struct mwl_hal_vap *hvap = MWL_VAP(vap)->mv_hvap;
 	struct ieee80211com *ic = vap->iv_ic;
 
 	KASSERT(hvap != NULL, ("no beacon"));
 	switch (item) {
 	case IEEE80211_BEACON_ERP:
 		mwl_updateslot(ic->ic_ifp);
 		break;
 	case IEEE80211_BEACON_HTINFO:
 		mwl_hal_setnprotmode(hvap,
 		    MS(ic->ic_curhtprotmode, IEEE80211_HTINFO_OPMODE));
 		break;
 	case IEEE80211_BEACON_CAPS:
 	case IEEE80211_BEACON_WME:
 	case IEEE80211_BEACON_APPIE:
 	case IEEE80211_BEACON_CSA:
 		break;
 	case IEEE80211_BEACON_TIM:
 		/* NB: firmware always forms TIM */
 		return;
 	}
 	/* XXX retain beacon frame and update */
 	mwl_beacon_setup(vap);
 }
 
 static void
 mwl_load_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
 {
 	bus_addr_t *paddr = (bus_addr_t*) arg;
 	KASSERT(error == 0, ("error %u on bus_dma callback", error));
 	*paddr = segs->ds_addr;
 }
 
 #ifdef MWL_HOST_PS_SUPPORT
 /*
  * Handle power save station occupancy changes.
  */
 static void
 mwl_update_ps(struct ieee80211vap *vap, int nsta)
 {
 	struct mwl_vap *mvp = MWL_VAP(vap);
 
 	if (nsta == 0 || mvp->mv_last_ps_sta == 0)
 		mwl_hal_setpowersave_bss(mvp->mv_hvap, nsta);
 	mvp->mv_last_ps_sta = nsta;
 }
 
 /*
  * Handle associated station power save state changes.
  */
 static int
 mwl_set_tim(struct ieee80211_node *ni, int set)
 {
 	struct ieee80211vap *vap = ni->ni_vap;
 	struct mwl_vap *mvp = MWL_VAP(vap);
 
 	if (mvp->mv_set_tim(ni, set)) {		/* NB: state change */
 		mwl_hal_setpowersave_sta(mvp->mv_hvap,
 		    IEEE80211_AID(ni->ni_associd), set);
 		return 1;
 	} else
 		return 0;
 }
 #endif /* MWL_HOST_PS_SUPPORT */
 
 static int
 mwl_desc_setup(struct mwl_softc *sc, const char *name,
 	struct mwl_descdma *dd,
 	int nbuf, size_t bufsize, int ndesc, size_t descsize)
 {
 	struct ifnet *ifp = sc->sc_ifp;
 	uint8_t *ds;
 	int error;
 
 	DPRINTF(sc, MWL_DEBUG_RESET,
 	    "%s: %s DMA: %u bufs (%ju) %u desc/buf (%ju)\n",
 	    __func__, name, nbuf, (uintmax_t) bufsize,
 	    ndesc, (uintmax_t) descsize);
 
 	dd->dd_name = name;
 	dd->dd_desc_len = nbuf * ndesc * descsize;
 
 	/*
 	 * Setup DMA descriptor area.
 	 */
 	error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev),	/* parent */
 		       PAGE_SIZE, 0,		/* alignment, bounds */
 		       BUS_SPACE_MAXADDR_32BIT,	/* lowaddr */
 		       BUS_SPACE_MAXADDR,	/* highaddr */
 		       NULL, NULL,		/* filter, filterarg */
 		       dd->dd_desc_len,		/* maxsize */
 		       1,			/* nsegments */
 		       dd->dd_desc_len,		/* maxsegsize */
 		       BUS_DMA_ALLOCNOW,	/* flags */
 		       NULL,			/* lockfunc */
 		       NULL,			/* lockarg */
 		       &dd->dd_dmat);
 	if (error != 0) {
 		if_printf(ifp, "cannot allocate %s DMA tag\n", dd->dd_name);
 		return error;
 	}
 
 	/* allocate descriptors */
-	error = bus_dmamap_create(dd->dd_dmat, BUS_DMA_NOWAIT, &dd->dd_dmamap);
-	if (error != 0) {
-		if_printf(ifp, "unable to create dmamap for %s descriptors, "
-			"error %u\n", dd->dd_name, error);
-		goto fail0;
-	}
-
 	error = bus_dmamem_alloc(dd->dd_dmat, (void**) &dd->dd_desc,
 				 BUS_DMA_NOWAIT | BUS_DMA_COHERENT, 
 				 &dd->dd_dmamap);
 	if (error != 0) {
 		if_printf(ifp, "unable to alloc memory for %u %s descriptors, "
 			"error %u\n", nbuf * ndesc, dd->dd_name, error);
 		goto fail1;
 	}
 
 	error = bus_dmamap_load(dd->dd_dmat, dd->dd_dmamap,
 				dd->dd_desc, dd->dd_desc_len,
 				mwl_load_cb, &dd->dd_desc_paddr,
 				BUS_DMA_NOWAIT);
 	if (error != 0) {
 		if_printf(ifp, "unable to map %s descriptors, error %u\n",
 			dd->dd_name, error);
 		goto fail2;
 	}
 
 	ds = dd->dd_desc;
 	memset(ds, 0, dd->dd_desc_len);
 	DPRINTF(sc, MWL_DEBUG_RESET, "%s: %s DMA map: %p (%lu) -> %p (%lu)\n",
 	    __func__, dd->dd_name, ds, (u_long) dd->dd_desc_len,
 	    (caddr_t) dd->dd_desc_paddr, /*XXX*/ (u_long) dd->dd_desc_len);
 
 	return 0;
 fail2:
 	bus_dmamem_free(dd->dd_dmat, dd->dd_desc, dd->dd_dmamap);
 fail1:
-	bus_dmamap_destroy(dd->dd_dmat, dd->dd_dmamap);
-fail0:
 	bus_dma_tag_destroy(dd->dd_dmat);
 	memset(dd, 0, sizeof(*dd));
 	return error;
 #undef DS2PHYS
 }
 
 static void
 mwl_desc_cleanup(struct mwl_softc *sc, struct mwl_descdma *dd)
 {
 	bus_dmamap_unload(dd->dd_dmat, dd->dd_dmamap);
 	bus_dmamem_free(dd->dd_dmat, dd->dd_desc, dd->dd_dmamap);
-	bus_dmamap_destroy(dd->dd_dmat, dd->dd_dmamap);
 	bus_dma_tag_destroy(dd->dd_dmat);
 
 	memset(dd, 0, sizeof(*dd));
 }
 
 /* 
  * Construct a tx q's free list.  The order of entries on
  * the list must reflect the physical layout of tx descriptors
  * because the firmware pre-fetches descriptors.
  *
  * XXX might be better to use indices into the buffer array.
  */
 static void
 mwl_txq_reset(struct mwl_softc *sc, struct mwl_txq *txq)
 {
 	struct mwl_txbuf *bf;
 	int i;
 
 	bf = txq->dma.dd_bufptr;
 	STAILQ_INIT(&txq->free);
 	for (i = 0; i < mwl_txbuf; i++, bf++)
 		STAILQ_INSERT_TAIL(&txq->free, bf, bf_list);
 	txq->nfree = i;
 }
 
 #define	DS2PHYS(_dd, _ds) \
 	((_dd)->dd_desc_paddr + ((caddr_t)(_ds) - (caddr_t)(_dd)->dd_desc))
 
 static int
 mwl_txdma_setup(struct mwl_softc *sc, struct mwl_txq *txq)
 {
 	struct ifnet *ifp = sc->sc_ifp;
 	int error, bsize, i;
 	struct mwl_txbuf *bf;
 	struct mwl_txdesc *ds;
 
 	error = mwl_desc_setup(sc, "tx", &txq->dma,
 			mwl_txbuf, sizeof(struct mwl_txbuf),
 			MWL_TXDESC, sizeof(struct mwl_txdesc));
 	if (error != 0)
 		return error;
 
 	/* allocate and setup tx buffers */
 	bsize = mwl_txbuf * sizeof(struct mwl_txbuf);
 	bf = malloc(bsize, M_MWLDEV, M_NOWAIT | M_ZERO);
 	if (bf == NULL) {
 		if_printf(ifp, "malloc of %u tx buffers failed\n",
 			mwl_txbuf);
 		return ENOMEM;
 	}
 	txq->dma.dd_bufptr = bf;
 
 	ds = txq->dma.dd_desc;
 	for (i = 0; i < mwl_txbuf; i++, bf++, ds += MWL_TXDESC) {
 		bf->bf_desc = ds;
 		bf->bf_daddr = DS2PHYS(&txq->dma, ds);
 		error = bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT,
 				&bf->bf_dmamap);
 		if (error != 0) {
 			if_printf(ifp, "unable to create dmamap for tx "
 				"buffer %u, error %u\n", i, error);
 			return error;
 		}
 	}
 	mwl_txq_reset(sc, txq);
 	return 0;
 }
 
 static void
 mwl_txdma_cleanup(struct mwl_softc *sc, struct mwl_txq *txq)
 {
 	struct mwl_txbuf *bf;
 	int i;
 
 	bf = txq->dma.dd_bufptr;
 	for (i = 0; i < mwl_txbuf; i++, bf++) {
 		KASSERT(bf->bf_m == NULL, ("mbuf on free list"));
 		KASSERT(bf->bf_node == NULL, ("node on free list"));
 		if (bf->bf_dmamap != NULL)
 			bus_dmamap_destroy(sc->sc_dmat, bf->bf_dmamap);
 	}
 	STAILQ_INIT(&txq->free);
 	txq->nfree = 0;
 	if (txq->dma.dd_bufptr != NULL) {
 		free(txq->dma.dd_bufptr, M_MWLDEV);
 		txq->dma.dd_bufptr = NULL;
 	}
 	if (txq->dma.dd_desc_len != 0)
 		mwl_desc_cleanup(sc, &txq->dma);
 }
 
 static int
 mwl_rxdma_setup(struct mwl_softc *sc)
 {
 	struct ifnet *ifp = sc->sc_ifp;
 	int error, jumbosize, bsize, i;
 	struct mwl_rxbuf *bf;
 	struct mwl_jumbo *rbuf;
 	struct mwl_rxdesc *ds;
 	caddr_t data;
 
 	error = mwl_desc_setup(sc, "rx", &sc->sc_rxdma,
 			mwl_rxdesc, sizeof(struct mwl_rxbuf),
 			1, sizeof(struct mwl_rxdesc));
 	if (error != 0)
 		return error;
 
 	/*
 	 * Receive is done to a private pool of jumbo buffers.
 	 * This allows us to attach to mbuf's and avoid re-mapping
 	 * memory on each rx we post.  We allocate a large chunk
 	 * of memory and manage it in the driver.  The mbuf free
 	 * callback method is used to reclaim frames after sending
 	 * them up the stack.  By default we allocate 2x the number of
 	 * rx descriptors configured so we have some slop to hold
 	 * us while frames are processed.
 	 */
 	if (mwl_rxbuf < 2*mwl_rxdesc) {
 		if_printf(ifp,
 		    "too few rx dma buffers (%d); increasing to %d\n",
 		    mwl_rxbuf, 2*mwl_rxdesc);
 		mwl_rxbuf = 2*mwl_rxdesc;
 	}
 	jumbosize = roundup(MWL_AGGR_SIZE, PAGE_SIZE);
 	sc->sc_rxmemsize = mwl_rxbuf*jumbosize;
 
 	error = bus_dma_tag_create(sc->sc_dmat,	/* parent */
 		       PAGE_SIZE, 0,		/* alignment, bounds */
 		       BUS_SPACE_MAXADDR_32BIT,	/* lowaddr */
 		       BUS_SPACE_MAXADDR,	/* highaddr */
 		       NULL, NULL,		/* filter, filterarg */
 		       sc->sc_rxmemsize,	/* maxsize */
 		       1,			/* nsegments */
 		       sc->sc_rxmemsize,	/* maxsegsize */
 		       BUS_DMA_ALLOCNOW,	/* flags */
 		       NULL,			/* lockfunc */
 		       NULL,			/* lockarg */
 		       &sc->sc_rxdmat);
-	error = bus_dmamap_create(sc->sc_rxdmat, BUS_DMA_NOWAIT, &sc->sc_rxmap);
 	if (error != 0) {
-		if_printf(ifp, "could not create rx DMA map\n");
+		if_printf(ifp, "could not create rx DMA tag\n");
 		return error;
 	}
 
 	error = bus_dmamem_alloc(sc->sc_rxdmat, (void**) &sc->sc_rxmem,
 				 BUS_DMA_NOWAIT | BUS_DMA_COHERENT, 
 				 &sc->sc_rxmap);
 	if (error != 0) {
 		if_printf(ifp, "could not alloc %ju bytes of rx DMA memory\n",
 		    (uintmax_t) sc->sc_rxmemsize);
 		return error;
 	}
 
 	error = bus_dmamap_load(sc->sc_rxdmat, sc->sc_rxmap,
 				sc->sc_rxmem, sc->sc_rxmemsize,
 				mwl_load_cb, &sc->sc_rxmem_paddr,
 				BUS_DMA_NOWAIT);
 	if (error != 0) {
 		if_printf(ifp, "could not load rx DMA map\n");
 		return error;
 	}
 
 	/*
 	 * Allocate rx buffers and set them up.
 	 */
 	bsize = mwl_rxdesc * sizeof(struct mwl_rxbuf);
 	bf = malloc(bsize, M_MWLDEV, M_NOWAIT | M_ZERO);
 	if (bf == NULL) {
 		if_printf(ifp, "malloc of %u rx buffers failed\n", bsize);
 		return error;
 	}
 	sc->sc_rxdma.dd_bufptr = bf;
 
 	STAILQ_INIT(&sc->sc_rxbuf);
 	ds = sc->sc_rxdma.dd_desc;
 	for (i = 0; i < mwl_rxdesc; i++, bf++, ds++) {
 		bf->bf_desc = ds;
 		bf->bf_daddr = DS2PHYS(&sc->sc_rxdma, ds);
 		/* pre-assign dma buffer */
 		bf->bf_data = ((uint8_t *)sc->sc_rxmem) + (i*jumbosize);
 		/* NB: tail is intentional to preserve descriptor order */
 		STAILQ_INSERT_TAIL(&sc->sc_rxbuf, bf, bf_list);
 	}
 
 	/*
 	 * Place remainder of dma memory buffers on the free list.
 	 */
 	SLIST_INIT(&sc->sc_rxfree);
 	for (; i < mwl_rxbuf; i++) {
 		data = ((uint8_t *)sc->sc_rxmem) + (i*jumbosize);
 		rbuf = MWL_JUMBO_DATA2BUF(data);
 		SLIST_INSERT_HEAD(&sc->sc_rxfree, rbuf, next);
 		sc->sc_nrxfree++;
 	}
 	return 0;
 }
 #undef DS2PHYS
 
 static void
 mwl_rxdma_cleanup(struct mwl_softc *sc)
 {
-	if (sc->sc_rxmap != NULL)
+	if (sc->sc_rxmem_paddr != 0) {
 		bus_dmamap_unload(sc->sc_rxdmat, sc->sc_rxmap);
+		sc->sc_rxmem_paddr = 0;
+	}
 	if (sc->sc_rxmem != NULL) {
 		bus_dmamem_free(sc->sc_rxdmat, sc->sc_rxmem, sc->sc_rxmap);
 		sc->sc_rxmem = NULL;
-	}
-	if (sc->sc_rxmap != NULL) {
-		bus_dmamap_destroy(sc->sc_rxdmat, sc->sc_rxmap);
-		sc->sc_rxmap = NULL;
 	}
 	if (sc->sc_rxdma.dd_bufptr != NULL) {
 		free(sc->sc_rxdma.dd_bufptr, M_MWLDEV);
 		sc->sc_rxdma.dd_bufptr = NULL;
 	}
 	if (sc->sc_rxdma.dd_desc_len != 0)
 		mwl_desc_cleanup(sc, &sc->sc_rxdma);
 }
 
 static int
 mwl_dma_setup(struct mwl_softc *sc)
 {
 	int error, i;
 
 	error = mwl_rxdma_setup(sc);
 	if (error != 0) {
 		mwl_rxdma_cleanup(sc);
 		return error;
 	}
 
 	for (i = 0; i < MWL_NUM_TX_QUEUES; i++) {
 		error = mwl_txdma_setup(sc, &sc->sc_txq[i]);
 		if (error != 0) {
 			mwl_dma_cleanup(sc);
 			return error;
 		}
 	}
 	return 0;
 }
 
 static void
 mwl_dma_cleanup(struct mwl_softc *sc)
 {
 	int i;
 
 	for (i = 0; i < MWL_NUM_TX_QUEUES; i++)
 		mwl_txdma_cleanup(sc, &sc->sc_txq[i]);
 	mwl_rxdma_cleanup(sc);
 }
 
 static struct ieee80211_node *
 mwl_node_alloc(struct ieee80211vap *vap, const uint8_t mac[IEEE80211_ADDR_LEN])
 {
 	struct ieee80211com *ic = vap->iv_ic;
 	struct mwl_softc *sc = ic->ic_ifp->if_softc;
 	const size_t space = sizeof(struct mwl_node);
 	struct mwl_node *mn;
 
 	mn = malloc(space, M_80211_NODE, M_NOWAIT|M_ZERO);
 	if (mn == NULL) {
 		/* XXX stat+msg */
 		return NULL;
 	}
 	DPRINTF(sc, MWL_DEBUG_NODE, "%s: mn %p\n", __func__, mn);
 	return &mn->mn_node;
 }
 
 static void
 mwl_node_cleanup(struct ieee80211_node *ni)
 {
 	struct ieee80211com *ic = ni->ni_ic;
         struct mwl_softc *sc = ic->ic_ifp->if_softc;
 	struct mwl_node *mn = MWL_NODE(ni);
 
 	DPRINTF(sc, MWL_DEBUG_NODE, "%s: ni %p ic %p staid %d\n",
 	    __func__, ni, ni->ni_ic, mn->mn_staid);
 
 	if (mn->mn_staid != 0) {
 		struct ieee80211vap *vap = ni->ni_vap;
 
 		if (mn->mn_hvap != NULL) {
 			if (vap->iv_opmode == IEEE80211_M_STA)
 				mwl_hal_delstation(mn->mn_hvap, vap->iv_myaddr);
 			else
 				mwl_hal_delstation(mn->mn_hvap, ni->ni_macaddr);
 		}
 		/*
 		 * NB: legacy WDS peer sta db entry is installed using
 		 * the associate ap's hvap; use it again to delete it.
 		 * XXX can vap be NULL?
 		 */
 		else if (vap->iv_opmode == IEEE80211_M_WDS &&
 		    MWL_VAP(vap)->mv_ap_hvap != NULL)
 			mwl_hal_delstation(MWL_VAP(vap)->mv_ap_hvap,
 			    ni->ni_macaddr);
 		delstaid(sc, mn->mn_staid);
 		mn->mn_staid = 0;
 	}
 	sc->sc_node_cleanup(ni);
 }
 
 /*
  * Reclaim rx dma buffers from packets sitting on the ampdu
  * reorder queue for a station.  We replace buffers with a
  * system cluster (if available).
  */
 static void
 mwl_ampdu_rxdma_reclaim(struct ieee80211_rx_ampdu *rap)
 {
 #if 0
 	int i, n, off;
 	struct mbuf *m;
 	void *cl;
 
 	n = rap->rxa_qframes;
 	for (i = 0; i < rap->rxa_wnd && n > 0; i++) {
 		m = rap->rxa_m[i];
 		if (m == NULL)
 			continue;
 		n--;
 		/* our dma buffers have a well-known free routine */
 		if ((m->m_flags & M_EXT) == 0 ||
 		    m->m_ext.ext_free != mwl_ext_free)
 			continue;
 		/*
 		 * Try to allocate a cluster and move the data.
 		 */
 		off = m->m_data - m->m_ext.ext_buf;
 		if (off + m->m_pkthdr.len > MCLBYTES) {
 			/* XXX no AMSDU for now */
 			continue;
 		}
 		cl = pool_cache_get_paddr(&mclpool_cache, 0,
 		    &m->m_ext.ext_paddr);
 		if (cl != NULL) {
 			/*
 			 * Copy the existing data to the cluster, remove
 			 * the rx dma buffer, and attach the cluster in
 			 * its place.  Note we preserve the offset to the
 			 * data so frames being bridged can still prepend
 			 * their headers without adding another mbuf.
 			 */
 			memcpy((caddr_t) cl + off, m->m_data, m->m_pkthdr.len);
 			MEXTREMOVE(m);
 			MEXTADD(m, cl, MCLBYTES, 0, NULL, &mclpool_cache);
 			/* setup mbuf like _MCLGET does */
 			m->m_flags |= M_CLUSTER | M_EXT_RW;
 			_MOWNERREF(m, M_EXT | M_CLUSTER);
 			/* NB: m_data is clobbered by MEXTADDR, adjust */
 			m->m_data += off;
 		}
 	}
 #endif
 }
 
 /*
  * Callback to reclaim resources.  We first let the
  * net80211 layer do it's thing, then if we are still
  * blocked by a lack of rx dma buffers we walk the ampdu
  * reorder q's to reclaim buffers by copying to a system
  * cluster.
  */
 static void
 mwl_node_drain(struct ieee80211_node *ni)
 {
 	struct ieee80211com *ic = ni->ni_ic;
         struct mwl_softc *sc = ic->ic_ifp->if_softc;
 	struct mwl_node *mn = MWL_NODE(ni);
 
 	DPRINTF(sc, MWL_DEBUG_NODE, "%s: ni %p vap %p staid %d\n",
 	    __func__, ni, ni->ni_vap, mn->mn_staid);
 
 	/* NB: call up first to age out ampdu q's */
 	sc->sc_node_drain(ni);
 
 	/* XXX better to not check low water mark? */
 	if (sc->sc_rxblocked && mn->mn_staid != 0 &&
 	    (ni->ni_flags & IEEE80211_NODE_HT)) {
 		uint8_t tid;
 		/*
 		 * Walk the reorder q and reclaim rx dma buffers by copying
 		 * the packet contents into clusters.
 		 */
 		for (tid = 0; tid < WME_NUM_TID; tid++) {
 			struct ieee80211_rx_ampdu *rap;
 
 			rap = &ni->ni_rx_ampdu[tid];
 			if ((rap->rxa_flags & IEEE80211_AGGR_XCHGPEND) == 0)
 				continue;
 			if (rap->rxa_qframes)
 				mwl_ampdu_rxdma_reclaim(rap);
 		}
 	}
 }
 
 static void
 mwl_node_getsignal(const struct ieee80211_node *ni, int8_t *rssi, int8_t *noise)
 {
 	*rssi = ni->ni_ic->ic_node_getrssi(ni);
 #ifdef MWL_ANT_INFO_SUPPORT
 #if 0
 	/* XXX need to smooth data */
 	*noise = -MWL_NODE_CONST(ni)->mn_ai.nf;
 #else
 	*noise = -95;		/* XXX */
 #endif
 #else
 	*noise = -95;		/* XXX */
 #endif
 }
 
 /*
  * Convert Hardware per-antenna rssi info to common format:
  * Let a1, a2, a3 represent the amplitudes per chain
  * Let amax represent max[a1, a2, a3]
  * Rssi1_dBm = RSSI_dBm + 20*log10(a1/amax)
  * Rssi1_dBm = RSSI_dBm + 20*log10(a1) - 20*log10(amax)
  * We store a table that is 4*20*log10(idx) - the extra 4 is to store or
  * maintain some extra precision.
  *
  * Values are stored in .5 db format capped at 127.
  */
 static void
 mwl_node_getmimoinfo(const struct ieee80211_node *ni,
 	struct ieee80211_mimo_info *mi)
 {
 #define	CVT(_dst, _src) do {						\
 	(_dst) = rssi + ((logdbtbl[_src] - logdbtbl[rssi_max]) >> 2);	\
 	(_dst) = (_dst) > 64 ? 127 : ((_dst) << 1);			\
 } while (0)
 	static const int8_t logdbtbl[32] = {
 	       0,   0,  24,  38,  48,  56,  62,  68, 
 	      72,  76,  80,  83,  86,  89,  92,  94, 
 	      96,  98, 100, 102, 104, 106, 107, 109, 
 	     110, 112, 113, 115, 116, 117, 118, 119
 	};
 	const struct mwl_node *mn = MWL_NODE_CONST(ni);
 	uint8_t rssi = mn->mn_ai.rsvd1/2;		/* XXX */
 	uint32_t rssi_max;
 
 	rssi_max = mn->mn_ai.rssi_a;
 	if (mn->mn_ai.rssi_b > rssi_max)
 		rssi_max = mn->mn_ai.rssi_b;
 	if (mn->mn_ai.rssi_c > rssi_max)
 		rssi_max = mn->mn_ai.rssi_c;
 
 	CVT(mi->rssi[0], mn->mn_ai.rssi_a);
 	CVT(mi->rssi[1], mn->mn_ai.rssi_b);
 	CVT(mi->rssi[2], mn->mn_ai.rssi_c);
 
 	mi->noise[0] = mn->mn_ai.nf_a;
 	mi->noise[1] = mn->mn_ai.nf_b;
 	mi->noise[2] = mn->mn_ai.nf_c;
 #undef CVT
 }
 
 static __inline void *
 mwl_getrxdma(struct mwl_softc *sc)
 {
 	struct mwl_jumbo *buf;
 	void *data;
 
 	/*
 	 * Allocate from jumbo pool.
 	 */
 	MWL_RXFREE_LOCK(sc);
 	buf = SLIST_FIRST(&sc->sc_rxfree);
 	if (buf == NULL) {
 		DPRINTF(sc, MWL_DEBUG_ANY,
 		    "%s: out of rx dma buffers\n", __func__);
 		sc->sc_stats.mst_rx_nodmabuf++;
 		data = NULL;
 	} else {
 		SLIST_REMOVE_HEAD(&sc->sc_rxfree, next);
 		sc->sc_nrxfree--;
 		data = MWL_JUMBO_BUF2DATA(buf);
 	}
 	MWL_RXFREE_UNLOCK(sc);
 	return data;
 }
 
 static __inline void
 mwl_putrxdma(struct mwl_softc *sc, void *data)
 {
 	struct mwl_jumbo *buf;
 
 	/* XXX bounds check data */
 	MWL_RXFREE_LOCK(sc);
 	buf = MWL_JUMBO_DATA2BUF(data);
 	SLIST_INSERT_HEAD(&sc->sc_rxfree, buf, next);
 	sc->sc_nrxfree++;
 	MWL_RXFREE_UNLOCK(sc);
 }
 
 static int
 mwl_rxbuf_init(struct mwl_softc *sc, struct mwl_rxbuf *bf)
 {
 	struct mwl_rxdesc *ds;
 
 	ds = bf->bf_desc;
 	if (bf->bf_data == NULL) {
 		bf->bf_data = mwl_getrxdma(sc);
 		if (bf->bf_data == NULL) {
 			/* mark descriptor to be skipped */
 			ds->RxControl = EAGLE_RXD_CTRL_OS_OWN;
 			/* NB: don't need PREREAD */
 			MWL_RXDESC_SYNC(sc, ds, BUS_DMASYNC_PREWRITE);
 			sc->sc_stats.mst_rxbuf_failed++;
 			return ENOMEM;
 		}
 	}
 	/*
 	 * NB: DMA buffer contents is known to be unmodified
 	 *     so there's no need to flush the data cache.
 	 */
 
 	/*
 	 * Setup descriptor.
 	 */
 	ds->QosCtrl = 0;
 	ds->RSSI = 0;
 	ds->Status = EAGLE_RXD_STATUS_IDLE;
 	ds->Channel = 0;
 	ds->PktLen = htole16(MWL_AGGR_SIZE);
 	ds->SQ2 = 0;
 	ds->pPhysBuffData = htole32(MWL_JUMBO_DMA_ADDR(sc, bf->bf_data));
 	/* NB: don't touch pPhysNext, set once */
 	ds->RxControl = EAGLE_RXD_CTRL_DRIVER_OWN;
 	MWL_RXDESC_SYNC(sc, ds, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 
 	return 0;
 }
 
 static int
 mwl_ext_free(struct mbuf *m, void *data, void *arg)
 {
 	struct mwl_softc *sc = arg;
 
 	/* XXX bounds check data */
 	mwl_putrxdma(sc, data);
 	/*
 	 * If we were previously blocked by a lack of rx dma buffers
 	 * check if we now have enough to restart rx interrupt handling.
 	 * NB: we know we are called at splvm which is above splnet.
 	 */
 	if (sc->sc_rxblocked && sc->sc_nrxfree > mwl_rxdmalow) {
 		sc->sc_rxblocked = 0;
 		mwl_hal_intrset(sc->sc_mh, sc->sc_imask);
 	}
 	return (EXT_FREE_OK);
 }
 
 struct mwl_frame_bar {
 	u_int8_t	i_fc[2];
 	u_int8_t	i_dur[2];
 	u_int8_t	i_ra[IEEE80211_ADDR_LEN];
 	u_int8_t	i_ta[IEEE80211_ADDR_LEN];
 	/* ctl, seq, FCS */
 } __packed;
 
 /*
  * Like ieee80211_anyhdrsize, but handles BAR frames
  * specially so the logic below to piece the 802.11
  * header together works.
  */
 static __inline int
 mwl_anyhdrsize(const void *data)
 {
 	const struct ieee80211_frame *wh = data;
 
 	if ((wh->i_fc[0]&IEEE80211_FC0_TYPE_MASK) == IEEE80211_FC0_TYPE_CTL) {
 		switch (wh->i_fc[0] & IEEE80211_FC0_SUBTYPE_MASK) {
 		case IEEE80211_FC0_SUBTYPE_CTS:
 		case IEEE80211_FC0_SUBTYPE_ACK:
 			return sizeof(struct ieee80211_frame_ack);
 		case IEEE80211_FC0_SUBTYPE_BAR:
 			return sizeof(struct mwl_frame_bar);
 		}
 		return sizeof(struct ieee80211_frame_min);
 	} else
 		return ieee80211_hdrsize(data);
 }
 
 static void
 mwl_handlemicerror(struct ieee80211com *ic, const uint8_t *data)
 {
 	const struct ieee80211_frame *wh;
 	struct ieee80211_node *ni;
 
 	wh = (const struct ieee80211_frame *)(data + sizeof(uint16_t));
 	ni = ieee80211_find_rxnode(ic, (const struct ieee80211_frame_min *) wh);
 	if (ni != NULL) {
 		ieee80211_notify_michael_failure(ni->ni_vap, wh, 0);
 		ieee80211_free_node(ni);
 	}
 }
 
 /*
  * Convert hardware signal strength to rssi.  The value
  * provided by the device has the noise floor added in;
  * we need to compensate for this but we don't have that
  * so we use a fixed value.
  *
  * The offset of 8 is good for both 2.4 and 5GHz.  The LNA
  * offset is already set as part of the initial gain.  This
  * will give at least +/- 3dB for 2.4GHz and +/- 5dB for 5GHz.
  */
 static __inline int
 cvtrssi(uint8_t ssi)
 {
 	int rssi = (int) ssi + 8;
 	/* XXX hack guess until we have a real noise floor */
 	rssi = 2*(87 - rssi);	/* NB: .5 dBm units */
 	return (rssi < 0 ? 0 : rssi > 127 ? 127 : rssi);
 }
 
 static void
 mwl_rx_proc(void *arg, int npending)
 {
 #define	IEEE80211_DIR_DSTODS(wh) \
 	((((const struct ieee80211_frame *)wh)->i_fc[1] & IEEE80211_FC1_DIR_MASK) == IEEE80211_FC1_DIR_DSTODS)
 	struct mwl_softc *sc = arg;
 	struct ifnet *ifp = sc->sc_ifp;
 	struct ieee80211com *ic = ifp->if_l2com;
 	struct mwl_rxbuf *bf;
 	struct mwl_rxdesc *ds;
 	struct mbuf *m;
 	struct ieee80211_qosframe *wh;
 	struct ieee80211_qosframe_addr4 *wh4;
 	struct ieee80211_node *ni;
 	struct mwl_node *mn;
 	int off, len, hdrlen, pktlen, rssi, ntodo;
 	uint8_t *data, status;
 	void *newdata;
 	int16_t nf;
 
 	DPRINTF(sc, MWL_DEBUG_RX_PROC, "%s: pending %u rdptr 0x%x wrptr 0x%x\n",
 	    __func__, npending, RD4(sc, sc->sc_hwspecs.rxDescRead),
 	    RD4(sc, sc->sc_hwspecs.rxDescWrite));
 	nf = -96;			/* XXX */
 	bf = sc->sc_rxnext;
 	for (ntodo = mwl_rxquota; ntodo > 0; ntodo--) {
 		if (bf == NULL)
 			bf = STAILQ_FIRST(&sc->sc_rxbuf);
 		ds = bf->bf_desc;
 		data = bf->bf_data;
 		if (data == NULL) {
 			/*
 			 * If data allocation failed previously there
 			 * will be no buffer; try again to re-populate it.
 			 * Note the firmware will not advance to the next
 			 * descriptor with a dma buffer so we must mimic
 			 * this or we'll get out of sync.
 			 */ 
 			DPRINTF(sc, MWL_DEBUG_ANY,
 			    "%s: rx buf w/o dma memory\n", __func__);
 			(void) mwl_rxbuf_init(sc, bf);
 			sc->sc_stats.mst_rx_dmabufmissing++;
 			break;
 		}
 		MWL_RXDESC_SYNC(sc, ds,
 		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 		if (ds->RxControl != EAGLE_RXD_CTRL_DMA_OWN)
 			break;
 #ifdef MWL_DEBUG
 		if (sc->sc_debug & MWL_DEBUG_RECV_DESC)
 			mwl_printrxbuf(bf, 0);
 #endif
 		status = ds->Status;
 		if (status & EAGLE_RXD_STATUS_DECRYPT_ERR_MASK) {
 			ifp->if_ierrors++;
 			sc->sc_stats.mst_rx_crypto++;
 			/*
 			 * NB: Check EAGLE_RXD_STATUS_GENERAL_DECRYPT_ERR
 			 *     for backwards compatibility.
 			 */
 			if (status != EAGLE_RXD_STATUS_GENERAL_DECRYPT_ERR &&
 			    (status & EAGLE_RXD_STATUS_TKIP_MIC_DECRYPT_ERR)) {
 				/*
 				 * MIC error, notify upper layers.
 				 */
 				bus_dmamap_sync(sc->sc_rxdmat, sc->sc_rxmap,
 				    BUS_DMASYNC_POSTREAD);
 				mwl_handlemicerror(ic, data);
 				sc->sc_stats.mst_rx_tkipmic++;
 			}
 			/* XXX too painful to tap packets */
 			goto rx_next;
 		}
 		/*
 		 * Sync the data buffer.
 		 */
 		len = le16toh(ds->PktLen);
 		bus_dmamap_sync(sc->sc_rxdmat, sc->sc_rxmap, BUS_DMASYNC_POSTREAD);
 		/*
 		 * The 802.11 header is provided all or in part at the front;
 		 * use it to calculate the true size of the header that we'll
 		 * construct below.  We use this to figure out where to copy
 		 * payload prior to constructing the header.
 		 */
 		hdrlen = mwl_anyhdrsize(data + sizeof(uint16_t));
 		off = sizeof(uint16_t) + sizeof(struct ieee80211_frame_addr4);
 
 		/* calculate rssi early so we can re-use for each aggregate */
 		rssi = cvtrssi(ds->RSSI);
 
 		pktlen = hdrlen + (len - off);
 		/*
 		 * NB: we know our frame is at least as large as
 		 * IEEE80211_MIN_LEN because there is a 4-address
 		 * frame at the front.  Hence there's no need to
 		 * vet the packet length.  If the frame in fact
 		 * is too small it should be discarded at the
 		 * net80211 layer.
 		 */
 
 		/*
 		 * Attach dma buffer to an mbuf.  We tried
 		 * doing this based on the packet size (i.e.
 		 * copying small packets) but it turns out to
 		 * be a net loss.  The tradeoff might be system
 		 * dependent (cache architecture is important).
 		 */
 		MGETHDR(m, M_NOWAIT, MT_DATA);
 		if (m == NULL) {
 			DPRINTF(sc, MWL_DEBUG_ANY,
 			    "%s: no rx mbuf\n", __func__);
 			sc->sc_stats.mst_rx_nombuf++;
 			goto rx_next;
 		}
 		/*
 		 * Acquire the replacement dma buffer before
 		 * processing the frame.  If we're out of dma
 		 * buffers we disable rx interrupts and wait
 		 * for the free pool to reach mlw_rxdmalow buffers
 		 * before starting to do work again.  If the firmware
 		 * runs out of descriptors then it will toss frames
 		 * which is better than our doing it as that can
 		 * starve our processing.  It is also important that
 		 * we always process rx'd frames in case they are
 		 * A-MPDU as otherwise the host's view of the BA
 		 * window may get out of sync with the firmware.
 		 */
 		newdata = mwl_getrxdma(sc);
 		if (newdata == NULL) {
 			/* NB: stat+msg in mwl_getrxdma */
 			m_free(m);
 			/* disable RX interrupt and mark state */
 			mwl_hal_intrset(sc->sc_mh,
 			    sc->sc_imask &~ MACREG_A2HRIC_BIT_RX_RDY);
 			sc->sc_rxblocked = 1;
 			ieee80211_drain(ic);
 			/* XXX check rxblocked and immediately start again? */
 			goto rx_stop;
 		}
 		bf->bf_data = newdata;
 		/*
 		 * Attach the dma buffer to the mbuf;
 		 * mwl_rxbuf_init will re-setup the rx
 		 * descriptor using the replacement dma
 		 * buffer we just installed above.
 		 */
 		MEXTADD(m, data, MWL_AGGR_SIZE, mwl_ext_free,
 		    data, sc, 0, EXT_NET_DRV);
 		m->m_data += off - hdrlen;
 		m->m_pkthdr.len = m->m_len = pktlen;
 		m->m_pkthdr.rcvif = ifp;
 		/* NB: dma buffer assumed read-only */
 
 		/*
 		 * Piece 802.11 header together.
 		 */
 		wh = mtod(m, struct ieee80211_qosframe *);
 		/* NB: don't need to do this sometimes but ... */
 		/* XXX special case so we can memcpy after m_devget? */
 		ovbcopy(data + sizeof(uint16_t), wh, hdrlen);
 		if (IEEE80211_QOS_HAS_SEQ(wh)) {
 			if (IEEE80211_DIR_DSTODS(wh)) {
 				wh4 = mtod(m,
 				    struct ieee80211_qosframe_addr4*);
 				*(uint16_t *)wh4->i_qos = ds->QosCtrl;
 			} else {
 				*(uint16_t *)wh->i_qos = ds->QosCtrl;
 			}
 		}
 		/*
 		 * The f/w strips WEP header but doesn't clear
 		 * the WEP bit; mark the packet with M_WEP so
 		 * net80211 will treat the data as decrypted.
 		 * While here also clear the PWR_MGT bit since
 		 * power save is handled by the firmware and
 		 * passing this up will potentially cause the
 		 * upper layer to put a station in power save
 		 * (except when configured with MWL_HOST_PS_SUPPORT).
 		 */
 		if (wh->i_fc[1] & IEEE80211_FC1_PROTECTED)
 			m->m_flags |= M_WEP;
 #ifdef MWL_HOST_PS_SUPPORT
 		wh->i_fc[1] &= ~IEEE80211_FC1_PROTECTED;
 #else
 		wh->i_fc[1] &= ~(IEEE80211_FC1_PROTECTED |
 		    IEEE80211_FC1_PWR_MGT);
 #endif
 
 		if (ieee80211_radiotap_active(ic)) {
 			struct mwl_rx_radiotap_header *tap = &sc->sc_rx_th;
 
 			tap->wr_flags = 0;
 			tap->wr_rate = ds->Rate;
 			tap->wr_antsignal = rssi + nf;
 			tap->wr_antnoise = nf;
 		}
 		if (IFF_DUMPPKTS_RECV(sc, wh)) {
 			ieee80211_dump_pkt(ic, mtod(m, caddr_t),
 			    len, ds->Rate, rssi);
 		}
 		ifp->if_ipackets++;
 
 		/* dispatch */
 		ni = ieee80211_find_rxnode(ic,
 		    (const struct ieee80211_frame_min *) wh);
 		if (ni != NULL) {
 			mn = MWL_NODE(ni);
 #ifdef MWL_ANT_INFO_SUPPORT
 			mn->mn_ai.rssi_a = ds->ai.rssi_a;
 			mn->mn_ai.rssi_b = ds->ai.rssi_b;
 			mn->mn_ai.rssi_c = ds->ai.rssi_c;
 			mn->mn_ai.rsvd1 = rssi;
 #endif
 			/* tag AMPDU aggregates for reorder processing */
 			if (ni->ni_flags & IEEE80211_NODE_HT)
 				m->m_flags |= M_AMPDU;
 			(void) ieee80211_input(ni, m, rssi, nf);
 			ieee80211_free_node(ni);
 		} else
 			(void) ieee80211_input_all(ic, m, rssi, nf);
 rx_next:
 		/* NB: ignore ENOMEM so we process more descriptors */
 		(void) mwl_rxbuf_init(sc, bf);
 		bf = STAILQ_NEXT(bf, bf_list);
 	}
 rx_stop:
 	sc->sc_rxnext = bf;
 
 	if ((ifp->if_drv_flags & IFF_DRV_OACTIVE) == 0 &&
 	    !IFQ_IS_EMPTY(&ifp->if_snd)) {
 		/* NB: kick fw; the tx thread may have been preempted */
 		mwl_hal_txstart(sc->sc_mh, 0);
 		mwl_start(ifp);
 	}
 #undef IEEE80211_DIR_DSTODS
 }
 
 static void
 mwl_txq_init(struct mwl_softc *sc, struct mwl_txq *txq, int qnum)
 {
 	struct mwl_txbuf *bf, *bn;
 	struct mwl_txdesc *ds;
 
 	MWL_TXQ_LOCK_INIT(sc, txq);
 	txq->qnum = qnum;
 	txq->txpri = 0;	/* XXX */
 #if 0
 	/* NB: q setup by mwl_txdma_setup XXX */
 	STAILQ_INIT(&txq->free);
 #endif
 	STAILQ_FOREACH(bf, &txq->free, bf_list) {
 		bf->bf_txq = txq;
 
 		ds = bf->bf_desc;
 		bn = STAILQ_NEXT(bf, bf_list);
 		if (bn == NULL)
 			bn = STAILQ_FIRST(&txq->free);
 		ds->pPhysNext = htole32(bn->bf_daddr);
 	}
 	STAILQ_INIT(&txq->active);
 }
 
 /*
  * Setup a hardware data transmit queue for the specified
  * access control.  We record the mapping from ac's
  * to h/w queues for use by mwl_tx_start.
  */
 static int
 mwl_tx_setup(struct mwl_softc *sc, int ac, int mvtype)
 {
 #define	N(a)	(sizeof(a)/sizeof(a[0]))
 	struct mwl_txq *txq;
 
 	if (ac >= N(sc->sc_ac2q)) {
 		device_printf(sc->sc_dev, "AC %u out of range, max %zu!\n",
 			ac, N(sc->sc_ac2q));
 		return 0;
 	}
 	if (mvtype >= MWL_NUM_TX_QUEUES) {
 		device_printf(sc->sc_dev, "mvtype %u out of range, max %u!\n",
 			mvtype, MWL_NUM_TX_QUEUES);
 		return 0;
 	}
 	txq = &sc->sc_txq[mvtype];
 	mwl_txq_init(sc, txq, mvtype);
 	sc->sc_ac2q[ac] = txq;
 	return 1;
 #undef N
 }
 
 /*
  * Update WME parameters for a transmit queue.
  */
 static int
 mwl_txq_update(struct mwl_softc *sc, int ac)
 {
 #define	MWL_EXPONENT_TO_VALUE(v)	((1<<v)-1)
 	struct ifnet *ifp = sc->sc_ifp;
 	struct ieee80211com *ic = ifp->if_l2com;
 	struct mwl_txq *txq = sc->sc_ac2q[ac];
 	struct wmeParams *wmep = &ic->ic_wme.wme_chanParams.cap_wmeParams[ac];
 	struct mwl_hal *mh = sc->sc_mh;
 	int aifs, cwmin, cwmax, txoplim;
 
 	aifs = wmep->wmep_aifsn;
 	/* XXX in sta mode need to pass log values for cwmin/max */
 	cwmin = MWL_EXPONENT_TO_VALUE(wmep->wmep_logcwmin);
 	cwmax = MWL_EXPONENT_TO_VALUE(wmep->wmep_logcwmax);
 	txoplim = wmep->wmep_txopLimit;		/* NB: units of 32us */
 
 	if (mwl_hal_setedcaparams(mh, txq->qnum, cwmin, cwmax, aifs, txoplim)) {
 		device_printf(sc->sc_dev, "unable to update hardware queue "
 			"parameters for %s traffic!\n",
 			ieee80211_wme_acnames[ac]);
 		return 0;
 	}
 	return 1;
 #undef MWL_EXPONENT_TO_VALUE
 }
 
 /*
  * Callback from the 802.11 layer to update WME parameters.
  */
 static int
 mwl_wme_update(struct ieee80211com *ic)
 {
 	struct mwl_softc *sc = ic->ic_ifp->if_softc;
 
 	return !mwl_txq_update(sc, WME_AC_BE) ||
 	    !mwl_txq_update(sc, WME_AC_BK) ||
 	    !mwl_txq_update(sc, WME_AC_VI) ||
 	    !mwl_txq_update(sc, WME_AC_VO) ? EIO : 0;
 }
 
 /*
  * Reclaim resources for a setup queue.
  */
 static void
 mwl_tx_cleanupq(struct mwl_softc *sc, struct mwl_txq *txq)
 {
 	/* XXX hal work? */
 	MWL_TXQ_LOCK_DESTROY(txq);
 }
 
 /*
  * Reclaim all tx queue resources.
  */
 static void
 mwl_tx_cleanup(struct mwl_softc *sc)
 {
 	int i;
 
 	for (i = 0; i < MWL_NUM_TX_QUEUES; i++)
 		mwl_tx_cleanupq(sc, &sc->sc_txq[i]);
 }
 
 static int
 mwl_tx_dmasetup(struct mwl_softc *sc, struct mwl_txbuf *bf, struct mbuf *m0)
 {
 	struct mbuf *m;
 	int error;
 
 	/*
 	 * Load the DMA map so any coalescing is done.  This
 	 * also calculates the number of descriptors we need.
 	 */
 	error = bus_dmamap_load_mbuf_sg(sc->sc_dmat, bf->bf_dmamap, m0,
 				     bf->bf_segs, &bf->bf_nseg,
 				     BUS_DMA_NOWAIT);
 	if (error == EFBIG) {
 		/* XXX packet requires too many descriptors */
 		bf->bf_nseg = MWL_TXDESC+1;
 	} else if (error != 0) {
 		sc->sc_stats.mst_tx_busdma++;
 		m_freem(m0);
 		return error;
 	}
 	/*
 	 * Discard null packets and check for packets that
 	 * require too many TX descriptors.  We try to convert
 	 * the latter to a cluster.
 	 */
 	if (error == EFBIG) {		/* too many desc's, linearize */
 		sc->sc_stats.mst_tx_linear++;
 #if MWL_TXDESC > 1
 		m = m_collapse(m0, M_NOWAIT, MWL_TXDESC);
 #else
 		m = m_defrag(m0, M_NOWAIT);
 #endif
 		if (m == NULL) {
 			m_freem(m0);
 			sc->sc_stats.mst_tx_nombuf++;
 			return ENOMEM;
 		}
 		m0 = m;
 		error = bus_dmamap_load_mbuf_sg(sc->sc_dmat, bf->bf_dmamap, m0,
 					     bf->bf_segs, &bf->bf_nseg,
 					     BUS_DMA_NOWAIT);
 		if (error != 0) {
 			sc->sc_stats.mst_tx_busdma++;
 			m_freem(m0);
 			return error;
 		}
 		KASSERT(bf->bf_nseg <= MWL_TXDESC,
 		    ("too many segments after defrag; nseg %u", bf->bf_nseg));
 	} else if (bf->bf_nseg == 0) {		/* null packet, discard */
 		sc->sc_stats.mst_tx_nodata++;
 		m_freem(m0);
 		return EIO;
 	}
 	DPRINTF(sc, MWL_DEBUG_XMIT, "%s: m %p len %u\n",
 		__func__, m0, m0->m_pkthdr.len);
 	bus_dmamap_sync(sc->sc_dmat, bf->bf_dmamap, BUS_DMASYNC_PREWRITE);
 	bf->bf_m = m0;
 
 	return 0;
 }
 
 static __inline int
 mwl_cvtlegacyrate(int rate)
 {
 	switch (rate) {
 	case 2:	 return 0;
 	case 4:	 return 1;
 	case 11: return 2;
 	case 22: return 3;
 	case 44: return 4;
 	case 12: return 5;
 	case 18: return 6;
 	case 24: return 7;
 	case 36: return 8;
 	case 48: return 9;
 	case 72: return 10;
 	case 96: return 11;
 	case 108:return 12;
 	}
 	return 0;
 }
 
 /*
  * Calculate fixed tx rate information per client state;
  * this value is suitable for writing to the Format field
  * of a tx descriptor.
  */
 static uint16_t
 mwl_calcformat(uint8_t rate, const struct ieee80211_node *ni)
 {
 	uint16_t fmt;
 
 	fmt = SM(3, EAGLE_TXD_ANTENNA)
 	    | (IEEE80211_IS_CHAN_HT40D(ni->ni_chan) ?
 		EAGLE_TXD_EXTCHAN_LO : EAGLE_TXD_EXTCHAN_HI);
 	if (rate & IEEE80211_RATE_MCS) {	/* HT MCS */
 		fmt |= EAGLE_TXD_FORMAT_HT
 		    /* NB: 0x80 implicitly stripped from ucastrate */
 		    | SM(rate, EAGLE_TXD_RATE);
 		/* XXX short/long GI may be wrong; re-check */
 		if (IEEE80211_IS_CHAN_HT40(ni->ni_chan)) {
 			fmt |= EAGLE_TXD_CHW_40
 			    | (ni->ni_htcap & IEEE80211_HTCAP_SHORTGI40 ?
 			        EAGLE_TXD_GI_SHORT : EAGLE_TXD_GI_LONG);
 		} else {
 			fmt |= EAGLE_TXD_CHW_20
 			    | (ni->ni_htcap & IEEE80211_HTCAP_SHORTGI20 ?
 			        EAGLE_TXD_GI_SHORT : EAGLE_TXD_GI_LONG);
 		}
 	} else {			/* legacy rate */
 		fmt |= EAGLE_TXD_FORMAT_LEGACY
 		    | SM(mwl_cvtlegacyrate(rate), EAGLE_TXD_RATE)
 		    | EAGLE_TXD_CHW_20
 		    /* XXX iv_flags & IEEE80211_F_SHPREAMBLE? */
 		    | (ni->ni_capinfo & IEEE80211_CAPINFO_SHORT_PREAMBLE ?
 			EAGLE_TXD_PREAMBLE_SHORT : EAGLE_TXD_PREAMBLE_LONG);
 	}
 	return fmt;
 }
 
 static int
 mwl_tx_start(struct mwl_softc *sc, struct ieee80211_node *ni, struct mwl_txbuf *bf,
     struct mbuf *m0)
 {
 #define	IEEE80211_DIR_DSTODS(wh) \
 	((wh->i_fc[1] & IEEE80211_FC1_DIR_MASK) == IEEE80211_FC1_DIR_DSTODS)
 	struct ifnet *ifp = sc->sc_ifp;
 	struct ieee80211com *ic = ifp->if_l2com;
 	struct ieee80211vap *vap = ni->ni_vap;
 	int error, iswep, ismcast;
 	int hdrlen, copyhdrlen, pktlen;
 	struct mwl_txdesc *ds;
 	struct mwl_txq *txq;
 	struct ieee80211_frame *wh;
 	struct mwltxrec *tr;
 	struct mwl_node *mn;
 	uint16_t qos;
 #if MWL_TXDESC > 1
 	int i;
 #endif
 
 	wh = mtod(m0, struct ieee80211_frame *);
 	iswep = wh->i_fc[1] & IEEE80211_FC1_PROTECTED;
 	ismcast = IEEE80211_IS_MULTICAST(wh->i_addr1);
 	hdrlen = ieee80211_anyhdrsize(wh);
 	copyhdrlen = hdrlen;
 	pktlen = m0->m_pkthdr.len;
 	if (IEEE80211_QOS_HAS_SEQ(wh)) {
 		if (IEEE80211_DIR_DSTODS(wh)) {
 			qos = *(uint16_t *)
 			    (((struct ieee80211_qosframe_addr4 *) wh)->i_qos);
 			copyhdrlen -= sizeof(qos);
 		} else
 			qos = *(uint16_t *)
 			    (((struct ieee80211_qosframe *) wh)->i_qos);
 	} else
 		qos = 0;
 
 	if (iswep) {
 		const struct ieee80211_cipher *cip;
 		struct ieee80211_key *k;
 
 		/*
 		 * Construct the 802.11 header+trailer for an encrypted
 		 * frame. The only reason this can fail is because of an
 		 * unknown or unsupported cipher/key type.
 		 *
 		 * NB: we do this even though the firmware will ignore
 		 *     what we've done for WEP and TKIP as we need the
 		 *     ExtIV filled in for CCMP and this also adjusts
 		 *     the headers which simplifies our work below.
 		 */
 		k = ieee80211_crypto_encap(ni, m0);
 		if (k == NULL) {
 			/*
 			 * This can happen when the key is yanked after the
 			 * frame was queued.  Just discard the frame; the
 			 * 802.11 layer counts failures and provides
 			 * debugging/diagnostics.
 			 */
 			m_freem(m0);
 			return EIO;
 		}
 		/*
 		 * Adjust the packet length for the crypto additions
 		 * done during encap and any other bits that the f/w
 		 * will add later on.
 		 */
 		cip = k->wk_cipher;
 		pktlen += cip->ic_header + cip->ic_miclen + cip->ic_trailer;
 
 		/* packet header may have moved, reset our local pointer */
 		wh = mtod(m0, struct ieee80211_frame *);
 	}
 
 	if (ieee80211_radiotap_active_vap(vap)) {
 		sc->sc_tx_th.wt_flags = 0;	/* XXX */
 		if (iswep)
 			sc->sc_tx_th.wt_flags |= IEEE80211_RADIOTAP_F_WEP;
 #if 0
 		sc->sc_tx_th.wt_rate = ds->DataRate;
 #endif
 		sc->sc_tx_th.wt_txpower = ni->ni_txpower;
 		sc->sc_tx_th.wt_antenna = sc->sc_txantenna;
 
 		ieee80211_radiotap_tx(vap, m0);
 	}
 	/*
 	 * Copy up/down the 802.11 header; the firmware requires
 	 * we present a 2-byte payload length followed by a
 	 * 4-address header (w/o QoS), followed (optionally) by
 	 * any WEP/ExtIV header (but only filled in for CCMP).
 	 * We are assured the mbuf has sufficient headroom to
 	 * prepend in-place by the setup of ic_headroom in
 	 * mwl_attach.
 	 */
 	if (hdrlen < sizeof(struct mwltxrec)) {
 		const int space = sizeof(struct mwltxrec) - hdrlen;
 		if (M_LEADINGSPACE(m0) < space) {
 			/* NB: should never happen */
 			device_printf(sc->sc_dev,
 			    "not enough headroom, need %d found %zd, "
 			    "m_flags 0x%x m_len %d\n",
 			    space, M_LEADINGSPACE(m0), m0->m_flags, m0->m_len);
 			ieee80211_dump_pkt(ic,
 			    mtod(m0, const uint8_t *), m0->m_len, 0, -1);
 			m_freem(m0);
 			sc->sc_stats.mst_tx_noheadroom++;
 			return EIO;
 		}
 		M_PREPEND(m0, space, M_NOWAIT);
 	}
 	tr = mtod(m0, struct mwltxrec *);
 	if (wh != (struct ieee80211_frame *) &tr->wh)
 		ovbcopy(wh, &tr->wh, hdrlen);
 	/*
 	 * Note: the "firmware length" is actually the length
 	 * of the fully formed "802.11 payload".  That is, it's
 	 * everything except for the 802.11 header.  In particular
 	 * this includes all crypto material including the MIC!
 	 */
 	tr->fwlen = htole16(pktlen - hdrlen);
 
 	/*
 	 * Load the DMA map so any coalescing is done.  This
 	 * also calculates the number of descriptors we need.
 	 */
 	error = mwl_tx_dmasetup(sc, bf, m0);
 	if (error != 0) {
 		/* NB: stat collected in mwl_tx_dmasetup */
 		DPRINTF(sc, MWL_DEBUG_XMIT,
 		    "%s: unable to setup dma\n", __func__);
 		return error;
 	}
 	bf->bf_node = ni;			/* NB: held reference */
 	m0 = bf->bf_m;				/* NB: may have changed */
 	tr = mtod(m0, struct mwltxrec *);
 	wh = (struct ieee80211_frame *)&tr->wh;
 
 	/*
 	 * Formulate tx descriptor.
 	 */
 	ds = bf->bf_desc;
 	txq = bf->bf_txq;
 
 	ds->QosCtrl = qos;			/* NB: already little-endian */
 #if MWL_TXDESC == 1
 	/*
 	 * NB: multiframes should be zero because the descriptors
 	 *     are initialized to zero.  This should handle the case
 	 *     where the driver is built with MWL_TXDESC=1 but we are
 	 *     using firmware with multi-segment support.
 	 */
 	ds->PktPtr = htole32(bf->bf_segs[0].ds_addr);
 	ds->PktLen = htole16(bf->bf_segs[0].ds_len);
 #else
 	ds->multiframes = htole32(bf->bf_nseg);
 	ds->PktLen = htole16(m0->m_pkthdr.len);
 	for (i = 0; i < bf->bf_nseg; i++) {
 		ds->PktPtrArray[i] = htole32(bf->bf_segs[i].ds_addr);
 		ds->PktLenArray[i] = htole16(bf->bf_segs[i].ds_len);
 	}
 #endif
 	/* NB: pPhysNext, DataRate, and SapPktInfo setup once, don't touch */
 	ds->Format = 0;
 	ds->pad = 0;
 	ds->ack_wcb_addr = 0;
 
 	mn = MWL_NODE(ni);
 	/*
 	 * Select transmit rate.
 	 */
 	switch (wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK) {
 	case IEEE80211_FC0_TYPE_MGT:
 		sc->sc_stats.mst_tx_mgmt++;
 		/* fall thru... */
 	case IEEE80211_FC0_TYPE_CTL:
 		/* NB: assign to BE q to avoid bursting */
 		ds->TxPriority = MWL_WME_AC_BE;
 		break;
 	case IEEE80211_FC0_TYPE_DATA:
 		if (!ismcast) {
 			const struct ieee80211_txparam *tp = ni->ni_txparms;
 			/*
 			 * EAPOL frames get forced to a fixed rate and w/o
 			 * aggregation; otherwise check for any fixed rate
 			 * for the client (may depend on association state).
 			 */
 			if (m0->m_flags & M_EAPOL) {
 				const struct mwl_vap *mvp = MWL_VAP_CONST(vap);
 				ds->Format = mvp->mv_eapolformat;
 				ds->pad = htole16(
 				    EAGLE_TXD_FIXED_RATE | EAGLE_TXD_DONT_AGGR);
 			} else if (tp->ucastrate != IEEE80211_FIXED_RATE_NONE) {
 				/* XXX pre-calculate per node */
 				ds->Format = htole16(
 				    mwl_calcformat(tp->ucastrate, ni));
 				ds->pad = htole16(EAGLE_TXD_FIXED_RATE);
 			}
 			/* NB: EAPOL frames will never have qos set */
 			if (qos == 0)
 				ds->TxPriority = txq->qnum;
 #if MWL_MAXBA > 3
 			else if (mwl_bastream_match(&mn->mn_ba[3], qos))
 				ds->TxPriority = mn->mn_ba[3].txq;
 #endif
 #if MWL_MAXBA > 2
 			else if (mwl_bastream_match(&mn->mn_ba[2], qos))
 				ds->TxPriority = mn->mn_ba[2].txq;
 #endif
 #if MWL_MAXBA > 1
 			else if (mwl_bastream_match(&mn->mn_ba[1], qos))
 				ds->TxPriority = mn->mn_ba[1].txq;
 #endif
 #if MWL_MAXBA > 0
 			else if (mwl_bastream_match(&mn->mn_ba[0], qos))
 				ds->TxPriority = mn->mn_ba[0].txq;
 #endif
 			else
 				ds->TxPriority = txq->qnum;
 		} else
 			ds->TxPriority = txq->qnum;
 		break;
 	default:
 		if_printf(ifp, "bogus frame type 0x%x (%s)\n",
 			wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK, __func__);
 		sc->sc_stats.mst_tx_badframetype++;
 		m_freem(m0);
 		return EIO;
 	}
 
 	if (IFF_DUMPPKTS_XMIT(sc))
 		ieee80211_dump_pkt(ic,
 		    mtod(m0, const uint8_t *)+sizeof(uint16_t),
 		    m0->m_len - sizeof(uint16_t), ds->DataRate, -1);
 
 	MWL_TXQ_LOCK(txq);
 	ds->Status = htole32(EAGLE_TXD_STATUS_FW_OWNED);
 	STAILQ_INSERT_TAIL(&txq->active, bf, bf_list);
 	MWL_TXDESC_SYNC(txq, ds, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 
 	ifp->if_opackets++;
 	sc->sc_tx_timer = 5;
 	MWL_TXQ_UNLOCK(txq);
 
 	return 0;
 #undef	IEEE80211_DIR_DSTODS
 }
 
 static __inline int
 mwl_cvtlegacyrix(int rix)
 {
 #define	N(x)	(sizeof(x)/sizeof(x[0]))
 	static const int ieeerates[] =
 	    { 2, 4, 11, 22, 44, 12, 18, 24, 36, 48, 72, 96, 108 };
 	return (rix < N(ieeerates) ? ieeerates[rix] : 0);
 #undef N
 }
 
 /*
  * Process completed xmit descriptors from the specified queue.
  */
 static int
 mwl_tx_processq(struct mwl_softc *sc, struct mwl_txq *txq)
 {
 #define	EAGLE_TXD_STATUS_MCAST \
 	(EAGLE_TXD_STATUS_MULTICAST_TX | EAGLE_TXD_STATUS_BROADCAST_TX)
 	struct ifnet *ifp = sc->sc_ifp;
 	struct ieee80211com *ic = ifp->if_l2com;
 	struct mwl_txbuf *bf;
 	struct mwl_txdesc *ds;
 	struct ieee80211_node *ni;
 	struct mwl_node *an;
 	int nreaped;
 	uint32_t status;
 
 	DPRINTF(sc, MWL_DEBUG_TX_PROC, "%s: tx queue %u\n", __func__, txq->qnum);
 	for (nreaped = 0;; nreaped++) {
 		MWL_TXQ_LOCK(txq);
 		bf = STAILQ_FIRST(&txq->active);
 		if (bf == NULL) {
 			MWL_TXQ_UNLOCK(txq);
 			break;
 		}
 		ds = bf->bf_desc;
 		MWL_TXDESC_SYNC(txq, ds,
 		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 		if (ds->Status & htole32(EAGLE_TXD_STATUS_FW_OWNED)) {
 			MWL_TXQ_UNLOCK(txq);
 			break;
 		}
 		STAILQ_REMOVE_HEAD(&txq->active, bf_list);
 		MWL_TXQ_UNLOCK(txq);
 
 #ifdef MWL_DEBUG
 		if (sc->sc_debug & MWL_DEBUG_XMIT_DESC)
 			mwl_printtxbuf(bf, txq->qnum, nreaped);
 #endif
 		ni = bf->bf_node;
 		if (ni != NULL) {
 			an = MWL_NODE(ni);
 			status = le32toh(ds->Status);
 			if (status & EAGLE_TXD_STATUS_OK) {
 				uint16_t Format = le16toh(ds->Format);
 				uint8_t txant = MS(Format, EAGLE_TXD_ANTENNA);
 
 				sc->sc_stats.mst_ant_tx[txant]++;
 				if (status & EAGLE_TXD_STATUS_OK_RETRY)
 					sc->sc_stats.mst_tx_retries++;
 				if (status & EAGLE_TXD_STATUS_OK_MORE_RETRY)
 					sc->sc_stats.mst_tx_mretries++;
 				if (txq->qnum >= MWL_WME_AC_VO)
 					ic->ic_wme.wme_hipri_traffic++;
 				ni->ni_txrate = MS(Format, EAGLE_TXD_RATE);
 				if ((Format & EAGLE_TXD_FORMAT_HT) == 0) {
 					ni->ni_txrate = mwl_cvtlegacyrix(
 					    ni->ni_txrate);
 				} else
 					ni->ni_txrate |= IEEE80211_RATE_MCS;
 				sc->sc_stats.mst_tx_rate = ni->ni_txrate;
 			} else {
 				if (status & EAGLE_TXD_STATUS_FAILED_LINK_ERROR)
 					sc->sc_stats.mst_tx_linkerror++;
 				if (status & EAGLE_TXD_STATUS_FAILED_XRETRY)
 					sc->sc_stats.mst_tx_xretries++;
 				if (status & EAGLE_TXD_STATUS_FAILED_AGING)
 					sc->sc_stats.mst_tx_aging++;
 				if (bf->bf_m->m_flags & M_FF)
 					sc->sc_stats.mst_ff_txerr++;
 			}
 			/*
 			 * Do any tx complete callback.  Note this must
 			 * be done before releasing the node reference.
 			 * XXX no way to figure out if frame was ACK'd
 			 */
 			if (bf->bf_m->m_flags & M_TXCB) {
 				/* XXX strip fw len in case header inspected */
 				m_adj(bf->bf_m, sizeof(uint16_t));
 				ieee80211_process_callback(ni, bf->bf_m,
 					(status & EAGLE_TXD_STATUS_OK) == 0);
 			}
 			/*
 			 * Reclaim reference to node.
 			 *
 			 * NB: the node may be reclaimed here if, for example
 			 *     this is a DEAUTH message that was sent and the
 			 *     node was timed out due to inactivity.
 			 */
 			ieee80211_free_node(ni);
 		}
 		ds->Status = htole32(EAGLE_TXD_STATUS_IDLE);
 
 		bus_dmamap_sync(sc->sc_dmat, bf->bf_dmamap,
 		    BUS_DMASYNC_POSTWRITE);
 		bus_dmamap_unload(sc->sc_dmat, bf->bf_dmamap);
 		m_freem(bf->bf_m);
 
 		mwl_puttxbuf_tail(txq, bf);
 	}
 	return nreaped;
 #undef EAGLE_TXD_STATUS_MCAST
 }
 
 /*
  * Deferred processing of transmit interrupt; special-cased
  * for four hardware queues, 0-3.
  */
 static void
 mwl_tx_proc(void *arg, int npending)
 {
 	struct mwl_softc *sc = arg;
 	struct ifnet *ifp = sc->sc_ifp;
 	int nreaped;
 
 	/*
 	 * Process each active queue.
 	 */
 	nreaped = 0;
 	if (!STAILQ_EMPTY(&sc->sc_txq[0].active))
 		nreaped += mwl_tx_processq(sc, &sc->sc_txq[0]);
 	if (!STAILQ_EMPTY(&sc->sc_txq[1].active))
 		nreaped += mwl_tx_processq(sc, &sc->sc_txq[1]);
 	if (!STAILQ_EMPTY(&sc->sc_txq[2].active))
 		nreaped += mwl_tx_processq(sc, &sc->sc_txq[2]);
 	if (!STAILQ_EMPTY(&sc->sc_txq[3].active))
 		nreaped += mwl_tx_processq(sc, &sc->sc_txq[3]);
 
 	if (nreaped != 0) {
 		ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
 		sc->sc_tx_timer = 0;
 		if (!IFQ_IS_EMPTY(&ifp->if_snd)) {
 			/* NB: kick fw; the tx thread may have been preempted */
 			mwl_hal_txstart(sc->sc_mh, 0);
 			mwl_start(ifp);
 		}
 	}
 }
 
 static void
 mwl_tx_draintxq(struct mwl_softc *sc, struct mwl_txq *txq)
 {
 	struct ieee80211_node *ni;
 	struct mwl_txbuf *bf;
 	u_int ix;
 
 	/*
 	 * NB: this assumes output has been stopped and
 	 *     we do not need to block mwl_tx_tasklet
 	 */
 	for (ix = 0;; ix++) {
 		MWL_TXQ_LOCK(txq);
 		bf = STAILQ_FIRST(&txq->active);
 		if (bf == NULL) {
 			MWL_TXQ_UNLOCK(txq);
 			break;
 		}
 		STAILQ_REMOVE_HEAD(&txq->active, bf_list);
 		MWL_TXQ_UNLOCK(txq);
 #ifdef MWL_DEBUG
 		if (sc->sc_debug & MWL_DEBUG_RESET) {
 			struct ifnet *ifp = sc->sc_ifp;
 			struct ieee80211com *ic = ifp->if_l2com;
 			const struct mwltxrec *tr =
 			    mtod(bf->bf_m, const struct mwltxrec *);
 			mwl_printtxbuf(bf, txq->qnum, ix);
 			ieee80211_dump_pkt(ic, (const uint8_t *)&tr->wh,
 				bf->bf_m->m_len - sizeof(tr->fwlen), 0, -1);
 		}
 #endif /* MWL_DEBUG */
 		bus_dmamap_unload(sc->sc_dmat, bf->bf_dmamap);
 		ni = bf->bf_node;
 		if (ni != NULL) {
 			/*
 			 * Reclaim node reference.
 			 */
 			ieee80211_free_node(ni);
 		}
 		m_freem(bf->bf_m);
 
 		mwl_puttxbuf_tail(txq, bf);
 	}
 }
 
 /*
  * Drain the transmit queues and reclaim resources.
  */
 static void
 mwl_draintxq(struct mwl_softc *sc)
 {
 	struct ifnet *ifp = sc->sc_ifp;
 	int i;
 
 	for (i = 0; i < MWL_NUM_TX_QUEUES; i++)
 		mwl_tx_draintxq(sc, &sc->sc_txq[i]);
 	ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
 	sc->sc_tx_timer = 0;
 }
 
 #ifdef MWL_DIAGAPI
 /*
  * Reset the transmit queues to a pristine state after a fw download.
  */
 static void
 mwl_resettxq(struct mwl_softc *sc)
 {
 	int i;
 
 	for (i = 0; i < MWL_NUM_TX_QUEUES; i++)
 		mwl_txq_reset(sc, &sc->sc_txq[i]);
 }
 #endif /* MWL_DIAGAPI */
 
 /*
  * Clear the transmit queues of any frames submitted for the
  * specified vap.  This is done when the vap is deleted so we
  * don't potentially reference the vap after it is gone.
  * Note we cannot remove the frames; we only reclaim the node
  * reference.
  */
 static void
 mwl_cleartxq(struct mwl_softc *sc, struct ieee80211vap *vap)
 {
 	struct mwl_txq *txq;
 	struct mwl_txbuf *bf;
 	int i;
 
 	for (i = 0; i < MWL_NUM_TX_QUEUES; i++) {
 		txq = &sc->sc_txq[i];
 		MWL_TXQ_LOCK(txq);
 		STAILQ_FOREACH(bf, &txq->active, bf_list) {
 			struct ieee80211_node *ni = bf->bf_node;
 			if (ni != NULL && ni->ni_vap == vap) {
 				bf->bf_node = NULL;
 				ieee80211_free_node(ni);
 			}
 		}
 		MWL_TXQ_UNLOCK(txq);
 	}
 }
 
 static int
 mwl_recv_action(struct ieee80211_node *ni, const struct ieee80211_frame *wh,
 	const uint8_t *frm, const uint8_t *efrm)
 {
 	struct mwl_softc *sc = ni->ni_ic->ic_ifp->if_softc;
 	const struct ieee80211_action *ia;
 
 	ia = (const struct ieee80211_action *) frm;
 	if (ia->ia_category == IEEE80211_ACTION_CAT_HT &&
 	    ia->ia_action == IEEE80211_ACTION_HT_MIMOPWRSAVE) {
 		const struct ieee80211_action_ht_mimopowersave *mps =
 		    (const struct ieee80211_action_ht_mimopowersave *) ia;
 
 		mwl_hal_setmimops(sc->sc_mh, ni->ni_macaddr,
 		    mps->am_control & IEEE80211_A_HT_MIMOPWRSAVE_ENA,
 		    MS(mps->am_control, IEEE80211_A_HT_MIMOPWRSAVE_MODE));
 		return 0;
 	} else
 		return sc->sc_recv_action(ni, wh, frm, efrm);
 }
 
 static int
 mwl_addba_request(struct ieee80211_node *ni, struct ieee80211_tx_ampdu *tap,
 	int dialogtoken, int baparamset, int batimeout)
 {
 	struct mwl_softc *sc = ni->ni_ic->ic_ifp->if_softc;
 	struct ieee80211vap *vap = ni->ni_vap;
 	struct mwl_node *mn = MWL_NODE(ni);
 	struct mwl_bastate *bas;
 
 	bas = tap->txa_private;
 	if (bas == NULL) {
 		const MWL_HAL_BASTREAM *sp;
 		/*
 		 * Check for a free BA stream slot.
 		 */
 #if MWL_MAXBA > 3
 		if (mn->mn_ba[3].bastream == NULL)
 			bas = &mn->mn_ba[3];
 		else
 #endif
 #if MWL_MAXBA > 2
 		if (mn->mn_ba[2].bastream == NULL)
 			bas = &mn->mn_ba[2];
 		else
 #endif
 #if MWL_MAXBA > 1
 		if (mn->mn_ba[1].bastream == NULL)
 			bas = &mn->mn_ba[1];
 		else
 #endif
 #if MWL_MAXBA > 0
 		if (mn->mn_ba[0].bastream == NULL)
 			bas = &mn->mn_ba[0];
 		else 
 #endif
 		{
 			/* sta already has max BA streams */
 			/* XXX assign BA stream to highest priority tid */
 			DPRINTF(sc, MWL_DEBUG_AMPDU,
 			    "%s: already has max bastreams\n", __func__);
 			sc->sc_stats.mst_ampdu_reject++;
 			return 0;
 		}
 		/* NB: no held reference to ni */
 		sp = mwl_hal_bastream_alloc(MWL_VAP(vap)->mv_hvap,
 		    (baparamset & IEEE80211_BAPS_POLICY_IMMEDIATE) != 0,
 		    ni->ni_macaddr, tap->txa_tid, ni->ni_htparam,
 		    ni, tap);
 		if (sp == NULL) {
 			/*
 			 * No available stream, return 0 so no
 			 * a-mpdu aggregation will be done.
 			 */
 			DPRINTF(sc, MWL_DEBUG_AMPDU,
 			    "%s: no bastream available\n", __func__);
 			sc->sc_stats.mst_ampdu_nostream++;
 			return 0;
 		}
 		DPRINTF(sc, MWL_DEBUG_AMPDU, "%s: alloc bastream %p\n",
 		    __func__, sp);
 		/* NB: qos is left zero so we won't match in mwl_tx_start */
 		bas->bastream = sp;
 		tap->txa_private = bas;
 	}
 	/* fetch current seq# from the firmware; if available */
 	if (mwl_hal_bastream_get_seqno(sc->sc_mh, bas->bastream,
 	    vap->iv_opmode == IEEE80211_M_STA ? vap->iv_myaddr : ni->ni_macaddr,
 	    &tap->txa_start) != 0)
 		tap->txa_start = 0;
 	return sc->sc_addba_request(ni, tap, dialogtoken, baparamset, batimeout);
 }
 
 static int
 mwl_addba_response(struct ieee80211_node *ni, struct ieee80211_tx_ampdu *tap,
 	int code, int baparamset, int batimeout)
 {
 	struct mwl_softc *sc = ni->ni_ic->ic_ifp->if_softc;
 	struct mwl_bastate *bas;
 
 	bas = tap->txa_private;
 	if (bas == NULL) {
 		/* XXX should not happen */
 		DPRINTF(sc, MWL_DEBUG_AMPDU,
 		    "%s: no BA stream allocated, TID %d\n",
 		    __func__, tap->txa_tid);
 		sc->sc_stats.mst_addba_nostream++;
 		return 0;
 	}
 	if (code == IEEE80211_STATUS_SUCCESS) {
 		struct ieee80211vap *vap = ni->ni_vap;
 		int bufsiz, error;
 
 		/*
 		 * Tell the firmware to setup the BA stream;
 		 * we know resources are available because we
 		 * pre-allocated one before forming the request.
 		 */
 		bufsiz = MS(baparamset, IEEE80211_BAPS_BUFSIZ);
 		if (bufsiz == 0)
 			bufsiz = IEEE80211_AGGR_BAWMAX;
 		error = mwl_hal_bastream_create(MWL_VAP(vap)->mv_hvap,
 		    bas->bastream, bufsiz, bufsiz, tap->txa_start);
 		if (error != 0) {
 			/*
 			 * Setup failed, return immediately so no a-mpdu
 			 * aggregation will be done.
 			 */
 			mwl_hal_bastream_destroy(sc->sc_mh, bas->bastream);
 			mwl_bastream_free(bas);
 			tap->txa_private = NULL;
 
 			DPRINTF(sc, MWL_DEBUG_AMPDU,
 			    "%s: create failed, error %d, bufsiz %d TID %d "
 			    "htparam 0x%x\n", __func__, error, bufsiz,
 			    tap->txa_tid, ni->ni_htparam);
 			sc->sc_stats.mst_bacreate_failed++;
 			return 0;
 		}
 		/* NB: cache txq to avoid ptr indirect */
 		mwl_bastream_setup(bas, tap->txa_tid, bas->bastream->txq);
 		DPRINTF(sc, MWL_DEBUG_AMPDU,
 		    "%s: bastream %p assigned to txq %d TID %d bufsiz %d "
 		    "htparam 0x%x\n", __func__, bas->bastream,
 		    bas->txq, tap->txa_tid, bufsiz, ni->ni_htparam);
 	} else {
 		/*
 		 * Other side NAK'd us; return the resources.
 		 */
 		DPRINTF(sc, MWL_DEBUG_AMPDU,
 		    "%s: request failed with code %d, destroy bastream %p\n",
 		    __func__, code, bas->bastream);
 		mwl_hal_bastream_destroy(sc->sc_mh, bas->bastream);
 		mwl_bastream_free(bas);
 		tap->txa_private = NULL;
 	}
 	/* NB: firmware sends BAR so we don't need to */
 	return sc->sc_addba_response(ni, tap, code, baparamset, batimeout);
 }
 
 static void
 mwl_addba_stop(struct ieee80211_node *ni, struct ieee80211_tx_ampdu *tap)
 {
 	struct mwl_softc *sc = ni->ni_ic->ic_ifp->if_softc;
 	struct mwl_bastate *bas;
 
 	bas = tap->txa_private;
 	if (bas != NULL) {
 		DPRINTF(sc, MWL_DEBUG_AMPDU, "%s: destroy bastream %p\n",
 		    __func__, bas->bastream);
 		mwl_hal_bastream_destroy(sc->sc_mh, bas->bastream);
 		mwl_bastream_free(bas);
 		tap->txa_private = NULL;
 	}
 	sc->sc_addba_stop(ni, tap);
 }
 
 /*
  * Setup the rx data structures.  This should only be
  * done once or we may get out of sync with the firmware.
  */
 static int
 mwl_startrecv(struct mwl_softc *sc)
 {
 	if (!sc->sc_recvsetup) {
 		struct mwl_rxbuf *bf, *prev;
 		struct mwl_rxdesc *ds;
 
 		prev = NULL;
 		STAILQ_FOREACH(bf, &sc->sc_rxbuf, bf_list) {
 			int error = mwl_rxbuf_init(sc, bf);
 			if (error != 0) {
 				DPRINTF(sc, MWL_DEBUG_RECV,
 					"%s: mwl_rxbuf_init failed %d\n",
 					__func__, error);
 				return error;
 			}
 			if (prev != NULL) {
 				ds = prev->bf_desc;
 				ds->pPhysNext = htole32(bf->bf_daddr);
 			}
 			prev = bf;
 		}
 		if (prev != NULL) {
 			ds = prev->bf_desc;
 			ds->pPhysNext =
 			    htole32(STAILQ_FIRST(&sc->sc_rxbuf)->bf_daddr);
 		}
 		sc->sc_recvsetup = 1;
 	}
 	mwl_mode_init(sc);		/* set filters, etc. */
 	return 0;
 }
 
 static MWL_HAL_APMODE
 mwl_getapmode(const struct ieee80211vap *vap, struct ieee80211_channel *chan)
 {
 	MWL_HAL_APMODE mode;
 
 	if (IEEE80211_IS_CHAN_HT(chan)) {
 		if (vap->iv_flags_ht & IEEE80211_FHT_PUREN)
 			mode = AP_MODE_N_ONLY;
 		else if (IEEE80211_IS_CHAN_5GHZ(chan))
 			mode = AP_MODE_AandN;
 		else if (vap->iv_flags & IEEE80211_F_PUREG)
 			mode = AP_MODE_GandN;
 		else
 			mode = AP_MODE_BandGandN;
 	} else if (IEEE80211_IS_CHAN_ANYG(chan)) {
 		if (vap->iv_flags & IEEE80211_F_PUREG)
 			mode = AP_MODE_G_ONLY;
 		else
 			mode = AP_MODE_MIXED;
 	} else if (IEEE80211_IS_CHAN_B(chan))
 		mode = AP_MODE_B_ONLY;
 	else if (IEEE80211_IS_CHAN_A(chan))
 		mode = AP_MODE_A_ONLY;
 	else
 		mode = AP_MODE_MIXED;		/* XXX should not happen? */
 	return mode;
 }
 
 static int
 mwl_setapmode(struct ieee80211vap *vap, struct ieee80211_channel *chan)
 {
 	struct mwl_hal_vap *hvap = MWL_VAP(vap)->mv_hvap;
 	return mwl_hal_setapmode(hvap, mwl_getapmode(vap, chan));
 }
 
 /*
  * Set/change channels.
  */
 static int
 mwl_chan_set(struct mwl_softc *sc, struct ieee80211_channel *chan)
 {
 	struct mwl_hal *mh = sc->sc_mh;
 	struct ifnet *ifp = sc->sc_ifp;
 	struct ieee80211com *ic = ifp->if_l2com;
 	MWL_HAL_CHANNEL hchan;
 	int maxtxpow;
 
 	DPRINTF(sc, MWL_DEBUG_RESET, "%s: chan %u MHz/flags 0x%x\n",
 	    __func__, chan->ic_freq, chan->ic_flags);
 
 	/*
 	 * Convert to a HAL channel description with
 	 * the flags constrained to reflect the current
 	 * operating mode.
 	 */
 	mwl_mapchan(&hchan, chan);
 	mwl_hal_intrset(mh, 0);		/* disable interrupts */
 #if 0
 	mwl_draintxq(sc);		/* clear pending tx frames */
 #endif
 	mwl_hal_setchannel(mh, &hchan);
 	/*
 	 * Tx power is cap'd by the regulatory setting and
 	 * possibly a user-set limit.  We pass the min of
 	 * these to the hal to apply them to the cal data
 	 * for this channel.
 	 * XXX min bound?
 	 */
 	maxtxpow = 2*chan->ic_maxregpower;
 	if (maxtxpow > ic->ic_txpowlimit)
 		maxtxpow = ic->ic_txpowlimit;
 	mwl_hal_settxpower(mh, &hchan, maxtxpow / 2);
 	/* NB: potentially change mcast/mgt rates */
 	mwl_setcurchanrates(sc);
 
 	/*
 	 * Update internal state.
 	 */
 	sc->sc_tx_th.wt_chan_freq = htole16(chan->ic_freq);
 	sc->sc_rx_th.wr_chan_freq = htole16(chan->ic_freq);
 	if (IEEE80211_IS_CHAN_A(chan)) {
 		sc->sc_tx_th.wt_chan_flags = htole16(IEEE80211_CHAN_A);
 		sc->sc_rx_th.wr_chan_flags = htole16(IEEE80211_CHAN_A);
 	} else if (IEEE80211_IS_CHAN_ANYG(chan)) {
 		sc->sc_tx_th.wt_chan_flags = htole16(IEEE80211_CHAN_G);
 		sc->sc_rx_th.wr_chan_flags = htole16(IEEE80211_CHAN_G);
 	} else {
 		sc->sc_tx_th.wt_chan_flags = htole16(IEEE80211_CHAN_B);
 		sc->sc_rx_th.wr_chan_flags = htole16(IEEE80211_CHAN_B);
 	}
 	sc->sc_curchan = hchan;
 	mwl_hal_intrset(mh, sc->sc_imask);
 
 	return 0;
 }
 
 static void
 mwl_scan_start(struct ieee80211com *ic)
 {
 	struct ifnet *ifp = ic->ic_ifp;
 	struct mwl_softc *sc = ifp->if_softc;
 
 	DPRINTF(sc, MWL_DEBUG_STATE, "%s\n", __func__);
 }
 
 static void
 mwl_scan_end(struct ieee80211com *ic)
 {
 	struct ifnet *ifp = ic->ic_ifp;
 	struct mwl_softc *sc = ifp->if_softc;
 
 	DPRINTF(sc, MWL_DEBUG_STATE, "%s\n", __func__);
 }
 
 static void
 mwl_set_channel(struct ieee80211com *ic)
 {
 	struct ifnet *ifp = ic->ic_ifp;
 	struct mwl_softc *sc = ifp->if_softc;
 
 	(void) mwl_chan_set(sc, ic->ic_curchan);
 }
 
 /* 
  * Handle a channel switch request.  We inform the firmware
  * and mark the global state to suppress various actions.
  * NB: we issue only one request to the fw; we may be called
  * multiple times if there are multiple vap's.
  */
 static void
 mwl_startcsa(struct ieee80211vap *vap)
 {
 	struct ieee80211com *ic = vap->iv_ic;
 	struct mwl_softc *sc = ic->ic_ifp->if_softc;
 	MWL_HAL_CHANNEL hchan;
 
 	if (sc->sc_csapending)
 		return;
 
 	mwl_mapchan(&hchan, ic->ic_csa_newchan);
 	/* 1 =>'s quiet channel */
 	mwl_hal_setchannelswitchie(sc->sc_mh, &hchan, 1, ic->ic_csa_count);
 	sc->sc_csapending = 1;
 }
 
 /*
  * Plumb any static WEP key for the station.  This is
  * necessary as we must propagate the key from the
  * global key table of the vap to each sta db entry.
  */
 static void
 mwl_setanywepkey(struct ieee80211vap *vap, const uint8_t mac[IEEE80211_ADDR_LEN])
 {
 	if ((vap->iv_flags & (IEEE80211_F_PRIVACY|IEEE80211_F_WPA)) ==
 		IEEE80211_F_PRIVACY &&
 	    vap->iv_def_txkey != IEEE80211_KEYIX_NONE &&
 	    vap->iv_nw_keys[vap->iv_def_txkey].wk_keyix != IEEE80211_KEYIX_NONE)
 		(void) mwl_key_set(vap, &vap->iv_nw_keys[vap->iv_def_txkey], mac);
 }
 
 static int
 mwl_peerstadb(struct ieee80211_node *ni, int aid, int staid, MWL_HAL_PEERINFO *pi)
 {
 #define	WME(ie) ((const struct ieee80211_wme_info *) ie)
 	struct ieee80211vap *vap = ni->ni_vap;
 	struct mwl_hal_vap *hvap;
 	int error;
 
 	if (vap->iv_opmode == IEEE80211_M_WDS) {
 		/*
 		 * WDS vap's do not have a f/w vap; instead they piggyback
 		 * on an AP vap and we must install the sta db entry and
 		 * crypto state using that AP's handle (the WDS vap has none).
 		 */
 		hvap = MWL_VAP(vap)->mv_ap_hvap;
 	} else
 		hvap = MWL_VAP(vap)->mv_hvap;
 	error = mwl_hal_newstation(hvap, ni->ni_macaddr,
 	    aid, staid, pi,
 	    ni->ni_flags & (IEEE80211_NODE_QOS | IEEE80211_NODE_HT),
 	    ni->ni_ies.wme_ie != NULL ? WME(ni->ni_ies.wme_ie)->wme_info : 0);
 	if (error == 0) {
 		/*
 		 * Setup security for this station.  For sta mode this is
 		 * needed even though do the same thing on transition to
 		 * AUTH state because the call to mwl_hal_newstation
 		 * clobbers the crypto state we setup.
 		 */
 		mwl_setanywepkey(vap, ni->ni_macaddr);
 	}
 	return error;
 #undef WME
 }
 
 static void
 mwl_setglobalkeys(struct ieee80211vap *vap)
 {
 	struct ieee80211_key *wk;
 
 	wk = &vap->iv_nw_keys[0];
 	for (; wk < &vap->iv_nw_keys[IEEE80211_WEP_NKID]; wk++)
 		if (wk->wk_keyix != IEEE80211_KEYIX_NONE)
 			(void) mwl_key_set(vap, wk, vap->iv_myaddr);
 }
 
 /*
  * Convert a legacy rate set to a firmware bitmask.
  */
 static uint32_t
 get_rate_bitmap(const struct ieee80211_rateset *rs)
 {
 	uint32_t rates;
 	int i;
 
 	rates = 0;
 	for (i = 0; i < rs->rs_nrates; i++)
 		switch (rs->rs_rates[i] & IEEE80211_RATE_VAL) {
 		case 2:	  rates |= 0x001; break;
 		case 4:	  rates |= 0x002; break;
 		case 11:  rates |= 0x004; break;
 		case 22:  rates |= 0x008; break;
 		case 44:  rates |= 0x010; break;
 		case 12:  rates |= 0x020; break;
 		case 18:  rates |= 0x040; break;
 		case 24:  rates |= 0x080; break;
 		case 36:  rates |= 0x100; break;
 		case 48:  rates |= 0x200; break;
 		case 72:  rates |= 0x400; break;
 		case 96:  rates |= 0x800; break;
 		case 108: rates |= 0x1000; break;
 		}
 	return rates;
 }
 
 /*
  * Construct an HT firmware bitmask from an HT rate set.
  */
 static uint32_t
 get_htrate_bitmap(const struct ieee80211_htrateset *rs)
 {
 	uint32_t rates;
 	int i;
 
 	rates = 0;
 	for (i = 0; i < rs->rs_nrates; i++) {
 		if (rs->rs_rates[i] < 16)
 			rates |= 1<<rs->rs_rates[i];
 	}
 	return rates;
 }
 
 /*
  * Craft station database entry for station.
  * NB: use host byte order here, the hal handles byte swapping.
  */
 static MWL_HAL_PEERINFO *
 mkpeerinfo(MWL_HAL_PEERINFO *pi, const struct ieee80211_node *ni)
 {
 	const struct ieee80211vap *vap = ni->ni_vap;
 
 	memset(pi, 0, sizeof(*pi));
 	pi->LegacyRateBitMap = get_rate_bitmap(&ni->ni_rates);
 	pi->CapInfo = ni->ni_capinfo;
 	if (ni->ni_flags & IEEE80211_NODE_HT) {
 		/* HT capabilities, etc */
 		pi->HTCapabilitiesInfo = ni->ni_htcap;
 		/* XXX pi.HTCapabilitiesInfo */
 	        pi->MacHTParamInfo = ni->ni_htparam;	
 		pi->HTRateBitMap = get_htrate_bitmap(&ni->ni_htrates);
 		pi->AddHtInfo.ControlChan = ni->ni_htctlchan;
 		pi->AddHtInfo.AddChan = ni->ni_ht2ndchan;
 		pi->AddHtInfo.OpMode = ni->ni_htopmode;
 		pi->AddHtInfo.stbc = ni->ni_htstbc;
 
 		/* constrain according to local configuration */
 		if ((vap->iv_flags_ht & IEEE80211_FHT_SHORTGI40) == 0)
 			pi->HTCapabilitiesInfo &= ~IEEE80211_HTCAP_SHORTGI40;
 		if ((vap->iv_flags_ht & IEEE80211_FHT_SHORTGI20) == 0)
 			pi->HTCapabilitiesInfo &= ~IEEE80211_HTCAP_SHORTGI20;
 		if (ni->ni_chw != 40)
 			pi->HTCapabilitiesInfo &= ~IEEE80211_HTCAP_CHWIDTH40;
 	}
 	return pi;
 }
 
 /*
  * Re-create the local sta db entry for a vap to ensure
  * up to date WME state is pushed to the firmware.  Because
  * this resets crypto state this must be followed by a
  * reload of any keys in the global key table.
  */
 static int
 mwl_localstadb(struct ieee80211vap *vap)
 {
 #define	WME(ie) ((const struct ieee80211_wme_info *) ie)
 	struct mwl_hal_vap *hvap = MWL_VAP(vap)->mv_hvap;
 	struct ieee80211_node *bss;
 	MWL_HAL_PEERINFO pi;
 	int error;
 
 	switch (vap->iv_opmode) {
 	case IEEE80211_M_STA:
 		bss = vap->iv_bss;
 		error = mwl_hal_newstation(hvap, vap->iv_myaddr, 0, 0,
 		    vap->iv_state == IEEE80211_S_RUN ?
 			mkpeerinfo(&pi, bss) : NULL,
 		    (bss->ni_flags & (IEEE80211_NODE_QOS | IEEE80211_NODE_HT)),
 		    bss->ni_ies.wme_ie != NULL ?
 			WME(bss->ni_ies.wme_ie)->wme_info : 0);
 		if (error == 0)
 			mwl_setglobalkeys(vap);
 		break;
 	case IEEE80211_M_HOSTAP:
 	case IEEE80211_M_MBSS:
 		error = mwl_hal_newstation(hvap, vap->iv_myaddr,
 		    0, 0, NULL, vap->iv_flags & IEEE80211_F_WME, 0);
 		if (error == 0)
 			mwl_setglobalkeys(vap);
 		break;
 	default:
 		error = 0;
 		break;
 	}
 	return error;
 #undef WME
 }
 
 static int
 mwl_newstate(struct ieee80211vap *vap, enum ieee80211_state nstate, int arg)
 {
 	struct mwl_vap *mvp = MWL_VAP(vap);
 	struct mwl_hal_vap *hvap = mvp->mv_hvap;
 	struct ieee80211com *ic = vap->iv_ic;
 	struct ieee80211_node *ni = NULL;
 	struct ifnet *ifp = ic->ic_ifp;
 	struct mwl_softc *sc = ifp->if_softc;
 	struct mwl_hal *mh = sc->sc_mh;
 	enum ieee80211_state ostate = vap->iv_state;
 	int error;
 
 	DPRINTF(sc, MWL_DEBUG_STATE, "%s: %s: %s -> %s\n",
 	    vap->iv_ifp->if_xname, __func__,
 	    ieee80211_state_name[ostate], ieee80211_state_name[nstate]);
 
 	callout_stop(&sc->sc_timer);
 	/*
 	 * Clear current radar detection state.
 	 */
 	if (ostate == IEEE80211_S_CAC) {
 		/* stop quiet mode radar detection */
 		mwl_hal_setradardetection(mh, DR_CHK_CHANNEL_AVAILABLE_STOP);
 	} else if (sc->sc_radarena) {
 		/* stop in-service radar detection */
 		mwl_hal_setradardetection(mh, DR_DFS_DISABLE);
 		sc->sc_radarena = 0;
 	}
 	/*
 	 * Carry out per-state actions before doing net80211 work.
 	 */
 	if (nstate == IEEE80211_S_INIT) {
 		/* NB: only ap+sta vap's have a fw entity */
 		if (hvap != NULL)
 			mwl_hal_stop(hvap);
 	} else if (nstate == IEEE80211_S_SCAN) {
 		mwl_hal_start(hvap);
 		/* NB: this disables beacon frames */
 		mwl_hal_setinframode(hvap);
 	} else if (nstate == IEEE80211_S_AUTH) {
 		/*
 		 * Must create a sta db entry in case a WEP key needs to
 		 * be plumbed.  This entry will be overwritten if we
 		 * associate; otherwise it will be reclaimed on node free.
 		 */
 		ni = vap->iv_bss;
 		MWL_NODE(ni)->mn_hvap = hvap;
 		(void) mwl_peerstadb(ni, 0, 0, NULL);
 	} else if (nstate == IEEE80211_S_CSA) {
 		/* XXX move to below? */
 		if (vap->iv_opmode == IEEE80211_M_HOSTAP ||
 		    vap->iv_opmode == IEEE80211_M_MBSS)
 			mwl_startcsa(vap);
 	} else if (nstate == IEEE80211_S_CAC) {
 		/* XXX move to below? */
 		/* stop ap xmit and enable quiet mode radar detection */
 		mwl_hal_setradardetection(mh, DR_CHK_CHANNEL_AVAILABLE_START);
 	}
 
 	/*
 	 * Invoke the parent method to do net80211 work.
 	 */
 	error = mvp->mv_newstate(vap, nstate, arg);
 
 	/*
 	 * Carry out work that must be done after net80211 runs;
 	 * this work requires up to date state (e.g. iv_bss).
 	 */
 	if (error == 0 && nstate == IEEE80211_S_RUN) {
 		/* NB: collect bss node again, it may have changed */
 		ni = vap->iv_bss;
 
 		DPRINTF(sc, MWL_DEBUG_STATE,
 		    "%s: %s(RUN): iv_flags 0x%08x bintvl %d bssid %s "
 		    "capinfo 0x%04x chan %d\n",
 		    vap->iv_ifp->if_xname, __func__, vap->iv_flags,
 		    ni->ni_intval, ether_sprintf(ni->ni_bssid), ni->ni_capinfo,
 		    ieee80211_chan2ieee(ic, ic->ic_curchan));
 
 		/*
 		 * Recreate local sta db entry to update WME/HT state.
 		 */
 		mwl_localstadb(vap);
 		switch (vap->iv_opmode) {
 		case IEEE80211_M_HOSTAP:
 		case IEEE80211_M_MBSS:
 			if (ostate == IEEE80211_S_CAC) {
 				/* enable in-service radar detection */
 				mwl_hal_setradardetection(mh,
 				    DR_IN_SERVICE_MONITOR_START);
 				sc->sc_radarena = 1;
 			}
 			/*
 			 * Allocate and setup the beacon frame
 			 * (and related state).
 			 */
 			error = mwl_reset_vap(vap, IEEE80211_S_RUN);
 			if (error != 0) {
 				DPRINTF(sc, MWL_DEBUG_STATE,
 				    "%s: beacon setup failed, error %d\n",
 				    __func__, error);
 				goto bad;
 			}
 			/* NB: must be after setting up beacon */
 			mwl_hal_start(hvap);
 			break;
 		case IEEE80211_M_STA:
 			DPRINTF(sc, MWL_DEBUG_STATE, "%s: %s: aid 0x%x\n",
 			    vap->iv_ifp->if_xname, __func__, ni->ni_associd);
 			/*
 			 * Set state now that we're associated.
 			 */
 			mwl_hal_setassocid(hvap, ni->ni_bssid, ni->ni_associd);
 			mwl_setrates(vap);
 			mwl_hal_setrtsthreshold(hvap, vap->iv_rtsthreshold);
 			if ((vap->iv_flags & IEEE80211_F_DWDS) &&
 			    sc->sc_ndwdsvaps++ == 0)
 				mwl_hal_setdwds(mh, 1);
 			break;
 		case IEEE80211_M_WDS:
 			DPRINTF(sc, MWL_DEBUG_STATE, "%s: %s: bssid %s\n",
 			    vap->iv_ifp->if_xname, __func__,
 			    ether_sprintf(ni->ni_bssid));
 			mwl_seteapolformat(vap);
 			break;
 		default:
 			break;
 		}
 		/*
 		 * Set CS mode according to operating channel;
 		 * this mostly an optimization for 5GHz.
 		 *
 		 * NB: must follow mwl_hal_start which resets csmode
 		 */
 		if (IEEE80211_IS_CHAN_5GHZ(ic->ic_bsschan))
 			mwl_hal_setcsmode(mh, CSMODE_AGGRESSIVE);
 		else
 			mwl_hal_setcsmode(mh, CSMODE_AUTO_ENA);
 		/*
 		 * Start timer to prod firmware.
 		 */
 		if (sc->sc_ageinterval != 0)
 			callout_reset(&sc->sc_timer, sc->sc_ageinterval*hz,
 			    mwl_agestations, sc);
 	} else if (nstate == IEEE80211_S_SLEEP) {
 		/* XXX set chip in power save */
 	} else if ((vap->iv_flags & IEEE80211_F_DWDS) &&
 	    --sc->sc_ndwdsvaps == 0)
 		mwl_hal_setdwds(mh, 0);
 bad:
 	return error;
 }
 
 /*
  * Manage station id's; these are separate from AID's
  * as AID's may have values out of the range of possible
  * station id's acceptable to the firmware.
  */
 static int
 allocstaid(struct mwl_softc *sc, int aid)
 {
 	int staid;
 
 	if (!(0 < aid && aid < MWL_MAXSTAID) || isset(sc->sc_staid, aid)) {
 		/* NB: don't use 0 */
 		for (staid = 1; staid < MWL_MAXSTAID; staid++)
 			if (isclr(sc->sc_staid, staid))
 				break;
 	} else
 		staid = aid;
 	setbit(sc->sc_staid, staid);
 	return staid;
 }
 
 static void
 delstaid(struct mwl_softc *sc, int staid)
 {
 	clrbit(sc->sc_staid, staid);
 }
 
 /*
  * Setup driver-specific state for a newly associated node.
  * Note that we're called also on a re-associate, the isnew
  * param tells us if this is the first time or not.
  */
 static void
 mwl_newassoc(struct ieee80211_node *ni, int isnew)
 {
 	struct ieee80211vap *vap = ni->ni_vap;
         struct mwl_softc *sc = vap->iv_ic->ic_ifp->if_softc;
 	struct mwl_node *mn = MWL_NODE(ni);
 	MWL_HAL_PEERINFO pi;
 	uint16_t aid;
 	int error;
 
 	aid = IEEE80211_AID(ni->ni_associd);
 	if (isnew) {
 		mn->mn_staid = allocstaid(sc, aid);
 		mn->mn_hvap = MWL_VAP(vap)->mv_hvap;
 	} else {
 		mn = MWL_NODE(ni);
 		/* XXX reset BA stream? */
 	}
 	DPRINTF(sc, MWL_DEBUG_NODE, "%s: mac %s isnew %d aid %d staid %d\n",
 	    __func__, ether_sprintf(ni->ni_macaddr), isnew, aid, mn->mn_staid);
 	error = mwl_peerstadb(ni, aid, mn->mn_staid, mkpeerinfo(&pi, ni));
 	if (error != 0) {
 		DPRINTF(sc, MWL_DEBUG_NODE,
 		    "%s: error %d creating sta db entry\n",
 		    __func__, error);
 		/* XXX how to deal with error? */
 	}
 }
 
 /*
  * Periodically poke the firmware to age out station state
  * (power save queues, pending tx aggregates).
  */
 static void
 mwl_agestations(void *arg)
 {
 	struct mwl_softc *sc = arg;
 
 	mwl_hal_setkeepalive(sc->sc_mh);
 	if (sc->sc_ageinterval != 0)		/* NB: catch dynamic changes */
 		callout_schedule(&sc->sc_timer, sc->sc_ageinterval*hz);
 }
 
 static const struct mwl_hal_channel *
 findhalchannel(const MWL_HAL_CHANNELINFO *ci, int ieee)
 {
 	int i;
 
 	for (i = 0; i < ci->nchannels; i++) {
 		const struct mwl_hal_channel *hc = &ci->channels[i];
 		if (hc->ieee == ieee)
 			return hc;
 	}
 	return NULL;
 }
 
 static int
 mwl_setregdomain(struct ieee80211com *ic, struct ieee80211_regdomain *rd,
 	int nchan, struct ieee80211_channel chans[])
 {
 	struct mwl_softc *sc = ic->ic_ifp->if_softc;
 	struct mwl_hal *mh = sc->sc_mh;
 	const MWL_HAL_CHANNELINFO *ci;
 	int i;
 
 	for (i = 0; i < nchan; i++) {
 		struct ieee80211_channel *c = &chans[i];
 		const struct mwl_hal_channel *hc;
 
 		if (IEEE80211_IS_CHAN_2GHZ(c)) {
 			mwl_hal_getchannelinfo(mh, MWL_FREQ_BAND_2DOT4GHZ,
 			    IEEE80211_IS_CHAN_HT40(c) ?
 				MWL_CH_40_MHz_WIDTH : MWL_CH_20_MHz_WIDTH, &ci);
 		} else if (IEEE80211_IS_CHAN_5GHZ(c)) {
 			mwl_hal_getchannelinfo(mh, MWL_FREQ_BAND_5GHZ,
 			    IEEE80211_IS_CHAN_HT40(c) ?
 				MWL_CH_40_MHz_WIDTH : MWL_CH_20_MHz_WIDTH, &ci);
 		} else {
 			if_printf(ic->ic_ifp,
 			    "%s: channel %u freq %u/0x%x not 2.4/5GHz\n",
 			    __func__, c->ic_ieee, c->ic_freq, c->ic_flags);
 			return EINVAL;
 		}
 		/* 
 		 * Verify channel has cal data and cap tx power.
 		 */
 		hc = findhalchannel(ci, c->ic_ieee);
 		if (hc != NULL) {
 			if (c->ic_maxpower > 2*hc->maxTxPow)
 				c->ic_maxpower = 2*hc->maxTxPow;
 			goto next;
 		}
 		if (IEEE80211_IS_CHAN_HT40(c)) {
 			/*
 			 * Look for the extension channel since the
 			 * hal table only has the primary channel.
 			 */
 			hc = findhalchannel(ci, c->ic_extieee);
 			if (hc != NULL) {
 				if (c->ic_maxpower > 2*hc->maxTxPow)
 					c->ic_maxpower = 2*hc->maxTxPow;
 				goto next;
 			}
 		}
 		if_printf(ic->ic_ifp,
 		    "%s: no cal data for channel %u ext %u freq %u/0x%x\n",
 		    __func__, c->ic_ieee, c->ic_extieee,
 		    c->ic_freq, c->ic_flags);
 		return EINVAL;
 	next:
 		;
 	}
 	return 0;
 }
 
 #define	IEEE80211_CHAN_HTG	(IEEE80211_CHAN_HT|IEEE80211_CHAN_G)
 #define	IEEE80211_CHAN_HTA	(IEEE80211_CHAN_HT|IEEE80211_CHAN_A)
 
 static void
 addchan(struct ieee80211_channel *c, int freq, int flags, int ieee, int txpow)
 {
 	c->ic_freq = freq;
 	c->ic_flags = flags;
 	c->ic_ieee = ieee;
 	c->ic_minpower = 0;
 	c->ic_maxpower = 2*txpow;
 	c->ic_maxregpower = txpow;
 }
 
 static const struct ieee80211_channel *
 findchannel(const struct ieee80211_channel chans[], int nchans,
 	int freq, int flags)
 {
 	const struct ieee80211_channel *c;
 	int i;
 
 	for (i = 0; i < nchans; i++) {
 		c = &chans[i];
 		if (c->ic_freq == freq && c->ic_flags == flags)
 			return c;
 	}
 	return NULL;
 }
 
 static void
 addht40channels(struct ieee80211_channel chans[], int maxchans, int *nchans,
 	const MWL_HAL_CHANNELINFO *ci, int flags)
 {
 	struct ieee80211_channel *c;
 	const struct ieee80211_channel *extc;
 	const struct mwl_hal_channel *hc;
 	int i;
 
 	c = &chans[*nchans];
 
 	flags &= ~IEEE80211_CHAN_HT;
 	for (i = 0; i < ci->nchannels; i++) {
 		/*
 		 * Each entry defines an HT40 channel pair; find the
 		 * extension channel above and the insert the pair.
 		 */
 		hc = &ci->channels[i];
 		extc = findchannel(chans, *nchans, hc->freq+20,
 		    flags | IEEE80211_CHAN_HT20);
 		if (extc != NULL) {
 			if (*nchans >= maxchans)
 				break;
 			addchan(c, hc->freq, flags | IEEE80211_CHAN_HT40U,
 			    hc->ieee, hc->maxTxPow);
 			c->ic_extieee = extc->ic_ieee;
 			c++, (*nchans)++;
 			if (*nchans >= maxchans)
 				break;
 			addchan(c, extc->ic_freq, flags | IEEE80211_CHAN_HT40D,
 			    extc->ic_ieee, hc->maxTxPow);
 			c->ic_extieee = hc->ieee;
 			c++, (*nchans)++;
 		}
 	}
 }
 
 static void
 addchannels(struct ieee80211_channel chans[], int maxchans, int *nchans,
 	const MWL_HAL_CHANNELINFO *ci, int flags)
 {
 	struct ieee80211_channel *c;
 	int i;
 
 	c = &chans[*nchans];
 
 	for (i = 0; i < ci->nchannels; i++) {
 		const struct mwl_hal_channel *hc;
 
 		hc = &ci->channels[i];
 		if (*nchans >= maxchans)
 			break;
 		addchan(c, hc->freq, flags, hc->ieee, hc->maxTxPow);
 		c++, (*nchans)++;
 		if (flags == IEEE80211_CHAN_G || flags == IEEE80211_CHAN_HTG) {
 			/* g channel have a separate b-only entry */
 			if (*nchans >= maxchans)
 				break;
 			c[0] = c[-1];
 			c[-1].ic_flags = IEEE80211_CHAN_B;
 			c++, (*nchans)++;
 		}
 		if (flags == IEEE80211_CHAN_HTG) {
 			/* HT g channel have a separate g-only entry */
 			if (*nchans >= maxchans)
 				break;
 			c[-1].ic_flags = IEEE80211_CHAN_G;
 			c[0] = c[-1];
 			c[0].ic_flags &= ~IEEE80211_CHAN_HT;
 			c[0].ic_flags |= IEEE80211_CHAN_HT20;	/* HT20 */
 			c++, (*nchans)++;
 		}
 		if (flags == IEEE80211_CHAN_HTA) {
 			/* HT a channel have a separate a-only entry */
 			if (*nchans >= maxchans)
 				break;
 			c[-1].ic_flags = IEEE80211_CHAN_A;
 			c[0] = c[-1];
 			c[0].ic_flags &= ~IEEE80211_CHAN_HT;
 			c[0].ic_flags |= IEEE80211_CHAN_HT20;	/* HT20 */
 			c++, (*nchans)++;
 		}
 	}
 }
 
 static void
 getchannels(struct mwl_softc *sc, int maxchans, int *nchans,
 	struct ieee80211_channel chans[])
 {
 	const MWL_HAL_CHANNELINFO *ci;
 
 	/*
 	 * Use the channel info from the hal to craft the
 	 * channel list.  Note that we pass back an unsorted
 	 * list; the caller is required to sort it for us
 	 * (if desired).
 	 */
 	*nchans = 0;
 	if (mwl_hal_getchannelinfo(sc->sc_mh,
 	    MWL_FREQ_BAND_2DOT4GHZ, MWL_CH_20_MHz_WIDTH, &ci) == 0)
 		addchannels(chans, maxchans, nchans, ci, IEEE80211_CHAN_HTG);
 	if (mwl_hal_getchannelinfo(sc->sc_mh,
 	    MWL_FREQ_BAND_5GHZ, MWL_CH_20_MHz_WIDTH, &ci) == 0)
 		addchannels(chans, maxchans, nchans, ci, IEEE80211_CHAN_HTA);
 	if (mwl_hal_getchannelinfo(sc->sc_mh,
 	    MWL_FREQ_BAND_2DOT4GHZ, MWL_CH_40_MHz_WIDTH, &ci) == 0)
 		addht40channels(chans, maxchans, nchans, ci, IEEE80211_CHAN_HTG);
 	if (mwl_hal_getchannelinfo(sc->sc_mh,
 	    MWL_FREQ_BAND_5GHZ, MWL_CH_40_MHz_WIDTH, &ci) == 0)
 		addht40channels(chans, maxchans, nchans, ci, IEEE80211_CHAN_HTA);
 }
 
 static void
 mwl_getradiocaps(struct ieee80211com *ic,
 	int maxchans, int *nchans, struct ieee80211_channel chans[])
 {
 	struct mwl_softc *sc = ic->ic_ifp->if_softc;
 
 	getchannels(sc, maxchans, nchans, chans);
 }
 
 static int
 mwl_getchannels(struct mwl_softc *sc)
 {
 	struct ifnet *ifp = sc->sc_ifp;
 	struct ieee80211com *ic = ifp->if_l2com;
 
 	/*
 	 * Use the channel info from the hal to craft the
 	 * channel list for net80211.  Note that we pass up
 	 * an unsorted list; net80211 will sort it for us.
 	 */
 	memset(ic->ic_channels, 0, sizeof(ic->ic_channels));
 	ic->ic_nchans = 0;
 	getchannels(sc, IEEE80211_CHAN_MAX, &ic->ic_nchans, ic->ic_channels);
 
 	ic->ic_regdomain.regdomain = SKU_DEBUG;
 	ic->ic_regdomain.country = CTRY_DEFAULT;
 	ic->ic_regdomain.location = 'I';
 	ic->ic_regdomain.isocc[0] = ' ';	/* XXX? */
 	ic->ic_regdomain.isocc[1] = ' ';
 	return (ic->ic_nchans == 0 ? EIO : 0);
 }
 #undef IEEE80211_CHAN_HTA
 #undef IEEE80211_CHAN_HTG
 
 #ifdef MWL_DEBUG
 static void
 mwl_printrxbuf(const struct mwl_rxbuf *bf, u_int ix)
 {
 	const struct mwl_rxdesc *ds = bf->bf_desc;
 	uint32_t status = le32toh(ds->Status);
 
 	printf("R[%2u] (DS.V:%p DS.P:%p) NEXT:%08x DATA:%08x RC:%02x%s\n"
 	       "      STAT:%02x LEN:%04x RSSI:%02x CHAN:%02x RATE:%02x QOS:%04x HT:%04x\n",
 	    ix, ds, (const struct mwl_desc *)bf->bf_daddr,
 	    le32toh(ds->pPhysNext), le32toh(ds->pPhysBuffData),
 	    ds->RxControl, 
 	    ds->RxControl != EAGLE_RXD_CTRL_DRIVER_OWN ?
 	        "" : (status & EAGLE_RXD_STATUS_OK) ? " *" : " !",
 	    ds->Status, le16toh(ds->PktLen), ds->RSSI, ds->Channel,
 	    ds->Rate, le16toh(ds->QosCtrl), le16toh(ds->HtSig2));
 }
 
 static void
 mwl_printtxbuf(const struct mwl_txbuf *bf, u_int qnum, u_int ix)
 {
 	const struct mwl_txdesc *ds = bf->bf_desc;
 	uint32_t status = le32toh(ds->Status);
 
 	printf("Q%u[%3u]", qnum, ix);
 	printf(" (DS.V:%p DS.P:%p)\n",
 	    ds, (const struct mwl_txdesc *)bf->bf_daddr);
 	printf("    NEXT:%08x DATA:%08x LEN:%04x STAT:%08x%s\n",
 	    le32toh(ds->pPhysNext),
 	    le32toh(ds->PktPtr), le16toh(ds->PktLen), status,
 	    status & EAGLE_TXD_STATUS_USED ?
 		"" : (status & 3) != 0 ? " *" : " !");
 	printf("    RATE:%02x PRI:%x QOS:%04x SAP:%08x FORMAT:%04x\n",
 	    ds->DataRate, ds->TxPriority, le16toh(ds->QosCtrl),
 	    le32toh(ds->SapPktInfo), le16toh(ds->Format));
 #if MWL_TXDESC > 1
 	printf("    MULTIFRAMES:%u LEN:%04x %04x %04x %04x %04x %04x\n"
 	    , le32toh(ds->multiframes)
 	    , le16toh(ds->PktLenArray[0]), le16toh(ds->PktLenArray[1])
 	    , le16toh(ds->PktLenArray[2]), le16toh(ds->PktLenArray[3])
 	    , le16toh(ds->PktLenArray[4]), le16toh(ds->PktLenArray[5])
 	);
 	printf("    DATA:%08x %08x %08x %08x %08x %08x\n"
 	    , le32toh(ds->PktPtrArray[0]), le32toh(ds->PktPtrArray[1])
 	    , le32toh(ds->PktPtrArray[2]), le32toh(ds->PktPtrArray[3])
 	    , le32toh(ds->PktPtrArray[4]), le32toh(ds->PktPtrArray[5])
 	);
 #endif
 #if 0
 { const uint8_t *cp = (const uint8_t *) ds;
   int i;
   for (i = 0; i < sizeof(struct mwl_txdesc); i++) {
 	printf("%02x ", cp[i]);
 	if (((i+1) % 16) == 0)
 		printf("\n");
   }
   printf("\n");
 }
 #endif
 }
 #endif /* MWL_DEBUG */
 
 #if 0
 static void
 mwl_txq_dump(struct mwl_txq *txq)
 {
 	struct mwl_txbuf *bf;
 	int i = 0;
 
 	MWL_TXQ_LOCK(txq);
 	STAILQ_FOREACH(bf, &txq->active, bf_list) {
 		struct mwl_txdesc *ds = bf->bf_desc;
 		MWL_TXDESC_SYNC(txq, ds,
 		    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 #ifdef MWL_DEBUG
 		mwl_printtxbuf(bf, txq->qnum, i);
 #endif
 		i++;
 	}
 	MWL_TXQ_UNLOCK(txq);
 }
 #endif
 
 static void
 mwl_watchdog(void *arg)
 {
 	struct mwl_softc *sc;
 	struct ifnet *ifp;
 
 	sc = arg;
 	callout_reset(&sc->sc_watchdog, hz, mwl_watchdog, sc);
 	if (sc->sc_tx_timer == 0 || --sc->sc_tx_timer > 0)
 		return;
 
 	ifp = sc->sc_ifp;
 	if ((ifp->if_drv_flags & IFF_DRV_RUNNING) && !sc->sc_invalid) {
 		if (mwl_hal_setkeepalive(sc->sc_mh))
 			if_printf(ifp, "transmit timeout (firmware hung?)\n");
 		else
 			if_printf(ifp, "transmit timeout\n");
 #if 0
 		mwl_reset(ifp);
 mwl_txq_dump(&sc->sc_txq[0]);/*XXX*/
 #endif
 		ifp->if_oerrors++;
 		sc->sc_stats.mst_watchdog++;
 	}
 }
 
 #ifdef MWL_DIAGAPI
 /*
  * Diagnostic interface to the HAL.  This is used by various
  * tools to do things like retrieve register contents for
  * debugging.  The mechanism is intentionally opaque so that
  * it can change frequently w/o concern for compatiblity.
  */
 static int
 mwl_ioctl_diag(struct mwl_softc *sc, struct mwl_diag *md)
 {
 	struct mwl_hal *mh = sc->sc_mh;
 	u_int id = md->md_id & MWL_DIAG_ID;
 	void *indata = NULL;
 	void *outdata = NULL;
 	u_int32_t insize = md->md_in_size;
 	u_int32_t outsize = md->md_out_size;
 	int error = 0;
 
 	if (md->md_id & MWL_DIAG_IN) {
 		/*
 		 * Copy in data.
 		 */
 		indata = malloc(insize, M_TEMP, M_NOWAIT);
 		if (indata == NULL) {
 			error = ENOMEM;
 			goto bad;
 		}
 		error = copyin(md->md_in_data, indata, insize);
 		if (error)
 			goto bad;
 	}
 	if (md->md_id & MWL_DIAG_DYN) {
 		/*
 		 * Allocate a buffer for the results (otherwise the HAL
 		 * returns a pointer to a buffer where we can read the
 		 * results).  Note that we depend on the HAL leaving this
 		 * pointer for us to use below in reclaiming the buffer;
 		 * may want to be more defensive.
 		 */
 		outdata = malloc(outsize, M_TEMP, M_NOWAIT);
 		if (outdata == NULL) {
 			error = ENOMEM;
 			goto bad;
 		}
 	}
 	if (mwl_hal_getdiagstate(mh, id, indata, insize, &outdata, &outsize)) {
 		if (outsize < md->md_out_size)
 			md->md_out_size = outsize;
 		if (outdata != NULL)
 			error = copyout(outdata, md->md_out_data,
 					md->md_out_size);
 	} else {
 		error = EINVAL;
 	}
 bad:
 	if ((md->md_id & MWL_DIAG_IN) && indata != NULL)
 		free(indata, M_TEMP);
 	if ((md->md_id & MWL_DIAG_DYN) && outdata != NULL)
 		free(outdata, M_TEMP);
 	return error;
 }
 
 static int
 mwl_ioctl_reset(struct mwl_softc *sc, struct mwl_diag *md)
 {
 	struct mwl_hal *mh = sc->sc_mh;
 	int error;
 
 	MWL_LOCK_ASSERT(sc);
 
 	if (md->md_id == 0 && mwl_hal_fwload(mh, NULL) != 0) {
 		device_printf(sc->sc_dev, "unable to load firmware\n");
 		return EIO;
 	}
 	if (mwl_hal_gethwspecs(mh, &sc->sc_hwspecs) != 0) {
 		device_printf(sc->sc_dev, "unable to fetch h/w specs\n");
 		return EIO;
 	}
 	error = mwl_setupdma(sc);
 	if (error != 0) {
 		/* NB: mwl_setupdma prints a msg */
 		return error;
 	}
 	/*
 	 * Reset tx/rx data structures; after reload we must
 	 * re-start the driver's notion of the next xmit/recv.
 	 */
 	mwl_draintxq(sc);		/* clear pending frames */
 	mwl_resettxq(sc);		/* rebuild tx q lists */
 	sc->sc_rxnext = NULL;		/* force rx to start at the list head */
 	return 0;
 }
 #endif /* MWL_DIAGAPI */
 
 static int
 mwl_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data)
 {
 #define	IS_RUNNING(ifp) \
 	((ifp->if_flags & IFF_UP) && (ifp->if_drv_flags & IFF_DRV_RUNNING))
 	struct mwl_softc *sc = ifp->if_softc;
 	struct ieee80211com *ic = ifp->if_l2com;
 	struct ifreq *ifr = (struct ifreq *)data;
 	int error = 0, startall;
 
 	switch (cmd) {
 	case SIOCSIFFLAGS:
 		MWL_LOCK(sc);
 		startall = 0;
 		if (IS_RUNNING(ifp)) {
 			/*
 			 * To avoid rescanning another access point,
 			 * do not call mwl_init() here.  Instead,
 			 * only reflect promisc mode settings.
 			 */
 			mwl_mode_init(sc);
 		} else if (ifp->if_flags & IFF_UP) {
 			/*
 			 * Beware of being called during attach/detach
 			 * to reset promiscuous mode.  In that case we
 			 * will still be marked UP but not RUNNING.
 			 * However trying to re-init the interface
 			 * is the wrong thing to do as we've already
 			 * torn down much of our state.  There's
 			 * probably a better way to deal with this.
 			 */
 			if (!sc->sc_invalid) {
 				mwl_init_locked(sc);	/* XXX lose error */
 				startall = 1;
 			}
 		} else
 			mwl_stop_locked(ifp, 1);
 		MWL_UNLOCK(sc);
 		if (startall)
 			ieee80211_start_all(ic);
 		break;
 	case SIOCGMVSTATS:
 		mwl_hal_gethwstats(sc->sc_mh, &sc->sc_stats.hw_stats);
 		/* NB: embed these numbers to get a consistent view */
 		sc->sc_stats.mst_tx_packets = ifp->if_opackets;
 		sc->sc_stats.mst_rx_packets = ifp->if_ipackets;
 		/*
 		 * NB: Drop the softc lock in case of a page fault;
 		 * we'll accept any potential inconsisentcy in the
 		 * statistics.  The alternative is to copy the data
 		 * to a local structure.
 		 */
 		return copyout(&sc->sc_stats,
 				ifr->ifr_data, sizeof (sc->sc_stats));
 #ifdef MWL_DIAGAPI
 	case SIOCGMVDIAG:
 		/* XXX check privs */
 		return mwl_ioctl_diag(sc, (struct mwl_diag *) ifr);
 	case SIOCGMVRESET:
 		/* XXX check privs */
 		MWL_LOCK(sc);
 		error = mwl_ioctl_reset(sc,(struct mwl_diag *) ifr); 
 		MWL_UNLOCK(sc);
 		break;
 #endif /* MWL_DIAGAPI */
 	case SIOCGIFMEDIA:
 		error = ifmedia_ioctl(ifp, ifr, &ic->ic_media, cmd);
 		break;
 	case SIOCGIFADDR:
 		error = ether_ioctl(ifp, cmd, data);
 		break;
 	default:
 		error = EINVAL;
 		break;
 	}
 	return error;
 #undef IS_RUNNING
 }
 
 #ifdef	MWL_DEBUG
 static int
 mwl_sysctl_debug(SYSCTL_HANDLER_ARGS)
 {
 	struct mwl_softc *sc = arg1;
 	int debug, error;
 
 	debug = sc->sc_debug | (mwl_hal_getdebug(sc->sc_mh) << 24);
 	error = sysctl_handle_int(oidp, &debug, 0, req);
 	if (error || !req->newptr)
 		return error;
 	mwl_hal_setdebug(sc->sc_mh, debug >> 24);
 	sc->sc_debug = debug & 0x00ffffff;
 	return 0;
 }
 #endif /* MWL_DEBUG */
 
 static void
 mwl_sysctlattach(struct mwl_softc *sc)
 {
 #ifdef	MWL_DEBUG
 	struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->sc_dev);
 	struct sysctl_oid *tree = device_get_sysctl_tree(sc->sc_dev);
 
 	sc->sc_debug = mwl_debug;
 	SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
 		"debug", CTLTYPE_INT | CTLFLAG_RW, sc, 0,
 		mwl_sysctl_debug, "I", "control debugging printfs");
 #endif
 }
 
 /*
  * Announce various information on device/driver attach.
  */
 static void
 mwl_announce(struct mwl_softc *sc)
 {
 	struct ifnet *ifp = sc->sc_ifp;
 
 	if_printf(ifp, "Rev A%d hardware, v%d.%d.%d.%d firmware (regioncode %d)\n",
 		sc->sc_hwspecs.hwVersion,
 		(sc->sc_hwspecs.fwReleaseNumber>>24) & 0xff,
 		(sc->sc_hwspecs.fwReleaseNumber>>16) & 0xff,
 		(sc->sc_hwspecs.fwReleaseNumber>>8) & 0xff,
 		(sc->sc_hwspecs.fwReleaseNumber>>0) & 0xff,
 		sc->sc_hwspecs.regionCode);
 	sc->sc_fwrelease = sc->sc_hwspecs.fwReleaseNumber;
 
 	if (bootverbose) {
 		int i;
 		for (i = 0; i <= WME_AC_VO; i++) {
 			struct mwl_txq *txq = sc->sc_ac2q[i];
 			if_printf(ifp, "Use hw queue %u for %s traffic\n",
 				txq->qnum, ieee80211_wme_acnames[i]);
 		}
 	}
 	if (bootverbose || mwl_rxdesc != MWL_RXDESC)
 		if_printf(ifp, "using %u rx descriptors\n", mwl_rxdesc);
 	if (bootverbose || mwl_rxbuf != MWL_RXBUF)
 		if_printf(ifp, "using %u rx buffers\n", mwl_rxbuf);
 	if (bootverbose || mwl_txbuf != MWL_TXBUF)
 		if_printf(ifp, "using %u tx buffers\n", mwl_txbuf);
 	if (bootverbose && mwl_hal_ismbsscapable(sc->sc_mh))
 		if_printf(ifp, "multi-bss support\n");
 #ifdef MWL_TX_NODROP
 	if (bootverbose)
 		if_printf(ifp, "no tx drop\n");
 #endif
 }
Index: projects/release-debugdist/sys/dev/mwl/mwlhal.c
===================================================================
--- projects/release-debugdist/sys/dev/mwl/mwlhal.c	(revision 267343)
+++ projects/release-debugdist/sys/dev/mwl/mwlhal.c	(revision 267344)
@@ -1,2787 +1,2778 @@
 /*-
  * Copyright (c) 2007-2009 Sam Leffler, Errno Consulting
  * Copyright (c) 2007-2009 Marvell Semiconductor, Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer,
  *    without modification.
  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  *    similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any
  *    redistribution must be conditioned upon including a substantially
  *    similar Disclaimer requirement for further binary redistribution.
  *
  * NO WARRANTY
  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY
  * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
  * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY,
  * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
  * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  * THE POSSIBILITY OF SUCH DAMAGES.
  *
  * $FreeBSD$
  */
 
 #include <sys/param.h>
 #include <sys/systm.h> 
 #include <sys/sysctl.h>
 #include <sys/malloc.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <sys/kernel.h>
 #include <sys/errno.h>
 #include <sys/bus.h>
 #include <sys/endian.h>
 
 #include <sys/linker.h>
 #include <sys/firmware.h>
 
 #include <machine/bus.h>
 
 #include <dev/mwl/mwlhal.h>
 #include <dev/mwl/mwlreg.h>
 
 #include <sys/socket.h>
 #include <sys/sockio.h>
 #include <net/if.h>
 #include <dev/mwl/mwldiag.h>
 
 #define	MWLHAL_DEBUG			/* debug msgs */
 
 typedef enum {
     WL_ANTENNAMODE_RX = 0xffff,
     WL_ANTENNAMODE_TX = 2,
 } wlantennamode_e;
 
 typedef enum {
     WL_TX_POWERLEVEL_LOW = 5,
     WL_TX_POWERLEVEL_MEDIUM = 10,
     WL_TX_POWERLEVEL_HIGH = 15,
 } wltxpowerlevel_e;
 
 #define	MWL_CMDBUF_SIZE	0x4000		/* size of f/w command buffer */
 #define	MWL_BASTREAMS_MAX	7	/* max BA streams (NB: fw >3.3.5.9) */
 #define	MWL_BAQID_MAX		8	/* max BA Q id's (NB: fw >3.3.5.9) */
 #define	MWL_MBSS_AP_MAX		8	/* max ap vap's */
 #define	MWL_MBSS_STA_MAX	24	/* max station/client vap's */
 #define	MWL_MBSS_MAX	(MWL_MBSS_AP_MAX+MWL_MBSS_STA_MAX)
 
 /*
  * BA stream -> queue ID mapping
  *
  * The first 2 streams map to h/w; the remaining streams are
  * implemented in firmware.
  */
 static const int ba2qid[MWL_BASTREAMS_MAX] = {
 	5, 6				/* h/w supported */
 #if MWL_BASTREAMS_MAX == 7
 	, 7, 0, 1, 2, 3 		/* f/w supported */
 #endif
 };
 static int qid2ba[MWL_BAQID_MAX];
 
 #define	IEEE80211_ADDR_LEN	6	/* XXX */
 #define	IEEE80211_ADDR_COPY(_dst, _src) \
 	memcpy(_dst, _src, IEEE80211_ADDR_LEN)
 #define	IEEE80211_ADDR_EQ(_dst, _src) \
 	(memcmp(_dst, _src, IEEE80211_ADDR_LEN) == 0)
 
 #define	_CMD_SETUP(pCmd, type, cmd) do {				\
 	pCmd = (type *)&mh->mh_cmdbuf[0];				\
 	memset(pCmd, 0, sizeof(type));					\
 	pCmd->CmdHdr.Cmd = htole16(cmd);				\
 	pCmd->CmdHdr.Length = htole16(sizeof(type));			\
 } while (0)
 
 #define	_VCMD_SETUP(vap, pCmd, type, cmd) do {				\
 	_CMD_SETUP(pCmd, type, cmd);					\
 	pCmd->CmdHdr.MacId = vap->macid;				\
 } while (0)
 
 #define	PWTAGETRATETABLE20M	14*4
 #define	PWTAGETRATETABLE40M	9*4
 #define	PWTAGETRATETABLE20M_5G	35*4
 #define	PWTAGETRATETABLE40M_5G	16*4
 
 struct mwl_hal_bastream {
 	MWL_HAL_BASTREAM public;	/* public state */
 	uint8_t	stream;			/* stream # */
 	uint8_t	setup;			/* f/w cmd sent */
 	uint8_t ba_policy;		/* direct/delayed BA policy */
 	uint8_t	tid;
 	uint8_t	paraminfo;
 	uint8_t macaddr[IEEE80211_ADDR_LEN];
 };
 
 struct mwl_hal_priv;
 
 struct mwl_hal_vap {
 	struct mwl_hal_priv *mh;	/* back pointer */
 	uint16_t bss_type;		/* f/w type */
 	uint8_t vap_type;		/* MWL_HAL_BSSTYPE */
 	uint8_t	macid;			/* for passing to f/w */
 	uint8_t flags;
 #define	MVF_RUNNING	0x01		/* BSS_START issued */
 #define	MVF_STATION	0x02		/* sta db entry created */
 	uint8_t mac[IEEE80211_ADDR_LEN];/* mac address */
 };
 #define	MWLVAP(_vap)	((_vap)->mh)
 
 /*
  * Per-device state.  We allocate a single cmd buffer for
  * submitting operations to the firmware.  Access to this
  * buffer (and the f/w) are single-threaded.  At present
  * we spin waiting for cmds to complete which is bad.  Not
  * sure if it's possible to submit multiple requests or
  * control when we get cmd done interrupts.  There's no
  * documentation and no example code to indicate what can
  * or cannot be done so all we can do right now is follow the
  * linux driver logic.  This falls apart when the f/w fails;
  * the system comes to a crawl as we spin waiting for operations
  * to finish.
  */
 struct mwl_hal_priv {
 	struct mwl_hal	public;		/* public area */
 	device_t	mh_dev;
 	char		mh_mtxname[12];
 	struct mtx	mh_mtx;
 	bus_dma_tag_t	mh_dmat;	/* bus DMA tag for cmd buffer */
 	bus_dma_segment_t mh_seg;	/* segment for cmd buffer */
 	bus_dmamap_t	mh_dmamap;	/* DMA map for cmd buffer */
 	uint16_t	*mh_cmdbuf;	/* f/w cmd buffer */
 	bus_addr_t	mh_cmdaddr;	/* physaddr of cmd buffer */
 	int		mh_flags;
 #define	MHF_CALDATA	0x0001		/* cal data retrieved */
 #define	MHF_FWHANG	0x0002		/* fw appears hung */
 #define	MHF_MBSS	0x0004		/* mbss enabled */
 	struct mwl_hal_vap mh_vaps[MWL_MBSS_MAX+1];
 	int		mh_bastreams;	/* bit mask of available BA streams */
 	int		mh_regioncode;	/* XXX last region code sent to fw */
 	struct mwl_hal_bastream mh_streams[MWL_BASTREAMS_MAX];
 	int		mh_debug;
 	MWL_HAL_CHANNELINFO mh_20M;
 	MWL_HAL_CHANNELINFO mh_40M;
 	MWL_HAL_CHANNELINFO mh_20M_5G;
 	MWL_HAL_CHANNELINFO mh_40M_5G;
 	int		mh_SDRAMSIZE_Addr;
 	uint32_t	mh_RTSSuccesses;/* cumulative stats for read-on-clear */
 	uint32_t	mh_RTSFailures;
 	uint32_t	mh_RxDuplicateFrames;
 	uint32_t	mh_FCSErrorCount;
 	MWL_DIAG_REVS	mh_revs;
 };
 #define	MWLPRIV(_mh)	((struct mwl_hal_priv *)(_mh))
 
 static int mwl_hal_setmac_locked(struct mwl_hal_vap *,
 	const uint8_t addr[IEEE80211_ADDR_LEN]);
 static int mwlExecuteCmd(struct mwl_hal_priv *, unsigned short cmd);
 static int mwlGetPwrCalTable(struct mwl_hal_priv *);
 #ifdef MWLHAL_DEBUG
 static const char *mwlcmdname(int cmd);
 static void dumpresult(struct mwl_hal_priv *, int showresult);
 #endif /* MWLHAL_DEBUG */
 
 SYSCTL_DECL(_hw_mwl);
 static SYSCTL_NODE(_hw_mwl, OID_AUTO, hal, CTLFLAG_RD, 0,
     "Marvell HAL parameters");
 
 static __inline void
 MWL_HAL_LOCK(struct mwl_hal_priv *mh)
 {
 	mtx_lock(&mh->mh_mtx);
 }
 
 static __inline void
 MWL_HAL_LOCK_ASSERT(struct mwl_hal_priv *mh)
 {
 	mtx_assert(&mh->mh_mtx, MA_OWNED);
 }
 
 static __inline void
 MWL_HAL_UNLOCK(struct mwl_hal_priv *mh)
 {
 	mtx_unlock(&mh->mh_mtx);
 }
 
 static __inline uint32_t
 RD4(struct mwl_hal_priv *mh, bus_size_t off)
 {
 	return bus_space_read_4(mh->public.mh_iot, mh->public.mh_ioh, off);
 }
 
 static __inline void
 WR4(struct mwl_hal_priv *mh, bus_size_t off, uint32_t val)
 {
 	bus_space_write_4(mh->public.mh_iot, mh->public.mh_ioh, off, val);
 }
 
 static void
 mwl_hal_load_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
 {
 	bus_addr_t *paddr = (bus_addr_t*) arg;
 	KASSERT(error == 0, ("error %u on bus_dma callback", error));
 	*paddr = segs->ds_addr;
 }
 
 /*
  * Setup for communication with the device.  We allocate
  * a command buffer and map it for bus dma use.  The pci
  * device id is used to identify whether the device has
  * SRAM on it (in which case f/w download must include a
  * memory controller reset).  All bus i/o operations happen
  * in BAR 1; the driver passes in the tag and handle we need.
  */
 struct mwl_hal *
 mwl_hal_attach(device_t dev, uint16_t devid,
     bus_space_handle_t ioh, bus_space_tag_t iot, bus_dma_tag_t tag)
 {
 	struct mwl_hal_priv *mh;
 	struct mwl_hal_vap *hvap;
 	int error, i;
 
 	mh = malloc(sizeof(struct mwl_hal_priv), M_DEVBUF, M_NOWAIT | M_ZERO);
 	if (mh == NULL)
 		return NULL;
 	mh->mh_dev = dev;
 	mh->public.mh_ioh = ioh;
 	mh->public.mh_iot = iot;
 	for (i = 0; i < MWL_BASTREAMS_MAX; i++) {
 		mh->mh_streams[i].public.txq = ba2qid[i];
 		mh->mh_streams[i].stream = i;
 		/* construct back-mapping while we're at it */
 		if (mh->mh_streams[i].public.txq < MWL_BAQID_MAX)
 			qid2ba[mh->mh_streams[i].public.txq] = i;
 		else
 			device_printf(dev, "unexpected BA tx qid %d for "
 			    "stream %d\n", mh->mh_streams[i].public.txq, i);
 	}
 	/* setup constant portion of vap state */
 	/* XXX should get max ap/client vap's from f/w */
 	i = 0;
 	hvap = &mh->mh_vaps[i];
 	hvap->vap_type = MWL_HAL_AP;
 	hvap->bss_type = htole16(WL_MAC_TYPE_PRIMARY_AP);
 	hvap->macid = 0;
 	for (i++; i < MWL_MBSS_AP_MAX; i++) {
 		hvap = &mh->mh_vaps[i];
 		hvap->vap_type = MWL_HAL_AP;
 		hvap->bss_type = htole16(WL_MAC_TYPE_SECONDARY_AP);
 		hvap->macid = i;
 	}
 	hvap = &mh->mh_vaps[i];
 	hvap->vap_type = MWL_HAL_STA;
 	hvap->bss_type = htole16(WL_MAC_TYPE_PRIMARY_CLIENT);
 	hvap->macid = i;
 	for (i++; i < MWL_MBSS_MAX; i++) {
 		hvap = &mh->mh_vaps[i];
 		hvap->vap_type = MWL_HAL_STA;
 		hvap->bss_type = htole16(WL_MAC_TYPE_SECONDARY_CLIENT);
 		hvap->macid = i;
 	}
 	mh->mh_revs.mh_devid = devid;
 	snprintf(mh->mh_mtxname, sizeof(mh->mh_mtxname),
 	    "%s_hal", device_get_nameunit(dev));
 	mtx_init(&mh->mh_mtx, mh->mh_mtxname, NULL, MTX_DEF);
 
 	/*
 	 * Allocate the command buffer and map into the address
 	 * space of the h/w.  We request "coherent" memory which
 	 * will be uncached on some architectures.
 	 */
 	error = bus_dma_tag_create(tag,		/* parent */
 		       PAGE_SIZE, 0,		/* alignment, bounds */
 		       BUS_SPACE_MAXADDR_32BIT,	/* lowaddr */
 		       BUS_SPACE_MAXADDR,	/* highaddr */
 		       NULL, NULL,		/* filter, filterarg */
 		       MWL_CMDBUF_SIZE,		/* maxsize */
 		       1,			/* nsegments */
 		       MWL_CMDBUF_SIZE,		/* maxsegsize */
 		       BUS_DMA_ALLOCNOW,	/* flags */
 		       NULL,			/* lockfunc */
 		       NULL,			/* lockarg */
 		       &mh->mh_dmat);
 	if (error != 0) {
-		device_printf(dev, "unable to allocate memory for cmd buffer, "
+		device_printf(dev, "unable to allocate memory for cmd tag, "
 			"error %u\n", error);
 		goto fail0;
 	}
 
 	/* allocate descriptors */
-	error = bus_dmamap_create(mh->mh_dmat, BUS_DMA_NOWAIT, &mh->mh_dmamap);
-	if (error != 0) {
-		device_printf(dev, "unable to create dmamap for cmd buffers, "
-			"error %u\n", error);
-		goto fail0;
-	}
-
 	error = bus_dmamem_alloc(mh->mh_dmat, (void**) &mh->mh_cmdbuf,
 				 BUS_DMA_NOWAIT | BUS_DMA_COHERENT, 
 				 &mh->mh_dmamap);
 	if (error != 0) {
 		device_printf(dev, "unable to allocate memory for cmd buffer, "
 			"error %u\n", error);
 		goto fail1;
 	}
 
 	error = bus_dmamap_load(mh->mh_dmat, mh->mh_dmamap,
 				mh->mh_cmdbuf, MWL_CMDBUF_SIZE,
 				mwl_hal_load_cb, &mh->mh_cmdaddr,
 				BUS_DMA_NOWAIT);
 	if (error != 0) {
 		device_printf(dev, "unable to load cmd buffer, error %u\n",
 			error);
 		goto fail2;
 	}
 
 	/*
 	 * Some cards have SDRAM.  When loading firmware we need
 	 * to reset the SDRAM controller prior to doing this.
 	 * When the SDRAMSIZE is non-zero we do that work in
 	 * mwl_hal_fwload.
 	 */
 	switch (devid) {
 	case 0x2a02:		/* CB82 */
 	case 0x2a03:		/* CB85 */
 	case 0x2a08:		/* MC85_B1 */
 	case 0x2a0b:		/* CB85AP */
 	case 0x2a24:
 		mh->mh_SDRAMSIZE_Addr = 0x40fe70b7;	/* 8M SDRAM */
 		break;
 	case 0x2a04:		/* MC85 */
 		mh->mh_SDRAMSIZE_Addr = 0x40fc70b7;	/* 16M SDRAM */
 		break;
 	default:
 		break;
 	}
 	return &mh->public;
 fail2:
 	bus_dmamem_free(mh->mh_dmat, mh->mh_cmdbuf, mh->mh_dmamap);
 fail1:
-	bus_dmamap_destroy(mh->mh_dmat, mh->mh_dmamap);
-fail0:
 	bus_dma_tag_destroy(mh->mh_dmat);
+fail0:
 	mtx_destroy(&mh->mh_mtx);
 	free(mh, M_DEVBUF);
 	return NULL;
 }
 
 void
 mwl_hal_detach(struct mwl_hal *mh0)
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 
 	bus_dmamem_free(mh->mh_dmat, mh->mh_cmdbuf, mh->mh_dmamap);
-	bus_dmamap_destroy(mh->mh_dmat, mh->mh_dmamap);
 	bus_dma_tag_destroy(mh->mh_dmat);
 	mtx_destroy(&mh->mh_mtx);
 	free(mh, M_DEVBUF);
 }
 
 /*
  * Reset internal state after a firmware download.
  */
 static int
 mwlResetHalState(struct mwl_hal_priv *mh)
 {
 	int i;
 
 	/* XXX get from f/w */
 	mh->mh_bastreams = (1<<MWL_BASTREAMS_MAX)-1;
 	for (i = 0; i < MWL_MBSS_MAX; i++)
 		mh->mh_vaps[i].mh = NULL;
 	/*
 	 * Clear cumulative stats.
 	 */
 	mh->mh_RTSSuccesses = 0;
 	mh->mh_RTSFailures = 0;
 	mh->mh_RxDuplicateFrames = 0;
 	mh->mh_FCSErrorCount = 0;
 	/*
 	 * Fetch cal data for later use.
 	 * XXX may want to fetch other stuff too.
 	 */
 	/* XXX check return */
 	if ((mh->mh_flags & MHF_CALDATA) == 0)
 		mwlGetPwrCalTable(mh);
 	return 0;
 }
 
 struct mwl_hal_vap *
 mwl_hal_newvap(struct mwl_hal *mh0, MWL_HAL_BSSTYPE type,
 	const uint8_t mac[IEEE80211_ADDR_LEN])
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 	struct mwl_hal_vap *vap;
 	int i;
 
 	MWL_HAL_LOCK(mh);
 	/* NB: could optimize but not worth it w/ max 32 bss */
 	for (i = 0; i < MWL_MBSS_MAX; i++) {
 		vap = &mh->mh_vaps[i];
 		if (vap->vap_type == type && vap->mh == NULL) {
 			vap->mh = mh;
 			mwl_hal_setmac_locked(vap, mac);
 			break;
 		}
 	}
 	MWL_HAL_UNLOCK(mh);
 	return (i < MWL_MBSS_MAX) ? vap : NULL;
 }
 
 void
 mwl_hal_delvap(struct mwl_hal_vap *vap)
 {
 	/* NB: locking not needed for single write */
 	vap->mh = NULL;
 }
 
 /*
  * Manipulate the debug mask.  Note debug
  * msgs are only provided when this code is
  * compiled with MWLHAL_DEBUG defined.
  */
 
 void
 mwl_hal_setdebug(struct mwl_hal *mh, int debug)
 {
 	MWLPRIV(mh)->mh_debug = debug;
 }
 
 int
 mwl_hal_getdebug(struct mwl_hal *mh)
 {
 	return MWLPRIV(mh)->mh_debug;
 }
 
 void
 mwl_hal_setbastreams(struct mwl_hal *mh, int mask)
 {
 	MWLPRIV(mh)->mh_bastreams = mask & ((1<<MWL_BASTREAMS_MAX)-1);
 }
 
 int
 mwl_hal_getbastreams(struct mwl_hal *mh)
 {
 	return MWLPRIV(mh)->mh_bastreams;
 }
 
 int
 mwl_hal_ismbsscapable(struct mwl_hal *mh)
 {
 	return (MWLPRIV(mh)->mh_flags & MHF_MBSS) != 0;
 }
 
 #if 0
 /* XXX inlined */
 /*
  * Return the current ISR setting and clear the cause.
  * XXX maybe make inline
  */
 void
 mwl_hal_getisr(struct mwl_hal *mh0, uint32_t *status)
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 	uint32_t cause;
 
 	cause = RD4(mh, MACREG_REG_A2H_INTERRUPT_CAUSE);
 	if (cause == 0xffffffff) {	/* card removed */
 device_printf(mh->mh_dev, "%s: cause 0x%x\n", __func__, cause);
 		cause = 0;
 	} else if (cause != 0) {
 		/* clear cause bits */
 		WR4(mh, MACREG_REG_A2H_INTERRUPT_CAUSE,
 		    cause &~ mh->public.mh_imask);
 		RD4(mh, MACREG_REG_INT_CODE);	/* XXX flush write? */
 	}
 	*status = cause;
 }
 #endif
 
 /*
  * Set the interrupt mask.
  */
 void
 mwl_hal_intrset(struct mwl_hal *mh0, uint32_t mask)
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 
 	WR4(mh, MACREG_REG_A2H_INTERRUPT_MASK, 0);
 	RD4(mh, MACREG_REG_INT_CODE);
 
 	mh->public.mh_imask = mask;
 	WR4(mh, MACREG_REG_A2H_INTERRUPT_MASK, mask);
 	RD4(mh, MACREG_REG_INT_CODE);
 }
 
 #if 0
 /* XXX inlined */
 /*
  * Kick the firmware to tell it there are new tx descriptors
  * for processing.  The driver says what h/w q has work in
  * case the f/w ever gets smarter.
  */
 void
 mwl_hal_txstart(struct mwl_hal *mh0, int qnum)
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 	uint32_t dummy;
 
 	WR4(mh, MACREG_REG_H2A_INTERRUPT_EVENTS, MACREG_H2ARIC_BIT_PPA_READY);
 	dummy = RD4(mh, MACREG_REG_INT_CODE);
 }
 #endif
 
 /*
  * Callback from the driver on a cmd done interrupt.
  * Nothing to do right now as we spin waiting for
  * cmd completion.
  */
 void
 mwl_hal_cmddone(struct mwl_hal *mh0)
 {
 #if 0
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 
 	if (mh->mh_debug & MWL_HAL_DEBUG_CMDDONE) {
 		device_printf(mh->mh_dev, "cmd done interrupt:\n");
 		dumpresult(mh, 1);
 	}
 #endif
 }
 
 /*
  * Return "hw specs".  Note this must be the first
  * cmd MUST be done after a firmware download or the
  * f/w will lockup.
  * XXX move into the hal so driver doesn't need to be responsible
  */
 int
 mwl_hal_gethwspecs(struct mwl_hal *mh0, struct mwl_hal_hwspec *hw)
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 	HostCmd_DS_GET_HW_SPEC *pCmd;
 	int retval, minrev;
 
 	MWL_HAL_LOCK(mh);
 	_CMD_SETUP(pCmd, HostCmd_DS_GET_HW_SPEC, HostCmd_CMD_GET_HW_SPEC);
 	memset(&pCmd->PermanentAddr[0], 0xff, IEEE80211_ADDR_LEN);
 	pCmd->ulFwAwakeCookie = htole32((unsigned int)mh->mh_cmdaddr+2048);
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_GET_HW_SPEC);
 	if (retval == 0) {
 		IEEE80211_ADDR_COPY(hw->macAddr, pCmd->PermanentAddr);
 		hw->wcbBase[0] = le32toh(pCmd->WcbBase0) & 0x0000ffff;
 		hw->wcbBase[1] = le32toh(pCmd->WcbBase1[0]) & 0x0000ffff;
 		hw->wcbBase[2] = le32toh(pCmd->WcbBase1[1]) & 0x0000ffff;
 		hw->wcbBase[3] = le32toh(pCmd->WcbBase1[2]) & 0x0000ffff;
 		hw->rxDescRead = le32toh(pCmd->RxPdRdPtr)& 0x0000ffff;
 		hw->rxDescWrite = le32toh(pCmd->RxPdWrPtr)& 0x0000ffff;
 		hw->regionCode = le16toh(pCmd->RegionCode) & 0x00ff;
 		hw->fwReleaseNumber = le32toh(pCmd->FWReleaseNumber);
 		hw->maxNumWCB = le16toh(pCmd->NumOfWCB);
 		hw->maxNumMCAddr = le16toh(pCmd->NumOfMCastAddr);
 		hw->numAntennas = le16toh(pCmd->NumberOfAntenna);
 		hw->hwVersion = pCmd->Version;
 		hw->hostInterface = pCmd->HostIf;
 
 		mh->mh_revs.mh_macRev = hw->hwVersion;		/* XXX */
 		mh->mh_revs.mh_phyRev = hw->hostInterface;	/* XXX */
 
 		minrev = ((hw->fwReleaseNumber) >> 16) & 0xff;
 		if (minrev >= 4) {
 			/* starting with 3.4.x.x s/w BA streams supported */
 			mh->mh_bastreams &= (1<<MWL_BASTREAMS_MAX)-1;
 		} else
 			mh->mh_bastreams &= (1<<2)-1;
 	}
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 /*
  * Inform the f/w about location of the tx/rx dma data structures
  * and related state.  This cmd must be done immediately after a
  * mwl_hal_gethwspecs call or the f/w will lockup.
  */
 int
 mwl_hal_sethwdma(struct mwl_hal *mh0, const struct mwl_hal_txrxdma *dma)
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 	HostCmd_DS_SET_HW_SPEC *pCmd;
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	_CMD_SETUP(pCmd, HostCmd_DS_SET_HW_SPEC, HostCmd_CMD_SET_HW_SPEC);
 	pCmd->WcbBase[0] = htole32(dma->wcbBase[0]);
 	pCmd->WcbBase[1] = htole32(dma->wcbBase[1]);
 	pCmd->WcbBase[2] = htole32(dma->wcbBase[2]);
 	pCmd->WcbBase[3] = htole32(dma->wcbBase[3]);
 	pCmd->TxWcbNumPerQueue = htole32(dma->maxNumTxWcb);
 	pCmd->NumTxQueues = htole32(dma->maxNumWCB);
 	pCmd->TotalRxWcb = htole32(1);		/* XXX */
 	pCmd->RxPdWrPtr = htole32(dma->rxDescRead);
 	pCmd->Flags = htole32(SET_HW_SPEC_HOSTFORM_BEACON
 #ifdef MWL_HOST_PS_SUPPORT
 		    | SET_HW_SPEC_HOST_POWERSAVE
 #endif
 		    | SET_HW_SPEC_HOSTFORM_PROBERESP);
 	/* disable multi-bss operation for A1-A4 parts */
 	if (mh->mh_revs.mh_macRev < 5)
 		pCmd->Flags |= htole32(SET_HW_SPEC_DISABLEMBSS);
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_SET_HW_SPEC);
 	if (retval == 0) {
 		if (pCmd->Flags & htole32(SET_HW_SPEC_DISABLEMBSS))
 			mh->mh_flags &= ~MHF_MBSS;
 		else
 			mh->mh_flags |= MHF_MBSS;
 	}
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 /*
  * Retrieve statistics from the f/w.
  * XXX should be in memory shared w/ driver
  */
 int
 mwl_hal_gethwstats(struct mwl_hal *mh0, struct mwl_hal_hwstats *stats)
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 	HostCmd_DS_802_11_GET_STAT *pCmd;
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	_CMD_SETUP(pCmd, HostCmd_DS_802_11_GET_STAT,
 		HostCmd_CMD_802_11_GET_STAT);
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_802_11_GET_STAT);
 	if (retval == 0) {
 		const uint32_t *sp = (const uint32_t *)&pCmd->TxRetrySuccesses;
 		uint32_t *dp = (uint32_t *)&stats->TxRetrySuccesses;
 		int i;
 
 		for (i = 0; i < sizeof(*stats)/sizeof(uint32_t); i++)
 			dp[i] = le32toh(sp[i]);
 		/*
 		 * Update stats not returned by f/w but available
 		 * through public registers.  Note these registers
 		 * are "clear on read" so we maintain cumulative data.
 		 * XXX register defines
 		 */
 		mh->mh_RTSSuccesses += RD4(mh, 0xa834);
 		mh->mh_RTSFailures += RD4(mh, 0xa830);
 		mh->mh_RxDuplicateFrames += RD4(mh, 0xa84c);
 		mh->mh_FCSErrorCount += RD4(mh, 0xa840);
 	}
 	MWL_HAL_UNLOCK(mh);
 
 	stats->RTSSuccesses = mh->mh_RTSSuccesses;
 	stats->RTSFailures = mh->mh_RTSFailures;
 	stats->RxDuplicateFrames = mh->mh_RxDuplicateFrames;
 	stats->FCSErrorCount = mh->mh_FCSErrorCount;
 	return retval;
 }
 
 /*
  * Set HT guard interval handling.
  * Takes effect immediately.
  */
 int
 mwl_hal_sethtgi(struct mwl_hal_vap *vap, int GIType)
 {
 	struct mwl_hal_priv *mh = MWLVAP(vap);
 	HostCmd_FW_HT_GUARD_INTERVAL *pCmd;
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	_VCMD_SETUP(vap, pCmd, HostCmd_FW_HT_GUARD_INTERVAL,
 		HostCmd_CMD_HT_GUARD_INTERVAL);
 	pCmd->Action = htole32(HostCmd_ACT_GEN_SET);
 
 	if (GIType == 0) {
 		pCmd->GIType = htole32(GI_TYPE_LONG);
 	} else if (GIType == 1) {
 		pCmd->GIType = htole32(GI_TYPE_LONG | GI_TYPE_SHORT);
 	} else {
 		pCmd->GIType = htole32(GI_TYPE_LONG);
 	}
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_HT_GUARD_INTERVAL);
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 /*
  * Configure radio.
  * Takes effect immediately.
  * XXX preamble installed after set fixed rate cmd
  */
 int
 mwl_hal_setradio(struct mwl_hal *mh0, int onoff, MWL_HAL_PREAMBLE preamble)
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 	HostCmd_DS_802_11_RADIO_CONTROL *pCmd;
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	_CMD_SETUP(pCmd, HostCmd_DS_802_11_RADIO_CONTROL,
 		HostCmd_CMD_802_11_RADIO_CONTROL);
 	pCmd->Action = htole16(HostCmd_ACT_GEN_SET);
 	if (onoff == 0)
 		pCmd->Control = 0;
 	else
 		pCmd->Control = htole16(preamble);
 	pCmd->RadioOn = htole16(onoff);
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_802_11_RADIO_CONTROL);
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 /*
  * Configure antenna use.
  * Takes effect immediately.
  * XXX tx antenna setting ignored
  * XXX rx antenna setting should always be 3 (for now)
  */
 int
 mwl_hal_setantenna(struct mwl_hal *mh0, MWL_HAL_ANTENNA dirSet, int ant)
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 	HostCmd_DS_802_11_RF_ANTENNA *pCmd;
 	int retval;
 
 	if (!(dirSet == WL_ANTENNATYPE_RX || dirSet == WL_ANTENNATYPE_TX))
 		return EINVAL;
 
 	MWL_HAL_LOCK(mh);
 	_CMD_SETUP(pCmd, HostCmd_DS_802_11_RF_ANTENNA,
 		HostCmd_CMD_802_11_RF_ANTENNA);
 	pCmd->Action = htole16(dirSet);
 	if (ant == 0)			/* default to all/both antennae */
 		ant = 3;
 	pCmd->AntennaMode = htole16(ant);
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_802_11_RF_ANTENNA);
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 /*
  * Set packet size threshold for implicit use of RTS.
  * Takes effect immediately.
  * XXX packet length > threshold =>'s RTS
  */
 int
 mwl_hal_setrtsthreshold(struct mwl_hal_vap *vap, int threshold)
 {
 	struct mwl_hal_priv *mh = MWLVAP(vap);
 	HostCmd_DS_802_11_RTS_THSD *pCmd;
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	_VCMD_SETUP(vap, pCmd, HostCmd_DS_802_11_RTS_THSD,
 		HostCmd_CMD_802_11_RTS_THSD);
 	pCmd->Action  = htole16(HostCmd_ACT_GEN_SET);
 	pCmd->Threshold = htole16(threshold);
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_802_11_RTS_THSD);
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 /*
  * Enable sta-mode operation (disables beacon frame xmit).
  */
 int
 mwl_hal_setinframode(struct mwl_hal_vap *vap)
 {
 	struct mwl_hal_priv *mh = MWLVAP(vap);
 	HostCmd_FW_SET_INFRA_MODE *pCmd;
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	_VCMD_SETUP(vap, pCmd, HostCmd_FW_SET_INFRA_MODE,
 		HostCmd_CMD_SET_INFRA_MODE);
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_SET_INFRA_MODE);
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 /*
  * Configure radar detection in support of 802.11h.
  */
 int
 mwl_hal_setradardetection(struct mwl_hal *mh0, MWL_HAL_RADAR action)
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 	HostCmd_802_11h_Detect_Radar *pCmd;
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	_CMD_SETUP(pCmd, HostCmd_802_11h_Detect_Radar,
 		HostCmd_CMD_802_11H_DETECT_RADAR);
 	pCmd->CmdHdr.Length = htole16(sizeof(HostCmd_802_11h_Detect_Radar));
 	pCmd->Action = htole16(action);
 	if (mh->mh_regioncode == DOMAIN_CODE_ETSI_131)
 		pCmd->RadarTypeCode = htole16(131);
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_802_11H_DETECT_RADAR);
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 } 
 
 /*
  * Convert public channel flags definition to a
  * value suitable for feeding to the firmware.
  * Note this includes byte swapping.
  */
 static uint32_t
 cvtChannelFlags(const MWL_HAL_CHANNEL *chan)
 {
 	uint32_t w;
 
 	/*
 	 * NB: f/w only understands FREQ_BAND_5GHZ, supplying the more
 	 *     precise band info causes it to lockup (sometimes).
 	 */
 	w = (chan->channelFlags.FreqBand == MWL_FREQ_BAND_2DOT4GHZ) ?
 		FREQ_BAND_2DOT4GHZ : FREQ_BAND_5GHZ;
 	switch (chan->channelFlags.ChnlWidth) {
 	case MWL_CH_10_MHz_WIDTH:
 		w |= CH_10_MHz_WIDTH;
 		break;
 	case MWL_CH_20_MHz_WIDTH:
 		w |= CH_20_MHz_WIDTH;
 		break;
 	case MWL_CH_40_MHz_WIDTH:
 	default:
 		w |= CH_40_MHz_WIDTH;
 		break;
 	}
 	switch (chan->channelFlags.ExtChnlOffset) {
 	case MWL_EXT_CH_NONE:
 		w |= EXT_CH_NONE;
 		break;
 	case MWL_EXT_CH_ABOVE_CTRL_CH:
 		w |= EXT_CH_ABOVE_CTRL_CH;
 		break;
 	case MWL_EXT_CH_BELOW_CTRL_CH:
 		w |= EXT_CH_BELOW_CTRL_CH;
 		break;
 	}
 	return htole32(w);
 }
 
 /*
  * Start a channel switch announcement countdown.  The IE
  * in the beacon frame is allowed to go out and the firmware
  * counts down and notifies the host when it's time to switch
  * channels.
  */
 int
 mwl_hal_setchannelswitchie(struct mwl_hal *mh0,
 	const MWL_HAL_CHANNEL *nextchan, uint32_t mode, uint32_t count)
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 	HostCmd_SET_SWITCH_CHANNEL *pCmd;
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	_CMD_SETUP(pCmd, HostCmd_SET_SWITCH_CHANNEL,
 		HostCmd_CMD_SET_SWITCH_CHANNEL);
 	pCmd->Next11hChannel = htole32(nextchan->channel);
 	pCmd->Mode = htole32(mode);
 	pCmd->InitialCount = htole32(count+1);
 	pCmd->ChannelFlags = cvtChannelFlags(nextchan);
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_SET_SWITCH_CHANNEL);
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 /*
  * Set the region code that selects the radar bin'ing agorithm.
  */
 int
 mwl_hal_setregioncode(struct mwl_hal *mh0, int regionCode)
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 	HostCmd_SET_REGIONCODE_INFO *pCmd;
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	_CMD_SETUP(pCmd, HostCmd_SET_REGIONCODE_INFO,
 		HostCmd_CMD_SET_REGION_CODE);
 	/* XXX map pseudo-codes to fw codes */
 	switch (regionCode) {
 	case DOMAIN_CODE_ETSI_131:
 		pCmd->regionCode = htole16(DOMAIN_CODE_ETSI);
 		break;
 	default:
 		pCmd->regionCode = htole16(regionCode);
 		break;
 	}
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_SET_REGION_CODE);
 	if (retval == 0)
 		mh->mh_regioncode = regionCode;
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 #define	RATEVAL(r)	((r) &~ RATE_MCS)
 #define	RATETYPE(r)	(((r) & RATE_MCS) ? HT_RATE_TYPE : LEGACY_RATE_TYPE)
 
 int
 mwl_hal_settxrate(struct mwl_hal_vap *vap, MWL_HAL_TXRATE_HANDLING handling,
 	const MWL_HAL_TXRATE *rate)
 {
 	struct mwl_hal_priv *mh = MWLVAP(vap);
 	HostCmd_FW_USE_FIXED_RATE *pCmd;
 	FIXED_RATE_ENTRY *fp;
 	int retval, i, n;
 
 	MWL_HAL_LOCK(mh);
 	_VCMD_SETUP(vap, pCmd, HostCmd_FW_USE_FIXED_RATE,
 		HostCmd_CMD_SET_FIXED_RATE);
 
 	pCmd->MulticastRate = RATEVAL(rate->McastRate);
 	pCmd->MultiRateTxType = RATETYPE(rate->McastRate);
 	/* NB: no rate type field */
 	pCmd->ManagementRate = RATEVAL(rate->MgtRate);
 	memset(pCmd->FixedRateTable, 0, sizeof(pCmd->FixedRateTable));
 	if (handling == RATE_FIXED) {
 		pCmd->Action = htole32(HostCmd_ACT_GEN_SET);
 		pCmd->AllowRateDrop = htole32(FIXED_RATE_WITHOUT_AUTORATE_DROP);
 		fp = pCmd->FixedRateTable;
 		fp->FixedRate =
 		    htole32(RATEVAL(rate->RateSeries[0].Rate));
 		fp->FixRateTypeFlags.FixRateType =
 		    htole32(RATETYPE(rate->RateSeries[0].Rate));
 		pCmd->EntryCount = htole32(1);
 	} else if (handling == RATE_FIXED_DROP) {
 		pCmd->Action = htole32(HostCmd_ACT_GEN_SET);
 		pCmd->AllowRateDrop = htole32(FIXED_RATE_WITH_AUTO_RATE_DROP);
 		n = 0;
 		fp = pCmd->FixedRateTable;
 		for (i = 0; i < 4; i++) {
 			if (rate->RateSeries[0].TryCount == 0)
 				break;
 			fp->FixRateTypeFlags.FixRateType =
 			    htole32(RATETYPE(rate->RateSeries[i].Rate));
 			fp->FixedRate =
 			    htole32(RATEVAL(rate->RateSeries[i].Rate));
 			fp->FixRateTypeFlags.RetryCountValid =
 			    htole32(RETRY_COUNT_VALID);
 			fp->RetryCount =
 			    htole32(rate->RateSeries[i].TryCount-1);
 			n++;
 		}
 		pCmd->EntryCount = htole32(n);
 	} else 
 		pCmd->Action = htole32(HostCmd_ACT_NOT_USE_FIXED_RATE);
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_SET_FIXED_RATE);
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 int
 mwl_hal_settxrate_auto(struct mwl_hal *mh0, const MWL_HAL_TXRATE *rate)
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 	HostCmd_FW_USE_FIXED_RATE *pCmd;
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	_CMD_SETUP(pCmd, HostCmd_FW_USE_FIXED_RATE,
 		HostCmd_CMD_SET_FIXED_RATE);
 
 	pCmd->MulticastRate = RATEVAL(rate->McastRate);
 	pCmd->MultiRateTxType = RATETYPE(rate->McastRate);
 	/* NB: no rate type field */
 	pCmd->ManagementRate = RATEVAL(rate->MgtRate);
 	memset(pCmd->FixedRateTable, 0, sizeof(pCmd->FixedRateTable));
 	pCmd->Action = htole32(HostCmd_ACT_NOT_USE_FIXED_RATE);
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_SET_FIXED_RATE);
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 #undef RATEVAL
 #undef RATETYPE
 
 int
 mwl_hal_setslottime(struct mwl_hal *mh0, int usecs)
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 	HostCmd_FW_SET_SLOT *pCmd;
 	int retval;
 
 	if (usecs != 9 && usecs != 20)
 		return EINVAL;
 
 	MWL_HAL_LOCK(mh);
 	_CMD_SETUP(pCmd, HostCmd_FW_SET_SLOT,
 	    HostCmd_CMD_802_11_SET_SLOT);
 	pCmd->Action = htole16(HostCmd_ACT_GEN_SET);
 	pCmd->Slot = (usecs == 9 ? 1 : 0);
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_802_11_SET_SLOT);
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 int
 mwl_hal_adjusttxpower(struct mwl_hal *mh0, uint32_t level)
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 	HostCmd_DS_802_11_RF_TX_POWER *pCmd;
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	_CMD_SETUP(pCmd, HostCmd_DS_802_11_RF_TX_POWER,
 	    HostCmd_CMD_802_11_RF_TX_POWER);
 	pCmd->Action = htole16(HostCmd_ACT_GEN_SET);
 
 	if (level < 30) {
 		pCmd->SupportTxPowerLevel = htole16(WL_TX_POWERLEVEL_LOW);
 	} else if (level >= 30 && level < 60) {
 		pCmd->SupportTxPowerLevel = htole16(WL_TX_POWERLEVEL_MEDIUM);
 	} else {
 		pCmd->SupportTxPowerLevel = htole16(WL_TX_POWERLEVEL_HIGH);
 	}
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_802_11_RF_TX_POWER);
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 static const struct mwl_hal_channel *
 findchannel(const struct mwl_hal_priv *mh, const MWL_HAL_CHANNEL *c)
 {
 	const struct mwl_hal_channel *hc;
 	const MWL_HAL_CHANNELINFO *ci;
 	int chan = c->channel, i;
 
 	if (c->channelFlags.FreqBand == MWL_FREQ_BAND_2DOT4GHZ) {
 		i = chan - 1;
 		if (c->channelFlags.ChnlWidth == MWL_CH_40_MHz_WIDTH) {
 			ci = &mh->mh_40M;
 			if (c->channelFlags.ExtChnlOffset == MWL_EXT_CH_BELOW_CTRL_CH)
 				i -= 4;
 		} else
 			ci = &mh->mh_20M;
 		/* 2.4G channel table is directly indexed */
 		hc = ((unsigned)i < ci->nchannels) ? &ci->channels[i] : NULL;
 	} else if (c->channelFlags.FreqBand == MWL_FREQ_BAND_5GHZ) {
 		if (c->channelFlags.ChnlWidth == MWL_CH_40_MHz_WIDTH) {
 			ci = &mh->mh_40M_5G;
 			if (c->channelFlags.ExtChnlOffset == MWL_EXT_CH_BELOW_CTRL_CH)
 				chan -= 4;
 		} else
 			ci = &mh->mh_20M_5G;
 		/* 5GHz channel table is sparse and must be searched */
 		for (i = 0; i < ci->nchannels; i++)
 			if (ci->channels[i].ieee == chan)
 				break;
 		hc = (i < ci->nchannels) ? &ci->channels[i] : NULL;
 	} else
 		hc = NULL;
 	return hc;
 }
 
 int
 mwl_hal_settxpower(struct mwl_hal *mh0, const MWL_HAL_CHANNEL *c, uint8_t maxtxpow)
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 	HostCmd_DS_802_11_RF_TX_POWER *pCmd;
 	const struct mwl_hal_channel *hc;
 	int i, retval;
 
 	hc = findchannel(mh, c);
 	if (hc == NULL) {
 		/* XXX temp while testing */
 		device_printf(mh->mh_dev,
 		    "%s: no cal data for channel %u band %u width %u ext %u\n",
 		    __func__, c->channel, c->channelFlags.FreqBand,
 		    c->channelFlags.ChnlWidth, c->channelFlags.ExtChnlOffset);
 		return EINVAL;
 	}
 
 	MWL_HAL_LOCK(mh);
 	_CMD_SETUP(pCmd, HostCmd_DS_802_11_RF_TX_POWER,
 	    HostCmd_CMD_802_11_RF_TX_POWER);
 	pCmd->Action = htole16(HostCmd_ACT_GEN_SET_LIST);
 	i = 0;
 	/* NB: 5Ghz cal data have the channel # in [0]; don't truncate */
 	if (c->channelFlags.FreqBand == MWL_FREQ_BAND_5GHZ)
 		pCmd->PowerLevelList[i++] = htole16(hc->targetPowers[0]);
 	for (; i < 4; i++) {
 		uint16_t pow = hc->targetPowers[i];
 		if (pow > maxtxpow)
 			pow = maxtxpow;
 		pCmd->PowerLevelList[i] = htole16(pow);
 	}
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_802_11_RF_TX_POWER);
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 int
 mwl_hal_getchannelinfo(struct mwl_hal *mh0, int band, int chw,
 	const MWL_HAL_CHANNELINFO **ci)
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 
 	switch (band) {
 	case MWL_FREQ_BAND_2DOT4GHZ:
 		*ci = (chw == MWL_CH_20_MHz_WIDTH) ? &mh->mh_20M : &mh->mh_40M;
 		break;
 	case MWL_FREQ_BAND_5GHZ:
 		*ci = (chw == MWL_CH_20_MHz_WIDTH) ?
 		     &mh->mh_20M_5G : &mh->mh_40M_5G;
 		break;
 	default:
 		return EINVAL;
 	}
 	return ((*ci)->freqLow == (*ci)->freqHigh) ? EINVAL : 0;
 }
 
 int
 mwl_hal_setmcast(struct mwl_hal *mh0, int nmc, const uint8_t macs[])
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 	HostCmd_DS_MAC_MULTICAST_ADR *pCmd;
 	int retval;
 
 	if (nmc > MWL_HAL_MCAST_MAX)
 		return EINVAL;
 
 	MWL_HAL_LOCK(mh);
 	_CMD_SETUP(pCmd, HostCmd_DS_MAC_MULTICAST_ADR,
 		HostCmd_CMD_MAC_MULTICAST_ADR);
 	memcpy(pCmd->MACList, macs, nmc*IEEE80211_ADDR_LEN);
 	pCmd->NumOfAdrs = htole16(nmc);
 	pCmd->Action = htole16(0xffff);
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_MAC_MULTICAST_ADR);
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 int
 mwl_hal_keyset(struct mwl_hal_vap *vap, const MWL_HAL_KEYVAL *kv,
 	const uint8_t mac[IEEE80211_ADDR_LEN])
 {
 	struct mwl_hal_priv *mh = MWLVAP(vap);
 	HostCmd_FW_UPDATE_ENCRYPTION_SET_KEY *pCmd;
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	_VCMD_SETUP(vap, pCmd, HostCmd_FW_UPDATE_ENCRYPTION_SET_KEY,
 		HostCmd_CMD_UPDATE_ENCRYPTION);
 	if (kv->keyFlags & (KEY_FLAG_TXGROUPKEY|KEY_FLAG_RXGROUPKEY))
 		pCmd->ActionType = htole32(EncrActionTypeSetGroupKey);
 	else
 		pCmd->ActionType = htole32(EncrActionTypeSetKey);
 	pCmd->KeyParam.Length = htole16(sizeof(pCmd->KeyParam));
 	pCmd->KeyParam.KeyTypeId = htole16(kv->keyTypeId);
 	pCmd->KeyParam.KeyInfo = htole32(kv->keyFlags);
 	pCmd->KeyParam.KeyIndex = htole32(kv->keyIndex);
 	/* NB: includes TKIP MIC keys */
 	memcpy(&pCmd->KeyParam.Key, &kv->key, kv->keyLen);
 	switch (kv->keyTypeId) {
 	case KEY_TYPE_ID_WEP:
 		pCmd->KeyParam.KeyLen = htole16(kv->keyLen);
 		break;
 	case KEY_TYPE_ID_TKIP:
 		pCmd->KeyParam.KeyLen = htole16(sizeof(TKIP_TYPE_KEY));
 		pCmd->KeyParam.Key.TkipKey.TkipRsc.low =
 			htole16(kv->key.tkip.rsc.low);
 		pCmd->KeyParam.Key.TkipKey.TkipRsc.high =
 			htole32(kv->key.tkip.rsc.high);
 		pCmd->KeyParam.Key.TkipKey.TkipTsc.low =
 			htole16(kv->key.tkip.tsc.low);
 		pCmd->KeyParam.Key.TkipKey.TkipTsc.high =
 			htole32(kv->key.tkip.tsc.high);
 		break;
 	case KEY_TYPE_ID_AES:
 		pCmd->KeyParam.KeyLen = htole16(sizeof(AES_TYPE_KEY));
 		break;
 	}
 #ifdef MWL_MBSS_SUPPORT
 	IEEE80211_ADDR_COPY(pCmd->KeyParam.Macaddr, mac);
 #else
 	IEEE80211_ADDR_COPY(pCmd->Macaddr, mac);
 #endif
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_UPDATE_ENCRYPTION);
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 int
 mwl_hal_keyreset(struct mwl_hal_vap *vap, const MWL_HAL_KEYVAL *kv, const uint8_t mac[IEEE80211_ADDR_LEN])
 {
 	struct mwl_hal_priv *mh = MWLVAP(vap);
 	HostCmd_FW_UPDATE_ENCRYPTION_SET_KEY *pCmd;
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	_VCMD_SETUP(vap, pCmd, HostCmd_FW_UPDATE_ENCRYPTION_SET_KEY,
 		HostCmd_CMD_UPDATE_ENCRYPTION);
 	pCmd->ActionType = htole16(EncrActionTypeRemoveKey);
 	pCmd->KeyParam.Length = htole16(sizeof(pCmd->KeyParam));
 	pCmd->KeyParam.KeyTypeId = htole16(kv->keyTypeId);
 	pCmd->KeyParam.KeyInfo = htole32(kv->keyFlags);
 	pCmd->KeyParam.KeyIndex = htole32(kv->keyIndex);
 #ifdef MWL_MBSS_SUPPORT
 	IEEE80211_ADDR_COPY(pCmd->KeyParam.Macaddr, mac);
 #else
 	IEEE80211_ADDR_COPY(pCmd->Macaddr, mac);
 #endif
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_UPDATE_ENCRYPTION);
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 static int
 mwl_hal_setmac_locked(struct mwl_hal_vap *vap,
 	const uint8_t addr[IEEE80211_ADDR_LEN])
 {
 	struct mwl_hal_priv *mh = MWLVAP(vap);
 	HostCmd_DS_SET_MAC *pCmd;
 
 	_VCMD_SETUP(vap, pCmd, HostCmd_DS_SET_MAC, HostCmd_CMD_SET_MAC_ADDR);
 	IEEE80211_ADDR_COPY(&pCmd->MacAddr[0], addr);
 #ifdef MWL_MBSS_SUPPORT
 	pCmd->MacType = vap->bss_type;		/* NB: already byte swapped */
 	IEEE80211_ADDR_COPY(vap->mac, addr);	/* XXX do only if success */
 #endif
 	return mwlExecuteCmd(mh, HostCmd_CMD_SET_MAC_ADDR);
 }
 
 int
 mwl_hal_setmac(struct mwl_hal_vap *vap, const uint8_t addr[IEEE80211_ADDR_LEN])
 {
 	struct mwl_hal_priv *mh = MWLVAP(vap);
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	retval = mwl_hal_setmac_locked(vap, addr);
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 int
 mwl_hal_setbeacon(struct mwl_hal_vap *vap, const void *frame, size_t frameLen)
 {
 	struct mwl_hal_priv *mh = MWLVAP(vap);
 	HostCmd_DS_SET_BEACON *pCmd;
 	int retval;
 
 	/* XXX verify frameLen fits */
 	MWL_HAL_LOCK(mh);
 	_VCMD_SETUP(vap, pCmd, HostCmd_DS_SET_BEACON, HostCmd_CMD_SET_BEACON);
 	/* XXX override _VCMD_SETUP */
 	pCmd->CmdHdr.Length = htole16(sizeof(HostCmd_DS_SET_BEACON)-1+frameLen);
 	pCmd->FrmBodyLen = htole16(frameLen);
 	memcpy(pCmd->FrmBody, frame, frameLen);
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_SET_BEACON);
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 int
 mwl_hal_setpowersave_bss(struct mwl_hal_vap *vap, uint8_t nsta)
 {
 	struct mwl_hal_priv *mh = MWLVAP(vap);
 	HostCmd_SET_POWERSAVESTATION *pCmd;
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	_VCMD_SETUP(vap, pCmd, HostCmd_SET_POWERSAVESTATION,
 		HostCmd_CMD_SET_POWERSAVESTATION);
 	pCmd->NumberOfPowersave = nsta;
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_SET_POWERSAVESTATION);
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 int
 mwl_hal_setpowersave_sta(struct mwl_hal_vap *vap, uint16_t aid, int ena)
 {
 	struct mwl_hal_priv *mh = MWLVAP(vap);
 	HostCmd_SET_TIM *pCmd;
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	_VCMD_SETUP(vap, pCmd, HostCmd_SET_TIM, HostCmd_CMD_SET_TIM);
 	pCmd->Aid = htole16(aid);
 	pCmd->Set = htole32(ena);
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_SET_TIM);
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 int
 mwl_hal_setassocid(struct mwl_hal_vap *vap,
 	const uint8_t bssId[IEEE80211_ADDR_LEN], uint16_t assocId)
 {
 	struct mwl_hal_priv *mh = MWLVAP(vap);
 	HostCmd_FW_SET_AID *pCmd = (HostCmd_FW_SET_AID *) &mh->mh_cmdbuf[0];
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	_VCMD_SETUP(vap, pCmd, HostCmd_FW_SET_AID, HostCmd_CMD_SET_AID);
 	pCmd->AssocID = htole16(assocId);
 	IEEE80211_ADDR_COPY(&pCmd->MacAddr[0], bssId);
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_SET_AID);
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 int
 mwl_hal_setchannel(struct mwl_hal *mh0, const MWL_HAL_CHANNEL *chan)
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 	HostCmd_FW_SET_RF_CHANNEL *pCmd;
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	_CMD_SETUP(pCmd, HostCmd_FW_SET_RF_CHANNEL, HostCmd_CMD_SET_RF_CHANNEL);
 	pCmd->Action = htole16(HostCmd_ACT_GEN_SET);
 	pCmd->CurrentChannel = chan->channel;
 	pCmd->ChannelFlags = cvtChannelFlags(chan);	/* NB: byte-swapped */
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_SET_RF_CHANNEL);
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 static int
 bastream_check_available(struct mwl_hal_vap *vap, int qid,
 	const uint8_t Macaddr[IEEE80211_ADDR_LEN],
 	uint8_t Tid, uint8_t ParamInfo)
 {
 	struct mwl_hal_priv *mh = MWLVAP(vap);
 	HostCmd_FW_BASTREAM *pCmd;
 	int retval;
 
 	MWL_HAL_LOCK_ASSERT(mh);
 
 	_VCMD_SETUP(vap, pCmd, HostCmd_FW_BASTREAM, HostCmd_CMD_BASTREAM);
 	pCmd->ActionType = htole32(BaCheckCreateStream);
 	pCmd->BaInfo.CreateParams.BarThrs = htole32(63);
 	pCmd->BaInfo.CreateParams.WindowSize = htole32(64); 
 	pCmd->BaInfo.CreateParams.IdleThrs = htole32(0x22000);
 	IEEE80211_ADDR_COPY(&pCmd->BaInfo.CreateParams.PeerMacAddr[0], Macaddr);
 	pCmd->BaInfo.CreateParams.DialogToken = 10;
 	pCmd->BaInfo.CreateParams.Tid = Tid;
 	pCmd->BaInfo.CreateParams.QueueId = qid;
 	pCmd->BaInfo.CreateParams.ParamInfo = (uint8_t) ParamInfo;
 #if 0
 	cvtBAFlags(&pCmd->BaInfo.CreateParams.Flags, sp->ba_policy, 0);
 #else
 	pCmd->BaInfo.CreateParams.Flags =
 			  htole32(BASTREAM_FLAG_IMMEDIATE_TYPE)
 			| htole32(BASTREAM_FLAG_DIRECTION_UPSTREAM)
 			;
 #endif
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_BASTREAM);
 	if (retval == 0) {
 		/*
 		 * NB: BA stream create may fail when the stream is
 		 * h/w backed under some (as yet not understood) conditions.
 		 * Check the result code to catch this.
 		 */
 		if (le16toh(pCmd->CmdHdr.Result) != HostCmd_RESULT_OK)
 			retval = EIO;
 	}
 	return retval;
 }
 
 const MWL_HAL_BASTREAM *
 mwl_hal_bastream_alloc(struct mwl_hal_vap *vap, int ba_policy,
 	const uint8_t Macaddr[IEEE80211_ADDR_LEN],
 	uint8_t Tid, uint8_t ParamInfo, void *a1, void *a2)
 {
 	struct mwl_hal_priv *mh = MWLVAP(vap);
 	struct mwl_hal_bastream *sp;
 	int s;
 
 	MWL_HAL_LOCK(mh);
 	if (mh->mh_bastreams == 0) {
 		/* no streams available */
 		MWL_HAL_UNLOCK(mh);
 		return NULL;
 	}
 	for (s = 0; (mh->mh_bastreams & (1<<s)) == 0; s++)
 		;
 	if (bastream_check_available(vap, s, Macaddr, Tid, ParamInfo)) {
 		MWL_HAL_UNLOCK(mh);
 		return NULL;
 	}
 	sp = &mh->mh_streams[s];
 	mh->mh_bastreams &= ~(1<<s);
 	sp->public.data[0] = a1;
 	sp->public.data[1] = a2;
 	IEEE80211_ADDR_COPY(sp->macaddr, Macaddr);
 	sp->tid = Tid;
 	sp->paraminfo = ParamInfo;
 	sp->setup = 0;
 	sp->ba_policy = ba_policy;
 	MWL_HAL_UNLOCK(mh);
 	return sp != NULL ? &sp->public : NULL;
 }
 
 const MWL_HAL_BASTREAM *
 mwl_hal_bastream_lookup(struct mwl_hal *mh0, int s)
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 
 	if (!(0 <= s && s < MWL_BASTREAMS_MAX))
 		return NULL;
 	if (mh->mh_bastreams & (1<<s))
 		return NULL;
 	return &mh->mh_streams[s].public;
 }
 
 #ifndef	__DECONST
 #define	__DECONST(type, var)	((type)(uintptr_t)(const void *)(var))
 #endif
 
 int
 mwl_hal_bastream_create(struct mwl_hal_vap *vap,
 	const MWL_HAL_BASTREAM *s, int BarThrs, int WindowSize, uint16_t seqno)
 {
 	struct mwl_hal_priv *mh = MWLVAP(vap);
 	struct mwl_hal_bastream *sp = __DECONST(struct mwl_hal_bastream *, s);
 	HostCmd_FW_BASTREAM *pCmd;
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	_VCMD_SETUP(vap, pCmd, HostCmd_FW_BASTREAM, HostCmd_CMD_BASTREAM);
 	pCmd->ActionType = htole32(BaCreateStream);
 	pCmd->BaInfo.CreateParams.BarThrs = htole32(BarThrs);
 	pCmd->BaInfo.CreateParams.WindowSize = htole32(WindowSize);
 	pCmd->BaInfo.CreateParams.IdleThrs = htole32(0x22000);
 	IEEE80211_ADDR_COPY(&pCmd->BaInfo.CreateParams.PeerMacAddr[0],
 	    sp->macaddr);
 	/* XXX proxy STA */
 	memset(&pCmd->BaInfo.CreateParams.StaSrcMacAddr, 0, IEEE80211_ADDR_LEN);
 #if 0
 	pCmd->BaInfo.CreateParams.DialogToken = DialogToken;
 #else
 	pCmd->BaInfo.CreateParams.DialogToken = 10;
 #endif
 	pCmd->BaInfo.CreateParams.Tid = sp->tid;
 	pCmd->BaInfo.CreateParams.QueueId = sp->stream;
 	pCmd->BaInfo.CreateParams.ParamInfo = sp->paraminfo;
 	/* NB: ResetSeqNo known to be zero */
 	pCmd->BaInfo.CreateParams.StartSeqNo = htole16(seqno);
 #if 0
 	cvtBAFlags(&pCmd->BaInfo.CreateParams.Flags, sp->ba_policy, 0);
 #else
 	pCmd->BaInfo.CreateParams.Flags =
 			  htole32(BASTREAM_FLAG_IMMEDIATE_TYPE)
 			| htole32(BASTREAM_FLAG_DIRECTION_UPSTREAM)
 			;
 #endif
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_BASTREAM);
 	if (retval == 0) {
 		/*
 		 * NB: BA stream create may fail when the stream is
 		 * h/w backed under some (as yet not understood) conditions.
 		 * Check the result code to catch this.
 		 */
 		if (le16toh(pCmd->CmdHdr.Result) != HostCmd_RESULT_OK)
 			retval = EIO;
 		else
 			sp->setup = 1;
 	}
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 int
 mwl_hal_bastream_destroy(struct mwl_hal *mh0, const MWL_HAL_BASTREAM *s)
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 	struct mwl_hal_bastream *sp = __DECONST(struct mwl_hal_bastream *, s);
 	HostCmd_FW_BASTREAM *pCmd;
 	int retval;
 
 	if (sp->stream >= MWL_BASTREAMS_MAX) {
 		/* XXX */
 		return EINVAL;
 	}
 	MWL_HAL_LOCK(mh);
 	if (sp->setup) {
 		_CMD_SETUP(pCmd, HostCmd_FW_BASTREAM, HostCmd_CMD_BASTREAM);
 		pCmd->ActionType = htole32(BaDestroyStream);
 		pCmd->BaInfo.DestroyParams.FwBaContext.Context =
 		    htole32(sp->stream);
 
 		retval = mwlExecuteCmd(mh, HostCmd_CMD_BASTREAM);
 	} else
 		retval = 0;
 	/* NB: always reclaim stream */
 	mh->mh_bastreams |= 1<<sp->stream;
 	sp->public.data[0] = NULL;
 	sp->public.data[1] = NULL;
 	sp->setup = 0;
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 int
 mwl_hal_bastream_get_seqno(struct mwl_hal *mh0,
 	const MWL_HAL_BASTREAM *s, const uint8_t Macaddr[IEEE80211_ADDR_LEN],
 	uint16_t *pseqno)
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 	struct mwl_hal_bastream *sp = __DECONST(struct mwl_hal_bastream *, s);
 	HostCmd_GET_SEQNO *pCmd;
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	_CMD_SETUP(pCmd, HostCmd_GET_SEQNO, HostCmd_CMD_GET_SEQNO);
 	IEEE80211_ADDR_COPY(pCmd->MacAddr, Macaddr);
 	pCmd->TID = sp->tid;
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_GET_SEQNO);
 	if (retval == 0)
 		*pseqno = le16toh(pCmd->SeqNo);
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }	
 
 int
 mwl_hal_getwatchdogbitmap(struct mwl_hal *mh0, uint8_t bitmap[1])
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 	HostCmd_FW_GET_WATCHDOG_BITMAP *pCmd;
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	_CMD_SETUP(pCmd, HostCmd_FW_GET_WATCHDOG_BITMAP,
 		HostCmd_CMD_GET_WATCHDOG_BITMAP);
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_GET_WATCHDOG_BITMAP);
 	if (retval == 0) {
 		bitmap[0] = pCmd->Watchdogbitmap;
 		/* fw returns qid, map it to BA stream */
 		if (bitmap[0] < MWL_BAQID_MAX)
 			bitmap[0] = qid2ba[bitmap[0]];
 	}
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 /*
  * Configure aggressive Ampdu rate mode.
  */
 int
 mwl_hal_setaggampduratemode(struct mwl_hal *mh0, int mode, int threshold)
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 	HostCmd_FW_AMPDU_RETRY_RATEDROP_MODE *pCmd;
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	_CMD_SETUP(pCmd, HostCmd_FW_AMPDU_RETRY_RATEDROP_MODE,
 		HostCmd_CMD_AMPDU_RETRY_RATEDROP_MODE);
 	pCmd->Action = htole16(1);
 	pCmd->Option = htole32(mode);
 	pCmd->Threshold = htole32(threshold);
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_AMPDU_RETRY_RATEDROP_MODE);
 	MWL_HAL_UNLOCK(mh);   
 	return retval;
 }
 
 int
 mwl_hal_getaggampduratemode(struct mwl_hal *mh0, int *mode, int *threshold)
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 	HostCmd_FW_AMPDU_RETRY_RATEDROP_MODE *pCmd;
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	_CMD_SETUP(pCmd, HostCmd_FW_AMPDU_RETRY_RATEDROP_MODE,
 		HostCmd_CMD_AMPDU_RETRY_RATEDROP_MODE);
 	pCmd->Action = htole16(0);
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_AMPDU_RETRY_RATEDROP_MODE);
 	MWL_HAL_UNLOCK(mh);   
 	*mode =  le32toh(pCmd->Option);
 	*threshold = le32toh(pCmd->Threshold);
 	return retval;
 }
 
 /*
  * Set CFEND status Enable/Disable
  */
 int
 mwl_hal_setcfend(struct mwl_hal *mh0, int ena)
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 	HostCmd_CFEND_ENABLE *pCmd;
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	_CMD_SETUP(pCmd, HostCmd_CFEND_ENABLE,
 		HostCmd_CMD_CFEND_ENABLE);
 	pCmd->Enable = htole32(ena);
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_CFEND_ENABLE);
 	MWL_HAL_UNLOCK(mh); 
 	return retval;
 }
 
 int
 mwl_hal_setdwds(struct mwl_hal *mh0, int ena)
 {
 	HostCmd_DWDS_ENABLE *pCmd;
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
    	int retval;
 
 	MWL_HAL_LOCK(mh);
 	_CMD_SETUP(pCmd, HostCmd_DWDS_ENABLE, HostCmd_CMD_DWDS_ENABLE);
 	pCmd->Enable = htole32(ena);
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_DWDS_ENABLE);
   	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 static void
 cvtPeerInfo(PeerInfo_t *to, const MWL_HAL_PEERINFO *from)
 {
 	to->LegacyRateBitMap = htole32(from->LegacyRateBitMap);
 	to->HTRateBitMap = htole32(from->HTRateBitMap);
 	to->CapInfo = htole16(from->CapInfo);
 	to->HTCapabilitiesInfo = htole16(from->HTCapabilitiesInfo);
 	to->MacHTParamInfo = from->MacHTParamInfo;
 	to->AddHtInfo.ControlChan = from->AddHtInfo.ControlChan;
 	to->AddHtInfo.AddChan = from->AddHtInfo.AddChan;
 	to->AddHtInfo.OpMode = htole16(from->AddHtInfo.OpMode);
 	to->AddHtInfo.stbc = htole16(from->AddHtInfo.stbc);
 }
 
 /* XXX station id must be in [0..63] */
 int
 mwl_hal_newstation(struct mwl_hal_vap *vap,
 	const uint8_t addr[IEEE80211_ADDR_LEN], uint16_t aid, uint16_t sid, 
 	const MWL_HAL_PEERINFO *peer, int isQosSta, int wmeInfo)
 {
 	struct mwl_hal_priv *mh = MWLVAP(vap);
 	HostCmd_FW_SET_NEW_STN *pCmd;
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	_VCMD_SETUP(vap, pCmd, HostCmd_FW_SET_NEW_STN, HostCmd_CMD_SET_NEW_STN);
 	pCmd->AID = htole16(aid);
 	pCmd->StnId = htole16(sid);
 	pCmd->Action = htole16(0);	/* SET */
 	if (peer != NULL) {
 		/* NB: must fix up byte order */
 		cvtPeerInfo(&pCmd->PeerInfo, peer);
 	}
 	IEEE80211_ADDR_COPY(&pCmd->MacAddr[0], addr);
 	pCmd->Qosinfo = wmeInfo;
 	pCmd->isQosSta = (isQosSta != 0);
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_SET_NEW_STN);
 	if (retval == 0 && IEEE80211_ADDR_EQ(vap->mac, addr))
 		vap->flags |= MVF_STATION;
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 int
 mwl_hal_delstation(struct mwl_hal_vap *vap,
 	const uint8_t addr[IEEE80211_ADDR_LEN])
 {
 	struct mwl_hal_priv *mh = MWLVAP(vap);
 	HostCmd_FW_SET_NEW_STN *pCmd;
 	int retval, islocal;
 
 	MWL_HAL_LOCK(mh);
 	islocal = IEEE80211_ADDR_EQ(vap->mac, addr);
 	if (!islocal || (vap->flags & MVF_STATION)) {
 		_VCMD_SETUP(vap, pCmd, HostCmd_FW_SET_NEW_STN,
 		    HostCmd_CMD_SET_NEW_STN);
 		pCmd->Action = htole16(2);	/* REMOVE */
 		IEEE80211_ADDR_COPY(&pCmd->MacAddr[0], addr);
 		retval = mwlExecuteCmd(mh, HostCmd_CMD_SET_NEW_STN);
 		if (islocal)
 			vap->flags &= ~MVF_STATION;
 	} else
 		retval = 0;
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 /*
  * Prod the firmware to age packets on station power
  * save queues and reap frames on the tx aggregation q's.
  */
 int
 mwl_hal_setkeepalive(struct mwl_hal *mh0)
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 	HostCmd_FW_SET_KEEP_ALIVE_TICK *pCmd;
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	_CMD_SETUP(pCmd, HostCmd_FW_SET_KEEP_ALIVE_TICK,
 		HostCmd_CMD_SET_KEEP_ALIVE);
 	/*
 	 * NB: tick must be 0 to prod the f/w;
 	 *     a non-zero value is a noop.
 	 */
 	pCmd->tick = 0;
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_SET_KEEP_ALIVE);
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 int
 mwl_hal_setapmode(struct mwl_hal_vap *vap, MWL_HAL_APMODE ApMode)
 {
 	struct mwl_hal_priv *mh = MWLVAP(vap);
 	HostCmd_FW_SET_APMODE *pCmd;
 	int retval;
 
 	/* XXX validate ApMode? */
 
 	MWL_HAL_LOCK(mh);
 	_VCMD_SETUP(vap, pCmd, HostCmd_FW_SET_APMODE, HostCmd_CMD_SET_APMODE);
 	pCmd->ApMode = ApMode;
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_SET_APMODE);
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 int
 mwl_hal_stop(struct mwl_hal_vap *vap)
 {
 	struct mwl_hal_priv *mh = MWLVAP(vap);
 	HostCmd_DS_BSS_START *pCmd;
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	if (vap->flags & MVF_RUNNING) {
 		_VCMD_SETUP(vap, pCmd, HostCmd_DS_BSS_START,
 		    HostCmd_CMD_BSS_START);
 		pCmd->Enable = htole32(HostCmd_ACT_GEN_OFF);
 		retval = mwlExecuteCmd(mh, HostCmd_CMD_BSS_START);
 	} else
 		retval = 0;
 	/* NB: mark !running regardless */
 	vap->flags &= ~MVF_RUNNING;
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 int
 mwl_hal_start(struct mwl_hal_vap *vap)
 {
 	struct mwl_hal_priv *mh = MWLVAP(vap);
 	HostCmd_DS_BSS_START *pCmd;
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	_VCMD_SETUP(vap, pCmd, HostCmd_DS_BSS_START, HostCmd_CMD_BSS_START);
 	pCmd->Enable = htole32(HostCmd_ACT_GEN_ON);
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_BSS_START);
 	if (retval == 0)
 		vap->flags |= MVF_RUNNING;
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 int
 mwl_hal_setgprot(struct mwl_hal *mh0, int prot)
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 	HostCmd_FW_SET_G_PROTECT_FLAG *pCmd;
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	_CMD_SETUP(pCmd, HostCmd_FW_SET_G_PROTECT_FLAG,
 		HostCmd_CMD_SET_G_PROTECT_FLAG);
 	pCmd->GProtectFlag  = htole32(prot);
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_SET_G_PROTECT_FLAG);
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 int
 mwl_hal_setwmm(struct mwl_hal *mh0, int onoff)
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 	HostCmd_FW_SetWMMMode *pCmd;
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	_CMD_SETUP(pCmd, HostCmd_FW_SetWMMMode,
 		HostCmd_CMD_SET_WMM_MODE);
 	pCmd->Action = htole16(onoff);
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_SET_WMM_MODE);
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 int
 mwl_hal_setedcaparams(struct mwl_hal *mh0, uint8_t qnum,
 	uint32_t CWmin, uint32_t CWmax, uint8_t AIFSN,  uint16_t TXOPLimit)
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 	HostCmd_FW_SET_EDCA_PARAMS *pCmd;
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	_CMD_SETUP(pCmd, HostCmd_FW_SET_EDCA_PARAMS,
 		HostCmd_CMD_SET_EDCA_PARAMS);
 	/*
 	 * NB: CWmin and CWmax are always set.
 	 *     TxOpLimit is set if bit 0x2 is marked in Action
 	 *     AIFSN is set if bit 0x4 is marked in Action
 	 */
 	pCmd->Action = htole16(0xffff);	/* NB: set everything */
 	pCmd->TxOP = htole16(TXOPLimit);
 	pCmd->CWMax = htole32(CWmax);
 	pCmd->CWMin = htole32(CWmin);
 	pCmd->AIFSN = AIFSN;
 	pCmd->TxQNum = qnum;		/* XXX check */
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_SET_EDCA_PARAMS);
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 /* XXX 0 = indoor, 1 = outdoor */
 int
 mwl_hal_setrateadaptmode(struct mwl_hal *mh0, uint16_t mode)
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 	HostCmd_DS_SET_RATE_ADAPT_MODE *pCmd;
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	_CMD_SETUP(pCmd, HostCmd_DS_SET_RATE_ADAPT_MODE,
 		HostCmd_CMD_SET_RATE_ADAPT_MODE);
 	pCmd->Action = htole16(HostCmd_ACT_GEN_SET);
 	pCmd->RateAdaptMode = htole16(mode);
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_SET_RATE_ADAPT_MODE);
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 int
 mwl_hal_setcsmode(struct mwl_hal *mh0, MWL_HAL_CSMODE csmode)
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 	HostCmd_DS_SET_LINKADAPT_CS_MODE *pCmd;
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	_CMD_SETUP(pCmd, HostCmd_DS_SET_LINKADAPT_CS_MODE,
 		HostCmd_CMD_SET_LINKADAPT_CS_MODE);
 	pCmd->Action = htole16(HostCmd_ACT_GEN_SET);
 	pCmd->CSMode = htole16(csmode);
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_SET_LINKADAPT_CS_MODE);
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 int
 mwl_hal_setnprot(struct mwl_hal_vap *vap, MWL_HAL_HTPROTECT mode)
 {
 	struct mwl_hal_priv *mh = MWLVAP(vap);
 	HostCmd_FW_SET_N_PROTECT_FLAG *pCmd;
 	int retval;
 
 	/* XXX validate mode */
 	MWL_HAL_LOCK(mh);
 	_VCMD_SETUP(vap, pCmd, HostCmd_FW_SET_N_PROTECT_FLAG,
 		HostCmd_CMD_SET_N_PROTECT_FLAG);
 	pCmd->NProtectFlag  = htole32(mode);
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_SET_N_PROTECT_FLAG);
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 int
 mwl_hal_setnprotmode(struct mwl_hal_vap *vap, uint8_t mode)
 {
 	struct mwl_hal_priv *mh = MWLVAP(vap);
 	HostCmd_FW_SET_N_PROTECT_OPMODE *pCmd;
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	_VCMD_SETUP(vap, pCmd, HostCmd_FW_SET_N_PROTECT_OPMODE,
 		HostCmd_CMD_SET_N_PROTECT_OPMODE);
 	pCmd->NProtectOpMode = mode;
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_SET_N_PROTECT_OPMODE);
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 int
 mwl_hal_setoptimizationlevel(struct mwl_hal *mh0, int level)
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 	HostCmd_FW_SET_OPTIMIZATION_LEVEL *pCmd;
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	_CMD_SETUP(pCmd, HostCmd_FW_SET_OPTIMIZATION_LEVEL,
 		HostCmd_CMD_SET_OPTIMIZATION_LEVEL);
 	pCmd->OptLevel = level;
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_SET_OPTIMIZATION_LEVEL);
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 int
 mwl_hal_setmimops(struct mwl_hal *mh0, const uint8_t addr[IEEE80211_ADDR_LEN],
 	uint8_t enable, uint8_t mode)
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 	HostCmd_FW_SET_MIMOPSHT *pCmd;
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	_CMD_SETUP(pCmd, HostCmd_FW_SET_MIMOPSHT, HostCmd_CMD_SET_MIMOPSHT);
 	IEEE80211_ADDR_COPY(pCmd->Addr, addr);
 	pCmd->Enable = enable;
 	pCmd->Mode = mode;
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_SET_MIMOPSHT);
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 static int
 mwlGetCalTable(struct mwl_hal_priv *mh, uint8_t annex, uint8_t index)
 {
 	HostCmd_FW_GET_CALTABLE *pCmd;
 	int retval;
 
 	MWL_HAL_LOCK_ASSERT(mh);
 
 	_CMD_SETUP(pCmd, HostCmd_FW_GET_CALTABLE, HostCmd_CMD_GET_CALTABLE);
 	pCmd->annex = annex;
 	pCmd->index = index;
 	memset(pCmd->calTbl, 0, sizeof(pCmd->calTbl));
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_GET_CALTABLE);
 	if (retval == 0 &&
 	    pCmd->calTbl[0] != annex && annex != 0 && annex != 255)
 		retval = EIO;
 	return retval;
 }							  
 
 /*
  * Calculate the max tx power from the channel's cal data.
  */
 static void
 setmaxtxpow(struct mwl_hal_channel *hc, int i, int maxix)
 {
 	hc->maxTxPow = hc->targetPowers[i];
 	for (i++; i < maxix; i++)
 		if (hc->targetPowers[i] > hc->maxTxPow)
 			hc->maxTxPow = hc->targetPowers[i];
 }
 
 /*
  * Construct channel info for 5GHz channels from cal data.
  */
 static void
 get5Ghz(MWL_HAL_CHANNELINFO *ci, const uint8_t table[], int len)
 {
 	int i, j, f, l, h;
 
 	l = 32000;
 	h = 0;
 	j = 0;
 	for (i = 0; i < len; i += 4) {
 		struct mwl_hal_channel *hc;
 
 		if (table[i] == 0)
 			continue;
 		f = 5000 + 5*table[i];
 		if (f < l)
 			l = f;
 		if (f > h)
 			h = f;
 		hc = &ci->channels[j];
 		hc->freq = f;
 		hc->ieee = table[i];
 		memcpy(hc->targetPowers, &table[i], 4);
 		setmaxtxpow(hc, 1, 4);		/* NB: col 1 is the freq, skip*/
 		j++;
 	}
 	ci->nchannels = j;
 	ci->freqLow = (l == 32000) ? 0 : l;
 	ci->freqHigh = h;
 }
 
 static uint16_t
 ieee2mhz(int chan)
 {
 	if (chan == 14)
 		return 2484;
 	if (chan < 14)
 		return 2407 + chan*5;
 	return 2512 + (chan-15)*20;
 }
 
 /*
  * Construct channel info for 2.4GHz channels from cal data.
  */
 static void
 get2Ghz(MWL_HAL_CHANNELINFO *ci, const uint8_t table[], int len)
 {
 	int i, j;
 
 	j = 0;
 	for (i = 0; i < len; i += 4) {
 		struct mwl_hal_channel *hc = &ci->channels[j];
 		hc->ieee = 1+j;
 		hc->freq = ieee2mhz(1+j);
 		memcpy(hc->targetPowers, &table[i], 4);
 		setmaxtxpow(hc, 0, 4);
 		j++;
 	}
 	ci->nchannels = j;
 	ci->freqLow = ieee2mhz(1);
 	ci->freqHigh = ieee2mhz(j);
 }
 
 #undef DUMPCALDATA
 #ifdef DUMPCALDATA
 static void
 dumpcaldata(const char *name, const uint8_t *table, int n)
 {
 	int i;
 	printf("\n%s:\n", name);
 	for (i = 0; i < n; i += 4)
 		printf("[%2d] %3d %3d %3d %3d\n", i/4, table[i+0], table[i+1], table[i+2], table[i+3]);
 }
 #endif
 
 static int
 mwlGetPwrCalTable(struct mwl_hal_priv *mh)
 {
 	const uint8_t *data;
 	MWL_HAL_CHANNELINFO *ci;
 	int len;
 
 	MWL_HAL_LOCK(mh);
 	/* NB: we hold the lock so it's ok to use cmdbuf */
 	data = ((const HostCmd_FW_GET_CALTABLE *) mh->mh_cmdbuf)->calTbl;
 	if (mwlGetCalTable(mh, 33, 0) == 0) {
 		len = (data[2] | (data[3] << 8)) - 12;
 		if (len > PWTAGETRATETABLE20M)
 			len = PWTAGETRATETABLE20M;
 #ifdef DUMPCALDATA
 dumpcaldata("2.4G 20M", &data[12], len);/*XXX*/
 #endif
 		get2Ghz(&mh->mh_20M, &data[12], len);
 	}
 	if (mwlGetCalTable(mh, 34, 0) == 0) {
 		len = (data[2] | (data[3] << 8)) - 12;
 		if (len > PWTAGETRATETABLE40M)
 			len = PWTAGETRATETABLE40M;
 #ifdef DUMPCALDATA
 dumpcaldata("2.4G 40M", &data[12], len);/*XXX*/
 #endif
 		ci = &mh->mh_40M;
 		get2Ghz(ci, &data[12], len);
 	}
 	if (mwlGetCalTable(mh, 35, 0) == 0) {
 		len = (data[2] | (data[3] << 8)) - 20;
 		if (len > PWTAGETRATETABLE20M_5G)
 			len = PWTAGETRATETABLE20M_5G;
 #ifdef DUMPCALDATA
 dumpcaldata("5G 20M", &data[20], len);/*XXX*/
 #endif
 		get5Ghz(&mh->mh_20M_5G, &data[20], len);
 	}
 	if (mwlGetCalTable(mh, 36, 0) == 0) {
 		len = (data[2] | (data[3] << 8)) - 20;
 		if (len > PWTAGETRATETABLE40M_5G)
 			len = PWTAGETRATETABLE40M_5G;
 #ifdef DUMPCALDATA
 dumpcaldata("5G 40M", &data[20], len);/*XXX*/
 #endif
 		ci = &mh->mh_40M_5G;
 		get5Ghz(ci, &data[20], len);
 	}
 	mh->mh_flags |= MHF_CALDATA;
 	MWL_HAL_UNLOCK(mh);
 	return 0;
 }
 
 int
 mwl_hal_getregioncode(struct mwl_hal *mh0, uint8_t *countryCode)
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	retval = mwlGetCalTable(mh, 0, 0);
 	if (retval == 0) {
 		const HostCmd_FW_GET_CALTABLE *pCmd =
 		    (const HostCmd_FW_GET_CALTABLE *) mh->mh_cmdbuf;
 		*countryCode = pCmd->calTbl[16];
 	}
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 int
 mwl_hal_setpromisc(struct mwl_hal *mh0, int ena)
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 	uint32_t v;
 
 	MWL_HAL_LOCK(mh);
 	v = RD4(mh, MACREG_REG_PROMISCUOUS);
 	WR4(mh, MACREG_REG_PROMISCUOUS, ena ? v | 1 : v &~ 1);
 	MWL_HAL_UNLOCK(mh);
 	return 0;
 }
 
 int
 mwl_hal_getpromisc(struct mwl_hal *mh0)
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 	uint32_t v;
 
 	MWL_HAL_LOCK(mh);
 	v = RD4(mh, MACREG_REG_PROMISCUOUS);
 	MWL_HAL_UNLOCK(mh);
 	return (v & 1) != 0;
 }
 
 int
 mwl_hal_GetBeacon(struct mwl_hal *mh0, uint8_t *pBcn, uint16_t *pLen)
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 	HostCmd_FW_GET_BEACON *pCmd;
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	_CMD_SETUP(pCmd, HostCmd_FW_GET_BEACON, HostCmd_CMD_GET_BEACON);
 	pCmd->Bcnlen = htole16(0);
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_GET_BEACON);
 	if (retval == 0) {
 		/* XXX bounds check */
 		memcpy(pBcn, &pCmd->Bcn, pCmd->Bcnlen);
 		*pLen = pCmd->Bcnlen;
 	}
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }	
 
 int
 mwl_hal_SetRifs(struct mwl_hal *mh0, uint8_t QNum)
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 	HostCmd_FW_SET_RIFS  *pCmd;
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	_CMD_SETUP(pCmd, HostCmd_FW_SET_RIFS, HostCmd_CMD_SET_RIFS);
 	pCmd->QNum = QNum;
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_SET_RIFS);
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 /*
  * Diagnostic api's for set/get registers.
  */
 
 static int
 getRFReg(struct mwl_hal_priv *mh, int flag, uint32_t reg, uint32_t *val)
 {
 	HostCmd_DS_RF_REG_ACCESS *pCmd;
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	_CMD_SETUP(pCmd, HostCmd_DS_RF_REG_ACCESS, HostCmd_CMD_RF_REG_ACCESS);
 	pCmd->Offset =  htole16(reg);
 	pCmd->Action = htole16(flag);
 	pCmd->Value = htole32(*val);
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_RF_REG_ACCESS);
 	if (retval == 0)
 		*val = pCmd->Value;
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 static int
 getBBReg(struct mwl_hal_priv *mh, int flag, uint32_t reg, uint32_t *val)
 {
 	HostCmd_DS_BBP_REG_ACCESS *pCmd;
 	int retval;
 
 	MWL_HAL_LOCK(mh);
 	_CMD_SETUP(pCmd, HostCmd_DS_BBP_REG_ACCESS, HostCmd_CMD_BBP_REG_ACCESS);
 	pCmd->Offset =  htole16(reg);
 	pCmd->Action = htole16(flag);
 	pCmd->Value = htole32(*val);
 
 	retval = mwlExecuteCmd(mh, HostCmd_CMD_BBP_REG_ACCESS);
 	if (retval == 0)
 		*val = pCmd->Value;
 	MWL_HAL_UNLOCK(mh);
 	return retval;
 }
 
 static u_int
 mwl_hal_getregdump(struct mwl_hal_priv *mh, const MWL_DIAG_REGRANGE *regs,
 	void *dstbuf, int space)
 {
 	uint32_t *dp = dstbuf;
 	int i;
 
 	for (i = 0; space >= 2*sizeof(uint32_t); i++) {
 		u_int r = regs[i].start;
 		u_int e = regs[i].end;
 		*dp++ = (r<<16) | e;
 		space -= sizeof(uint32_t);
 		do {
 			if (MWL_DIAG_ISMAC(r))
 				*dp = RD4(mh, r);
 			else if (MWL_DIAG_ISBB(r))
 				getBBReg(mh, HostCmd_ACT_GEN_READ,
 				    r - MWL_DIAG_BASE_BB, dp);
 			else if (MWL_DIAG_ISRF(r))
 				getRFReg(mh, HostCmd_ACT_GEN_READ,
 				    r - MWL_DIAG_BASE_RF, dp);
 			else if (r < 0x1000 || r == MACREG_REG_FW_PRESENT)
 				*dp = RD4(mh, r);
 			else
 				*dp = 0xffffffff;
 			dp++;
 			r += sizeof(uint32_t);
 			space -= sizeof(uint32_t);
 		} while (r <= e && space >= sizeof(uint32_t));
 	}
 	return (char *) dp - (char *) dstbuf;
 }
 
 int
 mwl_hal_getdiagstate(struct mwl_hal *mh0, int request,
 	const void *args, uint32_t argsize,
 	void **result, uint32_t *resultsize)
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 
 	switch (request) {
 	case MWL_DIAG_CMD_REVS:
 		*result = &mh->mh_revs;
 		*resultsize = sizeof(mh->mh_revs);
 		return 1;
 	case MWL_DIAG_CMD_REGS:
 		*resultsize = mwl_hal_getregdump(mh, args, *result, *resultsize);
 		return 1;
 	case MWL_DIAG_CMD_HOSTCMD: {
 		FWCmdHdr *pCmd = (FWCmdHdr *) &mh->mh_cmdbuf[0];
 		int retval;
 
 		MWL_HAL_LOCK(mh);
 		memcpy(pCmd, args, argsize);
 		retval = mwlExecuteCmd(mh, le16toh(pCmd->Cmd));
 		*result = (*resultsize != 0) ? pCmd : NULL;
 		MWL_HAL_UNLOCK(mh);
 		return (retval == 0);
 	}
 	case MWL_DIAG_CMD_FWLOAD:
 		if (mwl_hal_fwload(mh0, __DECONST(void *, args))) {
 			device_printf(mh->mh_dev, "problem loading fw image\n");
 			return 0;
 		}
 		return 1;
 	}
 	return 0;
 }
 
 /*
  * Low level firmware cmd block handshake support.
  */
 
 static void
 mwlSendCmd(struct mwl_hal_priv *mh)
 {
 	uint32_t dummy;
 
 	bus_dmamap_sync(mh->mh_dmat, mh->mh_dmamap,
 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 
 	WR4(mh, MACREG_REG_GEN_PTR, mh->mh_cmdaddr);
 	dummy = RD4(mh, MACREG_REG_INT_CODE);
 
 	WR4(mh, MACREG_REG_H2A_INTERRUPT_EVENTS, MACREG_H2ARIC_BIT_DOOR_BELL);
 }
 
 static int
 mwlWaitForCmdComplete(struct mwl_hal_priv *mh, uint16_t cmdCode)
 {
 #define MAX_WAIT_FW_COMPLETE_ITERATIONS 10000
 	int i;
 
 	for (i = 0; i < MAX_WAIT_FW_COMPLETE_ITERATIONS; i++) {
 		if (mh->mh_cmdbuf[0] == le16toh(cmdCode))
 			return 1;
 		DELAY(1*1000);
 	}
 	return 0;
 #undef MAX_WAIT_FW_COMPLETE_ITERATIONS
 }
 
 static int
 mwlExecuteCmd(struct mwl_hal_priv *mh, unsigned short cmd)
 {
 
 	MWL_HAL_LOCK_ASSERT(mh);
 
 	if ((mh->mh_flags & MHF_FWHANG) &&
 	    (mh->mh_debug & MWL_HAL_DEBUG_IGNHANG) == 0) {
 #ifdef MWLHAL_DEBUG
 		device_printf(mh->mh_dev, "firmware hung, skipping cmd %s\n",
 			mwlcmdname(cmd));
 #else
 		device_printf(mh->mh_dev, "firmware hung, skipping cmd 0x%x\n",
 			cmd);
 #endif
 		return ENXIO;
 	}
 	if (RD4(mh,  MACREG_REG_INT_CODE) == 0xffffffff) {
 		device_printf(mh->mh_dev, "%s: device not present!\n",
 		    __func__);
 		return EIO;
 	}
 #ifdef MWLHAL_DEBUG
 	if (mh->mh_debug & MWL_HAL_DEBUG_SENDCMD)
 		dumpresult(mh, 0);
 #endif
 	mwlSendCmd(mh);
 	if (!mwlWaitForCmdComplete(mh, 0x8000 | cmd)) {
 #ifdef MWLHAL_DEBUG
 		device_printf(mh->mh_dev,
 		    "timeout waiting for f/w cmd %s\n", mwlcmdname(cmd));
 #else
 		device_printf(mh->mh_dev,
 		    "timeout waiting for f/w cmd 0x%x\n", cmd);
 #endif
 		mh->mh_flags |= MHF_FWHANG;
 		return ETIMEDOUT;
 	}
 	bus_dmamap_sync(mh->mh_dmat, mh->mh_dmamap,
 	    BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
 #ifdef MWLHAL_DEBUG
 	if (mh->mh_debug & MWL_HAL_DEBUG_CMDDONE)
 		dumpresult(mh, 1);
 #endif
 	return 0;
 }
 
 /*
  * Firmware download support.
  */
 #define FW_DOWNLOAD_BLOCK_SIZE	256  
 #define FW_CHECK_USECS		(5*1000)	/* 5ms */
 #define FW_MAX_NUM_CHECKS	200  
 
 #if 0
 /* XXX read f/w from file */
 #include <dev/mwl/mwlbootfw.h>
 #include <dev/mwl/mwl88W8363fw.h>
 #endif
 
 static void
 mwlFwReset(struct mwl_hal_priv *mh)
 {
 	if (RD4(mh,  MACREG_REG_INT_CODE) == 0xffffffff) {
 		device_printf(mh->mh_dev, "%s: device not present!\n",
 		    __func__);
 		return;
 	}
 	WR4(mh, MACREG_REG_H2A_INTERRUPT_EVENTS, ISR_RESET);
 	mh->mh_flags &= ~MHF_FWHANG;
 }
 
 static void
 mwlTriggerPciCmd(struct mwl_hal_priv *mh)
 {
 	uint32_t dummy;
 
 	bus_dmamap_sync(mh->mh_dmat, mh->mh_dmamap, BUS_DMASYNC_PREWRITE);
 
 	WR4(mh, MACREG_REG_GEN_PTR, mh->mh_cmdaddr);
 	dummy = RD4(mh, MACREG_REG_INT_CODE);
 
 	WR4(mh, MACREG_REG_INT_CODE, 0x00);
 	dummy = RD4(mh, MACREG_REG_INT_CODE);
 
 	WR4(mh, MACREG_REG_H2A_INTERRUPT_EVENTS, MACREG_H2ARIC_BIT_DOOR_BELL);
 	dummy = RD4(mh, MACREG_REG_INT_CODE);
 }
 
 static int
 mwlWaitFor(struct mwl_hal_priv *mh, uint32_t val)
 {
 	int i;
 
 	for (i = 0; i < FW_MAX_NUM_CHECKS; i++) {
 		DELAY(FW_CHECK_USECS);
 		if (RD4(mh, MACREG_REG_INT_CODE) == val)
 			return 1;
 	}
 	return 0;
 }
 
 /*
  * Firmware block xmit when talking to the boot-rom.
  */
 static int
 mwlSendBlock(struct mwl_hal_priv *mh, int bsize, const void *data, size_t dsize)
 {
 	mh->mh_cmdbuf[0] = htole16(HostCmd_CMD_CODE_DNLD);
 	mh->mh_cmdbuf[1] = htole16(bsize);
 	memcpy(&mh->mh_cmdbuf[4], data , dsize);
 	mwlTriggerPciCmd(mh);
 	/* XXX 2000 vs 200 */
 	if (mwlWaitFor(mh, MACREG_INT_CODE_CMD_FINISHED)) {
 		WR4(mh, MACREG_REG_INT_CODE, 0);
 		return 1;
 	}
 	device_printf(mh->mh_dev,
 	    "%s: timeout waiting for CMD_FINISHED, INT_CODE 0x%x\n",
 	    __func__, RD4(mh, MACREG_REG_INT_CODE));
 	return 0;
 }
 
 /*
  * Firmware block xmit when talking to the 1st-stage loader.
  */
 static int
 mwlSendBlock2(struct mwl_hal_priv *mh, const void *data, size_t dsize)
 {
 	memcpy(&mh->mh_cmdbuf[0], data, dsize);
 	mwlTriggerPciCmd(mh);
 	if (mwlWaitFor(mh, MACREG_INT_CODE_CMD_FINISHED)) {
 		WR4(mh, MACREG_REG_INT_CODE, 0);
 		return 1;
 	}
 	device_printf(mh->mh_dev,
 	    "%s: timeout waiting for CMD_FINISHED, INT_CODE 0x%x\n",
 	    __func__, RD4(mh, MACREG_REG_INT_CODE));
 	return 0;
 }
 
 static void
 mwlPokeSdramController(struct mwl_hal_priv *mh, int SDRAMSIZE_Addr)
 {
 	/** Set up sdram controller for superflyv2 **/
 	WR4(mh, 0x00006014, 0x33);
 	WR4(mh, 0x00006018, 0xa3a2632);
 	WR4(mh, 0x00006010, SDRAMSIZE_Addr);
 }
 
 int
 mwl_hal_fwload(struct mwl_hal *mh0, void *fwargs)
 {
 	struct mwl_hal_priv *mh = MWLPRIV(mh0);
 	const char *fwname = "mw88W8363fw";
 	const char *fwbootname = "mwlboot";
 	const struct firmware *fwboot = NULL;
 	const struct firmware *fw;
 	/* XXX get from firmware header */
 	uint32_t FwReadySignature = HostCmd_SOFTAP_FWRDY_SIGNATURE;
 	uint32_t OpMode = HostCmd_SOFTAP_MODE;
 	const uint8_t *fp, *ep;
 	const uint8_t *fmdata;
 	uint32_t blocksize, nbytes, fmsize;
 	int i, error, ntries;
 
 	fw = firmware_get(fwname);
 	if (fw == NULL) {
 		device_printf(mh->mh_dev,
 		    "could not load firmware image %s\n", fwname);
 		return ENXIO;
 	}
 	fmdata = fw->data;
 	fmsize = fw->datasize;
 	if (fmsize < 4) {
 		device_printf(mh->mh_dev, "firmware image %s too small\n",
 		    fwname);
 		error = ENXIO;
 		goto bad2;
 	}
 	if (fmdata[0] == 0x01 && fmdata[1] == 0x00 &&
 	    fmdata[2] == 0x00 && fmdata[3] == 0x00) {
 		/*
 		 * 2-stage load, get the boot firmware.
 		 */
 		fwboot = firmware_get(fwbootname);
 		if (fwboot == NULL) {
 			device_printf(mh->mh_dev,
 			    "could not load firmware image %s\n", fwbootname);
 			error = ENXIO;
 			goto bad2;
 		}
 	} else
 		fwboot = NULL;
 
 	mwlFwReset(mh);
 
 	WR4(mh, MACREG_REG_A2H_INTERRUPT_CLEAR_SEL, MACREG_A2HRIC_BIT_MASK);
 	WR4(mh, MACREG_REG_A2H_INTERRUPT_CAUSE, 0x00);
 	WR4(mh, MACREG_REG_A2H_INTERRUPT_MASK, 0x00);
 	WR4(mh, MACREG_REG_A2H_INTERRUPT_STATUS_MASK, MACREG_A2HRIC_BIT_MASK);
 	if (mh->mh_SDRAMSIZE_Addr != 0) {
 		/** Set up sdram controller for superflyv2 **/
 		mwlPokeSdramController(mh, mh->mh_SDRAMSIZE_Addr);
 	}
 	device_printf(mh->mh_dev, "load %s firmware image (%u bytes)\n",
 	    fwname, fmsize);
 	if (fwboot != NULL) {
 		/*
 		 * Do 2-stage load.  The 1st stage loader is setup
 		 * with the bootrom loader then we load the real
 		 * image using a different handshake. With this
 		 * mechanism the firmware is segmented into chunks
 		 * that have a CRC.  If a chunk is incorrect we'll
 		 * be told to retransmit.
 		 */
 		/* XXX assumes hlpimage fits in a block */
 		/* NB: zero size block indicates download is finished */
 		if (!mwlSendBlock(mh, fwboot->datasize, fwboot->data, fwboot->datasize) ||
 		    !mwlSendBlock(mh, 0, NULL, 0)) {
 			error = ETIMEDOUT;
 			goto bad;
 		}
 		DELAY(200*FW_CHECK_USECS);
 		if (mh->mh_SDRAMSIZE_Addr != 0) {
 			/** Set up sdram controller for superflyv2 **/
 			mwlPokeSdramController(mh, mh->mh_SDRAMSIZE_Addr);
 		}
 		nbytes = ntries = 0;		/* NB: silence compiler */
 		for (fp = fmdata, ep = fp + fmsize; fp < ep; ) {
 			WR4(mh, MACREG_REG_INT_CODE, 0);
 			blocksize = RD4(mh, MACREG_REG_SCRATCH);
 			if (blocksize == 0)	/* download complete */
 				break;
 			if (blocksize > 0x00000c00) {
 				error = EINVAL;
 				goto bad;
 			}
 			if ((blocksize & 0x1) == 0) {
 				/* block successfully downloaded, advance */
 				fp += nbytes;
 				ntries = 0;
 			} else {
 				if (++ntries > 2) {
 					/*
 					 * Guard against f/w telling us to
 					 * retry infinitely.
 					 */
 					error = ELOOP;
 					goto bad;
 				}
 				/* clear NAK bit/flag */
 				blocksize &= ~0x1;
 			}
 			if (blocksize > ep - fp) {
 				/* XXX this should not happen, what to do? */
 				blocksize = ep - fp;
 			}
 			nbytes = blocksize;
 			if (!mwlSendBlock2(mh, fp, nbytes)) {
 				error = ETIMEDOUT;
 				goto bad;
 			}
 		}
 	} else {
 		for (fp = fmdata, ep = fp + fmsize; fp < ep;) {
 			nbytes = ep - fp;
 			if (nbytes > FW_DOWNLOAD_BLOCK_SIZE)
 				nbytes = FW_DOWNLOAD_BLOCK_SIZE;
 			if (!mwlSendBlock(mh, FW_DOWNLOAD_BLOCK_SIZE, fp, nbytes)) {
 				error = EIO;
 				goto bad;
 			}
 			fp += nbytes;
 		}
 	}
 	/* done with firmware... */
 	if (fwboot != NULL)
 		firmware_put(fwboot, FIRMWARE_UNLOAD);
 	firmware_put(fw, FIRMWARE_UNLOAD);
 	/*
 	 * Wait for firmware to startup; we monitor the
 	 * INT_CODE register waiting for a signature to
 	 * written back indicating it's ready to go.
 	 */
 	mh->mh_cmdbuf[1] = 0;
 	/*
 	 * XXX WAR for mfg fw download
 	 */
 	if (OpMode != HostCmd_STA_MODE)
 		mwlTriggerPciCmd(mh);
 	for (i = 0; i < FW_MAX_NUM_CHECKS; i++) {
 		WR4(mh, MACREG_REG_GEN_PTR, OpMode);
 		DELAY(FW_CHECK_USECS);
 		if (RD4(mh, MACREG_REG_INT_CODE) == FwReadySignature) {
 			WR4(mh, MACREG_REG_INT_CODE, 0x00);
 			return mwlResetHalState(mh);
 		}
 	}
 	return ETIMEDOUT;
 bad:
 	mwlFwReset(mh);
 bad2:
 	/* done with firmware... */
 	if (fwboot != NULL)
 		firmware_put(fwboot, FIRMWARE_UNLOAD);
 	firmware_put(fw, FIRMWARE_UNLOAD);
 	return error;
 }
 
 #ifdef MWLHAL_DEBUG
 static const char *
 mwlcmdname(int cmd)
 {
 	static char buf[12];
 #define	CMD(x)	case HostCmd_CMD_##x: return #x
 	switch (cmd) {
 	CMD(CODE_DNLD);
 	CMD(GET_HW_SPEC);
 	CMD(SET_HW_SPEC);
 	CMD(MAC_MULTICAST_ADR);
 	CMD(802_11_GET_STAT);
 	CMD(MAC_REG_ACCESS);
 	CMD(BBP_REG_ACCESS);
 	CMD(RF_REG_ACCESS);
 	CMD(802_11_RADIO_CONTROL);
 	CMD(802_11_RF_TX_POWER);
 	CMD(802_11_RF_ANTENNA);
 	CMD(SET_BEACON);
 	CMD(SET_RF_CHANNEL);
 	CMD(SET_AID);
 	CMD(SET_INFRA_MODE);
 	CMD(SET_G_PROTECT_FLAG);
 	CMD(802_11_RTS_THSD);
 	CMD(802_11_SET_SLOT);
 	CMD(SET_EDCA_PARAMS);
 	CMD(802_11H_DETECT_RADAR);
 	CMD(SET_WMM_MODE);
 	CMD(HT_GUARD_INTERVAL);
 	CMD(SET_FIXED_RATE);
 	CMD(SET_LINKADAPT_CS_MODE);
 	CMD(SET_MAC_ADDR);
 	CMD(SET_RATE_ADAPT_MODE);
 	CMD(BSS_START);
 	CMD(SET_NEW_STN);
 	CMD(SET_KEEP_ALIVE);
 	CMD(SET_APMODE);
 	CMD(SET_SWITCH_CHANNEL);
 	CMD(UPDATE_ENCRYPTION);
 	CMD(BASTREAM);
 	CMD(SET_RIFS);
 	CMD(SET_N_PROTECT_FLAG);
 	CMD(SET_N_PROTECT_OPMODE);
 	CMD(SET_OPTIMIZATION_LEVEL);
 	CMD(GET_CALTABLE);
 	CMD(SET_MIMOPSHT);
 	CMD(GET_BEACON);
 	CMD(SET_REGION_CODE);
 	CMD(SET_POWERSAVESTATION);
 	CMD(SET_TIM);
 	CMD(GET_TIM);
 	CMD(GET_SEQNO);
 	CMD(DWDS_ENABLE);
 	CMD(AMPDU_RETRY_RATEDROP_MODE);
 	CMD(CFEND_ENABLE);
 	}
 	snprintf(buf, sizeof(buf), "0x%x", cmd);
 	return buf;
 #undef CMD
 }
 
 static void
 dumpresult(struct mwl_hal_priv *mh, int showresult)
 {
 	const FWCmdHdr *h = (const FWCmdHdr *)mh->mh_cmdbuf;
 	const uint8_t *cp;
 	int len, i;
 
 	len = le16toh(h->Length);
 #ifdef MWL_MBSS_SUPPORT
 	device_printf(mh->mh_dev, "Cmd %s Length %d SeqNum %d MacId %d",
 	    mwlcmdname(le16toh(h->Cmd) &~ 0x8000), len, h->SeqNum, h->MacId);
 #else
 	device_printf(mh->mh_dev, "Cmd %s Length %d SeqNum %d",
 	    mwlcmdname(le16toh(h->Cmd) &~ 0x8000), len, le16toh(h->SeqNum));
 #endif
 	if (showresult) {
 		const char *results[] =
 		    { "OK", "ERROR", "NOT_SUPPORT", "PENDING", "BUSY",
 		      "PARTIAL_DATA" };
 		int result = le16toh(h->Result);
 
 		if (result <= HostCmd_RESULT_PARTIAL_DATA)
 			printf(" Result %s", results[result]);
 		else
 			printf(" Result %d", result);
 	}
 	cp = (const uint8_t *)h;
 	for (i = 0; i < len; i++) {
 		if ((i % 16) == 0)
 			printf("\n%02x", cp[i]);
 		else
 			printf(" %02x", cp[i]);
 	}
 	printf("\n");
 }
 #endif /* MWLHAL_DEBUG */
Index: projects/release-debugdist/sys/dev/netmap/netmap_generic.c
===================================================================
--- projects/release-debugdist/sys/dev/netmap/netmap_generic.c	(revision 267343)
+++ projects/release-debugdist/sys/dev/netmap/netmap_generic.c	(revision 267344)
@@ -1,838 +1,845 @@
 /*
  * Copyright (C) 2013-2014 Universita` di Pisa. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  *   1. Redistributions of source code must retain the above copyright
  *      notice, this list of conditions and the following disclaimer.
  *   2. Redistributions in binary form must reproduce the above copyright
  *      notice, this list of conditions and the following disclaimer in the
  *      documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 /*
  * This module implements netmap support on top of standard,
  * unmodified device drivers.
  *
  * A NIOCREGIF request is handled here if the device does not
  * have native support. TX and RX rings are emulated as follows:
  *
  * NIOCREGIF
  *	We preallocate a block of TX mbufs (roughly as many as
  *	tx descriptors; the number is not critical) to speed up
  *	operation during transmissions. The refcount on most of
  *	these buffers is artificially bumped up so we can recycle
  *	them more easily. Also, the destructor is intercepted
  *	so we use it as an interrupt notification to wake up
  *	processes blocked on a poll().
  *
  *	For each receive ring we allocate one "struct mbq"
  *	(an mbuf tailq plus a spinlock). We intercept packets
  *	(through if_input)
  *	on the receive path and put them in the mbq from which
  *	netmap receive routines can grab them.
  *
  * TX:
  *	in the generic_txsync() routine, netmap buffers are copied
  *	(or linked, in a future) to the preallocated mbufs
  *	and pushed to the transmit queue. Some of these mbufs
  *	(those with NS_REPORT, or otherwise every half ring)
  *	have the refcount=1, others have refcount=2.
  *	When the destructor is invoked, we take that as
  *	a notification that all mbufs up to that one in
  *	the specific ring have been completed, and generate
  *	the equivalent of a transmit interrupt.
  *
  * RX:
  *
  */
 
 #ifdef __FreeBSD__
 
 #include <sys/cdefs.h> /* prerequisite */
 __FBSDID("$FreeBSD$");
 
 #include <sys/types.h>
 #include <sys/errno.h>
 #include <sys/malloc.h>
 #include <sys/lock.h>   /* PROT_EXEC */
 #include <sys/rwlock.h>
 #include <sys/socket.h> /* sockaddrs */
 #include <sys/selinfo.h>
 #include <net/if.h>
 #include <net/if_var.h>
 #include <machine/bus.h>        /* bus_dmamap_* in netmap_kern.h */
 
 // XXX temporary - D() defined here
 #include <net/netmap.h>
 #include <dev/netmap/netmap_kern.h>
 #include <dev/netmap/netmap_mem2.h>
 
 #define rtnl_lock()	ND("rtnl_lock called")
 #define rtnl_unlock()	ND("rtnl_unlock called")
 #define MBUF_TXQ(m)	((m)->m_pkthdr.flowid)
 #define MBUF_RXQ(m)	((m)->m_pkthdr.flowid)
 #define smp_mb()
 
 /*
  * FreeBSD mbuf allocator/deallocator in emulation mode:
  *
  * We allocate EXT_PACKET mbuf+clusters, but need to set M_NOFREE
  * so that the destructor, if invoked, will not free the packet.
  *    In principle we should set the destructor only on demand,
  * but since there might be a race we better do it on allocation.
  * As a consequence, we also need to set the destructor or we
  * would leak buffers.
  */
 
 /*
  * mbuf wrappers
  */
 
-/* mbuf destructor, also need to change the type to EXT_EXTREF,
+/*
+ * mbuf destructor, also need to change the type to EXT_EXTREF,
  * add an M_NOFREE flag, and then clear the flag and
  * chain into uma_zfree(zone_pack, mf)
  * (or reinstall the buffer ?)
+ *
+ * On FreeBSD 9 the destructor is called as ext_free(ext_arg1, ext_arg2)
+ * whereas newer version have ext_free(m, ext_arg1, ext_arg2)
+ * For compatibility we set ext_arg1 = m on allocation so we have
+ * the same code on both.
  */
 #define SET_MBUF_DESTRUCTOR(m, fn)	do {		\
-	(m)->m_ext.ext_free = (void *)fn;	\
-	(m)->m_ext.ext_type = EXT_EXTREF;	\
-} while (0)
+		(m)->m_ext.ext_free = (void *)fn;	\
+		(m)->m_ext.ext_type = EXT_EXTREF;	\
+	} while (0)
 
 static void 
-netmap_default_mbuf_destructor(struct mbuf *m) 
+netmap_default_mbuf_destructor(struct mbuf *m)
 { 
-	/* restore original mbuf */
-	m->m_ext.ext_buf = m->m_data = m->m_ext.ext_arg1;
-	m->m_ext.ext_arg1 = NULL;
+	/* restore original data pointer and type */
+	m->m_ext.ext_buf = m->m_data = m->m_ext.ext_arg2;
 	m->m_ext.ext_type = EXT_PACKET;
 	m->m_ext.ext_free = NULL;
+	m->m_ext.ext_arg1 = m->m_ext.ext_arg2 = NULL;
 	if (*(m->m_ext.ref_cnt) == 0)
 		*(m->m_ext.ref_cnt) = 1;
 	uma_zfree(zone_pack, m);
 } 
 
 static inline struct mbuf * 
 netmap_get_mbuf(int len) 
 { 
 	struct mbuf *m;
 	m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR | M_NOFREE);
 	if (m) {
-		m->m_ext.ext_arg1 = m->m_ext.ext_buf; // XXX save
+		m->m_ext.ext_arg1 = m; /* FreeBSD 9 compat */
+		m->m_ext.ext_arg2 = m->m_ext.ext_buf; /* save original */
 		m->m_ext.ext_free = (void *)netmap_default_mbuf_destructor;
 		m->m_ext.ext_type = EXT_EXTREF;
 		ND(5, "create m %p refcnt %d", m, *m->m_ext.ref_cnt);
 	}
 	return m;
 } 
 
 #define GET_MBUF_REFCNT(m)	((m)->m_ext.ref_cnt ? *(m)->m_ext.ref_cnt : -1)
 
 
 
 #else /* linux */
 
 #include "bsd_glue.h"
 
 #include <linux/rtnetlink.h>    /* rtnl_[un]lock() */
 #include <linux/ethtool.h>      /* struct ethtool_ops, get_ringparam */
 #include <linux/hrtimer.h>
 
 //#define RATE  /* Enables communication statistics. */
 
 //#define REG_RESET
 
 #endif /* linux */
 
 
 /* Common headers. */
 #include <net/netmap.h>
 #include <dev/netmap/netmap_kern.h>
 #include <dev/netmap/netmap_mem2.h>
 
 
 
 /* ======================== usage stats =========================== */
 
 #ifdef RATE
 #define IFRATE(x) x
 struct rate_stats {
 	unsigned long txpkt;
 	unsigned long txsync;
 	unsigned long txirq;
 	unsigned long rxpkt;
 	unsigned long rxirq;
 	unsigned long rxsync;
 };
 
 struct rate_context {
 	unsigned refcount;
 	struct timer_list timer;
 	struct rate_stats new;
 	struct rate_stats old;
 };
 
 #define RATE_PRINTK(_NAME_) \
 	printk( #_NAME_ " = %lu Hz\n", (cur._NAME_ - ctx->old._NAME_)/RATE_PERIOD);
 #define RATE_PERIOD  2
 static void rate_callback(unsigned long arg)
 {
 	struct rate_context * ctx = (struct rate_context *)arg;
 	struct rate_stats cur = ctx->new;
 	int r;
 
 	RATE_PRINTK(txpkt);
 	RATE_PRINTK(txsync);
 	RATE_PRINTK(txirq);
 	RATE_PRINTK(rxpkt);
 	RATE_PRINTK(rxsync);
 	RATE_PRINTK(rxirq);
 	printk("\n");
 
 	ctx->old = cur;
 	r = mod_timer(&ctx->timer, jiffies +
 			msecs_to_jiffies(RATE_PERIOD * 1000));
 	if (unlikely(r))
 		D("[v1000] Error: mod_timer()");
 }
 
 static struct rate_context rate_ctx;
 
 #else /* !RATE */
 #define IFRATE(x)
 #endif /* !RATE */
 
 
 /* =============== GENERIC NETMAP ADAPTER SUPPORT ================= */
 #define GENERIC_BUF_SIZE        netmap_buf_size    /* Size of the mbufs in the Tx pool. */
 
 /*
  * Wrapper used by the generic adapter layer to notify
  * the poller threads. Differently from netmap_rx_irq(), we check
  * only IFCAP_NETMAP instead of NAF_NATIVE_ON to enable the irq.
  */
 static void
 netmap_generic_irq(struct ifnet *ifp, u_int q, u_int *work_done)
 {
 	if (unlikely(!(ifp->if_capenable & IFCAP_NETMAP)))
 		return;
 
 	netmap_common_irq(ifp, q, work_done);
 }
 
 
 /* Enable/disable netmap mode for a generic network interface. */
 static int
 generic_netmap_register(struct netmap_adapter *na, int enable)
 {
 	struct ifnet *ifp = na->ifp;
 	struct netmap_generic_adapter *gna = (struct netmap_generic_adapter *)na;
 	struct mbuf *m;
 	int error;
 	int i, r;
 
 	if (!na)
 		return EINVAL;
 
 #ifdef REG_RESET
 	error = ifp->netdev_ops->ndo_stop(ifp);
 	if (error) {
 		return error;
 	}
 #endif /* REG_RESET */
 
 	if (enable) { /* Enable netmap mode. */
 		/* Init the mitigation support on all the rx queues. */
 		gna->mit = malloc(na->num_rx_rings * sizeof(struct nm_generic_mit),
 					M_DEVBUF, M_NOWAIT | M_ZERO);
 		if (!gna->mit) {
 			D("mitigation allocation failed");
 			error = ENOMEM;
 			goto out;
 		}
 		for (r=0; r<na->num_rx_rings; r++)
 			netmap_mitigation_init(&gna->mit[r], na);
 
 		/* Initialize the rx queue, as generic_rx_handler() can
 		 * be called as soon as netmap_catch_rx() returns.
 		 */
 		for (r=0; r<na->num_rx_rings; r++) {
 			mbq_safe_init(&na->rx_rings[r].rx_queue);
 		}
 
 		/*
 		 * Preallocate packet buffers for the tx rings.
 		 */
 		for (r=0; r<na->num_tx_rings; r++)
 			na->tx_rings[r].tx_pool = NULL;
 		for (r=0; r<na->num_tx_rings; r++) {
 			na->tx_rings[r].tx_pool = malloc(na->num_tx_desc * sizeof(struct mbuf *),
 					M_DEVBUF, M_NOWAIT | M_ZERO);
 			if (!na->tx_rings[r].tx_pool) {
 				D("tx_pool allocation failed");
 				error = ENOMEM;
 				goto free_tx_pools;
 			}
 			for (i=0; i<na->num_tx_desc; i++)
 				na->tx_rings[r].tx_pool[i] = NULL;
 			for (i=0; i<na->num_tx_desc; i++) {
 				m = netmap_get_mbuf(GENERIC_BUF_SIZE);
 				if (!m) {
 					D("tx_pool[%d] allocation failed", i);
 					error = ENOMEM;
 					goto free_tx_pools;
 				}
 				na->tx_rings[r].tx_pool[i] = m;
 			}
 		}
 		rtnl_lock();
 		/* Prepare to intercept incoming traffic. */
 		error = netmap_catch_rx(na, 1);
 		if (error) {
 			D("netdev_rx_handler_register() failed (%d)", error);
 			goto register_handler;
 		}
 		ifp->if_capenable |= IFCAP_NETMAP;
 
 		/* Make netmap control the packet steering. */
 		netmap_catch_tx(gna, 1);
 
 		rtnl_unlock();
 
 #ifdef RATE
 		if (rate_ctx.refcount == 0) {
 			D("setup_timer()");
 			memset(&rate_ctx, 0, sizeof(rate_ctx));
 			setup_timer(&rate_ctx.timer, &rate_callback, (unsigned long)&rate_ctx);
 			if (mod_timer(&rate_ctx.timer, jiffies + msecs_to_jiffies(1500))) {
 				D("Error: mod_timer()");
 			}
 		}
 		rate_ctx.refcount++;
 #endif /* RATE */
 
 	} else if (na->tx_rings[0].tx_pool) {
 		/* Disable netmap mode. We enter here only if the previous
 		   generic_netmap_register(na, 1) was successfull.
 		   If it was not, na->tx_rings[0].tx_pool was set to NULL by the
 		   error handling code below. */
 		rtnl_lock();
 
 		ifp->if_capenable &= ~IFCAP_NETMAP;
 
 		/* Release packet steering control. */
 		netmap_catch_tx(gna, 0);
 
 		/* Do not intercept packets on the rx path. */
 		netmap_catch_rx(na, 0);
 
 		rtnl_unlock();
 
 		/* Free the mbufs going to the netmap rings */
 		for (r=0; r<na->num_rx_rings; r++) {
 			mbq_safe_purge(&na->rx_rings[r].rx_queue);
 			mbq_safe_destroy(&na->rx_rings[r].rx_queue);
 		}
 
 		for (r=0; r<na->num_rx_rings; r++)
 			netmap_mitigation_cleanup(&gna->mit[r]);
 		free(gna->mit, M_DEVBUF);
 
 		for (r=0; r<na->num_tx_rings; r++) {
 			for (i=0; i<na->num_tx_desc; i++) {
 				m_freem(na->tx_rings[r].tx_pool[i]);
 			}
 			free(na->tx_rings[r].tx_pool, M_DEVBUF);
 		}
 
 #ifdef RATE
 		if (--rate_ctx.refcount == 0) {
 			D("del_timer()");
 			del_timer(&rate_ctx.timer);
 		}
 #endif
 	}
 
 #ifdef REG_RESET
 	error = ifp->netdev_ops->ndo_open(ifp);
 	if (error) {
 		goto free_tx_pools;
 	}
 #endif
 
 	return 0;
 
 register_handler:
 	rtnl_unlock();
 free_tx_pools:
 	for (r=0; r<na->num_tx_rings; r++) {
 		if (na->tx_rings[r].tx_pool == NULL)
 			continue;
 		for (i=0; i<na->num_tx_desc; i++)
 			if (na->tx_rings[r].tx_pool[i])
 				m_freem(na->tx_rings[r].tx_pool[i]);
 		free(na->tx_rings[r].tx_pool, M_DEVBUF);
 		na->tx_rings[r].tx_pool = NULL;
 	}
 	for (r=0; r<na->num_rx_rings; r++) {
 		netmap_mitigation_cleanup(&gna->mit[r]);
 		mbq_safe_destroy(&na->rx_rings[r].rx_queue);
 	}
 	free(gna->mit, M_DEVBUF);
 out:
 
 	return error;
 }
 
 /*
  * Callback invoked when the device driver frees an mbuf used
  * by netmap to transmit a packet. This usually happens when
  * the NIC notifies the driver that transmission is completed.
  */
 static void
 generic_mbuf_destructor(struct mbuf *m)
 {
 	netmap_generic_irq(MBUF_IFP(m), MBUF_TXQ(m), NULL);
 #ifdef __FreeBSD__
 	if (netmap_verbose)
 		RD(5, "Tx irq (%p) queue %d index %d" , m, MBUF_TXQ(m), (int)(uintptr_t)m->m_ext.ext_arg1);
 	netmap_default_mbuf_destructor(m);
 #endif /* __FreeBSD__ */
 	IFRATE(rate_ctx.new.txirq++);
 }
 
 /* Record completed transmissions and update hwtail.
  *
  * The oldest tx buffer not yet completed is at nr_hwtail + 1,
  * nr_hwcur is the first unsent buffer.
  */
 static u_int
 generic_netmap_tx_clean(struct netmap_kring *kring)
 {
 	u_int const lim = kring->nkr_num_slots - 1;
 	u_int nm_i = nm_next(kring->nr_hwtail, lim);
 	u_int hwcur = kring->nr_hwcur;
 	u_int n = 0;
 	struct mbuf **tx_pool = kring->tx_pool;
 
 	while (nm_i != hwcur) { /* buffers not completed */
 		struct mbuf *m = tx_pool[nm_i];
 
 		if (unlikely(m == NULL)) {
 			/* this is done, try to replenish the entry */
 			tx_pool[nm_i] = m = netmap_get_mbuf(GENERIC_BUF_SIZE);
 			if (unlikely(m == NULL)) {
 				D("mbuf allocation failed, XXX error");
 				// XXX how do we proceed ? break ?
 				return -ENOMEM;
 			}
 		} else if (GET_MBUF_REFCNT(m) != 1) {
 			break; /* This mbuf is still busy: its refcnt is 2. */
 		}
 		n++;
 		nm_i = nm_next(nm_i, lim);
 	}
 	kring->nr_hwtail = nm_prev(nm_i, lim);
 	ND("tx completed [%d] -> hwtail %d", n, kring->nr_hwtail);
 
 	return n;
 }
 
 
 /*
  * We have pending packets in the driver between nr_hwtail +1 and hwcur.
  * Compute a position in the middle, to be used to generate
  * a notification.
  */
 static inline u_int
 generic_tx_event_middle(struct netmap_kring *kring, u_int hwcur)
 {
 	u_int n = kring->nkr_num_slots;
 	u_int ntc = nm_next(kring->nr_hwtail, n-1);
 	u_int e;
 
 	if (hwcur >= ntc) {
 		e = (hwcur + ntc) / 2;
 	} else { /* wrap around */
 		e = (hwcur + n + ntc) / 2;
 		if (e >= n) {
 			e -= n;
 		}
 	}
 
 	if (unlikely(e >= n)) {
 		D("This cannot happen");
 		e = 0;
 	}
 
 	return e;
 }
 
 /*
  * We have pending packets in the driver between nr_hwtail+1 and hwcur.
  * Schedule a notification approximately in the middle of the two.
  * There is a race but this is only called within txsync which does
  * a double check.
  */
 static void
 generic_set_tx_event(struct netmap_kring *kring, u_int hwcur)
 {
 	struct mbuf *m;
 	u_int e;
 
 	if (nm_next(kring->nr_hwtail, kring->nkr_num_slots -1) == hwcur) {
 		return; /* all buffers are free */
 	}
 	e = generic_tx_event_middle(kring, hwcur);
 
 	m = kring->tx_pool[e];
 	ND(5, "Request Event at %d mbuf %p refcnt %d", e, m, m ? GET_MBUF_REFCNT(m) : -2 );
 	if (m == NULL) {
 		/* This can happen if there is already an event on the netmap
 		   slot 'e': There is nothing to do. */
 		return;
 	}
 	kring->tx_pool[e] = NULL;
 	SET_MBUF_DESTRUCTOR(m, generic_mbuf_destructor);
 
 	// XXX wmb() ?
 	/* Decrement the refcount an free it if we have the last one. */
 	m_freem(m);
 	smp_mb();
 }
 
 
 /*
  * generic_netmap_txsync() transforms netmap buffers into mbufs
  * and passes them to the standard device driver
  * (ndo_start_xmit() or ifp->if_transmit() ).
  * On linux this is not done directly, but using dev_queue_xmit(),
  * since it implements the TX flow control (and takes some locks).
  */
 static int
 generic_netmap_txsync(struct netmap_adapter *na, u_int ring_nr, int flags)
 {
 	struct ifnet *ifp = na->ifp;
 	struct netmap_kring *kring = &na->tx_rings[ring_nr];
 	struct netmap_ring *ring = kring->ring;
 	u_int nm_i;	/* index into the netmap ring */ // j
 	u_int const lim = kring->nkr_num_slots - 1;
 	u_int const head = kring->rhead;
 
 	IFRATE(rate_ctx.new.txsync++);
 
 	// TODO: handle the case of mbuf allocation failure
 
 	rmb();
 
 	/*
 	 * First part: process new packets to send.
 	 */
 	nm_i = kring->nr_hwcur;
 	if (nm_i != head) {	/* we have new packets to send */
 		while (nm_i != head) {
 			struct netmap_slot *slot = &ring->slot[nm_i];
 			u_int len = slot->len;
 			void *addr = NMB(slot);
 
 			/* device-specific */
 			struct mbuf *m;
 			int tx_ret;
 
 			NM_CHECK_ADDR_LEN(addr, len);
 
 			/* Tale a mbuf from the tx pool and copy in the user packet. */
 			m = kring->tx_pool[nm_i];
 			if (unlikely(!m)) {
 				RD(5, "This should never happen");
 				kring->tx_pool[nm_i] = m = netmap_get_mbuf(GENERIC_BUF_SIZE);
 				if (unlikely(m == NULL)) {
 					D("mbuf allocation failed");
 					break;
 				}
 			}
 			/* XXX we should ask notifications when NS_REPORT is set,
 			 * or roughly every half frame. We can optimize this
 			 * by lazily requesting notifications only when a
 			 * transmission fails. Probably the best way is to
 			 * break on failures and set notifications when
 			 * ring->cur == ring->tail || nm_i != cur
 			 */
 			tx_ret = generic_xmit_frame(ifp, m, addr, len, ring_nr);
 			if (unlikely(tx_ret)) {
 				RD(5, "start_xmit failed: err %d [nm_i %u, head %u, hwtail %u]",
 						tx_ret, nm_i, head, kring->nr_hwtail);
 				/*
 				 * No room for this mbuf in the device driver.
 				 * Request a notification FOR A PREVIOUS MBUF,
 				 * then call generic_netmap_tx_clean(kring) to do the
 				 * double check and see if we can free more buffers.
 				 * If there is space continue, else break;
 				 * NOTE: the double check is necessary if the problem
 				 * occurs in the txsync call after selrecord().
 				 * Also, we need some way to tell the caller that not
 				 * all buffers were queued onto the device (this was
 				 * not a problem with native netmap driver where space
 				 * is preallocated). The bridge has a similar problem
 				 * and we solve it there by dropping the excess packets.
 				 */
 				generic_set_tx_event(kring, nm_i);
 				if (generic_netmap_tx_clean(kring)) { /* space now available */
 					continue;
 				} else {
 					break;
 				}
 			}
 			slot->flags &= ~(NS_REPORT | NS_BUF_CHANGED);
 			nm_i = nm_next(nm_i, lim);
 			IFRATE(rate_ctx.new.txpkt ++);
 		}
 
 		/* Update hwcur to the next slot to transmit. */
 		kring->nr_hwcur = nm_i; /* not head, we could break early */
 	}
 
 	/*
 	 * Second, reclaim completed buffers
 	 */
 	if (flags & NAF_FORCE_RECLAIM || nm_kr_txempty(kring)) {
 		/* No more available slots? Set a notification event
 		 * on a netmap slot that will be cleaned in the future.
 		 * No doublecheck is performed, since txsync() will be
 		 * called twice by netmap_poll().
 		 */
 		generic_set_tx_event(kring, nm_i);
 	}
 	ND("tx #%d, hwtail = %d", n, kring->nr_hwtail);
 
 	generic_netmap_tx_clean(kring);
 
 	nm_txsync_finalize(kring);
 
 	return 0;
 }
 
 
 /*
  * This handler is registered (through netmap_catch_rx())
  * within the attached network interface
  * in the RX subsystem, so that every mbuf passed up by
  * the driver can be stolen to the network stack.
  * Stolen packets are put in a queue where the
  * generic_netmap_rxsync() callback can extract them.
  */
 void
 generic_rx_handler(struct ifnet *ifp, struct mbuf *m)
 {
 	struct netmap_adapter *na = NA(ifp);
 	struct netmap_generic_adapter *gna = (struct netmap_generic_adapter *)na;
 	u_int work_done;
 	u_int rr = MBUF_RXQ(m); // receive ring number
 
 	if (rr >= na->num_rx_rings) {
 		rr = rr % na->num_rx_rings; // XXX expensive...
 	}
 
 	/* limit the size of the queue */
 	if (unlikely(mbq_len(&na->rx_rings[rr].rx_queue) > 1024)) {
 		m_freem(m);
 	} else {
 		mbq_safe_enqueue(&na->rx_rings[rr].rx_queue, m);
 	}
 
 	if (netmap_generic_mit < 32768) {
 		/* no rx mitigation, pass notification up */
 		netmap_generic_irq(na->ifp, rr, &work_done);
 		IFRATE(rate_ctx.new.rxirq++);
 	} else {
 		/* same as send combining, filter notification if there is a
 		 * pending timer, otherwise pass it up and start a timer.
 		 */
 		if (likely(netmap_mitigation_active(&gna->mit[rr]))) {
 			/* Record that there is some pending work. */
 			gna->mit[rr].mit_pending = 1;
 		} else {
 			netmap_generic_irq(na->ifp, rr, &work_done);
 			IFRATE(rate_ctx.new.rxirq++);
 			netmap_mitigation_start(&gna->mit[rr]);
 		}
 	}
 }
 
 /*
  * generic_netmap_rxsync() extracts mbufs from the queue filled by
  * generic_netmap_rx_handler() and puts their content in the netmap
  * receive ring.
  * Access must be protected because the rx handler is asynchronous,
  */
 static int
 generic_netmap_rxsync(struct netmap_adapter *na, u_int ring_nr, int flags)
 {
 	struct netmap_kring *kring = &na->rx_rings[ring_nr];
 	struct netmap_ring *ring = kring->ring;
 	u_int nm_i;	/* index into the netmap ring */ //j,
 	u_int n;
 	u_int const lim = kring->nkr_num_slots - 1;
 	u_int const head = nm_rxsync_prologue(kring);
 	int force_update = (flags & NAF_FORCE_READ) || kring->nr_kflags & NKR_PENDINTR;
 
 	if (head > lim)
 		return netmap_ring_reinit(kring);
 
 	/*
 	 * First part: import newly received packets.
 	 */
 	if (netmap_no_pendintr || force_update) {
 		/* extract buffers from the rx queue, stop at most one
 		 * slot before nr_hwcur (stop_i)
 		 */
 		uint16_t slot_flags = kring->nkr_slot_flags;
 		u_int stop_i = nm_prev(kring->nr_hwcur, lim);
 
 		nm_i = kring->nr_hwtail; /* first empty slot in the receive ring */
 		for (n = 0; nm_i != stop_i; n++) {
 			int len;
 			void *addr = NMB(&ring->slot[nm_i]);
 			struct mbuf *m;
 
 			/* we only check the address here on generic rx rings */
 			if (addr == netmap_buffer_base) { /* Bad buffer */
 				return netmap_ring_reinit(kring);
 			}
 			/*
 			 * Call the locked version of the function.
 			 * XXX Ideally we could grab a batch of mbufs at once
 			 * and save some locking overhead.
 			 */
 			m = mbq_safe_dequeue(&kring->rx_queue);
 			if (!m)	/* no more data */
 				break;
 			len = MBUF_LEN(m);
 			m_copydata(m, 0, len, addr);
 			ring->slot[nm_i].len = len;
 			ring->slot[nm_i].flags = slot_flags;
 			m_freem(m);
 			nm_i = nm_next(nm_i, lim);
 		}
 		if (n) {
 			kring->nr_hwtail = nm_i;
 			IFRATE(rate_ctx.new.rxpkt += n);
 		}
 		kring->nr_kflags &= ~NKR_PENDINTR;
 	}
 
 	// XXX should we invert the order ?
 	/*
 	 * Second part: skip past packets that userspace has released.
 	 */
 	nm_i = kring->nr_hwcur;
 	if (nm_i != head) {
 		/* Userspace has released some packets. */
 		for (n = 0; nm_i != head; n++) {
 			struct netmap_slot *slot = &ring->slot[nm_i];
 
 			slot->flags &= ~NS_BUF_CHANGED;
 			nm_i = nm_next(nm_i, lim);
 		}
 		kring->nr_hwcur = head;
 	}
 	/* tell userspace that there might be new packets. */
 	nm_rxsync_finalize(kring);
 	IFRATE(rate_ctx.new.rxsync++);
 
 	return 0;
 }
 
 static void
 generic_netmap_dtor(struct netmap_adapter *na)
 {
 	struct ifnet *ifp = na->ifp;
 	struct netmap_generic_adapter *gna = (struct netmap_generic_adapter*)na;
 	struct netmap_adapter *prev_na = gna->prev;
 
 	if (prev_na != NULL) {
 		D("Released generic NA %p", gna);
 		if_rele(na->ifp);
 		netmap_adapter_put(prev_na);
 	}
 	if (ifp != NULL) {
 		WNA(ifp) = prev_na;
 		D("Restored native NA %p", prev_na);
 		na->ifp = NULL;
 	}
 }
 
 /*
  * generic_netmap_attach() makes it possible to use netmap on
  * a device without native netmap support.
  * This is less performant than native support but potentially
  * faster than raw sockets or similar schemes.
  *
  * In this "emulated" mode, netmap rings do not necessarily
  * have the same size as those in the NIC. We use a default
  * value and possibly override it if the OS has ways to fetch the
  * actual configuration.
  */
 int
 generic_netmap_attach(struct ifnet *ifp)
 {
 	struct netmap_adapter *na;
 	struct netmap_generic_adapter *gna;
 	int retval;
 	u_int num_tx_desc, num_rx_desc;
 
 	num_tx_desc = num_rx_desc = netmap_generic_ringsize; /* starting point */
 
 	generic_find_num_desc(ifp, &num_tx_desc, &num_rx_desc);
 	ND("Netmap ring size: TX = %d, RX = %d", num_tx_desc, num_rx_desc);
 	if (num_tx_desc == 0 || num_rx_desc == 0) {
 		D("Device has no hw slots (tx %u, rx %u)", num_tx_desc, num_rx_desc);
 		return EINVAL;
 	}
 
 	gna = malloc(sizeof(*gna), M_DEVBUF, M_NOWAIT | M_ZERO);
 	if (gna == NULL) {
 		D("no memory on attach, give up");
 		return ENOMEM;
 	}
 	na = (struct netmap_adapter *)gna;
 	na->ifp = ifp;
 	na->num_tx_desc = num_tx_desc;
 	na->num_rx_desc = num_rx_desc;
 	na->nm_register = &generic_netmap_register;
 	na->nm_txsync = &generic_netmap_txsync;
 	na->nm_rxsync = &generic_netmap_rxsync;
 	na->nm_dtor = &generic_netmap_dtor;
 	/* when using generic, IFCAP_NETMAP is set so we force
 	 * NAF_SKIP_INTR to use the regular interrupt handler
 	 */
 	na->na_flags = NAF_SKIP_INTR | NAF_HOST_RINGS;
 
 	ND("[GNA] num_tx_queues(%d), real_num_tx_queues(%d), len(%lu)",
 			ifp->num_tx_queues, ifp->real_num_tx_queues,
 			ifp->tx_queue_len);
 	ND("[GNA] num_rx_queues(%d), real_num_rx_queues(%d)",
 			ifp->num_rx_queues, ifp->real_num_rx_queues);
 
 	generic_find_num_queues(ifp, &na->num_tx_rings, &na->num_rx_rings);
 
 	retval = netmap_attach_common(na);
 	if (retval) {
 		free(gna, M_DEVBUF);
 	}
 
 	return retval;
 }
Index: projects/release-debugdist/sys/dev/nvme/nvme_ctrlr_cmd.c
===================================================================
--- projects/release-debugdist/sys/dev/nvme/nvme_ctrlr_cmd.c	(revision 267343)
+++ projects/release-debugdist/sys/dev/nvme/nvme_ctrlr_cmd.c	(revision 267344)
@@ -1,325 +1,325 @@
 /*-
  * Copyright (C) 2012-2013 Intel Corporation
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include "nvme_private.h"
 
 void
 nvme_ctrlr_cmd_identify_controller(struct nvme_controller *ctrlr, void *payload,
 	nvme_cb_fn_t cb_fn, void *cb_arg)
 {
 	struct nvme_request *req;
 	struct nvme_command *cmd;
 
 	req = nvme_allocate_request_vaddr(payload,
 	    sizeof(struct nvme_controller_data), cb_fn, cb_arg);
 
 	cmd = &req->cmd;
 	cmd->opc = NVME_OPC_IDENTIFY;
 
 	/*
 	 * TODO: create an identify command data structure, which
 	 *  includes this CNS bit in cdw10.
 	 */
 	cmd->cdw10 = 1;
 
 	nvme_ctrlr_submit_admin_request(ctrlr, req);
 }
 
 void
 nvme_ctrlr_cmd_identify_namespace(struct nvme_controller *ctrlr, uint16_t nsid,
 	void *payload, nvme_cb_fn_t cb_fn, void *cb_arg)
 {
 	struct nvme_request *req;
 	struct nvme_command *cmd;
 
 	req = nvme_allocate_request_vaddr(payload,
 	    sizeof(struct nvme_namespace_data), cb_fn, cb_arg);
 
 	cmd = &req->cmd;
 	cmd->opc = NVME_OPC_IDENTIFY;
 
 	/*
 	 * TODO: create an identify command data structure
 	 */
 	cmd->nsid = nsid;
 
 	nvme_ctrlr_submit_admin_request(ctrlr, req);
 }
 
 void
 nvme_ctrlr_cmd_create_io_cq(struct nvme_controller *ctrlr,
     struct nvme_qpair *io_que, uint16_t vector, nvme_cb_fn_t cb_fn,
     void *cb_arg)
 {
 	struct nvme_request *req;
 	struct nvme_command *cmd;
 
 	req = nvme_allocate_request_null(cb_fn, cb_arg);
 
 	cmd = &req->cmd;
 	cmd->opc = NVME_OPC_CREATE_IO_CQ;
 
 	/*
 	 * TODO: create a create io completion queue command data
 	 *  structure.
 	 */
 	cmd->cdw10 = ((io_que->num_entries-1) << 16) | io_que->id;
 	/* 0x3 = interrupts enabled | physically contiguous */
 	cmd->cdw11 = (vector << 16) | 0x3;
 	cmd->prp1 = io_que->cpl_bus_addr;
 
 	nvme_ctrlr_submit_admin_request(ctrlr, req);
 }
 
 void
 nvme_ctrlr_cmd_create_io_sq(struct nvme_controller *ctrlr,
     struct nvme_qpair *io_que, nvme_cb_fn_t cb_fn, void *cb_arg)
 {
 	struct nvme_request *req;
 	struct nvme_command *cmd;
 
 	req = nvme_allocate_request_null(cb_fn, cb_arg);
 
 	cmd = &req->cmd;
 	cmd->opc = NVME_OPC_CREATE_IO_SQ;
 
 	/*
 	 * TODO: create a create io submission queue command data
 	 *  structure.
 	 */
 	cmd->cdw10 = ((io_que->num_entries-1) << 16) | io_que->id;
 	/* 0x1 = physically contiguous */
 	cmd->cdw11 = (io_que->id << 16) | 0x1;
 	cmd->prp1 = io_que->cmd_bus_addr;
 
 	nvme_ctrlr_submit_admin_request(ctrlr, req);
 }
 
 void
 nvme_ctrlr_cmd_delete_io_cq(struct nvme_controller *ctrlr,
     struct nvme_qpair *io_que, nvme_cb_fn_t cb_fn, void *cb_arg)
 {
 	struct nvme_request *req;
 	struct nvme_command *cmd;
 
 	req = nvme_allocate_request_null(cb_fn, cb_arg);
 
 	cmd = &req->cmd;
 	cmd->opc = NVME_OPC_DELETE_IO_CQ;
 
 	/*
 	 * TODO: create a delete io completion queue command data
 	 *  structure.
 	 */
 	cmd->cdw10 = io_que->id;
 
 	nvme_ctrlr_submit_admin_request(ctrlr, req);
 }
 
 void
 nvme_ctrlr_cmd_delete_io_sq(struct nvme_controller *ctrlr,
     struct nvme_qpair *io_que, nvme_cb_fn_t cb_fn, void *cb_arg)
 {
 	struct nvme_request *req;
 	struct nvme_command *cmd;
 
 	req = nvme_allocate_request_null(cb_fn, cb_arg);
 
 	cmd = &req->cmd;
 	cmd->opc = NVME_OPC_DELETE_IO_SQ;
 
 	/*
 	 * TODO: create a delete io submission queue command data
 	 *  structure.
 	 */
 	cmd->cdw10 = io_que->id;
 
 	nvme_ctrlr_submit_admin_request(ctrlr, req);
 }
 
 void
 nvme_ctrlr_cmd_set_feature(struct nvme_controller *ctrlr, uint8_t feature,
     uint32_t cdw11, void *payload, uint32_t payload_size,
     nvme_cb_fn_t cb_fn, void *cb_arg)
 {
 	struct nvme_request *req;
 	struct nvme_command *cmd;
 
 	req = nvme_allocate_request_null(cb_fn, cb_arg);
 
 	cmd = &req->cmd;
 	cmd->opc = NVME_OPC_SET_FEATURES;
 	cmd->cdw10 = feature;
 	cmd->cdw11 = cdw11;
 
 	nvme_ctrlr_submit_admin_request(ctrlr, req);
 }
 
 void
 nvme_ctrlr_cmd_get_feature(struct nvme_controller *ctrlr, uint8_t feature,
     uint32_t cdw11, void *payload, uint32_t payload_size,
     nvme_cb_fn_t cb_fn, void *cb_arg)
 {
 	struct nvme_request *req;
 	struct nvme_command *cmd;
 
 	req = nvme_allocate_request_null(cb_fn, cb_arg);
 
 	cmd = &req->cmd;
 	cmd->opc = NVME_OPC_GET_FEATURES;
 	cmd->cdw10 = feature;
 	cmd->cdw11 = cdw11;
 
 	nvme_ctrlr_submit_admin_request(ctrlr, req);
 }
 
 void
 nvme_ctrlr_cmd_set_num_queues(struct nvme_controller *ctrlr,
     uint32_t num_queues, nvme_cb_fn_t cb_fn, void *cb_arg)
 {
 	uint32_t cdw11;
 
-	cdw11 = ((num_queues - 1) << 16) || (num_queues - 1);
+	cdw11 = ((num_queues - 1) << 16) | (num_queues - 1);
 	nvme_ctrlr_cmd_set_feature(ctrlr, NVME_FEAT_NUMBER_OF_QUEUES, cdw11,
 	    NULL, 0, cb_fn, cb_arg);
 }
 
 void
 nvme_ctrlr_cmd_set_async_event_config(struct nvme_controller *ctrlr,
     union nvme_critical_warning_state state, nvme_cb_fn_t cb_fn,
     void *cb_arg)
 {
 	uint32_t cdw11;
 
 	cdw11 = state.raw;
 	nvme_ctrlr_cmd_set_feature(ctrlr,
 	    NVME_FEAT_ASYNC_EVENT_CONFIGURATION, cdw11, NULL, 0, cb_fn,
 	    cb_arg);
 }
 
 void
 nvme_ctrlr_cmd_set_interrupt_coalescing(struct nvme_controller *ctrlr,
     uint32_t microseconds, uint32_t threshold, nvme_cb_fn_t cb_fn, void *cb_arg)
 {
 	uint32_t cdw11;
 
 	if ((microseconds/100) >= 0x100) {
 		nvme_printf(ctrlr, "invalid coal time %d, disabling\n",
 		    microseconds);
 		microseconds = 0;
 		threshold = 0;
 	}
 
 	if (threshold >= 0x100) {
 		nvme_printf(ctrlr, "invalid threshold %d, disabling\n",
 		    threshold);
 		threshold = 0;
 		microseconds = 0;
 	}
 
 	cdw11 = ((microseconds/100) << 8) | threshold;
 	nvme_ctrlr_cmd_set_feature(ctrlr, NVME_FEAT_INTERRUPT_COALESCING, cdw11,
 	    NULL, 0, cb_fn, cb_arg);
 }
 
 void
 nvme_ctrlr_cmd_get_log_page(struct nvme_controller *ctrlr, uint8_t log_page,
     uint32_t nsid, void *payload, uint32_t payload_size, nvme_cb_fn_t cb_fn,
     void *cb_arg)
 {
 	struct nvme_request *req;
 	struct nvme_command *cmd;
 
 	req = nvme_allocate_request_vaddr(payload, payload_size, cb_fn, cb_arg);
 
 	cmd = &req->cmd;
 	cmd->opc = NVME_OPC_GET_LOG_PAGE;
 	cmd->nsid = nsid;
 	cmd->cdw10 = ((payload_size/sizeof(uint32_t)) - 1) << 16;
 	cmd->cdw10 |= log_page;
 
 	nvme_ctrlr_submit_admin_request(ctrlr, req);
 }
 
 void
 nvme_ctrlr_cmd_get_error_page(struct nvme_controller *ctrlr,
     struct nvme_error_information_entry *payload, uint32_t num_entries,
     nvme_cb_fn_t cb_fn, void *cb_arg)
 {
 
 	KASSERT(num_entries > 0, ("%s called with num_entries==0\n", __func__));
 
 	/* Controller's error log page entries is 0-based. */
 	KASSERT(num_entries <= (ctrlr->cdata.elpe + 1),
 	    ("%s called with num_entries=%d but (elpe+1)=%d\n", __func__,
 	    num_entries, ctrlr->cdata.elpe + 1));
 
 	if (num_entries > (ctrlr->cdata.elpe + 1))
 		num_entries = ctrlr->cdata.elpe + 1;
 
 	nvme_ctrlr_cmd_get_log_page(ctrlr, NVME_LOG_ERROR,
 	    NVME_GLOBAL_NAMESPACE_TAG, payload, sizeof(*payload) * num_entries,
 	    cb_fn, cb_arg);
 }
 
 void
 nvme_ctrlr_cmd_get_health_information_page(struct nvme_controller *ctrlr,
     uint32_t nsid, struct nvme_health_information_page *payload,
     nvme_cb_fn_t cb_fn, void *cb_arg)
 {
 
 	nvme_ctrlr_cmd_get_log_page(ctrlr, NVME_LOG_HEALTH_INFORMATION,
 	    nsid, payload, sizeof(*payload), cb_fn, cb_arg);
 }
 
 void
 nvme_ctrlr_cmd_get_firmware_page(struct nvme_controller *ctrlr,
     struct nvme_firmware_page *payload, nvme_cb_fn_t cb_fn, void *cb_arg)
 {
 
 	nvme_ctrlr_cmd_get_log_page(ctrlr, NVME_LOG_FIRMWARE_SLOT, 
 	    NVME_GLOBAL_NAMESPACE_TAG, payload, sizeof(*payload), cb_fn,
 	    cb_arg);
 }
 
 void
 nvme_ctrlr_cmd_abort(struct nvme_controller *ctrlr, uint16_t cid,
     uint16_t sqid, nvme_cb_fn_t cb_fn, void *cb_arg)
 {
 	struct nvme_request *req;
 	struct nvme_command *cmd;
 
 	req = nvme_allocate_request_null(cb_fn, cb_arg);
 
 	cmd = &req->cmd;
 	cmd->opc = NVME_OPC_ABORT;
 	cmd->cdw10 = (cid << 16) | sqid;
 
 	nvme_ctrlr_submit_admin_request(ctrlr, req);
 }
Index: projects/release-debugdist/sys/dev/safe/safe.c
===================================================================
--- projects/release-debugdist/sys/dev/safe/safe.c	(revision 267343)
+++ projects/release-debugdist/sys/dev/safe/safe.c	(revision 267344)
@@ -1,2243 +1,2232 @@
 /*-
  * Copyright (c) 2003 Sam Leffler, Errno Consulting
  * Copyright (c) 2003 Global Technology Associates, Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 /*
  * SafeNet SafeXcel-1141 hardware crypto accelerator
  */
 #include "opt_safe.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/proc.h>
 #include <sys/errno.h>
 #include <sys/malloc.h>
 #include <sys/kernel.h>
 #include <sys/mbuf.h>
 #include <sys/module.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <sys/sysctl.h>
 #include <sys/endian.h>
 
 #include <vm/vm.h>
 #include <vm/pmap.h>
 
 #include <machine/bus.h>
 #include <machine/resource.h>
 #include <sys/bus.h>
 #include <sys/rman.h>
 
 #include <crypto/sha1.h>
 #include <opencrypto/cryptodev.h>
 #include <opencrypto/cryptosoft.h>
 #include <sys/md5.h>
 #include <sys/random.h>
 #include <sys/kobj.h>
 
 #include "cryptodev_if.h"
 
 #include <dev/pci/pcivar.h>
 #include <dev/pci/pcireg.h>
 
 #ifdef SAFE_RNDTEST
 #include <dev/rndtest/rndtest.h>
 #endif
 #include <dev/safe/safereg.h>
 #include <dev/safe/safevar.h>
 
 #ifndef bswap32
 #define	bswap32	NTOHL
 #endif
 
 /*
  * Prototypes and count for the pci_device structure
  */
 static	int safe_probe(device_t);
 static	int safe_attach(device_t);
 static	int safe_detach(device_t);
 static	int safe_suspend(device_t);
 static	int safe_resume(device_t);
 static	int safe_shutdown(device_t);
 
 static	int safe_newsession(device_t, u_int32_t *, struct cryptoini *);
 static	int safe_freesession(device_t, u_int64_t);
 static	int safe_process(device_t, struct cryptop *, int);
 
 static device_method_t safe_methods[] = {
 	/* Device interface */
 	DEVMETHOD(device_probe,		safe_probe),
 	DEVMETHOD(device_attach,	safe_attach),
 	DEVMETHOD(device_detach,	safe_detach),
 	DEVMETHOD(device_suspend,	safe_suspend),
 	DEVMETHOD(device_resume,	safe_resume),
 	DEVMETHOD(device_shutdown,	safe_shutdown),
 
 	/* crypto device methods */
 	DEVMETHOD(cryptodev_newsession,	safe_newsession),
 	DEVMETHOD(cryptodev_freesession,safe_freesession),
 	DEVMETHOD(cryptodev_process,	safe_process),
 
 	DEVMETHOD_END
 };
 static driver_t safe_driver = {
 	"safe",
 	safe_methods,
 	sizeof (struct safe_softc)
 };
 static devclass_t safe_devclass;
 
 DRIVER_MODULE(safe, pci, safe_driver, safe_devclass, 0, 0);
 MODULE_DEPEND(safe, crypto, 1, 1, 1);
 #ifdef SAFE_RNDTEST
 MODULE_DEPEND(safe, rndtest, 1, 1, 1);
 #endif
 
 static	void safe_intr(void *);
 static	void safe_callback(struct safe_softc *, struct safe_ringentry *);
 static	void safe_feed(struct safe_softc *, struct safe_ringentry *);
 static	void safe_mcopy(struct mbuf *, struct mbuf *, u_int);
 #ifndef SAFE_NO_RNG
 static	void safe_rng_init(struct safe_softc *);
 static	void safe_rng(void *);
 #endif /* SAFE_NO_RNG */
 static	int safe_dma_malloc(struct safe_softc *, bus_size_t,
 	        struct safe_dma_alloc *, int);
 #define	safe_dma_sync(_dma, _flags) \
 	bus_dmamap_sync((_dma)->dma_tag, (_dma)->dma_map, (_flags))
 static	void safe_dma_free(struct safe_softc *, struct safe_dma_alloc *);
 static	int safe_dmamap_aligned(const struct safe_operand *);
 static	int safe_dmamap_uniform(const struct safe_operand *);
 
 static	void safe_reset_board(struct safe_softc *);
 static	void safe_init_board(struct safe_softc *);
 static	void safe_init_pciregs(device_t dev);
 static	void safe_cleanchip(struct safe_softc *);
 static	void safe_totalreset(struct safe_softc *);
 
 static	int safe_free_entry(struct safe_softc *, struct safe_ringentry *);
 
 static SYSCTL_NODE(_hw, OID_AUTO, safe, CTLFLAG_RD, 0,
     "SafeNet driver parameters");
 
 #ifdef SAFE_DEBUG
 static	void safe_dump_dmastatus(struct safe_softc *, const char *);
 static	void safe_dump_ringstate(struct safe_softc *, const char *);
 static	void safe_dump_intrstate(struct safe_softc *, const char *);
 static	void safe_dump_request(struct safe_softc *, const char *,
 		struct safe_ringentry *);
 
 static	struct safe_softc *safec;		/* for use by hw.safe.dump */
 
 static	int safe_debug = 0;
 SYSCTL_INT(_hw_safe, OID_AUTO, debug, CTLFLAG_RW, &safe_debug,
 	    0, "control debugging msgs");
 #define	DPRINTF(_x)	if (safe_debug) printf _x
 #else
 #define	DPRINTF(_x)
 #endif
 
 #define	READ_REG(sc,r) \
 	bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r))
 
 #define WRITE_REG(sc,reg,val) \
 	bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val)
 
 struct safe_stats safestats;
 SYSCTL_STRUCT(_hw_safe, OID_AUTO, stats, CTLFLAG_RD, &safestats,
 	    safe_stats, "driver statistics");
 #ifndef SAFE_NO_RNG
 static	int safe_rnginterval = 1;		/* poll once a second */
 SYSCTL_INT(_hw_safe, OID_AUTO, rnginterval, CTLFLAG_RW, &safe_rnginterval,
 	    0, "RNG polling interval (secs)");
 static	int safe_rngbufsize = 16;		/* 64 bytes each poll  */
 SYSCTL_INT(_hw_safe, OID_AUTO, rngbufsize, CTLFLAG_RW, &safe_rngbufsize,
 	    0, "RNG polling buffer size (32-bit words)");
 static	int safe_rngmaxalarm = 8;		/* max alarms before reset */
 SYSCTL_INT(_hw_safe, OID_AUTO, rngmaxalarm, CTLFLAG_RW, &safe_rngmaxalarm,
 	    0, "RNG max alarms before reset");
 #endif /* SAFE_NO_RNG */
 
 static int
 safe_probe(device_t dev)
 {
 	if (pci_get_vendor(dev) == PCI_VENDOR_SAFENET &&
 	    pci_get_device(dev) == PCI_PRODUCT_SAFEXCEL)
 		return (BUS_PROBE_DEFAULT);
 	return (ENXIO);
 }
 
 static const char*
 safe_partname(struct safe_softc *sc)
 {
 	/* XXX sprintf numbers when not decoded */
 	switch (pci_get_vendor(sc->sc_dev)) {
 	case PCI_VENDOR_SAFENET:
 		switch (pci_get_device(sc->sc_dev)) {
 		case PCI_PRODUCT_SAFEXCEL: return "SafeNet SafeXcel-1141";
 		}
 		return "SafeNet unknown-part";
 	}
 	return "Unknown-vendor unknown-part";
 }
 
 #ifndef SAFE_NO_RNG
 static void
 default_harvest(struct rndtest_state *rsp, void *buf, u_int count)
 {
 	random_harvest(buf, count, count*NBBY/2, RANDOM_PURE_SAFE);
 }
 #endif /* SAFE_NO_RNG */
 
 static int
 safe_attach(device_t dev)
 {
 	struct safe_softc *sc = device_get_softc(dev);
 	u_int32_t raddr;
 	u_int32_t i, devinfo;
 	int rid;
 
 	bzero(sc, sizeof (*sc));
 	sc->sc_dev = dev;
 
 	/* XXX handle power management */
 
 	pci_enable_busmaster(dev);
 
 	/* 
 	 * Setup memory-mapping of PCI registers.
 	 */
 	rid = BS_BAR;
 	sc->sc_sr = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 					   RF_ACTIVE);
 	if (sc->sc_sr == NULL) {
 		device_printf(dev, "cannot map register space\n");
 		goto bad;
 	}
 	sc->sc_st = rman_get_bustag(sc->sc_sr);
 	sc->sc_sh = rman_get_bushandle(sc->sc_sr);
 
 	/*
 	 * Arrange interrupt line.
 	 */
 	rid = 0;
 	sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
 					    RF_SHAREABLE|RF_ACTIVE);
 	if (sc->sc_irq == NULL) {
 		device_printf(dev, "could not map interrupt\n");
 		goto bad1;
 	}
 	/*
 	 * NB: Network code assumes we are blocked with splimp()
 	 *     so make sure the IRQ is mapped appropriately.
 	 */
 	if (bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET | INTR_MPSAFE,
 			   NULL, safe_intr, sc, &sc->sc_ih)) {
 		device_printf(dev, "could not establish interrupt\n");
 		goto bad2;
 	}
 
 	sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE);
 	if (sc->sc_cid < 0) {
 		device_printf(dev, "could not get crypto driver id\n");
 		goto bad3;
 	}
 
 	sc->sc_chiprev = READ_REG(sc, SAFE_DEVINFO) &
 		(SAFE_DEVINFO_REV_MAJ | SAFE_DEVINFO_REV_MIN);
 
 	/*
 	 * Setup DMA descriptor area.
 	 */
 	if (bus_dma_tag_create(bus_get_dma_tag(dev),	/* parent */
 			       1,			/* alignment */
 			       SAFE_DMA_BOUNDARY,	/* boundary */
 			       BUS_SPACE_MAXADDR_32BIT,	/* lowaddr */
 			       BUS_SPACE_MAXADDR,	/* highaddr */
 			       NULL, NULL,		/* filter, filterarg */
 			       SAFE_MAX_DMA,		/* maxsize */
 			       SAFE_MAX_PART,		/* nsegments */
 			       SAFE_MAX_SSIZE,		/* maxsegsize */
 			       BUS_DMA_ALLOCNOW,	/* flags */
 			       NULL, NULL,		/* locking */
 			       &sc->sc_srcdmat)) {
 		device_printf(dev, "cannot allocate DMA tag\n");
 		goto bad4;
 	}
 	if (bus_dma_tag_create(bus_get_dma_tag(dev),	/* parent */
 			       1,			/* alignment */
 			       SAFE_MAX_DSIZE,		/* boundary */
 			       BUS_SPACE_MAXADDR_32BIT,	/* lowaddr */
 			       BUS_SPACE_MAXADDR,	/* highaddr */
 			       NULL, NULL,		/* filter, filterarg */
 			       SAFE_MAX_DMA,		/* maxsize */
 			       SAFE_MAX_PART,		/* nsegments */
 			       SAFE_MAX_DSIZE,		/* maxsegsize */
 			       BUS_DMA_ALLOCNOW,	/* flags */
 			       NULL, NULL,		/* locking */
 			       &sc->sc_dstdmat)) {
 		device_printf(dev, "cannot allocate DMA tag\n");
 		goto bad4;
 	}
 
 	/*
 	 * Allocate packet engine descriptors.
 	 */
 	if (safe_dma_malloc(sc,
 	    SAFE_MAX_NQUEUE * sizeof (struct safe_ringentry),
 	    &sc->sc_ringalloc, 0)) {
 		device_printf(dev, "cannot allocate PE descriptor ring\n");
 		bus_dma_tag_destroy(sc->sc_srcdmat);
 		goto bad4;
 	}
 	/*
 	 * Hookup the static portion of all our data structures.
 	 */
 	sc->sc_ring = (struct safe_ringentry *) sc->sc_ringalloc.dma_vaddr;
 	sc->sc_ringtop = sc->sc_ring + SAFE_MAX_NQUEUE;
 	sc->sc_front = sc->sc_ring;
 	sc->sc_back = sc->sc_ring;
 	raddr = sc->sc_ringalloc.dma_paddr;
 	bzero(sc->sc_ring, SAFE_MAX_NQUEUE * sizeof(struct safe_ringentry));
 	for (i = 0; i < SAFE_MAX_NQUEUE; i++) {
 		struct safe_ringentry *re = &sc->sc_ring[i];
 
 		re->re_desc.d_sa = raddr +
 			offsetof(struct safe_ringentry, re_sa);
 		re->re_sa.sa_staterec = raddr +
 			offsetof(struct safe_ringentry, re_sastate);
 
 		raddr += sizeof (struct safe_ringentry);
 	}
 	mtx_init(&sc->sc_ringmtx, device_get_nameunit(dev),
 		"packet engine ring", MTX_DEF);
 
 	/*
 	 * Allocate scatter and gather particle descriptors.
 	 */
 	if (safe_dma_malloc(sc, SAFE_TOTAL_SPART * sizeof (struct safe_pdesc),
 	    &sc->sc_spalloc, 0)) {
 		device_printf(dev, "cannot allocate source particle "
 			"descriptor ring\n");
 		mtx_destroy(&sc->sc_ringmtx);
 		safe_dma_free(sc, &sc->sc_ringalloc);
 		bus_dma_tag_destroy(sc->sc_srcdmat);
 		goto bad4;
 	}
 	sc->sc_spring = (struct safe_pdesc *) sc->sc_spalloc.dma_vaddr;
 	sc->sc_springtop = sc->sc_spring + SAFE_TOTAL_SPART;
 	sc->sc_spfree = sc->sc_spring;
 	bzero(sc->sc_spring, SAFE_TOTAL_SPART * sizeof(struct safe_pdesc));
 
 	if (safe_dma_malloc(sc, SAFE_TOTAL_DPART * sizeof (struct safe_pdesc),
 	    &sc->sc_dpalloc, 0)) {
 		device_printf(dev, "cannot allocate destination particle "
 			"descriptor ring\n");
 		mtx_destroy(&sc->sc_ringmtx);
 		safe_dma_free(sc, &sc->sc_spalloc);
 		safe_dma_free(sc, &sc->sc_ringalloc);
 		bus_dma_tag_destroy(sc->sc_dstdmat);
 		goto bad4;
 	}
 	sc->sc_dpring = (struct safe_pdesc *) sc->sc_dpalloc.dma_vaddr;
 	sc->sc_dpringtop = sc->sc_dpring + SAFE_TOTAL_DPART;
 	sc->sc_dpfree = sc->sc_dpring;
 	bzero(sc->sc_dpring, SAFE_TOTAL_DPART * sizeof(struct safe_pdesc));
 
 	device_printf(sc->sc_dev, "%s", safe_partname(sc));
 
 	devinfo = READ_REG(sc, SAFE_DEVINFO);
 	if (devinfo & SAFE_DEVINFO_RNG) {
 		sc->sc_flags |= SAFE_FLAGS_RNG;
 		printf(" rng");
 	}
 	if (devinfo & SAFE_DEVINFO_PKEY) {
 #if 0
 		printf(" key");
 		sc->sc_flags |= SAFE_FLAGS_KEY;
 		crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0);
 		crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0);
 #endif
 	}
 	if (devinfo & SAFE_DEVINFO_DES) {
 		printf(" des/3des");
 		crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
 		crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
 	}
 	if (devinfo & SAFE_DEVINFO_AES) {
 		printf(" aes");
 		crypto_register(sc->sc_cid, CRYPTO_AES_CBC, 0, 0);
 	}
 	if (devinfo & SAFE_DEVINFO_MD5) {
 		printf(" md5");
 		crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
 	}
 	if (devinfo & SAFE_DEVINFO_SHA1) {
 		printf(" sha1");
 		crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
 	}
 	printf(" null");
 	crypto_register(sc->sc_cid, CRYPTO_NULL_CBC, 0, 0);
 	crypto_register(sc->sc_cid, CRYPTO_NULL_HMAC, 0, 0);
 	/* XXX other supported algorithms */
 	printf("\n");
 
 	safe_reset_board(sc);		/* reset h/w */
 	safe_init_pciregs(dev);		/* init pci settings */
 	safe_init_board(sc);		/* init h/w */
 
 #ifndef SAFE_NO_RNG
 	if (sc->sc_flags & SAFE_FLAGS_RNG) {
 #ifdef SAFE_RNDTEST
 		sc->sc_rndtest = rndtest_attach(dev);
 		if (sc->sc_rndtest)
 			sc->sc_harvest = rndtest_harvest;
 		else
 			sc->sc_harvest = default_harvest;
 #else
 		sc->sc_harvest = default_harvest;
 #endif
 		safe_rng_init(sc);
 
 		callout_init(&sc->sc_rngto, CALLOUT_MPSAFE);
 		callout_reset(&sc->sc_rngto, hz*safe_rnginterval, safe_rng, sc);
 	}
 #endif /* SAFE_NO_RNG */
 #ifdef SAFE_DEBUG
 	safec = sc;			/* for use by hw.safe.dump */
 #endif
 	return (0);
 bad4:
 	crypto_unregister_all(sc->sc_cid);
 bad3:
 	bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
 bad2:
 	bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
 bad1:
 	bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr);
 bad:
 	return (ENXIO);
 }
 
 /*
  * Detach a device that successfully probed.
  */
 static int
 safe_detach(device_t dev)
 {
 	struct safe_softc *sc = device_get_softc(dev);
 
 	/* XXX wait/abort active ops */
 
 	WRITE_REG(sc, SAFE_HI_MASK, 0);		/* disable interrupts */
 
 	callout_stop(&sc->sc_rngto);
 
 	crypto_unregister_all(sc->sc_cid);
 
 #ifdef SAFE_RNDTEST
 	if (sc->sc_rndtest)
 		rndtest_detach(sc->sc_rndtest);
 #endif
 
 	safe_cleanchip(sc);
 	safe_dma_free(sc, &sc->sc_dpalloc);
 	safe_dma_free(sc, &sc->sc_spalloc);
 	mtx_destroy(&sc->sc_ringmtx);
 	safe_dma_free(sc, &sc->sc_ringalloc);
 
 	bus_generic_detach(dev);
 	bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
 	bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
 
 	bus_dma_tag_destroy(sc->sc_srcdmat);
 	bus_dma_tag_destroy(sc->sc_dstdmat);
 	bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr);
 
 	return (0);
 }
 
 /*
  * Stop all chip i/o so that the kernel's probe routines don't
  * get confused by errant DMAs when rebooting.
  */
 static int
 safe_shutdown(device_t dev)
 {
 #ifdef notyet
 	safe_stop(device_get_softc(dev));
 #endif
 	return (0);
 }
 
 /*
  * Device suspend routine.
  */
 static int
 safe_suspend(device_t dev)
 {
 	struct safe_softc *sc = device_get_softc(dev);
 
 #ifdef notyet
 	/* XXX stop the device and save PCI settings */
 #endif
 	sc->sc_suspended = 1;
 
 	return (0);
 }
 
 static int
 safe_resume(device_t dev)
 {
 	struct safe_softc *sc = device_get_softc(dev);
 
 #ifdef notyet
 	/* XXX retore PCI settings and start the device */
 #endif
 	sc->sc_suspended = 0;
 	return (0);
 }
 
 /*
  * SafeXcel Interrupt routine
  */
 static void
 safe_intr(void *arg)
 {
 	struct safe_softc *sc = arg;
 	volatile u_int32_t stat;
 
 	stat = READ_REG(sc, SAFE_HM_STAT);
 	if (stat == 0)			/* shared irq, not for us */
 		return;
 
 	WRITE_REG(sc, SAFE_HI_CLR, stat);	/* IACK */
 
 	if ((stat & SAFE_INT_PE_DDONE)) {
 		/*
 		 * Descriptor(s) done; scan the ring and
 		 * process completed operations.
 		 */
 		mtx_lock(&sc->sc_ringmtx);
 		while (sc->sc_back != sc->sc_front) {
 			struct safe_ringentry *re = sc->sc_back;
 #ifdef SAFE_DEBUG
 			if (safe_debug) {
 				safe_dump_ringstate(sc, __func__);
 				safe_dump_request(sc, __func__, re);
 			}
 #endif
 			/*
 			 * safe_process marks ring entries that were allocated
 			 * but not used with a csr of zero.  This insures the
 			 * ring front pointer never needs to be set backwards
 			 * in the event that an entry is allocated but not used
 			 * because of a setup error.
 			 */
 			if (re->re_desc.d_csr != 0) {
 				if (!SAFE_PE_CSR_IS_DONE(re->re_desc.d_csr))
 					break;
 				if (!SAFE_PE_LEN_IS_DONE(re->re_desc.d_len))
 					break;
 				sc->sc_nqchip--;
 				safe_callback(sc, re);
 			}
 			if (++(sc->sc_back) == sc->sc_ringtop)
 				sc->sc_back = sc->sc_ring;
 		}
 		mtx_unlock(&sc->sc_ringmtx);
 	}
 
 	/*
 	 * Check to see if we got any DMA Error
 	 */
 	if (stat & SAFE_INT_PE_ERROR) {
 		DPRINTF(("dmaerr dmastat %08x\n",
 			READ_REG(sc, SAFE_PE_DMASTAT)));
 		safestats.st_dmaerr++;
 		safe_totalreset(sc);
 #if 0
 		safe_feed(sc);
 #endif
 	}
 
 	if (sc->sc_needwakeup) {		/* XXX check high watermark */
 		int wakeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ);
 		DPRINTF(("%s: wakeup crypto %x\n", __func__,
 			sc->sc_needwakeup));
 		sc->sc_needwakeup &= ~wakeup;
 		crypto_unblock(sc->sc_cid, wakeup);
 	}
 }
 
 /*
  * safe_feed() - post a request to chip
  */
 static void
 safe_feed(struct safe_softc *sc, struct safe_ringentry *re)
 {
 	bus_dmamap_sync(sc->sc_srcdmat, re->re_src_map, BUS_DMASYNC_PREWRITE);
 	if (re->re_dst_map != NULL)
 		bus_dmamap_sync(sc->sc_dstdmat, re->re_dst_map,
 			BUS_DMASYNC_PREREAD);
 	/* XXX have no smaller granularity */
 	safe_dma_sync(&sc->sc_ringalloc,
 		BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 	safe_dma_sync(&sc->sc_spalloc, BUS_DMASYNC_PREWRITE);
 	safe_dma_sync(&sc->sc_dpalloc, BUS_DMASYNC_PREWRITE);
 
 #ifdef SAFE_DEBUG
 	if (safe_debug) {
 		safe_dump_ringstate(sc, __func__);
 		safe_dump_request(sc, __func__, re);
 	}
 #endif
 	sc->sc_nqchip++;
 	if (sc->sc_nqchip > safestats.st_maxqchip)
 		safestats.st_maxqchip = sc->sc_nqchip;
 	/* poke h/w to check descriptor ring, any value can be written */
 	WRITE_REG(sc, SAFE_HI_RD_DESCR, 0);
 }
 
 #define	N(a)	(sizeof(a) / sizeof (a[0]))
 static void
 safe_setup_enckey(struct safe_session *ses, caddr_t key)
 {
 	int i;
 
 	bcopy(key, ses->ses_key, ses->ses_klen / 8);
 
 	/* PE is little-endian, insure proper byte order */
 	for (i = 0; i < N(ses->ses_key); i++)
 		ses->ses_key[i] = htole32(ses->ses_key[i]);
 }
 
 static void
 safe_setup_mackey(struct safe_session *ses, int algo, caddr_t key, int klen)
 {
 	MD5_CTX md5ctx;
 	SHA1_CTX sha1ctx;
 	int i;
 
 
 	for (i = 0; i < klen; i++)
 		key[i] ^= HMAC_IPAD_VAL;
 
 	if (algo == CRYPTO_MD5_HMAC) {
 		MD5Init(&md5ctx);
 		MD5Update(&md5ctx, key, klen);
 		MD5Update(&md5ctx, hmac_ipad_buffer, MD5_HMAC_BLOCK_LEN - klen);
 		bcopy(md5ctx.state, ses->ses_hminner, sizeof(md5ctx.state));
 	} else {
 		SHA1Init(&sha1ctx);
 		SHA1Update(&sha1ctx, key, klen);
 		SHA1Update(&sha1ctx, hmac_ipad_buffer,
 		    SHA1_HMAC_BLOCK_LEN - klen);
 		bcopy(sha1ctx.h.b32, ses->ses_hminner, sizeof(sha1ctx.h.b32));
 	}
 
 	for (i = 0; i < klen; i++)
 		key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
 
 	if (algo == CRYPTO_MD5_HMAC) {
 		MD5Init(&md5ctx);
 		MD5Update(&md5ctx, key, klen);
 		MD5Update(&md5ctx, hmac_opad_buffer, MD5_HMAC_BLOCK_LEN - klen);
 		bcopy(md5ctx.state, ses->ses_hmouter, sizeof(md5ctx.state));
 	} else {
 		SHA1Init(&sha1ctx);
 		SHA1Update(&sha1ctx, key, klen);
 		SHA1Update(&sha1ctx, hmac_opad_buffer,
 		    SHA1_HMAC_BLOCK_LEN - klen);
 		bcopy(sha1ctx.h.b32, ses->ses_hmouter, sizeof(sha1ctx.h.b32));
 	}
 
 	for (i = 0; i < klen; i++)
 		key[i] ^= HMAC_OPAD_VAL;
 
 	/* PE is little-endian, insure proper byte order */
 	for (i = 0; i < N(ses->ses_hminner); i++) {
 		ses->ses_hminner[i] = htole32(ses->ses_hminner[i]);
 		ses->ses_hmouter[i] = htole32(ses->ses_hmouter[i]);
 	}
 }
 #undef N
 
 /*
  * Allocate a new 'session' and return an encoded session id.  'sidp'
  * contains our registration id, and should contain an encoded session
  * id on successful allocation.
  */
 static int
 safe_newsession(device_t dev, u_int32_t *sidp, struct cryptoini *cri)
 {
 	struct safe_softc *sc = device_get_softc(dev);
 	struct cryptoini *c, *encini = NULL, *macini = NULL;
 	struct safe_session *ses = NULL;
 	int sesn;
 
 	if (sidp == NULL || cri == NULL || sc == NULL)
 		return (EINVAL);
 
 	for (c = cri; c != NULL; c = c->cri_next) {
 		if (c->cri_alg == CRYPTO_MD5_HMAC ||
 		    c->cri_alg == CRYPTO_SHA1_HMAC ||
 		    c->cri_alg == CRYPTO_NULL_HMAC) {
 			if (macini)
 				return (EINVAL);
 			macini = c;
 		} else if (c->cri_alg == CRYPTO_DES_CBC ||
 		    c->cri_alg == CRYPTO_3DES_CBC ||
 		    c->cri_alg == CRYPTO_AES_CBC ||
 		    c->cri_alg == CRYPTO_NULL_CBC) {
 			if (encini)
 				return (EINVAL);
 			encini = c;
 		} else
 			return (EINVAL);
 	}
 	if (encini == NULL && macini == NULL)
 		return (EINVAL);
 	if (encini) {			/* validate key length */
 		switch (encini->cri_alg) {
 		case CRYPTO_DES_CBC:
 			if (encini->cri_klen != 64)
 				return (EINVAL);
 			break;
 		case CRYPTO_3DES_CBC:
 			if (encini->cri_klen != 192)
 				return (EINVAL);
 			break;
 		case CRYPTO_AES_CBC:
 			if (encini->cri_klen != 128 &&
 			    encini->cri_klen != 192 &&
 			    encini->cri_klen != 256)
 				return (EINVAL);
 			break;
 		}
 	}
 
 	if (sc->sc_sessions == NULL) {
 		ses = sc->sc_sessions = (struct safe_session *)malloc(
 		    sizeof(struct safe_session), M_DEVBUF, M_NOWAIT);
 		if (ses == NULL)
 			return (ENOMEM);
 		sesn = 0;
 		sc->sc_nsessions = 1;
 	} else {
 		for (sesn = 0; sesn < sc->sc_nsessions; sesn++) {
 			if (sc->sc_sessions[sesn].ses_used == 0) {
 				ses = &sc->sc_sessions[sesn];
 				break;
 			}
 		}
 
 		if (ses == NULL) {
 			sesn = sc->sc_nsessions;
 			ses = (struct safe_session *)malloc((sesn + 1) *
 			    sizeof(struct safe_session), M_DEVBUF, M_NOWAIT);
 			if (ses == NULL)
 				return (ENOMEM);
 			bcopy(sc->sc_sessions, ses, sesn *
 			    sizeof(struct safe_session));
 			bzero(sc->sc_sessions, sesn *
 			    sizeof(struct safe_session));
 			free(sc->sc_sessions, M_DEVBUF);
 			sc->sc_sessions = ses;
 			ses = &sc->sc_sessions[sesn];
 			sc->sc_nsessions++;
 		}
 	}
 
 	bzero(ses, sizeof(struct safe_session));
 	ses->ses_used = 1;
 
 	if (encini) {
 		/* get an IV */
 		/* XXX may read fewer than requested */
 		read_random(ses->ses_iv, sizeof(ses->ses_iv));
 
 		ses->ses_klen = encini->cri_klen;
 		if (encini->cri_key != NULL)
 			safe_setup_enckey(ses, encini->cri_key);
 	}
 
 	if (macini) {
 		ses->ses_mlen = macini->cri_mlen;
 		if (ses->ses_mlen == 0) {
 			if (macini->cri_alg == CRYPTO_MD5_HMAC)
 				ses->ses_mlen = MD5_HASH_LEN;
 			else
 				ses->ses_mlen = SHA1_HASH_LEN;
 		}
 
 		if (macini->cri_key != NULL) {
 			safe_setup_mackey(ses, macini->cri_alg, macini->cri_key,
 			    macini->cri_klen / 8);
 		}
 	}
 
 	*sidp = SAFE_SID(device_get_unit(sc->sc_dev), sesn);
 	return (0);
 }
 
 /*
  * Deallocate a session.
  */
 static int
 safe_freesession(device_t dev, u_int64_t tid)
 {
 	struct safe_softc *sc = device_get_softc(dev);
 	int session, ret;
 	u_int32_t sid = ((u_int32_t) tid) & 0xffffffff;
 
 	if (sc == NULL)
 		return (EINVAL);
 
 	session = SAFE_SESSION(sid);
 	if (session < sc->sc_nsessions) {
 		bzero(&sc->sc_sessions[session], sizeof(sc->sc_sessions[session]));
 		ret = 0;
 	} else
 		ret = EINVAL;
 	return (ret);
 }
 
 static void
 safe_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error)
 {
 	struct safe_operand *op = arg;
 
 	DPRINTF(("%s: mapsize %u nsegs %d error %d\n", __func__,
 		(u_int) mapsize, nsegs, error));
 	if (error != 0)
 		return;
 	op->mapsize = mapsize;
 	op->nsegs = nsegs;
 	bcopy(seg, op->segs, nsegs * sizeof (seg[0]));
 }
 
 static int
 safe_process(device_t dev, struct cryptop *crp, int hint)
 {
 	struct safe_softc *sc = device_get_softc(dev);
 	int err = 0, i, nicealign, uniform;
 	struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
 	int bypass, oplen, ivsize;
 	caddr_t iv;
 	int16_t coffset;
 	struct safe_session *ses;
 	struct safe_ringentry *re;
 	struct safe_sarec *sa;
 	struct safe_pdesc *pd;
 	u_int32_t cmd0, cmd1, staterec;
 
 	if (crp == NULL || crp->crp_callback == NULL || sc == NULL) {
 		safestats.st_invalid++;
 		return (EINVAL);
 	}
 	if (SAFE_SESSION(crp->crp_sid) >= sc->sc_nsessions) {
 		safestats.st_badsession++;
 		return (EINVAL);
 	}
 
 	mtx_lock(&sc->sc_ringmtx);
 	if (sc->sc_front == sc->sc_back && sc->sc_nqchip != 0) {
 		safestats.st_ringfull++;
 		sc->sc_needwakeup |= CRYPTO_SYMQ;
 		mtx_unlock(&sc->sc_ringmtx);
 		return (ERESTART);
 	}
 	re = sc->sc_front;
 
 	staterec = re->re_sa.sa_staterec;	/* save */
 	/* NB: zero everything but the PE descriptor */
 	bzero(&re->re_sa, sizeof(struct safe_ringentry) - sizeof(re->re_desc));
 	re->re_sa.sa_staterec = staterec;	/* restore */
 
 	re->re_crp = crp;
 	re->re_sesn = SAFE_SESSION(crp->crp_sid);
 
 	if (crp->crp_flags & CRYPTO_F_IMBUF) {
 		re->re_src_m = (struct mbuf *)crp->crp_buf;
 		re->re_dst_m = (struct mbuf *)crp->crp_buf;
 	} else if (crp->crp_flags & CRYPTO_F_IOV) {
 		re->re_src_io = (struct uio *)crp->crp_buf;
 		re->re_dst_io = (struct uio *)crp->crp_buf;
 	} else {
 		safestats.st_badflags++;
 		err = EINVAL;
 		goto errout;	/* XXX we don't handle contiguous blocks! */
 	}
 
 	sa = &re->re_sa;
 	ses = &sc->sc_sessions[re->re_sesn];
 
 	crd1 = crp->crp_desc;
 	if (crd1 == NULL) {
 		safestats.st_nodesc++;
 		err = EINVAL;
 		goto errout;
 	}
 	crd2 = crd1->crd_next;
 
 	cmd0 = SAFE_SA_CMD0_BASIC;		/* basic group operation */
 	cmd1 = 0;
 	if (crd2 == NULL) {
 		if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
 		    crd1->crd_alg == CRYPTO_SHA1_HMAC ||
 		    crd1->crd_alg == CRYPTO_NULL_HMAC) {
 			maccrd = crd1;
 			enccrd = NULL;
 			cmd0 |= SAFE_SA_CMD0_OP_HASH;
 		} else if (crd1->crd_alg == CRYPTO_DES_CBC ||
 		    crd1->crd_alg == CRYPTO_3DES_CBC ||
 		    crd1->crd_alg == CRYPTO_AES_CBC ||
 		    crd1->crd_alg == CRYPTO_NULL_CBC) {
 			maccrd = NULL;
 			enccrd = crd1;
 			cmd0 |= SAFE_SA_CMD0_OP_CRYPT;
 		} else {
 			safestats.st_badalg++;
 			err = EINVAL;
 			goto errout;
 		}
 	} else {
 		if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
 		    crd1->crd_alg == CRYPTO_SHA1_HMAC ||
 		    crd1->crd_alg == CRYPTO_NULL_HMAC) &&
 		    (crd2->crd_alg == CRYPTO_DES_CBC ||
 			crd2->crd_alg == CRYPTO_3DES_CBC ||
 		        crd2->crd_alg == CRYPTO_AES_CBC ||
 		        crd2->crd_alg == CRYPTO_NULL_CBC) &&
 		    ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
 			maccrd = crd1;
 			enccrd = crd2;
 		} else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
 		    crd1->crd_alg == CRYPTO_3DES_CBC ||
 		    crd1->crd_alg == CRYPTO_AES_CBC ||
 		    crd1->crd_alg == CRYPTO_NULL_CBC) &&
 		    (crd2->crd_alg == CRYPTO_MD5_HMAC ||
 			crd2->crd_alg == CRYPTO_SHA1_HMAC ||
 			crd2->crd_alg == CRYPTO_NULL_HMAC) &&
 		    (crd1->crd_flags & CRD_F_ENCRYPT)) {
 			enccrd = crd1;
 			maccrd = crd2;
 		} else {
 			safestats.st_badalg++;
 			err = EINVAL;
 			goto errout;
 		}
 		cmd0 |= SAFE_SA_CMD0_OP_BOTH;
 	}
 
 	if (enccrd) {
 		if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT)
 			safe_setup_enckey(ses, enccrd->crd_key);
 
 		if (enccrd->crd_alg == CRYPTO_DES_CBC) {
 			cmd0 |= SAFE_SA_CMD0_DES;
 			cmd1 |= SAFE_SA_CMD1_CBC;
 			ivsize = 2*sizeof(u_int32_t);
 		} else if (enccrd->crd_alg == CRYPTO_3DES_CBC) {
 			cmd0 |= SAFE_SA_CMD0_3DES;
 			cmd1 |= SAFE_SA_CMD1_CBC;
 			ivsize = 2*sizeof(u_int32_t);
 		} else if (enccrd->crd_alg == CRYPTO_AES_CBC) {
 			cmd0 |= SAFE_SA_CMD0_AES;
 			cmd1 |= SAFE_SA_CMD1_CBC;
 			if (ses->ses_klen == 128)
 			     cmd1 |=  SAFE_SA_CMD1_AES128;
 			else if (ses->ses_klen == 192)
 			     cmd1 |=  SAFE_SA_CMD1_AES192;
 			else
 			     cmd1 |=  SAFE_SA_CMD1_AES256;
 			ivsize = 4*sizeof(u_int32_t);
 		} else {
 			cmd0 |= SAFE_SA_CMD0_CRYPT_NULL;
 			ivsize = 0;
 		}
 
 		/*
 		 * Setup encrypt/decrypt state.  When using basic ops
 		 * we can't use an inline IV because hash/crypt offset
 		 * must be from the end of the IV to the start of the
 		 * crypt data and this leaves out the preceding header
 		 * from the hash calculation.  Instead we place the IV
 		 * in the state record and set the hash/crypt offset to
 		 * copy both the header+IV.
 		 */
 		if (enccrd->crd_flags & CRD_F_ENCRYPT) {
 			cmd0 |= SAFE_SA_CMD0_OUTBOUND;
 
 			if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
 				iv = enccrd->crd_iv;
 			else
 				iv = (caddr_t) ses->ses_iv;
 			if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
 				crypto_copyback(crp->crp_flags, crp->crp_buf,
 				    enccrd->crd_inject, ivsize, iv);
 			}
 			bcopy(iv, re->re_sastate.sa_saved_iv, ivsize);
 			cmd0 |= SAFE_SA_CMD0_IVLD_STATE | SAFE_SA_CMD0_SAVEIV;
 			re->re_flags |= SAFE_QFLAGS_COPYOUTIV;
 		} else {
 			cmd0 |= SAFE_SA_CMD0_INBOUND;
 
 			if (enccrd->crd_flags & CRD_F_IV_EXPLICIT) {
 				bcopy(enccrd->crd_iv,
 					re->re_sastate.sa_saved_iv, ivsize);
 			} else {
 				crypto_copydata(crp->crp_flags, crp->crp_buf,
 				    enccrd->crd_inject, ivsize,
 				    (caddr_t)re->re_sastate.sa_saved_iv);
 			}
 			cmd0 |= SAFE_SA_CMD0_IVLD_STATE;
 		}
 		/*
 		 * For basic encryption use the zero pad algorithm.
 		 * This pads results to an 8-byte boundary and
 		 * suppresses padding verification for inbound (i.e.
 		 * decrypt) operations.
 		 *
 		 * NB: Not sure if the 8-byte pad boundary is a problem.
 		 */
 		cmd0 |= SAFE_SA_CMD0_PAD_ZERO;
 
 		/* XXX assert key bufs have the same size */
 		bcopy(ses->ses_key, sa->sa_key, sizeof(sa->sa_key));
 	}
 
 	if (maccrd) {
 		if (maccrd->crd_flags & CRD_F_KEY_EXPLICIT) {
 			safe_setup_mackey(ses, maccrd->crd_alg,
 			    maccrd->crd_key, maccrd->crd_klen / 8);
 		}
 
 		if (maccrd->crd_alg == CRYPTO_MD5_HMAC) {
 			cmd0 |= SAFE_SA_CMD0_MD5;
 			cmd1 |= SAFE_SA_CMD1_HMAC;	/* NB: enable HMAC */
 		} else if (maccrd->crd_alg == CRYPTO_SHA1_HMAC) {
 			cmd0 |= SAFE_SA_CMD0_SHA1;
 			cmd1 |= SAFE_SA_CMD1_HMAC;	/* NB: enable HMAC */
 		} else {
 			cmd0 |= SAFE_SA_CMD0_HASH_NULL;
 		}
 		/*
 		 * Digest data is loaded from the SA and the hash
 		 * result is saved to the state block where we
 		 * retrieve it for return to the caller.
 		 */
 		/* XXX assert digest bufs have the same size */
 		bcopy(ses->ses_hminner, sa->sa_indigest,
 			sizeof(sa->sa_indigest));
 		bcopy(ses->ses_hmouter, sa->sa_outdigest,
 			sizeof(sa->sa_outdigest));
 
 		cmd0 |= SAFE_SA_CMD0_HSLD_SA | SAFE_SA_CMD0_SAVEHASH;
 		re->re_flags |= SAFE_QFLAGS_COPYOUTICV;
 	}
 
 	if (enccrd && maccrd) {
 		/*
 		 * The offset from hash data to the start of
 		 * crypt data is the difference in the skips.
 		 */
 		bypass = maccrd->crd_skip;
 		coffset = enccrd->crd_skip - maccrd->crd_skip;
 		if (coffset < 0) {
 			DPRINTF(("%s: hash does not precede crypt; "
 				"mac skip %u enc skip %u\n",
 				__func__, maccrd->crd_skip, enccrd->crd_skip));
 			safestats.st_skipmismatch++;
 			err = EINVAL;
 			goto errout;
 		}
 		oplen = enccrd->crd_skip + enccrd->crd_len;
 		if (maccrd->crd_skip + maccrd->crd_len != oplen) {
 			DPRINTF(("%s: hash amount %u != crypt amount %u\n",
 				__func__, maccrd->crd_skip + maccrd->crd_len,
 				oplen));
 			safestats.st_lenmismatch++;
 			err = EINVAL;
 			goto errout;
 		}
 #ifdef SAFE_DEBUG
 		if (safe_debug) {
 			printf("mac: skip %d, len %d, inject %d\n",
 			    maccrd->crd_skip, maccrd->crd_len,
 			    maccrd->crd_inject);
 			printf("enc: skip %d, len %d, inject %d\n",
 			    enccrd->crd_skip, enccrd->crd_len,
 			    enccrd->crd_inject);
 			printf("bypass %d coffset %d oplen %d\n",
 				bypass, coffset, oplen);
 		}
 #endif
 		if (coffset & 3) {	/* offset must be 32-bit aligned */
 			DPRINTF(("%s: coffset %u misaligned\n",
 				__func__, coffset));
 			safestats.st_coffmisaligned++;
 			err = EINVAL;
 			goto errout;
 		}
 		coffset >>= 2;
 		if (coffset > 255) {	/* offset must be <256 dwords */
 			DPRINTF(("%s: coffset %u too big\n",
 				__func__, coffset));
 			safestats.st_cofftoobig++;
 			err = EINVAL;
 			goto errout;
 		}
 		/*
 		 * Tell the hardware to copy the header to the output.
 		 * The header is defined as the data from the end of
 		 * the bypass to the start of data to be encrypted. 
 		 * Typically this is the inline IV.  Note that you need
 		 * to do this even if src+dst are the same; it appears
 		 * that w/o this bit the crypted data is written
 		 * immediately after the bypass data.
 		 */
 		cmd1 |= SAFE_SA_CMD1_HDRCOPY;
 		/*
 		 * Disable IP header mutable bit handling.  This is
 		 * needed to get correct HMAC calculations.
 		 */
 		cmd1 |= SAFE_SA_CMD1_MUTABLE;
 	} else {
 		if (enccrd) {
 			bypass = enccrd->crd_skip;
 			oplen = bypass + enccrd->crd_len;
 		} else {
 			bypass = maccrd->crd_skip;
 			oplen = bypass + maccrd->crd_len;
 		}
 		coffset = 0;
 	}
 	/* XXX verify multiple of 4 when using s/g */
 	if (bypass > 96) {		/* bypass offset must be <= 96 bytes */
 		DPRINTF(("%s: bypass %u too big\n", __func__, bypass));
 		safestats.st_bypasstoobig++;
 		err = EINVAL;
 		goto errout;
 	}
 
 	if (bus_dmamap_create(sc->sc_srcdmat, BUS_DMA_NOWAIT, &re->re_src_map)) {
 		safestats.st_nomap++;
 		err = ENOMEM;
 		goto errout;
 	}
 	if (crp->crp_flags & CRYPTO_F_IMBUF) {
 		if (bus_dmamap_load_mbuf(sc->sc_srcdmat, re->re_src_map,
 		    re->re_src_m, safe_op_cb,
 		    &re->re_src, BUS_DMA_NOWAIT) != 0) {
 			bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map);
 			re->re_src_map = NULL;
 			safestats.st_noload++;
 			err = ENOMEM;
 			goto errout;
 		}
 	} else if (crp->crp_flags & CRYPTO_F_IOV) {
 		if (bus_dmamap_load_uio(sc->sc_srcdmat, re->re_src_map,
 		    re->re_src_io, safe_op_cb,
 		    &re->re_src, BUS_DMA_NOWAIT) != 0) {
 			bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map);
 			re->re_src_map = NULL;
 			safestats.st_noload++;
 			err = ENOMEM;
 			goto errout;
 		}
 	}
 	nicealign = safe_dmamap_aligned(&re->re_src);
 	uniform = safe_dmamap_uniform(&re->re_src);
 
 	DPRINTF(("src nicealign %u uniform %u nsegs %u\n",
 		nicealign, uniform, re->re_src.nsegs));
 	if (re->re_src.nsegs > 1) {
 		re->re_desc.d_src = sc->sc_spalloc.dma_paddr +
 			((caddr_t) sc->sc_spfree - (caddr_t) sc->sc_spring);
 		for (i = 0; i < re->re_src_nsegs; i++) {
 			/* NB: no need to check if there's space */
 			pd = sc->sc_spfree;
 			if (++(sc->sc_spfree) == sc->sc_springtop)
 				sc->sc_spfree = sc->sc_spring;
 
 			KASSERT((pd->pd_flags&3) == 0 ||
 				(pd->pd_flags&3) == SAFE_PD_DONE,
 				("bogus source particle descriptor; flags %x",
 				pd->pd_flags));
 			pd->pd_addr = re->re_src_segs[i].ds_addr;
 			pd->pd_size = re->re_src_segs[i].ds_len;
 			pd->pd_flags = SAFE_PD_READY;
 		}
 		cmd0 |= SAFE_SA_CMD0_IGATHER;
 	} else {
 		/*
 		 * No need for gather, reference the operand directly.
 		 */
 		re->re_desc.d_src = re->re_src_segs[0].ds_addr;
 	}
 
 	if (enccrd == NULL && maccrd != NULL) {
 		/*
 		 * Hash op; no destination needed.
 		 */
 	} else {
 		if (crp->crp_flags & CRYPTO_F_IOV) {
 			if (!nicealign) {
 				safestats.st_iovmisaligned++;
 				err = EINVAL;
 				goto errout;
 			}
 			if (uniform != 1) {
 				/*
 				 * Source is not suitable for direct use as
 				 * the destination.  Create a new scatter/gather
 				 * list based on the destination requirements
 				 * and check if that's ok.
 				 */
 				if (bus_dmamap_create(sc->sc_dstdmat,
 				    BUS_DMA_NOWAIT, &re->re_dst_map)) {
 					safestats.st_nomap++;
 					err = ENOMEM;
 					goto errout;
 				}
 				if (bus_dmamap_load_uio(sc->sc_dstdmat,
 				    re->re_dst_map, re->re_dst_io,
 				    safe_op_cb, &re->re_dst,
 				    BUS_DMA_NOWAIT) != 0) {
 					bus_dmamap_destroy(sc->sc_dstdmat,
 						re->re_dst_map);
 					re->re_dst_map = NULL;
 					safestats.st_noload++;
 					err = ENOMEM;
 					goto errout;
 				}
 				uniform = safe_dmamap_uniform(&re->re_dst);
 				if (!uniform) {
 					/*
 					 * There's no way to handle the DMA
 					 * requirements with this uio.  We
 					 * could create a separate DMA area for
 					 * the result and then copy it back,
 					 * but for now we just bail and return
 					 * an error.  Note that uio requests
 					 * > SAFE_MAX_DSIZE are handled because
 					 * the DMA map and segment list for the
 					 * destination wil result in a
 					 * destination particle list that does
 					 * the necessary scatter DMA.
 					 */ 
 					safestats.st_iovnotuniform++;
 					err = EINVAL;
 					goto errout;
 				}
 			} else
 				re->re_dst = re->re_src;
 		} else if (crp->crp_flags & CRYPTO_F_IMBUF) {
 			if (nicealign && uniform == 1) {
 				/*
 				 * Source layout is suitable for direct
 				 * sharing of the DMA map and segment list.
 				 */
 				re->re_dst = re->re_src;
 			} else if (nicealign && uniform == 2) {
 				/*
 				 * The source is properly aligned but requires a
 				 * different particle list to handle DMA of the
 				 * result.  Create a new map and do the load to
 				 * create the segment list.  The particle
 				 * descriptor setup code below will handle the
 				 * rest.
 				 */
 				if (bus_dmamap_create(sc->sc_dstdmat,
 				    BUS_DMA_NOWAIT, &re->re_dst_map)) {
 					safestats.st_nomap++;
 					err = ENOMEM;
 					goto errout;
 				}
 				if (bus_dmamap_load_mbuf(sc->sc_dstdmat,
 				    re->re_dst_map, re->re_dst_m,
 				    safe_op_cb, &re->re_dst,
 				    BUS_DMA_NOWAIT) != 0) {
 					bus_dmamap_destroy(sc->sc_dstdmat,
 						re->re_dst_map);
 					re->re_dst_map = NULL;
 					safestats.st_noload++;
 					err = ENOMEM;
 					goto errout;
 				}
 			} else {		/* !(aligned and/or uniform) */
 				int totlen, len;
 				struct mbuf *m, *top, **mp;
 
 				/*
 				 * DMA constraints require that we allocate a
 				 * new mbuf chain for the destination.  We
 				 * allocate an entire new set of mbufs of
 				 * optimal/required size and then tell the
 				 * hardware to copy any bits that are not
 				 * created as a byproduct of the operation.
 				 */
 				if (!nicealign)
 					safestats.st_unaligned++;
 				if (!uniform)
 					safestats.st_notuniform++;
 				totlen = re->re_src_mapsize;
 				if (re->re_src_m->m_flags & M_PKTHDR) {
 					len = MHLEN;
 					MGETHDR(m, M_NOWAIT, MT_DATA);
 					if (m && !m_dup_pkthdr(m, re->re_src_m,
 					    M_NOWAIT)) {
 						m_free(m);
 						m = NULL;
 					}
 				} else {
 					len = MLEN;
 					MGET(m, M_NOWAIT, MT_DATA);
 				}
 				if (m == NULL) {
 					safestats.st_nombuf++;
 					err = sc->sc_nqchip ? ERESTART : ENOMEM;
 					goto errout;
 				}
 				if (totlen >= MINCLSIZE) {
 					MCLGET(m, M_NOWAIT);
 					if ((m->m_flags & M_EXT) == 0) {
 						m_free(m);
 						safestats.st_nomcl++;
 						err = sc->sc_nqchip ?
 							ERESTART : ENOMEM;
 						goto errout;
 					}
 					len = MCLBYTES;
 				}
 				m->m_len = len;
 				top = NULL;
 				mp = &top;
 
 				while (totlen > 0) {
 					if (top) {
 						MGET(m, M_NOWAIT, MT_DATA);
 						if (m == NULL) {
 							m_freem(top);
 							safestats.st_nombuf++;
 							err = sc->sc_nqchip ?
 							    ERESTART : ENOMEM;
 							goto errout;
 						}
 						len = MLEN;
 					}
 					if (top && totlen >= MINCLSIZE) {
 						MCLGET(m, M_NOWAIT);
 						if ((m->m_flags & M_EXT) == 0) {
 							*mp = m;
 							m_freem(top);
 							safestats.st_nomcl++;
 							err = sc->sc_nqchip ?
 							    ERESTART : ENOMEM;
 							goto errout;
 						}
 						len = MCLBYTES;
 					}
 					m->m_len = len = min(totlen, len);
 					totlen -= len;
 					*mp = m;
 					mp = &m->m_next;
 				}
 				re->re_dst_m = top;
 				if (bus_dmamap_create(sc->sc_dstdmat, 
 				    BUS_DMA_NOWAIT, &re->re_dst_map) != 0) {
 					safestats.st_nomap++;
 					err = ENOMEM;
 					goto errout;
 				}
 				if (bus_dmamap_load_mbuf(sc->sc_dstdmat,
 				    re->re_dst_map, re->re_dst_m,
 				    safe_op_cb, &re->re_dst,
 				    BUS_DMA_NOWAIT) != 0) {
 					bus_dmamap_destroy(sc->sc_dstdmat,
 					re->re_dst_map);
 					re->re_dst_map = NULL;
 					safestats.st_noload++;
 					err = ENOMEM;
 					goto errout;
 				}
 				if (re->re_src.mapsize > oplen) {
 					/*
 					 * There's data following what the
 					 * hardware will copy for us.  If this
 					 * isn't just the ICV (that's going to
 					 * be written on completion), copy it
 					 * to the new mbufs
 					 */
 					if (!(maccrd &&
 					    (re->re_src.mapsize-oplen) == 12 &&
 					    maccrd->crd_inject == oplen))
 						safe_mcopy(re->re_src_m,
 							   re->re_dst_m,
 							   oplen);
 					else
 						safestats.st_noicvcopy++;
 				}
 			}
 		} else {
 			safestats.st_badflags++;
 			err = EINVAL;
 			goto errout;
 		}
 
 		if (re->re_dst.nsegs > 1) {
 			re->re_desc.d_dst = sc->sc_dpalloc.dma_paddr +
 			    ((caddr_t) sc->sc_dpfree - (caddr_t) sc->sc_dpring);
 			for (i = 0; i < re->re_dst_nsegs; i++) {
 				pd = sc->sc_dpfree;
 				KASSERT((pd->pd_flags&3) == 0 ||
 					(pd->pd_flags&3) == SAFE_PD_DONE,
 					("bogus dest particle descriptor; flags %x",
 						pd->pd_flags));
 				if (++(sc->sc_dpfree) == sc->sc_dpringtop)
 					sc->sc_dpfree = sc->sc_dpring;
 				pd->pd_addr = re->re_dst_segs[i].ds_addr;
 				pd->pd_flags = SAFE_PD_READY;
 			}
 			cmd0 |= SAFE_SA_CMD0_OSCATTER;
 		} else {
 			/*
 			 * No need for scatter, reference the operand directly.
 			 */
 			re->re_desc.d_dst = re->re_dst_segs[0].ds_addr;
 		}
 	}
 
 	/*
 	 * All done with setup; fillin the SA command words
 	 * and the packet engine descriptor.  The operation
 	 * is now ready for submission to the hardware.
 	 */
 	sa->sa_cmd0 = cmd0 | SAFE_SA_CMD0_IPCI | SAFE_SA_CMD0_OPCI;
 	sa->sa_cmd1 = cmd1
 		    | (coffset << SAFE_SA_CMD1_OFFSET_S)
 		    | SAFE_SA_CMD1_SAREV1	/* Rev 1 SA data structure */
 		    | SAFE_SA_CMD1_SRPCI
 		    ;
 	/*
 	 * NB: the order of writes is important here.  In case the
 	 * chip is scanning the ring because of an outstanding request
 	 * it might nab this one too.  In that case we need to make
 	 * sure the setup is complete before we write the length
 	 * field of the descriptor as it signals the descriptor is
 	 * ready for processing.
 	 */
 	re->re_desc.d_csr = SAFE_PE_CSR_READY | SAFE_PE_CSR_SAPCI;
 	if (maccrd)
 		re->re_desc.d_csr |= SAFE_PE_CSR_LOADSA | SAFE_PE_CSR_HASHFINAL;
 	re->re_desc.d_len = oplen
 			  | SAFE_PE_LEN_READY
 			  | (bypass << SAFE_PE_LEN_BYPASS_S)
 			  ;
 
 	safestats.st_ipackets++;
 	safestats.st_ibytes += oplen;
 
 	if (++(sc->sc_front) == sc->sc_ringtop)
 		sc->sc_front = sc->sc_ring;
 
 	/* XXX honor batching */
 	safe_feed(sc, re);
 	mtx_unlock(&sc->sc_ringmtx);
 	return (0);
 
 errout:
 	if ((re->re_dst_m != NULL) && (re->re_src_m != re->re_dst_m))
 		m_freem(re->re_dst_m);
 
 	if (re->re_dst_map != NULL && re->re_dst_map != re->re_src_map) {
 		bus_dmamap_unload(sc->sc_dstdmat, re->re_dst_map);
 		bus_dmamap_destroy(sc->sc_dstdmat, re->re_dst_map);
 	}
 	if (re->re_src_map != NULL) {
 		bus_dmamap_unload(sc->sc_srcdmat, re->re_src_map);
 		bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map);
 	}
 	mtx_unlock(&sc->sc_ringmtx);
 	if (err != ERESTART) {
 		crp->crp_etype = err;
 		crypto_done(crp);
 	} else {
 		sc->sc_needwakeup |= CRYPTO_SYMQ;
 	}
 	return (err);
 }
 
 static void
 safe_callback(struct safe_softc *sc, struct safe_ringentry *re)
 {
 	struct cryptop *crp = (struct cryptop *)re->re_crp;
 	struct cryptodesc *crd;
 
 	safestats.st_opackets++;
 	safestats.st_obytes += re->re_dst.mapsize;
 
 	safe_dma_sync(&sc->sc_ringalloc,
 		BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
 	if (re->re_desc.d_csr & SAFE_PE_CSR_STATUS) {
 		device_printf(sc->sc_dev, "csr 0x%x cmd0 0x%x cmd1 0x%x\n",
 			re->re_desc.d_csr,
 			re->re_sa.sa_cmd0, re->re_sa.sa_cmd1);
 		safestats.st_peoperr++;
 		crp->crp_etype = EIO;		/* something more meaningful? */
 	}
 	if (re->re_dst_map != NULL && re->re_dst_map != re->re_src_map) {
 		bus_dmamap_sync(sc->sc_dstdmat, re->re_dst_map,
 		    BUS_DMASYNC_POSTREAD);
 		bus_dmamap_unload(sc->sc_dstdmat, re->re_dst_map);
 		bus_dmamap_destroy(sc->sc_dstdmat, re->re_dst_map);
 	}
 	bus_dmamap_sync(sc->sc_srcdmat, re->re_src_map, BUS_DMASYNC_POSTWRITE);
 	bus_dmamap_unload(sc->sc_srcdmat, re->re_src_map);
 	bus_dmamap_destroy(sc->sc_srcdmat, re->re_src_map);
 
 	/* 
 	 * If result was written to a differet mbuf chain, swap
 	 * it in as the return value and reclaim the original.
 	 */
 	if ((crp->crp_flags & CRYPTO_F_IMBUF) && re->re_src_m != re->re_dst_m) {
 		m_freem(re->re_src_m);
 		crp->crp_buf = (caddr_t)re->re_dst_m;
 	}
 
 	if (re->re_flags & SAFE_QFLAGS_COPYOUTIV) {
 		/* copy out IV for future use */
 		for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
 			int ivsize;
 
 			if (crd->crd_alg == CRYPTO_DES_CBC ||
 			    crd->crd_alg == CRYPTO_3DES_CBC) {
 				ivsize = 2*sizeof(u_int32_t);
 			} else if (crd->crd_alg == CRYPTO_AES_CBC) {
 				ivsize = 4*sizeof(u_int32_t);
 			} else
 				continue;
 			crypto_copydata(crp->crp_flags, crp->crp_buf,
 			    crd->crd_skip + crd->crd_len - ivsize, ivsize,
 			    (caddr_t)sc->sc_sessions[re->re_sesn].ses_iv);
 			break;
 		}
 	}
 
 	if (re->re_flags & SAFE_QFLAGS_COPYOUTICV) {
 		/* copy out ICV result */
 		for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
 			if (!(crd->crd_alg == CRYPTO_MD5_HMAC ||
 			    crd->crd_alg == CRYPTO_SHA1_HMAC ||
 			    crd->crd_alg == CRYPTO_NULL_HMAC))
 				continue;
 			if (crd->crd_alg == CRYPTO_SHA1_HMAC) {
 				/*
 				 * SHA-1 ICV's are byte-swapped; fix 'em up
 				 * before copy them to their destination.
 				 */
 				re->re_sastate.sa_saved_indigest[0] =
 				    bswap32(re->re_sastate.sa_saved_indigest[0]);
 				re->re_sastate.sa_saved_indigest[1] =
 				    bswap32(re->re_sastate.sa_saved_indigest[1]);
 				re->re_sastate.sa_saved_indigest[2] =
 				    bswap32(re->re_sastate.sa_saved_indigest[2]);
 			}
 			crypto_copyback(crp->crp_flags, crp->crp_buf,
 			    crd->crd_inject,
 			    sc->sc_sessions[re->re_sesn].ses_mlen,
 			    (caddr_t)re->re_sastate.sa_saved_indigest);
 			break;
 		}
 	}
 	crypto_done(crp);
 }
 
 /*
  * Copy all data past offset from srcm to dstm.
  */
 static void
 safe_mcopy(struct mbuf *srcm, struct mbuf *dstm, u_int offset)
 {
 	u_int j, dlen, slen;
 	caddr_t dptr, sptr;
 
 	/*
 	 * Advance src and dst to offset.
 	 */
 	j = offset;
 	while (j >= 0) {
 		if (srcm->m_len > j)
 			break;
 		j -= srcm->m_len;
 		srcm = srcm->m_next;
 		if (srcm == NULL)
 			return;
 	}
 	sptr = mtod(srcm, caddr_t) + j;
 	slen = srcm->m_len - j;
 
 	j = offset;
 	while (j >= 0) {
 		if (dstm->m_len > j)
 			break;
 		j -= dstm->m_len;
 		dstm = dstm->m_next;
 		if (dstm == NULL)
 			return;
 	}
 	dptr = mtod(dstm, caddr_t) + j;
 	dlen = dstm->m_len - j;
 
 	/*
 	 * Copy everything that remains.
 	 */
 	for (;;) {
 		j = min(slen, dlen);
 		bcopy(sptr, dptr, j);
 		if (slen == j) {
 			srcm = srcm->m_next;
 			if (srcm == NULL)
 				return;
 			sptr = srcm->m_data;
 			slen = srcm->m_len;
 		} else
 			sptr += j, slen -= j;
 		if (dlen == j) {
 			dstm = dstm->m_next;
 			if (dstm == NULL)
 				return;
 			dptr = dstm->m_data;
 			dlen = dstm->m_len;
 		} else
 			dptr += j, dlen -= j;
 	}
 }
 
 #ifndef SAFE_NO_RNG
 #define	SAFE_RNG_MAXWAIT	1000
 
 static void
 safe_rng_init(struct safe_softc *sc)
 {
 	u_int32_t w, v;
 	int i;
 
 	WRITE_REG(sc, SAFE_RNG_CTRL, 0);
 	/* use default value according to the manual */
 	WRITE_REG(sc, SAFE_RNG_CNFG, 0x834);	/* magic from SafeNet */
 	WRITE_REG(sc, SAFE_RNG_ALM_CNT, 0);
 
 	/*
 	 * There is a bug in rev 1.0 of the 1140 that when the RNG
 	 * is brought out of reset the ready status flag does not
 	 * work until the RNG has finished its internal initialization.
 	 *
 	 * So in order to determine the device is through its
 	 * initialization we must read the data register, using the
 	 * status reg in the read in case it is initialized.  Then read
 	 * the data register until it changes from the first read.
 	 * Once it changes read the data register until it changes
 	 * again.  At this time the RNG is considered initialized. 
 	 * This could take between 750ms - 1000ms in time.
 	 */
 	i = 0;
 	w = READ_REG(sc, SAFE_RNG_OUT);
 	do {
 		v = READ_REG(sc, SAFE_RNG_OUT);
 		if (v != w) {
 			w = v;
 			break;
 		}
 		DELAY(10);
 	} while (++i < SAFE_RNG_MAXWAIT);
 
 	/* Wait Until data changes again */
 	i = 0;
 	do {
 		v = READ_REG(sc, SAFE_RNG_OUT);
 		if (v != w)
 			break;
 		DELAY(10);
 	} while (++i < SAFE_RNG_MAXWAIT);
 }
 
 static __inline void
 safe_rng_disable_short_cycle(struct safe_softc *sc)
 {
 	WRITE_REG(sc, SAFE_RNG_CTRL,
 		READ_REG(sc, SAFE_RNG_CTRL) &~ SAFE_RNG_CTRL_SHORTEN);
 }
 
 static __inline void
 safe_rng_enable_short_cycle(struct safe_softc *sc)
 {
 	WRITE_REG(sc, SAFE_RNG_CTRL, 
 		READ_REG(sc, SAFE_RNG_CTRL) | SAFE_RNG_CTRL_SHORTEN);
 }
 
 static __inline u_int32_t
 safe_rng_read(struct safe_softc *sc)
 {
 	int i;
 
 	i = 0;
 	while (READ_REG(sc, SAFE_RNG_STAT) != 0 && ++i < SAFE_RNG_MAXWAIT)
 		;
 	return READ_REG(sc, SAFE_RNG_OUT);
 }
 
 static void
 safe_rng(void *arg)
 {
 	struct safe_softc *sc = arg;
 	u_int32_t buf[SAFE_RNG_MAXBUFSIZ];	/* NB: maybe move to softc */
 	u_int maxwords;
 	int i;
 
 	safestats.st_rng++;
 	/*
 	 * Fetch the next block of data.
 	 */
 	maxwords = safe_rngbufsize;
 	if (maxwords > SAFE_RNG_MAXBUFSIZ)
 		maxwords = SAFE_RNG_MAXBUFSIZ;
 retry:
 	for (i = 0; i < maxwords; i++)
 		buf[i] = safe_rng_read(sc);
 	/*
 	 * Check the comparator alarm count and reset the h/w if
 	 * it exceeds our threshold.  This guards against the
 	 * hardware oscillators resonating with external signals.
 	 */
 	if (READ_REG(sc, SAFE_RNG_ALM_CNT) > safe_rngmaxalarm) {
 		u_int32_t freq_inc, w;
 
 		DPRINTF(("%s: alarm count %u exceeds threshold %u\n", __func__,
 			READ_REG(sc, SAFE_RNG_ALM_CNT), safe_rngmaxalarm));
 		safestats.st_rngalarm++;
 		safe_rng_enable_short_cycle(sc);
 		freq_inc = 18;
 		for (i = 0; i < 64; i++) {
 			w = READ_REG(sc, SAFE_RNG_CNFG);
 			freq_inc = ((w + freq_inc) & 0x3fL);
 			w = ((w & ~0x3fL) | freq_inc);
 			WRITE_REG(sc, SAFE_RNG_CNFG, w);
 
 			WRITE_REG(sc, SAFE_RNG_ALM_CNT, 0);
 
 			(void) safe_rng_read(sc);
 			DELAY(25);
 
 			if (READ_REG(sc, SAFE_RNG_ALM_CNT) == 0) {
 				safe_rng_disable_short_cycle(sc);
 				goto retry;
 			}
 			freq_inc = 1;
 		}
 		safe_rng_disable_short_cycle(sc);
 	} else
 		WRITE_REG(sc, SAFE_RNG_ALM_CNT, 0);
 
 	(*sc->sc_harvest)(sc->sc_rndtest, buf, maxwords*sizeof (u_int32_t));
 	callout_reset(&sc->sc_rngto,
 		hz * (safe_rnginterval ? safe_rnginterval : 1), safe_rng, sc);
 }
 #endif /* SAFE_NO_RNG */
 
 static void
 safe_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 {
 	bus_addr_t *paddr = (bus_addr_t*) arg;
 	*paddr = segs->ds_addr;
 }
 
 static int
 safe_dma_malloc(
 	struct safe_softc *sc,
 	bus_size_t size,
 	struct safe_dma_alloc *dma,
 	int mapflags
 )
 {
 	int r;
 
 	r = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev),	/* parent */
 			       sizeof(u_int32_t), 0,	/* alignment, bounds */
 			       BUS_SPACE_MAXADDR_32BIT,	/* lowaddr */
 			       BUS_SPACE_MAXADDR,	/* highaddr */
 			       NULL, NULL,		/* filter, filterarg */
 			       size,			/* maxsize */
 			       1,			/* nsegments */
 			       size,			/* maxsegsize */
 			       BUS_DMA_ALLOCNOW,	/* flags */
 			       NULL, NULL,		/* locking */
 			       &dma->dma_tag);
 	if (r != 0) {
 		device_printf(sc->sc_dev, "safe_dma_malloc: "
 			"bus_dma_tag_create failed; error %u\n", r);
 		goto fail_0;
 	}
 
-	r = bus_dmamap_create(dma->dma_tag, BUS_DMA_NOWAIT, &dma->dma_map);
-	if (r != 0) {
-		device_printf(sc->sc_dev, "safe_dma_malloc: "
-			"bus_dmamap_create failed; error %u\n", r);
-		goto fail_1;
-	}
-
 	r = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
 			     BUS_DMA_NOWAIT, &dma->dma_map);
 	if (r != 0) {
 		device_printf(sc->sc_dev, "safe_dma_malloc: "
 			"bus_dmammem_alloc failed; size %zu, error %u\n",
 			size, r);
-		goto fail_2;
+		goto fail_1;
 	}
 
 	r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
 		            size,
 			    safe_dmamap_cb,
 			    &dma->dma_paddr,
 			    mapflags | BUS_DMA_NOWAIT);
 	if (r != 0) {
 		device_printf(sc->sc_dev, "safe_dma_malloc: "
 			"bus_dmamap_load failed; error %u\n", r);
-		goto fail_3;
+		goto fail_2;
 	}
 
 	dma->dma_size = size;
 	return (0);
 
-fail_3:
 	bus_dmamap_unload(dma->dma_tag, dma->dma_map);
 fail_2:
 	bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
 fail_1:
-	bus_dmamap_destroy(dma->dma_tag, dma->dma_map);
 	bus_dma_tag_destroy(dma->dma_tag);
 fail_0:
-	dma->dma_map = NULL;
 	dma->dma_tag = NULL;
 	return (r);
 }
 
 static void
 safe_dma_free(struct safe_softc *sc, struct safe_dma_alloc *dma)
 {
 	bus_dmamap_unload(dma->dma_tag, dma->dma_map);
 	bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
-	bus_dmamap_destroy(dma->dma_tag, dma->dma_map);
 	bus_dma_tag_destroy(dma->dma_tag);
 }
 
 /*
  * Resets the board.  Values in the regesters are left as is
  * from the reset (i.e. initial values are assigned elsewhere).
  */
 static void
 safe_reset_board(struct safe_softc *sc)
 {
 	u_int32_t v;
 	/*
 	 * Reset the device.  The manual says no delay
 	 * is needed between marking and clearing reset.
 	 */
 	v = READ_REG(sc, SAFE_PE_DMACFG) &~
 		(SAFE_PE_DMACFG_PERESET | SAFE_PE_DMACFG_PDRRESET |
 		 SAFE_PE_DMACFG_SGRESET);
 	WRITE_REG(sc, SAFE_PE_DMACFG, v
 				    | SAFE_PE_DMACFG_PERESET
 				    | SAFE_PE_DMACFG_PDRRESET
 				    | SAFE_PE_DMACFG_SGRESET);
 	WRITE_REG(sc, SAFE_PE_DMACFG, v);
 }
 
 /*
  * Initialize registers we need to touch only once.
  */
 static void
 safe_init_board(struct safe_softc *sc)
 {
 	u_int32_t v, dwords;
 
 	v = READ_REG(sc, SAFE_PE_DMACFG);
 	v &=~ SAFE_PE_DMACFG_PEMODE;
 	v |= SAFE_PE_DMACFG_FSENA		/* failsafe enable */
 	  |  SAFE_PE_DMACFG_GPRPCI		/* gather ring on PCI */
 	  |  SAFE_PE_DMACFG_SPRPCI		/* scatter ring on PCI */
 	  |  SAFE_PE_DMACFG_ESDESC		/* endian-swap descriptors */
 	  |  SAFE_PE_DMACFG_ESSA		/* endian-swap SA's */
 	  |  SAFE_PE_DMACFG_ESPDESC		/* endian-swap part. desc's */
 	  ;
 	WRITE_REG(sc, SAFE_PE_DMACFG, v);
 #if 0
 	/* XXX select byte swap based on host byte order */
 	WRITE_REG(sc, SAFE_ENDIAN, 0x1b);
 #endif
 	if (sc->sc_chiprev == SAFE_REV(1,0)) {
 		/*
 		 * Avoid large PCI DMA transfers.  Rev 1.0 has a bug where
 		 * "target mode transfers" done while the chip is DMA'ing
 		 * >1020 bytes cause the hardware to lockup.  To avoid this
 		 * we reduce the max PCI transfer size and use small source
 		 * particle descriptors (<= 256 bytes).
 		 */
 		WRITE_REG(sc, SAFE_DMA_CFG, 256);
 		device_printf(sc->sc_dev,
 			"Reduce max DMA size to %u words for rev %u.%u WAR\n",
 			(READ_REG(sc, SAFE_DMA_CFG)>>2) & 0xff,
 			SAFE_REV_MAJ(sc->sc_chiprev),
 			SAFE_REV_MIN(sc->sc_chiprev));
 	}
 
 	/* NB: operands+results are overlaid */
 	WRITE_REG(sc, SAFE_PE_PDRBASE, sc->sc_ringalloc.dma_paddr);
 	WRITE_REG(sc, SAFE_PE_RDRBASE, sc->sc_ringalloc.dma_paddr);
 	/*
 	 * Configure ring entry size and number of items in the ring.
 	 */
 	KASSERT((sizeof(struct safe_ringentry) % sizeof(u_int32_t)) == 0,
 		("PE ring entry not 32-bit aligned!"));
 	dwords = sizeof(struct safe_ringentry) / sizeof(u_int32_t);
 	WRITE_REG(sc, SAFE_PE_RINGCFG,
 		(dwords << SAFE_PE_RINGCFG_OFFSET_S) | SAFE_MAX_NQUEUE);
 	WRITE_REG(sc, SAFE_PE_RINGPOLL, 0);	/* disable polling */
 
 	WRITE_REG(sc, SAFE_PE_GRNGBASE, sc->sc_spalloc.dma_paddr);
 	WRITE_REG(sc, SAFE_PE_SRNGBASE, sc->sc_dpalloc.dma_paddr);
 	WRITE_REG(sc, SAFE_PE_PARTSIZE,
 		(SAFE_TOTAL_DPART<<16) | SAFE_TOTAL_SPART);
 	/*
 	 * NB: destination particles are fixed size.  We use
 	 *     an mbuf cluster and require all results go to
 	 *     clusters or smaller.
 	 */
 	WRITE_REG(sc, SAFE_PE_PARTCFG, SAFE_MAX_DSIZE);
 
 	/* it's now safe to enable PE mode, do it */
 	WRITE_REG(sc, SAFE_PE_DMACFG, v | SAFE_PE_DMACFG_PEMODE);
 
 	/*
 	 * Configure hardware to use level-triggered interrupts and
 	 * to interrupt after each descriptor is processed.
 	 */
 	WRITE_REG(sc, SAFE_HI_CFG, SAFE_HI_CFG_LEVEL);
 	WRITE_REG(sc, SAFE_HI_DESC_CNT, 1);
 	WRITE_REG(sc, SAFE_HI_MASK, SAFE_INT_PE_DDONE | SAFE_INT_PE_ERROR);
 }
 
 /*
  * Init PCI registers
  */
 static void
 safe_init_pciregs(device_t dev)
 {
 }
 
 /*
  * Clean up after a chip crash.
  * It is assumed that the caller in splimp()
  */
 static void
 safe_cleanchip(struct safe_softc *sc)
 {
 
 	if (sc->sc_nqchip != 0) {
 		struct safe_ringentry *re = sc->sc_back;
 
 		while (re != sc->sc_front) {
 			if (re->re_desc.d_csr != 0)
 				safe_free_entry(sc, re);
 			if (++re == sc->sc_ringtop)
 				re = sc->sc_ring;
 		}
 		sc->sc_back = re;
 		sc->sc_nqchip = 0;
 	}
 }
 
 /*
  * free a safe_q
  * It is assumed that the caller is within splimp().
  */
 static int
 safe_free_entry(struct safe_softc *sc, struct safe_ringentry *re)
 {
 	struct cryptop *crp;
 
 	/*
 	 * Free header MCR
 	 */
 	if ((re->re_dst_m != NULL) && (re->re_src_m != re->re_dst_m))
 		m_freem(re->re_dst_m);
 
 	crp = (struct cryptop *)re->re_crp;
 	
 	re->re_desc.d_csr = 0;
 	
 	crp->crp_etype = EFAULT;
 	crypto_done(crp);
 	return(0);
 }
 
 /*
  * Routine to reset the chip and clean up.
  * It is assumed that the caller is in splimp()
  */
 static void
 safe_totalreset(struct safe_softc *sc)
 {
 	safe_reset_board(sc);
 	safe_init_board(sc);
 	safe_cleanchip(sc);
 }
 
 /*
  * Is the operand suitable aligned for direct DMA.  Each
  * segment must be aligned on a 32-bit boundary and all
  * but the last segment must be a multiple of 4 bytes.
  */
 static int
 safe_dmamap_aligned(const struct safe_operand *op)
 {
 	int i;
 
 	for (i = 0; i < op->nsegs; i++) {
 		if (op->segs[i].ds_addr & 3)
 			return (0);
 		if (i != (op->nsegs - 1) && (op->segs[i].ds_len & 3))
 			return (0);
 	}
 	return (1);
 }
 
 /*
  * Is the operand suitable for direct DMA as the destination
  * of an operation.  The hardware requires that each ``particle''
  * but the last in an operation result have the same size.  We
  * fix that size at SAFE_MAX_DSIZE bytes.  This routine returns
  * 0 if some segment is not a multiple of of this size, 1 if all
  * segments are exactly this size, or 2 if segments are at worst
  * a multple of this size.
  */
 static int
 safe_dmamap_uniform(const struct safe_operand *op)
 {
 	int result = 1;
 
 	if (op->nsegs > 0) {
 		int i;
 
 		for (i = 0; i < op->nsegs-1; i++) {
 			if (op->segs[i].ds_len % SAFE_MAX_DSIZE)
 				return (0);
 			if (op->segs[i].ds_len != SAFE_MAX_DSIZE)
 				result = 2;
 		}
 	}
 	return (result);
 }
 
 #ifdef SAFE_DEBUG
 static void
 safe_dump_dmastatus(struct safe_softc *sc, const char *tag)
 {
 	printf("%s: ENDIAN 0x%x SRC 0x%x DST 0x%x STAT 0x%x\n"
 		, tag
 		, READ_REG(sc, SAFE_DMA_ENDIAN)
 		, READ_REG(sc, SAFE_DMA_SRCADDR)
 		, READ_REG(sc, SAFE_DMA_DSTADDR)
 		, READ_REG(sc, SAFE_DMA_STAT)
 	);
 }
 
 static void
 safe_dump_intrstate(struct safe_softc *sc, const char *tag)
 {
 	printf("%s: HI_CFG 0x%x HI_MASK 0x%x HI_DESC_CNT 0x%x HU_STAT 0x%x HM_STAT 0x%x\n"
 		, tag
 		, READ_REG(sc, SAFE_HI_CFG)
 		, READ_REG(sc, SAFE_HI_MASK)
 		, READ_REG(sc, SAFE_HI_DESC_CNT)
 		, READ_REG(sc, SAFE_HU_STAT)
 		, READ_REG(sc, SAFE_HM_STAT)
 	);
 }
 
 static void
 safe_dump_ringstate(struct safe_softc *sc, const char *tag)
 {
 	u_int32_t estat = READ_REG(sc, SAFE_PE_ERNGSTAT);
 
 	/* NB: assume caller has lock on ring */
 	printf("%s: ERNGSTAT %x (next %u) back %lu front %lu\n",
 		tag,
 		estat, (estat >> SAFE_PE_ERNGSTAT_NEXT_S),
 		(unsigned long)(sc->sc_back - sc->sc_ring),
 		(unsigned long)(sc->sc_front - sc->sc_ring));
 }
 
 static void
 safe_dump_request(struct safe_softc *sc, const char* tag, struct safe_ringentry *re)
 {
 	int ix, nsegs;
 
 	ix = re - sc->sc_ring;
 	printf("%s: %p (%u): csr %x src %x dst %x sa %x len %x\n"
 		, tag
 		, re, ix
 		, re->re_desc.d_csr
 		, re->re_desc.d_src
 		, re->re_desc.d_dst
 		, re->re_desc.d_sa
 		, re->re_desc.d_len
 	);
 	if (re->re_src.nsegs > 1) {
 		ix = (re->re_desc.d_src - sc->sc_spalloc.dma_paddr) /
 			sizeof(struct safe_pdesc);
 		for (nsegs = re->re_src.nsegs; nsegs; nsegs--) {
 			printf(" spd[%u] %p: %p size %u flags %x"
 				, ix, &sc->sc_spring[ix]
 				, (caddr_t)(uintptr_t) sc->sc_spring[ix].pd_addr
 				, sc->sc_spring[ix].pd_size
 				, sc->sc_spring[ix].pd_flags
 			);
 			if (sc->sc_spring[ix].pd_size == 0)
 				printf(" (zero!)");
 			printf("\n");
 			if (++ix == SAFE_TOTAL_SPART)
 				ix = 0;
 		}
 	}
 	if (re->re_dst.nsegs > 1) {
 		ix = (re->re_desc.d_dst - sc->sc_dpalloc.dma_paddr) /
 			sizeof(struct safe_pdesc);
 		for (nsegs = re->re_dst.nsegs; nsegs; nsegs--) {
 			printf(" dpd[%u] %p: %p flags %x\n"
 				, ix, &sc->sc_dpring[ix]
 				, (caddr_t)(uintptr_t) sc->sc_dpring[ix].pd_addr
 				, sc->sc_dpring[ix].pd_flags
 			);
 			if (++ix == SAFE_TOTAL_DPART)
 				ix = 0;
 		}
 	}
 	printf("sa: cmd0 %08x cmd1 %08x staterec %x\n",
 		re->re_sa.sa_cmd0, re->re_sa.sa_cmd1, re->re_sa.sa_staterec);
 	printf("sa: key %x %x %x %x %x %x %x %x\n"
 		, re->re_sa.sa_key[0]
 		, re->re_sa.sa_key[1]
 		, re->re_sa.sa_key[2]
 		, re->re_sa.sa_key[3]
 		, re->re_sa.sa_key[4]
 		, re->re_sa.sa_key[5]
 		, re->re_sa.sa_key[6]
 		, re->re_sa.sa_key[7]
 	);
 	printf("sa: indigest %x %x %x %x %x\n"
 		, re->re_sa.sa_indigest[0]
 		, re->re_sa.sa_indigest[1]
 		, re->re_sa.sa_indigest[2]
 		, re->re_sa.sa_indigest[3]
 		, re->re_sa.sa_indigest[4]
 	);
 	printf("sa: outdigest %x %x %x %x %x\n"
 		, re->re_sa.sa_outdigest[0]
 		, re->re_sa.sa_outdigest[1]
 		, re->re_sa.sa_outdigest[2]
 		, re->re_sa.sa_outdigest[3]
 		, re->re_sa.sa_outdigest[4]
 	);
 	printf("sr: iv %x %x %x %x\n"
 		, re->re_sastate.sa_saved_iv[0]
 		, re->re_sastate.sa_saved_iv[1]
 		, re->re_sastate.sa_saved_iv[2]
 		, re->re_sastate.sa_saved_iv[3]
 	);
 	printf("sr: hashbc %u indigest %x %x %x %x %x\n"
 		, re->re_sastate.sa_saved_hashbc
 		, re->re_sastate.sa_saved_indigest[0]
 		, re->re_sastate.sa_saved_indigest[1]
 		, re->re_sastate.sa_saved_indigest[2]
 		, re->re_sastate.sa_saved_indigest[3]
 		, re->re_sastate.sa_saved_indigest[4]
 	);
 }
 
 static void
 safe_dump_ring(struct safe_softc *sc, const char *tag)
 {
 	mtx_lock(&sc->sc_ringmtx);
 	printf("\nSafeNet Ring State:\n");
 	safe_dump_intrstate(sc, tag);
 	safe_dump_dmastatus(sc, tag);
 	safe_dump_ringstate(sc, tag);
 	if (sc->sc_nqchip) {
 		struct safe_ringentry *re = sc->sc_back;
 		do {
 			safe_dump_request(sc, tag, re);
 			if (++re == sc->sc_ringtop)
 				re = sc->sc_ring;
 		} while (re != sc->sc_front);
 	}
 	mtx_unlock(&sc->sc_ringmtx);
 }
 
 static int
 sysctl_hw_safe_dump(SYSCTL_HANDLER_ARGS)
 {
 	char dmode[64];
 	int error;
 
 	strncpy(dmode, "", sizeof(dmode) - 1);
 	dmode[sizeof(dmode) - 1] = '\0';
 	error = sysctl_handle_string(oidp, &dmode[0], sizeof(dmode), req);
 
 	if (error == 0 && req->newptr != NULL) {
 		struct safe_softc *sc = safec;
 
 		if (!sc)
 			return EINVAL;
 		if (strncmp(dmode, "dma", 3) == 0)
 			safe_dump_dmastatus(sc, "safe0");
 		else if (strncmp(dmode, "int", 3) == 0)
 			safe_dump_intrstate(sc, "safe0");
 		else if (strncmp(dmode, "ring", 4) == 0)
 			safe_dump_ring(sc, "safe0");
 		else
 			return EINVAL;
 	}
 	return error;
 }
 SYSCTL_PROC(_hw_safe, OID_AUTO, dump, CTLTYPE_STRING | CTLFLAG_RW,
 	0, 0, sysctl_hw_safe_dump, "A", "Dump driver state");
 #endif /* SAFE_DEBUG */
Index: projects/release-debugdist/sys/dev/sym/sym_hipd.c
===================================================================
--- projects/release-debugdist/sys/dev/sym/sym_hipd.c	(revision 267343)
+++ projects/release-debugdist/sys/dev/sym/sym_hipd.c	(revision 267344)
@@ -1,9626 +1,9622 @@
 /*-
  *  Device driver optimized for the Symbios/LSI 53C896/53C895A/53C1010
  *  PCI-SCSI controllers.
  *
  *  Copyright (C) 1999-2001  Gerard Roudier <groudier@free.fr>
  *
  *  This driver also supports the following Symbios/LSI PCI-SCSI chips:
  *	53C810A, 53C825A, 53C860, 53C875, 53C876, 53C885, 53C895,
  *	53C810,  53C815,  53C825 and the 53C1510D is 53C8XX mode.
  *
  *
  *  This driver for FreeBSD-CAM is derived from the Linux sym53c8xx driver.
  *  Copyright (C) 1998-1999  Gerard Roudier
  *
  *  The sym53c8xx driver is derived from the ncr53c8xx driver that had been
  *  a port of the FreeBSD ncr driver to Linux-1.2.13.
  *
  *  The original ncr driver has been written for 386bsd and FreeBSD by
  *          Wolfgang Stanglmeier        <wolf@cologne.de>
  *          Stefan Esser                <se@mi.Uni-Koeln.de>
  *  Copyright (C) 1994  Wolfgang Stanglmeier
  *
  *  The initialisation code, and part of the code that addresses
  *  FreeBSD-CAM services is based on the aic7xxx driver for FreeBSD-CAM
  *  written by Justin T. Gibbs.
  *
  *  Other major contributions:
  *
  *  NVRAM detection and reading.
  *  Copyright (C) 1997 Richard Waltham <dormouse@farsrobt.demon.co.uk>
  *
  *-----------------------------------------------------------------------------
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  * 3. The name of the author may not be used to endorse or promote products
  *    derived from this software without specific prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #define SYM_DRIVER_NAME	"sym-1.6.5-20000902"
 
 /* #define SYM_DEBUG_GENERIC_SUPPORT */
 
 #include <sys/param.h>
 
 /*
  *  Driver configuration options.
  */
 #include "opt_sym.h"
 #include <dev/sym/sym_conf.h>
 
 #include <sys/systm.h>
 #include <sys/malloc.h>
 #include <sys/endian.h>
 #include <sys/kernel.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <sys/module.h>
 #include <sys/bus.h>
 
 #include <sys/proc.h>
 
 #include <dev/pci/pcireg.h>
 #include <dev/pci/pcivar.h>
 
 #include <machine/bus.h>
 #include <machine/resource.h>
 #include <machine/atomic.h>
 
 #ifdef __sparc64__
 #include <dev/ofw/openfirm.h>
 #include <machine/ofw_machdep.h>
 #endif
 
 #include <sys/rman.h>
 
 #include <cam/cam.h>
 #include <cam/cam_ccb.h>
 #include <cam/cam_sim.h>
 #include <cam/cam_xpt_sim.h>
 #include <cam/cam_debug.h>
 
 #include <cam/scsi/scsi_all.h>
 #include <cam/scsi/scsi_message.h>
 
 /* Short and quite clear integer types */
 typedef int8_t    s8;
 typedef int16_t   s16;
 typedef	int32_t   s32;
 typedef u_int8_t  u8;
 typedef u_int16_t u16;
 typedef	u_int32_t u32;
 
 /*
  *  Driver definitions.
  */
 #include <dev/sym/sym_defs.h>
 #include <dev/sym/sym_fw.h>
 
 /*
  *  IA32 architecture does not reorder STORES and prevents
  *  LOADS from passing STORES. It is called `program order'
  *  by Intel and allows device drivers to deal with memory
  *  ordering by only ensuring that the code is not reordered
  *  by the compiler when ordering is required.
  *  Other architectures implement a weaker ordering that
  *  requires memory barriers (and also IO barriers when they
  *  make sense) to be used.
  */
 #if	defined	__i386__ || defined __amd64__
 #define MEMORY_BARRIER()	do { ; } while(0)
 #elif	defined	__powerpc__
 #define MEMORY_BARRIER()	__asm__ volatile("eieio; sync" : : : "memory")
 #elif	defined	__ia64__
 #define MEMORY_BARRIER()	__asm__ volatile("mf.a; mf" : : : "memory")
 #elif	defined	__sparc64__
 #define MEMORY_BARRIER()	__asm__ volatile("membar #Sync" : : : "memory")
 #elif	defined	__arm__
 #define MEMORY_BARRIER()	dmb()
 #else
 #error	"Not supported platform"
 #endif
 
 /*
  *  A la VMS/CAM-3 queue management.
  */
 typedef struct sym_quehead {
 	struct sym_quehead *flink;	/* Forward  pointer */
 	struct sym_quehead *blink;	/* Backward pointer */
 } SYM_QUEHEAD;
 
 #define sym_que_init(ptr) do { \
 	(ptr)->flink = (ptr); (ptr)->blink = (ptr); \
 } while (0)
 
 static __inline void __sym_que_add(struct sym_quehead * new,
 	struct sym_quehead * blink,
 	struct sym_quehead * flink)
 {
 	flink->blink	= new;
 	new->flink	= flink;
 	new->blink	= blink;
 	blink->flink	= new;
 }
 
 static __inline void __sym_que_del(struct sym_quehead * blink,
 	struct sym_quehead * flink)
 {
 	flink->blink = blink;
 	blink->flink = flink;
 }
 
 static __inline int sym_que_empty(struct sym_quehead *head)
 {
 	return head->flink == head;
 }
 
 static __inline void sym_que_splice(struct sym_quehead *list,
 	struct sym_quehead *head)
 {
 	struct sym_quehead *first = list->flink;
 
 	if (first != list) {
 		struct sym_quehead *last = list->blink;
 		struct sym_quehead *at   = head->flink;
 
 		first->blink = head;
 		head->flink  = first;
 
 		last->flink = at;
 		at->blink   = last;
 	}
 }
 
 #define sym_que_entry(ptr, type, member) \
 	((type *)((char *)(ptr)-(size_t)(&((type *)0)->member)))
 
 #define sym_insque(new, pos)		__sym_que_add(new, pos, (pos)->flink)
 
 #define sym_remque(el)			__sym_que_del((el)->blink, (el)->flink)
 
 #define sym_insque_head(new, head)	__sym_que_add(new, head, (head)->flink)
 
 static __inline struct sym_quehead *sym_remque_head(struct sym_quehead *head)
 {
 	struct sym_quehead *elem = head->flink;
 
 	if (elem != head)
 		__sym_que_del(head, elem->flink);
 	else
 		elem = NULL;
 	return elem;
 }
 
 #define sym_insque_tail(new, head)	__sym_que_add(new, (head)->blink, head)
 
 /*
  *  This one may be useful.
  */
 #define FOR_EACH_QUEUED_ELEMENT(head, qp) \
 	for (qp = (head)->flink; qp != (head); qp = qp->flink)
 /*
  *  FreeBSD does not offer our kind of queue in the CAM CCB.
  *  So, we have to cast.
  */
 #define sym_qptr(p)	((struct sym_quehead *) (p))
 
 /*
  *  Simple bitmap operations.
  */
 #define sym_set_bit(p, n)	(((u32 *)(p))[(n)>>5] |=  (1<<((n)&0x1f)))
 #define sym_clr_bit(p, n)	(((u32 *)(p))[(n)>>5] &= ~(1<<((n)&0x1f)))
 #define sym_is_bit(p, n)	(((u32 *)(p))[(n)>>5] &   (1<<((n)&0x1f)))
 
 /*
  *  Number of tasks per device we want to handle.
  */
 #if	SYM_CONF_MAX_TAG_ORDER > 8
 #error	"more than 256 tags per logical unit not allowed."
 #endif
 #define	SYM_CONF_MAX_TASK	(1<<SYM_CONF_MAX_TAG_ORDER)
 
 /*
  *  Donnot use more tasks that we can handle.
  */
 #ifndef	SYM_CONF_MAX_TAG
 #define	SYM_CONF_MAX_TAG	SYM_CONF_MAX_TASK
 #endif
 #if	SYM_CONF_MAX_TAG > SYM_CONF_MAX_TASK
 #undef	SYM_CONF_MAX_TAG
 #define	SYM_CONF_MAX_TAG	SYM_CONF_MAX_TASK
 #endif
 
 /*
  *    This one means 'NO TAG for this job'
  */
 #define NO_TAG	(256)
 
 /*
  *  Number of SCSI targets.
  */
 #if	SYM_CONF_MAX_TARGET > 16
 #error	"more than 16 targets not allowed."
 #endif
 
 /*
  *  Number of logical units per target.
  */
 #if	SYM_CONF_MAX_LUN > 64
 #error	"more than 64 logical units per target not allowed."
 #endif
 
 /*
  *    Asynchronous pre-scaler (ns). Shall be 40 for
  *    the SCSI timings to be compliant.
  */
 #define	SYM_CONF_MIN_ASYNC (40)
 
 /*
  *  Number of entries in the START and DONE queues.
  *
  *  We limit to 1 PAGE in order to succeed allocation of
  *  these queues. Each entry is 8 bytes long (2 DWORDS).
  */
 #ifdef	SYM_CONF_MAX_START
 #define	SYM_CONF_MAX_QUEUE (SYM_CONF_MAX_START+2)
 #else
 #define	SYM_CONF_MAX_QUEUE (7*SYM_CONF_MAX_TASK+2)
 #define	SYM_CONF_MAX_START (SYM_CONF_MAX_QUEUE-2)
 #endif
 
 #if	SYM_CONF_MAX_QUEUE > PAGE_SIZE/8
 #undef	SYM_CONF_MAX_QUEUE
 #define	SYM_CONF_MAX_QUEUE   PAGE_SIZE/8
 #undef	SYM_CONF_MAX_START
 #define	SYM_CONF_MAX_START (SYM_CONF_MAX_QUEUE-2)
 #endif
 
 /*
  *  For this one, we want a short name :-)
  */
 #define MAX_QUEUE	SYM_CONF_MAX_QUEUE
 
 /*
  *  Active debugging tags and verbosity.
  */
 #define DEBUG_ALLOC	(0x0001)
 #define DEBUG_PHASE	(0x0002)
 #define DEBUG_POLL	(0x0004)
 #define DEBUG_QUEUE	(0x0008)
 #define DEBUG_RESULT	(0x0010)
 #define DEBUG_SCATTER	(0x0020)
 #define DEBUG_SCRIPT	(0x0040)
 #define DEBUG_TINY	(0x0080)
 #define DEBUG_TIMING	(0x0100)
 #define DEBUG_NEGO	(0x0200)
 #define DEBUG_TAGS	(0x0400)
 #define DEBUG_POINTER	(0x0800)
 
 #if 0
 static int sym_debug = 0;
 	#define DEBUG_FLAGS sym_debug
 #else
 /*	#define DEBUG_FLAGS (0x0631) */
 	#define DEBUG_FLAGS (0x0000)
 
 #endif
 #define sym_verbose	(np->verbose)
 
 /*
  *  Insert a delay in micro-seconds and milli-seconds.
  */
 static void UDELAY(int us) { DELAY(us); }
 static void MDELAY(int ms) { while (ms--) UDELAY(1000); }
 
 /*
  *  Simple power of two buddy-like allocator.
  *
  *  This simple code is not intended to be fast, but to
  *  provide power of 2 aligned memory allocations.
  *  Since the SCRIPTS processor only supplies 8 bit arithmetic,
  *  this allocator allows simple and fast address calculations
  *  from the SCRIPTS code. In addition, cache line alignment
  *  is guaranteed for power of 2 cache line size.
  *
  *  This allocator has been developed for the Linux sym53c8xx
  *  driver, since this O/S does not provide naturally aligned
  *  allocations.
  *  It has the advantage of allowing the driver to use private
  *  pages of memory that will be useful if we ever need to deal
  *  with IO MMUs for PCI.
  */
 #define MEMO_SHIFT	4	/* 16 bytes minimum memory chunk */
 #define MEMO_PAGE_ORDER	0	/* 1 PAGE  maximum */
 #if 0
 #define MEMO_FREE_UNUSED	/* Free unused pages immediately */
 #endif
 #define MEMO_WARN	1
 #define MEMO_CLUSTER_SHIFT	(PAGE_SHIFT+MEMO_PAGE_ORDER)
 #define MEMO_CLUSTER_SIZE	(1UL << MEMO_CLUSTER_SHIFT)
 #define MEMO_CLUSTER_MASK	(MEMO_CLUSTER_SIZE-1)
 
 #define get_pages()		malloc(MEMO_CLUSTER_SIZE, M_DEVBUF, M_NOWAIT)
 #define free_pages(p)		free((p), M_DEVBUF)
 
 typedef u_long m_addr_t;	/* Enough bits to bit-hack addresses */
 
 typedef struct m_link {		/* Link between free memory chunks */
 	struct m_link *next;
 } m_link_s;
 
 typedef struct m_vtob {		/* Virtual to Bus address translation */
 	struct m_vtob	*next;
 	bus_dmamap_t	dmamap;	/* Map for this chunk */
 	m_addr_t	vaddr;	/* Virtual address */
 	m_addr_t	baddr;	/* Bus physical address */
 } m_vtob_s;
 /* Hash this stuff a bit to speed up translations */
 #define VTOB_HASH_SHIFT		5
 #define VTOB_HASH_SIZE		(1UL << VTOB_HASH_SHIFT)
 #define VTOB_HASH_MASK		(VTOB_HASH_SIZE-1)
 #define VTOB_HASH_CODE(m)	\
 	((((m_addr_t) (m)) >> MEMO_CLUSTER_SHIFT) & VTOB_HASH_MASK)
 
 typedef struct m_pool {		/* Memory pool of a given kind */
 	bus_dma_tag_t	 dev_dmat;	/* Identifies the pool */
 	bus_dma_tag_t	 dmat;		/* Tag for our fixed allocations */
 	m_addr_t (*getp)(struct m_pool *);
 #ifdef	MEMO_FREE_UNUSED
 	void (*freep)(struct m_pool *, m_addr_t);
 #endif
 #define M_GETP()		mp->getp(mp)
 #define M_FREEP(p)		mp->freep(mp, p)
 	int nump;
 	m_vtob_s *(vtob[VTOB_HASH_SIZE]);
 	struct m_pool *next;
 	struct m_link h[MEMO_CLUSTER_SHIFT - MEMO_SHIFT + 1];
 } m_pool_s;
 
 static void *___sym_malloc(m_pool_s *mp, int size)
 {
 	int i = 0;
 	int s = (1 << MEMO_SHIFT);
 	int j;
 	m_addr_t a;
 	m_link_s *h = mp->h;
 
 	if (size > MEMO_CLUSTER_SIZE)
 		return NULL;
 
 	while (size > s) {
 		s <<= 1;
 		++i;
 	}
 
 	j = i;
 	while (!h[j].next) {
 		if (s == MEMO_CLUSTER_SIZE) {
 			h[j].next = (m_link_s *) M_GETP();
 			if (h[j].next)
 				h[j].next->next = NULL;
 			break;
 		}
 		++j;
 		s <<= 1;
 	}
 	a = (m_addr_t) h[j].next;
 	if (a) {
 		h[j].next = h[j].next->next;
 		while (j > i) {
 			j -= 1;
 			s >>= 1;
 			h[j].next = (m_link_s *) (a+s);
 			h[j].next->next = NULL;
 		}
 	}
 #ifdef DEBUG
 	printf("___sym_malloc(%d) = %p\n", size, (void *) a);
 #endif
 	return (void *) a;
 }
 
 static void ___sym_mfree(m_pool_s *mp, void *ptr, int size)
 {
 	int i = 0;
 	int s = (1 << MEMO_SHIFT);
 	m_link_s *q;
 	m_addr_t a, b;
 	m_link_s *h = mp->h;
 
 #ifdef DEBUG
 	printf("___sym_mfree(%p, %d)\n", ptr, size);
 #endif
 
 	if (size > MEMO_CLUSTER_SIZE)
 		return;
 
 	while (size > s) {
 		s <<= 1;
 		++i;
 	}
 
 	a = (m_addr_t) ptr;
 
 	while (1) {
 #ifdef MEMO_FREE_UNUSED
 		if (s == MEMO_CLUSTER_SIZE) {
 			M_FREEP(a);
 			break;
 		}
 #endif
 		b = a ^ s;
 		q = &h[i];
 		while (q->next && q->next != (m_link_s *) b) {
 			q = q->next;
 		}
 		if (!q->next) {
 			((m_link_s *) a)->next = h[i].next;
 			h[i].next = (m_link_s *) a;
 			break;
 		}
 		q->next = q->next->next;
 		a = a & b;
 		s <<= 1;
 		++i;
 	}
 }
 
 static void *__sym_calloc2(m_pool_s *mp, int size, char *name, int uflags)
 {
 	void *p;
 
 	p = ___sym_malloc(mp, size);
 
 	if (DEBUG_FLAGS & DEBUG_ALLOC)
 		printf ("new %-10s[%4d] @%p.\n", name, size, p);
 
 	if (p)
 		bzero(p, size);
 	else if (uflags & MEMO_WARN)
 		printf ("__sym_calloc2: failed to allocate %s[%d]\n", name, size);
 
 	return p;
 }
 
 #define __sym_calloc(mp, s, n)	__sym_calloc2(mp, s, n, MEMO_WARN)
 
 static void __sym_mfree(m_pool_s *mp, void *ptr, int size, char *name)
 {
 	if (DEBUG_FLAGS & DEBUG_ALLOC)
 		printf ("freeing %-10s[%4d] @%p.\n", name, size, ptr);
 
 	___sym_mfree(mp, ptr, size);
 
 }
 
 /*
  * Default memory pool we donnot need to involve in DMA.
  */
 /*
  * With the `bus dma abstraction', we use a separate pool for
  * memory we donnot need to involve in DMA.
  */
 static m_addr_t ___mp0_getp(m_pool_s *mp)
 {
 	m_addr_t m = (m_addr_t) get_pages();
 	if (m)
 		++mp->nump;
 	return m;
 }
 
 #ifdef	MEMO_FREE_UNUSED
 static void ___mp0_freep(m_pool_s *mp, m_addr_t m)
 {
 	free_pages(m);
 	--mp->nump;
 }
 #endif
 
 #ifdef	MEMO_FREE_UNUSED
 static m_pool_s mp0 = {0, 0, ___mp0_getp, ___mp0_freep};
 #else
 static m_pool_s mp0 = {0, 0, ___mp0_getp};
 #endif
 
 /*
  * Actual memory allocation routine for non-DMAed memory.
  */
 static void *sym_calloc(int size, char *name)
 {
 	void *m;
 	/* Lock */
 	m = __sym_calloc(&mp0, size, name);
 	/* Unlock */
 	return m;
 }
 
 /*
  * Actual memory allocation routine for non-DMAed memory.
  */
 static void sym_mfree(void *ptr, int size, char *name)
 {
 	/* Lock */
 	__sym_mfree(&mp0, ptr, size, name);
 	/* Unlock */
 }
 
 /*
  * DMAable pools.
  */
 /*
  * With `bus dma abstraction', we use a separate pool per parent
  * BUS handle. A reverse table (hashed) is maintained for virtual
  * to BUS address translation.
  */
 static void getbaddrcb(void *arg, bus_dma_segment_t *segs, int nseg __unused,
     int error)
 {
 	bus_addr_t *baddr;
 
 	KASSERT(nseg == 1, ("%s: too many DMA segments (%d)", __func__, nseg));
 
 	baddr = (bus_addr_t *)arg;
 	if (error)
 		*baddr = 0;
 	else
 		*baddr = segs->ds_addr;
 }
 
 static m_addr_t ___dma_getp(m_pool_s *mp)
 {
 	m_vtob_s *vbp;
 	void *vaddr = NULL;
 	bus_addr_t baddr = 0;
 
 	vbp = __sym_calloc(&mp0, sizeof(*vbp), "VTOB");
 	if (!vbp)
 		goto out_err;
 
 	if (bus_dmamem_alloc(mp->dmat, &vaddr,
 			BUS_DMA_COHERENT | BUS_DMA_WAITOK, &vbp->dmamap))
 		goto out_err;
 	bus_dmamap_load(mp->dmat, vbp->dmamap, vaddr,
 			MEMO_CLUSTER_SIZE, getbaddrcb, &baddr, BUS_DMA_NOWAIT);
 	if (baddr) {
 		int hc = VTOB_HASH_CODE(vaddr);
 		vbp->vaddr = (m_addr_t) vaddr;
 		vbp->baddr = (m_addr_t) baddr;
 		vbp->next = mp->vtob[hc];
 		mp->vtob[hc] = vbp;
 		++mp->nump;
 		return (m_addr_t) vaddr;
 	}
 out_err:
 	if (baddr)
 		bus_dmamap_unload(mp->dmat, vbp->dmamap);
 	if (vaddr)
 		bus_dmamem_free(mp->dmat, vaddr, vbp->dmamap);
-	if (vbp) {
-		if (vbp->dmamap)
-			bus_dmamap_destroy(mp->dmat, vbp->dmamap);
+	if (vbp)
 		__sym_mfree(&mp0, vbp, sizeof(*vbp), "VTOB");
-	}
 	return 0;
 }
 
 #ifdef	MEMO_FREE_UNUSED
 static void ___dma_freep(m_pool_s *mp, m_addr_t m)
 {
 	m_vtob_s **vbpp, *vbp;
 	int hc = VTOB_HASH_CODE(m);
 
 	vbpp = &mp->vtob[hc];
 	while (*vbpp && (*vbpp)->vaddr != m)
 		vbpp = &(*vbpp)->next;
 	if (*vbpp) {
 		vbp = *vbpp;
 		*vbpp = (*vbpp)->next;
 		bus_dmamap_unload(mp->dmat, vbp->dmamap);
 		bus_dmamem_free(mp->dmat, (void *) vbp->vaddr, vbp->dmamap);
-		bus_dmamap_destroy(mp->dmat, vbp->dmamap);
 		__sym_mfree(&mp0, vbp, sizeof(*vbp), "VTOB");
 		--mp->nump;
 	}
 }
 #endif
 
 static __inline m_pool_s *___get_dma_pool(bus_dma_tag_t dev_dmat)
 {
 	m_pool_s *mp;
 	for (mp = mp0.next; mp && mp->dev_dmat != dev_dmat; mp = mp->next);
 	return mp;
 }
 
 static m_pool_s *___cre_dma_pool(bus_dma_tag_t dev_dmat)
 {
 	m_pool_s *mp = NULL;
 
 	mp = __sym_calloc(&mp0, sizeof(*mp), "MPOOL");
 	if (mp) {
 		mp->dev_dmat = dev_dmat;
 		if (!bus_dma_tag_create(dev_dmat, 1, MEMO_CLUSTER_SIZE,
 			       BUS_SPACE_MAXADDR_32BIT,
 			       BUS_SPACE_MAXADDR,
 			       NULL, NULL, MEMO_CLUSTER_SIZE, 1,
 			       MEMO_CLUSTER_SIZE, 0,
 			       NULL, NULL, &mp->dmat)) {
 			mp->getp = ___dma_getp;
 #ifdef	MEMO_FREE_UNUSED
 			mp->freep = ___dma_freep;
 #endif
 			mp->next = mp0.next;
 			mp0.next = mp;
 			return mp;
 		}
 	}
 	if (mp)
 		__sym_mfree(&mp0, mp, sizeof(*mp), "MPOOL");
 	return NULL;
 }
 
 #ifdef	MEMO_FREE_UNUSED
 static void ___del_dma_pool(m_pool_s *p)
 {
 	struct m_pool **pp = &mp0.next;
 
 	while (*pp && *pp != p)
 		pp = &(*pp)->next;
 	if (*pp) {
 		*pp = (*pp)->next;
 		bus_dma_tag_destroy(p->dmat);
 		__sym_mfree(&mp0, p, sizeof(*p), "MPOOL");
 	}
 }
 #endif
 
 static void *__sym_calloc_dma(bus_dma_tag_t dev_dmat, int size, char *name)
 {
 	struct m_pool *mp;
 	void *m = NULL;
 
 	/* Lock */
 	mp = ___get_dma_pool(dev_dmat);
 	if (!mp)
 		mp = ___cre_dma_pool(dev_dmat);
 	if (mp)
 		m = __sym_calloc(mp, size, name);
 #ifdef	MEMO_FREE_UNUSED
 	if (mp && !mp->nump)
 		___del_dma_pool(mp);
 #endif
 	/* Unlock */
 
 	return m;
 }
 
 static void
 __sym_mfree_dma(bus_dma_tag_t dev_dmat, void *m, int size, char *name)
 {
 	struct m_pool *mp;
 
 	/* Lock */
 	mp = ___get_dma_pool(dev_dmat);
 	if (mp)
 		__sym_mfree(mp, m, size, name);
 #ifdef	MEMO_FREE_UNUSED
 	if (mp && !mp->nump)
 		___del_dma_pool(mp);
 #endif
 	/* Unlock */
 }
 
 static m_addr_t __vtobus(bus_dma_tag_t dev_dmat, void *m)
 {
 	m_pool_s *mp;
 	int hc = VTOB_HASH_CODE(m);
 	m_vtob_s *vp = NULL;
 	m_addr_t a = ((m_addr_t) m) & ~MEMO_CLUSTER_MASK;
 
 	/* Lock */
 	mp = ___get_dma_pool(dev_dmat);
 	if (mp) {
 		vp = mp->vtob[hc];
 		while (vp && (m_addr_t) vp->vaddr != a)
 			vp = vp->next;
 	}
 	/* Unlock */
 	if (!vp)
 		panic("sym: VTOBUS FAILED!\n");
 	return vp ? vp->baddr + (((m_addr_t) m) - a) : 0;
 }
 
 /*
  * Verbs for DMAable memory handling.
  * The _uvptv_ macro avoids a nasty warning about pointer to volatile
  * being discarded.
  */
 #define _uvptv_(p) ((void *)((vm_offset_t)(p)))
 #define _sym_calloc_dma(np, s, n)	__sym_calloc_dma(np->bus_dmat, s, n)
 #define _sym_mfree_dma(np, p, s, n)	\
 				__sym_mfree_dma(np->bus_dmat, _uvptv_(p), s, n)
 #define sym_calloc_dma(s, n)		_sym_calloc_dma(np, s, n)
 #define sym_mfree_dma(p, s, n)		_sym_mfree_dma(np, p, s, n)
 #define _vtobus(np, p)			__vtobus(np->bus_dmat, _uvptv_(p))
 #define vtobus(p)			_vtobus(np, p)
 
 /*
  *  Print a buffer in hexadecimal format.
  */
 static void sym_printb_hex (u_char *p, int n)
 {
 	while (n-- > 0)
 		printf (" %x", *p++);
 }
 
 /*
  *  Same with a label at beginning and .\n at end.
  */
 static void sym_printl_hex (char *label, u_char *p, int n)
 {
 	printf ("%s", label);
 	sym_printb_hex (p, n);
 	printf (".\n");
 }
 
 /*
  *  Return a string for SCSI BUS mode.
  */
 static const char *sym_scsi_bus_mode(int mode)
 {
 	switch(mode) {
 	case SMODE_HVD:	return "HVD";
 	case SMODE_SE:	return "SE";
 	case SMODE_LVD: return "LVD";
 	}
 	return "??";
 }
 
 /*
  *  Some poor and bogus sync table that refers to Tekram NVRAM layout.
  */
 #ifdef SYM_CONF_NVRAM_SUPPORT
 static const u_char Tekram_sync[16] =
 	{25,31,37,43, 50,62,75,125, 12,15,18,21, 6,7,9,10};
 #endif
 
 /*
  *  Union of supported NVRAM formats.
  */
 struct sym_nvram {
 	int type;
 #define	SYM_SYMBIOS_NVRAM	(1)
 #define	SYM_TEKRAM_NVRAM	(2)
 #ifdef	SYM_CONF_NVRAM_SUPPORT
 	union {
 		Symbios_nvram Symbios;
 		Tekram_nvram Tekram;
 	} data;
 #endif
 };
 
 /*
  *  This one is hopefully useless, but actually useful. :-)
  */
 #ifndef assert
 #define	assert(expression) { \
 	if (!(expression)) { \
 		(void)panic( \
 			"assertion \"%s\" failed: file \"%s\", line %d\n", \
 			#expression, \
 			__FILE__, __LINE__); \
 	} \
 }
 #endif
 
 /*
  *  Some provision for a possible big endian mode supported by
  *  Symbios chips (never seen, by the way).
  *  For now, this stuff does not deserve any comments. :)
  */
 #define sym_offb(o)	(o)
 #define sym_offw(o)	(o)
 
 /*
  *  Some provision for support for BIG ENDIAN CPU.
  */
 #define cpu_to_scr(dw)	htole32(dw)
 #define scr_to_cpu(dw)	le32toh(dw)
 
 /*
  *  Access to the chip IO registers and on-chip RAM.
  *  We use the `bus space' interface under FreeBSD-4 and
  *  later kernel versions.
  */
 #if defined(SYM_CONF_IOMAPPED)
 
 #define INB_OFF(o)	bus_read_1(np->io_res, (o))
 #define INW_OFF(o)	bus_read_2(np->io_res, (o))
 #define INL_OFF(o)	bus_read_4(np->io_res, (o))
 
 #define OUTB_OFF(o, v)	bus_write_1(np->io_res, (o), (v))
 #define OUTW_OFF(o, v)	bus_write_2(np->io_res, (o), (v))
 #define OUTL_OFF(o, v)	bus_write_4(np->io_res, (o), (v))
 
 #else	/* Memory mapped IO */
 
 #define INB_OFF(o)	bus_read_1(np->mmio_res, (o))
 #define INW_OFF(o)	bus_read_2(np->mmio_res, (o))
 #define INL_OFF(o)	bus_read_4(np->mmio_res, (o))
 
 #define OUTB_OFF(o, v)	bus_write_1(np->mmio_res, (o), (v))
 #define OUTW_OFF(o, v)	bus_write_2(np->mmio_res, (o), (v))
 #define OUTL_OFF(o, v)	bus_write_4(np->mmio_res, (o), (v))
 
 #endif	/* SYM_CONF_IOMAPPED */
 
 #define OUTRAM_OFF(o, a, l)	\
 	bus_write_region_1(np->ram_res, (o), (a), (l))
 
 /*
  *  Common definitions for both bus space and legacy IO methods.
  */
 #define INB(r)		INB_OFF(offsetof(struct sym_reg,r))
 #define INW(r)		INW_OFF(offsetof(struct sym_reg,r))
 #define INL(r)		INL_OFF(offsetof(struct sym_reg,r))
 
 #define OUTB(r, v)	OUTB_OFF(offsetof(struct sym_reg,r), (v))
 #define OUTW(r, v)	OUTW_OFF(offsetof(struct sym_reg,r), (v))
 #define OUTL(r, v)	OUTL_OFF(offsetof(struct sym_reg,r), (v))
 
 #define OUTONB(r, m)	OUTB(r, INB(r) | (m))
 #define OUTOFFB(r, m)	OUTB(r, INB(r) & ~(m))
 #define OUTONW(r, m)	OUTW(r, INW(r) | (m))
 #define OUTOFFW(r, m)	OUTW(r, INW(r) & ~(m))
 #define OUTONL(r, m)	OUTL(r, INL(r) | (m))
 #define OUTOFFL(r, m)	OUTL(r, INL(r) & ~(m))
 
 /*
  *  We normally want the chip to have a consistent view
  *  of driver internal data structures when we restart it.
  *  Thus these macros.
  */
 #define OUTL_DSP(v)				\
 	do {					\
 		MEMORY_BARRIER();		\
 		OUTL (nc_dsp, (v));		\
 	} while (0)
 
 #define OUTONB_STD()				\
 	do {					\
 		MEMORY_BARRIER();		\
 		OUTONB (nc_dcntl, (STD|NOCOM));	\
 	} while (0)
 
 /*
  *  Command control block states.
  */
 #define HS_IDLE		(0)
 #define HS_BUSY		(1)
 #define HS_NEGOTIATE	(2)	/* sync/wide data transfer*/
 #define HS_DISCONNECT	(3)	/* Disconnected by target */
 #define HS_WAIT		(4)	/* waiting for resource	  */
 
 #define HS_DONEMASK	(0x80)
 #define HS_COMPLETE	(4|HS_DONEMASK)
 #define HS_SEL_TIMEOUT	(5|HS_DONEMASK)	/* Selection timeout      */
 #define HS_UNEXPECTED	(6|HS_DONEMASK)	/* Unexpected disconnect  */
 #define HS_COMP_ERR	(7|HS_DONEMASK)	/* Completed with error	  */
 
 /*
  *  Software Interrupt Codes
  */
 #define	SIR_BAD_SCSI_STATUS	(1)
 #define	SIR_SEL_ATN_NO_MSG_OUT	(2)
 #define	SIR_MSG_RECEIVED	(3)
 #define	SIR_MSG_WEIRD		(4)
 #define	SIR_NEGO_FAILED		(5)
 #define	SIR_NEGO_PROTO		(6)
 #define	SIR_SCRIPT_STOPPED	(7)
 #define	SIR_REJECT_TO_SEND	(8)
 #define	SIR_SWIDE_OVERRUN	(9)
 #define	SIR_SODL_UNDERRUN	(10)
 #define	SIR_RESEL_NO_MSG_IN	(11)
 #define	SIR_RESEL_NO_IDENTIFY	(12)
 #define	SIR_RESEL_BAD_LUN	(13)
 #define	SIR_TARGET_SELECTED	(14)
 #define	SIR_RESEL_BAD_I_T_L	(15)
 #define	SIR_RESEL_BAD_I_T_L_Q	(16)
 #define	SIR_ABORT_SENT		(17)
 #define	SIR_RESEL_ABORTED	(18)
 #define	SIR_MSG_OUT_DONE	(19)
 #define	SIR_COMPLETE_ERROR	(20)
 #define	SIR_DATA_OVERRUN	(21)
 #define	SIR_BAD_PHASE		(22)
 #define	SIR_MAX			(22)
 
 /*
  *  Extended error bit codes.
  *  xerr_status field of struct sym_ccb.
  */
 #define	XE_EXTRA_DATA	(1)	/* unexpected data phase	 */
 #define	XE_BAD_PHASE	(1<<1)	/* illegal phase (4/5)		 */
 #define	XE_PARITY_ERR	(1<<2)	/* unrecovered SCSI parity error */
 #define	XE_SODL_UNRUN	(1<<3)	/* ODD transfer in DATA OUT phase */
 #define	XE_SWIDE_OVRUN	(1<<4)	/* ODD transfer in DATA IN phase */
 
 /*
  *  Negotiation status.
  *  nego_status field of struct sym_ccb.
  */
 #define NS_SYNC		(1)
 #define NS_WIDE		(2)
 #define NS_PPR		(3)
 
 /*
  *  A CCB hashed table is used to retrieve CCB address
  *  from DSA value.
  */
 #define CCB_HASH_SHIFT		8
 #define CCB_HASH_SIZE		(1UL << CCB_HASH_SHIFT)
 #define CCB_HASH_MASK		(CCB_HASH_SIZE-1)
 #define CCB_HASH_CODE(dsa)	(((dsa) >> 9) & CCB_HASH_MASK)
 
 /*
  *  Device flags.
  */
 #define SYM_DISC_ENABLED	(1)
 #define SYM_TAGS_ENABLED	(1<<1)
 #define SYM_SCAN_BOOT_DISABLED	(1<<2)
 #define SYM_SCAN_LUNS_DISABLED	(1<<3)
 
 /*
  *  Host adapter miscellaneous flags.
  */
 #define SYM_AVOID_BUS_RESET	(1)
 #define SYM_SCAN_TARGETS_HILO	(1<<1)
 
 /*
  *  Device quirks.
  *  Some devices, for example the CHEETAH 2 LVD, disconnects without
  *  saving the DATA POINTER then reselects and terminates the IO.
  *  On reselection, the automatic RESTORE DATA POINTER makes the
  *  CURRENT DATA POINTER not point at the end of the IO.
  *  This behaviour just breaks our calculation of the residual.
  *  For now, we just force an AUTO SAVE on disconnection and will
  *  fix that in a further driver version.
  */
 #define SYM_QUIRK_AUTOSAVE 1
 
 /*
  *  Misc.
  */
 #define	SYM_LOCK()		mtx_lock(&np->mtx)
 #define	SYM_LOCK_ASSERT(_what)	mtx_assert(&np->mtx, (_what))
 #define	SYM_LOCK_DESTROY()	mtx_destroy(&np->mtx)
 #define	SYM_LOCK_INIT()		mtx_init(&np->mtx, "sym_lock", NULL, MTX_DEF)
 #define	SYM_LOCK_INITIALIZED()	mtx_initialized(&np->mtx)
 #define	SYM_UNLOCK()		mtx_unlock(&np->mtx)
 
 #define SYM_SNOOP_TIMEOUT (10000000)
 #define SYM_PCI_IO	PCIR_BAR(0)
 #define SYM_PCI_MMIO	PCIR_BAR(1)
 #define SYM_PCI_RAM	PCIR_BAR(2)
 #define SYM_PCI_RAM64	PCIR_BAR(3)
 
 /*
  *  Back-pointer from the CAM CCB to our data structures.
  */
 #define sym_hcb_ptr	spriv_ptr0
 /* #define sym_ccb_ptr	spriv_ptr1 */
 
 /*
  *  We mostly have to deal with pointers.
  *  Thus these typedef's.
  */
 typedef struct sym_tcb *tcb_p;
 typedef struct sym_lcb *lcb_p;
 typedef struct sym_ccb *ccb_p;
 typedef struct sym_hcb *hcb_p;
 
 /*
  *  Gather negotiable parameters value
  */
 struct sym_trans {
 	u8 scsi_version;
 	u8 spi_version;
 	u8 period;
 	u8 offset;
 	u8 width;
 	u8 options;	/* PPR options */
 };
 
 struct sym_tinfo {
 	struct sym_trans current;
 	struct sym_trans goal;
 	struct sym_trans user;
 };
 
 #define BUS_8_BIT	MSG_EXT_WDTR_BUS_8_BIT
 #define BUS_16_BIT	MSG_EXT_WDTR_BUS_16_BIT
 
 /*
  *  Global TCB HEADER.
  *
  *  Due to lack of indirect addressing on earlier NCR chips,
  *  this substructure is copied from the TCB to a global
  *  address after selection.
  *  For SYMBIOS chips that support LOAD/STORE this copy is
  *  not needed and thus not performed.
  */
 struct sym_tcbh {
 	/*
 	 *  Scripts bus addresses of LUN table accessed from scripts.
 	 *  LUN #0 is a special case, since multi-lun devices are rare,
 	 *  and we we want to speed-up the general case and not waste
 	 *  resources.
 	 */
 	u32	luntbl_sa;	/* bus address of this table	*/
 	u32	lun0_sa;	/* bus address of LCB #0	*/
 	/*
 	 *  Actual SYNC/WIDE IO registers value for this target.
 	 *  'sval', 'wval' and 'uval' are read from SCRIPTS and
 	 *  so have alignment constraints.
 	 */
 /*0*/	u_char	uval;		/* -> SCNTL4 register		*/
 /*1*/	u_char	sval;		/* -> SXFER  io register	*/
 /*2*/	u_char	filler1;
 /*3*/	u_char	wval;		/* -> SCNTL3 io register	*/
 };
 
 /*
  *  Target Control Block
  */
 struct sym_tcb {
 	/*
 	 *  TCB header.
 	 *  Assumed at offset 0.
 	 */
 /*0*/	struct sym_tcbh head;
 
 	/*
 	 *  LUN table used by the SCRIPTS processor.
 	 *  An array of bus addresses is used on reselection.
 	 */
 	u32	*luntbl;	/* LCBs bus address table	*/
 
 	/*
 	 *  LUN table used by the C code.
 	 */
 	lcb_p	lun0p;		/* LCB of LUN #0 (usual case)	*/
 #if SYM_CONF_MAX_LUN > 1
 	lcb_p	*lunmp;		/* Other LCBs [1..MAX_LUN]	*/
 #endif
 
 	/*
 	 *  Bitmap that tells about LUNs that succeeded at least
 	 *  1 IO and therefore assumed to be a real device.
 	 *  Avoid useless allocation of the LCB structure.
 	 */
 	u32	lun_map[(SYM_CONF_MAX_LUN+31)/32];
 
 	/*
 	 *  Bitmap that tells about LUNs that haven't yet an LCB
 	 *  allocated (not discovered or LCB allocation failed).
 	 */
 	u32	busy0_map[(SYM_CONF_MAX_LUN+31)/32];
 
 	/*
 	 *  Transfer capabilities (SIP)
 	 */
 	struct sym_tinfo tinfo;
 
 	/*
 	 * Keep track of the CCB used for the negotiation in order
 	 * to ensure that only 1 negotiation is queued at a time.
 	 */
 	ccb_p   nego_cp;	/* CCB used for the nego		*/
 
 	/*
 	 *  Set when we want to reset the device.
 	 */
 	u_char	to_reset;
 
 	/*
 	 *  Other user settable limits and options.
 	 *  These limits are read from the NVRAM if present.
 	 */
 	u_char	usrflags;
 	u_short	usrtags;
 };
 
 /*
  *  Assert some alignments required by the chip.
  */
 CTASSERT(((offsetof(struct sym_reg, nc_sxfer) ^
     offsetof(struct sym_tcb, head.sval)) &3) == 0);
 CTASSERT(((offsetof(struct sym_reg, nc_scntl3) ^
     offsetof(struct sym_tcb, head.wval)) &3) == 0);
 
 /*
  *  Global LCB HEADER.
  *
  *  Due to lack of indirect addressing on earlier NCR chips,
  *  this substructure is copied from the LCB to a global
  *  address after selection.
  *  For SYMBIOS chips that support LOAD/STORE this copy is
  *  not needed and thus not performed.
  */
 struct sym_lcbh {
 	/*
 	 *  SCRIPTS address jumped by SCRIPTS on reselection.
 	 *  For not probed logical units, this address points to
 	 *  SCRIPTS that deal with bad LU handling (must be at
 	 *  offset zero of the LCB for that reason).
 	 */
 /*0*/	u32	resel_sa;
 
 	/*
 	 *  Task (bus address of a CCB) read from SCRIPTS that points
 	 *  to the unique ITL nexus allowed to be disconnected.
 	 */
 	u32	itl_task_sa;
 
 	/*
 	 *  Task table bus address (read from SCRIPTS).
 	 */
 	u32	itlq_tbl_sa;
 };
 
 /*
  *  Logical Unit Control Block
  */
 struct sym_lcb {
 	/*
 	 *  TCB header.
 	 *  Assumed at offset 0.
 	 */
 /*0*/	struct sym_lcbh head;
 
 	/*
 	 *  Task table read from SCRIPTS that contains pointers to
 	 *  ITLQ nexuses. The bus address read from SCRIPTS is
 	 *  inside the header.
 	 */
 	u32	*itlq_tbl;	/* Kernel virtual address	*/
 
 	/*
 	 *  Busy CCBs management.
 	 */
 	u_short	busy_itlq;	/* Number of busy tagged CCBs	*/
 	u_short	busy_itl;	/* Number of busy untagged CCBs	*/
 
 	/*
 	 *  Circular tag allocation buffer.
 	 */
 	u_short	ia_tag;		/* Tag allocation index		*/
 	u_short	if_tag;		/* Tag release index		*/
 	u_char	*cb_tags;	/* Circular tags buffer		*/
 
 	/*
 	 *  Set when we want to clear all tasks.
 	 */
 	u_char to_clear;
 
 	/*
 	 *  Capabilities.
 	 */
 	u_char	user_flags;
 	u_char	current_flags;
 };
 
 /*
  *  Action from SCRIPTS on a task.
  *  Is part of the CCB, but is also used separately to plug
  *  error handling action to perform from SCRIPTS.
  */
 struct sym_actscr {
 	u32	start;		/* Jumped by SCRIPTS after selection	*/
 	u32	restart;	/* Jumped by SCRIPTS on relection	*/
 };
 
 /*
  *  Phase mismatch context.
  *
  *  It is part of the CCB and is used as parameters for the
  *  DATA pointer. We need two contexts to handle correctly the
  *  SAVED DATA POINTER.
  */
 struct sym_pmc {
 	struct	sym_tblmove sg;	/* Updated interrupted SG block	*/
 	u32	ret;		/* SCRIPT return address	*/
 };
 
 /*
  *  LUN control block lookup.
  *  We use a direct pointer for LUN #0, and a table of
  *  pointers which is only allocated for devices that support
  *  LUN(s) > 0.
  */
 #if SYM_CONF_MAX_LUN <= 1
 #define sym_lp(tp, lun) (!lun) ? (tp)->lun0p : 0
 #else
 #define sym_lp(tp, lun) \
 	(!lun) ? (tp)->lun0p : (tp)->lunmp ? (tp)->lunmp[(lun)] : 0
 #endif
 
 /*
  *  Status are used by the host and the script processor.
  *
  *  The last four bytes (status[4]) are copied to the
  *  scratchb register (declared as scr0..scr3) just after the
  *  select/reselect, and copied back just after disconnecting.
  *  Inside the script the XX_REG are used.
  */
 
 /*
  *  Last four bytes (script)
  */
 #define  QU_REG	scr0
 #define  HS_REG	scr1
 #define  HS_PRT	nc_scr1
 #define  SS_REG	scr2
 #define  SS_PRT	nc_scr2
 #define  HF_REG	scr3
 #define  HF_PRT	nc_scr3
 
 /*
  *  Last four bytes (host)
  */
 #define  actualquirks  phys.head.status[0]
 #define  host_status   phys.head.status[1]
 #define  ssss_status   phys.head.status[2]
 #define  host_flags    phys.head.status[3]
 
 /*
  *  Host flags
  */
 #define HF_IN_PM0	1u
 #define HF_IN_PM1	(1u<<1)
 #define HF_ACT_PM	(1u<<2)
 #define HF_DP_SAVED	(1u<<3)
 #define HF_SENSE	(1u<<4)
 #define HF_EXT_ERR	(1u<<5)
 #define HF_DATA_IN	(1u<<6)
 #ifdef SYM_CONF_IARB_SUPPORT
 #define HF_HINT_IARB	(1u<<7)
 #endif
 
 /*
  *  Global CCB HEADER.
  *
  *  Due to lack of indirect addressing on earlier NCR chips,
  *  this substructure is copied from the ccb to a global
  *  address after selection (or reselection) and copied back
  *  before disconnect.
  *  For SYMBIOS chips that support LOAD/STORE this copy is
  *  not needed and thus not performed.
  */
 struct sym_ccbh {
 	/*
 	 *  Start and restart SCRIPTS addresses (must be at 0).
 	 */
 /*0*/	struct sym_actscr go;
 
 	/*
 	 *  SCRIPTS jump address that deal with data pointers.
 	 *  'savep' points to the position in the script responsible
 	 *  for the actual transfer of data.
 	 *  It's written on reception of a SAVE_DATA_POINTER message.
 	 */
 	u32	savep;		/* Jump address to saved data pointer	*/
 	u32	lastp;		/* SCRIPTS address at end of data	*/
 	u32	goalp;		/* Not accessed for now from SCRIPTS	*/
 
 	/*
 	 *  Status fields.
 	 */
 	u8	status[4];
 };
 
 /*
  *  Data Structure Block
  *
  *  During execution of a ccb by the script processor, the
  *  DSA (data structure address) register points to this
  *  substructure of the ccb.
  */
 struct sym_dsb {
 	/*
 	 *  CCB header.
 	 *  Also assumed at offset 0 of the sym_ccb structure.
 	 */
 /*0*/	struct sym_ccbh head;
 
 	/*
 	 *  Phase mismatch contexts.
 	 *  We need two to handle correctly the SAVED DATA POINTER.
 	 *  MUST BOTH BE AT OFFSET < 256, due to using 8 bit arithmetic
 	 *  for address calculation from SCRIPTS.
 	 */
 	struct sym_pmc pm0;
 	struct sym_pmc pm1;
 
 	/*
 	 *  Table data for Script
 	 */
 	struct sym_tblsel  select;
 	struct sym_tblmove smsg;
 	struct sym_tblmove smsg_ext;
 	struct sym_tblmove cmd;
 	struct sym_tblmove sense;
 	struct sym_tblmove wresid;
 	struct sym_tblmove data [SYM_CONF_MAX_SG];
 };
 
 /*
  *  Our Command Control Block
  */
 struct sym_ccb {
 	/*
 	 *  This is the data structure which is pointed by the DSA
 	 *  register when it is executed by the script processor.
 	 *  It must be the first entry.
 	 */
 	struct sym_dsb phys;
 
 	/*
 	 *  Pointer to CAM ccb and related stuff.
 	 */
 	struct callout ch;	/* callout handle		*/
 	union ccb *cam_ccb;	/* CAM scsiio ccb		*/
 	u8	cdb_buf[16];	/* Copy of CDB			*/
 	u8	*sns_bbuf;	/* Bounce buffer for sense data	*/
 #define SYM_SNS_BBUF_LEN	sizeof(struct scsi_sense_data)
 	int	data_len;	/* Total data length		*/
 	int	segments;	/* Number of SG segments	*/
 
 	/*
 	 *  Miscellaneous status'.
 	 */
 	u_char	nego_status;	/* Negotiation status		*/
 	u_char	xerr_status;	/* Extended error flags		*/
 	u32	extra_bytes;	/* Extraneous bytes transferred	*/
 
 	/*
 	 *  Message areas.
 	 *  We prepare a message to be sent after selection.
 	 *  We may use a second one if the command is rescheduled
 	 *  due to CHECK_CONDITION or COMMAND TERMINATED.
 	 *  Contents are IDENTIFY and SIMPLE_TAG.
 	 *  While negotiating sync or wide transfer,
 	 *  a SDTR or WDTR message is appended.
 	 */
 	u_char	scsi_smsg [12];
 	u_char	scsi_smsg2[12];
 
 	/*
 	 *  Auto request sense related fields.
 	 */
 	u_char	sensecmd[6];	/* Request Sense command	*/
 	u_char	sv_scsi_status;	/* Saved SCSI status 		*/
 	u_char	sv_xerr_status;	/* Saved extended status	*/
 	int	sv_resid;	/* Saved residual		*/
 
 	/*
 	 *  Map for the DMA of user data.
 	 */
 	void		*arg;	/* Argument for some callback	*/
 	bus_dmamap_t	dmamap;	/* DMA map for user data	*/
 	u_char		dmamapped;
 #define SYM_DMA_NONE	0
 #define SYM_DMA_READ	1
 #define SYM_DMA_WRITE	2
 	/*
 	 *  Other fields.
 	 */
 	u32	ccb_ba;		/* BUS address of this CCB	*/
 	u_short	tag;		/* Tag for this transfer	*/
 				/*  NO_TAG means no tag		*/
 	u_char	target;
 	u_char	lun;
 	ccb_p	link_ccbh;	/* Host adapter CCB hash chain	*/
 	SYM_QUEHEAD
 		link_ccbq;	/* Link to free/busy CCB queue	*/
 	u32	startp;		/* Initial data pointer		*/
 	int	ext_sg;		/* Extreme data pointer, used	*/
 	int	ext_ofs;	/*  to calculate the residual.	*/
 	u_char	to_abort;	/* Want this IO to be aborted	*/
 };
 
 #define CCB_BA(cp,lbl)	(cp->ccb_ba + offsetof(struct sym_ccb, lbl))
 
 /*
  *  Host Control Block
  */
 struct sym_hcb {
 	struct mtx	mtx;
 
 	/*
 	 *  Global headers.
 	 *  Due to poorness of addressing capabilities, earlier
 	 *  chips (810, 815, 825) copy part of the data structures
 	 *  (CCB, TCB and LCB) in fixed areas.
 	 */
 #ifdef	SYM_CONF_GENERIC_SUPPORT
 	struct sym_ccbh	ccb_head;
 	struct sym_tcbh	tcb_head;
 	struct sym_lcbh	lcb_head;
 #endif
 	/*
 	 *  Idle task and invalid task actions and
 	 *  their bus addresses.
 	 */
 	struct sym_actscr idletask, notask, bad_itl, bad_itlq;
 	vm_offset_t idletask_ba, notask_ba, bad_itl_ba, bad_itlq_ba;
 
 	/*
 	 *  Dummy lun table to protect us against target
 	 *  returning bad lun number on reselection.
 	 */
 	u32	*badluntbl;	/* Table physical address	*/
 	u32	badlun_sa;	/* SCRIPT handler BUS address	*/
 
 	/*
 	 *  Bus address of this host control block.
 	 */
 	u32	hcb_ba;
 
 	/*
 	 *  Bit 32-63 of the on-chip RAM bus address in LE format.
 	 *  The START_RAM64 script loads the MMRS and MMWS from this
 	 *  field.
 	 */
 	u32	scr_ram_seg;
 
 	/*
 	 *  Chip and controller indentification.
 	 */
 	device_t device;
 
 	/*
 	 *  Initial value of some IO register bits.
 	 *  These values are assumed to have been set by BIOS, and may
 	 *  be used to probe adapter implementation differences.
 	 */
 	u_char	sv_scntl0, sv_scntl3, sv_dmode, sv_dcntl, sv_ctest3, sv_ctest4,
 		sv_ctest5, sv_gpcntl, sv_stest2, sv_stest4, sv_scntl4,
 		sv_stest1;
 
 	/*
 	 *  Actual initial value of IO register bits used by the
 	 *  driver. They are loaded at initialisation according to
 	 *  features that are to be enabled/disabled.
 	 */
 	u_char	rv_scntl0, rv_scntl3, rv_dmode, rv_dcntl, rv_ctest3, rv_ctest4,
 		rv_ctest5, rv_stest2, rv_ccntl0, rv_ccntl1, rv_scntl4;
 
 	/*
 	 *  Target data.
 	 */
 #ifdef __amd64__
 	struct sym_tcb	*target;
 #else
 	struct sym_tcb	target[SYM_CONF_MAX_TARGET];
 #endif
 
 	/*
 	 *  Target control block bus address array used by the SCRIPT
 	 *  on reselection.
 	 */
 	u32		*targtbl;
 	u32		targtbl_ba;
 
 	/*
 	 *  CAM SIM information for this instance.
 	 */
 	struct		cam_sim  *sim;
 	struct		cam_path *path;
 
 	/*
 	 *  Allocated hardware resources.
 	 */
 	struct resource	*irq_res;
 	struct resource	*io_res;
 	struct resource	*mmio_res;
 	struct resource	*ram_res;
 	int		ram_id;
 	void *intr;
 
 	/*
 	 *  Bus stuff.
 	 *
 	 *  My understanding of PCI is that all agents must share the
 	 *  same addressing range and model.
 	 *  But some hardware architecture guys provide complex and
 	 *  brain-deaded stuff that makes shit.
 	 *  This driver only support PCI compliant implementations and
 	 *  deals with part of the BUS stuff complexity only to fit O/S
 	 *  requirements.
 	 */
 
 	/*
 	 *  DMA stuff.
 	 */
 	bus_dma_tag_t	bus_dmat;	/* DMA tag from parent BUS	*/
 	bus_dma_tag_t	data_dmat;	/* DMA tag for user data	*/
 	/*
 	 *  BUS addresses of the chip
 	 */
 	vm_offset_t	mmio_ba;	/* MMIO BUS address		*/
 	int		mmio_ws;	/* MMIO Window size		*/
 
 	vm_offset_t	ram_ba;		/* RAM BUS address		*/
 	int		ram_ws;		/* RAM window size		*/
 
 	/*
 	 *  SCRIPTS virtual and physical bus addresses.
 	 *  'script'  is loaded in the on-chip RAM if present.
 	 *  'scripth' stays in main memory for all chips except the
 	 *  53C895A, 53C896 and 53C1010 that provide 8K on-chip RAM.
 	 */
 	u_char		*scripta0;	/* Copies of script and scripth	*/
 	u_char		*scriptb0;	/* Copies of script and scripth	*/
 	vm_offset_t	scripta_ba;	/* Actual script and scripth	*/
 	vm_offset_t	scriptb_ba;	/*  bus addresses.		*/
 	vm_offset_t	scriptb0_ba;
 	u_short		scripta_sz;	/* Actual size of script A	*/
 	u_short		scriptb_sz;	/* Actual size of script B	*/
 
 	/*
 	 *  Bus addresses, setup and patch methods for
 	 *  the selected firmware.
 	 */
 	struct sym_fwa_ba fwa_bas;	/* Useful SCRIPTA bus addresses	*/
 	struct sym_fwb_ba fwb_bas;	/* Useful SCRIPTB bus addresses	*/
 	void		(*fw_setup)(hcb_p np, const struct sym_fw *fw);
 	void		(*fw_patch)(hcb_p np);
 	const char	*fw_name;
 
 	/*
 	 *  General controller parameters and configuration.
 	 */
 	u_short	device_id;	/* PCI device id		*/
 	u_char	revision_id;	/* PCI device revision id	*/
 	u_int	features;	/* Chip features map		*/
 	u_char	myaddr;		/* SCSI id of the adapter	*/
 	u_char	maxburst;	/* log base 2 of dwords burst	*/
 	u_char	maxwide;	/* Maximum transfer width	*/
 	u_char	minsync;	/* Min sync period factor (ST)	*/
 	u_char	maxsync;	/* Max sync period factor (ST)	*/
 	u_char	maxoffs;	/* Max scsi offset        (ST)	*/
 	u_char	minsync_dt;	/* Min sync period factor (DT)	*/
 	u_char	maxsync_dt;	/* Max sync period factor (DT)	*/
 	u_char	maxoffs_dt;	/* Max scsi offset        (DT)	*/
 	u_char	multiplier;	/* Clock multiplier (1,2,4)	*/
 	u_char	clock_divn;	/* Number of clock divisors	*/
 	u32	clock_khz;	/* SCSI clock frequency in KHz	*/
 	u32	pciclk_khz;	/* Estimated PCI clock  in KHz	*/
 	/*
 	 *  Start queue management.
 	 *  It is filled up by the host processor and accessed by the
 	 *  SCRIPTS processor in order to start SCSI commands.
 	 */
 	volatile		/* Prevent code optimizations	*/
 	u32	*squeue;	/* Start queue virtual address	*/
 	u32	squeue_ba;	/* Start queue BUS address	*/
 	u_short	squeueput;	/* Next free slot of the queue	*/
 	u_short	actccbs;	/* Number of allocated CCBs	*/
 
 	/*
 	 *  Command completion queue.
 	 *  It is the same size as the start queue to avoid overflow.
 	 */
 	u_short	dqueueget;	/* Next position to scan	*/
 	volatile		/* Prevent code optimizations	*/
 	u32	*dqueue;	/* Completion (done) queue	*/
 	u32	dqueue_ba;	/* Done queue BUS address	*/
 
 	/*
 	 *  Miscellaneous buffers accessed by the scripts-processor.
 	 *  They shall be DWORD aligned, because they may be read or
 	 *  written with a script command.
 	 */
 	u_char		msgout[8];	/* Buffer for MESSAGE OUT 	*/
 	u_char		msgin [8];	/* Buffer for MESSAGE IN	*/
 	u32		lastmsg;	/* Last SCSI message sent	*/
 	u_char		scratch;	/* Scratch for SCSI receive	*/
 
 	/*
 	 *  Miscellaneous configuration and status parameters.
 	 */
 	u_char		usrflags;	/* Miscellaneous user flags	*/
 	u_char		scsi_mode;	/* Current SCSI BUS mode	*/
 	u_char		verbose;	/* Verbosity for this controller*/
 	u32		cache;		/* Used for cache test at init.	*/
 
 	/*
 	 *  CCB lists and queue.
 	 */
 	ccb_p ccbh[CCB_HASH_SIZE];	/* CCB hashed by DSA value	*/
 	SYM_QUEHEAD	free_ccbq;	/* Queue of available CCBs	*/
 	SYM_QUEHEAD	busy_ccbq;	/* Queue of busy CCBs		*/
 
 	/*
 	 *  During error handling and/or recovery,
 	 *  active CCBs that are to be completed with
 	 *  error or requeued are moved from the busy_ccbq
 	 *  to the comp_ccbq prior to completion.
 	 */
 	SYM_QUEHEAD	comp_ccbq;
 
 	/*
 	 *  CAM CCB pending queue.
 	 */
 	SYM_QUEHEAD	cam_ccbq;
 
 	/*
 	 *  IMMEDIATE ARBITRATION (IARB) control.
 	 *
 	 *  We keep track in 'last_cp' of the last CCB that has been
 	 *  queued to the SCRIPTS processor and clear 'last_cp' when
 	 *  this CCB completes. If last_cp is not zero at the moment
 	 *  we queue a new CCB, we set a flag in 'last_cp' that is
 	 *  used by the SCRIPTS as a hint for setting IARB.
 	 *  We donnot set more than 'iarb_max' consecutive hints for
 	 *  IARB in order to leave devices a chance to reselect.
 	 *  By the way, any non zero value of 'iarb_max' is unfair. :)
 	 */
 #ifdef SYM_CONF_IARB_SUPPORT
 	u_short		iarb_max;	/* Max. # consecutive IARB hints*/
 	u_short		iarb_count;	/* Actual # of these hints	*/
 	ccb_p		last_cp;
 #endif
 
 	/*
 	 *  Command abort handling.
 	 *  We need to synchronize tightly with the SCRIPTS
 	 *  processor in order to handle things correctly.
 	 */
 	u_char		abrt_msg[4];	/* Message to send buffer	*/
 	struct sym_tblmove abrt_tbl;	/* Table for the MOV of it 	*/
 	struct sym_tblsel  abrt_sel;	/* Sync params for selection	*/
 	u_char		istat_sem;	/* Tells the chip to stop (SEM)	*/
 };
 
 #define HCB_BA(np, lbl)	    (np->hcb_ba      + offsetof(struct sym_hcb, lbl))
 
 /*
  *  Return the name of the controller.
  */
 static __inline const char *sym_name(hcb_p np)
 {
 	return device_get_nameunit(np->device);
 }
 
 /*--------------------------------------------------------------------------*/
 /*------------------------------ FIRMWARES ---------------------------------*/
 /*--------------------------------------------------------------------------*/
 
 /*
  *  This stuff will be moved to a separate source file when
  *  the driver will be broken into several source modules.
  */
 
 /*
  *  Macros used for all firmwares.
  */
 #define	SYM_GEN_A(s, label)	((short) offsetof(s, label)),
 #define	SYM_GEN_B(s, label)	((short) offsetof(s, label)),
 #define	PADDR_A(label)		SYM_GEN_PADDR_A(struct SYM_FWA_SCR, label)
 #define	PADDR_B(label)		SYM_GEN_PADDR_B(struct SYM_FWB_SCR, label)
 
 #ifdef	SYM_CONF_GENERIC_SUPPORT
 /*
  *  Allocate firmware #1 script area.
  */
 #define	SYM_FWA_SCR		sym_fw1a_scr
 #define	SYM_FWB_SCR		sym_fw1b_scr
 #include <dev/sym/sym_fw1.h>
 static const struct sym_fwa_ofs sym_fw1a_ofs = {
 	SYM_GEN_FW_A(struct SYM_FWA_SCR)
 };
 static const struct sym_fwb_ofs sym_fw1b_ofs = {
 	SYM_GEN_FW_B(struct SYM_FWB_SCR)
 };
 #undef	SYM_FWA_SCR
 #undef	SYM_FWB_SCR
 #endif	/* SYM_CONF_GENERIC_SUPPORT */
 
 /*
  *  Allocate firmware #2 script area.
  */
 #define	SYM_FWA_SCR		sym_fw2a_scr
 #define	SYM_FWB_SCR		sym_fw2b_scr
 #include <dev/sym/sym_fw2.h>
 static const struct sym_fwa_ofs sym_fw2a_ofs = {
 	SYM_GEN_FW_A(struct SYM_FWA_SCR)
 };
 static const struct sym_fwb_ofs sym_fw2b_ofs = {
 	SYM_GEN_FW_B(struct SYM_FWB_SCR)
 	SYM_GEN_B(struct SYM_FWB_SCR, start64)
 	SYM_GEN_B(struct SYM_FWB_SCR, pm_handle)
 };
 #undef	SYM_FWA_SCR
 #undef	SYM_FWB_SCR
 
 #undef	SYM_GEN_A
 #undef	SYM_GEN_B
 #undef	PADDR_A
 #undef	PADDR_B
 
 #ifdef	SYM_CONF_GENERIC_SUPPORT
 /*
  *  Patch routine for firmware #1.
  */
 static void
 sym_fw1_patch(hcb_p np)
 {
 	struct sym_fw1a_scr *scripta0;
 	struct sym_fw1b_scr *scriptb0;
 
 	scripta0 = (struct sym_fw1a_scr *) np->scripta0;
 	scriptb0 = (struct sym_fw1b_scr *) np->scriptb0;
 
 	/*
 	 *  Remove LED support if not needed.
 	 */
 	if (!(np->features & FE_LED0)) {
 		scripta0->idle[0]	= cpu_to_scr(SCR_NO_OP);
 		scripta0->reselected[0]	= cpu_to_scr(SCR_NO_OP);
 		scripta0->start[0]	= cpu_to_scr(SCR_NO_OP);
 	}
 
 #ifdef SYM_CONF_IARB_SUPPORT
 	/*
 	 *    If user does not want to use IMMEDIATE ARBITRATION
 	 *    when we are reselected while attempting to arbitrate,
 	 *    patch the SCRIPTS accordingly with a SCRIPT NO_OP.
 	 */
 	if (!SYM_CONF_SET_IARB_ON_ARB_LOST)
 		scripta0->ungetjob[0] = cpu_to_scr(SCR_NO_OP);
 #endif
 	/*
 	 *  Patch some data in SCRIPTS.
 	 *  - start and done queue initial bus address.
 	 *  - target bus address table bus address.
 	 */
 	scriptb0->startpos[0]	= cpu_to_scr(np->squeue_ba);
 	scriptb0->done_pos[0]	= cpu_to_scr(np->dqueue_ba);
 	scriptb0->targtbl[0]	= cpu_to_scr(np->targtbl_ba);
 }
 #endif	/* SYM_CONF_GENERIC_SUPPORT */
 
 /*
  *  Patch routine for firmware #2.
  */
 static void
 sym_fw2_patch(hcb_p np)
 {
 	struct sym_fw2a_scr *scripta0;
 	struct sym_fw2b_scr *scriptb0;
 
 	scripta0 = (struct sym_fw2a_scr *) np->scripta0;
 	scriptb0 = (struct sym_fw2b_scr *) np->scriptb0;
 
 	/*
 	 *  Remove LED support if not needed.
 	 */
 	if (!(np->features & FE_LED0)) {
 		scripta0->idle[0]	= cpu_to_scr(SCR_NO_OP);
 		scripta0->reselected[0]	= cpu_to_scr(SCR_NO_OP);
 		scripta0->start[0]	= cpu_to_scr(SCR_NO_OP);
 	}
 
 #ifdef SYM_CONF_IARB_SUPPORT
 	/*
 	 *    If user does not want to use IMMEDIATE ARBITRATION
 	 *    when we are reselected while attempting to arbitrate,
 	 *    patch the SCRIPTS accordingly with a SCRIPT NO_OP.
 	 */
 	if (!SYM_CONF_SET_IARB_ON_ARB_LOST)
 		scripta0->ungetjob[0] = cpu_to_scr(SCR_NO_OP);
 #endif
 	/*
 	 *  Patch some variable in SCRIPTS.
 	 *  - start and done queue initial bus address.
 	 *  - target bus address table bus address.
 	 */
 	scriptb0->startpos[0]	= cpu_to_scr(np->squeue_ba);
 	scriptb0->done_pos[0]	= cpu_to_scr(np->dqueue_ba);
 	scriptb0->targtbl[0]	= cpu_to_scr(np->targtbl_ba);
 
 	/*
 	 *  Remove the load of SCNTL4 on reselection if not a C10.
 	 */
 	if (!(np->features & FE_C10)) {
 		scripta0->resel_scntl4[0] = cpu_to_scr(SCR_NO_OP);
 		scripta0->resel_scntl4[1] = cpu_to_scr(0);
 	}
 
 	/*
 	 *  Remove a couple of work-arounds specific to C1010 if
 	 *  they are not desirable. See `sym_fw2.h' for more details.
 	 */
 	if (!(np->device_id == PCI_ID_LSI53C1010_2 &&
 	      np->revision_id < 0x1 &&
 	      np->pciclk_khz < 60000)) {
 		scripta0->datao_phase[0] = cpu_to_scr(SCR_NO_OP);
 		scripta0->datao_phase[1] = cpu_to_scr(0);
 	}
 	if (!(np->device_id == PCI_ID_LSI53C1010 &&
 	      /* np->revision_id < 0xff */ 1)) {
 		scripta0->sel_done[0] = cpu_to_scr(SCR_NO_OP);
 		scripta0->sel_done[1] = cpu_to_scr(0);
 	}
 
 	/*
 	 *  Patch some other variables in SCRIPTS.
 	 *  These ones are loaded by the SCRIPTS processor.
 	 */
 	scriptb0->pm0_data_addr[0] =
 		cpu_to_scr(np->scripta_ba +
 			   offsetof(struct sym_fw2a_scr, pm0_data));
 	scriptb0->pm1_data_addr[0] =
 		cpu_to_scr(np->scripta_ba +
 			   offsetof(struct sym_fw2a_scr, pm1_data));
 }
 
 /*
  *  Fill the data area in scripts.
  *  To be done for all firmwares.
  */
 static void
 sym_fw_fill_data (u32 *in, u32 *out)
 {
 	int	i;
 
 	for (i = 0; i < SYM_CONF_MAX_SG; i++) {
 		*in++  = SCR_CHMOV_TBL ^ SCR_DATA_IN;
 		*in++  = offsetof (struct sym_dsb, data[i]);
 		*out++ = SCR_CHMOV_TBL ^ SCR_DATA_OUT;
 		*out++ = offsetof (struct sym_dsb, data[i]);
 	}
 }
 
 /*
  *  Setup useful script bus addresses.
  *  To be done for all firmwares.
  */
 static void
 sym_fw_setup_bus_addresses(hcb_p np, const struct sym_fw *fw)
 {
 	u32 *pa;
 	const u_short *po;
 	int i;
 
 	/*
 	 *  Build the bus address table for script A
 	 *  from the script A offset table.
 	 */
 	po = (const u_short *) fw->a_ofs;
 	pa = (u32 *) &np->fwa_bas;
 	for (i = 0 ; i < sizeof(np->fwa_bas)/sizeof(u32) ; i++)
 		pa[i] = np->scripta_ba + po[i];
 
 	/*
 	 *  Same for script B.
 	 */
 	po = (const u_short *) fw->b_ofs;
 	pa = (u32 *) &np->fwb_bas;
 	for (i = 0 ; i < sizeof(np->fwb_bas)/sizeof(u32) ; i++)
 		pa[i] = np->scriptb_ba + po[i];
 }
 
 #ifdef	SYM_CONF_GENERIC_SUPPORT
 /*
  *  Setup routine for firmware #1.
  */
 static void
 sym_fw1_setup(hcb_p np, const struct sym_fw *fw)
 {
 	struct sym_fw1a_scr *scripta0;
 
 	scripta0 = (struct sym_fw1a_scr *) np->scripta0;
 
 	/*
 	 *  Fill variable parts in scripts.
 	 */
 	sym_fw_fill_data(scripta0->data_in, scripta0->data_out);
 
 	/*
 	 *  Setup bus addresses used from the C code..
 	 */
 	sym_fw_setup_bus_addresses(np, fw);
 }
 #endif	/* SYM_CONF_GENERIC_SUPPORT */
 
 /*
  *  Setup routine for firmware #2.
  */
 static void
 sym_fw2_setup(hcb_p np, const struct sym_fw *fw)
 {
 	struct sym_fw2a_scr *scripta0;
 
 	scripta0 = (struct sym_fw2a_scr *) np->scripta0;
 
 	/*
 	 *  Fill variable parts in scripts.
 	 */
 	sym_fw_fill_data(scripta0->data_in, scripta0->data_out);
 
 	/*
 	 *  Setup bus addresses used from the C code..
 	 */
 	sym_fw_setup_bus_addresses(np, fw);
 }
 
 /*
  *  Allocate firmware descriptors.
  */
 #ifdef	SYM_CONF_GENERIC_SUPPORT
 static const struct sym_fw sym_fw1 = SYM_FW_ENTRY(sym_fw1, "NCR-generic");
 #endif	/* SYM_CONF_GENERIC_SUPPORT */
 static const struct sym_fw sym_fw2 = SYM_FW_ENTRY(sym_fw2, "LOAD/STORE-based");
 
 /*
  *  Find the most appropriate firmware for a chip.
  */
 static const struct sym_fw *
 sym_find_firmware(const struct sym_pci_chip *chip)
 {
 	if (chip->features & FE_LDSTR)
 		return &sym_fw2;
 #ifdef	SYM_CONF_GENERIC_SUPPORT
 	else if (!(chip->features & (FE_PFEN|FE_NOPM|FE_DAC)))
 		return &sym_fw1;
 #endif
 	else
 		return NULL;
 }
 
 /*
  *  Bind a script to physical addresses.
  */
 static void sym_fw_bind_script (hcb_p np, u32 *start, int len)
 {
 	u32 opcode, new, old, tmp1, tmp2;
 	u32 *end, *cur;
 	int relocs;
 
 	cur = start;
 	end = start + len/4;
 
 	while (cur < end) {
 
 		opcode = *cur;
 
 		/*
 		 *  If we forget to change the length
 		 *  in scripts, a field will be
 		 *  padded with 0. This is an illegal
 		 *  command.
 		 */
 		if (opcode == 0) {
 			printf ("%s: ERROR0 IN SCRIPT at %d.\n",
 				sym_name(np), (int) (cur-start));
 			MDELAY (10000);
 			++cur;
 			continue;
 		};
 
 		/*
 		 *  We use the bogus value 0xf00ff00f ;-)
 		 *  to reserve data area in SCRIPTS.
 		 */
 		if (opcode == SCR_DATA_ZERO) {
 			*cur++ = 0;
 			continue;
 		}
 
 		if (DEBUG_FLAGS & DEBUG_SCRIPT)
 			printf ("%d:  <%x>\n", (int) (cur-start),
 				(unsigned)opcode);
 
 		/*
 		 *  We don't have to decode ALL commands
 		 */
 		switch (opcode >> 28) {
 		case 0xf:
 			/*
 			 *  LOAD / STORE DSA relative, don't relocate.
 			 */
 			relocs = 0;
 			break;
 		case 0xe:
 			/*
 			 *  LOAD / STORE absolute.
 			 */
 			relocs = 1;
 			break;
 		case 0xc:
 			/*
 			 *  COPY has TWO arguments.
 			 */
 			relocs = 2;
 			tmp1 = cur[1];
 			tmp2 = cur[2];
 			if ((tmp1 ^ tmp2) & 3) {
 				printf ("%s: ERROR1 IN SCRIPT at %d.\n",
 					sym_name(np), (int) (cur-start));
 				MDELAY (10000);
 			}
 			/*
 			 *  If PREFETCH feature not enabled, remove
 			 *  the NO FLUSH bit if present.
 			 */
 			if ((opcode & SCR_NO_FLUSH) &&
 			    !(np->features & FE_PFEN)) {
 				opcode = (opcode & ~SCR_NO_FLUSH);
 			}
 			break;
 		case 0x0:
 			/*
 			 *  MOVE/CHMOV (absolute address)
 			 */
 			if (!(np->features & FE_WIDE))
 				opcode = (opcode | OPC_MOVE);
 			relocs = 1;
 			break;
 		case 0x1:
 			/*
 			 *  MOVE/CHMOV (table indirect)
 			 */
 			if (!(np->features & FE_WIDE))
 				opcode = (opcode | OPC_MOVE);
 			relocs = 0;
 			break;
 		case 0x8:
 			/*
 			 *  JUMP / CALL
 			 *  dont't relocate if relative :-)
 			 */
 			if (opcode & 0x00800000)
 				relocs = 0;
 			else if ((opcode & 0xf8400000) == 0x80400000)/*JUMP64*/
 				relocs = 2;
 			else
 				relocs = 1;
 			break;
 		case 0x4:
 		case 0x5:
 		case 0x6:
 		case 0x7:
 			relocs = 1;
 			break;
 		default:
 			relocs = 0;
 			break;
 		};
 
 		/*
 		 *  Scriptify:) the opcode.
 		 */
 		*cur++ = cpu_to_scr(opcode);
 
 		/*
 		 *  If no relocation, assume 1 argument
 		 *  and just scriptize:) it.
 		 */
 		if (!relocs) {
 			*cur = cpu_to_scr(*cur);
 			++cur;
 			continue;
 		}
 
 		/*
 		 *  Otherwise performs all needed relocations.
 		 */
 		while (relocs--) {
 			old = *cur;
 
 			switch (old & RELOC_MASK) {
 			case RELOC_REGISTER:
 				new = (old & ~RELOC_MASK) + np->mmio_ba;
 				break;
 			case RELOC_LABEL_A:
 				new = (old & ~RELOC_MASK) + np->scripta_ba;
 				break;
 			case RELOC_LABEL_B:
 				new = (old & ~RELOC_MASK) + np->scriptb_ba;
 				break;
 			case RELOC_SOFTC:
 				new = (old & ~RELOC_MASK) + np->hcb_ba;
 				break;
 			case 0:
 				/*
 				 *  Don't relocate a 0 address.
 				 *  They are mostly used for patched or
 				 *  script self-modified areas.
 				 */
 				if (old == 0) {
 					new = old;
 					break;
 				}
 				/* fall through */
 			default:
 				new = 0;
 				panic("sym_fw_bind_script: "
 				      "weird relocation %x\n", old);
 				break;
 			}
 
 			*cur++ = cpu_to_scr(new);
 		}
 	};
 }
 
 /*---------------------------------------------------------------------------*/
 /*--------------------------- END OF FIRMWARES  -----------------------------*/
 /*---------------------------------------------------------------------------*/
 
 /*
  *  Function prototypes.
  */
 static void sym_save_initial_setting (hcb_p np);
 static int  sym_prepare_setting (hcb_p np, struct sym_nvram *nvram);
 static int  sym_prepare_nego (hcb_p np, ccb_p cp, int nego, u_char *msgptr);
 static void sym_put_start_queue (hcb_p np, ccb_p cp);
 static void sym_chip_reset (hcb_p np);
 static void sym_soft_reset (hcb_p np);
 static void sym_start_reset (hcb_p np);
 static int  sym_reset_scsi_bus (hcb_p np, int enab_int);
 static int  sym_wakeup_done (hcb_p np);
 static void sym_flush_busy_queue (hcb_p np, int cam_status);
 static void sym_flush_comp_queue (hcb_p np, int cam_status);
 static void sym_init (hcb_p np, int reason);
 static int  sym_getsync(hcb_p np, u_char dt, u_char sfac, u_char *divp,
 		        u_char *fakp);
 static void sym_setsync (hcb_p np, ccb_p cp, u_char ofs, u_char per,
 			 u_char div, u_char fak);
 static void sym_setwide (hcb_p np, ccb_p cp, u_char wide);
 static void sym_setpprot(hcb_p np, ccb_p cp, u_char dt, u_char ofs,
 			 u_char per, u_char wide, u_char div, u_char fak);
 static void sym_settrans(hcb_p np, ccb_p cp, u_char dt, u_char ofs,
 			 u_char per, u_char wide, u_char div, u_char fak);
 static void sym_log_hard_error (hcb_p np, u_short sist, u_char dstat);
 static void sym_intr (void *arg);
 static void sym_poll (struct cam_sim *sim);
 static void sym_recover_scsi_int (hcb_p np, u_char hsts);
 static void sym_int_sto (hcb_p np);
 static void sym_int_udc (hcb_p np);
 static void sym_int_sbmc (hcb_p np);
 static void sym_int_par (hcb_p np, u_short sist);
 static void sym_int_ma (hcb_p np);
 static int  sym_dequeue_from_squeue(hcb_p np, int i, int target, int lun,
 				    int task);
 static void sym_sir_bad_scsi_status (hcb_p np, ccb_p cp);
 static int  sym_clear_tasks (hcb_p np, int status, int targ, int lun, int task);
 static void sym_sir_task_recovery (hcb_p np, int num);
 static int  sym_evaluate_dp (hcb_p np, ccb_p cp, u32 scr, int *ofs);
 static void sym_modify_dp(hcb_p np, ccb_p cp, int ofs);
 static int  sym_compute_residual (hcb_p np, ccb_p cp);
 static int  sym_show_msg (u_char * msg);
 static void sym_print_msg (ccb_p cp, char *label, u_char *msg);
 static void sym_sync_nego (hcb_p np, tcb_p tp, ccb_p cp);
 static void sym_ppr_nego (hcb_p np, tcb_p tp, ccb_p cp);
 static void sym_wide_nego (hcb_p np, tcb_p tp, ccb_p cp);
 static void sym_nego_default (hcb_p np, tcb_p tp, ccb_p cp);
 static void sym_nego_rejected (hcb_p np, tcb_p tp, ccb_p cp);
 static void sym_int_sir (hcb_p np);
 static void sym_free_ccb (hcb_p np, ccb_p cp);
 static ccb_p sym_get_ccb (hcb_p np, u_char tn, u_char ln, u_char tag_order);
 static ccb_p sym_alloc_ccb (hcb_p np);
 static ccb_p sym_ccb_from_dsa (hcb_p np, u32 dsa);
 static lcb_p sym_alloc_lcb (hcb_p np, u_char tn, u_char ln);
 static void sym_alloc_lcb_tags (hcb_p np, u_char tn, u_char ln);
 static int  sym_snooptest (hcb_p np);
 static void sym_selectclock(hcb_p np, u_char scntl3);
 static void sym_getclock (hcb_p np, int mult);
 static int  sym_getpciclock (hcb_p np);
 static void sym_complete_ok (hcb_p np, ccb_p cp);
 static void sym_complete_error (hcb_p np, ccb_p cp);
 static void sym_callout (void *arg);
 static int  sym_abort_scsiio (hcb_p np, union ccb *ccb, int timed_out);
 static void sym_reset_dev (hcb_p np, union ccb *ccb);
 static void sym_action (struct cam_sim *sim, union ccb *ccb);
 static int  sym_setup_cdb (hcb_p np, struct ccb_scsiio *csio, ccb_p cp);
 static void sym_setup_data_and_start (hcb_p np, struct ccb_scsiio *csio,
 				      ccb_p cp);
 static int sym_fast_scatter_sg_physical(hcb_p np, ccb_p cp,
 					bus_dma_segment_t *psegs, int nsegs);
 static int sym_scatter_sg_physical (hcb_p np, ccb_p cp,
 				    bus_dma_segment_t *psegs, int nsegs);
 static void sym_action2 (struct cam_sim *sim, union ccb *ccb);
 static void sym_update_trans(hcb_p np, struct sym_trans *tip,
 			      struct ccb_trans_settings *cts);
 static void sym_update_dflags(hcb_p np, u_char *flags,
 			      struct ccb_trans_settings *cts);
 
 static const struct sym_pci_chip *sym_find_pci_chip (device_t dev);
 static int  sym_pci_probe (device_t dev);
 static int  sym_pci_attach (device_t dev);
 
 static void sym_pci_free (hcb_p np);
 static int  sym_cam_attach (hcb_p np);
 static void sym_cam_free (hcb_p np);
 
 static void sym_nvram_setup_host (hcb_p np, struct sym_nvram *nvram);
 static void sym_nvram_setup_target (hcb_p np, int targ, struct sym_nvram *nvp);
 static int sym_read_nvram (hcb_p np, struct sym_nvram *nvp);
 
 /*
  *  Print something which allows to retrieve the controller type,
  *  unit, target, lun concerned by a kernel message.
  */
 static void PRINT_TARGET (hcb_p np, int target)
 {
 	printf ("%s:%d:", sym_name(np), target);
 }
 
 static void PRINT_LUN(hcb_p np, int target, int lun)
 {
 	printf ("%s:%d:%d:", sym_name(np), target, lun);
 }
 
 static void PRINT_ADDR (ccb_p cp)
 {
 	if (cp && cp->cam_ccb)
 		xpt_print_path(cp->cam_ccb->ccb_h.path);
 }
 
 /*
  *  Take into account this ccb in the freeze count.
  */
 static void sym_freeze_cam_ccb(union ccb *ccb)
 {
 	if (!(ccb->ccb_h.flags & CAM_DEV_QFRZDIS)) {
 		if (!(ccb->ccb_h.status & CAM_DEV_QFRZN)) {
 			ccb->ccb_h.status |= CAM_DEV_QFRZN;
 			xpt_freeze_devq(ccb->ccb_h.path, 1);
 		}
 	}
 }
 
 /*
  *  Set the status field of a CAM CCB.
  */
 static __inline void sym_set_cam_status(union ccb *ccb, cam_status status)
 {
 	ccb->ccb_h.status &= ~CAM_STATUS_MASK;
 	ccb->ccb_h.status |= status;
 }
 
 /*
  *  Get the status field of a CAM CCB.
  */
 static __inline int sym_get_cam_status(union ccb *ccb)
 {
 	return ccb->ccb_h.status & CAM_STATUS_MASK;
 }
 
 /*
  *  Enqueue a CAM CCB.
  */
 static void sym_enqueue_cam_ccb(ccb_p cp)
 {
 	hcb_p np;
 	union ccb *ccb;
 
 	ccb = cp->cam_ccb;
 	np = (hcb_p) cp->arg;
 
 	assert(!(ccb->ccb_h.status & CAM_SIM_QUEUED));
 	ccb->ccb_h.status = CAM_REQ_INPROG;
 
 	callout_reset(&cp->ch, ccb->ccb_h.timeout * hz / 1000, sym_callout,
 			(caddr_t) ccb);
 	ccb->ccb_h.status |= CAM_SIM_QUEUED;
 	ccb->ccb_h.sym_hcb_ptr = np;
 
 	sym_insque_tail(sym_qptr(&ccb->ccb_h.sim_links), &np->cam_ccbq);
 }
 
 /*
  *  Complete a pending CAM CCB.
  */
 
 static void sym_xpt_done(hcb_p np, union ccb *ccb, ccb_p cp)
 {
 
 	SYM_LOCK_ASSERT(MA_OWNED);
 
 	if (ccb->ccb_h.status & CAM_SIM_QUEUED) {
 		callout_stop(&cp->ch);
 		sym_remque(sym_qptr(&ccb->ccb_h.sim_links));
 		ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
 		ccb->ccb_h.sym_hcb_ptr = NULL;
 	}
 	xpt_done(ccb);
 }
 
 static void sym_xpt_done2(hcb_p np, union ccb *ccb, int cam_status)
 {
 
 	SYM_LOCK_ASSERT(MA_OWNED);
 
 	sym_set_cam_status(ccb, cam_status);
 	xpt_done(ccb);
 }
 
 /*
  *  SYMBIOS chip clock divisor table.
  *
  *  Divisors are multiplied by 10,000,000 in order to make
  *  calculations more simple.
  */
 #define _5M 5000000
 static const u32 div_10M[] =
 	{2*_5M, 3*_5M, 4*_5M, 6*_5M, 8*_5M, 12*_5M, 16*_5M};
 
 /*
  *  SYMBIOS chips allow burst lengths of 2, 4, 8, 16, 32, 64,
  *  128 transfers. All chips support at least 16 transfers
  *  bursts. The 825A, 875 and 895 chips support bursts of up
  *  to 128 transfers and the 895A and 896 support bursts of up
  *  to 64 transfers. All other chips support up to 16
  *  transfers bursts.
  *
  *  For PCI 32 bit data transfers each transfer is a DWORD.
  *  It is a QUADWORD (8 bytes) for PCI 64 bit data transfers.
  *
  *  We use log base 2 (burst length) as internal code, with
  *  value 0 meaning "burst disabled".
  */
 
 /*
  *  Burst length from burst code.
  */
 #define burst_length(bc) (!(bc))? 0 : 1 << (bc)
 
 /*
  *  Burst code from io register bits.
  */
 #define burst_code(dmode, ctest4, ctest5) \
 	(ctest4) & 0x80? 0 : (((dmode) & 0xc0) >> 6) + ((ctest5) & 0x04) + 1
 
 /*
  *  Set initial io register bits from burst code.
  */
 static __inline void sym_init_burst(hcb_p np, u_char bc)
 {
 	np->rv_ctest4	&= ~0x80;
 	np->rv_dmode	&= ~(0x3 << 6);
 	np->rv_ctest5	&= ~0x4;
 
 	if (!bc) {
 		np->rv_ctest4	|= 0x80;
 	}
 	else {
 		--bc;
 		np->rv_dmode	|= ((bc & 0x3) << 6);
 		np->rv_ctest5	|= (bc & 0x4);
 	}
 }
 
 /*
  * Print out the list of targets that have some flag disabled by user.
  */
 static void sym_print_targets_flag(hcb_p np, int mask, char *msg)
 {
 	int cnt;
 	int i;
 
 	for (cnt = 0, i = 0 ; i < SYM_CONF_MAX_TARGET ; i++) {
 		if (i == np->myaddr)
 			continue;
 		if (np->target[i].usrflags & mask) {
 			if (!cnt++)
 				printf("%s: %s disabled for targets",
 					sym_name(np), msg);
 			printf(" %d", i);
 		}
 	}
 	if (cnt)
 		printf(".\n");
 }
 
 /*
  *  Save initial settings of some IO registers.
  *  Assumed to have been set by BIOS.
  *  We cannot reset the chip prior to reading the
  *  IO registers, since informations will be lost.
  *  Since the SCRIPTS processor may be running, this
  *  is not safe on paper, but it seems to work quite
  *  well. :)
  */
 static void sym_save_initial_setting (hcb_p np)
 {
 	np->sv_scntl0	= INB(nc_scntl0) & 0x0a;
 	np->sv_scntl3	= INB(nc_scntl3) & 0x07;
 	np->sv_dmode	= INB(nc_dmode)  & 0xce;
 	np->sv_dcntl	= INB(nc_dcntl)  & 0xa8;
 	np->sv_ctest3	= INB(nc_ctest3) & 0x01;
 	np->sv_ctest4	= INB(nc_ctest4) & 0x80;
 	np->sv_gpcntl	= INB(nc_gpcntl);
 	np->sv_stest1	= INB(nc_stest1);
 	np->sv_stest2	= INB(nc_stest2) & 0x20;
 	np->sv_stest4	= INB(nc_stest4);
 	if (np->features & FE_C10) {	/* Always large DMA fifo + ultra3 */
 		np->sv_scntl4	= INB(nc_scntl4);
 		np->sv_ctest5	= INB(nc_ctest5) & 0x04;
 	}
 	else
 		np->sv_ctest5	= INB(nc_ctest5) & 0x24;
 }
 
 /*
  *  Prepare io register values used by sym_init() according
  *  to selected and supported features.
  */
 static int sym_prepare_setting(hcb_p np, struct sym_nvram *nvram)
 {
 	u_char	burst_max;
 	u32	period;
 	int i;
 
 	/*
 	 *  Wide ?
 	 */
 	np->maxwide	= (np->features & FE_WIDE)? 1 : 0;
 
 	/*
 	 *  Get the frequency of the chip's clock.
 	 */
 	if	(np->features & FE_QUAD)
 		np->multiplier	= 4;
 	else if	(np->features & FE_DBLR)
 		np->multiplier	= 2;
 	else
 		np->multiplier	= 1;
 
 	np->clock_khz	= (np->features & FE_CLK80)? 80000 : 40000;
 	np->clock_khz	*= np->multiplier;
 
 	if (np->clock_khz != 40000)
 		sym_getclock(np, np->multiplier);
 
 	/*
 	 * Divisor to be used for async (timer pre-scaler).
 	 */
 	i = np->clock_divn - 1;
 	while (--i >= 0) {
 		if (10ul * SYM_CONF_MIN_ASYNC * np->clock_khz > div_10M[i]) {
 			++i;
 			break;
 		}
 	}
 	np->rv_scntl3 = i+1;
 
 	/*
 	 * The C1010 uses hardwired divisors for async.
 	 * So, we just throw away, the async. divisor.:-)
 	 */
 	if (np->features & FE_C10)
 		np->rv_scntl3 = 0;
 
 	/*
 	 * Minimum synchronous period factor supported by the chip.
 	 * Btw, 'period' is in tenths of nanoseconds.
 	 */
 	period = (4 * div_10M[0] + np->clock_khz - 1) / np->clock_khz;
 	if	(period <= 250)		np->minsync = 10;
 	else if	(period <= 303)		np->minsync = 11;
 	else if	(period <= 500)		np->minsync = 12;
 	else				np->minsync = (period + 40 - 1) / 40;
 
 	/*
 	 * Check against chip SCSI standard support (SCSI-2,ULTRA,ULTRA2).
 	 */
 	if	(np->minsync < 25 &&
 		 !(np->features & (FE_ULTRA|FE_ULTRA2|FE_ULTRA3)))
 		np->minsync = 25;
 	else if	(np->minsync < 12 &&
 		 !(np->features & (FE_ULTRA2|FE_ULTRA3)))
 		np->minsync = 12;
 
 	/*
 	 * Maximum synchronous period factor supported by the chip.
 	 */
 	period = (11 * div_10M[np->clock_divn - 1]) / (4 * np->clock_khz);
 	np->maxsync = period > 2540 ? 254 : period / 10;
 
 	/*
 	 * If chip is a C1010, guess the sync limits in DT mode.
 	 */
 	if ((np->features & (FE_C10|FE_ULTRA3)) == (FE_C10|FE_ULTRA3)) {
 		if (np->clock_khz == 160000) {
 			np->minsync_dt = 9;
 			np->maxsync_dt = 50;
 			np->maxoffs_dt = 62;
 		}
 	}
 
 	/*
 	 *  64 bit addressing  (895A/896/1010) ?
 	 */
 	if (np->features & FE_DAC)
 #ifdef __LP64__
 		np->rv_ccntl1	|= (XTIMOD | EXTIBMV);
 #else
 		np->rv_ccntl1	|= (DDAC);
 #endif
 
 	/*
 	 *  Phase mismatch handled by SCRIPTS (895A/896/1010) ?
   	 */
 	if (np->features & FE_NOPM)
 		np->rv_ccntl0	|= (ENPMJ);
 
  	/*
 	 *  C1010 Errata.
 	 *  In dual channel mode, contention occurs if internal cycles
 	 *  are used. Disable internal cycles.
 	 */
 	if (np->device_id == PCI_ID_LSI53C1010 &&
 	    np->revision_id < 0x2)
 		np->rv_ccntl0	|=  DILS;
 
 	/*
 	 *  Select burst length (dwords)
 	 */
 	burst_max	= SYM_SETUP_BURST_ORDER;
 	if (burst_max == 255)
 		burst_max = burst_code(np->sv_dmode, np->sv_ctest4,
 				       np->sv_ctest5);
 	if (burst_max > 7)
 		burst_max = 7;
 	if (burst_max > np->maxburst)
 		burst_max = np->maxburst;
 
 	/*
 	 *  DEL 352 - 53C810 Rev x11 - Part Number 609-0392140 - ITEM 2.
 	 *  This chip and the 860 Rev 1 may wrongly use PCI cache line
 	 *  based transactions on LOAD/STORE instructions. So we have
 	 *  to prevent these chips from using such PCI transactions in
 	 *  this driver. The generic ncr driver that does not use
 	 *  LOAD/STORE instructions does not need this work-around.
 	 */
 	if ((np->device_id == PCI_ID_SYM53C810 &&
 	     np->revision_id >= 0x10 && np->revision_id <= 0x11) ||
 	    (np->device_id == PCI_ID_SYM53C860 &&
 	     np->revision_id <= 0x1))
 		np->features &= ~(FE_WRIE|FE_ERL|FE_ERMP);
 
 	/*
 	 *  Select all supported special features.
 	 *  If we are using on-board RAM for scripts, prefetch (PFEN)
 	 *  does not help, but burst op fetch (BOF) does.
 	 *  Disabling PFEN makes sure BOF will be used.
 	 */
 	if (np->features & FE_ERL)
 		np->rv_dmode	|= ERL;		/* Enable Read Line */
 	if (np->features & FE_BOF)
 		np->rv_dmode	|= BOF;		/* Burst Opcode Fetch */
 	if (np->features & FE_ERMP)
 		np->rv_dmode	|= ERMP;	/* Enable Read Multiple */
 #if 1
 	if ((np->features & FE_PFEN) && !np->ram_ba)
 #else
 	if (np->features & FE_PFEN)
 #endif
 		np->rv_dcntl	|= PFEN;	/* Prefetch Enable */
 	if (np->features & FE_CLSE)
 		np->rv_dcntl	|= CLSE;	/* Cache Line Size Enable */
 	if (np->features & FE_WRIE)
 		np->rv_ctest3	|= WRIE;	/* Write and Invalidate */
 	if (np->features & FE_DFS)
 		np->rv_ctest5	|= DFS;		/* Dma Fifo Size */
 
 	/*
 	 *  Select some other
 	 */
 	if (SYM_SETUP_PCI_PARITY)
 		np->rv_ctest4	|= MPEE; /* Master parity checking */
 	if (SYM_SETUP_SCSI_PARITY)
 		np->rv_scntl0	|= 0x0a; /*  full arb., ena parity, par->ATN  */
 
 	/*
 	 *  Get parity checking, host ID and verbose mode from NVRAM
 	 */
 	np->myaddr = 255;
 	sym_nvram_setup_host (np, nvram);
 #ifdef __sparc64__
 	np->myaddr = OF_getscsinitid(np->device);
 #endif
 
 	/*
 	 *  Get SCSI addr of host adapter (set by bios?).
 	 */
 	if (np->myaddr == 255) {
 		np->myaddr = INB(nc_scid) & 0x07;
 		if (!np->myaddr)
 			np->myaddr = SYM_SETUP_HOST_ID;
 	}
 
 	/*
 	 *  Prepare initial io register bits for burst length
 	 */
 	sym_init_burst(np, burst_max);
 
 	/*
 	 *  Set SCSI BUS mode.
 	 *  - LVD capable chips (895/895A/896/1010) report the
 	 *    current BUS mode through the STEST4 IO register.
 	 *  - For previous generation chips (825/825A/875),
 	 *    user has to tell us how to check against HVD,
 	 *    since a 100% safe algorithm is not possible.
 	 */
 	np->scsi_mode = SMODE_SE;
 	if (np->features & (FE_ULTRA2|FE_ULTRA3))
 		np->scsi_mode = (np->sv_stest4 & SMODE);
 	else if	(np->features & FE_DIFF) {
 		if (SYM_SETUP_SCSI_DIFF == 1) {
 			if (np->sv_scntl3) {
 				if (np->sv_stest2 & 0x20)
 					np->scsi_mode = SMODE_HVD;
 			}
 			else if (nvram->type == SYM_SYMBIOS_NVRAM) {
 				if (!(INB(nc_gpreg) & 0x08))
 					np->scsi_mode = SMODE_HVD;
 			}
 		}
 		else if	(SYM_SETUP_SCSI_DIFF == 2)
 			np->scsi_mode = SMODE_HVD;
 	}
 	if (np->scsi_mode == SMODE_HVD)
 		np->rv_stest2 |= 0x20;
 
 	/*
 	 *  Set LED support from SCRIPTS.
 	 *  Ignore this feature for boards known to use a
 	 *  specific GPIO wiring and for the 895A, 896
 	 *  and 1010 that drive the LED directly.
 	 */
 	if ((SYM_SETUP_SCSI_LED ||
 	     (nvram->type == SYM_SYMBIOS_NVRAM ||
 	      (nvram->type == SYM_TEKRAM_NVRAM &&
 	       np->device_id == PCI_ID_SYM53C895))) &&
 	    !(np->features & FE_LEDC) && !(np->sv_gpcntl & 0x01))
 		np->features |= FE_LED0;
 
 	/*
 	 *  Set irq mode.
 	 */
 	switch(SYM_SETUP_IRQ_MODE & 3) {
 	case 2:
 		np->rv_dcntl	|= IRQM;
 		break;
 	case 1:
 		np->rv_dcntl	|= (np->sv_dcntl & IRQM);
 		break;
 	default:
 		break;
 	}
 
 	/*
 	 *  Configure targets according to driver setup.
 	 *  If NVRAM present get targets setup from NVRAM.
 	 */
 	for (i = 0 ; i < SYM_CONF_MAX_TARGET ; i++) {
 		tcb_p tp = &np->target[i];
 
 		tp->tinfo.user.scsi_version = tp->tinfo.current.scsi_version= 2;
 		tp->tinfo.user.spi_version  = tp->tinfo.current.spi_version = 2;
 		tp->tinfo.user.period = np->minsync;
 		if (np->features & FE_ULTRA3)
 			tp->tinfo.user.period = np->minsync_dt;
 		tp->tinfo.user.offset = np->maxoffs;
 		tp->tinfo.user.width  = np->maxwide ? BUS_16_BIT : BUS_8_BIT;
 		tp->usrflags |= (SYM_DISC_ENABLED | SYM_TAGS_ENABLED);
 		tp->usrtags = SYM_SETUP_MAX_TAG;
 
 		sym_nvram_setup_target (np, i, nvram);
 
 		/*
 		 *  For now, guess PPR/DT support from the period
 		 *  and BUS width.
 		 */
 		if (np->features & FE_ULTRA3) {
 			if (tp->tinfo.user.period <= 9	&&
 			    tp->tinfo.user.width == BUS_16_BIT) {
 				tp->tinfo.user.options |= PPR_OPT_DT;
 				tp->tinfo.user.offset   = np->maxoffs_dt;
 				tp->tinfo.user.spi_version = 3;
 			}
 		}
 
 		if (!tp->usrtags)
 			tp->usrflags &= ~SYM_TAGS_ENABLED;
 	}
 
 	/*
 	 *  Let user know about the settings.
 	 */
 	i = nvram->type;
 	printf("%s: %s NVRAM, ID %d, Fast-%d, %s, %s\n", sym_name(np),
 		i  == SYM_SYMBIOS_NVRAM ? "Symbios" :
 		(i == SYM_TEKRAM_NVRAM  ? "Tekram" : "No"),
 		np->myaddr,
 		(np->features & FE_ULTRA3) ? 80 :
 		(np->features & FE_ULTRA2) ? 40 :
 		(np->features & FE_ULTRA)  ? 20 : 10,
 		sym_scsi_bus_mode(np->scsi_mode),
 		(np->rv_scntl0 & 0xa)	? "parity checking" : "NO parity");
 	/*
 	 *  Tell him more on demand.
 	 */
 	if (sym_verbose) {
 		printf("%s: %s IRQ line driver%s\n",
 			sym_name(np),
 			np->rv_dcntl & IRQM ? "totem pole" : "open drain",
 			np->ram_ba ? ", using on-chip SRAM" : "");
 		printf("%s: using %s firmware.\n", sym_name(np), np->fw_name);
 		if (np->features & FE_NOPM)
 			printf("%s: handling phase mismatch from SCRIPTS.\n",
 			       sym_name(np));
 	}
 	/*
 	 *  And still more.
 	 */
 	if (sym_verbose > 1) {
 		printf ("%s: initial SCNTL3/DMODE/DCNTL/CTEST3/4/5 = "
 			"(hex) %02x/%02x/%02x/%02x/%02x/%02x\n",
 			sym_name(np), np->sv_scntl3, np->sv_dmode, np->sv_dcntl,
 			np->sv_ctest3, np->sv_ctest4, np->sv_ctest5);
 
 		printf ("%s: final   SCNTL3/DMODE/DCNTL/CTEST3/4/5 = "
 			"(hex) %02x/%02x/%02x/%02x/%02x/%02x\n",
 			sym_name(np), np->rv_scntl3, np->rv_dmode, np->rv_dcntl,
 			np->rv_ctest3, np->rv_ctest4, np->rv_ctest5);
 	}
 	/*
 	 *  Let user be aware of targets that have some disable flags set.
 	 */
 	sym_print_targets_flag(np, SYM_SCAN_BOOT_DISABLED, "SCAN AT BOOT");
 	if (sym_verbose)
 		sym_print_targets_flag(np, SYM_SCAN_LUNS_DISABLED,
 				       "SCAN FOR LUNS");
 
 	return 0;
 }
 
 /*
  *  Prepare the next negotiation message if needed.
  *
  *  Fill in the part of message buffer that contains the
  *  negotiation and the nego_status field of the CCB.
  *  Returns the size of the message in bytes.
  */
 static int sym_prepare_nego(hcb_p np, ccb_p cp, int nego, u_char *msgptr)
 {
 	tcb_p tp = &np->target[cp->target];
 	int msglen = 0;
 
 	/*
 	 *  Early C1010 chips need a work-around for DT
 	 *  data transfer to work.
 	 */
 	if (!(np->features & FE_U3EN))
 		tp->tinfo.goal.options = 0;
 	/*
 	 *  negotiate using PPR ?
 	 */
 	if (tp->tinfo.goal.options & PPR_OPT_MASK)
 		nego = NS_PPR;
 	/*
 	 *  negotiate wide transfers ?
 	 */
 	else if (tp->tinfo.current.width != tp->tinfo.goal.width)
 		nego = NS_WIDE;
 	/*
 	 *  negotiate synchronous transfers?
 	 */
 	else if (tp->tinfo.current.period != tp->tinfo.goal.period ||
 		 tp->tinfo.current.offset != tp->tinfo.goal.offset)
 		nego = NS_SYNC;
 
 	switch (nego) {
 	case NS_SYNC:
 		msgptr[msglen++] = M_EXTENDED;
 		msgptr[msglen++] = 3;
 		msgptr[msglen++] = M_X_SYNC_REQ;
 		msgptr[msglen++] = tp->tinfo.goal.period;
 		msgptr[msglen++] = tp->tinfo.goal.offset;
 		break;
 	case NS_WIDE:
 		msgptr[msglen++] = M_EXTENDED;
 		msgptr[msglen++] = 2;
 		msgptr[msglen++] = M_X_WIDE_REQ;
 		msgptr[msglen++] = tp->tinfo.goal.width;
 		break;
 	case NS_PPR:
 		msgptr[msglen++] = M_EXTENDED;
 		msgptr[msglen++] = 6;
 		msgptr[msglen++] = M_X_PPR_REQ;
 		msgptr[msglen++] = tp->tinfo.goal.period;
 		msgptr[msglen++] = 0;
 		msgptr[msglen++] = tp->tinfo.goal.offset;
 		msgptr[msglen++] = tp->tinfo.goal.width;
 		msgptr[msglen++] = tp->tinfo.goal.options & PPR_OPT_DT;
 		break;
 	};
 
 	cp->nego_status = nego;
 
 	if (nego) {
 		tp->nego_cp = cp; /* Keep track a nego will be performed */
 		if (DEBUG_FLAGS & DEBUG_NEGO) {
 			sym_print_msg(cp, nego == NS_SYNC ? "sync msgout" :
 					  nego == NS_WIDE ? "wide msgout" :
 					  "ppr msgout", msgptr);
 		};
 	};
 
 	return msglen;
 }
 
 /*
  *  Insert a job into the start queue.
  */
 static void sym_put_start_queue(hcb_p np, ccb_p cp)
 {
 	u_short	qidx;
 
 #ifdef SYM_CONF_IARB_SUPPORT
 	/*
 	 *  If the previously queued CCB is not yet done,
 	 *  set the IARB hint. The SCRIPTS will go with IARB
 	 *  for this job when starting the previous one.
 	 *  We leave devices a chance to win arbitration by
 	 *  not using more than 'iarb_max' consecutive
 	 *  immediate arbitrations.
 	 */
 	if (np->last_cp && np->iarb_count < np->iarb_max) {
 		np->last_cp->host_flags |= HF_HINT_IARB;
 		++np->iarb_count;
 	}
 	else
 		np->iarb_count = 0;
 	np->last_cp = cp;
 #endif
 
 	/*
 	 *  Insert first the idle task and then our job.
 	 *  The MB should ensure proper ordering.
 	 */
 	qidx = np->squeueput + 2;
 	if (qidx >= MAX_QUEUE*2) qidx = 0;
 
 	np->squeue [qidx]	   = cpu_to_scr(np->idletask_ba);
 	MEMORY_BARRIER();
 	np->squeue [np->squeueput] = cpu_to_scr(cp->ccb_ba);
 
 	np->squeueput = qidx;
 
 	if (DEBUG_FLAGS & DEBUG_QUEUE)
 		printf ("%s: queuepos=%d.\n", sym_name (np), np->squeueput);
 
 	/*
 	 *  Script processor may be waiting for reselect.
 	 *  Wake it up.
 	 */
 	MEMORY_BARRIER();
 	OUTB (nc_istat, SIGP|np->istat_sem);
 }
 
 /*
  *  Soft reset the chip.
  *
  *  Raising SRST when the chip is running may cause
  *  problems on dual function chips (see below).
  *  On the other hand, LVD devices need some delay
  *  to settle and report actual BUS mode in STEST4.
  */
 static void sym_chip_reset (hcb_p np)
 {
 	OUTB (nc_istat, SRST);
 	UDELAY (10);
 	OUTB (nc_istat, 0);
 	UDELAY(2000);	/* For BUS MODE to settle */
 }
 
 /*
  *  Soft reset the chip.
  *
  *  Some 896 and 876 chip revisions may hang-up if we set
  *  the SRST (soft reset) bit at the wrong time when SCRIPTS
  *  are running.
  *  So, we need to abort the current operation prior to
  *  soft resetting the chip.
  */
 static void sym_soft_reset (hcb_p np)
 {
 	u_char istat;
 	int i;
 
 	OUTB (nc_istat, CABRT);
 	for (i = 1000000 ; i ; --i) {
 		istat = INB (nc_istat);
 		if (istat & SIP) {
 			INW (nc_sist);
 			continue;
 		}
 		if (istat & DIP) {
 			OUTB (nc_istat, 0);
 			INB (nc_dstat);
 			break;
 		}
 	}
 	if (!i)
 		printf("%s: unable to abort current chip operation.\n",
 			sym_name(np));
 	sym_chip_reset (np);
 }
 
 /*
  *  Start reset process.
  *
  *  The interrupt handler will reinitialize the chip.
  */
 static void sym_start_reset(hcb_p np)
 {
 	(void) sym_reset_scsi_bus(np, 1);
 }
 
 static int sym_reset_scsi_bus(hcb_p np, int enab_int)
 {
 	u32 term;
 	int retv = 0;
 
 	sym_soft_reset(np);	/* Soft reset the chip */
 	if (enab_int)
 		OUTW (nc_sien, RST);
 	/*
 	 *  Enable Tolerant, reset IRQD if present and
 	 *  properly set IRQ mode, prior to resetting the bus.
 	 */
 	OUTB (nc_stest3, TE);
 	OUTB (nc_dcntl, (np->rv_dcntl & IRQM));
 	OUTB (nc_scntl1, CRST);
 	UDELAY (200);
 
 	if (!SYM_SETUP_SCSI_BUS_CHECK)
 		goto out;
 	/*
 	 *  Check for no terminators or SCSI bus shorts to ground.
 	 *  Read SCSI data bus, data parity bits and control signals.
 	 *  We are expecting RESET to be TRUE and other signals to be
 	 *  FALSE.
 	 */
 	term =	INB(nc_sstat0);
 	term =	((term & 2) << 7) + ((term & 1) << 17);	/* rst sdp0 */
 	term |= ((INB(nc_sstat2) & 0x01) << 26) |	/* sdp1     */
 		((INW(nc_sbdl) & 0xff)   << 9)  |	/* d7-0     */
 		((INW(nc_sbdl) & 0xff00) << 10) |	/* d15-8    */
 		INB(nc_sbcl);	/* req ack bsy sel atn msg cd io    */
 
 	if (!(np->features & FE_WIDE))
 		term &= 0x3ffff;
 
 	if (term != (2<<7)) {
 		printf("%s: suspicious SCSI data while resetting the BUS.\n",
 			sym_name(np));
 		printf("%s: %sdp0,d7-0,rst,req,ack,bsy,sel,atn,msg,c/d,i/o = "
 			"0x%lx, expecting 0x%lx\n",
 			sym_name(np),
 			(np->features & FE_WIDE) ? "dp1,d15-8," : "",
 			(u_long)term, (u_long)(2<<7));
 		if (SYM_SETUP_SCSI_BUS_CHECK == 1)
 			retv = 1;
 	}
 out:
 	OUTB (nc_scntl1, 0);
 	/* MDELAY(100); */
 	return retv;
 }
 
 /*
  *  The chip may have completed jobs. Look at the DONE QUEUE.
  *
  *  On architectures that may reorder LOAD/STORE operations,
  *  a memory barrier may be needed after the reading of the
  *  so-called `flag' and prior to dealing with the data.
  */
 static int sym_wakeup_done (hcb_p np)
 {
 	ccb_p cp;
 	int i, n;
 	u32 dsa;
 
 	SYM_LOCK_ASSERT(MA_OWNED);
 
 	n = 0;
 	i = np->dqueueget;
 	while (1) {
 		dsa = scr_to_cpu(np->dqueue[i]);
 		if (!dsa)
 			break;
 		np->dqueue[i] = 0;
 		if ((i = i+2) >= MAX_QUEUE*2)
 			i = 0;
 
 		cp = sym_ccb_from_dsa(np, dsa);
 		if (cp) {
 			MEMORY_BARRIER();
 			sym_complete_ok (np, cp);
 			++n;
 		}
 		else
 			printf ("%s: bad DSA (%x) in done queue.\n",
 				sym_name(np), (u_int) dsa);
 	}
 	np->dqueueget = i;
 
 	return n;
 }
 
 /*
  *  Complete all active CCBs with error.
  *  Used on CHIP/SCSI RESET.
  */
 static void sym_flush_busy_queue (hcb_p np, int cam_status)
 {
 	/*
 	 *  Move all active CCBs to the COMP queue
 	 *  and flush this queue.
 	 */
 	sym_que_splice(&np->busy_ccbq, &np->comp_ccbq);
 	sym_que_init(&np->busy_ccbq);
 	sym_flush_comp_queue(np, cam_status);
 }
 
 /*
  *  Start chip.
  *
  *  'reason' means:
  *     0: initialisation.
  *     1: SCSI BUS RESET delivered or received.
  *     2: SCSI BUS MODE changed.
  */
 static void sym_init (hcb_p np, int reason)
 {
  	int	i;
 	u32	phys;
 
 	SYM_LOCK_ASSERT(MA_OWNED);
 
  	/*
 	 *  Reset chip if asked, otherwise just clear fifos.
  	 */
 	if (reason == 1)
 		sym_soft_reset(np);
 	else {
 		OUTB (nc_stest3, TE|CSF);
 		OUTONB (nc_ctest3, CLF);
 	}
 
 	/*
 	 *  Clear Start Queue
 	 */
 	phys = np->squeue_ba;
 	for (i = 0; i < MAX_QUEUE*2; i += 2) {
 		np->squeue[i]   = cpu_to_scr(np->idletask_ba);
 		np->squeue[i+1] = cpu_to_scr(phys + (i+2)*4);
 	}
 	np->squeue[MAX_QUEUE*2-1] = cpu_to_scr(phys);
 
 	/*
 	 *  Start at first entry.
 	 */
 	np->squeueput = 0;
 
 	/*
 	 *  Clear Done Queue
 	 */
 	phys = np->dqueue_ba;
 	for (i = 0; i < MAX_QUEUE*2; i += 2) {
 		np->dqueue[i]   = 0;
 		np->dqueue[i+1] = cpu_to_scr(phys + (i+2)*4);
 	}
 	np->dqueue[MAX_QUEUE*2-1] = cpu_to_scr(phys);
 
 	/*
 	 *  Start at first entry.
 	 */
 	np->dqueueget = 0;
 
 	/*
 	 *  Install patches in scripts.
 	 *  This also let point to first position the start
 	 *  and done queue pointers used from SCRIPTS.
 	 */
 	np->fw_patch(np);
 
 	/*
 	 *  Wakeup all pending jobs.
 	 */
 	sym_flush_busy_queue(np, CAM_SCSI_BUS_RESET);
 
 	/*
 	 *  Init chip.
 	 */
 	OUTB (nc_istat,  0x00   );	/*  Remove Reset, abort */
 	UDELAY (2000);	/* The 895 needs time for the bus mode to settle */
 
 	OUTB (nc_scntl0, np->rv_scntl0 | 0xc0);
 					/*  full arb., ena parity, par->ATN  */
 	OUTB (nc_scntl1, 0x00);		/*  odd parity, and remove CRST!! */
 
 	sym_selectclock(np, np->rv_scntl3);	/* Select SCSI clock */
 
 	OUTB (nc_scid  , RRE|np->myaddr);	/* Adapter SCSI address */
 	OUTW (nc_respid, 1ul<<np->myaddr);	/* Id to respond to */
 	OUTB (nc_istat , SIGP	);		/*  Signal Process */
 	OUTB (nc_dmode , np->rv_dmode);		/* Burst length, dma mode */
 	OUTB (nc_ctest5, np->rv_ctest5);	/* Large fifo + large burst */
 
 	OUTB (nc_dcntl , NOCOM|np->rv_dcntl);	/* Protect SFBR */
 	OUTB (nc_ctest3, np->rv_ctest3);	/* Write and invalidate */
 	OUTB (nc_ctest4, np->rv_ctest4);	/* Master parity checking */
 
 	/* Extended Sreq/Sack filtering not supported on the C10 */
 	if (np->features & FE_C10)
 		OUTB (nc_stest2, np->rv_stest2);
 	else
 		OUTB (nc_stest2, EXT|np->rv_stest2);
 
 	OUTB (nc_stest3, TE);			/* TolerANT enable */
 	OUTB (nc_stime0, 0x0c);			/* HTH disabled  STO 0.25 sec */
 
 	/*
 	 *  For now, disable AIP generation on C1010-66.
 	 */
 	if (np->device_id == PCI_ID_LSI53C1010_2)
 		OUTB (nc_aipcntl1, DISAIP);
 
 	/*
 	 *  C10101 Errata.
 	 *  Errant SGE's when in narrow. Write bits 4 & 5 of
 	 *  STEST1 register to disable SGE. We probably should do
 	 *  that from SCRIPTS for each selection/reselection, but
 	 *  I just don't want. :)
 	 */
 	if (np->device_id == PCI_ID_LSI53C1010 &&
 	    /* np->revision_id < 0xff */ 1)
 		OUTB (nc_stest1, INB(nc_stest1) | 0x30);
 
 	/*
 	 *  DEL 441 - 53C876 Rev 5 - Part Number 609-0392787/2788 - ITEM 2.
 	 *  Disable overlapped arbitration for some dual function devices,
 	 *  regardless revision id (kind of post-chip-design feature. ;-))
 	 */
 	if (np->device_id == PCI_ID_SYM53C875)
 		OUTB (nc_ctest0, (1<<5));
 	else if (np->device_id == PCI_ID_SYM53C896)
 		np->rv_ccntl0 |= DPR;
 
 	/*
 	 *  Write CCNTL0/CCNTL1 for chips capable of 64 bit addressing
 	 *  and/or hardware phase mismatch, since only such chips
 	 *  seem to support those IO registers.
 	 */
 	if (np->features & (FE_DAC|FE_NOPM)) {
 		OUTB (nc_ccntl0, np->rv_ccntl0);
 		OUTB (nc_ccntl1, np->rv_ccntl1);
 	}
 
 	/*
 	 *  If phase mismatch handled by scripts (895A/896/1010),
 	 *  set PM jump addresses.
 	 */
 	if (np->features & FE_NOPM) {
 		OUTL (nc_pmjad1, SCRIPTB_BA (np, pm_handle));
 		OUTL (nc_pmjad2, SCRIPTB_BA (np, pm_handle));
 	}
 
 	/*
 	 *    Enable GPIO0 pin for writing if LED support from SCRIPTS.
 	 *    Also set GPIO5 and clear GPIO6 if hardware LED control.
 	 */
 	if (np->features & FE_LED0)
 		OUTB(nc_gpcntl, INB(nc_gpcntl) & ~0x01);
 	else if (np->features & FE_LEDC)
 		OUTB(nc_gpcntl, (INB(nc_gpcntl) & ~0x41) | 0x20);
 
 	/*
 	 *      enable ints
 	 */
 	OUTW (nc_sien , STO|HTH|MA|SGE|UDC|RST|PAR);
 	OUTB (nc_dien , MDPE|BF|SSI|SIR|IID);
 
 	/*
 	 *  For 895/6 enable SBMC interrupt and save current SCSI bus mode.
 	 *  Try to eat the spurious SBMC interrupt that may occur when
 	 *  we reset the chip but not the SCSI BUS (at initialization).
 	 */
 	if (np->features & (FE_ULTRA2|FE_ULTRA3)) {
 		OUTONW (nc_sien, SBMC);
 		if (reason == 0) {
 			MDELAY(100);
 			INW (nc_sist);
 		}
 		np->scsi_mode = INB (nc_stest4) & SMODE;
 	}
 
 	/*
 	 *  Fill in target structure.
 	 *  Reinitialize usrsync.
 	 *  Reinitialize usrwide.
 	 *  Prepare sync negotiation according to actual SCSI bus mode.
 	 */
 	for (i=0;i<SYM_CONF_MAX_TARGET;i++) {
 		tcb_p tp = &np->target[i];
 
 		tp->to_reset  = 0;
 		tp->head.sval = 0;
 		tp->head.wval = np->rv_scntl3;
 		tp->head.uval = 0;
 
 		tp->tinfo.current.period = 0;
 		tp->tinfo.current.offset = 0;
 		tp->tinfo.current.width  = BUS_8_BIT;
 		tp->tinfo.current.options = 0;
 	}
 
 	/*
 	 *  Download SCSI SCRIPTS to on-chip RAM if present,
 	 *  and start script processor.
 	 */
 	if (np->ram_ba) {
 		if (sym_verbose > 1)
 			printf ("%s: Downloading SCSI SCRIPTS.\n",
 				sym_name(np));
 		if (np->ram_ws == 8192) {
 			OUTRAM_OFF(4096, np->scriptb0, np->scriptb_sz);
 			OUTL (nc_mmws, np->scr_ram_seg);
 			OUTL (nc_mmrs, np->scr_ram_seg);
 			OUTL (nc_sfs,  np->scr_ram_seg);
 			phys = SCRIPTB_BA (np, start64);
 		}
 		else
 			phys = SCRIPTA_BA (np, init);
 		OUTRAM_OFF(0, np->scripta0, np->scripta_sz);
 	}
 	else
 		phys = SCRIPTA_BA (np, init);
 
 	np->istat_sem = 0;
 
 	OUTL (nc_dsa, np->hcb_ba);
 	OUTL_DSP (phys);
 
 	/*
 	 *  Notify the XPT about the RESET condition.
 	 */
 	if (reason != 0)
 		xpt_async(AC_BUS_RESET, np->path, NULL);
 }
 
 /*
  *  Get clock factor and sync divisor for a given
  *  synchronous factor period.
  */
 static int
 sym_getsync(hcb_p np, u_char dt, u_char sfac, u_char *divp, u_char *fakp)
 {
 	u32	clk = np->clock_khz;	/* SCSI clock frequency in kHz	*/
 	int	div = np->clock_divn;	/* Number of divisors supported	*/
 	u32	fak;			/* Sync factor in sxfer		*/
 	u32	per;			/* Period in tenths of ns	*/
 	u32	kpc;			/* (per * clk)			*/
 	int	ret;
 
 	/*
 	 *  Compute the synchronous period in tenths of nano-seconds
 	 */
 	if (dt && sfac <= 9)	per = 125;
 	else if	(sfac <= 10)	per = 250;
 	else if	(sfac == 11)	per = 303;
 	else if	(sfac == 12)	per = 500;
 	else			per = 40 * sfac;
 	ret = per;
 
 	kpc = per * clk;
 	if (dt)
 		kpc <<= 1;
 
 	/*
 	 *  For earliest C10 revision 0, we cannot use extra
 	 *  clocks for the setting of the SCSI clocking.
 	 *  Note that this limits the lowest sync data transfer
 	 *  to 5 Mega-transfers per second and may result in
 	 *  using higher clock divisors.
 	 */
 #if 1
 	if ((np->features & (FE_C10|FE_U3EN)) == FE_C10) {
 		/*
 		 *  Look for the lowest clock divisor that allows an
 		 *  output speed not faster than the period.
 		 */
 		while (div > 0) {
 			--div;
 			if (kpc > (div_10M[div] << 2)) {
 				++div;
 				break;
 			}
 		}
 		fak = 0;			/* No extra clocks */
 		if (div == np->clock_divn) {	/* Are we too fast ? */
 			ret = -1;
 		}
 		*divp = div;
 		*fakp = fak;
 		return ret;
 	}
 #endif
 
 	/*
 	 *  Look for the greatest clock divisor that allows an
 	 *  input speed faster than the period.
 	 */
 	while (div-- > 0)
 		if (kpc >= (div_10M[div] << 2)) break;
 
 	/*
 	 *  Calculate the lowest clock factor that allows an output
 	 *  speed not faster than the period, and the max output speed.
 	 *  If fak >= 1 we will set both XCLKH_ST and XCLKH_DT.
 	 *  If fak >= 2 we will also set XCLKS_ST and XCLKS_DT.
 	 */
 	if (dt) {
 		fak = (kpc - 1) / (div_10M[div] << 1) + 1 - 2;
 		/* ret = ((2+fak)*div_10M[div])/np->clock_khz; */
 	}
 	else {
 		fak = (kpc - 1) / div_10M[div] + 1 - 4;
 		/* ret = ((4+fak)*div_10M[div])/np->clock_khz; */
 	}
 
 	/*
 	 *  Check against our hardware limits, or bugs :).
 	 */
 	if (fak > 2)	{fak = 2; ret = -1;}
 
 	/*
 	 *  Compute and return sync parameters.
 	 */
 	*divp = div;
 	*fakp = fak;
 
 	return ret;
 }
 
 /*
  *  Tell the SCSI layer about the new transfer parameters.
  */
 static void
 sym_xpt_async_transfer_neg(hcb_p np, int target, u_int spi_valid)
 {
 	struct ccb_trans_settings cts;
 	struct cam_path *path;
 	int sts;
 	tcb_p tp = &np->target[target];
 
 	sts = xpt_create_path(&path, NULL, cam_sim_path(np->sim), target,
 	                      CAM_LUN_WILDCARD);
 	if (sts != CAM_REQ_CMP)
 		return;
 
 	bzero(&cts, sizeof(cts));
 
 #define	cts__scsi (cts.proto_specific.scsi)
 #define	cts__spi  (cts.xport_specific.spi)
 
 	cts.type      = CTS_TYPE_CURRENT_SETTINGS;
 	cts.protocol  = PROTO_SCSI;
 	cts.transport = XPORT_SPI;
 	cts.protocol_version  = tp->tinfo.current.scsi_version;
 	cts.transport_version = tp->tinfo.current.spi_version;
 
 	cts__spi.valid = spi_valid;
 	if (spi_valid & CTS_SPI_VALID_SYNC_RATE)
 		cts__spi.sync_period = tp->tinfo.current.period;
 	if (spi_valid & CTS_SPI_VALID_SYNC_OFFSET)
 		cts__spi.sync_offset = tp->tinfo.current.offset;
 	if (spi_valid & CTS_SPI_VALID_BUS_WIDTH)
 		cts__spi.bus_width   = tp->tinfo.current.width;
 	if (spi_valid & CTS_SPI_VALID_PPR_OPTIONS)
 		cts__spi.ppr_options = tp->tinfo.current.options;
 #undef cts__spi
 #undef cts__scsi
 	xpt_setup_ccb(&cts.ccb_h, path, /*priority*/1);
 	xpt_async(AC_TRANSFER_NEG, path, &cts);
 	xpt_free_path(path);
 }
 
 #define SYM_SPI_VALID_WDTR		\
 	CTS_SPI_VALID_BUS_WIDTH |	\
 	CTS_SPI_VALID_SYNC_RATE |	\
 	CTS_SPI_VALID_SYNC_OFFSET
 #define SYM_SPI_VALID_SDTR		\
 	CTS_SPI_VALID_SYNC_RATE |	\
 	CTS_SPI_VALID_SYNC_OFFSET
 #define SYM_SPI_VALID_PPR		\
 	CTS_SPI_VALID_PPR_OPTIONS |	\
 	CTS_SPI_VALID_BUS_WIDTH |	\
 	CTS_SPI_VALID_SYNC_RATE |	\
 	CTS_SPI_VALID_SYNC_OFFSET
 
 /*
  *  We received a WDTR.
  *  Let everything be aware of the changes.
  */
 static void sym_setwide(hcb_p np, ccb_p cp, u_char wide)
 {
 	tcb_p tp = &np->target[cp->target];
 
 	sym_settrans(np, cp, 0, 0, 0, wide, 0, 0);
 
 	/*
 	 *  Tell the SCSI layer about the new transfer parameters.
 	 */
 	tp->tinfo.goal.width = tp->tinfo.current.width = wide;
 	tp->tinfo.current.offset = 0;
 	tp->tinfo.current.period = 0;
 	tp->tinfo.current.options = 0;
 
 	sym_xpt_async_transfer_neg(np, cp->target, SYM_SPI_VALID_WDTR);
 }
 
 /*
  *  We received a SDTR.
  *  Let everything be aware of the changes.
  */
 static void
 sym_setsync(hcb_p np, ccb_p cp, u_char ofs, u_char per, u_char div, u_char fak)
 {
 	tcb_p tp = &np->target[cp->target];
 	u_char wide = (cp->phys.select.sel_scntl3 & EWS) ? 1 : 0;
 
 	sym_settrans(np, cp, 0, ofs, per, wide, div, fak);
 
 	/*
 	 *  Tell the SCSI layer about the new transfer parameters.
 	 */
 	tp->tinfo.goal.period	= tp->tinfo.current.period  = per;
 	tp->tinfo.goal.offset	= tp->tinfo.current.offset  = ofs;
 	tp->tinfo.goal.options	= tp->tinfo.current.options = 0;
 
 	sym_xpt_async_transfer_neg(np, cp->target, SYM_SPI_VALID_SDTR);
 }
 
 /*
  *  We received a PPR.
  *  Let everything be aware of the changes.
  */
 static void sym_setpprot(hcb_p np, ccb_p cp, u_char dt, u_char ofs,
 			 u_char per, u_char wide, u_char div, u_char fak)
 {
 	tcb_p tp = &np->target[cp->target];
 
 	sym_settrans(np, cp, dt, ofs, per, wide, div, fak);
 
 	/*
 	 *  Tell the SCSI layer about the new transfer parameters.
 	 */
 	tp->tinfo.goal.width	= tp->tinfo.current.width  = wide;
 	tp->tinfo.goal.period	= tp->tinfo.current.period = per;
 	tp->tinfo.goal.offset	= tp->tinfo.current.offset = ofs;
 	tp->tinfo.goal.options	= tp->tinfo.current.options = dt;
 
 	sym_xpt_async_transfer_neg(np, cp->target, SYM_SPI_VALID_PPR);
 }
 
 /*
  *  Switch trans mode for current job and it's target.
  */
 static void sym_settrans(hcb_p np, ccb_p cp, u_char dt, u_char ofs,
 			 u_char per, u_char wide, u_char div, u_char fak)
 {
 	SYM_QUEHEAD *qp;
 	union	ccb *ccb;
 	tcb_p tp;
 	u_char target = INB (nc_sdid) & 0x0f;
 	u_char sval, wval, uval;
 
 	assert (cp);
 	if (!cp) return;
 	ccb = cp->cam_ccb;
 	assert (ccb);
 	if (!ccb) return;
 	assert (target == (cp->target & 0xf));
 	tp = &np->target[target];
 
 	sval = tp->head.sval;
 	wval = tp->head.wval;
 	uval = tp->head.uval;
 
 #if 0
 	printf("XXXX sval=%x wval=%x uval=%x (%x)\n",
 		sval, wval, uval, np->rv_scntl3);
 #endif
 	/*
 	 *  Set the offset.
 	 */
 	if (!(np->features & FE_C10))
 		sval = (sval & ~0x1f) | ofs;
 	else
 		sval = (sval & ~0x3f) | ofs;
 
 	/*
 	 *  Set the sync divisor and extra clock factor.
 	 */
 	if (ofs != 0) {
 		wval = (wval & ~0x70) | ((div+1) << 4);
 		if (!(np->features & FE_C10))
 			sval = (sval & ~0xe0) | (fak << 5);
 		else {
 			uval = uval & ~(XCLKH_ST|XCLKH_DT|XCLKS_ST|XCLKS_DT);
 			if (fak >= 1) uval |= (XCLKH_ST|XCLKH_DT);
 			if (fak >= 2) uval |= (XCLKS_ST|XCLKS_DT);
 		}
 	}
 
 	/*
 	 *  Set the bus width.
 	 */
 	wval = wval & ~EWS;
 	if (wide != 0)
 		wval |= EWS;
 
 	/*
 	 *  Set misc. ultra enable bits.
 	 */
 	if (np->features & FE_C10) {
 		uval = uval & ~(U3EN|AIPCKEN);
 		if (dt)	{
 			assert(np->features & FE_U3EN);
 			uval |= U3EN;
 		}
 	}
 	else {
 		wval = wval & ~ULTRA;
 		if (per <= 12)	wval |= ULTRA;
 	}
 
 	/*
 	 *   Stop there if sync parameters are unchanged.
 	 */
 	if (tp->head.sval == sval &&
 	    tp->head.wval == wval &&
 	    tp->head.uval == uval)
 		return;
 	tp->head.sval = sval;
 	tp->head.wval = wval;
 	tp->head.uval = uval;
 
 	/*
 	 *  Disable extended Sreq/Sack filtering if per < 50.
 	 *  Not supported on the C1010.
 	 */
 	if (per < 50 && !(np->features & FE_C10))
 		OUTOFFB (nc_stest2, EXT);
 
 	/*
 	 *  set actual value and sync_status
 	 */
 	OUTB (nc_sxfer,  tp->head.sval);
 	OUTB (nc_scntl3, tp->head.wval);
 
 	if (np->features & FE_C10) {
 		OUTB (nc_scntl4, tp->head.uval);
 	}
 
 	/*
 	 *  patch ALL busy ccbs of this target.
 	 */
 	FOR_EACH_QUEUED_ELEMENT(&np->busy_ccbq, qp) {
 		cp = sym_que_entry(qp, struct sym_ccb, link_ccbq);
 		if (cp->target != target)
 			continue;
 		cp->phys.select.sel_scntl3 = tp->head.wval;
 		cp->phys.select.sel_sxfer  = tp->head.sval;
 		if (np->features & FE_C10) {
 			cp->phys.select.sel_scntl4 = tp->head.uval;
 		}
 	}
 }
 
 /*
  *  log message for real hard errors
  *
  *  sym0 targ 0?: ERROR (ds:si) (so-si-sd) (sxfer/scntl3) @ name (dsp:dbc).
  *  	      reg: r0 r1 r2 r3 r4 r5 r6 ..... rf.
  *
  *  exception register:
  *  	ds:	dstat
  *  	si:	sist
  *
  *  SCSI bus lines:
  *  	so:	control lines as driven by chip.
  *  	si:	control lines as seen by chip.
  *  	sd:	scsi data lines as seen by chip.
  *
  *  wide/fastmode:
  *  	sxfer:	(see the manual)
  *  	scntl3:	(see the manual)
  *
  *  current script command:
  *  	dsp:	script address (relative to start of script).
  *  	dbc:	first word of script command.
  *
  *  First 24 register of the chip:
  *  	r0..rf
  */
 static void sym_log_hard_error(hcb_p np, u_short sist, u_char dstat)
 {
 	u32	dsp;
 	int	script_ofs;
 	int	script_size;
 	char	*script_name;
 	u_char	*script_base;
 	int	i;
 
 	dsp	= INL (nc_dsp);
 
 	if	(dsp > np->scripta_ba &&
 		 dsp <= np->scripta_ba + np->scripta_sz) {
 		script_ofs	= dsp - np->scripta_ba;
 		script_size	= np->scripta_sz;
 		script_base	= (u_char *) np->scripta0;
 		script_name	= "scripta";
 	}
 	else if (np->scriptb_ba < dsp &&
 		 dsp <= np->scriptb_ba + np->scriptb_sz) {
 		script_ofs	= dsp - np->scriptb_ba;
 		script_size	= np->scriptb_sz;
 		script_base	= (u_char *) np->scriptb0;
 		script_name	= "scriptb";
 	} else {
 		script_ofs	= dsp;
 		script_size	= 0;
 		script_base	= 0;
 		script_name	= "mem";
 	}
 
 	printf ("%s:%d: ERROR (%x:%x) (%x-%x-%x) (%x/%x) @ (%s %x:%08x).\n",
 		sym_name (np), (unsigned)INB (nc_sdid)&0x0f, dstat, sist,
 		(unsigned)INB (nc_socl), (unsigned)INB (nc_sbcl),
 		(unsigned)INB (nc_sbdl), (unsigned)INB (nc_sxfer),
 		(unsigned)INB (nc_scntl3), script_name, script_ofs,
 		(unsigned)INL (nc_dbc));
 
 	if (((script_ofs & 3) == 0) &&
 	    (unsigned)script_ofs < script_size) {
 		printf ("%s: script cmd = %08x\n", sym_name(np),
 			scr_to_cpu((int) *(u32 *)(script_base + script_ofs)));
 	}
 
         printf ("%s: regdump:", sym_name(np));
         for (i=0; i<24;i++)
             printf (" %02x", (unsigned)INB_OFF(i));
         printf (".\n");
 
 	/*
 	 *  PCI BUS error, read the PCI ststus register.
 	 */
 	if (dstat & (MDPE|BF)) {
 		u_short pci_sts;
 		pci_sts = pci_read_config(np->device, PCIR_STATUS, 2);
 		if (pci_sts & 0xf900) {
 			pci_write_config(np->device, PCIR_STATUS, pci_sts, 2);
 			printf("%s: PCI STATUS = 0x%04x\n",
 				sym_name(np), pci_sts & 0xf900);
 		}
 	}
 }
 
 /*
  *  chip interrupt handler
  *
  *  In normal situations, interrupt conditions occur one at
  *  a time. But when something bad happens on the SCSI BUS,
  *  the chip may raise several interrupt flags before
  *  stopping and interrupting the CPU. The additionnal
  *  interrupt flags are stacked in some extra registers
  *  after the SIP and/or DIP flag has been raised in the
  *  ISTAT. After the CPU has read the interrupt condition
  *  flag from SIST or DSTAT, the chip unstacks the other
  *  interrupt flags and sets the corresponding bits in
  *  SIST or DSTAT. Since the chip starts stacking once the
  *  SIP or DIP flag is set, there is a small window of time
  *  where the stacking does not occur.
  *
  *  Typically, multiple interrupt conditions may happen in
  *  the following situations:
  *
  *  - SCSI parity error + Phase mismatch  (PAR|MA)
  *    When a parity error is detected in input phase
  *    and the device switches to msg-in phase inside a
  *    block MOV.
  *  - SCSI parity error + Unexpected disconnect (PAR|UDC)
  *    When a stupid device does not want to handle the
  *    recovery of an SCSI parity error.
  *  - Some combinations of STO, PAR, UDC, ...
  *    When using non compliant SCSI stuff, when user is
  *    doing non compliant hot tampering on the BUS, when
  *    something really bad happens to a device, etc ...
  *
  *  The heuristic suggested by SYMBIOS to handle
  *  multiple interrupts is to try unstacking all
  *  interrupts conditions and to handle them on some
  *  priority based on error severity.
  *  This will work when the unstacking has been
  *  successful, but we cannot be 100 % sure of that,
  *  since the CPU may have been faster to unstack than
  *  the chip is able to stack. Hmmm ... But it seems that
  *  such a situation is very unlikely to happen.
  *
  *  If this happen, for example STO caught by the CPU
  *  then UDC happenning before the CPU have restarted
  *  the SCRIPTS, the driver may wrongly complete the
  *  same command on UDC, since the SCRIPTS didn't restart
  *  and the DSA still points to the same command.
  *  We avoid this situation by setting the DSA to an
  *  invalid value when the CCB is completed and before
  *  restarting the SCRIPTS.
  *
  *  Another issue is that we need some section of our
  *  recovery procedures to be somehow uninterruptible but
  *  the SCRIPTS processor does not provides such a
  *  feature. For this reason, we handle recovery preferently
  *  from the C code and check against some SCRIPTS critical
  *  sections from the C code.
  *
  *  Hopefully, the interrupt handling of the driver is now
  *  able to resist to weird BUS error conditions, but donnot
  *  ask me for any guarantee that it will never fail. :-)
  *  Use at your own decision and risk.
  */
 static void sym_intr1 (hcb_p np)
 {
 	u_char	istat, istatc;
 	u_char	dstat;
 	u_short	sist;
 
 	SYM_LOCK_ASSERT(MA_OWNED);
 
 	/*
 	 *  interrupt on the fly ?
 	 *
 	 *  A `dummy read' is needed to ensure that the
 	 *  clear of the INTF flag reaches the device
 	 *  before the scanning of the DONE queue.
 	 */
 	istat = INB (nc_istat);
 	if (istat & INTF) {
 		OUTB (nc_istat, (istat & SIGP) | INTF | np->istat_sem);
 		istat = INB (nc_istat);		/* DUMMY READ */
 		if (DEBUG_FLAGS & DEBUG_TINY) printf ("F ");
 		(void)sym_wakeup_done (np);
 	};
 
 	if (!(istat & (SIP|DIP)))
 		return;
 
 #if 0	/* We should never get this one */
 	if (istat & CABRT)
 		OUTB (nc_istat, CABRT);
 #endif
 
 	/*
 	 *  PAR and MA interrupts may occur at the same time,
 	 *  and we need to know of both in order to handle
 	 *  this situation properly. We try to unstack SCSI
 	 *  interrupts for that reason. BTW, I dislike a LOT
 	 *  such a loop inside the interrupt routine.
 	 *  Even if DMA interrupt stacking is very unlikely to
 	 *  happen, we also try unstacking these ones, since
 	 *  this has no performance impact.
 	 */
 	sist	= 0;
 	dstat	= 0;
 	istatc	= istat;
 	do {
 		if (istatc & SIP)
 			sist  |= INW (nc_sist);
 		if (istatc & DIP)
 			dstat |= INB (nc_dstat);
 		istatc = INB (nc_istat);
 		istat |= istatc;
 	} while (istatc & (SIP|DIP));
 
 	if (DEBUG_FLAGS & DEBUG_TINY)
 		printf ("<%d|%x:%x|%x:%x>",
 			(int)INB(nc_scr0),
 			dstat,sist,
 			(unsigned)INL(nc_dsp),
 			(unsigned)INL(nc_dbc));
 	/*
 	 *  On paper, a memory barrier may be needed here.
 	 *  And since we are paranoid ... :)
 	 */
 	MEMORY_BARRIER();
 
 	/*
 	 *  First, interrupts we want to service cleanly.
 	 *
 	 *  Phase mismatch (MA) is the most frequent interrupt
 	 *  for chip earlier than the 896 and so we have to service
 	 *  it as quickly as possible.
 	 *  A SCSI parity error (PAR) may be combined with a phase
 	 *  mismatch condition (MA).
 	 *  Programmed interrupts (SIR) are used to call the C code
 	 *  from SCRIPTS.
 	 *  The single step interrupt (SSI) is not used in this
 	 *  driver.
 	 */
 	if (!(sist  & (STO|GEN|HTH|SGE|UDC|SBMC|RST)) &&
 	    !(dstat & (MDPE|BF|ABRT|IID))) {
 		if	(sist & PAR)	sym_int_par (np, sist);
 		else if (sist & MA)	sym_int_ma (np);
 		else if (dstat & SIR)	sym_int_sir (np);
 		else if (dstat & SSI)	OUTONB_STD ();
 		else			goto unknown_int;
 		return;
 	};
 
 	/*
 	 *  Now, interrupts that donnot happen in normal
 	 *  situations and that we may need to recover from.
 	 *
 	 *  On SCSI RESET (RST), we reset everything.
 	 *  On SCSI BUS MODE CHANGE (SBMC), we complete all
 	 *  active CCBs with RESET status, prepare all devices
 	 *  for negotiating again and restart the SCRIPTS.
 	 *  On STO and UDC, we complete the CCB with the corres-
 	 *  ponding status and restart the SCRIPTS.
 	 */
 	if (sist & RST) {
 		xpt_print_path(np->path);
 		printf("SCSI BUS reset detected.\n");
 		sym_init (np, 1);
 		return;
 	};
 
 	OUTB (nc_ctest3, np->rv_ctest3 | CLF);	/* clear dma fifo  */
 	OUTB (nc_stest3, TE|CSF);		/* clear scsi fifo */
 
 	if (!(sist  & (GEN|HTH|SGE)) &&
 	    !(dstat & (MDPE|BF|ABRT|IID))) {
 		if	(sist & SBMC)	sym_int_sbmc (np);
 		else if (sist & STO)	sym_int_sto (np);
 		else if (sist & UDC)	sym_int_udc (np);
 		else			goto unknown_int;
 		return;
 	};
 
 	/*
 	 *  Now, interrupts we are not able to recover cleanly.
 	 *
 	 *  Log message for hard errors.
 	 *  Reset everything.
 	 */
 
 	sym_log_hard_error(np, sist, dstat);
 
 	if ((sist & (GEN|HTH|SGE)) ||
 		(dstat & (MDPE|BF|ABRT|IID))) {
 		sym_start_reset(np);
 		return;
 	};
 
 unknown_int:
 	/*
 	 *  We just miss the cause of the interrupt. :(
 	 *  Print a message. The timeout will do the real work.
 	 */
 	printf(	"%s: unknown interrupt(s) ignored, "
 		"ISTAT=0x%x DSTAT=0x%x SIST=0x%x\n",
 		sym_name(np), istat, dstat, sist);
 }
 
 static void sym_intr(void *arg)
 {
 	hcb_p np = arg;
 
 	SYM_LOCK();
 
 	if (DEBUG_FLAGS & DEBUG_TINY) printf ("[");
 	sym_intr1((hcb_p) arg);
 	if (DEBUG_FLAGS & DEBUG_TINY) printf ("]");
 
 	SYM_UNLOCK();
 }
 
 static void sym_poll(struct cam_sim *sim)
 {
 	sym_intr1(cam_sim_softc(sim));
 }
 
 /*
  *  generic recovery from scsi interrupt
  *
  *  The doc says that when the chip gets an SCSI interrupt,
  *  it tries to stop in an orderly fashion, by completing
  *  an instruction fetch that had started or by flushing
  *  the DMA fifo for a write to memory that was executing.
  *  Such a fashion is not enough to know if the instruction
  *  that was just before the current DSP value has been
  *  executed or not.
  *
  *  There are some small SCRIPTS sections that deal with
  *  the start queue and the done queue that may break any
  *  assomption from the C code if we are interrupted
  *  inside, so we reset if this happens. Btw, since these
  *  SCRIPTS sections are executed while the SCRIPTS hasn't
  *  started SCSI operations, it is very unlikely to happen.
  *
  *  All the driver data structures are supposed to be
  *  allocated from the same 4 GB memory window, so there
  *  is a 1 to 1 relationship between DSA and driver data
  *  structures. Since we are careful :) to invalidate the
  *  DSA when we complete a command or when the SCRIPTS
  *  pushes a DSA into a queue, we can trust it when it
  *  points to a CCB.
  */
 static void sym_recover_scsi_int (hcb_p np, u_char hsts)
 {
 	u32	dsp	= INL (nc_dsp);
 	u32	dsa	= INL (nc_dsa);
 	ccb_p cp	= sym_ccb_from_dsa(np, dsa);
 
 	/*
 	 *  If we haven't been interrupted inside the SCRIPTS
 	 *  critical pathes, we can safely restart the SCRIPTS
 	 *  and trust the DSA value if it matches a CCB.
 	 */
 	if ((!(dsp > SCRIPTA_BA (np, getjob_begin) &&
 	       dsp < SCRIPTA_BA (np, getjob_end) + 1)) &&
 	    (!(dsp > SCRIPTA_BA (np, ungetjob) &&
 	       dsp < SCRIPTA_BA (np, reselect) + 1)) &&
 	    (!(dsp > SCRIPTB_BA (np, sel_for_abort) &&
 	       dsp < SCRIPTB_BA (np, sel_for_abort_1) + 1)) &&
 	    (!(dsp > SCRIPTA_BA (np, done) &&
 	       dsp < SCRIPTA_BA (np, done_end) + 1))) {
 		OUTB (nc_ctest3, np->rv_ctest3 | CLF);	/* clear dma fifo  */
 		OUTB (nc_stest3, TE|CSF);		/* clear scsi fifo */
 		/*
 		 *  If we have a CCB, let the SCRIPTS call us back for
 		 *  the handling of the error with SCRATCHA filled with
 		 *  STARTPOS. This way, we will be able to freeze the
 		 *  device queue and requeue awaiting IOs.
 		 */
 		if (cp) {
 			cp->host_status = hsts;
 			OUTL_DSP (SCRIPTA_BA (np, complete_error));
 		}
 		/*
 		 *  Otherwise just restart the SCRIPTS.
 		 */
 		else {
 			OUTL (nc_dsa, 0xffffff);
 			OUTL_DSP (SCRIPTA_BA (np, start));
 		}
 	}
 	else
 		goto reset_all;
 
 	return;
 
 reset_all:
 	sym_start_reset(np);
 }
 
 /*
  *  chip exception handler for selection timeout
  */
 static void sym_int_sto (hcb_p np)
 {
 	u32 dsp	= INL (nc_dsp);
 
 	if (DEBUG_FLAGS & DEBUG_TINY) printf ("T");
 
 	if (dsp == SCRIPTA_BA (np, wf_sel_done) + 8)
 		sym_recover_scsi_int(np, HS_SEL_TIMEOUT);
 	else
 		sym_start_reset(np);
 }
 
 /*
  *  chip exception handler for unexpected disconnect
  */
 static void sym_int_udc (hcb_p np)
 {
 	printf ("%s: unexpected disconnect\n", sym_name(np));
 	sym_recover_scsi_int(np, HS_UNEXPECTED);
 }
 
 /*
  *  chip exception handler for SCSI bus mode change
  *
  *  spi2-r12 11.2.3 says a transceiver mode change must
  *  generate a reset event and a device that detects a reset
  *  event shall initiate a hard reset. It says also that a
  *  device that detects a mode change shall set data transfer
  *  mode to eight bit asynchronous, etc...
  *  So, just reinitializing all except chip should be enough.
  */
 static void sym_int_sbmc (hcb_p np)
 {
 	u_char scsi_mode = INB (nc_stest4) & SMODE;
 
 	/*
 	 *  Notify user.
 	 */
 	xpt_print_path(np->path);
 	printf("SCSI BUS mode change from %s to %s.\n",
 		sym_scsi_bus_mode(np->scsi_mode), sym_scsi_bus_mode(scsi_mode));
 
 	/*
 	 *  Should suspend command processing for a few seconds and
 	 *  reinitialize all except the chip.
 	 */
 	sym_init (np, 2);
 }
 
 /*
  *  chip exception handler for SCSI parity error.
  *
  *  When the chip detects a SCSI parity error and is
  *  currently executing a (CH)MOV instruction, it does
  *  not interrupt immediately, but tries to finish the
  *  transfer of the current scatter entry before
  *  interrupting. The following situations may occur:
  *
  *  - The complete scatter entry has been transferred
  *    without the device having changed phase.
  *    The chip will then interrupt with the DSP pointing
  *    to the instruction that follows the MOV.
  *
  *  - A phase mismatch occurs before the MOV finished
  *    and phase errors are to be handled by the C code.
  *    The chip will then interrupt with both PAR and MA
  *    conditions set.
  *
  *  - A phase mismatch occurs before the MOV finished and
  *    phase errors are to be handled by SCRIPTS.
  *    The chip will load the DSP with the phase mismatch
  *    JUMP address and interrupt the host processor.
  */
 static void sym_int_par (hcb_p np, u_short sist)
 {
 	u_char	hsts	= INB (HS_PRT);
 	u32	dsp	= INL (nc_dsp);
 	u32	dbc	= INL (nc_dbc);
 	u32	dsa	= INL (nc_dsa);
 	u_char	sbcl	= INB (nc_sbcl);
 	u_char	cmd	= dbc >> 24;
 	int phase	= cmd & 7;
 	ccb_p	cp	= sym_ccb_from_dsa(np, dsa);
 
 	printf("%s: SCSI parity error detected: SCR1=%d DBC=%x SBCL=%x\n",
 		sym_name(np), hsts, dbc, sbcl);
 
 	/*
 	 *  Check that the chip is connected to the SCSI BUS.
 	 */
 	if (!(INB (nc_scntl1) & ISCON)) {
 		sym_recover_scsi_int(np, HS_UNEXPECTED);
 		return;
 	}
 
 	/*
 	 *  If the nexus is not clearly identified, reset the bus.
 	 *  We will try to do better later.
 	 */
 	if (!cp)
 		goto reset_all;
 
 	/*
 	 *  Check instruction was a MOV, direction was INPUT and
 	 *  ATN is asserted.
 	 */
 	if ((cmd & 0xc0) || !(phase & 1) || !(sbcl & 0x8))
 		goto reset_all;
 
 	/*
 	 *  Keep track of the parity error.
 	 */
 	OUTONB (HF_PRT, HF_EXT_ERR);
 	cp->xerr_status |= XE_PARITY_ERR;
 
 	/*
 	 *  Prepare the message to send to the device.
 	 */
 	np->msgout[0] = (phase == 7) ? M_PARITY : M_ID_ERROR;
 
 	/*
 	 *  If the old phase was DATA IN phase, we have to deal with
 	 *  the 3 situations described above.
 	 *  For other input phases (MSG IN and STATUS), the device
 	 *  must resend the whole thing that failed parity checking
 	 *  or signal error. So, jumping to dispatcher should be OK.
 	 */
 	if (phase == 1 || phase == 5) {
 		/* Phase mismatch handled by SCRIPTS */
 		if (dsp == SCRIPTB_BA (np, pm_handle))
 			OUTL_DSP (dsp);
 		/* Phase mismatch handled by the C code */
 		else if (sist & MA)
 			sym_int_ma (np);
 		/* No phase mismatch occurred */
 		else {
 			OUTL (nc_temp, dsp);
 			OUTL_DSP (SCRIPTA_BA (np, dispatch));
 		}
 	}
 	else
 		OUTL_DSP (SCRIPTA_BA (np, clrack));
 	return;
 
 reset_all:
 	sym_start_reset(np);
 }
 
 /*
  *  chip exception handler for phase errors.
  *
  *  We have to construct a new transfer descriptor,
  *  to transfer the rest of the current block.
  */
 static void sym_int_ma (hcb_p np)
 {
 	u32	dbc;
 	u32	rest;
 	u32	dsp;
 	u32	dsa;
 	u32	nxtdsp;
 	u32	*vdsp;
 	u32	oadr, olen;
 	u32	*tblp;
         u32	newcmd;
 	u_int	delta;
 	u_char	cmd;
 	u_char	hflags, hflags0;
 	struct	sym_pmc *pm;
 	ccb_p	cp;
 
 	dsp	= INL (nc_dsp);
 	dbc	= INL (nc_dbc);
 	dsa	= INL (nc_dsa);
 
 	cmd	= dbc >> 24;
 	rest	= dbc & 0xffffff;
 	delta	= 0;
 
 	/*
 	 *  locate matching cp if any.
 	 */
 	cp = sym_ccb_from_dsa(np, dsa);
 
 	/*
 	 *  Donnot take into account dma fifo and various buffers in
 	 *  INPUT phase since the chip flushes everything before
 	 *  raising the MA interrupt for interrupted INPUT phases.
 	 *  For DATA IN phase, we will check for the SWIDE later.
 	 */
 	if ((cmd & 7) != 1 && (cmd & 7) != 5) {
 		u_char ss0, ss2;
 
 		if (np->features & FE_DFBC)
 			delta = INW (nc_dfbc);
 		else {
 			u32 dfifo;
 
 			/*
 			 * Read DFIFO, CTEST[4-6] using 1 PCI bus ownership.
 			 */
 			dfifo = INL(nc_dfifo);
 
 			/*
 			 *  Calculate remaining bytes in DMA fifo.
 			 *  (CTEST5 = dfifo >> 16)
 			 */
 			if (dfifo & (DFS << 16))
 				delta = ((((dfifo >> 8) & 0x300) |
 				          (dfifo & 0xff)) - rest) & 0x3ff;
 			else
 				delta = ((dfifo & 0xff) - rest) & 0x7f;
 		}
 
 		/*
 		 *  The data in the dma fifo has not been transferred to
 		 *  the target -> add the amount to the rest
 		 *  and clear the data.
 		 *  Check the sstat2 register in case of wide transfer.
 		 */
 		rest += delta;
 		ss0  = INB (nc_sstat0);
 		if (ss0 & OLF) rest++;
 		if (!(np->features & FE_C10))
 			if (ss0 & ORF) rest++;
 		if (cp && (cp->phys.select.sel_scntl3 & EWS)) {
 			ss2 = INB (nc_sstat2);
 			if (ss2 & OLF1) rest++;
 			if (!(np->features & FE_C10))
 				if (ss2 & ORF1) rest++;
 		};
 
 		/*
 		 *  Clear fifos.
 		 */
 		OUTB (nc_ctest3, np->rv_ctest3 | CLF);	/* dma fifo  */
 		OUTB (nc_stest3, TE|CSF);		/* scsi fifo */
 	}
 
 	/*
 	 *  log the information
 	 */
 	if (DEBUG_FLAGS & (DEBUG_TINY|DEBUG_PHASE))
 		printf ("P%x%x RL=%d D=%d ", cmd&7, INB(nc_sbcl)&7,
 			(unsigned) rest, (unsigned) delta);
 
 	/*
 	 *  try to find the interrupted script command,
 	 *  and the address at which to continue.
 	 */
 	vdsp	= 0;
 	nxtdsp	= 0;
 	if	(dsp >  np->scripta_ba &&
 		 dsp <= np->scripta_ba + np->scripta_sz) {
 		vdsp = (u32 *)((char*)np->scripta0 + (dsp-np->scripta_ba-8));
 		nxtdsp = dsp;
 	}
 	else if	(dsp >  np->scriptb_ba &&
 		 dsp <= np->scriptb_ba + np->scriptb_sz) {
 		vdsp = (u32 *)((char*)np->scriptb0 + (dsp-np->scriptb_ba-8));
 		nxtdsp = dsp;
 	}
 
 	/*
 	 *  log the information
 	 */
 	if (DEBUG_FLAGS & DEBUG_PHASE) {
 		printf ("\nCP=%p DSP=%x NXT=%x VDSP=%p CMD=%x ",
 			cp, (unsigned)dsp, (unsigned)nxtdsp, vdsp, cmd);
 	};
 
 	if (!vdsp) {
 		printf ("%s: interrupted SCRIPT address not found.\n",
 			sym_name (np));
 		goto reset_all;
 	}
 
 	if (!cp) {
 		printf ("%s: SCSI phase error fixup: CCB already dequeued.\n",
 			sym_name (np));
 		goto reset_all;
 	}
 
 	/*
 	 *  get old startaddress and old length.
 	 */
 	oadr = scr_to_cpu(vdsp[1]);
 
 	if (cmd & 0x10) {	/* Table indirect */
 		tblp = (u32 *) ((char*) &cp->phys + oadr);
 		olen = scr_to_cpu(tblp[0]);
 		oadr = scr_to_cpu(tblp[1]);
 	} else {
 		tblp = (u32 *) 0;
 		olen = scr_to_cpu(vdsp[0]) & 0xffffff;
 	};
 
 	if (DEBUG_FLAGS & DEBUG_PHASE) {
 		printf ("OCMD=%x\nTBLP=%p OLEN=%x OADR=%x\n",
 			(unsigned) (scr_to_cpu(vdsp[0]) >> 24),
 			tblp,
 			(unsigned) olen,
 			(unsigned) oadr);
 	};
 
 	/*
 	 *  check cmd against assumed interrupted script command.
 	 *  If dt data phase, the MOVE instruction hasn't bit 4 of
 	 *  the phase.
 	 */
 	if (((cmd & 2) ? cmd : (cmd & ~4)) != (scr_to_cpu(vdsp[0]) >> 24)) {
 		PRINT_ADDR(cp);
 		printf ("internal error: cmd=%02x != %02x=(vdsp[0] >> 24)\n",
 			(unsigned)cmd, (unsigned)scr_to_cpu(vdsp[0]) >> 24);
 
 		goto reset_all;
 	};
 
 	/*
 	 *  if old phase not dataphase, leave here.
 	 */
 	if (cmd & 2) {
 		PRINT_ADDR(cp);
 		printf ("phase change %x-%x %d@%08x resid=%d.\n",
 			cmd&7, INB(nc_sbcl)&7, (unsigned)olen,
 			(unsigned)oadr, (unsigned)rest);
 		goto unexpected_phase;
 	};
 
 	/*
 	 *  Choose the correct PM save area.
 	 *
 	 *  Look at the PM_SAVE SCRIPT if you want to understand
 	 *  this stuff. The equivalent code is implemented in
 	 *  SCRIPTS for the 895A, 896 and 1010 that are able to
 	 *  handle PM from the SCRIPTS processor.
 	 */
 	hflags0 = INB (HF_PRT);
 	hflags = hflags0;
 
 	if (hflags & (HF_IN_PM0 | HF_IN_PM1 | HF_DP_SAVED)) {
 		if (hflags & HF_IN_PM0)
 			nxtdsp = scr_to_cpu(cp->phys.pm0.ret);
 		else if	(hflags & HF_IN_PM1)
 			nxtdsp = scr_to_cpu(cp->phys.pm1.ret);
 
 		if (hflags & HF_DP_SAVED)
 			hflags ^= HF_ACT_PM;
 	}
 
 	if (!(hflags & HF_ACT_PM)) {
 		pm = &cp->phys.pm0;
 		newcmd = SCRIPTA_BA (np, pm0_data);
 	}
 	else {
 		pm = &cp->phys.pm1;
 		newcmd = SCRIPTA_BA (np, pm1_data);
 	}
 
 	hflags &= ~(HF_IN_PM0 | HF_IN_PM1 | HF_DP_SAVED);
 	if (hflags != hflags0)
 		OUTB (HF_PRT, hflags);
 
 	/*
 	 *  fillin the phase mismatch context
 	 */
 	pm->sg.addr = cpu_to_scr(oadr + olen - rest);
 	pm->sg.size = cpu_to_scr(rest);
 	pm->ret     = cpu_to_scr(nxtdsp);
 
 	/*
 	 *  If we have a SWIDE,
 	 *  - prepare the address to write the SWIDE from SCRIPTS,
 	 *  - compute the SCRIPTS address to restart from,
 	 *  - move current data pointer context by one byte.
 	 */
 	nxtdsp = SCRIPTA_BA (np, dispatch);
 	if ((cmd & 7) == 1 && cp && (cp->phys.select.sel_scntl3 & EWS) &&
 	    (INB (nc_scntl2) & WSR)) {
 		u32 tmp;
 
 		/*
 		 *  Set up the table indirect for the MOVE
 		 *  of the residual byte and adjust the data
 		 *  pointer context.
 		 */
 		tmp = scr_to_cpu(pm->sg.addr);
 		cp->phys.wresid.addr = cpu_to_scr(tmp);
 		pm->sg.addr = cpu_to_scr(tmp + 1);
 		tmp = scr_to_cpu(pm->sg.size);
 		cp->phys.wresid.size = cpu_to_scr((tmp&0xff000000) | 1);
 		pm->sg.size = cpu_to_scr(tmp - 1);
 
 		/*
 		 *  If only the residual byte is to be moved,
 		 *  no PM context is needed.
 		 */
 		if ((tmp&0xffffff) == 1)
 			newcmd = pm->ret;
 
 		/*
 		 *  Prepare the address of SCRIPTS that will
 		 *  move the residual byte to memory.
 		 */
 		nxtdsp = SCRIPTB_BA (np, wsr_ma_helper);
 	}
 
 	if (DEBUG_FLAGS & DEBUG_PHASE) {
 		PRINT_ADDR(cp);
 		printf ("PM %x %x %x / %x %x %x.\n",
 			hflags0, hflags, newcmd,
 			(unsigned)scr_to_cpu(pm->sg.addr),
 			(unsigned)scr_to_cpu(pm->sg.size),
 			(unsigned)scr_to_cpu(pm->ret));
 	}
 
 	/*
 	 *  Restart the SCRIPTS processor.
 	 */
 	OUTL (nc_temp, newcmd);
 	OUTL_DSP (nxtdsp);
 	return;
 
 	/*
 	 *  Unexpected phase changes that occurs when the current phase
 	 *  is not a DATA IN or DATA OUT phase are due to error conditions.
 	 *  Such event may only happen when the SCRIPTS is using a
 	 *  multibyte SCSI MOVE.
 	 *
 	 *  Phase change		Some possible cause
 	 *
 	 *  COMMAND  --> MSG IN	SCSI parity error detected by target.
 	 *  COMMAND  --> STATUS	Bad command or refused by target.
 	 *  MSG OUT  --> MSG IN     Message rejected by target.
 	 *  MSG OUT  --> COMMAND    Bogus target that discards extended
 	 *  			negotiation messages.
 	 *
 	 *  The code below does not care of the new phase and so
 	 *  trusts the target. Why to annoy it ?
 	 *  If the interrupted phase is COMMAND phase, we restart at
 	 *  dispatcher.
 	 *  If a target does not get all the messages after selection,
 	 *  the code assumes blindly that the target discards extended
 	 *  messages and clears the negotiation status.
 	 *  If the target does not want all our response to negotiation,
 	 *  we force a SIR_NEGO_PROTO interrupt (it is a hack that avoids
 	 *  bloat for such a should_not_happen situation).
 	 *  In all other situation, we reset the BUS.
 	 *  Are these assumptions reasonnable ? (Wait and see ...)
 	 */
 unexpected_phase:
 	dsp -= 8;
 	nxtdsp = 0;
 
 	switch (cmd & 7) {
 	case 2:	/* COMMAND phase */
 		nxtdsp = SCRIPTA_BA (np, dispatch);
 		break;
 #if 0
 	case 3:	/* STATUS  phase */
 		nxtdsp = SCRIPTA_BA (np, dispatch);
 		break;
 #endif
 	case 6:	/* MSG OUT phase */
 		/*
 		 *  If the device may want to use untagged when we want
 		 *  tagged, we prepare an IDENTIFY without disc. granted,
 		 *  since we will not be able to handle reselect.
 		 *  Otherwise, we just don't care.
 		 */
 		if	(dsp == SCRIPTA_BA (np, send_ident)) {
 			if (cp->tag != NO_TAG && olen - rest <= 3) {
 				cp->host_status = HS_BUSY;
 				np->msgout[0] = M_IDENTIFY | cp->lun;
 				nxtdsp = SCRIPTB_BA (np, ident_break_atn);
 			}
 			else
 				nxtdsp = SCRIPTB_BA (np, ident_break);
 		}
 		else if	(dsp == SCRIPTB_BA (np, send_wdtr) ||
 			 dsp == SCRIPTB_BA (np, send_sdtr) ||
 			 dsp == SCRIPTB_BA (np, send_ppr)) {
 			nxtdsp = SCRIPTB_BA (np, nego_bad_phase);
 		}
 		break;
 #if 0
 	case 7:	/* MSG IN  phase */
 		nxtdsp = SCRIPTA_BA (np, clrack);
 		break;
 #endif
 	}
 
 	if (nxtdsp) {
 		OUTL_DSP (nxtdsp);
 		return;
 	}
 
 reset_all:
 	sym_start_reset(np);
 }
 
 /*
  *  Dequeue from the START queue all CCBs that match
  *  a given target/lun/task condition (-1 means all),
  *  and move them from the BUSY queue to the COMP queue
  *  with CAM_REQUEUE_REQ status condition.
  *  This function is used during error handling/recovery.
  *  It is called with SCRIPTS not running.
  */
 static int
 sym_dequeue_from_squeue(hcb_p np, int i, int target, int lun, int task)
 {
 	int j;
 	ccb_p cp;
 
 	/*
 	 *  Make sure the starting index is within range.
 	 */
 	assert((i >= 0) && (i < 2*MAX_QUEUE));
 
 	/*
 	 *  Walk until end of START queue and dequeue every job
 	 *  that matches the target/lun/task condition.
 	 */
 	j = i;
 	while (i != np->squeueput) {
 		cp = sym_ccb_from_dsa(np, scr_to_cpu(np->squeue[i]));
 		assert(cp);
 #ifdef SYM_CONF_IARB_SUPPORT
 		/* Forget hints for IARB, they may be no longer relevant */
 		cp->host_flags &= ~HF_HINT_IARB;
 #endif
 		if ((target == -1 || cp->target == target) &&
 		    (lun    == -1 || cp->lun    == lun)    &&
 		    (task   == -1 || cp->tag    == task)) {
 			sym_set_cam_status(cp->cam_ccb, CAM_REQUEUE_REQ);
 			sym_remque(&cp->link_ccbq);
 			sym_insque_tail(&cp->link_ccbq, &np->comp_ccbq);
 		}
 		else {
 			if (i != j)
 				np->squeue[j] = np->squeue[i];
 			if ((j += 2) >= MAX_QUEUE*2) j = 0;
 		}
 		if ((i += 2) >= MAX_QUEUE*2) i = 0;
 	}
 	if (i != j)		/* Copy back the idle task if needed */
 		np->squeue[j] = np->squeue[i];
 	np->squeueput = j;	/* Update our current start queue pointer */
 
 	return (i - j) / 2;
 }
 
 /*
  *  Complete all CCBs queued to the COMP queue.
  *
  *  These CCBs are assumed:
  *  - Not to be referenced either by devices or
  *    SCRIPTS-related queues and datas.
  *  - To have to be completed with an error condition
  *    or requeued.
  *
  *  The device queue freeze count is incremented
  *  for each CCB that does not prevent this.
  *  This function is called when all CCBs involved
  *  in error handling/recovery have been reaped.
  */
 static void
 sym_flush_comp_queue(hcb_p np, int cam_status)
 {
 	SYM_QUEHEAD *qp;
 	ccb_p cp;
 
 	while ((qp = sym_remque_head(&np->comp_ccbq)) != NULL) {
 		union ccb *ccb;
 		cp = sym_que_entry(qp, struct sym_ccb, link_ccbq);
 		sym_insque_tail(&cp->link_ccbq, &np->busy_ccbq);
 		/* Leave quiet CCBs waiting for resources */
 		if (cp->host_status == HS_WAIT)
 			continue;
 		ccb = cp->cam_ccb;
 		if (cam_status)
 			sym_set_cam_status(ccb, cam_status);
 		sym_freeze_cam_ccb(ccb);
 		sym_xpt_done(np, ccb, cp);
 		sym_free_ccb(np, cp);
 	}
 }
 
 /*
  *  chip handler for bad SCSI status condition
  *
  *  In case of bad SCSI status, we unqueue all the tasks
  *  currently queued to the controller but not yet started
  *  and then restart the SCRIPTS processor immediately.
  *
  *  QUEUE FULL and BUSY conditions are handled the same way.
  *  Basically all the not yet started tasks are requeued in
  *  device queue and the queue is frozen until a completion.
  *
  *  For CHECK CONDITION and COMMAND TERMINATED status, we use
  *  the CCB of the failed command to prepare a REQUEST SENSE
  *  SCSI command and queue it to the controller queue.
  *
  *  SCRATCHA is assumed to have been loaded with STARTPOS
  *  before the SCRIPTS called the C code.
  */
 static void sym_sir_bad_scsi_status(hcb_p np, ccb_p cp)
 {
 	tcb_p tp	= &np->target[cp->target];
 	u32		startp;
 	u_char		s_status = cp->ssss_status;
 	u_char		h_flags  = cp->host_flags;
 	int		msglen;
 	int		nego;
 	int		i;
 
 	SYM_LOCK_ASSERT(MA_OWNED);
 
 	/*
 	 *  Compute the index of the next job to start from SCRIPTS.
 	 */
 	i = (INL (nc_scratcha) - np->squeue_ba) / 4;
 
 	/*
 	 *  The last CCB queued used for IARB hint may be
 	 *  no longer relevant. Forget it.
 	 */
 #ifdef SYM_CONF_IARB_SUPPORT
 	if (np->last_cp)
 		np->last_cp = NULL;
 #endif
 
 	/*
 	 *  Now deal with the SCSI status.
 	 */
 	switch(s_status) {
 	case S_BUSY:
 	case S_QUEUE_FULL:
 		if (sym_verbose >= 2) {
 			PRINT_ADDR(cp);
 			printf (s_status == S_BUSY ? "BUSY" : "QUEUE FULL\n");
 		}
 	default:	/* S_INT, S_INT_COND_MET, S_CONFLICT */
 		sym_complete_error (np, cp);
 		break;
 	case S_TERMINATED:
 	case S_CHECK_COND:
 		/*
 		 *  If we get an SCSI error when requesting sense, give up.
 		 */
 		if (h_flags & HF_SENSE) {
 			sym_complete_error (np, cp);
 			break;
 		}
 
 		/*
 		 *  Dequeue all queued CCBs for that device not yet started,
 		 *  and restart the SCRIPTS processor immediately.
 		 */
 		(void) sym_dequeue_from_squeue(np, i, cp->target, cp->lun, -1);
 		OUTL_DSP (SCRIPTA_BA (np, start));
 
  		/*
 		 *  Save some info of the actual IO.
 		 *  Compute the data residual.
 		 */
 		cp->sv_scsi_status = cp->ssss_status;
 		cp->sv_xerr_status = cp->xerr_status;
 		cp->sv_resid = sym_compute_residual(np, cp);
 
 		/*
 		 *  Prepare all needed data structures for
 		 *  requesting sense data.
 		 */
 
 		/*
 		 *  identify message
 		 */
 		cp->scsi_smsg2[0] = M_IDENTIFY | cp->lun;
 		msglen = 1;
 
 		/*
 		 *  If we are currently using anything different from
 		 *  async. 8 bit data transfers with that target,
 		 *  start a negotiation, since the device may want
 		 *  to report us a UNIT ATTENTION condition due to
 		 *  a cause we currently ignore, and we donnot want
 		 *  to be stuck with WIDE and/or SYNC data transfer.
 		 *
 		 *  cp->nego_status is filled by sym_prepare_nego().
 		 */
 		cp->nego_status = 0;
 		nego = 0;
 		if	(tp->tinfo.current.options & PPR_OPT_MASK)
 			nego = NS_PPR;
 		else if	(tp->tinfo.current.width != BUS_8_BIT)
 			nego = NS_WIDE;
 		else if (tp->tinfo.current.offset != 0)
 			nego = NS_SYNC;
 		if (nego)
 			msglen +=
 			sym_prepare_nego (np,cp, nego, &cp->scsi_smsg2[msglen]);
 		/*
 		 *  Message table indirect structure.
 		 */
 		cp->phys.smsg.addr	= cpu_to_scr(CCB_BA (cp, scsi_smsg2));
 		cp->phys.smsg.size	= cpu_to_scr(msglen);
 
 		/*
 		 *  sense command
 		 */
 		cp->phys.cmd.addr	= cpu_to_scr(CCB_BA (cp, sensecmd));
 		cp->phys.cmd.size	= cpu_to_scr(6);
 
 		/*
 		 *  patch requested size into sense command
 		 */
 		cp->sensecmd[0]		= 0x03;
 		cp->sensecmd[1]		= cp->lun << 5;
 		if (tp->tinfo.current.scsi_version > 2 || cp->lun > 7)
 			cp->sensecmd[1]	= 0;
 		cp->sensecmd[4]		= SYM_SNS_BBUF_LEN;
 		cp->data_len		= SYM_SNS_BBUF_LEN;
 
 		/*
 		 *  sense data
 		 */
 		bzero(cp->sns_bbuf, SYM_SNS_BBUF_LEN);
 		cp->phys.sense.addr	= cpu_to_scr(vtobus(cp->sns_bbuf));
 		cp->phys.sense.size	= cpu_to_scr(SYM_SNS_BBUF_LEN);
 
 		/*
 		 *  requeue the command.
 		 */
 		startp = SCRIPTB_BA (np, sdata_in);
 
 		cp->phys.head.savep	= cpu_to_scr(startp);
 		cp->phys.head.goalp	= cpu_to_scr(startp + 16);
 		cp->phys.head.lastp	= cpu_to_scr(startp);
 		cp->startp	= cpu_to_scr(startp);
 
 		cp->actualquirks = SYM_QUIRK_AUTOSAVE;
 		cp->host_status	= cp->nego_status ? HS_NEGOTIATE : HS_BUSY;
 		cp->ssss_status = S_ILLEGAL;
 		cp->host_flags	= (HF_SENSE|HF_DATA_IN);
 		cp->xerr_status = 0;
 		cp->extra_bytes = 0;
 
 		cp->phys.head.go.start = cpu_to_scr(SCRIPTA_BA (np, select));
 
 		/*
 		 *  Requeue the command.
 		 */
 		sym_put_start_queue(np, cp);
 
 		/*
 		 *  Give back to upper layer everything we have dequeued.
 		 */
 		sym_flush_comp_queue(np, 0);
 		break;
 	}
 }
 
 /*
  *  After a device has accepted some management message
  *  as BUS DEVICE RESET, ABORT TASK, etc ..., or when
  *  a device signals a UNIT ATTENTION condition, some
  *  tasks are thrown away by the device. We are required
  *  to reflect that on our tasks list since the device
  *  will never complete these tasks.
  *
  *  This function move from the BUSY queue to the COMP
  *  queue all disconnected CCBs for a given target that
  *  match the following criteria:
  *  - lun=-1  means any logical UNIT otherwise a given one.
  *  - task=-1 means any task, otherwise a given one.
  */
 static int
 sym_clear_tasks(hcb_p np, int cam_status, int target, int lun, int task)
 {
 	SYM_QUEHEAD qtmp, *qp;
 	int i = 0;
 	ccb_p cp;
 
 	/*
 	 *  Move the entire BUSY queue to our temporary queue.
 	 */
 	sym_que_init(&qtmp);
 	sym_que_splice(&np->busy_ccbq, &qtmp);
 	sym_que_init(&np->busy_ccbq);
 
 	/*
 	 *  Put all CCBs that matches our criteria into
 	 *  the COMP queue and put back other ones into
 	 *  the BUSY queue.
 	 */
 	while ((qp = sym_remque_head(&qtmp)) != NULL) {
 		union ccb *ccb;
 		cp = sym_que_entry(qp, struct sym_ccb, link_ccbq);
 		ccb = cp->cam_ccb;
 		if (cp->host_status != HS_DISCONNECT ||
 		    cp->target != target	     ||
 		    (lun  != -1 && cp->lun != lun)   ||
 		    (task != -1 &&
 			(cp->tag != NO_TAG && cp->scsi_smsg[2] != task))) {
 			sym_insque_tail(&cp->link_ccbq, &np->busy_ccbq);
 			continue;
 		}
 		sym_insque_tail(&cp->link_ccbq, &np->comp_ccbq);
 
 		/* Preserve the software timeout condition */
 		if (sym_get_cam_status(ccb) != CAM_CMD_TIMEOUT)
 			sym_set_cam_status(ccb, cam_status);
 		++i;
 #if 0
 printf("XXXX TASK @%p CLEARED\n", cp);
 #endif
 	}
 	return i;
 }
 
 /*
  *  chip handler for TASKS recovery
  *
  *  We cannot safely abort a command, while the SCRIPTS
  *  processor is running, since we just would be in race
  *  with it.
  *
  *  As long as we have tasks to abort, we keep the SEM
  *  bit set in the ISTAT. When this bit is set, the
  *  SCRIPTS processor interrupts (SIR_SCRIPT_STOPPED)
  *  each time it enters the scheduler.
  *
  *  If we have to reset a target, clear tasks of a unit,
  *  or to perform the abort of a disconnected job, we
  *  restart the SCRIPTS for selecting the target. Once
  *  selected, the SCRIPTS interrupts (SIR_TARGET_SELECTED).
  *  If it loses arbitration, the SCRIPTS will interrupt again
  *  the next time it will enter its scheduler, and so on ...
  *
  *  On SIR_TARGET_SELECTED, we scan for the more
  *  appropriate thing to do:
  *
  *  - If nothing, we just sent a M_ABORT message to the
  *    target to get rid of the useless SCSI bus ownership.
  *    According to the specs, no tasks shall be affected.
  *  - If the target is to be reset, we send it a M_RESET
  *    message.
  *  - If a logical UNIT is to be cleared , we send the
  *    IDENTIFY(lun) + M_ABORT.
  *  - If an untagged task is to be aborted, we send the
  *    IDENTIFY(lun) + M_ABORT.
  *  - If a tagged task is to be aborted, we send the
  *    IDENTIFY(lun) + task attributes + M_ABORT_TAG.
  *
  *  Once our 'kiss of death' :) message has been accepted
  *  by the target, the SCRIPTS interrupts again
  *  (SIR_ABORT_SENT). On this interrupt, we complete
  *  all the CCBs that should have been aborted by the
  *  target according to our message.
  */
 static void sym_sir_task_recovery(hcb_p np, int num)
 {
 	SYM_QUEHEAD *qp;
 	ccb_p cp;
 	tcb_p tp;
 	int target=-1, lun=-1, task;
 	int i, k;
 
 	switch(num) {
 	/*
 	 *  The SCRIPTS processor stopped before starting
 	 *  the next command in order to allow us to perform
 	 *  some task recovery.
 	 */
 	case SIR_SCRIPT_STOPPED:
 		/*
 		 *  Do we have any target to reset or unit to clear ?
 		 */
 		for (i = 0 ; i < SYM_CONF_MAX_TARGET ; i++) {
 			tp = &np->target[i];
 			if (tp->to_reset ||
 			    (tp->lun0p && tp->lun0p->to_clear)) {
 				target = i;
 				break;
 			}
 			if (!tp->lunmp)
 				continue;
 			for (k = 1 ; k < SYM_CONF_MAX_LUN ; k++) {
 				if (tp->lunmp[k] && tp->lunmp[k]->to_clear) {
 					target	= i;
 					break;
 				}
 			}
 			if (target != -1)
 				break;
 		}
 
 		/*
 		 *  If not, walk the busy queue for any
 		 *  disconnected CCB to be aborted.
 		 */
 		if (target == -1) {
 			FOR_EACH_QUEUED_ELEMENT(&np->busy_ccbq, qp) {
 				cp = sym_que_entry(qp,struct sym_ccb,link_ccbq);
 				if (cp->host_status != HS_DISCONNECT)
 					continue;
 				if (cp->to_abort) {
 					target = cp->target;
 					break;
 				}
 			}
 		}
 
 		/*
 		 *  If some target is to be selected,
 		 *  prepare and start the selection.
 		 */
 		if (target != -1) {
 			tp = &np->target[target];
 			np->abrt_sel.sel_id	= target;
 			np->abrt_sel.sel_scntl3 = tp->head.wval;
 			np->abrt_sel.sel_sxfer  = tp->head.sval;
 			OUTL(nc_dsa, np->hcb_ba);
 			OUTL_DSP (SCRIPTB_BA (np, sel_for_abort));
 			return;
 		}
 
 		/*
 		 *  Now look for a CCB to abort that haven't started yet.
 		 *  Btw, the SCRIPTS processor is still stopped, so
 		 *  we are not in race.
 		 */
 		i = 0;
 		cp = NULL;
 		FOR_EACH_QUEUED_ELEMENT(&np->busy_ccbq, qp) {
 			cp = sym_que_entry(qp, struct sym_ccb, link_ccbq);
 			if (cp->host_status != HS_BUSY &&
 			    cp->host_status != HS_NEGOTIATE)
 				continue;
 			if (!cp->to_abort)
 				continue;
 #ifdef SYM_CONF_IARB_SUPPORT
 			/*
 			 *    If we are using IMMEDIATE ARBITRATION, we donnot
 			 *    want to cancel the last queued CCB, since the
 			 *    SCRIPTS may have anticipated the selection.
 			 */
 			if (cp == np->last_cp) {
 				cp->to_abort = 0;
 				continue;
 			}
 #endif
 			i = 1;	/* Means we have found some */
 			break;
 		}
 		if (!i) {
 			/*
 			 *  We are done, so we donnot need
 			 *  to synchronize with the SCRIPTS anylonger.
 			 *  Remove the SEM flag from the ISTAT.
 			 */
 			np->istat_sem = 0;
 			OUTB (nc_istat, SIGP);
 			break;
 		}
 		/*
 		 *  Compute index of next position in the start
 		 *  queue the SCRIPTS intends to start and dequeue
 		 *  all CCBs for that device that haven't been started.
 		 */
 		i = (INL (nc_scratcha) - np->squeue_ba) / 4;
 		i = sym_dequeue_from_squeue(np, i, cp->target, cp->lun, -1);
 
 		/*
 		 *  Make sure at least our IO to abort has been dequeued.
 		 */
 		assert(i && sym_get_cam_status(cp->cam_ccb) == CAM_REQUEUE_REQ);
 
 		/*
 		 *  Keep track in cam status of the reason of the abort.
 		 */
 		if (cp->to_abort == 2)
 			sym_set_cam_status(cp->cam_ccb, CAM_CMD_TIMEOUT);
 		else
 			sym_set_cam_status(cp->cam_ccb, CAM_REQ_ABORTED);
 
 		/*
 		 *  Complete with error everything that we have dequeued.
 	 	 */
 		sym_flush_comp_queue(np, 0);
 		break;
 	/*
 	 *  The SCRIPTS processor has selected a target
 	 *  we may have some manual recovery to perform for.
 	 */
 	case SIR_TARGET_SELECTED:
 		target = (INB (nc_sdid) & 0xf);
 		tp = &np->target[target];
 
 		np->abrt_tbl.addr = cpu_to_scr(vtobus(np->abrt_msg));
 
 		/*
 		 *  If the target is to be reset, prepare a
 		 *  M_RESET message and clear the to_reset flag
 		 *  since we donnot expect this operation to fail.
 		 */
 		if (tp->to_reset) {
 			np->abrt_msg[0] = M_RESET;
 			np->abrt_tbl.size = 1;
 			tp->to_reset = 0;
 			break;
 		}
 
 		/*
 		 *  Otherwise, look for some logical unit to be cleared.
 		 */
 		if (tp->lun0p && tp->lun0p->to_clear)
 			lun = 0;
 		else if (tp->lunmp) {
 			for (k = 1 ; k < SYM_CONF_MAX_LUN ; k++) {
 				if (tp->lunmp[k] && tp->lunmp[k]->to_clear) {
 					lun = k;
 					break;
 				}
 			}
 		}
 
 		/*
 		 *  If a logical unit is to be cleared, prepare
 		 *  an IDENTIFY(lun) + ABORT MESSAGE.
 		 */
 		if (lun != -1) {
 			lcb_p lp = sym_lp(tp, lun);
 			lp->to_clear = 0; /* We donnot expect to fail here */
 			np->abrt_msg[0] = M_IDENTIFY | lun;
 			np->abrt_msg[1] = M_ABORT;
 			np->abrt_tbl.size = 2;
 			break;
 		}
 
 		/*
 		 *  Otherwise, look for some disconnected job to
 		 *  abort for this target.
 		 */
 		i = 0;
 		cp = NULL;
 		FOR_EACH_QUEUED_ELEMENT(&np->busy_ccbq, qp) {
 			cp = sym_que_entry(qp, struct sym_ccb, link_ccbq);
 			if (cp->host_status != HS_DISCONNECT)
 				continue;
 			if (cp->target != target)
 				continue;
 			if (!cp->to_abort)
 				continue;
 			i = 1;	/* Means we have some */
 			break;
 		}
 
 		/*
 		 *  If we have none, probably since the device has
 		 *  completed the command before we won abitration,
 		 *  send a M_ABORT message without IDENTIFY.
 		 *  According to the specs, the device must just
 		 *  disconnect the BUS and not abort any task.
 		 */
 		if (!i) {
 			np->abrt_msg[0] = M_ABORT;
 			np->abrt_tbl.size = 1;
 			break;
 		}
 
 		/*
 		 *  We have some task to abort.
 		 *  Set the IDENTIFY(lun)
 		 */
 		np->abrt_msg[0] = M_IDENTIFY | cp->lun;
 
 		/*
 		 *  If we want to abort an untagged command, we
 		 *  will send an IDENTIFY + M_ABORT.
 		 *  Otherwise (tagged command), we will send
 		 *  an IDENTIFY + task attributes + ABORT TAG.
 		 */
 		if (cp->tag == NO_TAG) {
 			np->abrt_msg[1] = M_ABORT;
 			np->abrt_tbl.size = 2;
 		}
 		else {
 			np->abrt_msg[1] = cp->scsi_smsg[1];
 			np->abrt_msg[2] = cp->scsi_smsg[2];
 			np->abrt_msg[3] = M_ABORT_TAG;
 			np->abrt_tbl.size = 4;
 		}
 		/*
 		 *  Keep track of software timeout condition, since the
 		 *  peripheral driver may not count retries on abort
 		 *  conditions not due to timeout.
 		 */
 		if (cp->to_abort == 2)
 			sym_set_cam_status(cp->cam_ccb, CAM_CMD_TIMEOUT);
 		cp->to_abort = 0; /* We donnot expect to fail here */
 		break;
 
 	/*
 	 *  The target has accepted our message and switched
 	 *  to BUS FREE phase as we expected.
 	 */
 	case SIR_ABORT_SENT:
 		target = (INB (nc_sdid) & 0xf);
 		tp = &np->target[target];
 
 		/*
 		**  If we didn't abort anything, leave here.
 		*/
 		if (np->abrt_msg[0] == M_ABORT)
 			break;
 
 		/*
 		 *  If we sent a M_RESET, then a hardware reset has
 		 *  been performed by the target.
 		 *  - Reset everything to async 8 bit
 		 *  - Tell ourself to negotiate next time :-)
 		 *  - Prepare to clear all disconnected CCBs for
 		 *    this target from our task list (lun=task=-1)
 		 */
 		lun = -1;
 		task = -1;
 		if (np->abrt_msg[0] == M_RESET) {
 			tp->head.sval = 0;
 			tp->head.wval = np->rv_scntl3;
 			tp->head.uval = 0;
 			tp->tinfo.current.period = 0;
 			tp->tinfo.current.offset = 0;
 			tp->tinfo.current.width  = BUS_8_BIT;
 			tp->tinfo.current.options = 0;
 		}
 
 		/*
 		 *  Otherwise, check for the LUN and TASK(s)
 		 *  concerned by the cancelation.
 		 *  If it is not ABORT_TAG then it is CLEAR_QUEUE
 		 *  or an ABORT message :-)
 		 */
 		else {
 			lun = np->abrt_msg[0] & 0x3f;
 			if (np->abrt_msg[1] == M_ABORT_TAG)
 				task = np->abrt_msg[2];
 		}
 
 		/*
 		 *  Complete all the CCBs the device should have
 		 *  aborted due to our 'kiss of death' message.
 		 */
 		i = (INL (nc_scratcha) - np->squeue_ba) / 4;
 		(void) sym_dequeue_from_squeue(np, i, target, lun, -1);
 		(void) sym_clear_tasks(np, CAM_REQ_ABORTED, target, lun, task);
 		sym_flush_comp_queue(np, 0);
 
 		/*
 		 *  If we sent a BDR, make uper layer aware of that.
 		 */
 		if (np->abrt_msg[0] == M_RESET)
 			xpt_async(AC_SENT_BDR, np->path, NULL);
 		break;
 	}
 
 	/*
 	 *  Print to the log the message we intend to send.
 	 */
 	if (num == SIR_TARGET_SELECTED) {
 		PRINT_TARGET(np, target);
 		sym_printl_hex("control msgout:", np->abrt_msg,
 			      np->abrt_tbl.size);
 		np->abrt_tbl.size = cpu_to_scr(np->abrt_tbl.size);
 	}
 
 	/*
 	 *  Let the SCRIPTS processor continue.
 	 */
 	OUTONB_STD ();
 }
 
 /*
  *  Gerard's alchemy:) that deals with with the data
  *  pointer for both MDP and the residual calculation.
  *
  *  I didn't want to bloat the code by more than 200
  *  lignes for the handling of both MDP and the residual.
  *  This has been achieved by using a data pointer
  *  representation consisting in an index in the data
  *  array (dp_sg) and a negative offset (dp_ofs) that
  *  have the following meaning:
  *
  *  - dp_sg = SYM_CONF_MAX_SG
  *    we are at the end of the data script.
  *  - dp_sg < SYM_CONF_MAX_SG
  *    dp_sg points to the next entry of the scatter array
  *    we want to transfer.
  *  - dp_ofs < 0
  *    dp_ofs represents the residual of bytes of the
  *    previous entry scatter entry we will send first.
  *  - dp_ofs = 0
  *    no residual to send first.
  *
  *  The function sym_evaluate_dp() accepts an arbitray
  *  offset (basically from the MDP message) and returns
  *  the corresponding values of dp_sg and dp_ofs.
  */
 static int sym_evaluate_dp(hcb_p np, ccb_p cp, u32 scr, int *ofs)
 {
 	u32	dp_scr;
 	int	dp_ofs, dp_sg, dp_sgmin;
 	int	tmp;
 	struct sym_pmc *pm;
 
 	/*
 	 *  Compute the resulted data pointer in term of a script
 	 *  address within some DATA script and a signed byte offset.
 	 */
 	dp_scr = scr;
 	dp_ofs = *ofs;
 	if	(dp_scr == SCRIPTA_BA (np, pm0_data))
 		pm = &cp->phys.pm0;
 	else if (dp_scr == SCRIPTA_BA (np, pm1_data))
 		pm = &cp->phys.pm1;
 	else
 		pm = NULL;
 
 	if (pm) {
 		dp_scr  = scr_to_cpu(pm->ret);
 		dp_ofs -= scr_to_cpu(pm->sg.size);
 	}
 
 	/*
 	 *  If we are auto-sensing, then we are done.
 	 */
 	if (cp->host_flags & HF_SENSE) {
 		*ofs = dp_ofs;
 		return 0;
 	}
 
 	/*
 	 *  Deduce the index of the sg entry.
 	 *  Keep track of the index of the first valid entry.
 	 *  If result is dp_sg = SYM_CONF_MAX_SG, then we are at the
 	 *  end of the data.
 	 */
 	tmp = scr_to_cpu(cp->phys.head.goalp);
 	dp_sg = SYM_CONF_MAX_SG;
 	if (dp_scr != tmp)
 		dp_sg -= (tmp - 8 - (int)dp_scr) / (2*4);
 	dp_sgmin = SYM_CONF_MAX_SG - cp->segments;
 
 	/*
 	 *  Move to the sg entry the data pointer belongs to.
 	 *
 	 *  If we are inside the data area, we expect result to be:
 	 *
 	 *  Either,
 	 *      dp_ofs = 0 and dp_sg is the index of the sg entry
 	 *      the data pointer belongs to (or the end of the data)
 	 *  Or,
 	 *      dp_ofs < 0 and dp_sg is the index of the sg entry
 	 *      the data pointer belongs to + 1.
 	 */
 	if (dp_ofs < 0) {
 		int n;
 		while (dp_sg > dp_sgmin) {
 			--dp_sg;
 			tmp = scr_to_cpu(cp->phys.data[dp_sg].size);
 			n = dp_ofs + (tmp & 0xffffff);
 			if (n > 0) {
 				++dp_sg;
 				break;
 			}
 			dp_ofs = n;
 		}
 	}
 	else if (dp_ofs > 0) {
 		while (dp_sg < SYM_CONF_MAX_SG) {
 			tmp = scr_to_cpu(cp->phys.data[dp_sg].size);
 			dp_ofs -= (tmp & 0xffffff);
 			++dp_sg;
 			if (dp_ofs <= 0)
 				break;
 		}
 	}
 
 	/*
 	 *  Make sure the data pointer is inside the data area.
 	 *  If not, return some error.
 	 */
 	if	(dp_sg < dp_sgmin || (dp_sg == dp_sgmin && dp_ofs < 0))
 		goto out_err;
 	else if	(dp_sg > SYM_CONF_MAX_SG ||
 		 (dp_sg == SYM_CONF_MAX_SG && dp_ofs > 0))
 		goto out_err;
 
 	/*
 	 *  Save the extreme pointer if needed.
 	 */
 	if (dp_sg > cp->ext_sg ||
             (dp_sg == cp->ext_sg && dp_ofs > cp->ext_ofs)) {
 		cp->ext_sg  = dp_sg;
 		cp->ext_ofs = dp_ofs;
 	}
 
 	/*
 	 *  Return data.
 	 */
 	*ofs = dp_ofs;
 	return dp_sg;
 
 out_err:
 	return -1;
 }
 
 /*
  *  chip handler for MODIFY DATA POINTER MESSAGE
  *
  *  We also call this function on IGNORE WIDE RESIDUE
  *  messages that do not match a SWIDE full condition.
  *  Btw, we assume in that situation that such a message
  *  is equivalent to a MODIFY DATA POINTER (offset=-1).
  */
 static void sym_modify_dp(hcb_p np, ccb_p cp, int ofs)
 {
 	int dp_ofs	= ofs;
 	u32	dp_scr	= INL (nc_temp);
 	u32	dp_ret;
 	u32	tmp;
 	u_char	hflags;
 	int	dp_sg;
 	struct	sym_pmc *pm;
 
 	/*
 	 *  Not supported for auto-sense.
 	 */
 	if (cp->host_flags & HF_SENSE)
 		goto out_reject;
 
 	/*
 	 *  Apply our alchemy:) (see comments in sym_evaluate_dp()),
 	 *  to the resulted data pointer.
 	 */
 	dp_sg = sym_evaluate_dp(np, cp, dp_scr, &dp_ofs);
 	if (dp_sg < 0)
 		goto out_reject;
 
 	/*
 	 *  And our alchemy:) allows to easily calculate the data
 	 *  script address we want to return for the next data phase.
 	 */
 	dp_ret = cpu_to_scr(cp->phys.head.goalp);
 	dp_ret = dp_ret - 8 - (SYM_CONF_MAX_SG - dp_sg) * (2*4);
 
 	/*
 	 *  If offset / scatter entry is zero we donnot need
 	 *  a context for the new current data pointer.
 	 */
 	if (dp_ofs == 0) {
 		dp_scr = dp_ret;
 		goto out_ok;
 	}
 
 	/*
 	 *  Get a context for the new current data pointer.
 	 */
 	hflags = INB (HF_PRT);
 
 	if (hflags & HF_DP_SAVED)
 		hflags ^= HF_ACT_PM;
 
 	if (!(hflags & HF_ACT_PM)) {
 		pm  = &cp->phys.pm0;
 		dp_scr = SCRIPTA_BA (np, pm0_data);
 	}
 	else {
 		pm = &cp->phys.pm1;
 		dp_scr = SCRIPTA_BA (np, pm1_data);
 	}
 
 	hflags &= ~(HF_DP_SAVED);
 
 	OUTB (HF_PRT, hflags);
 
 	/*
 	 *  Set up the new current data pointer.
 	 *  ofs < 0 there, and for the next data phase, we
 	 *  want to transfer part of the data of the sg entry
 	 *  corresponding to index dp_sg-1 prior to returning
 	 *  to the main data script.
 	 */
 	pm->ret = cpu_to_scr(dp_ret);
 	tmp  = scr_to_cpu(cp->phys.data[dp_sg-1].addr);
 	tmp += scr_to_cpu(cp->phys.data[dp_sg-1].size) + dp_ofs;
 	pm->sg.addr = cpu_to_scr(tmp);
 	pm->sg.size = cpu_to_scr(-dp_ofs);
 
 out_ok:
 	OUTL (nc_temp, dp_scr);
 	OUTL_DSP (SCRIPTA_BA (np, clrack));
 	return;
 
 out_reject:
 	OUTL_DSP (SCRIPTB_BA (np, msg_bad));
 }
 
 /*
  *  chip calculation of the data residual.
  *
  *  As I used to say, the requirement of data residual
  *  in SCSI is broken, useless and cannot be achieved
  *  without huge complexity.
  *  But most OSes and even the official CAM require it.
  *  When stupidity happens to be so widely spread inside
  *  a community, it gets hard to convince.
  *
  *  Anyway, I don't care, since I am not going to use
  *  any software that considers this data residual as
  *  a relevant information. :)
  */
 static int sym_compute_residual(hcb_p np, ccb_p cp)
 {
 	int dp_sg, dp_sgmin, resid = 0;
 	int dp_ofs = 0;
 
 	/*
 	 *  Check for some data lost or just thrown away.
 	 *  We are not required to be quite accurate in this
 	 *  situation. Btw, if we are odd for output and the
 	 *  device claims some more data, it may well happen
 	 *  than our residual be zero. :-)
 	 */
 	if (cp->xerr_status & (XE_EXTRA_DATA|XE_SODL_UNRUN|XE_SWIDE_OVRUN)) {
 		if (cp->xerr_status & XE_EXTRA_DATA)
 			resid -= cp->extra_bytes;
 		if (cp->xerr_status & XE_SODL_UNRUN)
 			++resid;
 		if (cp->xerr_status & XE_SWIDE_OVRUN)
 			--resid;
 	}
 
 	/*
 	 *  If all data has been transferred,
 	 *  there is no residual.
 	 */
 	if (cp->phys.head.lastp == cp->phys.head.goalp)
 		return resid;
 
 	/*
 	 *  If no data transfer occurs, or if the data
 	 *  pointer is weird, return full residual.
 	 */
 	if (cp->startp == cp->phys.head.lastp ||
 	    sym_evaluate_dp(np, cp, scr_to_cpu(cp->phys.head.lastp),
 			    &dp_ofs) < 0) {
 		return cp->data_len;
 	}
 
 	/*
 	 *  If we were auto-sensing, then we are done.
 	 */
 	if (cp->host_flags & HF_SENSE) {
 		return -dp_ofs;
 	}
 
 	/*
 	 *  We are now full comfortable in the computation
 	 *  of the data residual (2's complement).
 	 */
 	dp_sgmin = SYM_CONF_MAX_SG - cp->segments;
 	resid = -cp->ext_ofs;
 	for (dp_sg = cp->ext_sg; dp_sg < SYM_CONF_MAX_SG; ++dp_sg) {
 		u_int tmp = scr_to_cpu(cp->phys.data[dp_sg].size);
 		resid += (tmp & 0xffffff);
 	}
 
 	/*
 	 *  Hopefully, the result is not too wrong.
 	 */
 	return resid;
 }
 
 /*
  *  Print out the content of a SCSI message.
  */
 static int sym_show_msg (u_char * msg)
 {
 	u_char i;
 	printf ("%x",*msg);
 	if (*msg==M_EXTENDED) {
 		for (i=1;i<8;i++) {
 			if (i-1>msg[1]) break;
 			printf ("-%x",msg[i]);
 		};
 		return (i+1);
 	} else if ((*msg & 0xf0) == 0x20) {
 		printf ("-%x",msg[1]);
 		return (2);
 	};
 	return (1);
 }
 
 static void sym_print_msg (ccb_p cp, char *label, u_char *msg)
 {
 	PRINT_ADDR(cp);
 	if (label)
 		printf ("%s: ", label);
 
 	(void) sym_show_msg (msg);
 	printf (".\n");
 }
 
 /*
  *  Negotiation for WIDE and SYNCHRONOUS DATA TRANSFER.
  *
  *  When we try to negotiate, we append the negotiation message
  *  to the identify and (maybe) simple tag message.
  *  The host status field is set to HS_NEGOTIATE to mark this
  *  situation.
  *
  *  If the target doesn't answer this message immediately
  *  (as required by the standard), the SIR_NEGO_FAILED interrupt
  *  will be raised eventually.
  *  The handler removes the HS_NEGOTIATE status, and sets the
  *  negotiated value to the default (async / nowide).
  *
  *  If we receive a matching answer immediately, we check it
  *  for validity, and set the values.
  *
  *  If we receive a Reject message immediately, we assume the
  *  negotiation has failed, and fall back to standard values.
  *
  *  If we receive a negotiation message while not in HS_NEGOTIATE
  *  state, it's a target initiated negotiation. We prepare a
  *  (hopefully) valid answer, set our parameters, and send back
  *  this answer to the target.
  *
  *  If the target doesn't fetch the answer (no message out phase),
  *  we assume the negotiation has failed, and fall back to default
  *  settings (SIR_NEGO_PROTO interrupt).
  *
  *  When we set the values, we adjust them in all ccbs belonging
  *  to this target, in the controller's register, and in the "phys"
  *  field of the controller's struct sym_hcb.
  */
 
 /*
  *  chip handler for SYNCHRONOUS DATA TRANSFER REQUEST (SDTR) message.
  */
 static void sym_sync_nego(hcb_p np, tcb_p tp, ccb_p cp)
 {
 	u_char	chg, ofs, per, fak, div;
 	int	req = 1;
 
 	/*
 	 *  Synchronous request message received.
 	 */
 	if (DEBUG_FLAGS & DEBUG_NEGO) {
 		sym_print_msg(cp, "sync msgin", np->msgin);
 	};
 
 	/*
 	 * request or answer ?
 	 */
 	if (INB (HS_PRT) == HS_NEGOTIATE) {
 		OUTB (HS_PRT, HS_BUSY);
 		if (cp->nego_status && cp->nego_status != NS_SYNC)
 			goto reject_it;
 		req = 0;
 	}
 
 	/*
 	 *  get requested values.
 	 */
 	chg = 0;
 	per = np->msgin[3];
 	ofs = np->msgin[4];
 
 	/*
 	 *  check values against our limits.
 	 */
 	if (ofs) {
 		if (ofs > np->maxoffs)
 			{chg = 1; ofs = np->maxoffs;}
 		if (req) {
 			if (ofs > tp->tinfo.user.offset)
 				{chg = 1; ofs = tp->tinfo.user.offset;}
 		}
 	}
 
 	if (ofs) {
 		if (per < np->minsync)
 			{chg = 1; per = np->minsync;}
 		if (req) {
 			if (per < tp->tinfo.user.period)
 				{chg = 1; per = tp->tinfo.user.period;}
 		}
 	}
 
 	div = fak = 0;
 	if (ofs && sym_getsync(np, 0, per, &div, &fak) < 0)
 		goto reject_it;
 
 	if (DEBUG_FLAGS & DEBUG_NEGO) {
 		PRINT_ADDR(cp);
 		printf ("sdtr: ofs=%d per=%d div=%d fak=%d chg=%d.\n",
 			ofs, per, div, fak, chg);
 	}
 
 	/*
 	 *  This was an answer message
 	 */
 	if (req == 0) {
 		if (chg) 	/* Answer wasn't acceptable. */
 			goto reject_it;
 		sym_setsync (np, cp, ofs, per, div, fak);
 		OUTL_DSP (SCRIPTA_BA (np, clrack));
 		return;
 	}
 
 	/*
 	 *  It was a request. Set value and
 	 *  prepare an answer message
 	 */
 	sym_setsync (np, cp, ofs, per, div, fak);
 
 	np->msgout[0] = M_EXTENDED;
 	np->msgout[1] = 3;
 	np->msgout[2] = M_X_SYNC_REQ;
 	np->msgout[3] = per;
 	np->msgout[4] = ofs;
 
 	cp->nego_status = NS_SYNC;
 
 	if (DEBUG_FLAGS & DEBUG_NEGO) {
 		sym_print_msg(cp, "sync msgout", np->msgout);
 	}
 
 	np->msgin [0] = M_NOOP;
 
 	OUTL_DSP (SCRIPTB_BA (np, sdtr_resp));
 	return;
 reject_it:
 	sym_setsync (np, cp, 0, 0, 0, 0);
 	OUTL_DSP (SCRIPTB_BA (np, msg_bad));
 }
 
 /*
  *  chip handler for PARALLEL PROTOCOL REQUEST (PPR) message.
  */
 static void sym_ppr_nego(hcb_p np, tcb_p tp, ccb_p cp)
 {
 	u_char	chg, ofs, per, fak, dt, div, wide;
 	int	req = 1;
 
 	/*
 	 * Synchronous request message received.
 	 */
 	if (DEBUG_FLAGS & DEBUG_NEGO) {
 		sym_print_msg(cp, "ppr msgin", np->msgin);
 	};
 
 	/*
 	 *  get requested values.
 	 */
 	chg  = 0;
 	per  = np->msgin[3];
 	ofs  = np->msgin[5];
 	wide = np->msgin[6];
 	dt   = np->msgin[7] & PPR_OPT_DT;
 
 	/*
 	 * request or answer ?
 	 */
 	if (INB (HS_PRT) == HS_NEGOTIATE) {
 		OUTB (HS_PRT, HS_BUSY);
 		if (cp->nego_status && cp->nego_status != NS_PPR)
 			goto reject_it;
 		req = 0;
 	}
 
 	/*
 	 *  check values against our limits.
 	 */
 	if (wide > np->maxwide)
 		{chg = 1; wide = np->maxwide;}
 	if (!wide || !(np->features & FE_ULTRA3))
 		dt &= ~PPR_OPT_DT;
 	if (req) {
 		if (wide > tp->tinfo.user.width)
 			{chg = 1; wide = tp->tinfo.user.width;}
 	}
 
 	if (!(np->features & FE_U3EN))	/* Broken U3EN bit not supported */
 		dt &= ~PPR_OPT_DT;
 
 	if (dt != (np->msgin[7] & PPR_OPT_MASK)) chg = 1;
 
 	if (ofs) {
 		if (dt) {
 			if (ofs > np->maxoffs_dt)
 				{chg = 1; ofs = np->maxoffs_dt;}
 		}
 		else if (ofs > np->maxoffs)
 			{chg = 1; ofs = np->maxoffs;}
 		if (req) {
 			if (ofs > tp->tinfo.user.offset)
 				{chg = 1; ofs = tp->tinfo.user.offset;}
 		}
 	}
 
 	if (ofs) {
 		if (dt) {
 			if (per < np->minsync_dt)
 				{chg = 1; per = np->minsync_dt;}
 		}
 		else if (per < np->minsync)
 			{chg = 1; per = np->minsync;}
 		if (req) {
 			if (per < tp->tinfo.user.period)
 				{chg = 1; per = tp->tinfo.user.period;}
 		}
 	}
 
 	div = fak = 0;
 	if (ofs && sym_getsync(np, dt, per, &div, &fak) < 0)
 		goto reject_it;
 
 	if (DEBUG_FLAGS & DEBUG_NEGO) {
 		PRINT_ADDR(cp);
 		printf ("ppr: "
 			"dt=%x ofs=%d per=%d wide=%d div=%d fak=%d chg=%d.\n",
 			dt, ofs, per, wide, div, fak, chg);
 	}
 
 	/*
 	 *  It was an answer.
 	 */
 	if (req == 0) {
 		if (chg) 	/* Answer wasn't acceptable */
 			goto reject_it;
 		sym_setpprot (np, cp, dt, ofs, per, wide, div, fak);
 		OUTL_DSP (SCRIPTA_BA (np, clrack));
 		return;
 	}
 
 	/*
 	 *  It was a request. Set value and
 	 *  prepare an answer message
 	 */
 	sym_setpprot (np, cp, dt, ofs, per, wide, div, fak);
 
 	np->msgout[0] = M_EXTENDED;
 	np->msgout[1] = 6;
 	np->msgout[2] = M_X_PPR_REQ;
 	np->msgout[3] = per;
 	np->msgout[4] = 0;
 	np->msgout[5] = ofs;
 	np->msgout[6] = wide;
 	np->msgout[7] = dt;
 
 	cp->nego_status = NS_PPR;
 
 	if (DEBUG_FLAGS & DEBUG_NEGO) {
 		sym_print_msg(cp, "ppr msgout", np->msgout);
 	}
 
 	np->msgin [0] = M_NOOP;
 
 	OUTL_DSP (SCRIPTB_BA (np, ppr_resp));
 	return;
 reject_it:
 	sym_setpprot (np, cp, 0, 0, 0, 0, 0, 0);
 	OUTL_DSP (SCRIPTB_BA (np, msg_bad));
 	/*
 	 *  If it was a device response that should result in
 	 *  ST, we may want to try a legacy negotiation later.
 	 */
 	if (!req && !dt) {
 		tp->tinfo.goal.options = 0;
 		tp->tinfo.goal.width   = wide;
 		tp->tinfo.goal.period  = per;
 		tp->tinfo.goal.offset  = ofs;
 	}
 }
 
 /*
  *  chip handler for WIDE DATA TRANSFER REQUEST (WDTR) message.
  */
 static void sym_wide_nego(hcb_p np, tcb_p tp, ccb_p cp)
 {
 	u_char	chg, wide;
 	int	req = 1;
 
 	/*
 	 *  Wide request message received.
 	 */
 	if (DEBUG_FLAGS & DEBUG_NEGO) {
 		sym_print_msg(cp, "wide msgin", np->msgin);
 	};
 
 	/*
 	 * Is it a request from the device?
 	 */
 	if (INB (HS_PRT) == HS_NEGOTIATE) {
 		OUTB (HS_PRT, HS_BUSY);
 		if (cp->nego_status && cp->nego_status != NS_WIDE)
 			goto reject_it;
 		req = 0;
 	}
 
 	/*
 	 *  get requested values.
 	 */
 	chg  = 0;
 	wide = np->msgin[3];
 
 	/*
 	 *  check values against driver limits.
 	 */
 	if (wide > np->maxwide)
 		{chg = 1; wide = np->maxwide;}
 	if (req) {
 		if (wide > tp->tinfo.user.width)
 			{chg = 1; wide = tp->tinfo.user.width;}
 	}
 
 	if (DEBUG_FLAGS & DEBUG_NEGO) {
 		PRINT_ADDR(cp);
 		printf ("wdtr: wide=%d chg=%d.\n", wide, chg);
 	}
 
 	/*
 	 * This was an answer message
 	 */
 	if (req == 0) {
 		if (chg)	/*  Answer wasn't acceptable. */
 			goto reject_it;
 		sym_setwide (np, cp, wide);
 
 		/*
 		 * Negotiate for SYNC immediately after WIDE response.
 		 * This allows to negotiate for both WIDE and SYNC on
 		 * a single SCSI command (Suggested by Justin Gibbs).
 		 */
 		if (tp->tinfo.goal.offset) {
 			np->msgout[0] = M_EXTENDED;
 			np->msgout[1] = 3;
 			np->msgout[2] = M_X_SYNC_REQ;
 			np->msgout[3] = tp->tinfo.goal.period;
 			np->msgout[4] = tp->tinfo.goal.offset;
 
 			if (DEBUG_FLAGS & DEBUG_NEGO) {
 				sym_print_msg(cp, "sync msgout", np->msgout);
 			}
 
 			cp->nego_status = NS_SYNC;
 			OUTB (HS_PRT, HS_NEGOTIATE);
 			OUTL_DSP (SCRIPTB_BA (np, sdtr_resp));
 			return;
 		}
 
 		OUTL_DSP (SCRIPTA_BA (np, clrack));
 		return;
 	};
 
 	/*
 	 *  It was a request, set value and
 	 *  prepare an answer message
 	 */
 	sym_setwide (np, cp, wide);
 
 	np->msgout[0] = M_EXTENDED;
 	np->msgout[1] = 2;
 	np->msgout[2] = M_X_WIDE_REQ;
 	np->msgout[3] = wide;
 
 	np->msgin [0] = M_NOOP;
 
 	cp->nego_status = NS_WIDE;
 
 	if (DEBUG_FLAGS & DEBUG_NEGO) {
 		sym_print_msg(cp, "wide msgout", np->msgout);
 	}
 
 	OUTL_DSP (SCRIPTB_BA (np, wdtr_resp));
 	return;
 reject_it:
 	OUTL_DSP (SCRIPTB_BA (np, msg_bad));
 }
 
 /*
  *  Reset SYNC or WIDE to default settings.
  *
  *  Called when a negotiation does not succeed either
  *  on rejection or on protocol error.
  *
  *  If it was a PPR that made problems, we may want to
  *  try a legacy negotiation later.
  */
 static void sym_nego_default(hcb_p np, tcb_p tp, ccb_p cp)
 {
 	/*
 	 *  any error in negotiation:
 	 *  fall back to default mode.
 	 */
 	switch (cp->nego_status) {
 	case NS_PPR:
 #if 0
 		sym_setpprot (np, cp, 0, 0, 0, 0, 0, 0);
 #else
 		tp->tinfo.goal.options = 0;
 		if (tp->tinfo.goal.period < np->minsync)
 			tp->tinfo.goal.period = np->minsync;
 		if (tp->tinfo.goal.offset > np->maxoffs)
 			tp->tinfo.goal.offset = np->maxoffs;
 #endif
 		break;
 	case NS_SYNC:
 		sym_setsync (np, cp, 0, 0, 0, 0);
 		break;
 	case NS_WIDE:
 		sym_setwide (np, cp, 0);
 		break;
 	};
 	np->msgin [0] = M_NOOP;
 	np->msgout[0] = M_NOOP;
 	cp->nego_status = 0;
 }
 
 /*
  *  chip handler for MESSAGE REJECT received in response to
  *  a WIDE or SYNCHRONOUS negotiation.
  */
 static void sym_nego_rejected(hcb_p np, tcb_p tp, ccb_p cp)
 {
 	sym_nego_default(np, tp, cp);
 	OUTB (HS_PRT, HS_BUSY);
 }
 
 /*
  *  chip exception handler for programmed interrupts.
  */
 static void sym_int_sir (hcb_p np)
 {
 	u_char	num	= INB (nc_dsps);
 	u32	dsa	= INL (nc_dsa);
 	ccb_p	cp	= sym_ccb_from_dsa(np, dsa);
 	u_char	target	= INB (nc_sdid) & 0x0f;
 	tcb_p	tp	= &np->target[target];
 	int	tmp;
 
 	SYM_LOCK_ASSERT(MA_OWNED);
 
 	if (DEBUG_FLAGS & DEBUG_TINY) printf ("I#%d", num);
 
 	switch (num) {
 	/*
 	 *  Command has been completed with error condition
 	 *  or has been auto-sensed.
 	 */
 	case SIR_COMPLETE_ERROR:
 		sym_complete_error(np, cp);
 		return;
 	/*
 	 *  The C code is currently trying to recover from something.
 	 *  Typically, user want to abort some command.
 	 */
 	case SIR_SCRIPT_STOPPED:
 	case SIR_TARGET_SELECTED:
 	case SIR_ABORT_SENT:
 		sym_sir_task_recovery(np, num);
 		return;
 	/*
 	 *  The device didn't go to MSG OUT phase after having
 	 *  been selected with ATN. We donnot want to handle
 	 *  that.
 	 */
 	case SIR_SEL_ATN_NO_MSG_OUT:
 		printf ("%s:%d: No MSG OUT phase after selection with ATN.\n",
 			sym_name (np), target);
 		goto out_stuck;
 	/*
 	 *  The device didn't switch to MSG IN phase after
 	 *  having reseleted the initiator.
 	 */
 	case SIR_RESEL_NO_MSG_IN:
 		printf ("%s:%d: No MSG IN phase after reselection.\n",
 			sym_name (np), target);
 		goto out_stuck;
 	/*
 	 *  After reselection, the device sent a message that wasn't
 	 *  an IDENTIFY.
 	 */
 	case SIR_RESEL_NO_IDENTIFY:
 		printf ("%s:%d: No IDENTIFY after reselection.\n",
 			sym_name (np), target);
 		goto out_stuck;
 	/*
 	 *  The device reselected a LUN we donnot know about.
 	 */
 	case SIR_RESEL_BAD_LUN:
 		np->msgout[0] = M_RESET;
 		goto out;
 	/*
 	 *  The device reselected for an untagged nexus and we
 	 *  haven't any.
 	 */
 	case SIR_RESEL_BAD_I_T_L:
 		np->msgout[0] = M_ABORT;
 		goto out;
 	/*
 	 *  The device reselected for a tagged nexus that we donnot
 	 *  have.
 	 */
 	case SIR_RESEL_BAD_I_T_L_Q:
 		np->msgout[0] = M_ABORT_TAG;
 		goto out;
 	/*
 	 *  The SCRIPTS let us know that the device has grabbed
 	 *  our message and will abort the job.
 	 */
 	case SIR_RESEL_ABORTED:
 		np->lastmsg = np->msgout[0];
 		np->msgout[0] = M_NOOP;
 		printf ("%s:%d: message %x sent on bad reselection.\n",
 			sym_name (np), target, np->lastmsg);
 		goto out;
 	/*
 	 *  The SCRIPTS let us know that a message has been
 	 *  successfully sent to the device.
 	 */
 	case SIR_MSG_OUT_DONE:
 		np->lastmsg = np->msgout[0];
 		np->msgout[0] = M_NOOP;
 		/* Should we really care of that */
 		if (np->lastmsg == M_PARITY || np->lastmsg == M_ID_ERROR) {
 			if (cp) {
 				cp->xerr_status &= ~XE_PARITY_ERR;
 				if (!cp->xerr_status)
 					OUTOFFB (HF_PRT, HF_EXT_ERR);
 			}
 		}
 		goto out;
 	/*
 	 *  The device didn't send a GOOD SCSI status.
 	 *  We may have some work to do prior to allow
 	 *  the SCRIPTS processor to continue.
 	 */
 	case SIR_BAD_SCSI_STATUS:
 		if (!cp)
 			goto out;
 		sym_sir_bad_scsi_status(np, cp);
 		return;
 	/*
 	 *  We are asked by the SCRIPTS to prepare a
 	 *  REJECT message.
 	 */
 	case SIR_REJECT_TO_SEND:
 		sym_print_msg(cp, "M_REJECT to send for ", np->msgin);
 		np->msgout[0] = M_REJECT;
 		goto out;
 	/*
 	 *  We have been ODD at the end of a DATA IN
 	 *  transfer and the device didn't send a
 	 *  IGNORE WIDE RESIDUE message.
 	 *  It is a data overrun condition.
 	 */
 	case SIR_SWIDE_OVERRUN:
 		if (cp) {
 			OUTONB (HF_PRT, HF_EXT_ERR);
 			cp->xerr_status |= XE_SWIDE_OVRUN;
 		}
 		goto out;
 	/*
 	 *  We have been ODD at the end of a DATA OUT
 	 *  transfer.
 	 *  It is a data underrun condition.
 	 */
 	case SIR_SODL_UNDERRUN:
 		if (cp) {
 			OUTONB (HF_PRT, HF_EXT_ERR);
 			cp->xerr_status |= XE_SODL_UNRUN;
 		}
 		goto out;
 	/*
 	 *  The device wants us to tranfer more data than
 	 *  expected or in the wrong direction.
 	 *  The number of extra bytes is in scratcha.
 	 *  It is a data overrun condition.
 	 */
 	case SIR_DATA_OVERRUN:
 		if (cp) {
 			OUTONB (HF_PRT, HF_EXT_ERR);
 			cp->xerr_status |= XE_EXTRA_DATA;
 			cp->extra_bytes += INL (nc_scratcha);
 		}
 		goto out;
 	/*
 	 *  The device switched to an illegal phase (4/5).
 	 */
 	case SIR_BAD_PHASE:
 		if (cp) {
 			OUTONB (HF_PRT, HF_EXT_ERR);
 			cp->xerr_status |= XE_BAD_PHASE;
 		}
 		goto out;
 	/*
 	 *  We received a message.
 	 */
 	case SIR_MSG_RECEIVED:
 		if (!cp)
 			goto out_stuck;
 		switch (np->msgin [0]) {
 		/*
 		 *  We received an extended message.
 		 *  We handle MODIFY DATA POINTER, SDTR, WDTR
 		 *  and reject all other extended messages.
 		 */
 		case M_EXTENDED:
 			switch (np->msgin [2]) {
 			case M_X_MODIFY_DP:
 				if (DEBUG_FLAGS & DEBUG_POINTER)
 					sym_print_msg(cp,"modify DP",np->msgin);
 				tmp = (np->msgin[3]<<24) + (np->msgin[4]<<16) +
 				      (np->msgin[5]<<8)  + (np->msgin[6]);
 				sym_modify_dp(np, cp, tmp);
 				return;
 			case M_X_SYNC_REQ:
 				sym_sync_nego(np, tp, cp);
 				return;
 			case M_X_PPR_REQ:
 				sym_ppr_nego(np, tp, cp);
 				return;
 			case M_X_WIDE_REQ:
 				sym_wide_nego(np, tp, cp);
 				return;
 			default:
 				goto out_reject;
 			}
 			break;
 		/*
 		 *  We received a 1/2 byte message not handled from SCRIPTS.
 		 *  We are only expecting MESSAGE REJECT and IGNORE WIDE
 		 *  RESIDUE messages that haven't been anticipated by
 		 *  SCRIPTS on SWIDE full condition. Unanticipated IGNORE
 		 *  WIDE RESIDUE messages are aliased as MODIFY DP (-1).
 		 */
 		case M_IGN_RESIDUE:
 			if (DEBUG_FLAGS & DEBUG_POINTER)
 				sym_print_msg(cp,"ign wide residue", np->msgin);
 			sym_modify_dp(np, cp, -1);
 			return;
 		case M_REJECT:
 			if (INB (HS_PRT) == HS_NEGOTIATE)
 				sym_nego_rejected(np, tp, cp);
 			else {
 				PRINT_ADDR(cp);
 				printf ("M_REJECT received (%x:%x).\n",
 					scr_to_cpu(np->lastmsg), np->msgout[0]);
 			}
 			goto out_clrack;
 			break;
 		default:
 			goto out_reject;
 		}
 		break;
 	/*
 	 *  We received an unknown message.
 	 *  Ignore all MSG IN phases and reject it.
 	 */
 	case SIR_MSG_WEIRD:
 		sym_print_msg(cp, "WEIRD message received", np->msgin);
 		OUTL_DSP (SCRIPTB_BA (np, msg_weird));
 		return;
 	/*
 	 *  Negotiation failed.
 	 *  Target does not send us the reply.
 	 *  Remove the HS_NEGOTIATE status.
 	 */
 	case SIR_NEGO_FAILED:
 		OUTB (HS_PRT, HS_BUSY);
 	/*
 	 *  Negotiation failed.
 	 *  Target does not want answer message.
 	 */
 	case SIR_NEGO_PROTO:
 		sym_nego_default(np, tp, cp);
 		goto out;
 	};
 
 out:
 	OUTONB_STD ();
 	return;
 out_reject:
 	OUTL_DSP (SCRIPTB_BA (np, msg_bad));
 	return;
 out_clrack:
 	OUTL_DSP (SCRIPTA_BA (np, clrack));
 	return;
 out_stuck:
 	return;
 }
 
 /*
  *  Acquire a control block
  */
 static	ccb_p sym_get_ccb (hcb_p np, u_char tn, u_char ln, u_char tag_order)
 {
 	tcb_p tp = &np->target[tn];
 	lcb_p lp = sym_lp(tp, ln);
 	u_short tag = NO_TAG;
 	SYM_QUEHEAD *qp;
 	ccb_p cp = (ccb_p) NULL;
 
 	/*
 	 *  Look for a free CCB
 	 */
 	if (sym_que_empty(&np->free_ccbq))
 		goto out;
 	qp = sym_remque_head(&np->free_ccbq);
 	if (!qp)
 		goto out;
 	cp = sym_que_entry(qp, struct sym_ccb, link_ccbq);
 
 	/*
 	 *  If the LCB is not yet available and the LUN
 	 *  has been probed ok, try to allocate the LCB.
 	 */
 	if (!lp && sym_is_bit(tp->lun_map, ln)) {
 		lp = sym_alloc_lcb(np, tn, ln);
 		if (!lp)
 			goto out_free;
 	}
 
 	/*
 	 *  If the LCB is not available here, then the
 	 *  logical unit is not yet discovered. For those
 	 *  ones only accept 1 SCSI IO per logical unit,
 	 *  since we cannot allow disconnections.
 	 */
 	if (!lp) {
 		if (!sym_is_bit(tp->busy0_map, ln))
 			sym_set_bit(tp->busy0_map, ln);
 		else
 			goto out_free;
 	} else {
 		/*
 		 *  If we have been asked for a tagged command.
 		 */
 		if (tag_order) {
 			/*
 			 *  Debugging purpose.
 			 */
 			assert(lp->busy_itl == 0);
 			/*
 			 *  Allocate resources for tags if not yet.
 			 */
 			if (!lp->cb_tags) {
 				sym_alloc_lcb_tags(np, tn, ln);
 				if (!lp->cb_tags)
 					goto out_free;
 			}
 			/*
 			 *  Get a tag for this SCSI IO and set up
 			 *  the CCB bus address for reselection,
 			 *  and count it for this LUN.
 			 *  Toggle reselect path to tagged.
 			 */
 			if (lp->busy_itlq < SYM_CONF_MAX_TASK) {
 				tag = lp->cb_tags[lp->ia_tag];
 				if (++lp->ia_tag == SYM_CONF_MAX_TASK)
 					lp->ia_tag = 0;
 				lp->itlq_tbl[tag] = cpu_to_scr(cp->ccb_ba);
 				++lp->busy_itlq;
 				lp->head.resel_sa =
 					cpu_to_scr(SCRIPTA_BA (np, resel_tag));
 			}
 			else
 				goto out_free;
 		}
 		/*
 		 *  This command will not be tagged.
 		 *  If we already have either a tagged or untagged
 		 *  one, refuse to overlap this untagged one.
 		 */
 		else {
 			/*
 			 *  Debugging purpose.
 			 */
 			assert(lp->busy_itl == 0 && lp->busy_itlq == 0);
 			/*
 			 *  Count this nexus for this LUN.
 			 *  Set up the CCB bus address for reselection.
 			 *  Toggle reselect path to untagged.
 			 */
 			if (++lp->busy_itl == 1) {
 				lp->head.itl_task_sa = cpu_to_scr(cp->ccb_ba);
 				lp->head.resel_sa =
 				      cpu_to_scr(SCRIPTA_BA (np, resel_no_tag));
 			}
 			else
 				goto out_free;
 		}
 	}
 	/*
 	 *  Put the CCB into the busy queue.
 	 */
 	sym_insque_tail(&cp->link_ccbq, &np->busy_ccbq);
 
 	/*
 	 *  Remember all informations needed to free this CCB.
 	 */
 	cp->to_abort = 0;
 	cp->tag	   = tag;
 	cp->target = tn;
 	cp->lun    = ln;
 
 	if (DEBUG_FLAGS & DEBUG_TAGS) {
 		PRINT_LUN(np, tn, ln);
 		printf ("ccb @%p using tag %d.\n", cp, tag);
 	}
 
 out:
 	return cp;
 out_free:
 	sym_insque_head(&cp->link_ccbq, &np->free_ccbq);
 	return NULL;
 }
 
 /*
  *  Release one control block
  */
 static void sym_free_ccb(hcb_p np, ccb_p cp)
 {
 	tcb_p tp = &np->target[cp->target];
 	lcb_p lp = sym_lp(tp, cp->lun);
 
 	if (DEBUG_FLAGS & DEBUG_TAGS) {
 		PRINT_LUN(np, cp->target, cp->lun);
 		printf ("ccb @%p freeing tag %d.\n", cp, cp->tag);
 	}
 
 	/*
 	 *  If LCB available,
 	 */
 	if (lp) {
 		/*
 		 *  If tagged, release the tag, set the relect path
 		 */
 		if (cp->tag != NO_TAG) {
 			/*
 			 *  Free the tag value.
 			 */
 			lp->cb_tags[lp->if_tag] = cp->tag;
 			if (++lp->if_tag == SYM_CONF_MAX_TASK)
 				lp->if_tag = 0;
 			/*
 			 *  Make the reselect path invalid,
 			 *  and uncount this CCB.
 			 */
 			lp->itlq_tbl[cp->tag] = cpu_to_scr(np->bad_itlq_ba);
 			--lp->busy_itlq;
 		} else {	/* Untagged */
 			/*
 			 *  Make the reselect path invalid,
 			 *  and uncount this CCB.
 			 */
 			lp->head.itl_task_sa = cpu_to_scr(np->bad_itl_ba);
 			--lp->busy_itl;
 		}
 		/*
 		 *  If no JOB active, make the LUN reselect path invalid.
 		 */
 		if (lp->busy_itlq == 0 && lp->busy_itl == 0)
 			lp->head.resel_sa =
 				cpu_to_scr(SCRIPTB_BA (np, resel_bad_lun));
 	}
 	/*
 	 *  Otherwise, we only accept 1 IO per LUN.
 	 *  Clear the bit that keeps track of this IO.
 	 */
 	else
 		sym_clr_bit(tp->busy0_map, cp->lun);
 
 	/*
 	 *  We donnot queue more than 1 ccb per target
 	 *  with negotiation at any time. If this ccb was
 	 *  used for negotiation, clear this info in the tcb.
 	 */
 	if (cp == tp->nego_cp)
 		tp->nego_cp = NULL;
 
 #ifdef SYM_CONF_IARB_SUPPORT
 	/*
 	 *  If we just complete the last queued CCB,
 	 *  clear this info that is no longer relevant.
 	 */
 	if (cp == np->last_cp)
 		np->last_cp = NULL;
 #endif
 
 	/*
 	 *  Unmap user data from DMA map if needed.
 	 */
 	if (cp->dmamapped) {
 		bus_dmamap_unload(np->data_dmat, cp->dmamap);
 		cp->dmamapped = 0;
 	}
 
 	/*
 	 *  Make this CCB available.
 	 */
 	cp->cam_ccb = NULL;
 	cp->host_status = HS_IDLE;
 	sym_remque(&cp->link_ccbq);
 	sym_insque_head(&cp->link_ccbq, &np->free_ccbq);
 }
 
 /*
  *  Allocate a CCB from memory and initialize its fixed part.
  */
 static ccb_p sym_alloc_ccb(hcb_p np)
 {
 	ccb_p cp = NULL;
 	int hcode;
 
 	SYM_LOCK_ASSERT(MA_NOTOWNED);
 
 	/*
 	 *  Prevent from allocating more CCBs than we can
 	 *  queue to the controller.
 	 */
 	if (np->actccbs >= SYM_CONF_MAX_START)
 		return NULL;
 
 	/*
 	 *  Allocate memory for this CCB.
 	 */
 	cp = sym_calloc_dma(sizeof(struct sym_ccb), "CCB");
 	if (!cp)
 		return NULL;
 
 	/*
 	 *  Allocate a bounce buffer for sense data.
 	 */
 	cp->sns_bbuf = sym_calloc_dma(SYM_SNS_BBUF_LEN, "SNS_BBUF");
 	if (!cp->sns_bbuf)
 		goto out_free;
 
 	/*
 	 *  Allocate a map for the DMA of user data.
 	 */
 	if (bus_dmamap_create(np->data_dmat, 0, &cp->dmamap))
 		goto out_free;
 	/*
 	 *  Count it.
 	 */
 	np->actccbs++;
 
 	/*
 	 * Initialize the callout.
 	 */
 	callout_init(&cp->ch, 1);
 
 	/*
 	 *  Compute the bus address of this ccb.
 	 */
 	cp->ccb_ba = vtobus(cp);
 
 	/*
 	 *  Insert this ccb into the hashed list.
 	 */
 	hcode = CCB_HASH_CODE(cp->ccb_ba);
 	cp->link_ccbh = np->ccbh[hcode];
 	np->ccbh[hcode] = cp;
 
 	/*
 	 *  Initialize the start and restart actions.
 	 */
 	cp->phys.head.go.start   = cpu_to_scr(SCRIPTA_BA (np, idle));
 	cp->phys.head.go.restart = cpu_to_scr(SCRIPTB_BA (np, bad_i_t_l));
 
  	/*
 	 *  Initilialyze some other fields.
 	 */
 	cp->phys.smsg_ext.addr = cpu_to_scr(HCB_BA(np, msgin[2]));
 
 	/*
 	 *  Chain into free ccb queue.
 	 */
 	sym_insque_head(&cp->link_ccbq, &np->free_ccbq);
 
 	return cp;
 out_free:
 	if (cp->sns_bbuf)
 		sym_mfree_dma(cp->sns_bbuf, SYM_SNS_BBUF_LEN, "SNS_BBUF");
 	sym_mfree_dma(cp, sizeof(*cp), "CCB");
 	return NULL;
 }
 
 /*
  *  Look up a CCB from a DSA value.
  */
 static ccb_p sym_ccb_from_dsa(hcb_p np, u32 dsa)
 {
 	int hcode;
 	ccb_p cp;
 
 	hcode = CCB_HASH_CODE(dsa);
 	cp = np->ccbh[hcode];
 	while (cp) {
 		if (cp->ccb_ba == dsa)
 			break;
 		cp = cp->link_ccbh;
 	}
 
 	return cp;
 }
 
 /*
  *  Lun control block allocation and initialization.
  */
 static lcb_p sym_alloc_lcb (hcb_p np, u_char tn, u_char ln)
 {
 	tcb_p tp = &np->target[tn];
 	lcb_p lp = sym_lp(tp, ln);
 
 	/*
 	 *  Already done, just return.
 	 */
 	if (lp)
 		return lp;
 	/*
 	 *  Check against some race.
 	 */
 	assert(!sym_is_bit(tp->busy0_map, ln));
 
 	/*
 	 *  Allocate the LCB bus address array.
 	 *  Compute the bus address of this table.
 	 */
 	if (ln && !tp->luntbl) {
 		int i;
 
 		tp->luntbl = sym_calloc_dma(256, "LUNTBL");
 		if (!tp->luntbl)
 			goto fail;
 		for (i = 0 ; i < 64 ; i++)
 			tp->luntbl[i] = cpu_to_scr(vtobus(&np->badlun_sa));
 		tp->head.luntbl_sa = cpu_to_scr(vtobus(tp->luntbl));
 	}
 
 	/*
 	 *  Allocate the table of pointers for LUN(s) > 0, if needed.
 	 */
 	if (ln && !tp->lunmp) {
 		tp->lunmp = sym_calloc(SYM_CONF_MAX_LUN * sizeof(lcb_p),
 				   "LUNMP");
 		if (!tp->lunmp)
 			goto fail;
 	}
 
 	/*
 	 *  Allocate the lcb.
 	 *  Make it available to the chip.
 	 */
 	lp = sym_calloc_dma(sizeof(struct sym_lcb), "LCB");
 	if (!lp)
 		goto fail;
 	if (ln) {
 		tp->lunmp[ln] = lp;
 		tp->luntbl[ln] = cpu_to_scr(vtobus(lp));
 	}
 	else {
 		tp->lun0p = lp;
 		tp->head.lun0_sa = cpu_to_scr(vtobus(lp));
 	}
 
 	/*
 	 *  Let the itl task point to error handling.
 	 */
 	lp->head.itl_task_sa = cpu_to_scr(np->bad_itl_ba);
 
 	/*
 	 *  Set the reselect pattern to our default. :)
 	 */
 	lp->head.resel_sa = cpu_to_scr(SCRIPTB_BA (np, resel_bad_lun));
 
 	/*
 	 *  Set user capabilities.
 	 */
 	lp->user_flags = tp->usrflags & (SYM_DISC_ENABLED | SYM_TAGS_ENABLED);
 
 fail:
 	return lp;
 }
 
 /*
  *  Allocate LCB resources for tagged command queuing.
  */
 static void sym_alloc_lcb_tags (hcb_p np, u_char tn, u_char ln)
 {
 	tcb_p tp = &np->target[tn];
 	lcb_p lp = sym_lp(tp, ln);
 	int i;
 
 	/*
 	 *  If LCB not available, try to allocate it.
 	 */
 	if (!lp && !(lp = sym_alloc_lcb(np, tn, ln)))
 		return;
 
 	/*
 	 *  Allocate the task table and and the tag allocation
 	 *  circular buffer. We want both or none.
 	 */
 	lp->itlq_tbl = sym_calloc_dma(SYM_CONF_MAX_TASK*4, "ITLQ_TBL");
 	if (!lp->itlq_tbl)
 		return;
 	lp->cb_tags = sym_calloc(SYM_CONF_MAX_TASK, "CB_TAGS");
 	if (!lp->cb_tags) {
 		sym_mfree_dma(lp->itlq_tbl, SYM_CONF_MAX_TASK*4, "ITLQ_TBL");
 		lp->itlq_tbl = 0;
 		return;
 	}
 
 	/*
 	 *  Initialize the task table with invalid entries.
 	 */
 	for (i = 0 ; i < SYM_CONF_MAX_TASK ; i++)
 		lp->itlq_tbl[i] = cpu_to_scr(np->notask_ba);
 
 	/*
 	 *  Fill up the tag buffer with tag numbers.
 	 */
 	for (i = 0 ; i < SYM_CONF_MAX_TASK ; i++)
 		lp->cb_tags[i] = i;
 
 	/*
 	 *  Make the task table available to SCRIPTS,
 	 *  And accept tagged commands now.
 	 */
 	lp->head.itlq_tbl_sa = cpu_to_scr(vtobus(lp->itlq_tbl));
 }
 
 /*
  *  Test the pci bus snoop logic :-(
  *
  *  Has to be called with interrupts disabled.
  */
 #ifndef SYM_CONF_IOMAPPED
 static int sym_regtest (hcb_p np)
 {
 	register volatile u32 data;
 	/*
 	 *  chip registers may NOT be cached.
 	 *  write 0xffffffff to a read only register area,
 	 *  and try to read it back.
 	 */
 	data = 0xffffffff;
 	OUTL_OFF(offsetof(struct sym_reg, nc_dstat), data);
 	data = INL_OFF(offsetof(struct sym_reg, nc_dstat));
 #if 1
 	if (data == 0xffffffff) {
 #else
 	if ((data & 0xe2f0fffd) != 0x02000080) {
 #endif
 		printf ("CACHE TEST FAILED: reg dstat-sstat2 readback %x.\n",
 			(unsigned) data);
 		return (0x10);
 	};
 	return (0);
 }
 #endif
 
 static int sym_snooptest (hcb_p np)
 {
 	u32	sym_rd, sym_wr, sym_bk, host_rd, host_wr, pc, dstat;
 	int	i, err=0;
 #ifndef SYM_CONF_IOMAPPED
 	err |= sym_regtest (np);
 	if (err) return (err);
 #endif
 restart_test:
 	/*
 	 *  Enable Master Parity Checking as we intend
 	 *  to enable it for normal operations.
 	 */
 	OUTB (nc_ctest4, (np->rv_ctest4 & MPEE));
 	/*
 	 *  init
 	 */
 	pc  = SCRIPTB0_BA (np, snooptest);
 	host_wr = 1;
 	sym_wr  = 2;
 	/*
 	 *  Set memory and register.
 	 */
 	np->cache = cpu_to_scr(host_wr);
 	OUTL (nc_temp, sym_wr);
 	/*
 	 *  Start script (exchange values)
 	 */
 	OUTL (nc_dsa, np->hcb_ba);
 	OUTL_DSP (pc);
 	/*
 	 *  Wait 'til done (with timeout)
 	 */
 	for (i=0; i<SYM_SNOOP_TIMEOUT; i++)
 		if (INB(nc_istat) & (INTF|SIP|DIP))
 			break;
 	if (i>=SYM_SNOOP_TIMEOUT) {
 		printf ("CACHE TEST FAILED: timeout.\n");
 		return (0x20);
 	};
 	/*
 	 *  Check for fatal DMA errors.
 	 */
 	dstat = INB (nc_dstat);
 #if 1	/* Band aiding for broken hardwares that fail PCI parity */
 	if ((dstat & MDPE) && (np->rv_ctest4 & MPEE)) {
 		printf ("%s: PCI DATA PARITY ERROR DETECTED - "
 			"DISABLING MASTER DATA PARITY CHECKING.\n",
 			sym_name(np));
 		np->rv_ctest4 &= ~MPEE;
 		goto restart_test;
 	}
 #endif
 	if (dstat & (MDPE|BF|IID)) {
 		printf ("CACHE TEST FAILED: DMA error (dstat=0x%02x).", dstat);
 		return (0x80);
 	}
 	/*
 	 *  Save termination position.
 	 */
 	pc = INL (nc_dsp);
 	/*
 	 *  Read memory and register.
 	 */
 	host_rd = scr_to_cpu(np->cache);
 	sym_rd  = INL (nc_scratcha);
 	sym_bk  = INL (nc_temp);
 
 	/*
 	 *  Check termination position.
 	 */
 	if (pc != SCRIPTB0_BA (np, snoopend)+8) {
 		printf ("CACHE TEST FAILED: script execution failed.\n");
 		printf ("start=%08lx, pc=%08lx, end=%08lx\n",
 			(u_long) SCRIPTB0_BA (np, snooptest), (u_long) pc,
 			(u_long) SCRIPTB0_BA (np, snoopend) +8);
 		return (0x40);
 	};
 	/*
 	 *  Show results.
 	 */
 	if (host_wr != sym_rd) {
 		printf ("CACHE TEST FAILED: host wrote %d, chip read %d.\n",
 			(int) host_wr, (int) sym_rd);
 		err |= 1;
 	};
 	if (host_rd != sym_wr) {
 		printf ("CACHE TEST FAILED: chip wrote %d, host read %d.\n",
 			(int) sym_wr, (int) host_rd);
 		err |= 2;
 	};
 	if (sym_bk != sym_wr) {
 		printf ("CACHE TEST FAILED: chip wrote %d, read back %d.\n",
 			(int) sym_wr, (int) sym_bk);
 		err |= 4;
 	};
 
 	return (err);
 }
 
 /*
  *  Determine the chip's clock frequency.
  *
  *  This is essential for the negotiation of the synchronous
  *  transfer rate.
  *
  *  Note: we have to return the correct value.
  *  THERE IS NO SAFE DEFAULT VALUE.
  *
  *  Most NCR/SYMBIOS boards are delivered with a 40 Mhz clock.
  *  53C860 and 53C875 rev. 1 support fast20 transfers but
  *  do not have a clock doubler and so are provided with a
  *  80 MHz clock. All other fast20 boards incorporate a doubler
  *  and so should be delivered with a 40 MHz clock.
  *  The recent fast40 chips (895/896/895A/1010) use a 40 Mhz base
  *  clock and provide a clock quadrupler (160 Mhz).
  */
 
 /*
  *  Select SCSI clock frequency
  */
 static void sym_selectclock(hcb_p np, u_char scntl3)
 {
 	/*
 	 *  If multiplier not present or not selected, leave here.
 	 */
 	if (np->multiplier <= 1) {
 		OUTB(nc_scntl3,	scntl3);
 		return;
 	}
 
 	if (sym_verbose >= 2)
 		printf ("%s: enabling clock multiplier\n", sym_name(np));
 
 	OUTB(nc_stest1, DBLEN);	   /* Enable clock multiplier		  */
 	/*
 	 *  Wait for the LCKFRQ bit to be set if supported by the chip.
 	 *  Otherwise wait 20 micro-seconds.
 	 */
 	if (np->features & FE_LCKFRQ) {
 		int i = 20;
 		while (!(INB(nc_stest4) & LCKFRQ) && --i > 0)
 			UDELAY (20);
 		if (!i)
 			printf("%s: the chip cannot lock the frequency\n",
 				sym_name(np));
 	} else
 		UDELAY (20);
 	OUTB(nc_stest3, HSC);		/* Halt the scsi clock		*/
 	OUTB(nc_scntl3,	scntl3);
 	OUTB(nc_stest1, (DBLEN|DBLSEL));/* Select clock multiplier	*/
 	OUTB(nc_stest3, 0x00);		/* Restart scsi clock 		*/
 }
 
 /*
  *  calculate SCSI clock frequency (in KHz)
  */
 static unsigned getfreq (hcb_p np, int gen)
 {
 	unsigned int ms = 0;
 	unsigned int f;
 
 	/*
 	 * Measure GEN timer delay in order
 	 * to calculate SCSI clock frequency
 	 *
 	 * This code will never execute too
 	 * many loop iterations (if DELAY is
 	 * reasonably correct). It could get
 	 * too low a delay (too high a freq.)
 	 * if the CPU is slow executing the
 	 * loop for some reason (an NMI, for
 	 * example). For this reason we will
 	 * if multiple measurements are to be
 	 * performed trust the higher delay
 	 * (lower frequency returned).
 	 */
 	OUTW (nc_sien , 0);	/* mask all scsi interrupts */
 	(void) INW (nc_sist);	/* clear pending scsi interrupt */
 	OUTB (nc_dien , 0);	/* mask all dma interrupts */
 	(void) INW (nc_sist);	/* another one, just to be sure :) */
 	OUTB (nc_scntl3, 4);	/* set pre-scaler to divide by 3 */
 	OUTB (nc_stime1, 0);	/* disable general purpose timer */
 	OUTB (nc_stime1, gen);	/* set to nominal delay of 1<<gen * 125us */
 	while (!(INW(nc_sist) & GEN) && ms++ < 100000)
 		UDELAY (1000);	/* count ms */
 	OUTB (nc_stime1, 0);	/* disable general purpose timer */
  	/*
  	 * set prescaler to divide by whatever 0 means
  	 * 0 ought to choose divide by 2, but appears
  	 * to set divide by 3.5 mode in my 53c810 ...
  	 */
  	OUTB (nc_scntl3, 0);
 
   	/*
  	 * adjust for prescaler, and convert into KHz
   	 */
 	f = ms ? ((1 << gen) * 4340) / ms : 0;
 
 	if (sym_verbose >= 2)
 		printf ("%s: Delay (GEN=%d): %u msec, %u KHz\n",
 			sym_name(np), gen, ms, f);
 
 	return f;
 }
 
 static unsigned sym_getfreq (hcb_p np)
 {
 	u_int f1, f2;
 	int gen = 11;
 
 	(void) getfreq (np, gen);	/* throw away first result */
 	f1 = getfreq (np, gen);
 	f2 = getfreq (np, gen);
 	if (f1 > f2) f1 = f2;		/* trust lower result	*/
 	return f1;
 }
 
 /*
  *  Get/probe chip SCSI clock frequency
  */
 static void sym_getclock (hcb_p np, int mult)
 {
 	unsigned char scntl3 = np->sv_scntl3;
 	unsigned char stest1 = np->sv_stest1;
 	unsigned f1;
 
 	/*
 	 *  For the C10 core, assume 40 MHz.
 	 */
 	if (np->features & FE_C10) {
 		np->multiplier = mult;
 		np->clock_khz = 40000 * mult;
 		return;
 	}
 
 	np->multiplier = 1;
 	f1 = 40000;
 	/*
 	 *  True with 875/895/896/895A with clock multiplier selected
 	 */
 	if (mult > 1 && (stest1 & (DBLEN+DBLSEL)) == DBLEN+DBLSEL) {
 		if (sym_verbose >= 2)
 			printf ("%s: clock multiplier found\n", sym_name(np));
 		np->multiplier = mult;
 	}
 
 	/*
 	 *  If multiplier not found or scntl3 not 7,5,3,
 	 *  reset chip and get frequency from general purpose timer.
 	 *  Otherwise trust scntl3 BIOS setting.
 	 */
 	if (np->multiplier != mult || (scntl3 & 7) < 3 || !(scntl3 & 1)) {
 		OUTB (nc_stest1, 0);		/* make sure doubler is OFF */
 		f1 = sym_getfreq (np);
 
 		if (sym_verbose)
 			printf ("%s: chip clock is %uKHz\n", sym_name(np), f1);
 
 		if	(f1 <	45000)		f1 =  40000;
 		else if (f1 <	55000)		f1 =  50000;
 		else				f1 =  80000;
 
 		if (f1 < 80000 && mult > 1) {
 			if (sym_verbose >= 2)
 				printf ("%s: clock multiplier assumed\n",
 					sym_name(np));
 			np->multiplier	= mult;
 		}
 	} else {
 		if	((scntl3 & 7) == 3)	f1 =  40000;
 		else if	((scntl3 & 7) == 5)	f1 =  80000;
 		else 				f1 = 160000;
 
 		f1 /= np->multiplier;
 	}
 
 	/*
 	 *  Compute controller synchronous parameters.
 	 */
 	f1		*= np->multiplier;
 	np->clock_khz	= f1;
 }
 
 /*
  *  Get/probe PCI clock frequency
  */
 static int sym_getpciclock (hcb_p np)
 {
 	int f = 0;
 
 	/*
 	 *  For the C1010-33, this doesn't work.
 	 *  For the C1010-66, this will be tested when I'll have
 	 *  such a beast to play with.
 	 */
 	if (!(np->features & FE_C10)) {
 		OUTB (nc_stest1, SCLK);	/* Use the PCI clock as SCSI clock */
 		f = (int) sym_getfreq (np);
 		OUTB (nc_stest1, 0);
 	}
 	np->pciclk_khz = f;
 
 	return f;
 }
 
 /*============= DRIVER ACTION/COMPLETION ====================*/
 
 /*
  *  Print something that tells about extended errors.
  */
 static void sym_print_xerr(ccb_p cp, int x_status)
 {
 	if (x_status & XE_PARITY_ERR) {
 		PRINT_ADDR(cp);
 		printf ("unrecovered SCSI parity error.\n");
 	}
 	if (x_status & XE_EXTRA_DATA) {
 		PRINT_ADDR(cp);
 		printf ("extraneous data discarded.\n");
 	}
 	if (x_status & XE_BAD_PHASE) {
 		PRINT_ADDR(cp);
 		printf ("illegal scsi phase (4/5).\n");
 	}
 	if (x_status & XE_SODL_UNRUN) {
 		PRINT_ADDR(cp);
 		printf ("ODD transfer in DATA OUT phase.\n");
 	}
 	if (x_status & XE_SWIDE_OVRUN) {
 		PRINT_ADDR(cp);
 		printf ("ODD transfer in DATA IN phase.\n");
 	}
 }
 
 /*
  *  Choose the more appropriate CAM status if
  *  the IO encountered an extended error.
  */
 static int sym_xerr_cam_status(int cam_status, int x_status)
 {
 	if (x_status) {
 		if	(x_status & XE_PARITY_ERR)
 			cam_status = CAM_UNCOR_PARITY;
 		else if	(x_status &(XE_EXTRA_DATA|XE_SODL_UNRUN|XE_SWIDE_OVRUN))
 			cam_status = CAM_DATA_RUN_ERR;
 		else if	(x_status & XE_BAD_PHASE)
 			cam_status = CAM_REQ_CMP_ERR;
 		else
 			cam_status = CAM_REQ_CMP_ERR;
 	}
 	return cam_status;
 }
 
 /*
  *  Complete execution of a SCSI command with extented
  *  error, SCSI status error, or having been auto-sensed.
  *
  *  The SCRIPTS processor is not running there, so we
  *  can safely access IO registers and remove JOBs from
  *  the START queue.
  *  SCRATCHA is assumed to have been loaded with STARTPOS
  *  before the SCRIPTS called the C code.
  */
 static void sym_complete_error (hcb_p np, ccb_p cp)
 {
 	struct ccb_scsiio *csio;
 	u_int cam_status;
 	int i, sense_returned;
 
 	SYM_LOCK_ASSERT(MA_OWNED);
 
 	/*
 	 *  Paranoid check. :)
 	 */
 	if (!cp || !cp->cam_ccb)
 		return;
 
 	if (DEBUG_FLAGS & (DEBUG_TINY|DEBUG_RESULT)) {
 		printf ("CCB=%lx STAT=%x/%x/%x DEV=%d/%d\n", (unsigned long)cp,
 			cp->host_status, cp->ssss_status, cp->host_flags,
 			cp->target, cp->lun);
 		MDELAY(100);
 	}
 
 	/*
 	 *  Get CAM command pointer.
 	 */
 	csio = &cp->cam_ccb->csio;
 
 	/*
 	 *  Check for extended errors.
 	 */
 	if (cp->xerr_status) {
 		if (sym_verbose)
 			sym_print_xerr(cp, cp->xerr_status);
 		if (cp->host_status == HS_COMPLETE)
 			cp->host_status = HS_COMP_ERR;
 	}
 
 	/*
 	 *  Calculate the residual.
 	 */
 	csio->sense_resid = 0;
 	csio->resid = sym_compute_residual(np, cp);
 
 	if (!SYM_CONF_RESIDUAL_SUPPORT) {/* If user does not want residuals */
 		csio->resid  = 0;	/* throw them away. :)		   */
 		cp->sv_resid = 0;
 	}
 
 	if (cp->host_flags & HF_SENSE) {		/* Auto sense     */
 		csio->scsi_status = cp->sv_scsi_status;	/* Restore status */
 		csio->sense_resid = csio->resid;	/* Swap residuals */
 		csio->resid       = cp->sv_resid;
 		cp->sv_resid	  = 0;
 		if (sym_verbose && cp->sv_xerr_status)
 			sym_print_xerr(cp, cp->sv_xerr_status);
 		if (cp->host_status == HS_COMPLETE &&
 		    cp->ssss_status == S_GOOD &&
 		    cp->xerr_status == 0) {
 			cam_status = sym_xerr_cam_status(CAM_SCSI_STATUS_ERROR,
 							 cp->sv_xerr_status);
 			cam_status |= CAM_AUTOSNS_VALID;
 			/*
 			 *  Bounce back the sense data to user and
 			 *  fix the residual.
 			 */
 			bzero(&csio->sense_data, sizeof(csio->sense_data));
 			sense_returned = SYM_SNS_BBUF_LEN - csio->sense_resid;
 			if (sense_returned < csio->sense_len)
 				csio->sense_resid = csio->sense_len -
 				    sense_returned;
 			else
 				csio->sense_resid = 0;
 			bcopy(cp->sns_bbuf, &csio->sense_data,
 			    MIN(csio->sense_len, sense_returned));
 #if 0
 			/*
 			 *  If the device reports a UNIT ATTENTION condition
 			 *  due to a RESET condition, we should consider all
 			 *  disconnect CCBs for this unit as aborted.
 			 */
 			if (1) {
 				u_char *p;
 				p  = (u_char *) csio->sense_data;
 				if (p[0]==0x70 && p[2]==0x6 && p[12]==0x29)
 					sym_clear_tasks(np, CAM_REQ_ABORTED,
 							cp->target,cp->lun, -1);
 			}
 #endif
 		}
 		else
 			cam_status = CAM_AUTOSENSE_FAIL;
 	}
 	else if (cp->host_status == HS_COMPLETE) {	/* Bad SCSI status */
 		csio->scsi_status = cp->ssss_status;
 		cam_status = CAM_SCSI_STATUS_ERROR;
 	}
 	else if (cp->host_status == HS_SEL_TIMEOUT)	/* Selection timeout */
 		cam_status = CAM_SEL_TIMEOUT;
 	else if (cp->host_status == HS_UNEXPECTED)	/* Unexpected BUS FREE*/
 		cam_status = CAM_UNEXP_BUSFREE;
 	else {						/* Extended error */
 		if (sym_verbose) {
 			PRINT_ADDR(cp);
 			printf ("COMMAND FAILED (%x %x %x).\n",
 				cp->host_status, cp->ssss_status,
 				cp->xerr_status);
 		}
 		csio->scsi_status = cp->ssss_status;
 		/*
 		 *  Set the most appropriate value for CAM status.
 		 */
 		cam_status = sym_xerr_cam_status(CAM_REQ_CMP_ERR,
 						 cp->xerr_status);
 	}
 
 	/*
 	 *  Dequeue all queued CCBs for that device
 	 *  not yet started by SCRIPTS.
 	 */
 	i = (INL (nc_scratcha) - np->squeue_ba) / 4;
 	(void) sym_dequeue_from_squeue(np, i, cp->target, cp->lun, -1);
 
 	/*
 	 *  Restart the SCRIPTS processor.
 	 */
 	OUTL_DSP (SCRIPTA_BA (np, start));
 
 	/*
 	 *  Synchronize DMA map if needed.
 	 */
 	if (cp->dmamapped) {
 		bus_dmamap_sync(np->data_dmat, cp->dmamap,
 			(cp->dmamapped == SYM_DMA_READ ?
 				BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE));
 	}
 	/*
 	 *  Add this one to the COMP queue.
 	 *  Complete all those commands with either error
 	 *  or requeue condition.
 	 */
 	sym_set_cam_status((union ccb *) csio, cam_status);
 	sym_remque(&cp->link_ccbq);
 	sym_insque_head(&cp->link_ccbq, &np->comp_ccbq);
 	sym_flush_comp_queue(np, 0);
 }
 
 /*
  *  Complete execution of a successful SCSI command.
  *
  *  Only successful commands go to the DONE queue,
  *  since we need to have the SCRIPTS processor
  *  stopped on any error condition.
  *  The SCRIPTS processor is running while we are
  *  completing successful commands.
  */
 static void sym_complete_ok (hcb_p np, ccb_p cp)
 {
 	struct ccb_scsiio *csio;
 	tcb_p tp;
 	lcb_p lp;
 
 	SYM_LOCK_ASSERT(MA_OWNED);
 
 	/*
 	 *  Paranoid check. :)
 	 */
 	if (!cp || !cp->cam_ccb)
 		return;
 	assert (cp->host_status == HS_COMPLETE);
 
 	/*
 	 *  Get command, target and lun pointers.
 	 */
 	csio = &cp->cam_ccb->csio;
 	tp = &np->target[cp->target];
 	lp = sym_lp(tp, cp->lun);
 
 	/*
 	 *  Assume device discovered on first success.
 	 */
 	if (!lp)
 		sym_set_bit(tp->lun_map, cp->lun);
 
 	/*
 	 *  If all data have been transferred, given than no
 	 *  extended error did occur, there is no residual.
 	 */
 	csio->resid = 0;
 	if (cp->phys.head.lastp != cp->phys.head.goalp)
 		csio->resid = sym_compute_residual(np, cp);
 
 	/*
 	 *  Wrong transfer residuals may be worse than just always
 	 *  returning zero. User can disable this feature from
 	 *  sym_conf.h. Residual support is enabled by default.
 	 */
 	if (!SYM_CONF_RESIDUAL_SUPPORT)
 		csio->resid  = 0;
 
 	/*
 	 *  Synchronize DMA map if needed.
 	 */
 	if (cp->dmamapped) {
 		bus_dmamap_sync(np->data_dmat, cp->dmamap,
 			(cp->dmamapped == SYM_DMA_READ ?
 				BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE));
 	}
 	/*
 	 *  Set status and complete the command.
 	 */
 	csio->scsi_status = cp->ssss_status;
 	sym_set_cam_status((union ccb *) csio, CAM_REQ_CMP);
 	sym_xpt_done(np, (union ccb *) csio, cp);
 	sym_free_ccb(np, cp);
 }
 
 /*
  *  Our callout handler
  */
 static void sym_callout(void *arg)
 {
 	union ccb *ccb = (union ccb *) arg;
 	hcb_p np = ccb->ccb_h.sym_hcb_ptr;
 
 	/*
 	 *  Check that the CAM CCB is still queued.
 	 */
 	if (!np)
 		return;
 
 	SYM_LOCK();
 
 	switch(ccb->ccb_h.func_code) {
 	case XPT_SCSI_IO:
 		(void) sym_abort_scsiio(np, ccb, 1);
 		break;
 	default:
 		break;
 	}
 
 	SYM_UNLOCK();
 }
 
 /*
  *  Abort an SCSI IO.
  */
 static int sym_abort_scsiio(hcb_p np, union ccb *ccb, int timed_out)
 {
 	ccb_p cp;
 	SYM_QUEHEAD *qp;
 
 	SYM_LOCK_ASSERT(MA_OWNED);
 
 	/*
 	 *  Look up our CCB control block.
 	 */
 	cp = NULL;
 	FOR_EACH_QUEUED_ELEMENT(&np->busy_ccbq, qp) {
 		ccb_p cp2 = sym_que_entry(qp, struct sym_ccb, link_ccbq);
 		if (cp2->cam_ccb == ccb) {
 			cp = cp2;
 			break;
 		}
 	}
 	if (!cp || cp->host_status == HS_WAIT)
 		return -1;
 
 	/*
 	 *  If a previous abort didn't succeed in time,
 	 *  perform a BUS reset.
 	 */
 	if (cp->to_abort) {
 		sym_reset_scsi_bus(np, 1);
 		return 0;
 	}
 
 	/*
 	 *  Mark the CCB for abort and allow time for.
 	 */
 	cp->to_abort = timed_out ? 2 : 1;
 	callout_reset(&cp->ch, 10 * hz, sym_callout, (caddr_t) ccb);
 
 	/*
 	 *  Tell the SCRIPTS processor to stop and synchronize with us.
 	 */
 	np->istat_sem = SEM;
 	OUTB (nc_istat, SIGP|SEM);
 	return 0;
 }
 
 /*
  *  Reset a SCSI device (all LUNs of a target).
  */
 static void sym_reset_dev(hcb_p np, union ccb *ccb)
 {
 	tcb_p tp;
 	struct ccb_hdr *ccb_h = &ccb->ccb_h;
 
 	SYM_LOCK_ASSERT(MA_OWNED);
 
 	if (ccb_h->target_id   == np->myaddr ||
 	    ccb_h->target_id   >= SYM_CONF_MAX_TARGET ||
 	    ccb_h->target_lun  >= SYM_CONF_MAX_LUN) {
 		sym_xpt_done2(np, ccb, CAM_DEV_NOT_THERE);
 		return;
 	}
 
 	tp = &np->target[ccb_h->target_id];
 
 	tp->to_reset = 1;
 	sym_xpt_done2(np, ccb, CAM_REQ_CMP);
 
 	np->istat_sem = SEM;
 	OUTB (nc_istat, SIGP|SEM);
 }
 
 /*
  *  SIM action entry point.
  */
 static void sym_action(struct cam_sim *sim, union ccb *ccb)
 {
 	hcb_p	np;
 	tcb_p	tp;
 	lcb_p	lp;
 	ccb_p	cp;
 	int 	tmp;
 	u_char	idmsg, *msgptr;
 	u_int   msglen;
 	struct	ccb_scsiio *csio;
 	struct	ccb_hdr  *ccb_h;
 
 	CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE, ("sym_action\n"));
 
 	/*
 	 *  Retrieve our controller data structure.
 	 */
 	np = (hcb_p) cam_sim_softc(sim);
 
 	SYM_LOCK_ASSERT(MA_OWNED);
 
 	/*
 	 *  The common case is SCSI IO.
 	 *  We deal with other ones elsewhere.
 	 */
 	if (ccb->ccb_h.func_code != XPT_SCSI_IO) {
 		sym_action2(sim, ccb);
 		return;
 	}
 	csio  = &ccb->csio;
 	ccb_h = &csio->ccb_h;
 
 	/*
 	 *  Work around races.
 	 */
 	if ((ccb_h->status & CAM_STATUS_MASK) != CAM_REQ_INPROG) {
 		xpt_done(ccb);
 		return;
 	}
 
 	/*
 	 *  Minimal checkings, so that we will not
 	 *  go outside our tables.
 	 */
 	if (ccb_h->target_id   == np->myaddr ||
 	    ccb_h->target_id   >= SYM_CONF_MAX_TARGET ||
 	    ccb_h->target_lun  >= SYM_CONF_MAX_LUN) {
 		sym_xpt_done2(np, ccb, CAM_DEV_NOT_THERE);
 		return;
         }
 
 	/*
 	 *  Retrieve the target and lun descriptors.
 	 */
 	tp = &np->target[ccb_h->target_id];
 	lp = sym_lp(tp, ccb_h->target_lun);
 
 	/*
 	 *  Complete the 1st INQUIRY command with error
 	 *  condition if the device is flagged NOSCAN
 	 *  at BOOT in the NVRAM. This may speed up
 	 *  the boot and maintain coherency with BIOS
 	 *  device numbering. Clearing the flag allows
 	 *  user to rescan skipped devices later.
 	 *  We also return error for devices not flagged
 	 *  for SCAN LUNS in the NVRAM since some mono-lun
 	 *  devices behave badly when asked for some non
 	 *  zero LUN. Btw, this is an absolute hack.:-)
 	 */
 	if (!(ccb_h->flags & CAM_CDB_PHYS) &&
 	    (0x12 == ((ccb_h->flags & CAM_CDB_POINTER) ?
 		  csio->cdb_io.cdb_ptr[0] : csio->cdb_io.cdb_bytes[0]))) {
 		if ((tp->usrflags & SYM_SCAN_BOOT_DISABLED) ||
 		    ((tp->usrflags & SYM_SCAN_LUNS_DISABLED) &&
 		     ccb_h->target_lun != 0)) {
 			tp->usrflags &= ~SYM_SCAN_BOOT_DISABLED;
 			sym_xpt_done2(np, ccb, CAM_DEV_NOT_THERE);
 			return;
 		}
 	}
 
 	/*
 	 *  Get a control block for this IO.
 	 */
 	tmp = ((ccb_h->flags & CAM_TAG_ACTION_VALID) != 0);
 	cp = sym_get_ccb(np, ccb_h->target_id, ccb_h->target_lun, tmp);
 	if (!cp) {
 		sym_xpt_done2(np, ccb, CAM_RESRC_UNAVAIL);
 		return;
 	}
 
 	/*
 	 *  Keep track of the IO in our CCB.
 	 */
 	cp->cam_ccb = ccb;
 
 	/*
 	 *  Build the IDENTIFY message.
 	 */
 	idmsg = M_IDENTIFY | cp->lun;
 	if (cp->tag != NO_TAG || (lp && (lp->current_flags & SYM_DISC_ENABLED)))
 		idmsg |= 0x40;
 
 	msgptr = cp->scsi_smsg;
 	msglen = 0;
 	msgptr[msglen++] = idmsg;
 
 	/*
 	 *  Build the tag message if present.
 	 */
 	if (cp->tag != NO_TAG) {
 		u_char order = csio->tag_action;
 
 		switch(order) {
 		case M_ORDERED_TAG:
 			break;
 		case M_HEAD_TAG:
 			break;
 		default:
 			order = M_SIMPLE_TAG;
 		}
 		msgptr[msglen++] = order;
 
 		/*
 		 *  For less than 128 tags, actual tags are numbered
 		 *  1,3,5,..2*MAXTAGS+1,since we may have to deal
 		 *  with devices that have problems with #TAG 0 or too
 		 *  great #TAG numbers. For more tags (up to 256),
 		 *  we use directly our tag number.
 		 */
 #if SYM_CONF_MAX_TASK > (512/4)
 		msgptr[msglen++] = cp->tag;
 #else
 		msgptr[msglen++] = (cp->tag << 1) + 1;
 #endif
 	}
 
 	/*
 	 *  Build a negotiation message if needed.
 	 *  (nego_status is filled by sym_prepare_nego())
 	 */
 	cp->nego_status = 0;
 	if (tp->tinfo.current.width   != tp->tinfo.goal.width  ||
 	    tp->tinfo.current.period  != tp->tinfo.goal.period ||
 	    tp->tinfo.current.offset  != tp->tinfo.goal.offset ||
 	    tp->tinfo.current.options != tp->tinfo.goal.options) {
 		if (!tp->nego_cp && lp)
 			msglen += sym_prepare_nego(np, cp, 0, msgptr + msglen);
 	}
 
 	/*
 	 *  Fill in our ccb
 	 */
 
 	/*
 	 *  Startqueue
 	 */
 	cp->phys.head.go.start   = cpu_to_scr(SCRIPTA_BA (np, select));
 	cp->phys.head.go.restart = cpu_to_scr(SCRIPTA_BA (np, resel_dsa));
 
 	/*
 	 *  select
 	 */
 	cp->phys.select.sel_id		= cp->target;
 	cp->phys.select.sel_scntl3	= tp->head.wval;
 	cp->phys.select.sel_sxfer	= tp->head.sval;
 	cp->phys.select.sel_scntl4	= tp->head.uval;
 
 	/*
 	 *  message
 	 */
 	cp->phys.smsg.addr	= cpu_to_scr(CCB_BA (cp, scsi_smsg));
 	cp->phys.smsg.size	= cpu_to_scr(msglen);
 
 	/*
 	 *  command
 	 */
 	if (sym_setup_cdb(np, csio, cp) < 0) {
 		sym_xpt_done(np, ccb, cp);
 		sym_free_ccb(np, cp);
 		return;
 	}
 
 	/*
 	 *  status
 	 */
 #if	0	/* Provision */
 	cp->actualquirks	= tp->quirks;
 #endif
 	cp->actualquirks	= SYM_QUIRK_AUTOSAVE;
 	cp->host_status		= cp->nego_status ? HS_NEGOTIATE : HS_BUSY;
 	cp->ssss_status		= S_ILLEGAL;
 	cp->xerr_status		= 0;
 	cp->host_flags		= 0;
 	cp->extra_bytes		= 0;
 
 	/*
 	 *  extreme data pointer.
 	 *  shall be positive, so -1 is lower than lowest.:)
 	 */
 	cp->ext_sg  = -1;
 	cp->ext_ofs = 0;
 
 	/*
 	 *  Build the data descriptor block
 	 *  and start the IO.
 	 */
 	sym_setup_data_and_start(np, csio, cp);
 }
 
 /*
  *  Setup buffers and pointers that address the CDB.
  *  I bet, physical CDBs will never be used on the planet,
  *  since they can be bounced without significant overhead.
  */
 static int sym_setup_cdb(hcb_p np, struct ccb_scsiio *csio, ccb_p cp)
 {
 	struct ccb_hdr *ccb_h;
 	u32	cmd_ba;
 	int	cmd_len;
 
 	SYM_LOCK_ASSERT(MA_OWNED);
 
 	ccb_h = &csio->ccb_h;
 
 	/*
 	 *  CDB is 16 bytes max.
 	 */
 	if (csio->cdb_len > sizeof(cp->cdb_buf)) {
 		sym_set_cam_status(cp->cam_ccb, CAM_REQ_INVALID);
 		return -1;
 	}
 	cmd_len = csio->cdb_len;
 
 	if (ccb_h->flags & CAM_CDB_POINTER) {
 		/* CDB is a pointer */
 		if (!(ccb_h->flags & CAM_CDB_PHYS)) {
 			/* CDB pointer is virtual */
 			bcopy(csio->cdb_io.cdb_ptr, cp->cdb_buf, cmd_len);
 			cmd_ba = CCB_BA (cp, cdb_buf[0]);
 		} else {
 			/* CDB pointer is physical */
 #if 0
 			cmd_ba = ((u32)csio->cdb_io.cdb_ptr) & 0xffffffff;
 #else
 			sym_set_cam_status(cp->cam_ccb, CAM_REQ_INVALID);
 			return -1;
 #endif
 		}
 	} else {
 		/* CDB is in the CAM ccb (buffer) */
 		bcopy(csio->cdb_io.cdb_bytes, cp->cdb_buf, cmd_len);
 		cmd_ba = CCB_BA (cp, cdb_buf[0]);
 	}
 
 	cp->phys.cmd.addr	= cpu_to_scr(cmd_ba);
 	cp->phys.cmd.size	= cpu_to_scr(cmd_len);
 
 	return 0;
 }
 
 /*
  *  Set up data pointers used by SCRIPTS.
  */
 static void __inline
 sym_setup_data_pointers(hcb_p np, ccb_p cp, int dir)
 {
 	u32 lastp, goalp;
 
 	SYM_LOCK_ASSERT(MA_OWNED);
 
 	/*
 	 *  No segments means no data.
 	 */
 	if (!cp->segments)
 		dir = CAM_DIR_NONE;
 
 	/*
 	 *  Set the data pointer.
 	 */
 	switch(dir) {
 	case CAM_DIR_OUT:
 		goalp = SCRIPTA_BA (np, data_out2) + 8;
 		lastp = goalp - 8 - (cp->segments * (2*4));
 		break;
 	case CAM_DIR_IN:
 		cp->host_flags |= HF_DATA_IN;
 		goalp = SCRIPTA_BA (np, data_in2) + 8;
 		lastp = goalp - 8 - (cp->segments * (2*4));
 		break;
 	case CAM_DIR_NONE:
 	default:
 		lastp = goalp = SCRIPTB_BA (np, no_data);
 		break;
 	}
 
 	cp->phys.head.lastp = cpu_to_scr(lastp);
 	cp->phys.head.goalp = cpu_to_scr(goalp);
 	cp->phys.head.savep = cpu_to_scr(lastp);
 	cp->startp	    = cp->phys.head.savep;
 }
 
 /*
  *  Call back routine for the DMA map service.
  *  If bounce buffers are used (why ?), we may sleep and then
  *  be called there in another context.
  */
 static void
 sym_execute_ccb(void *arg, bus_dma_segment_t *psegs, int nsegs, int error)
 {
 	ccb_p	cp;
 	hcb_p	np;
 	union	ccb *ccb;
 
 	cp  = (ccb_p) arg;
 	ccb = cp->cam_ccb;
 	np  = (hcb_p) cp->arg;
 
 	SYM_LOCK_ASSERT(MA_OWNED);
 
 	/*
 	 *  Deal with weird races.
 	 */
 	if (sym_get_cam_status(ccb) != CAM_REQ_INPROG)
 		goto out_abort;
 
 	/*
 	 *  Deal with weird errors.
 	 */
 	if (error) {
 		cp->dmamapped = 0;
 		sym_set_cam_status(cp->cam_ccb, CAM_REQ_ABORTED);
 		goto out_abort;
 	}
 
 	/*
 	 *  Build the data descriptor for the chip.
 	 */
 	if (nsegs) {
 		int retv;
 		/* 896 rev 1 requires to be careful about boundaries */
 		if (np->device_id == PCI_ID_SYM53C896 && np->revision_id <= 1)
 			retv = sym_scatter_sg_physical(np, cp, psegs, nsegs);
 		else
 			retv = sym_fast_scatter_sg_physical(np,cp, psegs,nsegs);
 		if (retv < 0) {
 			sym_set_cam_status(cp->cam_ccb, CAM_REQ_TOO_BIG);
 			goto out_abort;
 		}
 	}
 
 	/*
 	 *  Synchronize the DMA map only if we have
 	 *  actually mapped the data.
 	 */
 	if (cp->dmamapped) {
 		bus_dmamap_sync(np->data_dmat, cp->dmamap,
 			(cp->dmamapped == SYM_DMA_READ ?
 				BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE));
 	}
 
 	/*
 	 *  Set host status to busy state.
 	 *  May have been set back to HS_WAIT to avoid a race.
 	 */
 	cp->host_status	= cp->nego_status ? HS_NEGOTIATE : HS_BUSY;
 
 	/*
 	 *  Set data pointers.
 	 */
 	sym_setup_data_pointers(np, cp,  (ccb->ccb_h.flags & CAM_DIR_MASK));
 
 	/*
 	 *  Enqueue this IO in our pending queue.
 	 */
 	sym_enqueue_cam_ccb(cp);
 
 	/*
 	 *  When `#ifed 1', the code below makes the driver
 	 *  panic on the first attempt to write to a SCSI device.
 	 *  It is the first test we want to do after a driver
 	 *  change that does not seem obviously safe. :)
 	 */
 #if 0
 	switch (cp->cdb_buf[0]) {
 	case 0x0A: case 0x2A: case 0xAA:
 		panic("XXXXXXXXXXXXX WRITE NOT YET ALLOWED XXXXXXXXXXXXXX\n");
 		MDELAY(10000);
 		break;
 	default:
 		break;
 	}
 #endif
 	/*
 	 *  Activate this job.
 	 */
 	sym_put_start_queue(np, cp);
 	return;
 out_abort:
 	sym_xpt_done(np, ccb, cp);
 	sym_free_ccb(np, cp);
 }
 
 /*
  *  How complex it gets to deal with the data in CAM.
  *  The Bus Dma stuff makes things still more complex.
  */
 static void
 sym_setup_data_and_start(hcb_p np, struct ccb_scsiio *csio, ccb_p cp)
 {
 	struct ccb_hdr *ccb_h;
 	int dir, retv;
 
 	SYM_LOCK_ASSERT(MA_OWNED);
 
 	ccb_h = &csio->ccb_h;
 
 	/*
 	 *  Now deal with the data.
 	 */
 	cp->data_len = csio->dxfer_len;
 	cp->arg      = np;
 
 	/*
 	 *  No direction means no data.
 	 */
 	dir = (ccb_h->flags & CAM_DIR_MASK);
 	if (dir == CAM_DIR_NONE) {
 		sym_execute_ccb(cp, NULL, 0, 0);
 		return;
 	}
 
 	cp->dmamapped = (dir == CAM_DIR_IN) ?  SYM_DMA_READ : SYM_DMA_WRITE;
 	retv = bus_dmamap_load_ccb(np->data_dmat, cp->dmamap,
 			       (union ccb *)csio, sym_execute_ccb, cp, 0);
 	if (retv == EINPROGRESS) {
 		cp->host_status	= HS_WAIT;
 		xpt_freeze_simq(np->sim, 1);
 		csio->ccb_h.status |= CAM_RELEASE_SIMQ;
 	}
 }
 
 /*
  *  Move the scatter list to our data block.
  */
 static int
 sym_fast_scatter_sg_physical(hcb_p np, ccb_p cp,
 			     bus_dma_segment_t *psegs, int nsegs)
 {
 	struct sym_tblmove *data;
 	bus_dma_segment_t *psegs2;
 
 	SYM_LOCK_ASSERT(MA_OWNED);
 
 	if (nsegs > SYM_CONF_MAX_SG)
 		return -1;
 
 	data   = &cp->phys.data[SYM_CONF_MAX_SG-1];
 	psegs2 = &psegs[nsegs-1];
 	cp->segments = nsegs;
 
 	while (1) {
 		data->addr = cpu_to_scr(psegs2->ds_addr);
 		data->size = cpu_to_scr(psegs2->ds_len);
 		if (DEBUG_FLAGS & DEBUG_SCATTER) {
 			printf ("%s scatter: paddr=%lx len=%ld\n",
 				sym_name(np), (long) psegs2->ds_addr,
 				(long) psegs2->ds_len);
 		}
 		if (psegs2 != psegs) {
 			--data;
 			--psegs2;
 			continue;
 		}
 		break;
 	}
 	return 0;
 }
 
 /*
  *  Scatter a SG list with physical addresses into bus addressable chunks.
  */
 static int
 sym_scatter_sg_physical(hcb_p np, ccb_p cp, bus_dma_segment_t *psegs, int nsegs)
 {
 	u_long	ps, pe, pn;
 	u_long	k;
 	int s, t;
 
 	SYM_LOCK_ASSERT(MA_OWNED);
 
 	s  = SYM_CONF_MAX_SG - 1;
 	t  = nsegs - 1;
 	ps = psegs[t].ds_addr;
 	pe = ps + psegs[t].ds_len;
 
 	while (s >= 0) {
 		pn = (pe - 1) & ~(SYM_CONF_DMA_BOUNDARY - 1);
 		if (pn <= ps)
 			pn = ps;
 		k = pe - pn;
 		if (DEBUG_FLAGS & DEBUG_SCATTER) {
 			printf ("%s scatter: paddr=%lx len=%ld\n",
 				sym_name(np), pn, k);
 		}
 		cp->phys.data[s].addr = cpu_to_scr(pn);
 		cp->phys.data[s].size = cpu_to_scr(k);
 		--s;
 		if (pn == ps) {
 			if (--t < 0)
 				break;
 			ps = psegs[t].ds_addr;
 			pe = ps + psegs[t].ds_len;
 		}
 		else
 			pe = pn;
 	}
 
 	cp->segments = SYM_CONF_MAX_SG - 1 - s;
 
 	return t >= 0 ? -1 : 0;
 }
 
 /*
  *  SIM action for non performance critical stuff.
  */
 static void sym_action2(struct cam_sim *sim, union ccb *ccb)
 {
 	union ccb *abort_ccb;
 	struct ccb_hdr *ccb_h;
 	struct ccb_pathinq *cpi;
 	struct ccb_trans_settings *cts;
 	struct sym_trans *tip;
 	hcb_p	np;
 	tcb_p	tp;
 	lcb_p	lp;
 	u_char dflags;
 
 	/*
 	 *  Retrieve our controller data structure.
 	 */
 	np = (hcb_p) cam_sim_softc(sim);
 
 	SYM_LOCK_ASSERT(MA_OWNED);
 
 	ccb_h = &ccb->ccb_h;
 
 	switch (ccb_h->func_code) {
 	case XPT_SET_TRAN_SETTINGS:
 		cts  = &ccb->cts;
 		tp = &np->target[ccb_h->target_id];
 
 		/*
 		 *  Update SPI transport settings in TARGET control block.
 		 *  Update SCSI device settings in LUN control block.
 		 */
 		lp = sym_lp(tp, ccb_h->target_lun);
 		if (cts->type == CTS_TYPE_CURRENT_SETTINGS) {
 			sym_update_trans(np, &tp->tinfo.goal, cts);
 			if (lp)
 				sym_update_dflags(np, &lp->current_flags, cts);
 		}
 		if (cts->type == CTS_TYPE_USER_SETTINGS) {
 			sym_update_trans(np, &tp->tinfo.user, cts);
 			if (lp)
 				sym_update_dflags(np, &lp->user_flags, cts);
 		}
 
 		sym_xpt_done2(np, ccb, CAM_REQ_CMP);
 		break;
 	case XPT_GET_TRAN_SETTINGS:
 		cts = &ccb->cts;
 		tp = &np->target[ccb_h->target_id];
 		lp = sym_lp(tp, ccb_h->target_lun);
 
 #define	cts__scsi (&cts->proto_specific.scsi)
 #define	cts__spi  (&cts->xport_specific.spi)
 		if (cts->type == CTS_TYPE_CURRENT_SETTINGS) {
 			tip = &tp->tinfo.current;
 			dflags = lp ? lp->current_flags : 0;
 		}
 		else {
 			tip = &tp->tinfo.user;
 			dflags = lp ? lp->user_flags : tp->usrflags;
 		}
 
 		cts->protocol  = PROTO_SCSI;
 		cts->transport = XPORT_SPI;
 		cts->protocol_version  = tip->scsi_version;
 		cts->transport_version = tip->spi_version;
 
 		cts__spi->sync_period = tip->period;
 		cts__spi->sync_offset = tip->offset;
 		cts__spi->bus_width   = tip->width;
 		cts__spi->ppr_options = tip->options;
 
 		cts__spi->valid = CTS_SPI_VALID_SYNC_RATE
 		                | CTS_SPI_VALID_SYNC_OFFSET
 		                | CTS_SPI_VALID_BUS_WIDTH
 		                | CTS_SPI_VALID_PPR_OPTIONS;
 
 		cts__spi->flags &= ~CTS_SPI_FLAGS_DISC_ENB;
 		if (dflags & SYM_DISC_ENABLED)
 			cts__spi->flags |= CTS_SPI_FLAGS_DISC_ENB;
 		cts__spi->valid |= CTS_SPI_VALID_DISC;
 
 		cts__scsi->flags &= ~CTS_SCSI_FLAGS_TAG_ENB;
 		if (dflags & SYM_TAGS_ENABLED)
 			cts__scsi->flags |= CTS_SCSI_FLAGS_TAG_ENB;
 		cts__scsi->valid |= CTS_SCSI_VALID_TQ;
 #undef	cts__spi
 #undef	cts__scsi
 		sym_xpt_done2(np, ccb, CAM_REQ_CMP);
 		break;
 	case XPT_CALC_GEOMETRY:
 		cam_calc_geometry(&ccb->ccg, /*extended*/1);
 		sym_xpt_done2(np, ccb, CAM_REQ_CMP);
 		break;
 	case XPT_PATH_INQ:
 		cpi = &ccb->cpi;
 		cpi->version_num = 1;
 		cpi->hba_inquiry = PI_MDP_ABLE|PI_SDTR_ABLE|PI_TAG_ABLE;
 		if ((np->features & FE_WIDE) != 0)
 			cpi->hba_inquiry |= PI_WIDE_16;
 		cpi->target_sprt = 0;
 		cpi->hba_misc = PIM_UNMAPPED;
 		if (np->usrflags & SYM_SCAN_TARGETS_HILO)
 			cpi->hba_misc |= PIM_SCANHILO;
 		if (np->usrflags & SYM_AVOID_BUS_RESET)
 			cpi->hba_misc |= PIM_NOBUSRESET;
 		cpi->hba_eng_cnt = 0;
 		cpi->max_target = (np->features & FE_WIDE) ? 15 : 7;
 		/* Semantic problem:)LUN number max = max number of LUNs - 1 */
 		cpi->max_lun = SYM_CONF_MAX_LUN-1;
 		if (SYM_SETUP_MAX_LUN < SYM_CONF_MAX_LUN)
 			cpi->max_lun = SYM_SETUP_MAX_LUN-1;
 		cpi->bus_id = cam_sim_bus(sim);
 		cpi->initiator_id = np->myaddr;
 		cpi->base_transfer_speed = 3300;
 		strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
 		strncpy(cpi->hba_vid, "Symbios", HBA_IDLEN);
 		strncpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
 		cpi->unit_number = cam_sim_unit(sim);
 
 		cpi->protocol = PROTO_SCSI;
 		cpi->protocol_version = SCSI_REV_2;
 		cpi->transport = XPORT_SPI;
 		cpi->transport_version = 2;
 		cpi->xport_specific.spi.ppr_options = SID_SPI_CLOCK_ST;
 		if (np->features & FE_ULTRA3) {
 			cpi->transport_version = 3;
 			cpi->xport_specific.spi.ppr_options =
 			    SID_SPI_CLOCK_DT_ST;
 		}
 		cpi->maxio = SYM_CONF_MAX_SG * PAGE_SIZE;
 		sym_xpt_done2(np, ccb, CAM_REQ_CMP);
 		break;
 	case XPT_ABORT:
 		abort_ccb = ccb->cab.abort_ccb;
 		switch(abort_ccb->ccb_h.func_code) {
 		case XPT_SCSI_IO:
 			if (sym_abort_scsiio(np, abort_ccb, 0) == 0) {
 				sym_xpt_done2(np, ccb, CAM_REQ_CMP);
 				break;
 			}
 		default:
 			sym_xpt_done2(np, ccb, CAM_UA_ABORT);
 			break;
 		}
 		break;
 	case XPT_RESET_DEV:
 		sym_reset_dev(np, ccb);
 		break;
 	case XPT_RESET_BUS:
 		sym_reset_scsi_bus(np, 0);
 		if (sym_verbose) {
 			xpt_print_path(np->path);
 			printf("SCSI BUS reset delivered.\n");
 		}
 		sym_init (np, 1);
 		sym_xpt_done2(np, ccb, CAM_REQ_CMP);
 		break;
 	case XPT_ACCEPT_TARGET_IO:
 	case XPT_CONT_TARGET_IO:
 	case XPT_EN_LUN:
 	case XPT_NOTIFY_ACK:
 	case XPT_IMMED_NOTIFY:
 	case XPT_TERM_IO:
 	default:
 		sym_xpt_done2(np, ccb, CAM_REQ_INVALID);
 		break;
 	}
 }
 
 /*
  *  Asynchronous notification handler.
  */
 static void
 sym_async(void *cb_arg, u32 code, struct cam_path *path, void *args __unused)
 {
 	hcb_p np;
 	struct cam_sim *sim;
 	u_int tn;
 	tcb_p tp;
 
 	sim = (struct cam_sim *) cb_arg;
 	np  = (hcb_p) cam_sim_softc(sim);
 
 	SYM_LOCK_ASSERT(MA_OWNED);
 
 	switch (code) {
 	case AC_LOST_DEVICE:
 		tn = xpt_path_target_id(path);
 		if (tn >= SYM_CONF_MAX_TARGET)
 			break;
 
 		tp = &np->target[tn];
 
 		tp->to_reset  = 0;
 		tp->head.sval = 0;
 		tp->head.wval = np->rv_scntl3;
 		tp->head.uval = 0;
 
 		tp->tinfo.current.period  = tp->tinfo.goal.period = 0;
 		tp->tinfo.current.offset  = tp->tinfo.goal.offset = 0;
 		tp->tinfo.current.width   = tp->tinfo.goal.width  = BUS_8_BIT;
 		tp->tinfo.current.options = tp->tinfo.goal.options = 0;
 
 		break;
 	default:
 		break;
 	}
 }
 
 /*
  *  Update transfer settings of a target.
  */
 static void sym_update_trans(hcb_p np, struct sym_trans *tip,
     struct ccb_trans_settings *cts)
 {
 
 	SYM_LOCK_ASSERT(MA_OWNED);
 
 	/*
 	 *  Update the infos.
 	 */
 #define cts__spi (&cts->xport_specific.spi)
 	if ((cts__spi->valid & CTS_SPI_VALID_BUS_WIDTH) != 0)
 		tip->width = cts__spi->bus_width;
 	if ((cts__spi->valid & CTS_SPI_VALID_SYNC_OFFSET) != 0)
 		tip->offset = cts__spi->sync_offset;
 	if ((cts__spi->valid & CTS_SPI_VALID_SYNC_RATE) != 0)
 		tip->period = cts__spi->sync_period;
 	if ((cts__spi->valid & CTS_SPI_VALID_PPR_OPTIONS) != 0)
 		tip->options = (cts__spi->ppr_options & PPR_OPT_DT);
 	if (cts->protocol_version != PROTO_VERSION_UNSPECIFIED &&
 	    cts->protocol_version != PROTO_VERSION_UNKNOWN)
 		tip->scsi_version = cts->protocol_version;
 	if (cts->transport_version != XPORT_VERSION_UNSPECIFIED &&
 	    cts->transport_version != XPORT_VERSION_UNKNOWN)
 		tip->spi_version = cts->transport_version;
 #undef cts__spi
 	/*
 	 *  Scale against driver configuration limits.
 	 */
 	if (tip->width  > SYM_SETUP_MAX_WIDE) tip->width  = SYM_SETUP_MAX_WIDE;
 	if (tip->period && tip->offset) {
 		if (tip->offset > SYM_SETUP_MAX_OFFS) tip->offset = SYM_SETUP_MAX_OFFS;
 		if (tip->period < SYM_SETUP_MIN_SYNC) tip->period = SYM_SETUP_MIN_SYNC;
 	} else {
 		tip->offset = 0;
 		tip->period = 0;
 	}
 
 	/*
 	 *  Scale against actual controller BUS width.
 	 */
 	if (tip->width > np->maxwide)
 		tip->width  = np->maxwide;
 
 	/*
 	 *  Only accept DT if controller supports and SYNC/WIDE asked.
 	 */
 	if (!((np->features & (FE_C10|FE_ULTRA3)) == (FE_C10|FE_ULTRA3)) ||
 	    !(tip->width == BUS_16_BIT && tip->offset)) {
 		tip->options &= ~PPR_OPT_DT;
 	}
 
 	/*
 	 *  Scale period factor and offset against controller limits.
 	 */
 	if (tip->offset && tip->period) {
 		if (tip->options & PPR_OPT_DT) {
 			if (tip->period < np->minsync_dt)
 				tip->period = np->minsync_dt;
 			if (tip->period > np->maxsync_dt)
 				tip->period = np->maxsync_dt;
 			if (tip->offset > np->maxoffs_dt)
 				tip->offset = np->maxoffs_dt;
 		}
 		else {
 			if (tip->period < np->minsync)
 				tip->period = np->minsync;
 			if (tip->period > np->maxsync)
 				tip->period = np->maxsync;
 			if (tip->offset > np->maxoffs)
 				tip->offset = np->maxoffs;
 		}
 	}
 }
 
 /*
  *  Update flags for a device (logical unit).
  */
 static void
 sym_update_dflags(hcb_p np, u_char *flags, struct ccb_trans_settings *cts)
 {
 
 	SYM_LOCK_ASSERT(MA_OWNED);
 
 #define	cts__scsi (&cts->proto_specific.scsi)
 #define	cts__spi  (&cts->xport_specific.spi)
 	if ((cts__spi->valid & CTS_SPI_VALID_DISC) != 0) {
 		if ((cts__spi->flags & CTS_SPI_FLAGS_DISC_ENB) != 0)
 			*flags |= SYM_DISC_ENABLED;
 		else
 			*flags &= ~SYM_DISC_ENABLED;
 	}
 
 	if ((cts__scsi->valid & CTS_SCSI_VALID_TQ) != 0) {
 		if ((cts__scsi->flags & CTS_SCSI_FLAGS_TAG_ENB) != 0)
 			*flags |= SYM_TAGS_ENABLED;
 		else
 			*flags &= ~SYM_TAGS_ENABLED;
 	}
 #undef	cts__spi
 #undef	cts__scsi
 }
 
 /*============= DRIVER INITIALISATION ==================*/
 
 static device_method_t sym_pci_methods[] = {
 	DEVMETHOD(device_probe,	 sym_pci_probe),
 	DEVMETHOD(device_attach, sym_pci_attach),
 	DEVMETHOD_END
 };
 
 static driver_t sym_pci_driver = {
 	"sym",
 	sym_pci_methods,
 	1	/* no softc */
 };
 
 static devclass_t sym_devclass;
 
 DRIVER_MODULE(sym, pci, sym_pci_driver, sym_devclass, NULL, NULL);
 MODULE_DEPEND(sym, cam, 1, 1, 1);
 MODULE_DEPEND(sym, pci, 1, 1, 1);
 
 static const struct sym_pci_chip sym_pci_dev_table[] = {
  {PCI_ID_SYM53C810, 0x0f, "810", 4, 8, 4, 64,
  FE_ERL}
  ,
 #ifdef SYM_DEBUG_GENERIC_SUPPORT
  {PCI_ID_SYM53C810, 0xff, "810a", 4,  8, 4, 1,
  FE_BOF}
  ,
 #else
  {PCI_ID_SYM53C810, 0xff, "810a", 4,  8, 4, 1,
  FE_CACHE_SET|FE_LDSTR|FE_PFEN|FE_BOF}
  ,
 #endif
  {PCI_ID_SYM53C815, 0xff, "815", 4,  8, 4, 64,
  FE_BOF|FE_ERL}
  ,
  {PCI_ID_SYM53C825, 0x0f, "825", 6,  8, 4, 64,
  FE_WIDE|FE_BOF|FE_ERL|FE_DIFF}
  ,
  {PCI_ID_SYM53C825, 0xff, "825a", 6,  8, 4, 2,
  FE_WIDE|FE_CACHE0_SET|FE_BOF|FE_DFS|FE_LDSTR|FE_PFEN|FE_RAM|FE_DIFF}
  ,
  {PCI_ID_SYM53C860, 0xff, "860", 4,  8, 5, 1,
  FE_ULTRA|FE_CLK80|FE_CACHE_SET|FE_BOF|FE_LDSTR|FE_PFEN}
  ,
  {PCI_ID_SYM53C875, 0x01, "875", 6, 16, 5, 2,
  FE_WIDE|FE_ULTRA|FE_CLK80|FE_CACHE0_SET|FE_BOF|FE_DFS|FE_LDSTR|FE_PFEN|
  FE_RAM|FE_DIFF}
  ,
  {PCI_ID_SYM53C875, 0xff, "875", 6, 16, 5, 2,
  FE_WIDE|FE_ULTRA|FE_DBLR|FE_CACHE0_SET|FE_BOF|FE_DFS|FE_LDSTR|FE_PFEN|
  FE_RAM|FE_DIFF}
  ,
  {PCI_ID_SYM53C875_2, 0xff, "875", 6, 16, 5, 2,
  FE_WIDE|FE_ULTRA|FE_DBLR|FE_CACHE0_SET|FE_BOF|FE_DFS|FE_LDSTR|FE_PFEN|
  FE_RAM|FE_DIFF}
  ,
  {PCI_ID_SYM53C885, 0xff, "885", 6, 16, 5, 2,
  FE_WIDE|FE_ULTRA|FE_DBLR|FE_CACHE0_SET|FE_BOF|FE_DFS|FE_LDSTR|FE_PFEN|
  FE_RAM|FE_DIFF}
  ,
 #ifdef SYM_DEBUG_GENERIC_SUPPORT
  {PCI_ID_SYM53C895, 0xff, "895", 6, 31, 7, 2,
  FE_WIDE|FE_ULTRA2|FE_QUAD|FE_CACHE_SET|FE_BOF|FE_DFS|
  FE_RAM|FE_LCKFRQ}
  ,
 #else
  {PCI_ID_SYM53C895, 0xff, "895", 6, 31, 7, 2,
  FE_WIDE|FE_ULTRA2|FE_QUAD|FE_CACHE_SET|FE_BOF|FE_DFS|FE_LDSTR|FE_PFEN|
  FE_RAM|FE_LCKFRQ}
  ,
 #endif
  {PCI_ID_SYM53C896, 0xff, "896", 6, 31, 7, 4,
  FE_WIDE|FE_ULTRA2|FE_QUAD|FE_CACHE_SET|FE_BOF|FE_DFS|FE_LDSTR|FE_PFEN|
  FE_RAM|FE_RAM8K|FE_64BIT|FE_DAC|FE_IO256|FE_NOPM|FE_LEDC|FE_LCKFRQ}
  ,
  {PCI_ID_SYM53C895A, 0xff, "895a", 6, 31, 7, 4,
  FE_WIDE|FE_ULTRA2|FE_QUAD|FE_CACHE_SET|FE_BOF|FE_DFS|FE_LDSTR|FE_PFEN|
  FE_RAM|FE_RAM8K|FE_DAC|FE_IO256|FE_NOPM|FE_LEDC|FE_LCKFRQ}
  ,
  {PCI_ID_LSI53C1010, 0x00, "1010-33", 6, 31, 7, 8,
  FE_WIDE|FE_ULTRA3|FE_QUAD|FE_CACHE_SET|FE_BOF|FE_DFBC|FE_LDSTR|FE_PFEN|
  FE_RAM|FE_RAM8K|FE_64BIT|FE_DAC|FE_IO256|FE_NOPM|FE_LEDC|FE_CRC|
  FE_C10}
  ,
  {PCI_ID_LSI53C1010, 0xff, "1010-33", 6, 31, 7, 8,
  FE_WIDE|FE_ULTRA3|FE_QUAD|FE_CACHE_SET|FE_BOF|FE_DFBC|FE_LDSTR|FE_PFEN|
  FE_RAM|FE_RAM8K|FE_64BIT|FE_DAC|FE_IO256|FE_NOPM|FE_LEDC|FE_CRC|
  FE_C10|FE_U3EN}
  ,
  {PCI_ID_LSI53C1010_2, 0xff, "1010-66", 6, 31, 7, 8,
  FE_WIDE|FE_ULTRA3|FE_QUAD|FE_CACHE_SET|FE_BOF|FE_DFBC|FE_LDSTR|FE_PFEN|
  FE_RAM|FE_RAM8K|FE_64BIT|FE_DAC|FE_IO256|FE_NOPM|FE_LEDC|FE_66MHZ|FE_CRC|
  FE_C10|FE_U3EN}
  ,
  {PCI_ID_LSI53C1510D, 0xff, "1510d", 6, 31, 7, 4,
  FE_WIDE|FE_ULTRA2|FE_QUAD|FE_CACHE_SET|FE_BOF|FE_DFS|FE_LDSTR|FE_PFEN|
  FE_RAM|FE_IO256|FE_LEDC}
 };
 
 /*
  *  Look up the chip table.
  *
  *  Return a pointer to the chip entry if found,
  *  zero otherwise.
  */
 static const struct sym_pci_chip *
 sym_find_pci_chip(device_t dev)
 {
 	const struct	sym_pci_chip *chip;
 	int	i;
 	u_short	device_id;
 	u_char	revision;
 
 	if (pci_get_vendor(dev) != PCI_VENDOR_NCR)
 		return NULL;
 
 	device_id = pci_get_device(dev);
 	revision  = pci_get_revid(dev);
 
 	for (i = 0; i < nitems(sym_pci_dev_table); i++) {
 		chip = &sym_pci_dev_table[i];
 		if (device_id != chip->device_id)
 			continue;
 		if (revision > chip->revision_id)
 			continue;
 		return chip;
 	}
 
 	return NULL;
 }
 
 /*
  *  Tell upper layer if the chip is supported.
  */
 static int
 sym_pci_probe(device_t dev)
 {
 	const struct	sym_pci_chip *chip;
 
 	chip = sym_find_pci_chip(dev);
 	if (chip && sym_find_firmware(chip)) {
 		device_set_desc(dev, chip->name);
 		return (chip->lp_probe_bit & SYM_SETUP_LP_PROBE_MAP)?
 		  BUS_PROBE_LOW_PRIORITY : BUS_PROBE_DEFAULT;
 	}
 	return ENXIO;
 }
 
 /*
  *  Attach a sym53c8xx device.
  */
 static int
 sym_pci_attach(device_t dev)
 {
 	const struct	sym_pci_chip *chip;
 	u_short	command;
 	u_char	cachelnsz;
 	struct	sym_hcb *np = NULL;
 	struct	sym_nvram nvram;
 	const struct	sym_fw *fw = NULL;
 	int 	i;
 	bus_dma_tag_t	bus_dmat;
 
 	bus_dmat = bus_get_dma_tag(dev);
 
 	/*
 	 *  Only probed devices should be attached.
 	 *  We just enjoy being paranoid. :)
 	 */
 	chip = sym_find_pci_chip(dev);
 	if (chip == NULL || (fw = sym_find_firmware(chip)) == NULL)
 		return (ENXIO);
 
 	/*
 	 *  Allocate immediately the host control block,
 	 *  since we are only expecting to succeed. :)
 	 *  We keep track in the HCB of all the resources that
 	 *  are to be released on error.
 	 */
 	np = __sym_calloc_dma(bus_dmat, sizeof(*np), "HCB");
 	if (np)
 		np->bus_dmat = bus_dmat;
 	else
 		return (ENXIO);
 	device_set_softc(dev, np);
 
 	SYM_LOCK_INIT();
 
 	/*
 	 *  Copy some useful infos to the HCB.
 	 */
 	np->hcb_ba	 = vtobus(np);
 	np->verbose	 = bootverbose;
 	np->device	 = dev;
 	np->device_id	 = pci_get_device(dev);
 	np->revision_id  = pci_get_revid(dev);
 	np->features	 = chip->features;
 	np->clock_divn	 = chip->nr_divisor;
 	np->maxoffs	 = chip->offset_max;
 	np->maxburst	 = chip->burst_max;
 	np->scripta_sz	 = fw->a_size;
 	np->scriptb_sz	 = fw->b_size;
 	np->fw_setup	 = fw->setup;
 	np->fw_patch	 = fw->patch;
 	np->fw_name	 = fw->name;
 
 #ifdef __amd64__
 	np->target = sym_calloc_dma(SYM_CONF_MAX_TARGET * sizeof(*(np->target)),
 			"TARGET");
 	if (!np->target)
 		goto attach_failed;
 #endif
 
 	/*
 	 *  Initialize the CCB free and busy queues.
 	 */
 	sym_que_init(&np->free_ccbq);
 	sym_que_init(&np->busy_ccbq);
 	sym_que_init(&np->comp_ccbq);
 	sym_que_init(&np->cam_ccbq);
 
 	/*
 	 *  Allocate a tag for the DMA of user data.
 	 */
 	if (bus_dma_tag_create(np->bus_dmat, 1, SYM_CONF_DMA_BOUNDARY,
 	    BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL,
 	    BUS_SPACE_MAXSIZE_32BIT, SYM_CONF_MAX_SG, SYM_CONF_DMA_BOUNDARY,
 	    0, busdma_lock_mutex, &np->mtx, &np->data_dmat)) {
 		device_printf(dev, "failed to create DMA tag.\n");
 		goto attach_failed;
 	}
 
 	/*
 	 *  Read and apply some fix-ups to the PCI COMMAND
 	 *  register. We want the chip to be enabled for:
 	 *  - BUS mastering
 	 *  - PCI parity checking (reporting would also be fine)
 	 *  - Write And Invalidate.
 	 */
 	command = pci_read_config(dev, PCIR_COMMAND, 2);
 	command |= PCIM_CMD_BUSMASTEREN | PCIM_CMD_PERRESPEN |
 	    PCIM_CMD_MWRICEN;
 	pci_write_config(dev, PCIR_COMMAND, command, 2);
 
 	/*
 	 *  Let the device know about the cache line size,
 	 *  if it doesn't yet.
 	 */
 	cachelnsz = pci_read_config(dev, PCIR_CACHELNSZ, 1);
 	if (!cachelnsz) {
 		cachelnsz = 8;
 		pci_write_config(dev, PCIR_CACHELNSZ, cachelnsz, 1);
 	}
 
 	/*
 	 *  Alloc/get/map/retrieve everything that deals with MMIO.
 	 */
 	i = SYM_PCI_MMIO;
 	np->mmio_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &i,
 	    RF_ACTIVE);
 	if (!np->mmio_res) {
 		device_printf(dev, "failed to allocate MMIO resources\n");
 		goto attach_failed;
 	}
 	np->mmio_ba = rman_get_start(np->mmio_res);
 
 	/*
 	 *  Allocate the IRQ.
 	 */
 	i = 0;
 	np->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &i,
 					     RF_ACTIVE | RF_SHAREABLE);
 	if (!np->irq_res) {
 		device_printf(dev, "failed to allocate IRQ resource\n");
 		goto attach_failed;
 	}
 
 #ifdef	SYM_CONF_IOMAPPED
 	/*
 	 *  User want us to use normal IO with PCI.
 	 *  Alloc/get/map/retrieve everything that deals with IO.
 	 */
 	i = SYM_PCI_IO;
 	np->io_res = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &i, RF_ACTIVE);
 	if (!np->io_res) {
 		device_printf(dev, "failed to allocate IO resources\n");
 		goto attach_failed;
 	}
 
 #endif /* SYM_CONF_IOMAPPED */
 
 	/*
 	 *  If the chip has RAM.
 	 *  Alloc/get/map/retrieve the corresponding resources.
 	 */
 	if (np->features & (FE_RAM|FE_RAM8K)) {
 		int regs_id = SYM_PCI_RAM;
 		if (np->features & FE_64BIT)
 			regs_id = SYM_PCI_RAM64;
 		np->ram_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
 						     &regs_id, RF_ACTIVE);
 		if (!np->ram_res) {
 			device_printf(dev,"failed to allocate RAM resources\n");
 			goto attach_failed;
 		}
 		np->ram_id  = regs_id;
 		np->ram_ba = rman_get_start(np->ram_res);
 	}
 
 	/*
 	 *  Save setting of some IO registers, so we will
 	 *  be able to probe specific implementations.
 	 */
 	sym_save_initial_setting (np);
 
 	/*
 	 *  Reset the chip now, since it has been reported
 	 *  that SCSI clock calibration may not work properly
 	 *  if the chip is currently active.
 	 */
 	sym_chip_reset (np);
 
 	/*
 	 *  Try to read the user set-up.
 	 */
 	(void) sym_read_nvram(np, &nvram);
 
 	/*
 	 *  Prepare controller and devices settings, according
 	 *  to chip features, user set-up and driver set-up.
 	 */
 	(void) sym_prepare_setting(np, &nvram);
 
 	/*
 	 *  Check the PCI clock frequency.
 	 *  Must be performed after prepare_setting since it destroys
 	 *  STEST1 that is used to probe for the clock doubler.
 	 */
 	i = sym_getpciclock(np);
 	if (i > 37000)
 		device_printf(dev, "PCI BUS clock seems too high: %u KHz.\n",i);
 
 	/*
 	 *  Allocate the start queue.
 	 */
 	np->squeue = (u32 *) sym_calloc_dma(sizeof(u32)*(MAX_QUEUE*2),"SQUEUE");
 	if (!np->squeue)
 		goto attach_failed;
 	np->squeue_ba = vtobus(np->squeue);
 
 	/*
 	 *  Allocate the done queue.
 	 */
 	np->dqueue = (u32 *) sym_calloc_dma(sizeof(u32)*(MAX_QUEUE*2),"DQUEUE");
 	if (!np->dqueue)
 		goto attach_failed;
 	np->dqueue_ba = vtobus(np->dqueue);
 
 	/*
 	 *  Allocate the target bus address array.
 	 */
 	np->targtbl = (u32 *) sym_calloc_dma(256, "TARGTBL");
 	if (!np->targtbl)
 		goto attach_failed;
 	np->targtbl_ba = vtobus(np->targtbl);
 
 	/*
 	 *  Allocate SCRIPTS areas.
 	 */
 	np->scripta0 = sym_calloc_dma(np->scripta_sz, "SCRIPTA0");
 	np->scriptb0 = sym_calloc_dma(np->scriptb_sz, "SCRIPTB0");
 	if (!np->scripta0 || !np->scriptb0)
 		goto attach_failed;
 
 	/*
 	 *  Allocate the CCBs. We need at least ONE.
 	 */
 	for (i = 0; sym_alloc_ccb(np) != NULL; i++)
 		;
 	if (i < 1)
 		goto attach_failed;
 
 	/*
 	 *  Calculate BUS addresses where we are going
 	 *  to load the SCRIPTS.
 	 */
 	np->scripta_ba	= vtobus(np->scripta0);
 	np->scriptb_ba	= vtobus(np->scriptb0);
 	np->scriptb0_ba	= np->scriptb_ba;
 
 	if (np->ram_ba) {
 		np->scripta_ba	= np->ram_ba;
 		if (np->features & FE_RAM8K) {
 			np->ram_ws = 8192;
 			np->scriptb_ba = np->scripta_ba + 4096;
 #ifdef __LP64__
 			np->scr_ram_seg = cpu_to_scr(np->scripta_ba >> 32);
 #endif
 		}
 		else
 			np->ram_ws = 4096;
 	}
 
 	/*
 	 *  Copy scripts to controller instance.
 	 */
 	bcopy(fw->a_base, np->scripta0, np->scripta_sz);
 	bcopy(fw->b_base, np->scriptb0, np->scriptb_sz);
 
 	/*
 	 *  Setup variable parts in scripts and compute
 	 *  scripts bus addresses used from the C code.
 	 */
 	np->fw_setup(np, fw);
 
 	/*
 	 *  Bind SCRIPTS with physical addresses usable by the
 	 *  SCRIPTS processor (as seen from the BUS = BUS addresses).
 	 */
 	sym_fw_bind_script(np, (u32 *) np->scripta0, np->scripta_sz);
 	sym_fw_bind_script(np, (u32 *) np->scriptb0, np->scriptb_sz);
 
 #ifdef SYM_CONF_IARB_SUPPORT
 	/*
 	 *    If user wants IARB to be set when we win arbitration
 	 *    and have other jobs, compute the max number of consecutive
 	 *    settings of IARB hints before we leave devices a chance to
 	 *    arbitrate for reselection.
 	 */
 #ifdef	SYM_SETUP_IARB_MAX
 	np->iarb_max = SYM_SETUP_IARB_MAX;
 #else
 	np->iarb_max = 4;
 #endif
 #endif
 
 	/*
 	 *  Prepare the idle and invalid task actions.
 	 */
 	np->idletask.start	= cpu_to_scr(SCRIPTA_BA (np, idle));
 	np->idletask.restart	= cpu_to_scr(SCRIPTB_BA (np, bad_i_t_l));
 	np->idletask_ba		= vtobus(&np->idletask);
 
 	np->notask.start	= cpu_to_scr(SCRIPTA_BA (np, idle));
 	np->notask.restart	= cpu_to_scr(SCRIPTB_BA (np, bad_i_t_l));
 	np->notask_ba		= vtobus(&np->notask);
 
 	np->bad_itl.start	= cpu_to_scr(SCRIPTA_BA (np, idle));
 	np->bad_itl.restart	= cpu_to_scr(SCRIPTB_BA (np, bad_i_t_l));
 	np->bad_itl_ba		= vtobus(&np->bad_itl);
 
 	np->bad_itlq.start	= cpu_to_scr(SCRIPTA_BA (np, idle));
 	np->bad_itlq.restart	= cpu_to_scr(SCRIPTB_BA (np,bad_i_t_l_q));
 	np->bad_itlq_ba		= vtobus(&np->bad_itlq);
 
 	/*
 	 *  Allocate and prepare the lun JUMP table that is used
 	 *  for a target prior the probing of devices (bad lun table).
 	 *  A private table will be allocated for the target on the
 	 *  first INQUIRY response received.
 	 */
 	np->badluntbl = sym_calloc_dma(256, "BADLUNTBL");
 	if (!np->badluntbl)
 		goto attach_failed;
 
 	np->badlun_sa = cpu_to_scr(SCRIPTB_BA (np, resel_bad_lun));
 	for (i = 0 ; i < 64 ; i++)	/* 64 luns/target, no less */
 		np->badluntbl[i] = cpu_to_scr(vtobus(&np->badlun_sa));
 
 	/*
 	 *  Prepare the bus address array that contains the bus
 	 *  address of each target control block.
 	 *  For now, assume all logical units are wrong. :)
 	 */
 	for (i = 0 ; i < SYM_CONF_MAX_TARGET ; i++) {
 		np->targtbl[i] = cpu_to_scr(vtobus(&np->target[i]));
 		np->target[i].head.luntbl_sa =
 				cpu_to_scr(vtobus(np->badluntbl));
 		np->target[i].head.lun0_sa =
 				cpu_to_scr(vtobus(&np->badlun_sa));
 	}
 
 	/*
 	 *  Now check the cache handling of the pci chipset.
 	 */
 	if (sym_snooptest (np)) {
 		device_printf(dev, "CACHE INCORRECTLY CONFIGURED.\n");
 		goto attach_failed;
 	};
 
 	/*
 	 *  Now deal with CAM.
 	 *  Hopefully, we will succeed with that one.:)
 	 */
 	if (!sym_cam_attach(np))
 		goto attach_failed;
 
 	/*
 	 *  Sigh! we are done.
 	 */
 	return 0;
 
 	/*
 	 *  We have failed.
 	 *  We will try to free all the resources we have
 	 *  allocated, but if we are a boot device, this
 	 *  will not help that much.;)
 	 */
 attach_failed:
 	if (np)
 		sym_pci_free(np);
 	return ENXIO;
 }
 
 /*
  *  Free everything that have been allocated for this device.
  */
 static void sym_pci_free(hcb_p np)
 {
 	SYM_QUEHEAD *qp;
 	ccb_p cp;
 	tcb_p tp;
 	lcb_p lp;
 	int target, lun;
 
 	/*
 	 *  First free CAM resources.
 	 */
 	sym_cam_free(np);
 
 	/*
 	 *  Now every should be quiet for us to
 	 *  free other resources.
 	 */
 	if (np->ram_res)
 		bus_release_resource(np->device, SYS_RES_MEMORY,
 				     np->ram_id, np->ram_res);
 	if (np->mmio_res)
 		bus_release_resource(np->device, SYS_RES_MEMORY,
 				     SYM_PCI_MMIO, np->mmio_res);
 	if (np->io_res)
 		bus_release_resource(np->device, SYS_RES_IOPORT,
 				     SYM_PCI_IO, np->io_res);
 	if (np->irq_res)
 		bus_release_resource(np->device, SYS_RES_IRQ,
 				     0, np->irq_res);
 
 	if (np->scriptb0)
 		sym_mfree_dma(np->scriptb0, np->scriptb_sz, "SCRIPTB0");
 	if (np->scripta0)
 		sym_mfree_dma(np->scripta0, np->scripta_sz, "SCRIPTA0");
 	if (np->squeue)
 		sym_mfree_dma(np->squeue, sizeof(u32)*(MAX_QUEUE*2), "SQUEUE");
 	if (np->dqueue)
 		sym_mfree_dma(np->dqueue, sizeof(u32)*(MAX_QUEUE*2), "DQUEUE");
 
 	while ((qp = sym_remque_head(&np->free_ccbq)) != NULL) {
 		cp = sym_que_entry(qp, struct sym_ccb, link_ccbq);
 		bus_dmamap_destroy(np->data_dmat, cp->dmamap);
 		sym_mfree_dma(cp->sns_bbuf, SYM_SNS_BBUF_LEN, "SNS_BBUF");
 		sym_mfree_dma(cp, sizeof(*cp), "CCB");
 	}
 
 	if (np->badluntbl)
 		sym_mfree_dma(np->badluntbl, 256,"BADLUNTBL");
 
 	for (target = 0; target < SYM_CONF_MAX_TARGET ; target++) {
 		tp = &np->target[target];
 		for (lun = 0 ; lun < SYM_CONF_MAX_LUN ; lun++) {
 			lp = sym_lp(tp, lun);
 			if (!lp)
 				continue;
 			if (lp->itlq_tbl)
 				sym_mfree_dma(lp->itlq_tbl, SYM_CONF_MAX_TASK*4,
 				       "ITLQ_TBL");
 			if (lp->cb_tags)
 				sym_mfree(lp->cb_tags, SYM_CONF_MAX_TASK,
 				       "CB_TAGS");
 			sym_mfree_dma(lp, sizeof(*lp), "LCB");
 		}
 #if SYM_CONF_MAX_LUN > 1
 		if (tp->lunmp)
 			sym_mfree(tp->lunmp, SYM_CONF_MAX_LUN*sizeof(lcb_p),
 			       "LUNMP");
 #endif
 	}
 #ifdef __amd64__
 	if (np->target)
 		sym_mfree_dma(np->target,
 			SYM_CONF_MAX_TARGET * sizeof(*(np->target)), "TARGET");
 #endif
 	if (np->targtbl)
 		sym_mfree_dma(np->targtbl, 256, "TARGTBL");
 	if (np->data_dmat)
 		bus_dma_tag_destroy(np->data_dmat);
 	if (SYM_LOCK_INITIALIZED() != 0)
 		SYM_LOCK_DESTROY();
 	device_set_softc(np->device, NULL);
 	sym_mfree_dma(np, sizeof(*np), "HCB");
 }
 
 /*
  *  Allocate CAM resources and register a bus to CAM.
  */
 static int sym_cam_attach(hcb_p np)
 {
 	struct cam_devq *devq = NULL;
 	struct cam_sim *sim = NULL;
 	struct cam_path *path = NULL;
 	int err;
 
 	/*
 	 *  Establish our interrupt handler.
 	 */
 	err = bus_setup_intr(np->device, np->irq_res,
 			INTR_ENTROPY | INTR_MPSAFE | INTR_TYPE_CAM,
 			NULL, sym_intr, np, &np->intr);
 	if (err) {
 		device_printf(np->device, "bus_setup_intr() failed: %d\n",
 			      err);
 		goto fail;
 	}
 
 	/*
 	 *  Create the device queue for our sym SIM.
 	 */
 	devq = cam_simq_alloc(SYM_CONF_MAX_START);
 	if (!devq)
 		goto fail;
 
 	/*
 	 *  Construct our SIM entry.
 	 */
 	sim = cam_sim_alloc(sym_action, sym_poll, "sym", np,
 			device_get_unit(np->device),
 			&np->mtx, 1, SYM_SETUP_MAX_TAG, devq);
 	if (!sim)
 		goto fail;
 
 	SYM_LOCK();
 
 	if (xpt_bus_register(sim, np->device, 0) != CAM_SUCCESS)
 		goto fail;
 	np->sim = sim;
 
 	if (xpt_create_path(&path, NULL,
 			    cam_sim_path(np->sim), CAM_TARGET_WILDCARD,
 			    CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
 		goto fail;
 	}
 	np->path = path;
 
 	/*
 	 *  Establish our async notification handler.
 	 */
 	if (xpt_register_async(AC_LOST_DEVICE, sym_async, sim, path) !=
 	    CAM_REQ_CMP)
 		goto fail;
 
 	/*
 	 *  Start the chip now, without resetting the BUS, since
 	 *  it seems that this must stay under control of CAM.
 	 *  With LVD/SE capable chips and BUS in SE mode, we may
 	 *  get a spurious SMBC interrupt.
 	 */
 	sym_init (np, 0);
 
 	SYM_UNLOCK();
 
 	return 1;
 fail:
 	if (sim)
 		cam_sim_free(sim, FALSE);
 	if (devq)
 		cam_simq_free(devq);
 
 	SYM_UNLOCK();
 
 	sym_cam_free(np);
 
 	return 0;
 }
 
 /*
  *  Free everything that deals with CAM.
  */
 static void sym_cam_free(hcb_p np)
 {
 
 	SYM_LOCK_ASSERT(MA_NOTOWNED);
 
 	if (np->intr) {
 		bus_teardown_intr(np->device, np->irq_res, np->intr);
 		np->intr = NULL;
 	}
 
 	SYM_LOCK();
 
 	if (np->sim) {
 		xpt_bus_deregister(cam_sim_path(np->sim));
 		cam_sim_free(np->sim, /*free_devq*/ TRUE);
 		np->sim = NULL;
 	}
 	if (np->path) {
 		xpt_free_path(np->path);
 		np->path = NULL;
 	}
 
 	SYM_UNLOCK();
 }
 
 /*============ OPTIONNAL NVRAM SUPPORT =================*/
 
 /*
  *  Get host setup from NVRAM.
  */
 static void sym_nvram_setup_host (hcb_p np, struct sym_nvram *nvram)
 {
 #ifdef SYM_CONF_NVRAM_SUPPORT
 	/*
 	 *  Get parity checking, host ID, verbose mode
 	 *  and miscellaneous host flags from NVRAM.
 	 */
 	switch(nvram->type) {
 	case SYM_SYMBIOS_NVRAM:
 		if (!(nvram->data.Symbios.flags & SYMBIOS_PARITY_ENABLE))
 			np->rv_scntl0  &= ~0x0a;
 		np->myaddr = nvram->data.Symbios.host_id & 0x0f;
 		if (nvram->data.Symbios.flags & SYMBIOS_VERBOSE_MSGS)
 			np->verbose += 1;
 		if (nvram->data.Symbios.flags1 & SYMBIOS_SCAN_HI_LO)
 			np->usrflags |= SYM_SCAN_TARGETS_HILO;
 		if (nvram->data.Symbios.flags2 & SYMBIOS_AVOID_BUS_RESET)
 			np->usrflags |= SYM_AVOID_BUS_RESET;
 		break;
 	case SYM_TEKRAM_NVRAM:
 		np->myaddr = nvram->data.Tekram.host_id & 0x0f;
 		break;
 	default:
 		break;
 	}
 #endif
 }
 
 /*
  *  Get target setup from NVRAM.
  */
 #ifdef SYM_CONF_NVRAM_SUPPORT
 static void sym_Symbios_setup_target(hcb_p np,int target, Symbios_nvram *nvram);
 static void sym_Tekram_setup_target(hcb_p np,int target, Tekram_nvram *nvram);
 #endif
 
 static void
 sym_nvram_setup_target (hcb_p np, int target, struct sym_nvram *nvp)
 {
 #ifdef SYM_CONF_NVRAM_SUPPORT
 	switch(nvp->type) {
 	case SYM_SYMBIOS_NVRAM:
 		sym_Symbios_setup_target (np, target, &nvp->data.Symbios);
 		break;
 	case SYM_TEKRAM_NVRAM:
 		sym_Tekram_setup_target (np, target, &nvp->data.Tekram);
 		break;
 	default:
 		break;
 	}
 #endif
 }
 
 #ifdef SYM_CONF_NVRAM_SUPPORT
 /*
  *  Get target set-up from Symbios format NVRAM.
  */
 static void
 sym_Symbios_setup_target(hcb_p np, int target, Symbios_nvram *nvram)
 {
 	tcb_p tp = &np->target[target];
 	Symbios_target *tn = &nvram->target[target];
 
 	tp->tinfo.user.period = tn->sync_period ? (tn->sync_period + 3) / 4 : 0;
 	tp->tinfo.user.width  = tn->bus_width == 0x10 ? BUS_16_BIT : BUS_8_BIT;
 	tp->usrtags =
 		(tn->flags & SYMBIOS_QUEUE_TAGS_ENABLED)? SYM_SETUP_MAX_TAG : 0;
 
 	if (!(tn->flags & SYMBIOS_DISCONNECT_ENABLE))
 		tp->usrflags &= ~SYM_DISC_ENABLED;
 	if (!(tn->flags & SYMBIOS_SCAN_AT_BOOT_TIME))
 		tp->usrflags |= SYM_SCAN_BOOT_DISABLED;
 	if (!(tn->flags & SYMBIOS_SCAN_LUNS))
 		tp->usrflags |= SYM_SCAN_LUNS_DISABLED;
 }
 
 /*
  *  Get target set-up from Tekram format NVRAM.
  */
 static void
 sym_Tekram_setup_target(hcb_p np, int target, Tekram_nvram *nvram)
 {
 	tcb_p tp = &np->target[target];
 	struct Tekram_target *tn = &nvram->target[target];
 	int i;
 
 	if (tn->flags & TEKRAM_SYNC_NEGO) {
 		i = tn->sync_index & 0xf;
 		tp->tinfo.user.period = Tekram_sync[i];
 	}
 
 	tp->tinfo.user.width =
 		(tn->flags & TEKRAM_WIDE_NEGO) ? BUS_16_BIT : BUS_8_BIT;
 
 	if (tn->flags & TEKRAM_TAGGED_COMMANDS) {
 		tp->usrtags = 2 << nvram->max_tags_index;
 	}
 
 	if (tn->flags & TEKRAM_DISCONNECT_ENABLE)
 		tp->usrflags |= SYM_DISC_ENABLED;
 
 	/* If any device does not support parity, we will not use this option */
 	if (!(tn->flags & TEKRAM_PARITY_CHECK))
 		np->rv_scntl0  &= ~0x0a; /* SCSI parity checking disabled */
 }
 
 #ifdef	SYM_CONF_DEBUG_NVRAM
 /*
  *  Dump Symbios format NVRAM for debugging purpose.
  */
 static void sym_display_Symbios_nvram(hcb_p np, Symbios_nvram *nvram)
 {
 	int i;
 
 	/* display Symbios nvram host data */
 	printf("%s: HOST ID=%d%s%s%s%s%s%s\n",
 		sym_name(np), nvram->host_id & 0x0f,
 		(nvram->flags  & SYMBIOS_SCAM_ENABLE)	? " SCAM"	:"",
 		(nvram->flags  & SYMBIOS_PARITY_ENABLE)	? " PARITY"	:"",
 		(nvram->flags  & SYMBIOS_VERBOSE_MSGS)	? " VERBOSE"	:"",
 		(nvram->flags  & SYMBIOS_CHS_MAPPING)	? " CHS_ALT"	:"",
 		(nvram->flags2 & SYMBIOS_AVOID_BUS_RESET)?" NO_RESET"	:"",
 		(nvram->flags1 & SYMBIOS_SCAN_HI_LO)	? " HI_LO"	:"");
 
 	/* display Symbios nvram drive data */
 	for (i = 0 ; i < 15 ; i++) {
 		struct Symbios_target *tn = &nvram->target[i];
 		printf("%s-%d:%s%s%s%s WIDTH=%d SYNC=%d TMO=%d\n",
 		sym_name(np), i,
 		(tn->flags & SYMBIOS_DISCONNECT_ENABLE)	? " DISC"	: "",
 		(tn->flags & SYMBIOS_SCAN_AT_BOOT_TIME)	? " SCAN_BOOT"	: "",
 		(tn->flags & SYMBIOS_SCAN_LUNS)		? " SCAN_LUNS"	: "",
 		(tn->flags & SYMBIOS_QUEUE_TAGS_ENABLED)? " TCQ"	: "",
 		tn->bus_width,
 		tn->sync_period / 4,
 		tn->timeout);
 	}
 }
 
 /*
  *  Dump TEKRAM format NVRAM for debugging purpose.
  */
 static const u_char Tekram_boot_delay[7] = {3, 5, 10, 20, 30, 60, 120};
 static void sym_display_Tekram_nvram(hcb_p np, Tekram_nvram *nvram)
 {
 	int i, tags, boot_delay;
 	char *rem;
 
 	/* display Tekram nvram host data */
 	tags = 2 << nvram->max_tags_index;
 	boot_delay = 0;
 	if (nvram->boot_delay_index < 6)
 		boot_delay = Tekram_boot_delay[nvram->boot_delay_index];
 	switch((nvram->flags & TEKRAM_REMOVABLE_FLAGS) >> 6) {
 	default:
 	case 0:	rem = "";			break;
 	case 1: rem = " REMOVABLE=boot device";	break;
 	case 2: rem = " REMOVABLE=all";		break;
 	}
 
 	printf("%s: HOST ID=%d%s%s%s%s%s%s%s%s%s BOOT DELAY=%d tags=%d\n",
 		sym_name(np), nvram->host_id & 0x0f,
 		(nvram->flags1 & SYMBIOS_SCAM_ENABLE)	? " SCAM"	:"",
 		(nvram->flags & TEKRAM_MORE_THAN_2_DRIVES) ? " >2DRIVES"	:"",
 		(nvram->flags & TEKRAM_DRIVES_SUP_1GB)	? " >1GB"	:"",
 		(nvram->flags & TEKRAM_RESET_ON_POWER_ON) ? " RESET"	:"",
 		(nvram->flags & TEKRAM_ACTIVE_NEGATION)	? " ACT_NEG"	:"",
 		(nvram->flags & TEKRAM_IMMEDIATE_SEEK)	? " IMM_SEEK"	:"",
 		(nvram->flags & TEKRAM_SCAN_LUNS)	? " SCAN_LUNS"	:"",
 		(nvram->flags1 & TEKRAM_F2_F6_ENABLED)	? " F2_F6"	:"",
 		rem, boot_delay, tags);
 
 	/* display Tekram nvram drive data */
 	for (i = 0; i <= 15; i++) {
 		int sync, j;
 		struct Tekram_target *tn = &nvram->target[i];
 		j = tn->sync_index & 0xf;
 		sync = Tekram_sync[j];
 		printf("%s-%d:%s%s%s%s%s%s PERIOD=%d\n",
 		sym_name(np), i,
 		(tn->flags & TEKRAM_PARITY_CHECK)	? " PARITY"	: "",
 		(tn->flags & TEKRAM_SYNC_NEGO)		? " SYNC"	: "",
 		(tn->flags & TEKRAM_DISCONNECT_ENABLE)	? " DISC"	: "",
 		(tn->flags & TEKRAM_START_CMD)		? " START"	: "",
 		(tn->flags & TEKRAM_TAGGED_COMMANDS)	? " TCQ"	: "",
 		(tn->flags & TEKRAM_WIDE_NEGO)		? " WIDE"	: "",
 		sync);
 	}
 }
 #endif	/* SYM_CONF_DEBUG_NVRAM */
 #endif	/* SYM_CONF_NVRAM_SUPPORT */
 
 /*
  *  Try reading Symbios or Tekram NVRAM
  */
 #ifdef SYM_CONF_NVRAM_SUPPORT
 static int sym_read_Symbios_nvram (hcb_p np, Symbios_nvram *nvram);
 static int sym_read_Tekram_nvram  (hcb_p np, Tekram_nvram *nvram);
 #endif
 
 static int sym_read_nvram(hcb_p np, struct sym_nvram *nvp)
 {
 #ifdef SYM_CONF_NVRAM_SUPPORT
 	/*
 	 *  Try to read SYMBIOS nvram.
 	 *  Try to read TEKRAM nvram if Symbios nvram not found.
 	 */
 	if	(SYM_SETUP_SYMBIOS_NVRAM &&
 		 !sym_read_Symbios_nvram (np, &nvp->data.Symbios)) {
 		nvp->type = SYM_SYMBIOS_NVRAM;
 #ifdef SYM_CONF_DEBUG_NVRAM
 		sym_display_Symbios_nvram(np, &nvp->data.Symbios);
 #endif
 	}
 	else if	(SYM_SETUP_TEKRAM_NVRAM &&
 		 !sym_read_Tekram_nvram (np, &nvp->data.Tekram)) {
 		nvp->type = SYM_TEKRAM_NVRAM;
 #ifdef SYM_CONF_DEBUG_NVRAM
 		sym_display_Tekram_nvram(np, &nvp->data.Tekram);
 #endif
 	}
 	else
 		nvp->type = 0;
 #else
 	nvp->type = 0;
 #endif
 	return nvp->type;
 }
 
 #ifdef SYM_CONF_NVRAM_SUPPORT
 /*
  *  24C16 EEPROM reading.
  *
  *  GPOI0 - data in/data out
  *  GPIO1 - clock
  *  Symbios NVRAM wiring now also used by Tekram.
  */
 
 #define SET_BIT 0
 #define CLR_BIT 1
 #define SET_CLK 2
 #define CLR_CLK 3
 
 /*
  *  Set/clear data/clock bit in GPIO0
  */
 static void S24C16_set_bit(hcb_p np, u_char write_bit, u_char *gpreg,
 			  int bit_mode)
 {
 	UDELAY (5);
 	switch (bit_mode){
 	case SET_BIT:
 		*gpreg |= write_bit;
 		break;
 	case CLR_BIT:
 		*gpreg &= 0xfe;
 		break;
 	case SET_CLK:
 		*gpreg |= 0x02;
 		break;
 	case CLR_CLK:
 		*gpreg &= 0xfd;
 		break;
 
 	}
 	OUTB (nc_gpreg, *gpreg);
 	UDELAY (5);
 }
 
 /*
  *  Send START condition to NVRAM to wake it up.
  */
 static void S24C16_start(hcb_p np, u_char *gpreg)
 {
 	S24C16_set_bit(np, 1, gpreg, SET_BIT);
 	S24C16_set_bit(np, 0, gpreg, SET_CLK);
 	S24C16_set_bit(np, 0, gpreg, CLR_BIT);
 	S24C16_set_bit(np, 0, gpreg, CLR_CLK);
 }
 
 /*
  *  Send STOP condition to NVRAM - puts NVRAM to sleep... ZZzzzz!!
  */
 static void S24C16_stop(hcb_p np, u_char *gpreg)
 {
 	S24C16_set_bit(np, 0, gpreg, SET_CLK);
 	S24C16_set_bit(np, 1, gpreg, SET_BIT);
 }
 
 /*
  *  Read or write a bit to the NVRAM,
  *  read if GPIO0 input else write if GPIO0 output
  */
 static void S24C16_do_bit(hcb_p np, u_char *read_bit, u_char write_bit,
 			 u_char *gpreg)
 {
 	S24C16_set_bit(np, write_bit, gpreg, SET_BIT);
 	S24C16_set_bit(np, 0, gpreg, SET_CLK);
 	if (read_bit)
 		*read_bit = INB (nc_gpreg);
 	S24C16_set_bit(np, 0, gpreg, CLR_CLK);
 	S24C16_set_bit(np, 0, gpreg, CLR_BIT);
 }
 
 /*
  *  Output an ACK to the NVRAM after reading,
  *  change GPIO0 to output and when done back to an input
  */
 static void S24C16_write_ack(hcb_p np, u_char write_bit, u_char *gpreg,
 			    u_char *gpcntl)
 {
 	OUTB (nc_gpcntl, *gpcntl & 0xfe);
 	S24C16_do_bit(np, 0, write_bit, gpreg);
 	OUTB (nc_gpcntl, *gpcntl);
 }
 
 /*
  *  Input an ACK from NVRAM after writing,
  *  change GPIO0 to input and when done back to an output
  */
 static void S24C16_read_ack(hcb_p np, u_char *read_bit, u_char *gpreg,
 			   u_char *gpcntl)
 {
 	OUTB (nc_gpcntl, *gpcntl | 0x01);
 	S24C16_do_bit(np, read_bit, 1, gpreg);
 	OUTB (nc_gpcntl, *gpcntl);
 }
 
 /*
  *  WRITE a byte to the NVRAM and then get an ACK to see it was accepted OK,
  *  GPIO0 must already be set as an output
  */
 static void S24C16_write_byte(hcb_p np, u_char *ack_data, u_char write_data,
 			     u_char *gpreg, u_char *gpcntl)
 {
 	int x;
 
 	for (x = 0; x < 8; x++)
 		S24C16_do_bit(np, 0, (write_data >> (7 - x)) & 0x01, gpreg);
 
 	S24C16_read_ack(np, ack_data, gpreg, gpcntl);
 }
 
 /*
  *  READ a byte from the NVRAM and then send an ACK to say we have got it,
  *  GPIO0 must already be set as an input
  */
 static void S24C16_read_byte(hcb_p np, u_char *read_data, u_char ack_data,
 			    u_char *gpreg, u_char *gpcntl)
 {
 	int x;
 	u_char read_bit;
 
 	*read_data = 0;
 	for (x = 0; x < 8; x++) {
 		S24C16_do_bit(np, &read_bit, 1, gpreg);
 		*read_data |= ((read_bit & 0x01) << (7 - x));
 	}
 
 	S24C16_write_ack(np, ack_data, gpreg, gpcntl);
 }
 
 /*
  *  Read 'len' bytes starting at 'offset'.
  */
 static int sym_read_S24C16_nvram (hcb_p np, int offset, u_char *data, int len)
 {
 	u_char	gpcntl, gpreg;
 	u_char	old_gpcntl, old_gpreg;
 	u_char	ack_data;
 	int	retv = 1;
 	int	x;
 
 	/* save current state of GPCNTL and GPREG */
 	old_gpreg	= INB (nc_gpreg);
 	old_gpcntl	= INB (nc_gpcntl);
 	gpcntl		= old_gpcntl & 0x1c;
 
 	/* set up GPREG & GPCNTL to set GPIO0 and GPIO1 in to known state */
 	OUTB (nc_gpreg,  old_gpreg);
 	OUTB (nc_gpcntl, gpcntl);
 
 	/* this is to set NVRAM into a known state with GPIO0/1 both low */
 	gpreg = old_gpreg;
 	S24C16_set_bit(np, 0, &gpreg, CLR_CLK);
 	S24C16_set_bit(np, 0, &gpreg, CLR_BIT);
 
 	/* now set NVRAM inactive with GPIO0/1 both high */
 	S24C16_stop(np, &gpreg);
 
 	/* activate NVRAM */
 	S24C16_start(np, &gpreg);
 
 	/* write device code and random address MSB */
 	S24C16_write_byte(np, &ack_data,
 		0xa0 | ((offset >> 7) & 0x0e), &gpreg, &gpcntl);
 	if (ack_data & 0x01)
 		goto out;
 
 	/* write random address LSB */
 	S24C16_write_byte(np, &ack_data,
 		offset & 0xff, &gpreg, &gpcntl);
 	if (ack_data & 0x01)
 		goto out;
 
 	/* regenerate START state to set up for reading */
 	S24C16_start(np, &gpreg);
 
 	/* rewrite device code and address MSB with read bit set (lsb = 0x01) */
 	S24C16_write_byte(np, &ack_data,
 		0xa1 | ((offset >> 7) & 0x0e), &gpreg, &gpcntl);
 	if (ack_data & 0x01)
 		goto out;
 
 	/* now set up GPIO0 for inputting data */
 	gpcntl |= 0x01;
 	OUTB (nc_gpcntl, gpcntl);
 
 	/* input all requested data - only part of total NVRAM */
 	for (x = 0; x < len; x++)
 		S24C16_read_byte(np, &data[x], (x == (len-1)), &gpreg, &gpcntl);
 
 	/* finally put NVRAM back in inactive mode */
 	gpcntl &= 0xfe;
 	OUTB (nc_gpcntl, gpcntl);
 	S24C16_stop(np, &gpreg);
 	retv = 0;
 out:
 	/* return GPIO0/1 to original states after having accessed NVRAM */
 	OUTB (nc_gpcntl, old_gpcntl);
 	OUTB (nc_gpreg,  old_gpreg);
 
 	return retv;
 }
 
 #undef SET_BIT /* 0 */
 #undef CLR_BIT /* 1 */
 #undef SET_CLK /* 2 */
 #undef CLR_CLK /* 3 */
 
 /*
  *  Try reading Symbios NVRAM.
  *  Return 0 if OK.
  */
 static int sym_read_Symbios_nvram (hcb_p np, Symbios_nvram *nvram)
 {
 	static u_char Symbios_trailer[6] = {0xfe, 0xfe, 0, 0, 0, 0};
 	u_char *data = (u_char *) nvram;
 	int len  = sizeof(*nvram);
 	u_short	csum;
 	int x;
 
 	/* probe the 24c16 and read the SYMBIOS 24c16 area */
 	if (sym_read_S24C16_nvram (np, SYMBIOS_NVRAM_ADDRESS, data, len))
 		return 1;
 
 	/* check valid NVRAM signature, verify byte count and checksum */
 	if (nvram->type != 0 ||
 	    bcmp(nvram->trailer, Symbios_trailer, 6) ||
 	    nvram->byte_count != len - 12)
 		return 1;
 
 	/* verify checksum */
 	for (x = 6, csum = 0; x < len - 6; x++)
 		csum += data[x];
 	if (csum != nvram->checksum)
 		return 1;
 
 	return 0;
 }
 
 /*
  *  93C46 EEPROM reading.
  *
  *  GPOI0 - data in
  *  GPIO1 - data out
  *  GPIO2 - clock
  *  GPIO4 - chip select
  *
  *  Used by Tekram.
  */
 
 /*
  *  Pulse clock bit in GPIO0
  */
 static void T93C46_Clk(hcb_p np, u_char *gpreg)
 {
 	OUTB (nc_gpreg, *gpreg | 0x04);
 	UDELAY (2);
 	OUTB (nc_gpreg, *gpreg);
 }
 
 /*
  *  Read bit from NVRAM
  */
 static void T93C46_Read_Bit(hcb_p np, u_char *read_bit, u_char *gpreg)
 {
 	UDELAY (2);
 	T93C46_Clk(np, gpreg);
 	*read_bit = INB (nc_gpreg);
 }
 
 /*
  *  Write bit to GPIO0
  */
 static void T93C46_Write_Bit(hcb_p np, u_char write_bit, u_char *gpreg)
 {
 	if (write_bit & 0x01)
 		*gpreg |= 0x02;
 	else
 		*gpreg &= 0xfd;
 
 	*gpreg |= 0x10;
 
 	OUTB (nc_gpreg, *gpreg);
 	UDELAY (2);
 
 	T93C46_Clk(np, gpreg);
 }
 
 /*
  *  Send STOP condition to NVRAM - puts NVRAM to sleep... ZZZzzz!!
  */
 static void T93C46_Stop(hcb_p np, u_char *gpreg)
 {
 	*gpreg &= 0xef;
 	OUTB (nc_gpreg, *gpreg);
 	UDELAY (2);
 
 	T93C46_Clk(np, gpreg);
 }
 
 /*
  *  Send read command and address to NVRAM
  */
 static void T93C46_Send_Command(hcb_p np, u_short write_data,
 				u_char *read_bit, u_char *gpreg)
 {
 	int x;
 
 	/* send 9 bits, start bit (1), command (2), address (6)  */
 	for (x = 0; x < 9; x++)
 		T93C46_Write_Bit(np, (u_char) (write_data >> (8 - x)), gpreg);
 
 	*read_bit = INB (nc_gpreg);
 }
 
 /*
  *  READ 2 bytes from the NVRAM
  */
 static void T93C46_Read_Word(hcb_p np, u_short *nvram_data, u_char *gpreg)
 {
 	int x;
 	u_char read_bit;
 
 	*nvram_data = 0;
 	for (x = 0; x < 16; x++) {
 		T93C46_Read_Bit(np, &read_bit, gpreg);
 
 		if (read_bit & 0x01)
 			*nvram_data |=  (0x01 << (15 - x));
 		else
 			*nvram_data &= ~(0x01 << (15 - x));
 	}
 }
 
 /*
  *  Read Tekram NvRAM data.
  */
 static int T93C46_Read_Data(hcb_p np, u_short *data,int len,u_char *gpreg)
 {
 	u_char	read_bit;
 	int	x;
 
 	for (x = 0; x < len; x++)  {
 
 		/* output read command and address */
 		T93C46_Send_Command(np, 0x180 | x, &read_bit, gpreg);
 		if (read_bit & 0x01)
 			return 1; /* Bad */
 		T93C46_Read_Word(np, &data[x], gpreg);
 		T93C46_Stop(np, gpreg);
 	}
 
 	return 0;
 }
 
 /*
  *  Try reading 93C46 Tekram NVRAM.
  */
 static int sym_read_T93C46_nvram (hcb_p np, Tekram_nvram *nvram)
 {
 	u_char gpcntl, gpreg;
 	u_char old_gpcntl, old_gpreg;
 	int retv = 1;
 
 	/* save current state of GPCNTL and GPREG */
 	old_gpreg	= INB (nc_gpreg);
 	old_gpcntl	= INB (nc_gpcntl);
 
 	/* set up GPREG & GPCNTL to set GPIO0/1/2/4 in to known state, 0 in,
 	   1/2/4 out */
 	gpreg = old_gpreg & 0xe9;
 	OUTB (nc_gpreg, gpreg);
 	gpcntl = (old_gpcntl & 0xe9) | 0x09;
 	OUTB (nc_gpcntl, gpcntl);
 
 	/* input all of NVRAM, 64 words */
 	retv = T93C46_Read_Data(np, (u_short *) nvram,
 				sizeof(*nvram) / sizeof(short), &gpreg);
 
 	/* return GPIO0/1/2/4 to original states after having accessed NVRAM */
 	OUTB (nc_gpcntl, old_gpcntl);
 	OUTB (nc_gpreg,  old_gpreg);
 
 	return retv;
 }
 
 /*
  *  Try reading Tekram NVRAM.
  *  Return 0 if OK.
  */
 static int sym_read_Tekram_nvram (hcb_p np, Tekram_nvram *nvram)
 {
 	u_char *data = (u_char *) nvram;
 	int len = sizeof(*nvram);
 	u_short	csum;
 	int x;
 
 	switch (np->device_id) {
 	case PCI_ID_SYM53C885:
 	case PCI_ID_SYM53C895:
 	case PCI_ID_SYM53C896:
 		x = sym_read_S24C16_nvram(np, TEKRAM_24C16_NVRAM_ADDRESS,
 					  data, len);
 		break;
 	case PCI_ID_SYM53C875:
 		x = sym_read_S24C16_nvram(np, TEKRAM_24C16_NVRAM_ADDRESS,
 					  data, len);
 		if (!x)
 			break;
 	default:
 		x = sym_read_T93C46_nvram(np, nvram);
 		break;
 	}
 	if (x)
 		return 1;
 
 	/* verify checksum */
 	for (x = 0, csum = 0; x < len - 1; x += 2)
 		csum += data[x] + (data[x+1] << 8);
 	if (csum != 0x1234)
 		return 1;
 
 	return 0;
 }
 
 #endif	/* SYM_CONF_NVRAM_SUPPORT */
Index: projects/release-debugdist/sys/dev/trm/trm.c
===================================================================
--- projects/release-debugdist/sys/dev/trm/trm.c	(revision 267343)
+++ projects/release-debugdist/sys/dev/trm/trm.c	(revision 267344)
@@ -1,3714 +1,3707 @@
 /*
  *      O.S   : FreeBSD CAM
  *	FILE NAME  : trm.c					      
  *	     BY    : C.L. Huang 	(ching@tekram.com.tw)
  *                   Erich Chen     (erich@tekram.com.tw)
  *      Description: Device Driver for Tekram SCSI adapters
  *                   DC395U/UW/F ,DC315/U(TRM-S1040)
  *                   DC395U2D/U2W(TRM-S2080)
  *                   PCI SCSI Bus Master Host Adapter	
  *                   (SCSI chip set used Tekram ASIC TRM-S1040,TRM-S2080)
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 /*
  *	HISTORY:					
  *						
  *	REV#	DATE	NAME    	DESCRIPTION	
  *  1.05   05/01/1999  ERICH CHEN  First released for 3.x.x (CAM)
  *  1.06   07/29/1999  ERICH CHEN  Modify for NEW PCI
  *  1.07   12/12/1999  ERICH CHEN  Modify for 3.3.x ,DCB no free
  *  1.08   06/12/2000  ERICH CHEN  Modify for 4.x.x 
  *  1.09   11/03/2000  ERICH CHEN  Modify for 4.1.R ,new sim
  *  1.10   10/10/2001  Oscar Feng  Fixed CAM rescan hang up bug.	
  *  1.11   10/13/2001  Oscar Feng  Fixed wrong Async speed display bug.	
  */
 
 /*-
  * (C)Copyright 1995-2001 Tekram Technology Co.,Ltd.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  * 3. The name of the author may not be used to endorse or promote products
  *    derived from this software without specific prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
  */
 
 /*
  * Imported into FreeBSD source repository, and updated to compile under  
  * FreeBSD-3.0-DEVELOPMENT, by Stefan Esser <se@FreeBSD.Org>, 1996-12-17  
  */
 
 /*
  * Updated to compile under FreeBSD 5.0-CURRENT by Olivier Houchard
  * <doginou@ci0.org>, 2002-03-04
  */
 
 #include <sys/param.h>
 
 #include <sys/systm.h>
 #include <sys/malloc.h>
 #include <sys/queue.h>
 #if __FreeBSD_version >= 500000
 #include <sys/bio.h>
 #endif
 #include <sys/buf.h>
 #include <sys/bus.h>
 #include <sys/kernel.h>
 #include <sys/module.h>
 
 #include <vm/vm.h>
 #include <vm/pmap.h>
 
 #include <dev/pci/pcivar.h>
 #include <dev/pci/pcireg.h>
 #include <machine/resource.h>
 #include <machine/bus.h>
 #include <sys/rman.h>
 
 #include <cam/cam.h>
 #include <cam/cam_ccb.h>
 #include <cam/cam_sim.h>
 #include <cam/cam_xpt_sim.h>
 #include <cam/cam_debug.h>
 
 #include <cam/scsi/scsi_all.h>
 #include <cam/scsi/scsi_message.h>
 
 #include <dev/trm/trm.h>
 
 #define trm_reg_read8(reg)	bus_space_read_1(pACB->tag, pACB->bsh, reg)
 #define trm_reg_read16(reg)	bus_space_read_2(pACB->tag, pACB->bsh, reg)
 #define trm_reg_read32(reg)	bus_space_read_4(pACB->tag, pACB->bsh, reg)
 #define trm_reg_write8(value,reg)	bus_space_write_1(pACB->tag, pACB->bsh,\
 		reg, value)
 #define trm_reg_write16(value,reg)	bus_space_write_2(pACB->tag, pACB->bsh,\
 		reg, value)
 #define trm_reg_write32(value,reg)	bus_space_write_4(pACB->tag, pACB->bsh,\
 		reg, value)
 
 #define PCI_Vendor_ID_TEKRAM	0x1DE1
 #define PCI_Device_ID_TRM_S1040	0x0391
 #define PCI_DEVICEID_TRMS1040	0x03911DE1
 #define PCI_DEVICEID_TRMS2080   0x03921DE1
 
 #ifdef trm_DEBUG1
 #define TRM_DPRINTF(fmt, arg...) printf("trm: " fmt, ##arg)
 #else
 #define TRM_DPRINTF(fmt, arg...) {}
 #endif /* TRM_DEBUG */
 
 static void	trm_check_eeprom(PNVRAMTYPE pEEpromBuf,PACB pACB);
 static void	NVRAM_trm_read_all(PNVRAMTYPE pEEpromBuf,PACB pACB);
 static u_int8_t	NVRAM_trm_get_data(PACB pACB, u_int8_t bAddr);
 static void	NVRAM_trm_write_all(PNVRAMTYPE pEEpromBuf,PACB pACB);
 static void	NVRAM_trm_set_data(PACB pACB, u_int8_t bAddr, u_int8_t bData);
 static void	NVRAM_trm_write_cmd(PACB pACB, u_int8_t bCmd, u_int8_t bAddr);
 static void	NVRAM_trm_wait_30us(PACB pACB);
 
 static void	trm_Interrupt(void *vpACB);
 static void	trm_DataOutPhase0(PACB pACB, PSRB pSRB, 
 					 u_int16_t * pscsi_status);
 static void	trm_DataInPhase0(PACB pACB, PSRB pSRB, 
 					u_int16_t * pscsi_status);
 static void	trm_CommandPhase0(PACB pACB, PSRB pSRB, 
 					 u_int16_t * pscsi_status);
 static void	trm_StatusPhase0(PACB pACB, PSRB pSRB, 
 					u_int16_t * pscsi_status);
 static void	trm_MsgOutPhase0(PACB pACB, PSRB pSRB, 
 					u_int16_t * pscsi_status);
 static void	trm_MsgInPhase0(PACB pACB, PSRB pSRB, 
 					u_int16_t * pscsi_status);
 static void	trm_DataOutPhase1(PACB pACB, PSRB pSRB, 
 					 u_int16_t * pscsi_status);
 static void	trm_DataInPhase1(PACB pACB, PSRB pSRB, 
 					u_int16_t * pscsi_status);
 static void	trm_CommandPhase1(PACB pACB, PSRB pSRB, 
 					 u_int16_t * pscsi_status);
 static void	trm_StatusPhase1(PACB pACB, PSRB pSRB, 
 					u_int16_t * pscsi_status);
 static void	trm_MsgOutPhase1(PACB pACB, PSRB pSRB, 
 					u_int16_t * pscsi_status);
 static void	trm_MsgInPhase1(PACB pACB, PSRB pSRB, 
 					u_int16_t * pscsi_status);
 static void	trm_Nop0(PACB pACB, PSRB pSRB, u_int16_t * pscsi_status);
 static void	trm_Nop1(PACB pACB, PSRB pSRB, u_int16_t * pscsi_status);
 static void	trm_SetXferRate(PACB pACB, PSRB pSRB,PDCB pDCB);
 static void	trm_DataIO_transfer(PACB pACB, PSRB pSRB, u_int16_t ioDir);
 static void	trm_Disconnect(PACB pACB);
 static void	trm_Reselect(PACB pACB);
 static void	trm_SRBdone(PACB pACB, PDCB pDCB, PSRB pSRB);
 static void	trm_DoingSRB_Done(PACB pACB);
 static void	trm_ScsiRstDetect(PACB pACB);
 static void	trm_ResetSCSIBus(PACB pACB);
 static void	trm_RequestSense(PACB pACB, PDCB pDCB, PSRB pSRB);
 static void	trm_EnableMsgOutAbort2(PACB pACB, PSRB pSRB);
 static void	trm_EnableMsgOutAbort1(PACB pACB, PSRB pSRB);
 static void	trm_SendSRB(PACB pACB, PSRB pSRB);
 static int	trm_probe(device_t tag);
 static int	trm_attach(device_t tag);
 static void	trm_reset(PACB pACB);
 
 static u_int16_t	trm_StartSCSI(PACB pACB, PDCB pDCB, PSRB pSRB);
 
 static int	trm_initAdapter(PACB pACB, u_int16_t unit); 
 static void	trm_initDCB(PACB pACB, PDCB pDCB, u_int16_t unit, 
     					u_int32_t i, u_int32_t j);
 static int	trm_initSRB(PACB pACB);
 static void	trm_initACB(PACB pACB, u_int8_t adaptType, u_int16_t unit);
 /* CAM SIM entry points */
 #define ccb_trmsrb_ptr spriv_ptr0
 #define ccb_trmacb_ptr spriv_ptr1
 static void	trm_action(struct cam_sim *psim, union ccb *pccb);
 static void	trm_poll(struct cam_sim *psim);
 
 
 static void * trm_SCSI_phase0[] = {
 	trm_DataOutPhase0,    /* phase:0 */
 	trm_DataInPhase0,     /* phase:1 */
 	trm_CommandPhase0,    /* phase:2 */
 	trm_StatusPhase0,     /* phase:3 */
 	trm_Nop0,             /* phase:4 */
 	trm_Nop1,             /* phase:5 */
 	trm_MsgOutPhase0,     /* phase:6 */
 	trm_MsgInPhase0,      /* phase:7 */
 };
 
 /*
  *
  *          stateV = (void *) trm_SCSI_phase1[phase]
  *
  */
 static void * trm_SCSI_phase1[] = {
 	trm_DataOutPhase1,    /* phase:0 */
 	trm_DataInPhase1,     /* phase:1 */
 	trm_CommandPhase1,    /* phase:2 */
 	trm_StatusPhase1,     /* phase:3 */
 	trm_Nop0,             /* phase:4 */
 	trm_Nop1,             /* phase:5 */
 	trm_MsgOutPhase1,     /* phase:6 */
 	trm_MsgInPhase1,      /* phase:7 */
 };
 
 
 NVRAMTYPE trm_eepromBuf[TRM_MAX_ADAPTER_NUM];
 /*
  *Fast20:  000	 50ns, 20.0 Mbytes/s
  *	       001	 75ns, 13.3 Mbytes/s
  *	       010	100ns, 10.0 Mbytes/s
  *	       011	125ns,  8.0 Mbytes/s
  *	       100	150ns,  6.6 Mbytes/s
  *	       101	175ns,  5.7 Mbytes/s
  *	       110	200ns,  5.0 Mbytes/s
  *	       111	250ns,  4.0 Mbytes/s
  *
  *Fast40:  000	 25ns, 40.0 Mbytes/s
  *	       001	 50ns, 20.0 Mbytes/s
  *	       010	 75ns, 13.3 Mbytes/s
  *	       011	100ns, 10.0 Mbytes/s
  *	       100	125ns,  8.0 Mbytes/s
  *	       101	150ns,  6.6 Mbytes/s
  *	       110	175ns,  5.7 Mbytes/s
  *	       111	200ns,  5.0 Mbytes/s
  */
                                              /* real period: */
 u_int8_t dc395x_clock_period[] = {
 	12,/*  48  ns 20   MB/sec */
 	18,/*  72  ns 13.3 MB/sec */
 	25,/* 100  ns 10.0 MB/sec */
 	31,/* 124  ns  8.0 MB/sec */
 	37,/* 148  ns  6.6 MB/sec */
 	43,/* 172  ns  5.7 MB/sec */
 	50,/* 200  ns  5.0 MB/sec */
 	62 /* 248  ns  4.0 MB/sec */
 };
 
 u_int8_t dc395u2x_clock_period[]={
 	10,/*  25  ns 40.0 MB/sec */
 	12,/*  48  ns 20.0 MB/sec */
 	18,/*  72  ns 13.3 MB/sec */
 	25,/* 100  ns 10.0 MB/sec */
 	31,/* 124  ns  8.0 MB/sec */
 	37,/* 148  ns  6.6 MB/sec */
 	43,/* 172  ns  5.7 MB/sec */
 	50,/* 200  ns  5.0 MB/sec */
 };
 
 #define  dc395x_tinfo_period           dc395x_clock_period
 #define  dc395u2x_tinfo_period         dc395u2x_clock_period
 
 static PSRB
 trm_GetSRB(PACB pACB)
 {
 	int	intflag;
 	PSRB	pSRB;
 
 	intflag = splcam();
     	pSRB = pACB->pFreeSRB;
 	if (pSRB) {
 		pACB->pFreeSRB = pSRB->pNextSRB;
 		pSRB->pNextSRB = NULL;
 	}
 	splx(intflag);
     	return (pSRB);
 }
 
 static void
 trm_RewaitSRB0(PDCB pDCB, PSRB pSRB)
 {
 	PSRB	psrb1;
 	int	intflag;
 
 	intflag = splcam();
     	if ((psrb1 = pDCB->pWaitingSRB)) {
 		pSRB->pNextSRB = psrb1;
 		pDCB->pWaitingSRB = pSRB;
 	} else {
 	  	pSRB->pNextSRB = NULL;
 		pDCB->pWaitingSRB = pSRB;
 		pDCB->pWaitingLastSRB = pSRB;
 	}
 	splx(intflag);
 }
 
 static void
 trm_RewaitSRB(PDCB pDCB, PSRB pSRB)
 {
 	PSRB	psrb1;
 	int	intflag;
 
 	intflag = splcam();
     	pDCB->GoingSRBCnt--;
 	psrb1 = pDCB->pGoingSRB;
 	if (pSRB == psrb1)
 		/*
 		 * if this SRB is GoingSRB
 		 * remove this SRB from GoingSRB Q
 		 */
 		pDCB->pGoingSRB = psrb1->pNextSRB;
 	else {
 		/*
 		 * if this SRB is not current GoingSRB
 		 * remove this SRB from GoingSRB Q
 		 */
 		while (pSRB != psrb1->pNextSRB)
 			psrb1 = psrb1->pNextSRB;
 		psrb1->pNextSRB = pSRB->pNextSRB;
 		if (pSRB == pDCB->pGoingLastSRB)
 			pDCB->pGoingLastSRB = psrb1;
 	}
 	if ((psrb1 = pDCB->pWaitingSRB)) {
 		/*
 		 * if WaitingSRB Q is not NULL
 		 * Q back this SRB into WaitingSRB
 		 */
 
 		pSRB->pNextSRB = psrb1;
 		pDCB->pWaitingSRB = pSRB;
 	} else {
 		pSRB->pNextSRB = NULL;
 		pDCB->pWaitingSRB = pSRB;
 		pDCB->pWaitingLastSRB = pSRB;
 	}
 	splx(intflag);
 }
 
 static void
 trm_DoWaitingSRB(PACB pACB)
 {
 	int	intflag;
 	PDCB	ptr, ptr1;
 	PSRB	pSRB;
 
 	intflag = splcam();
     	if (!(pACB->pActiveDCB) && 
 	    !(pACB->ACBFlag & (RESET_DETECT+RESET_DONE+RESET_DEV))) {
 		ptr = pACB->pDCBRunRobin;
 		if (!ptr) {
 			ptr = pACB->pLinkDCB;
 			pACB->pDCBRunRobin = ptr;
 		}
 		ptr1 = ptr;
 		for (;ptr1 ;) {
 			pACB->pDCBRunRobin = ptr1->pNextDCB;
 			if (!(ptr1->MaxActiveCommandCnt > ptr1->GoingSRBCnt) 
 			    || !(pSRB = ptr1->pWaitingSRB)) {
 				if (pACB->pDCBRunRobin == ptr)
 					break;
 				ptr1 = ptr1->pNextDCB;
 			} else {
 				if (!trm_StartSCSI(pACB, ptr1, pSRB)) {
 				/* 
 				 * If trm_StartSCSI return 0 :
 				 * current interrupt status is interrupt enable 
 				 * It's said that SCSI processor is unoccupied 
 				 */
 					ptr1->GoingSRBCnt++;
 					if (ptr1->pWaitingLastSRB == pSRB) {
 						ptr1->pWaitingSRB = NULL;
 						ptr1->pWaitingLastSRB = NULL;
 					} else
 						ptr1->pWaitingSRB = pSRB->pNextSRB;
 					pSRB->pNextSRB = NULL;
 					if (ptr1->pGoingSRB) 
 						ptr1->pGoingLastSRB->pNextSRB = pSRB;
 					else
 						ptr1->pGoingSRB = pSRB;
 					ptr1->pGoingLastSRB = pSRB;
 				}
 				break;
 			}
 		}
 	}
 	splx(intflag);
 	return;
 }
 
 static void
 trm_SRBwaiting(PDCB pDCB, PSRB pSRB)
 {
   
 	if (pDCB->pWaitingSRB) {
 		pDCB->pWaitingLastSRB->pNextSRB = pSRB;
 		pDCB->pWaitingLastSRB = pSRB;
 		pSRB->pNextSRB = NULL;
 	} else {
 		pDCB->pWaitingSRB = pSRB;
 		pDCB->pWaitingLastSRB = pSRB;
 	}
 }
 
 static u_int32_t
 trm_get_sense_bufaddr(PACB pACB, PSRB pSRB)
 {
 	int offset;
 
 	offset = pSRB->TagNumber;
 	return (pACB->sense_busaddr + 
 	    (offset * sizeof(struct scsi_sense_data)));
 }
 
 static struct scsi_sense_data *
 trm_get_sense_buf(PACB pACB, PSRB pSRB)
 {
 	int offset;
 
 	offset = pSRB->TagNumber;
 	return (&pACB->sense_buffers[offset]);
 }
 static void
 trm_ExecuteSRB(void *arg, bus_dma_segment_t *dm_segs, int nseg, int error)
 {
 	int		flags;
 	PACB		pACB;
 	PSRB		pSRB;
 	union ccb	*ccb;
 	u_long		totalxferlen=0;
 
 	flags = splcam();
 	pSRB = (PSRB)arg;
 	ccb = pSRB->pccb;
 	pACB = (PACB)ccb->ccb_h.ccb_trmacb_ptr;
 	TRM_DPRINTF("trm_ExecuteSRB..........\n");        
 	if (nseg != 0) {
 		PSEG			psg;
 		bus_dma_segment_t	*end_seg;
 		int			op;
 
 		/* Copy the segments into our SG list */
 		end_seg = dm_segs + nseg;
 		psg = pSRB->pSRBSGL;
 		while (dm_segs < end_seg) {
 			psg->address = dm_segs->ds_addr;
 			psg->length = (u_long)dm_segs->ds_len;
 			totalxferlen += dm_segs->ds_len;
 			psg++;
 			dm_segs++;
 		}
 		if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
 			op = BUS_DMASYNC_PREREAD;
 		} else {
 			op = BUS_DMASYNC_PREWRITE;
 		}
 		bus_dmamap_sync(pACB->buffer_dmat, pSRB->dmamap, op);
 	}
 	pSRB->RetryCnt = 0;
 	pSRB->SRBTotalXferLength = totalxferlen;
 	pSRB->SRBSGCount = nseg;
 	pSRB->SRBSGIndex = 0;
 	pSRB->AdaptStatus = 0;
 	pSRB->TargetStatus = 0;
 	pSRB->MsgCnt = 0;
 	pSRB->SRBStatus = 0;
 	pSRB->SRBFlag = 0;
 	pSRB->SRBState = 0;
 	pSRB->ScsiPhase = PH_BUS_FREE; /* SCSI bus free Phase */
 
 	if (ccb->ccb_h.status != CAM_REQ_INPROG) {
 		if (nseg != 0)
 			bus_dmamap_unload(pACB->buffer_dmat, pSRB->dmamap);
 		pSRB->pNextSRB = pACB->pFreeSRB;
 		pACB->pFreeSRB = pSRB;
 		xpt_done(ccb);
 		splx(flags);
 		return;
 	}
 	ccb->ccb_h.status |= CAM_SIM_QUEUED;
 #if 0
 	/* XXX Need a timeout handler */
 	ccb->ccb_h.timeout_ch = timeout(trmtimeout, (caddr_t)srb, (ccb->ccb_h.timeout * hz) / 1000);
 #endif
 	trm_SendSRB(pACB, pSRB);
 	splx(flags);
 	return;
 }
 
 static void
 trm_SendSRB(PACB pACB, PSRB pSRB)
 {
 	PDCB	pDCB;
 
 	pDCB = pSRB->pSRBDCB;
 	if (!(pDCB->MaxActiveCommandCnt > pDCB->GoingSRBCnt) || (pACB->pActiveDCB)
 	    || (pACB->ACBFlag & (RESET_DETECT+RESET_DONE+RESET_DEV))) {
 		TRM_DPRINTF("pDCB->MaxCommand=%d \n",pDCB->MaxActiveCommandCnt);        
 		TRM_DPRINTF("pDCB->GoingSRBCnt=%d \n",pDCB->GoingSRBCnt);
 		TRM_DPRINTF("pACB->pActiveDCB=%8x \n",(u_int)pACB->pActiveDCB);
 		TRM_DPRINTF("pACB->ACBFlag=%x \n",pACB->ACBFlag);
 	    	trm_SRBwaiting(pDCB, pSRB);
 		goto SND_EXIT;
 	}
 
 	if (pDCB->pWaitingSRB) {
 		trm_SRBwaiting(pDCB, pSRB);
 		pSRB = pDCB->pWaitingSRB;
 		pDCB->pWaitingSRB = pSRB->pNextSRB;
 		pSRB->pNextSRB = NULL;
 	}
 
 	if (!trm_StartSCSI(pACB, pDCB, pSRB)) { 
 	/* 
 	 * If trm_StartSCSI return 0 :
 	 * current interrupt status is interrupt enable 
 	 * It's said that SCSI processor is unoccupied 
 	 */
 		pDCB->GoingSRBCnt++; /* stack waiting SRB*/
 		if (pDCB->pGoingSRB) {
 			pDCB->pGoingLastSRB->pNextSRB = pSRB;
 			pDCB->pGoingLastSRB = pSRB;
 		} else {
 			pDCB->pGoingSRB = pSRB;
 			pDCB->pGoingLastSRB = pSRB;
 		}
 	} else {
 	/* 
 	 * If trm_StartSCSI return 1 :
 	 * current interrupt status is interrupt disreenable 
 	 * It's said that SCSI processor has more one SRB need to do
 	 */
 		trm_RewaitSRB0(pDCB, pSRB);
 	}
 SND_EXIT:
 	return;
 }
 
 
 static void
 trm_action(struct cam_sim *psim, union ccb *pccb) 
 {
 	PACB	pACB;
 	int	actionflags;
 	u_int	target_id,target_lun;
 
 	CAM_DEBUG(pccb->ccb_h.path, CAM_DEBUG_TRACE, ("trm_action\n"));
 
 	actionflags = splcam();
 	pACB = (PACB) cam_sim_softc(psim);
     	target_id  = pccb->ccb_h.target_id;
 	target_lun = pccb->ccb_h.target_lun;
 
 	switch (pccb->ccb_h.func_code) {
 		case XPT_NOOP:	        	
 			TRM_DPRINTF(" XPT_NOOP \n");
 			pccb->ccb_h.status = CAM_REQ_INVALID;
 			xpt_done(pccb);
 			break;
 		/*
 		 * Execute the requested I/O operation 
 	 	 */
 		case XPT_SCSI_IO: {
 			PDCB			pDCB = NULL;
 			PSRB			pSRB;
 			struct ccb_scsiio	*pcsio;
 			int			error;
      
 			pcsio = &pccb->csio;
 			TRM_DPRINTF(" XPT_SCSI_IO \n");
 			TRM_DPRINTF("trm: target_id= %d target_lun= %d \n"
 			     ,target_id, target_lun);
 			TRM_DPRINTF(
 			    "pACB->scan_devices[target_id][target_lun]= %d \n"
 			    ,pACB->scan_devices[target_id][target_lun]);
 			if ((pccb->ccb_h.status & CAM_STATUS_MASK) !=
 			    CAM_REQ_INPROG) {
 				xpt_done(pccb);
 				splx(actionflags);
 				return;
 			}
 			pDCB = &pACB->DCBarray[target_id][target_lun];
 			if (!(pDCB->DCBstatus & DS_IN_QUEUE)) {
 				pACB->scan_devices[target_id][target_lun] = 1;
 				trm_initDCB(pACB, pDCB, pACB->AdapterUnit, 
 				    target_id, target_lun); 
 			}
 			/*
 			 * Assign an SRB and connect it with this ccb.
 			 */
 			pSRB = trm_GetSRB(pACB);
 			if (!pSRB) {
 				/* Freeze SIMQ */
 				pccb->ccb_h.status = CAM_RESRC_UNAVAIL;
 				xpt_done(pccb);
 				splx(actionflags);
 				return;
 			}
 	    		pSRB->pSRBDCB = pDCB;
 	    		pccb->ccb_h.ccb_trmsrb_ptr = pSRB;
 	    		pccb->ccb_h.ccb_trmacb_ptr = pACB;
 		    	pSRB->pccb = pccb;
 			pSRB->ScsiCmdLen = pcsio->cdb_len;
 			/* 
 			 * move layer of CAM command block to layer of SCSI
 			 * Request Block for SCSI processor command doing
 			 */
 			if ((pccb->ccb_h.flags & CAM_CDB_POINTER) != 0) {
 				if ((pccb->ccb_h.flags & CAM_CDB_PHYS) == 0) {
 					bcopy(pcsio->cdb_io.cdb_ptr,pSRB->CmdBlock
 					    ,pcsio->cdb_len);
 				} else {
 					pccb->ccb_h.status = CAM_REQ_INVALID;
 					pSRB->pNextSRB = pACB->pFreeSRB;
 					pACB->pFreeSRB=  pSRB;
 					xpt_done(pccb);
 					splx(actionflags);
 					return;
 				}		
 			} else
 				bcopy(pcsio->cdb_io.cdb_bytes,
 				    pSRB->CmdBlock, pcsio->cdb_len);
 			error = bus_dmamap_load_ccb(pACB->buffer_dmat,
 						    pSRB->dmamap,
 						    pccb,
 						    trm_ExecuteSRB,
 						    pSRB,
 						    0);
 			if (error == EINPROGRESS) {
 				xpt_freeze_simq(pACB->psim, 1);
 				pccb->ccb_h.status |= CAM_RELEASE_SIMQ;
 			}
 			break;
 		}
 		case XPT_GDEV_TYPE:		    
 			TRM_DPRINTF(" XPT_GDEV_TYPE \n");
 	    		pccb->ccb_h.status = CAM_REQ_INVALID;
 			xpt_done(pccb);
 			break;
 		case XPT_GDEVLIST:		    
 			TRM_DPRINTF(" XPT_GDEVLIST \n");
 			pccb->ccb_h.status = CAM_REQ_INVALID;
 			xpt_done(pccb);
 			break;
 		/*
 		 * Path routing inquiry 
 	       	 * Path Inquiry CCB 
 		 */
 		case XPT_PATH_INQ: {
 			struct ccb_pathinq *cpi = &pccb->cpi;
 
 			TRM_DPRINTF(" XPT_PATH_INQ \n");
 			cpi->version_num = 1; 
 			cpi->hba_inquiry = PI_SDTR_ABLE|PI_TAG_ABLE|PI_WIDE_16;
 			cpi->target_sprt = 0;
 			cpi->hba_misc = 0;
 			cpi->hba_eng_cnt = 0;
 			cpi->max_target = 15 ; 
 			cpi->max_lun = pACB->max_lun;        /* 7 or 0 */
 			cpi->initiator_id = pACB->AdaptSCSIID;
 			cpi->bus_id = cam_sim_bus(psim);
 			cpi->base_transfer_speed = 3300;
 			strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
 			strncpy(cpi->hba_vid, "Tekram_TRM", HBA_IDLEN);
 			strncpy(cpi->dev_name, cam_sim_name(psim), DEV_IDLEN);
 			cpi->unit_number = cam_sim_unit(psim);
 			cpi->transport = XPORT_SPI;
 			cpi->transport_version = 2;
 			cpi->protocol = PROTO_SCSI;
 			cpi->protocol_version = SCSI_REV_2;
 			cpi->ccb_h.status = CAM_REQ_CMP;
 			xpt_done(pccb);
 				   }
 			break;
 		/*
 		 * Release a frozen SIM queue 
 		 * Release SIM Queue 
 		 */
 		case XPT_REL_SIMQ:		    
 			TRM_DPRINTF(" XPT_REL_SIMQ \n");
 			pccb->ccb_h.status = CAM_REQ_INVALID;
 			xpt_done(pccb);
 			break;
 		/*
 		 * Set Asynchronous Callback Parameters 
 		 * Set Asynchronous Callback CCB
  		 */
 		case XPT_SASYNC_CB:		    
 			TRM_DPRINTF(" XPT_SASYNC_CB \n");
 			pccb->ccb_h.status = CAM_REQ_INVALID;
 			xpt_done(pccb);
 			break;
 		/*
 		 * Set device type information 
 		 * Set Device Type CCB 
  		 */
 		case XPT_SDEV_TYPE:		    
 			TRM_DPRINTF(" XPT_SDEV_TYPE \n");
 			pccb->ccb_h.status = CAM_REQ_INVALID;
 			xpt_done(pccb);
 			break;
 		/*
 		 * Get EDT entries matching the given pattern 
  		 */
 		case XPT_DEV_MATCH:	    	
 			TRM_DPRINTF(" XPT_DEV_MATCH \n");
 	    		pccb->ccb_h.status = CAM_REQ_INVALID;
 			xpt_done(pccb);
 			break;
 		/*
 		 * Turn on debugging for a bus, target or lun 
       		 */
 		case XPT_DEBUG:	    	    
 			TRM_DPRINTF(" XPT_DEBUG \n");
 			pccb->ccb_h.status = CAM_REQ_INVALID;
 			xpt_done(pccb);
 			break;
 			/*
 			 * XPT_ABORT = 0x10, Abort the specified CCB 
 			 * Abort XPT request CCB 
 			 */
 		case XPT_ABORT:             
 			TRM_DPRINTF(" XPT_ABORT \n");
 			pccb->ccb_h.status = CAM_REQ_INVALID;
 			xpt_done(pccb);
 			break;
 		/*
 		 * Reset the specified SCSI bus 
 		 * Reset SCSI Bus CCB 
  		 */
 		case XPT_RESET_BUS: {		
 			int i;
 
 			TRM_DPRINTF(" XPT_RESET_BUS \n");
 	    		trm_reset(pACB);
 			pACB->ACBFlag=0;
 			for (i=0; i<500; i++)
 				DELAY(1000);
 			pccb->ccb_h.status = CAM_REQ_CMP;
 			xpt_done(pccb);
 				    }
 			break;
 		/*
 		 * Bus Device Reset the specified SCSI device 
 		 * Reset SCSI Device CCB 
  		 */
 		case XPT_RESET_DEV:	    	
 		/*
 		 * Don't (yet?) support vendor
 		 * specific commands.
 		 */
 			TRM_DPRINTF(" XPT_RESET_DEV \n");
 	    		pccb->ccb_h.status = CAM_REQ_INVALID;
 			xpt_done(pccb);
 			break;
 		/*
 		 * Terminate the I/O process 
 		 * Terminate I/O Process Request CCB 
  		 */
 		case XPT_TERM_IO:	    	
 			TRM_DPRINTF(" XPT_TERM_IO \n");
 	    		pccb->ccb_h.status = CAM_REQ_INVALID;
 			xpt_done(pccb);
 			break;
 		/*
 		 * Get/Set transfer rate/width/disconnection/tag queueing 
 		 * settings 
 		 * (GET) default/user transfer settings for the target 
 	 	 */
 		case XPT_GET_TRAN_SETTINGS: {
 			struct	ccb_trans_settings *cts = &pccb->cts;
 			int	intflag;
 			struct	trm_transinfo *tinfo;
 			PDCB	pDCB;	
 			struct ccb_trans_settings_scsi *scsi =
 			    &cts->proto_specific.scsi;
 			struct ccb_trans_settings_spi *spi =
 			    &cts->xport_specific.spi;
 
 			cts->protocol = PROTO_SCSI;
 			cts->protocol_version = SCSI_REV_2;
 			cts->transport = XPORT_SPI;
 			cts->transport_version = 2;
 
 			TRM_DPRINTF(" XPT_GET_TRAN_SETTINGS \n");
 			pDCB = &pACB->DCBarray[target_id][target_lun];
 			intflag = splcam();
 			/*
 			 * disable interrupt
 			 */
 			if (cts->type == CTS_TYPE_CURRENT_SETTINGS) {
 				/* current transfer settings */
 				if (pDCB->tinfo.disc_tag & TRM_CUR_DISCENB)
 					spi->flags = CTS_SPI_FLAGS_DISC_ENB;
 				else
 					spi->flags = 0;/* no tag & disconnect */
 				if (pDCB->tinfo.disc_tag & TRM_CUR_TAGENB)
 					scsi->flags |= CTS_SCSI_FLAGS_TAG_ENB;
 				tinfo = &pDCB->tinfo.current;
 				TRM_DPRINTF("CURRENT:  cts->flags= %2x \n",
 				    cts->flags);
 			} else {
 		  	  /* default(user) transfer settings */
 				if (pDCB->tinfo.disc_tag & TRM_USR_DISCENB)
 					spi->flags = CTS_SPI_FLAGS_DISC_ENB;
 				else
 					spi->flags = 0;
 				if (pDCB->tinfo.disc_tag & TRM_USR_TAGENB)
 					scsi->flags |= CTS_SCSI_FLAGS_TAG_ENB;
 				tinfo = &pDCB->tinfo.user;
 				TRM_DPRINTF("USER: cts->flags= %2x \n",
 					cts->flags);
 			}
 			spi->sync_period = tinfo->period;
 			spi->sync_offset = tinfo->offset;
 			spi->bus_width   = tinfo->width;
 			TRM_DPRINTF("pDCB->SyncPeriod: %d  \n", 
 				pDCB->SyncPeriod);
 			TRM_DPRINTF("period: %d  \n", tinfo->period);
 			TRM_DPRINTF("offset: %d  \n", tinfo->offset);
 			TRM_DPRINTF("width: %d  \n", tinfo->width);
 
 	    		splx(intflag);
 			spi->valid = CTS_SPI_VALID_SYNC_RATE |
 			    CTS_SPI_VALID_SYNC_OFFSET |
 			    CTS_SPI_VALID_BUS_WIDTH |
 			    CTS_SPI_VALID_DISC;
 			scsi->valid = CTS_SCSI_VALID_TQ;
 			pccb->ccb_h.status = CAM_REQ_CMP;
 			xpt_done(pccb);
 					    }
 			break;
 		/* 
 		 * Get/Set transfer rate/width/disconnection/tag queueing 
 		 * settings
 		 * (Set) transfer rate/width negotiation settings 
 		 */
 		case XPT_SET_TRAN_SETTINGS: {
 			struct	ccb_trans_settings *cts =  &pccb->cts;
 			u_int	update_type;
 			int	intflag;
 			PDCB	pDCB;
 			struct ccb_trans_settings_scsi *scsi =
 			    &cts->proto_specific.scsi;
 			struct ccb_trans_settings_spi *spi =
 			    &cts->xport_specific.spi;
 
 			TRM_DPRINTF(" XPT_SET_TRAN_SETTINGS \n");
 			update_type = 0;
 			if (cts->type == CTS_TYPE_CURRENT_SETTINGS)
 				update_type |= TRM_TRANS_GOAL;
 			if (cts->type == CTS_TYPE_USER_SETTINGS)
 				update_type |= TRM_TRANS_USER;
 			intflag = splcam();
 	    		pDCB = &pACB->DCBarray[target_id][target_lun];
 
 			if ((spi->valid & CTS_SPI_VALID_DISC) != 0) {
 			  /*ccb disc enables */
 				if (update_type & TRM_TRANS_GOAL) {
 					if ((spi->flags & CTS_SPI_FLAGS_DISC_ENB)
 					    != 0) 
 				    		pDCB->tinfo.disc_tag 
 						    |= TRM_CUR_DISCENB;
 					else
 						pDCB->tinfo.disc_tag &=
 						    ~TRM_CUR_DISCENB;
 				}
 				if (update_type & TRM_TRANS_USER) {
 					if ((spi->flags & CTS_SPI_FLAGS_DISC_ENB)
 					    != 0)
 						pDCB->tinfo.disc_tag 
 						    |= TRM_USR_DISCENB;
 					else
 						pDCB->tinfo.disc_tag &=
 						    ~TRM_USR_DISCENB;
 				}
 			}
 			if ((scsi->valid & CTS_SCSI_VALID_TQ) != 0) {
 			  /* if ccb tag q active */
 				if (update_type & TRM_TRANS_GOAL) {
 					if ((scsi->flags & CTS_SCSI_FLAGS_TAG_ENB)
 					    != 0)
 						pDCB->tinfo.disc_tag |= 
 						    TRM_CUR_TAGENB;
 					else
 						pDCB->tinfo.disc_tag &= 
 						    ~TRM_CUR_TAGENB;
 				}
 				if (update_type & TRM_TRANS_USER) {
 					if ((scsi->flags & CTS_SCSI_FLAGS_TAG_ENB)
 					    != 0)
 				  		pDCB->tinfo.disc_tag |= 
 						    TRM_USR_TAGENB;
 					else
 						pDCB->tinfo.disc_tag &= 
 						    ~TRM_USR_TAGENB;
 				}	
 			}
 			/* Minimum sync period factor	*/
 
 			if ((spi->valid & CTS_SPI_VALID_SYNC_RATE) != 0) {
 				/* if ccb sync active */
 				/* TRM-S1040 MinSyncPeriod = 4 clocks/byte */
 				if ((spi->sync_period != 0) &&
 				    (spi->sync_period < 125))
 					spi->sync_period = 125;
 				/* 1/(125*4) minsync 2 MByte/sec */
 				if ((spi->valid & CTS_SPI_VALID_SYNC_OFFSET)
 				    != 0) {
 					if (spi->sync_offset == 0)
 				 		spi->sync_period = 0;
 					/* TRM-S1040 MaxSyncOffset = 15 bytes*/
 					if (spi->sync_offset > 15) 
 						spi->sync_offset = 15;
 				}
 			}
 			if ((update_type & TRM_TRANS_USER) != 0) {
 				pDCB->tinfo.user.period = spi->sync_period;
 				pDCB->tinfo.user.offset = spi->sync_offset;
 				pDCB->tinfo.user.width  = spi->bus_width;
 			}
 			if ((update_type & TRM_TRANS_GOAL) != 0) {
 				pDCB->tinfo.goal.period = spi->sync_period;
 				pDCB->tinfo.goal.offset = spi->sync_offset;
 				pDCB->tinfo.goal.width  = spi->bus_width;
 			}
 			splx(intflag);
 			pccb->ccb_h.status = CAM_REQ_CMP;
 			xpt_done(pccb);
 			break;
 					    }
 		/*
 		 * Calculate the geometry parameters for a device give
 		 * the sector size and volume size. 
    		 */
 		case XPT_CALC_GEOMETRY:
 			TRM_DPRINTF(" XPT_CALC_GEOMETRY \n");
 			cam_calc_geometry(&pccb->ccg, /*extended*/1);
 			xpt_done(pccb);
 			break;
 		case XPT_ENG_INQ:           
 			TRM_DPRINTF(" XPT_ENG_INQ \n");
 	    		pccb->ccb_h.status = CAM_REQ_INVALID;
 			xpt_done(pccb);
 			break;
 		/*
 		 * HBA execute engine request 
 		 * This structure must match SCSIIO size 
 		 */
 		case XPT_ENG_EXEC:		    
 			TRM_DPRINTF(" XPT_ENG_EXEC \n");
 	    		pccb->ccb_h.status = CAM_REQ_INVALID;
 			xpt_done(pccb);
 			break;
 		/*
 		 * XPT_EN_LUN = 0x30, Enable LUN as a target 
 		 * Target mode structures. 
 	 	 */
 		case XPT_EN_LUN:            
 		/*
 		 * Don't (yet?) support vendor
 		 * specific commands.
 		 */
 			TRM_DPRINTF(" XPT_EN_LUN \n");
 			pccb->ccb_h.status = CAM_REQ_INVALID;
 			xpt_done(pccb);
 			break;
 		/*
 		* Execute target I/O request 
 		*/
 		case XPT_TARGET_IO:		    
 		/*
 		 * Don't (yet?) support vendor
 		 * specific commands.
 		 */
 			TRM_DPRINTF(" XPT_TARGET_IO \n");
 			pccb->ccb_h.status = CAM_REQ_INVALID;
 			xpt_done(pccb);
 			break;
 		/*
 		 * Accept Host Target Mode CDB 
        		 */
 		case XPT_ACCEPT_TARGET_IO:	
 		/*
 		 * Don't (yet?) support vendor
 		 * specific commands.
 		 */
 			TRM_DPRINTF(" XPT_ACCEPT_TARGET_IO \n");
 			pccb->ccb_h.status = CAM_REQ_INVALID;
 			xpt_done(pccb);
 			break;
 		/*
 		 * Continue Host Target I/O Connection 
  		 */
 		case XPT_CONT_TARGET_IO:  	
 		/*
 		 * Don't (yet?) support vendor
 		 * specific commands.
 		 */
 			TRM_DPRINTF(" XPT_CONT_TARGET_IO \n");
 			pccb->ccb_h.status = CAM_REQ_INVALID;
 			xpt_done(pccb);
 			break;
 		/*
 		 * Notify Host Target driver of event 
  		 */
 		case XPT_IMMED_NOTIFY:	    
 			TRM_DPRINTF(" XPT_IMMED_NOTIFY \n");
 	    		pccb->ccb_h.status = CAM_REQ_INVALID;
 			xpt_done(pccb);
 			break;
 		/*
 		 * Acknowledgement of event
        		 */
 		case XPT_NOTIFY_ACK:	    
 			TRM_DPRINTF(" XPT_NOTIFY_ACK \n");
 	    		pccb->ccb_h.status = CAM_REQ_INVALID;
 			xpt_done(pccb);
 			break;
 		/*
 		 * XPT_VUNIQUE = 0x80
 		 */
 		case XPT_VUNIQUE:   
 			pccb->ccb_h.status = CAM_REQ_INVALID;
 			xpt_done(pccb);
 			break;
 		default:
 			pccb->ccb_h.status = CAM_REQ_INVALID;
 			xpt_done(pccb);
 			break;
 	}
 	splx(actionflags);
 }
 
 static void 
 trm_poll(struct cam_sim *psim)
 {       
 	trm_Interrupt(cam_sim_softc(psim));	
 }
 
 static void
 trm_ResetDevParam(PACB pACB)
 {
 	PDCB		pDCB, pdcb;
 	PNVRAMTYPE 	pEEpromBuf;
 	u_int8_t	PeriodIndex;
 
 	pDCB = pACB->pLinkDCB;
 	if (pDCB == NULL)
 		return;
 	pdcb = pDCB;
 	do {
 		pDCB->SyncMode  &= ~(SYNC_NEGO_DONE+ WIDE_NEGO_DONE);
 		pDCB->SyncPeriod = 0;
 		pDCB->SyncOffset = 0;
 		pEEpromBuf = &trm_eepromBuf[pACB->AdapterUnit];
 		pDCB->DevMode = 
 		  pEEpromBuf->NvramTarget[pDCB->TargetID].NvmTarCfg0;
 		pDCB->AdpMode = pEEpromBuf->NvramChannelCfg;
 		PeriodIndex =
 		   pEEpromBuf->NvramTarget[pDCB->TargetID].NvmTarPeriod & 0x07;
 		if (pACB->AdaptType == 1) /* is U2? */
 			pDCB->MaxNegoPeriod = dc395u2x_clock_period[PeriodIndex];
 		else
 			pDCB->MaxNegoPeriod = dc395x_clock_period[PeriodIndex];
 		if ((pDCB->DevMode & NTC_DO_WIDE_NEGO) && 
 		    (pACB->Config & HCC_WIDE_CARD))
 			pDCB->SyncMode |= WIDE_NEGO_ENABLE;
 		pDCB = pDCB->pNextDCB;
 	}
 	while (pdcb != pDCB);
 }
 
 static void
 trm_RecoverSRB(PACB pACB)
 {
 	PDCB		pDCB, pdcb;
 	PSRB		psrb, psrb2;
        	u_int16_t	cnt, i;
 
 	pDCB = pACB->pLinkDCB;
 	if (pDCB == NULL)
 		return;
 	pdcb = pDCB;
 	do {
 		cnt = pdcb->GoingSRBCnt;
 		psrb = pdcb->pGoingSRB;
 		for (i = 0; i < cnt; i++) {
 			psrb2 = psrb;
 			psrb = psrb->pNextSRB;
 			if (pdcb->pWaitingSRB) {
 				psrb2->pNextSRB = pdcb->pWaitingSRB;
 				pdcb->pWaitingSRB = psrb2;
 			} else {
 				pdcb->pWaitingSRB = psrb2;
 				pdcb->pWaitingLastSRB = psrb2;
 				psrb2->pNextSRB = NULL;
 			}
 		}
 		pdcb->GoingSRBCnt = 0;
 		pdcb->pGoingSRB = NULL;
 		pdcb = pdcb->pNextDCB;
 	}
 	while (pdcb != pDCB);
 }
 
 static void
 trm_reset(PACB pACB)
 {
 	int		intflag;
 	u_int16_t	i;
 
     	TRM_DPRINTF("trm: RESET");
     	intflag = splcam();
 	trm_reg_write8(0x00, TRMREG_DMA_INTEN);
 	trm_reg_write8(0x00, TRMREG_SCSI_INTEN);
 
 	trm_ResetSCSIBus(pACB);
 	for (i = 0; i < 500; i++)
 		DELAY(1000);
     	trm_reg_write8(0x7F, TRMREG_SCSI_INTEN); 
 	/* Enable DMA interrupt	*/
 	trm_reg_write8(EN_SCSIINTR, TRMREG_DMA_INTEN);
 	/* Clear DMA FIFO */
 	trm_reg_write8(CLRXFIFO, TRMREG_DMA_CONTROL);
 	/* Clear SCSI FIFO */
 	trm_reg_write16(DO_CLRFIFO,TRMREG_SCSI_CONTROL);
 	trm_ResetDevParam(pACB);
 	trm_DoingSRB_Done(pACB);
 	pACB->pActiveDCB = NULL;
 	pACB->ACBFlag = 0;/* RESET_DETECT, RESET_DONE ,RESET_DEV */
 	trm_DoWaitingSRB(pACB);
 	/* Tell the XPT layer that a bus reset occured    */
 	if (pACB->ppath != NULL)
 		xpt_async(AC_BUS_RESET, pACB->ppath, NULL);
 	splx(intflag);
     	return;
 }
 
 static u_int16_t
 trm_StartSCSI(PACB pACB, PDCB pDCB, PSRB pSRB)
 {
 	u_int16_t	return_code;
 	u_int8_t	scsicommand, i,command,identify_message;
 	u_int8_t *	ptr;
 	union  ccb	*pccb;
 	struct ccb_scsiio *pcsio;
 
 	pccb  = pSRB->pccb;
 	pcsio = &pccb->csio;
 
 	trm_reg_write8(pACB->AdaptSCSIID, TRMREG_SCSI_HOSTID);
 	trm_reg_write8(pDCB->TargetID, TRMREG_SCSI_TARGETID);
 	trm_reg_write8(pDCB->SyncPeriod, TRMREG_SCSI_SYNC);
 	trm_reg_write8(pDCB->SyncOffset, TRMREG_SCSI_OFFSET);
 	pSRB->ScsiPhase = PH_BUS_FREE;/* initial phase */
 	/* Flush FIFO */
 	trm_reg_write16(DO_CLRFIFO, TRMREG_SCSI_CONTROL);
 
 	identify_message = pDCB->IdentifyMsg;
 
    	if ((pSRB->CmdBlock[0] == INQUIRY) ||
 	    (pSRB->CmdBlock[0] == REQUEST_SENSE) ||
 	    (pSRB->SRBFlag & AUTO_REQSENSE)) {
 		if (((pDCB->SyncMode & WIDE_NEGO_ENABLE) &&
 		      !(pDCB->SyncMode & WIDE_NEGO_DONE)) 
 		|| ((pDCB->SyncMode & SYNC_NEGO_ENABLE) &&
 		  !(pDCB->SyncMode & SYNC_NEGO_DONE))) {
 			if (!(pDCB->IdentifyMsg & 7) ||
 			    (pSRB->CmdBlock[0] != INQUIRY)) {
 				scsicommand = SCMD_SEL_ATNSTOP;
 				pSRB->SRBState = SRB_MSGOUT;
 				goto polling;
 			}
 		}
        	/* 
        	* Send identify message	
        	*/
 		trm_reg_write8((identify_message & 0xBF) ,TRMREG_SCSI_FIFO); 
 		scsicommand = SCMD_SEL_ATN;
 		pSRB->SRBState = SRB_START_;
 	} else {
 		/* not inquiry,request sense,auto request sense */
 		/* 
 		 * Send identify message	
 		 */
 		trm_reg_write8(identify_message,TRMREG_SCSI_FIFO);
 		scsicommand = SCMD_SEL_ATN;
 		pSRB->SRBState = SRB_START_;
 		if (pDCB->SyncMode & EN_TAG_QUEUING) {
 		  /* Send Tag message */
  			trm_reg_write8(MSG_SIMPLE_QTAG, TRMREG_SCSI_FIFO);
 			trm_reg_write8(pSRB->TagNumber, TRMREG_SCSI_FIFO);
 			scsicommand = SCMD_SEL_ATN3;
 		}
 	}
 polling:
 	/*
 	 * 	 Send CDB ..command block .........			
 	 */
    	if (pSRB->SRBFlag & AUTO_REQSENSE) {
 		trm_reg_write8(REQUEST_SENSE, TRMREG_SCSI_FIFO);
 		trm_reg_write8((pDCB->IdentifyMsg << 5), TRMREG_SCSI_FIFO);
 		trm_reg_write8(0, TRMREG_SCSI_FIFO);
 		trm_reg_write8(0, TRMREG_SCSI_FIFO);
 		trm_reg_write8(pcsio->sense_len, TRMREG_SCSI_FIFO);
 		trm_reg_write8(0, TRMREG_SCSI_FIFO);
 	} else {
 		ptr = (u_int8_t *) pSRB->CmdBlock;
 		for (i = 0; i < pSRB->ScsiCmdLen ; i++) {
 			command = *ptr++;
 			trm_reg_write8(command,TRMREG_SCSI_FIFO);
 		}
 	}
 	if (trm_reg_read16(TRMREG_SCSI_STATUS) & SCSIINTERRUPT) { 
 	    /* 
 	     * If trm_StartSCSI return 1 :
 	     * current interrupt status is interrupt disreenable 
 	     * It's said that SCSI processor has more one SRB need to do,
      	     * SCSI processor has been occupied by one SRB.
 	     */
 		pSRB->SRBState = SRB_READY;
 		return_code = 1;
 	} else { 
 	  /* 
 	   * If trm_StartSCSI return 0 :
 	   * current interrupt status is interrupt enable 
 	   * It's said that SCSI processor is unoccupied 
 	   */
 		pSRB->ScsiPhase  = SCSI_NOP1; /* SCSI bus free Phase */
 		pACB->pActiveDCB = pDCB;
 		pDCB->pActiveSRB = pSRB;
 		return_code = 0;
 		trm_reg_write16(DO_DATALATCH | DO_HWRESELECT, 
 		    TRMREG_SCSI_CONTROL);/* it's important for atn stop*/
 		/*
 		 * SCSI cammand 
 		 */
 		trm_reg_write8(scsicommand,TRMREG_SCSI_COMMAND);
 	}
 	return (return_code);	
 }
 
 static void 
 trm_Interrupt(vpACB)
 void *vpACB;
 {
 	PACB		pACB;
 	PDCB		pDCB;
 	PSRB		pSRB;
 	u_int16_t	phase;
 	void		(*stateV)(PACB, PSRB, u_int16_t *);
 	u_int16_t	scsi_status=0;
 	u_int8_t	scsi_intstatus;
 
 	pACB = vpACB;
 
 	scsi_status = trm_reg_read16(TRMREG_SCSI_STATUS);
 	if (!(scsi_status & SCSIINTERRUPT)) {
 		TRM_DPRINTF("trm_Interrupt: TRMREG_SCSI_STATUS scsi_status = NULL ,return......");
 	    	return;
 	}
 	TRM_DPRINTF("scsi_status=%2x,",scsi_status);
 
     	scsi_intstatus = trm_reg_read8(TRMREG_SCSI_INTSTATUS);
 
 	TRM_DPRINTF("scsi_intstatus=%2x,",scsi_intstatus);
 
     	if (scsi_intstatus & (INT_SELTIMEOUT | INT_DISCONNECT)) {
 		trm_Disconnect(pACB);
 		return;
 	}
 
 	if (scsi_intstatus & INT_RESELECTED) {
 		trm_Reselect(pACB);
 		return;
 	}
 	if (scsi_intstatus & INT_SCSIRESET) {
 		trm_ScsiRstDetect(pACB);
 		return;
 	}
 
 	if (scsi_intstatus & (INT_BUSSERVICE | INT_CMDDONE)) {
 		pDCB = pACB->pActiveDCB;
 		KASSERT(pDCB != NULL, ("no active DCB"));
 		pSRB = pDCB->pActiveSRB;
 		if (pDCB->DCBFlag & ABORT_DEV_)
 				trm_EnableMsgOutAbort1(pACB, pSRB);
 		phase = (u_int16_t) pSRB->ScsiPhase;  /* phase: */
 		stateV = (void *) trm_SCSI_phase0[phase];
 		stateV(pACB, pSRB, &scsi_status);
 		pSRB->ScsiPhase = scsi_status & PHASEMASK; 
 		/* phase:0,1,2,3,4,5,6,7 */
 		phase = (u_int16_t) scsi_status & PHASEMASK;       
 		stateV = (void *) trm_SCSI_phase1[phase];
 		stateV(pACB, pSRB, &scsi_status);  
 	}
 }
 
 static void
 trm_MsgOutPhase0(PACB pACB, PSRB pSRB, u_int16_t *pscsi_status)
 {
 
 	if (pSRB->SRBState & (SRB_UNEXPECT_RESEL+SRB_ABORT_SENT))
 		*pscsi_status = PH_BUS_FREE;
 	/*.. initial phase*/
 }
 
 static void
 trm_MsgOutPhase1(PACB pACB, PSRB pSRB, u_int16_t *pscsi_status)
 {
 	u_int8_t	bval;
 	u_int16_t	i, cnt;
 	u_int8_t *	ptr;
 	PDCB		pDCB;
 
 	trm_reg_write16(DO_CLRFIFO, TRMREG_SCSI_CONTROL);
 	pDCB = pACB->pActiveDCB;
 	if (!(pSRB->SRBState & SRB_MSGOUT)) {
 		cnt = pSRB->MsgCnt;
 		if (cnt) {
 			ptr = (u_int8_t *) pSRB->MsgOutBuf;
 			for (i = 0; i < cnt; i++) {
 				trm_reg_write8(*ptr, TRMREG_SCSI_FIFO);
 				ptr++;
 			}
 			pSRB->MsgCnt = 0;
 			if ((pDCB->DCBFlag & ABORT_DEV_) &&
 			    (pSRB->MsgOutBuf[0] == MSG_ABORT)) {
 				pSRB->SRBState = SRB_ABORT_SENT;
 			}
 		} else {
 			bval = MSG_ABORT;	
 			if ((pSRB->CmdBlock[0] == INQUIRY) ||
 					(pSRB->CmdBlock[0] == REQUEST_SENSE) ||
 					(pSRB->SRBFlag & AUTO_REQSENSE)) {
 				if (pDCB->SyncMode & SYNC_NEGO_ENABLE) {
 					goto  mop1;
 				}
 			}
 			trm_reg_write8(bval, TRMREG_SCSI_FIFO);
 		}
 	} else {
 mop1:   /* message out phase */
 		if (!(pSRB->SRBState & SRB_DO_WIDE_NEGO)
 		    && (pDCB->SyncMode & WIDE_NEGO_ENABLE)) {
 		  /*
 	   	   * WIDE DATA TRANSFER REQUEST code (03h)
 		   */
 			pDCB->SyncMode &= ~(SYNC_NEGO_DONE | EN_ATN_STOP);
 			trm_reg_write8((pDCB->IdentifyMsg & 0xBF),
 			    TRMREG_SCSI_FIFO); 
 			trm_reg_write8(MSG_EXTENDED,TRMREG_SCSI_FIFO);
 			/* (01h) */
 			trm_reg_write8(2,TRMREG_SCSI_FIFO);	
 			/* Message length (02h) */
 			trm_reg_write8(3,TRMREG_SCSI_FIFO);
 			/* wide data xfer (03h) */
 			trm_reg_write8(1,TRMREG_SCSI_FIFO);
 			/* width:0(8bit),1(16bit),2(32bit) */
 			pSRB->SRBState |= SRB_DO_WIDE_NEGO; 
 		} else if (!(pSRB->SRBState & SRB_DO_SYNC_NEGO) 
 		    && (pDCB->SyncMode & SYNC_NEGO_ENABLE)) {
 		  /*
 	   	   * SYNCHRONOUS DATA TRANSFER REQUEST code (01h)
 		   */
 			if (!(pDCB->SyncMode & WIDE_NEGO_DONE))
 				trm_reg_write8((pDCB->IdentifyMsg & 0xBF),
 						TRMREG_SCSI_FIFO);
 			trm_reg_write8(MSG_EXTENDED,TRMREG_SCSI_FIFO);
 		  /* (01h) */
 			trm_reg_write8(3,TRMREG_SCSI_FIFO); 
 		  /* Message length (03h) */
 			trm_reg_write8(1,TRMREG_SCSI_FIFO);
 		  /* SYNCHRONOUS DATA TRANSFER REQUEST code (01h) */
 			trm_reg_write8(pDCB->MaxNegoPeriod,TRMREG_SCSI_FIFO);
 		  /* Transfer peeriod factor */
 			trm_reg_write8((pACB->AdaptType == 1) ? 31 : 15,
 			    TRMREG_SCSI_FIFO); 
 		  /* REQ/ACK offset */
 			pSRB->SRBState |= SRB_DO_SYNC_NEGO;
 		}
 	}
 	trm_reg_write16(DO_DATALATCH, TRMREG_SCSI_CONTROL);
 	/* it's important for atn stop */
 	/*
 	 * SCSI cammand 
 	 */
 	trm_reg_write8(SCMD_FIFO_OUT, TRMREG_SCSI_COMMAND);
 }
 
 static void 
 trm_CommandPhase0(PACB pACB, PSRB pSRB, u_int16_t *pscsi_status)
 {
 
 }
 
 static void 
 trm_CommandPhase1(PACB pACB, PSRB pSRB, u_int16_t *pscsi_status)
 {
 	PDCB			pDCB;
 	u_int8_t *		ptr;
 	u_int16_t		i, cnt;
 	union  ccb		*pccb;
 	struct ccb_scsiio	*pcsio;
 
 	pccb  = pSRB->pccb;
 	pcsio = &pccb->csio;
 
 	trm_reg_write16(DO_CLRATN | DO_CLRFIFO , TRMREG_SCSI_CONTROL);
 	if (!(pSRB->SRBFlag & AUTO_REQSENSE)) {
 		cnt = (u_int16_t) pSRB->ScsiCmdLen;
 		ptr = (u_int8_t *) pSRB->CmdBlock;
 		for (i = 0; i < cnt; i++) {
 			trm_reg_write8(*ptr, TRMREG_SCSI_FIFO);
 			ptr++;
 		}
 	} else {
 		trm_reg_write8(REQUEST_SENSE, TRMREG_SCSI_FIFO);
 		pDCB = pACB->pActiveDCB;
 		/* target id */
 		trm_reg_write8((pDCB->IdentifyMsg << 5), TRMREG_SCSI_FIFO);
 		trm_reg_write8(0, TRMREG_SCSI_FIFO);
 		trm_reg_write8(0, TRMREG_SCSI_FIFO);
 		/* sizeof(struct scsi_sense_data) */
 		trm_reg_write8(pcsio->sense_len, TRMREG_SCSI_FIFO);
 		trm_reg_write8(0, TRMREG_SCSI_FIFO);
 	}
 	pSRB->SRBState = SRB_COMMAND;
 	trm_reg_write16(DO_DATALATCH, TRMREG_SCSI_CONTROL);
 	/* it's important for atn stop*/
 	/*
 	 * SCSI cammand 
 	 */
 	trm_reg_write8(SCMD_FIFO_OUT, TRMREG_SCSI_COMMAND);
 }
 
 static void
 trm_DataOutPhase0(PACB pACB, PSRB pSRB, u_int16_t *pscsi_status)
 {
 	PDCB		pDCB;
 	u_int8_t	TempDMAstatus,SGIndexTemp;
 	u_int16_t	scsi_status;
 	PSEG		pseg;
 	u_long		TempSRBXferredLength,dLeftCounter=0;
 
 	pDCB = pSRB->pSRBDCB;
 	scsi_status = *pscsi_status;
 
 	if (!(pSRB->SRBState & SRB_XFERPAD)) {
 		if (scsi_status & PARITYERROR)
 			pSRB->SRBStatus |= PARITY_ERROR;
 		if (!(scsi_status & SCSIXFERDONE)) {
       		  /*
 		   * when data transfer from DMA FIFO to SCSI FIFO
 		   * if there was some data left in SCSI FIFO
 		   */
   			dLeftCounter = (u_long) 
 			  (trm_reg_read8(TRMREG_SCSI_FIFOCNT) & 0x3F);
 			if (pDCB->SyncPeriod & WIDE_SYNC) {
 			  /*
 		   	   * if WIDE scsi SCSI FIFOCNT unit is word
 	   		   * so need to * 2
    			   */
 				dLeftCounter <<= 1;
 			}
 		}
 		/*
 		 * caculate all the residue data that not yet tranfered
 		 * SCSI transfer counter + left in SCSI FIFO data
 		 *
 		 * .....TRM_SCSI_COUNTER (24bits)
 		 * The counter always decrement by one for every SCSI byte 
 		 *transfer.
 		 * .....TRM_SCSI_FIFOCNT (5bits)
 		 * The counter is SCSI FIFO offset counter
 		 */
 		dLeftCounter += trm_reg_read32(TRMREG_SCSI_COUNTER);
 		if (dLeftCounter == 1) {
 			dLeftCounter = 0;
 			trm_reg_write16(DO_CLRFIFO,TRMREG_SCSI_CONTROL);
 		}
 		if ((dLeftCounter == 0) || 
 		    (scsi_status & SCSIXFERCNT_2_ZERO)) {   
 			TempDMAstatus = trm_reg_read8(TRMREG_DMA_STATUS);
 			while (!(TempDMAstatus & DMAXFERCOMP)) {
 				TempDMAstatus = 
 				  trm_reg_read8(TRMREG_DMA_STATUS);
 			}
 			pSRB->SRBTotalXferLength = 0;
 		} else {
 		  /* Update SG list		*/
 		  /*
 	   	   * if transfer not yet complete
    		   * there were some data residue in SCSI FIFO or
 		   * SCSI transfer counter not empty
 		   */
 			if (pSRB->SRBTotalXferLength != dLeftCounter) {
 			  /*
 		  	   * data that had transferred length
 	   		   */
 				TempSRBXferredLength = 
 				  pSRB->SRBTotalXferLength - dLeftCounter;
 				/*
 				 * next time to be transferred length
 				 */
 				pSRB->SRBTotalXferLength = dLeftCounter;
 				/*
 				 * parsing from last time disconnect SRBSGIndex
 				 */
 				pseg = 
 				  pSRB->pSRBSGL + pSRB->SRBSGIndex;
 				for (SGIndexTemp = pSRB->SRBSGIndex;
 				    SGIndexTemp < pSRB->SRBSGCount; 
 				    SGIndexTemp++) {
 					/* 
 					 * find last time which SG transfer be 
 					 * disconnect 
 					 */
 					if (TempSRBXferredLength >= 
 					    pseg->length) 
 						TempSRBXferredLength -= 
 						  pseg->length;
 					else {
 				  	  /*
 			   		   * update last time disconnected SG 
 					   * list
 				   	   */
 						pseg->length -= 
 						  TempSRBXferredLength; 
 						/* residue data length  */
 						pseg->address += 
 						  TempSRBXferredLength;
 						/* residue data pointer */
 						pSRB->SRBSGIndex = SGIndexTemp;
 						break;
 					}
 					pseg++;
 				}
 			}
 		}
 	}
 	trm_reg_write8(STOPDMAXFER ,TRMREG_DMA_CONTROL);
 }
 
 
 static void
 trm_DataOutPhase1(PACB pACB, PSRB pSRB, u_int16_t *pscsi_status)
 {
 	u_int16_t	ioDir;
 	/*
 	 * do prepare befor transfer when data out phase
 	 */
 
 	ioDir = XFERDATAOUT;
 	trm_DataIO_transfer(pACB, pSRB, ioDir);
 }
 
 static void 
 trm_DataInPhase0(PACB pACB, PSRB pSRB, u_int16_t *pscsi_status)
 {
 	u_int8_t	TempDMAstatus, SGIndexTemp;
 	u_int16_t	scsi_status;
 	PSEG		pseg;
 	u_long		TempSRBXferredLength,dLeftCounter = 0;
 
     	scsi_status = *pscsi_status;
 	if (!(pSRB->SRBState & SRB_XFERPAD)) {
 		if (scsi_status & PARITYERROR)
 			pSRB->SRBStatus |= PARITY_ERROR;
 		dLeftCounter += trm_reg_read32(TRMREG_SCSI_COUNTER);
 		if ((dLeftCounter == 0) || (scsi_status & SCSIXFERCNT_2_ZERO)) {
 			TempDMAstatus = trm_reg_read8(TRMREG_DMA_STATUS);
 			while (!(TempDMAstatus & DMAXFERCOMP))
 				TempDMAstatus = trm_reg_read8(TRMREG_DMA_STATUS);
 			pSRB->SRBTotalXferLength = 0;
 		} else {  
 	  	  /*
    		   * parsing the case:
 	   	   * when a transfer not yet complete 
 	   	   * but be disconnected by uper layer
 	   	   * if transfer not yet complete
 	   	   * there were some data residue in SCSI FIFO or
 	   	   * SCSI transfer counter not empty
 	   	   */
 		  if (pSRB->SRBTotalXferLength != dLeftCounter) {
 				/*
 				 * data that had transferred length
 				 */
 		  	TempSRBXferredLength = 
 			  pSRB->SRBTotalXferLength - dLeftCounter;
 				/*
 			 	 * next time to be transferred length
 				 */
 			pSRB->SRBTotalXferLength = dLeftCounter;
 				/*
 				 * parsing from last time disconnect SRBSGIndex
 				 */
 			pseg = pSRB->pSRBSGL + pSRB->SRBSGIndex;
 			for (SGIndexTemp = pSRB->SRBSGIndex; 
 			    SGIndexTemp < pSRB->SRBSGCount;
 			    SGIndexTemp++) {
 			  /* 
 	   		   * find last time which SG transfer be disconnect 
 	   		   */
 	 			if (TempSRBXferredLength >= pseg->length)
 					TempSRBXferredLength -= pseg->length;
 				else {
 		  		  /*
    				   * update last time disconnected SG list
 				   */
 					pseg->length -= TempSRBXferredLength;
 					/* residue data length  */
 					pseg->address += TempSRBXferredLength;
 					/* residue data pointer */
 					pSRB->SRBSGIndex = SGIndexTemp;
 					break;
 				} 
 				pseg++;
 			}
 	  	  }
 		}
 	}
 }
 
 static void
 trm_DataInPhase1(PACB pACB, PSRB pSRB, u_int16_t *pscsi_status)
 {
 	u_int16_t	ioDir;
 	/*
 	 * do prepare befor transfer when data in phase
 	 */
 	
 	ioDir = XFERDATAIN;
 	trm_DataIO_transfer(pACB, pSRB, ioDir);
 }
 
 static void
 trm_DataIO_transfer(PACB pACB, PSRB pSRB, u_int16_t ioDir)
 {
 	u_int8_t	bval;
 	PDCB		pDCB;
 
 	pDCB = pSRB->pSRBDCB;
 	if (pSRB->SRBSGIndex < pSRB->SRBSGCount) {
 		if (pSRB->SRBTotalXferLength != 0) {
 			/* 
 			 * load what physical address of Scatter/Gather list 
 			 table want to be transfer
 			 */
 			TRM_DPRINTF(" SG->address=%8x \n",pSRB->pSRBSGL->address);
 			TRM_DPRINTF(" SG->length=%8x \n",pSRB->pSRBSGL->length);
 			TRM_DPRINTF(" pDCB->SyncPeriod=%x \n",pDCB->SyncPeriod);
 			TRM_DPRINTF(" pSRB->pSRBSGL=%8x \n",(unsigned int)pSRB->pSRBSGL);
 			TRM_DPRINTF(" pSRB->SRBSGPhyAddr=%8x \n",pSRB->SRBSGPhyAddr);
 			TRM_DPRINTF(" pSRB->SRBSGIndex=%d \n",pSRB->SRBSGIndex);
 			TRM_DPRINTF(" pSRB->SRBSGCount=%d \n",pSRB->SRBSGCount);
 			TRM_DPRINTF(" pSRB->SRBTotalXferLength=%d \n",pSRB->SRBTotalXferLength);
 
 			pSRB->SRBState = SRB_DATA_XFER;
 			trm_reg_write32(0, TRMREG_DMA_XHIGHADDR);
 			trm_reg_write32(
 			    (pSRB->SRBSGPhyAddr + 
 			     ((u_long)pSRB->SRBSGIndex << 3)),
 			    TRMREG_DMA_XLOWADDR);
 			/*
 			 * load how many bytes in the Scatter/Gather 
 			 * list table 
 			 */
 			trm_reg_write32(
 			    ((u_long)(pSRB->SRBSGCount - pSRB->SRBSGIndex) << 3),
 			    TRMREG_DMA_XCNT);			
 			/*
 			 * load total transfer length (24bits) max value
 			 * 16Mbyte 
 			 */
 			trm_reg_write32(pSRB->SRBTotalXferLength,
 			    TRMREG_SCSI_COUNTER);
 			/* Start DMA transfer */
 			trm_reg_write16(ioDir, TRMREG_DMA_COMMAND);
 			/* Start SCSI transfer */
 			trm_reg_write16(DO_DATALATCH, TRMREG_SCSI_CONTROL);
 			/* it's important for atn stop */
 			/*
 			 * SCSI cammand 
 			 */
 			bval = (ioDir == XFERDATAOUT) ?
 			  SCMD_DMA_OUT : SCMD_DMA_IN;
 			trm_reg_write8(bval, TRMREG_SCSI_COMMAND);
 		} else {
 		  /* xfer pad */
 			if (pSRB->SRBSGCount) {
 				pSRB->AdaptStatus = H_OVER_UNDER_RUN;
 				pSRB->SRBStatus |= OVER_RUN;
 			}
 			if (pDCB->SyncPeriod & WIDE_SYNC)
 				trm_reg_write32(2,TRMREG_SCSI_COUNTER);
 			else
 				trm_reg_write32(1,TRMREG_SCSI_COUNTER);
 			if (ioDir == XFERDATAOUT)
 				trm_reg_write16(0, TRMREG_SCSI_FIFO);
 			else
 				trm_reg_read16(TRMREG_SCSI_FIFO);
 			pSRB->SRBState |= SRB_XFERPAD;
 			trm_reg_write16(DO_DATALATCH, TRMREG_SCSI_CONTROL);
 			/* it's important for atn stop */
 			/*
 			 * SCSI cammand 
 			 */
 			bval = (ioDir == XFERDATAOUT) ? 
 			  SCMD_FIFO_OUT : SCMD_FIFO_IN;
 			trm_reg_write8(bval, TRMREG_SCSI_COMMAND);
 		}
 	}
 }
 
 static void
 trm_StatusPhase0(PACB pACB, PSRB pSRB, u_int16_t *pscsi_status)
 {
 
 	pSRB->TargetStatus = trm_reg_read8(TRMREG_SCSI_FIFO);
 	pSRB->SRBState = SRB_COMPLETED;
 	*pscsi_status = PH_BUS_FREE;  
 	/*.. initial phase*/
 	trm_reg_write16(DO_DATALATCH, TRMREG_SCSI_CONTROL);
 	/* it's important for atn stop */
 	/*
 	 * SCSI cammand 
 	 */
 	trm_reg_write8(SCMD_MSGACCEPT, TRMREG_SCSI_COMMAND);
 }
 
 
 
 static void
 trm_StatusPhase1(PACB pACB, PSRB pSRB, u_int16_t *pscsi_status)
 {
 
 	if (trm_reg_read16(TRMREG_DMA_COMMAND) & 0x0001) {
 		if (!(trm_reg_read8(TRMREG_SCSI_FIFOCNT) & 0x40))
 	       		trm_reg_write16(DO_CLRFIFO, TRMREG_SCSI_CONTROL);
 		if (!(trm_reg_read16(TRMREG_DMA_FIFOCNT) & 0x8000))
 			trm_reg_write8(CLRXFIFO, TRMREG_DMA_CONTROL);
 	} else {
 		if (!(trm_reg_read16(TRMREG_DMA_FIFOCNT) & 0x8000))
 			trm_reg_write8(CLRXFIFO, TRMREG_DMA_CONTROL);
 		if (!(trm_reg_read8(TRMREG_SCSI_FIFOCNT) & 0x40))
 			trm_reg_write16(DO_CLRFIFO, TRMREG_SCSI_CONTROL);
 	}
 	pSRB->SRBState = SRB_STATUS;
 	trm_reg_write16(DO_DATALATCH, TRMREG_SCSI_CONTROL);
 	/* it's important for atn stop */
 	/*
 	 * SCSI cammand 
 	 */
 	trm_reg_write8(SCMD_COMP, TRMREG_SCSI_COMMAND);
 }
 
 /*
  *scsiiom		 
  *       trm_MsgInPhase0: one of trm_SCSI_phase0[] vectors
  *            stateV = (void *) trm_SCSI_phase0[phase]
  *		           if phase =7    
  * extended message codes:
  *
  *   code        description
  *
  *    02h        Reserved
  *    00h        MODIFY DATA  POINTER
  *    01h        SYNCHRONOUS DATA TRANSFER REQUEST
  *    03h        WIDE DATA TRANSFER REQUEST
  * 04h - 7Fh     Reserved
  * 80h - FFh     Vendor specific  
  *		        
  */
 
 static void
 trm_MsgInPhase0(PACB pACB, PSRB pSRB, u_int16_t *pscsi_status)
 {
 	u_int8_t	message_in_code,bIndex,message_in_tag_id;
 	PDCB		pDCB;
 	PSRB		pSRBTemp;
 
 	pDCB = pACB->pActiveDCB;
 
 	message_in_code = trm_reg_read8(TRMREG_SCSI_FIFO);
 	if (!(pSRB->SRBState & SRB_EXTEND_MSGIN)) {
 		if (message_in_code == MSG_DISCONNECT) {
 			pSRB->SRBState = SRB_DISCONNECT;
 			*pscsi_status = PH_BUS_FREE; /* .. initial phase */
 			/* it's important for atn stop */
 			trm_reg_write16(DO_DATALATCH, TRMREG_SCSI_CONTROL);
 			/*
 			 * SCSI command
 			 */
 			trm_reg_write8(SCMD_MSGACCEPT, TRMREG_SCSI_COMMAND);
 			return;
 		} else if (message_in_code == MSG_SAVE_PTR) {
 			*pscsi_status = PH_BUS_FREE; /* .. initial phase */
 			/* it's important for atn stop */
 			trm_reg_write16(DO_DATALATCH, TRMREG_SCSI_CONTROL);
 			/*
 			 * SCSI command
 			 */
 			trm_reg_write8(SCMD_MSGACCEPT, TRMREG_SCSI_COMMAND);
 			return;
 		} else if ((message_in_code == MSG_EXTENDED) ||
 		    ((message_in_code >= MSG_SIMPLE_QTAG) &&
 		     (message_in_code <= MSG_ORDER_QTAG))) {
 			pSRB->SRBState |= SRB_EXTEND_MSGIN;
 		    	pSRB->MsgInBuf[0] = message_in_code;
 			/* extended message      (01h) */
 			pSRB->MsgCnt = 1;
 			pSRB->pMsgPtr = &pSRB->MsgInBuf[1];
 			/* extended message length (n) */
 			*pscsi_status = PH_BUS_FREE; /* .. initial phase */
 			/* it's important for atn stop */
 			trm_reg_write16(DO_DATALATCH, TRMREG_SCSI_CONTROL);
 			/*
 			 * SCSI command
 			 */
 			trm_reg_write8(SCMD_MSGACCEPT, TRMREG_SCSI_COMMAND);
 			return;
 		} else if (message_in_code == MSG_REJECT_) {
 			/* Reject message */
 			if (pDCB->SyncMode & WIDE_NEGO_ENABLE) {
 			  /* do wide nego reject */
 				pDCB = pSRB->pSRBDCB;
 				pDCB->SyncMode |= WIDE_NEGO_DONE;
 				pDCB->SyncMode &= ~(SYNC_NEGO_DONE | 
 				    EN_ATN_STOP | WIDE_NEGO_ENABLE);
 				pSRB->SRBState &= ~(SRB_DO_WIDE_NEGO+SRB_MSGIN);
 				if ((pDCB->SyncMode & SYNC_NEGO_ENABLE) 
 				    && !(pDCB->SyncMode & SYNC_NEGO_DONE)) {   
 				  /* Set ATN, in case ATN was clear */
 					pSRB->SRBState |= SRB_MSGOUT;
 					trm_reg_write16(
 					    DO_SETATN,
 					    TRMREG_SCSI_CONTROL);
 				} else {   
 			  	  /* Clear ATN */
 					trm_reg_write16(
 					    DO_CLRATN,
 					    TRMREG_SCSI_CONTROL);
 				}
 			} else if (pDCB->SyncMode & SYNC_NEGO_ENABLE) { 
 			  /* do sync nego reject */
 				trm_reg_write16(DO_CLRATN,TRMREG_SCSI_CONTROL);
 				if (pSRB->SRBState & SRB_DO_SYNC_NEGO) {
 					pDCB = pSRB->pSRBDCB;
 					pDCB->SyncMode &= 
 					  ~(SYNC_NEGO_ENABLE+SYNC_NEGO_DONE); 
 					pDCB->SyncPeriod = 0;
 					pDCB->SyncOffset = 0;
 					/*               
 					 *
 					 *   program SCSI control register
 					 *
 					 */
 					trm_reg_write8(pDCB->SyncPeriod,
 					    TRMREG_SCSI_SYNC);
 					trm_reg_write8(pDCB->SyncOffset,
 					    TRMREG_SCSI_OFFSET);
 					trm_SetXferRate(pACB,pSRB,pDCB);
 				}
 			}
 			*pscsi_status = PH_BUS_FREE; /* .. initial phase */
 			/* it's important for atn stop */
 			trm_reg_write16(DO_DATALATCH, TRMREG_SCSI_CONTROL);
 			/*
 			 * SCSI command
 			 */
 			trm_reg_write8(SCMD_MSGACCEPT, TRMREG_SCSI_COMMAND);
 			return;
 		} else if (message_in_code == MSG_IGNOREWIDE) {
 			trm_reg_write32(1, TRMREG_SCSI_COUNTER);
 			trm_reg_read8(TRMREG_SCSI_FIFO);
 			*pscsi_status = PH_BUS_FREE; /* .. initial phase */
 			/* it's important for atn stop */
 			trm_reg_write16(DO_DATALATCH, TRMREG_SCSI_CONTROL);
 			/*
 			 * SCSI command
 			 */
 			trm_reg_write8(SCMD_MSGACCEPT, TRMREG_SCSI_COMMAND);
 			return;
 		} else {
 	  	  /* Restore data pointer message */
   		  /* Save data pointer message	  */
 		  /* Completion message		  */
 		  /* NOP message       	          */
 			*pscsi_status = PH_BUS_FREE; /* .. initial phase */
 			/* it's important for atn stop */
 			trm_reg_write16(DO_DATALATCH, TRMREG_SCSI_CONTROL);
 			/*
 			 * SCSI command
 			 */
 			trm_reg_write8(SCMD_MSGACCEPT, TRMREG_SCSI_COMMAND);
 			return;
 		}
 	} else {	
 	  /* 
    	   * Parsing incomming extented messages 
 	   */
 		*pSRB->pMsgPtr = message_in_code;
 		pSRB->MsgCnt++;
 		pSRB->pMsgPtr++;
 		TRM_DPRINTF("pSRB->MsgInBuf[0]=%2x \n ",pSRB->MsgInBuf[0]);
 		TRM_DPRINTF("pSRB->MsgInBuf[1]=%2x \n ",pSRB->MsgInBuf[1]);
 		TRM_DPRINTF("pSRB->MsgInBuf[2]=%2x \n ",pSRB->MsgInBuf[2]);
 		TRM_DPRINTF("pSRB->MsgInBuf[3]=%2x \n ",pSRB->MsgInBuf[3]);
 		TRM_DPRINTF("pSRB->MsgInBuf[4]=%2x \n ",pSRB->MsgInBuf[4]);
 	    	if ((pSRB->MsgInBuf[0] >= MSG_SIMPLE_QTAG)
 		    && (pSRB->MsgInBuf[0] <= MSG_ORDER_QTAG)) {
 		  /*
 	   	   * is QUEUE tag message :
    		   *
 	   	   * byte 0:
 	   	   * HEAD    QUEUE TAG (20h)
 	   	   * ORDERED QUEUE TAG (21h)
 	   	   * SIMPLE  QUEUE TAG (22h)
 	   	   * byte 1:
 	   	   * Queue tag (00h - FFh)
 	   	   */
 			if (pSRB->MsgCnt == 2) {
 				pSRB->SRBState = 0;
 				message_in_tag_id = pSRB->MsgInBuf[1];
 				pSRB = pDCB->pGoingSRB;
 				pSRBTemp = pDCB->pGoingLastSRB;
 				if (pSRB) {
 					for (;;) {
 						if (pSRB->TagNumber != 
 						    message_in_tag_id) {
 							if (pSRB == pSRBTemp) {
 								goto  mingx0;
 							}
 							pSRB = pSRB->pNextSRB;
 						} else
 							break;
 					}
 					if (pDCB->DCBFlag & ABORT_DEV_) {
 						pSRB->SRBState = SRB_ABORT_SENT;
 						trm_EnableMsgOutAbort1(
 						    pACB, pSRB);
 					}
 					if (!(pSRB->SRBState & SRB_DISCONNECT)) {
 						TRM_DPRINTF("SRB not yet disconnect........ \n ");
 						goto  mingx0;
 					}
 					pDCB->pActiveSRB = pSRB;
 					pSRB->SRBState = SRB_DATA_XFER;
 				} else {
 mingx0:
 	     				pSRB = &pACB->TmpSRB;
 					pSRB->SRBState = SRB_UNEXPECT_RESEL;
 					pDCB->pActiveSRB = pSRB;
 					pSRB->MsgOutBuf[0] = MSG_ABORT_TAG;
 					trm_EnableMsgOutAbort2(
 					    pACB,
 					    pSRB);
 				}
 			}
 			*pscsi_status = PH_BUS_FREE;
 			/* .. initial phase */
 			trm_reg_write16(DO_DATALATCH, TRMREG_SCSI_CONTROL);
 			/* it's important for atn stop */
 			/*
 			 * SCSI command 
 			 */
 			trm_reg_write8(SCMD_MSGACCEPT, TRMREG_SCSI_COMMAND);
 			return;
 		} else if ((pSRB->MsgInBuf[0] == MSG_EXTENDED) &&
 		    (pSRB->MsgInBuf[2] == 3) && (pSRB->MsgCnt == 4)) {
 		  /*
 	   	   * is Wide data xfer Extended message :
 	   	   * ======================================
 	   	   * WIDE DATA TRANSFER REQUEST
    		   * ======================================
 		   * byte 0 :  Extended message (01h)
 		   * byte 1 :  Extended message length (02h)
 		   * byte 2 :  WIDE DATA TRANSFER code (03h)
 		   * byte 3 :  Transfer width exponent 
 		   */
 			pDCB = pSRB->pSRBDCB;
 			pSRB->SRBState &= ~(SRB_EXTEND_MSGIN+SRB_DO_WIDE_NEGO);
 			if ((pSRB->MsgInBuf[1] != 2)) {
 			  /* Length is wrong, reject it  */
 				pDCB->SyncMode &=
 				  ~(WIDE_NEGO_ENABLE+WIDE_NEGO_DONE); 
 				pSRB->MsgCnt = 1;
 				pSRB->MsgInBuf[0] = MSG_REJECT_;
 				trm_reg_write16(DO_SETATN, TRMREG_SCSI_CONTROL);
 				*pscsi_status = PH_BUS_FREE; /* .. initial phase */
 				/* it's important for atn stop */
 				trm_reg_write16(DO_DATALATCH, TRMREG_SCSI_CONTROL);
 				/*
 				 * SCSI command
 				 */
 				trm_reg_write8(SCMD_MSGACCEPT, TRMREG_SCSI_COMMAND);
 				return;
 			}
 			if (pDCB->SyncMode & WIDE_NEGO_ENABLE) {		
 			  /* Do wide negoniation */
 				if (pSRB->MsgInBuf[3] > 2) {
 				  /* > 32 bit	*/
 				  /* reject_msg: */
 					pDCB->SyncMode &= 
 					  ~(WIDE_NEGO_ENABLE+WIDE_NEGO_DONE); 
 					pSRB->MsgCnt = 1;
 					pSRB->MsgInBuf[0] = MSG_REJECT_;
 					trm_reg_write16(DO_SETATN,
 					    TRMREG_SCSI_CONTROL);
 					*pscsi_status = PH_BUS_FREE; /* .. initial phase */
 					/* it's important for atn stop */
 					trm_reg_write16(DO_DATALATCH, TRMREG_SCSI_CONTROL);
 					/*
 					 * SCSI command
 					 */
 					trm_reg_write8(SCMD_MSGACCEPT, TRMREG_SCSI_COMMAND);
 					return;
 				}
 				if (pSRB->MsgInBuf[3] == 2) {
 					pSRB->MsgInBuf[3] = 1;
 					/* do 16 bits	*/
 				} else {
 					if (!(pDCB->SyncMode 
 					      & WIDE_NEGO_DONE)) {
 						pSRB->SRBState &=
 						  ~(SRB_DO_WIDE_NEGO+SRB_MSGIN);
 						pDCB->SyncMode |= 
 						  WIDE_NEGO_DONE;
 						pDCB->SyncMode &=
 						  ~(SYNC_NEGO_DONE |
 						      EN_ATN_STOP |
 						      WIDE_NEGO_ENABLE);
 						if (pSRB->MsgInBuf[3] != 0) {
 						  /* is Wide data xfer */
 							pDCB->SyncPeriod |=
 							  WIDE_SYNC;
 							pDCB->tinfo.current.width 
 							  = MSG_EXT_WDTR_BUS_16_BIT;
 							pDCB->tinfo.goal.width
 							  = MSG_EXT_WDTR_BUS_16_BIT;
 						}
 					}
 				}
 			} else
 				pSRB->MsgInBuf[3] = 0;
 			pSRB->SRBState |= SRB_MSGOUT;
 			trm_reg_write16(DO_SETATN,TRMREG_SCSI_CONTROL);
 			*pscsi_status = PH_BUS_FREE; /* .. initial phase */
 			/* it's important for atn stop */
 			trm_reg_write16(DO_DATALATCH, TRMREG_SCSI_CONTROL);
 			/*
 			 * SCSI command
 			 */
 			trm_reg_write8(SCMD_MSGACCEPT, TRMREG_SCSI_COMMAND);
 			return;
 		} else if ((pSRB->MsgInBuf[0] == MSG_EXTENDED) &&
 		    (pSRB->MsgInBuf[2] == 1) && (pSRB->MsgCnt == 5)) {
 			/*
 			 * is 8bit transfer Extended message :
 			 * =================================
 			 * SYNCHRONOUS DATA TRANSFER REQUEST
 			 * =================================
 			 * byte 0 :  Extended message (01h)
 			 * byte 1 :  Extended message length (03)
 			 * byte 2 :  SYNCHRONOUS DATA TRANSFER code (01h)
 			 * byte 3 :  Transfer period factor 
 			 * byte 4 :  REQ/ACK offset  
 			 */
 			pSRB->SRBState &= ~(SRB_EXTEND_MSGIN+SRB_DO_SYNC_NEGO);
 			if ((pSRB->MsgInBuf[1] != 3) ||
 			    (pSRB->MsgInBuf[2] != 1)) {
 			  /* reject_msg: */
 				pSRB->MsgCnt = 1;
 				pSRB->MsgInBuf[0] = MSG_REJECT_;
 				trm_reg_write16(DO_SETATN, TRMREG_SCSI_CONTROL);
 				*pscsi_status = PH_BUS_FREE;
 				/* .. initial phase */
 				trm_reg_write16(DO_DATALATCH, TRMREG_SCSI_CONTROL);
 				/* it's important for atn stop */
 				/*
 				 * SCSI cammand 
 				 */
 				trm_reg_write8(SCMD_MSGACCEPT, TRMREG_SCSI_COMMAND);
 				return;
 			} else if (!(pSRB->MsgInBuf[3]) || !(pSRB->MsgInBuf[4])) {
 				/* set async */
 				pDCB = pSRB->pSRBDCB;
 				/* disable sync & sync nego */
 				pDCB->SyncMode &= 
 				  ~(SYNC_NEGO_ENABLE+SYNC_NEGO_DONE);
 				pDCB->SyncPeriod = 0;
 				pDCB->SyncOffset = 0;
 				pDCB->tinfo.goal.period = 0;
 				pDCB->tinfo.goal.offset = 0;
 				pDCB->tinfo.current.period = 0;
 				pDCB->tinfo.current.offset = 0;
 				pDCB->tinfo.current.width = 
 				  MSG_EXT_WDTR_BUS_8_BIT;
 				/*               
 				 *
 				 *   program SCSI control register
 				 *
 				 */
 				trm_reg_write8(pDCB->SyncPeriod,TRMREG_SCSI_SYNC);
 				trm_reg_write8(pDCB->SyncOffset,TRMREG_SCSI_OFFSET);
 				trm_SetXferRate(pACB,pSRB,pDCB);
 				*pscsi_status = PH_BUS_FREE;
 				/* .. initial phase */
 				trm_reg_write16(DO_DATALATCH, TRMREG_SCSI_CONTROL);
 				/* it's important for atn stop */
 				/*
 				 * SCSI cammand 
 				 */
 				trm_reg_write8(SCMD_MSGACCEPT, TRMREG_SCSI_COMMAND);
 				return;
 			} else {
 				/* set sync */
 				pDCB = pSRB->pSRBDCB;
 				pDCB->SyncMode |= 
 				  SYNC_NEGO_ENABLE+SYNC_NEGO_DONE;
 				pDCB->MaxNegoPeriod = pSRB->MsgInBuf[3];
 				/* Transfer period factor */
 				pDCB->SyncOffset = pSRB->MsgInBuf[4]; 
 				/* REQ/ACK offset */
 				if (pACB->AdaptType == 1) {
 					for(bIndex = 0; bIndex < 7; bIndex++) {
 						if (pSRB->MsgInBuf[3] <=
 					   dc395u2x_clock_period[bIndex]) {
 				            pDCB->tinfo.goal.period =
 						dc395u2x_tinfo_period[bIndex];
 				            pDCB->tinfo.current.period =
 						dc395u2x_tinfo_period[bIndex];
 			                pDCB->tinfo.goal.offset = 
 					    pDCB->SyncOffset;
 					pDCB->tinfo.current.offset = 
 					    pDCB->SyncOffset;
 					pDCB->SyncPeriod |= (bIndex|LVDS_SYNC);
 					break;
 						}
 					}
 				} else {
 					for(bIndex = 0; bIndex < 7; bIndex++) {
 						if (pSRB->MsgInBuf[3] <=
 						 dc395x_clock_period[bIndex]) {
 						   pDCB->tinfo.goal.period =
 						dc395x_tinfo_period[bIndex];
 						   pDCB->tinfo.current.period =
 						dc395x_tinfo_period[bIndex];
 						   pDCB->tinfo.goal.offset =
 						pDCB->SyncOffset;
 						   pDCB->tinfo.current.offset =
 					     	       pDCB->SyncOffset;
 					     	   pDCB->SyncPeriod |=
 				     		       (bIndex|ALT_SYNC);
 			     			   break;
 						}
 					}
 				}
 				/*               
 				 *
 	 			 *   program SCSI control register
 	 			 *
 	 			 */
 				trm_reg_write8(pDCB->SyncPeriod,
 				    TRMREG_SCSI_SYNC);
 				trm_reg_write8(pDCB->SyncOffset,
 				    TRMREG_SCSI_OFFSET);
 				trm_SetXferRate(pACB,pSRB,pDCB);
 				*pscsi_status=PH_BUS_FREE;/*.. initial phase*/
 				trm_reg_write16(DO_DATALATCH,TRMREG_SCSI_CONTROL);/* it's important for atn stop*/
 	            /*
 	            ** SCSI command 
 	            */
 				trm_reg_write8(SCMD_MSGACCEPT,TRMREG_SCSI_COMMAND);
 				return;
 			}
 		}
 	*pscsi_status = PH_BUS_FREE;
 	/* .. initial phase */
 	trm_reg_write16(DO_DATALATCH, TRMREG_SCSI_CONTROL);
 	/* it's important for atn stop */
 	/*
 	 * SCSI cammand 
 	 */
 	trm_reg_write8(SCMD_MSGACCEPT, TRMREG_SCSI_COMMAND);
 	}
 }
 
 static void
 trm_MsgInPhase1(PACB pACB, PSRB pSRB, u_int16_t *pscsi_status)
 {
 
 	trm_reg_write16(DO_CLRFIFO, TRMREG_SCSI_CONTROL);
 	trm_reg_write32(1,TRMREG_SCSI_COUNTER);
 	if (!(pSRB->SRBState & SRB_MSGIN)) {
 		pSRB->SRBState &= SRB_DISCONNECT;
 		pSRB->SRBState |= SRB_MSGIN;
 	}
 	trm_reg_write16(DO_DATALATCH, TRMREG_SCSI_CONTROL);
 	/* it's important for atn stop*/
 	/*
 	 * SCSI cammand 
 	 */
 	trm_reg_write8(SCMD_FIFO_IN, TRMREG_SCSI_COMMAND);
 }
 
 static void
 trm_Nop0(PACB pACB, PSRB pSRB, u_int16_t *pscsi_status)
 {
 
 }
 
 static void
 trm_Nop1(PACB pACB, PSRB pSRB, u_int16_t *pscsi_status)
 {
 
 }
 
 static void
 trm_SetXferRate(PACB pACB,PSRB pSRB, PDCB pDCB)
 {
 	union ccb	*pccb;
 	struct ccb_trans_settings neg;
 	u_int16_t	cnt, i;
 	u_int8_t	bval;
 	PDCB		pDCBTemp;
 
 	/*
 	 * set all lun device's  period , offset
 	 */
 	TRM_DPRINTF("trm_SetXferRate\n");
 	pccb = pSRB->pccb;
 	memset(&neg, 0, sizeof (neg));
 	neg.xport_specific.spi.sync_period = pDCB->tinfo.goal.period;
 	neg.xport_specific.spi.sync_offset = pDCB->tinfo.goal.offset;
 	neg.xport_specific.spi.valid =
 	    CTS_SPI_VALID_SYNC_RATE | CTS_SPI_VALID_SYNC_OFFSET;
 	xpt_setup_ccb(&neg.ccb_h, pccb->ccb_h.path, /* priority */1);
 	xpt_async(AC_TRANSFER_NEG, pccb->ccb_h.path, &neg);
 	if (!(pDCB->IdentifyMsg & 0x07)) {
 		pDCBTemp = pACB->pLinkDCB;
 		cnt = pACB->DeviceCnt;
 		bval = pDCB->TargetID;
 		for (i = 0; i < cnt; i++) {
 			if (pDCBTemp->TargetID == bval) {
 				pDCBTemp->SyncPeriod = pDCB->SyncPeriod;
 				pDCBTemp->SyncOffset = pDCB->SyncOffset;
 				pDCBTemp->SyncMode = pDCB->SyncMode;
 			}
 			pDCBTemp = pDCBTemp->pNextDCB;
 		}
 	}
 	return;
 }
 
 /*
  * scsiiom		
  *            trm_Interrupt       
  *		        
  *
  *    ---SCSI bus phase
  *
  * 	PH_DATA_OUT	        0x00	 Data out phase	              
  * 	PH_DATA_IN	        0x01	 Data in phase	            
  * 	PH_COMMAND	        0x02	 Command phase	 
  * 	PH_STATUS	        0x03	 Status phase
  *	PH_BUS_FREE	        0x04	 Invalid phase used as bus free	
  * 	PH_BUS_FREE	        0x05	 Invalid phase used as bus free	
  * 	PH_MSG_OUT	        0x06	 Message out phase
  * 	PH_MSG_IN	        0x07	 Message in phase
  *
  */
 static void 
 trm_Disconnect(PACB pACB)
 {
 	PDCB		pDCB;
 	PSRB		pSRB, psrb;
 	u_int16_t	i,j, cnt;
 	u_int		target_id,target_lun;
 	
 	TRM_DPRINTF("trm_Disconnect...............\n ");
 	
        	pDCB = pACB->pActiveDCB;
 	if (!pDCB) {
 		TRM_DPRINTF(" Exception Disconnect DCB=NULL..............\n ");
 		j = 400;
     		while (--j) 
 			DELAY(1);
 		/* 1 msec */
 		trm_reg_write16((DO_CLRFIFO | DO_HWRESELECT),
 		    TRMREG_SCSI_CONTROL);
 		return;
 	}
 	pSRB = pDCB->pActiveSRB; 
 	/* bug pSRB=0 */
 	target_id  = pSRB->pccb->ccb_h.target_id;
 	target_lun = pSRB->pccb->ccb_h.target_lun;
 	TRM_DPRINTF(":pDCB->pActiveSRB= %8x \n ",(u_int) pDCB->pActiveSRB);
 	pACB->pActiveDCB = 0;
 	pSRB->ScsiPhase = PH_BUS_FREE; 
 	/* SCSI bus free Phase */
 	trm_reg_write16((DO_CLRFIFO | DO_HWRESELECT), TRMREG_SCSI_CONTROL);
 	if (pSRB->SRBState & SRB_UNEXPECT_RESEL) {
 		pSRB->SRBState = 0;
 		trm_DoWaitingSRB(pACB);
 	} else if (pSRB->SRBState & SRB_ABORT_SENT) {
 		pDCB->DCBFlag = 0;
 		cnt = pDCB->GoingSRBCnt;
 		pDCB->GoingSRBCnt = 0;
 		pSRB = pDCB->pGoingSRB;
 		for (i = 0; i < cnt; i++) {
 			psrb = pSRB->pNextSRB;
 			pSRB->pNextSRB = pACB->pFreeSRB;
 			pACB->pFreeSRB = pSRB;
 			pSRB = psrb;
 		}
 		pDCB->pGoingSRB = 0;
 		trm_DoWaitingSRB(pACB);
 	} else {
 		if ((pSRB->SRBState & (SRB_START_+SRB_MSGOUT)) || 
 		    !(pSRB->SRBState & (SRB_DISCONNECT+SRB_COMPLETED))) {
 		  /* Selection time out */
 			if (!(pACB->scan_devices[target_id][target_lun]) &&
 			    pSRB->CmdBlock[0] != 0x00 && /* TEST UNIT READY */
 			    pSRB->CmdBlock[0] != INQUIRY) {
 				pSRB->SRBState = SRB_READY;
 				trm_RewaitSRB(pDCB, pSRB);
 			} else {
 				pSRB->TargetStatus = SCSI_STAT_SEL_TIMEOUT;
 				goto  disc1;
 			}
 		} else if (pSRB->SRBState & SRB_DISCONNECT) {
 			/*
 			 * SRB_DISCONNECT
 			 */
 			trm_DoWaitingSRB(pACB);
 		} else if (pSRB->SRBState & SRB_COMPLETED) {
 disc1:
 		  /*
 		   * SRB_COMPLETED
 		   */
 			pDCB->pActiveSRB = 0;
 			pSRB->SRBState = SRB_FREE;
 			trm_SRBdone(pACB, pDCB, pSRB);
 		}
 	}
 	return;
 }
 
 static void
 trm_Reselect(PACB pACB)
 {
 	PDCB		pDCB;
 	PSRB		pSRB;
 	u_int16_t	RselTarLunId;
 
 	TRM_DPRINTF("trm_Reselect................. \n");
 	pDCB = pACB->pActiveDCB;
 	if (pDCB) {
 	  /* Arbitration lost but Reselection win */
 		pSRB = pDCB->pActiveSRB;
 		pSRB->SRBState = SRB_READY;
 		trm_RewaitSRB(pDCB, pSRB);
 	}
 	/* Read Reselected Target Id and LUN */
 	RselTarLunId = trm_reg_read16(TRMREG_SCSI_TARGETID) & 0x1FFF;
 	pDCB = pACB->pLinkDCB;
 	while (RselTarLunId != *((u_int16_t *) &pDCB->TargetID)) {
 	  /* get pDCB of the reselect id */
 		pDCB = pDCB->pNextDCB;
 	}
 
 	pACB->pActiveDCB = pDCB;
 	if (pDCB->SyncMode & EN_TAG_QUEUING) {
 		pSRB = &pACB->TmpSRB;
 		pDCB->pActiveSRB = pSRB;
 	} else {
 		pSRB = pDCB->pActiveSRB;
 		if (!pSRB || !(pSRB->SRBState & SRB_DISCONNECT)) {
 		  /*
 	   	   * abort command
    		   */
 			pSRB = &pACB->TmpSRB;
 			pSRB->SRBState = SRB_UNEXPECT_RESEL;
 			pDCB->pActiveSRB = pSRB;
 			trm_EnableMsgOutAbort1(pACB, pSRB);
 		} else {
 			if (pDCB->DCBFlag & ABORT_DEV_) {
 				pSRB->SRBState = SRB_ABORT_SENT;
 				trm_EnableMsgOutAbort1(pACB, pSRB);
 			} else 
 				pSRB->SRBState = SRB_DATA_XFER;
 		}
 	}
 	pSRB->ScsiPhase = PH_BUS_FREE;
 	/* SCSI bus free Phase */
 	/* 
 	 * Program HA ID, target ID, period and offset
 	 */
 	trm_reg_write8((u_int8_t) RselTarLunId,TRMREG_SCSI_TARGETID);
 	/* target ID */
 	trm_reg_write8(pACB->AdaptSCSIID,TRMREG_SCSI_HOSTID);
 	/* host   ID */
 	trm_reg_write8(pDCB->SyncPeriod,TRMREG_SCSI_SYNC);
 	/* period    */
 	trm_reg_write8(pDCB->SyncOffset,TRMREG_SCSI_OFFSET); 
 	/* offset    */
 	trm_reg_write16(DO_DATALATCH, TRMREG_SCSI_CONTROL);
 	/* it's important for atn stop*/
 	/*
 	 * SCSI cammand 
 	 */
 	trm_reg_write8(SCMD_MSGACCEPT, TRMREG_SCSI_COMMAND);
 	/* to rls the /ACK signal */
 }
 
 static void
 trm_SRBdone(PACB pACB, PDCB pDCB, PSRB pSRB)
 {
 	PSRB			psrb;
 	u_int8_t		bval, bval1,status;
 	union ccb		*pccb;
 	struct ccb_scsiio	*pcsio;
 	PSCSI_INQDATA		ptr;
 	int			intflag;
 	u_int			target_id,target_lun;
 	PDCB			pTempDCB;
 
 	pccb  = pSRB->pccb;
 	if (pccb == NULL)
 		return;
 	pcsio = &pccb->csio;
 	target_id  = pSRB->pccb->ccb_h.target_id;
 	target_lun = pSRB->pccb->ccb_h.target_lun;
 	if ((pccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE) {
 		bus_dmasync_op_t op;
 		if ((pccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN)
 			op = BUS_DMASYNC_POSTREAD;
 		else
 			op = BUS_DMASYNC_POSTWRITE;
 		bus_dmamap_sync(pACB->buffer_dmat, pSRB->dmamap, op);
 		bus_dmamap_unload(pACB->buffer_dmat, pSRB->dmamap);
 	}
     	/*
 	 *
 	 * target status
 	 *
 	 */
 	status = pSRB->TargetStatus;
 	pcsio->scsi_status=SCSI_STAT_GOOD;
 	pccb->ccb_h.status = CAM_REQ_CMP;
 	if (pSRB->SRBFlag & AUTO_REQSENSE) {
 	  /* 
    	   * status of auto request sense 
 	   */
 		pSRB->SRBFlag &= ~AUTO_REQSENSE;
 		pSRB->AdaptStatus = 0;
 		pSRB->TargetStatus = SCSI_STATUS_CHECK_COND;
 		
 		if (status == SCSI_STATUS_CHECK_COND) {
 			pccb->ccb_h.status = CAM_SEL_TIMEOUT;
 			goto ckc_e;
 		}
 		*((u_long *) &(pSRB->CmdBlock[0])) = pSRB->Segment0[0];
 		*((u_long *) &(pSRB->CmdBlock[4])) = pSRB->Segment0[1];
 		pSRB->SRBTotalXferLength = pSRB->Segment1[1];
 		pSRB->pSRBSGL->address = pSRB->SgSenseTemp.address;
 		pSRB->pSRBSGL->length = pSRB->SgSenseTemp.length;
 		pcsio->scsi_status = SCSI_STATUS_CHECK_COND;
 		bcopy(trm_get_sense_buf(pACB, pSRB), &pcsio->sense_data,
 		    pcsio->sense_len);
 		pcsio->ccb_h.status = CAM_SCSI_STATUS_ERROR 
 		    | CAM_AUTOSNS_VALID;
 		goto ckc_e;
 	}
 	/*
 	 * target status
 	 */
 	if (status) {
 		if (status == SCSI_STATUS_CHECK_COND) {
 			if ((pcsio->ccb_h.flags & CAM_DIS_AUTOSENSE) == 0) {
 			  TRM_DPRINTF("trm_RequestSense..................\n");
 			  trm_RequestSense(pACB, pDCB, pSRB);
 			  return;
 			}
 			pcsio->scsi_status = SCSI_STATUS_CHECK_COND;
 			pccb->ccb_h.status = CAM_SCSI_STATUS_ERROR;
 			goto ckc_e;
 		} else if (status == SCSI_STAT_QUEUEFULL) {
 			bval = (u_int8_t) pDCB->GoingSRBCnt;
 			bval--;
 			pDCB->MaxActiveCommandCnt = bval;
 			trm_RewaitSRB(pDCB, pSRB);
 			pSRB->AdaptStatus = 0;
 			pSRB->TargetStatus = 0;
 			return;
 		} else if (status == SCSI_STAT_SEL_TIMEOUT) {
 			pSRB->AdaptStatus  = H_SEL_TIMEOUT;
 			pSRB->TargetStatus = 0;
 			pcsio->scsi_status = SCSI_STAT_SEL_TIMEOUT;
 			pccb->ccb_h.status = CAM_SEL_TIMEOUT;
 		} else if (status == SCSI_STAT_BUSY) {
 			TRM_DPRINTF("trm: target busy at %s %d\n",
 				__FILE__, __LINE__);
 			pcsio->scsi_status = SCSI_STAT_BUSY;
 			pccb->ccb_h.status = CAM_SCSI_BUSY;
 			return;
 		  /* The device busy, try again later?	  */
 		} else if (status == SCSI_STAT_RESCONFLICT) {
 			TRM_DPRINTF("trm: target reserved at %s %d\n",
 				__FILE__, __LINE__);
 			pcsio->scsi_status = SCSI_STAT_RESCONFLICT;
 			pccb->ccb_h.status = CAM_SCSI_STATUS_ERROR;	/*XXX*/
 			return;
 		} else {
 			pSRB->AdaptStatus = 0;
 			if (pSRB->RetryCnt) {
 				pSRB->RetryCnt--;
 				pSRB->TargetStatus = 0;
 				pSRB->SRBSGIndex = 0;
 				if (trm_StartSCSI(pACB, pDCB, pSRB)) {
 				  /* 
 				   * If trm_StartSCSI return 1 :
 				   * current interrupt status is interrupt 
 				   * disreenable 
 				   * It's said that SCSI processor has more 
 				   * one SRB need to do
 				   */
 					trm_RewaitSRB(pDCB, pSRB);
 				}
 				return;
 			} else {
         			TRM_DPRINTF("trm: driver stuffup at %s %d\n",
 					__FILE__, __LINE__);
 		      		pccb->ccb_h.status = CAM_SCSI_STATUS_ERROR;
 			}
 		}
 	} else {
         /* 
  	 * process initiator status..........................
 	 * Adapter (initiator) status
 	 */
 		status = pSRB->AdaptStatus;
 		if (status & H_OVER_UNDER_RUN) {
 			pSRB->TargetStatus = 0;
 			pccb->ccb_h.status = CAM_DATA_RUN_ERR;
 			/* Illegal length (over/under run) */
 		} else if (pSRB->SRBStatus & PARITY_ERROR) {
 			TRM_DPRINTF("trm: driver stuffup %s %d\n",
 				__FILE__, __LINE__);
 			pDCB->tinfo.goal.period = 0;
 			pDCB->tinfo.goal.offset = 0;
 			/* Driver failed to perform operation */
 			pccb->ccb_h.status = CAM_UNCOR_PARITY;
 		} else {
 		  /* no error */
 			pSRB->AdaptStatus = 0;
 			pSRB->TargetStatus = 0;
 			pccb->ccb_h.status = CAM_REQ_CMP;
 			/* there is no error, (sense is invalid) */
 		}
 	}
 ckc_e:
 	if (pACB->scan_devices[target_id][target_lun]) {
 	  /*
 	   *   if SCSI command in "scan devices" duty
 	   */
 		if (pSRB->CmdBlock[0] == TEST_UNIT_READY) 
 			pACB->scan_devices[target_id][target_lun] = 0;
 		/* SCSI command phase :test unit ready */
 		else if (pSRB->CmdBlock[0] == INQUIRY) {
 		  /* 
 		   * SCSI command phase :inquiry scsi device data 
 		   * (type,capacity,manufacture.... 
 		   */
 			if (pccb->ccb_h.status == CAM_SEL_TIMEOUT)
 				goto NO_DEV;
 			ptr = (PSCSI_INQDATA) pcsio->data_ptr;
 			/* page fault */
 			TRM_DPRINTF("trm_SRBdone..PSCSI_INQDATA:%2x \n",
 			    ptr->DevType);
 		  	bval1 = ptr->DevType & SCSI_DEVTYPE; 
 			if (bval1 == SCSI_NODEV) {
 NO_DEV:
 	  			TRM_DPRINTF("trm_SRBdone NO Device:target_id= %d ,target_lun= %d \n",
 				    target_id,
 				    target_lun);
 		      		intflag = splcam();
 				pACB->scan_devices[target_id][target_lun] = 0;
 				/* no device set scan device flag =0*/
 				/* pDCB Q link */
 				/* move the head of DCB to tempDCB*/
 				pTempDCB=pACB->pLinkDCB;  
 				/* search current DCB for pass link */
 				while (pTempDCB->pNextDCB != pDCB) {
 					pTempDCB = pTempDCB->pNextDCB;
 				}
 				/*
 				 * when the current DCB found than connect 
 				 * current DCB tail 
 				 */
 				/* to the DCB tail that before current DCB */
 				pTempDCB->pNextDCB = pDCB->pNextDCB;
 				/*
 				 * if there was only one DCB ,connect his tail
 				 * to his head 
 				 */
 				if (pACB->pLinkDCB == pDCB)
 					pACB->pLinkDCB = pTempDCB->pNextDCB;
 				if (pACB->pDCBRunRobin == pDCB)
 					pACB->pDCBRunRobin = pTempDCB->pNextDCB;
 				pDCB->DCBstatus &= ~DS_IN_QUEUE;
 				pACB->DeviceCnt--;
 				if (pACB->DeviceCnt == 0) {
 					pACB->pLinkDCB = NULL;
 					pACB->pDCBRunRobin = NULL;
 				}
 				splx(intflag);
 			} else { 
 #ifdef trm_DEBUG1
 				int j;
 				for (j = 0; j < 28; j++) {
 					TRM_DPRINTF("ptr=%2x ", 
 						((u_int8_t *)ptr)[j]);
 				}
 #endif
 	      			pDCB->DevType = bval1;
 				if (bval1 == SCSI_DASD ||
 				    bval1 == SCSI_OPTICAL) {
 					if ((((ptr->Vers & 0x07) >= 2) ||
 					      ((ptr->RDF & 0x0F) == 2)) && 
 					    (ptr->Flags & SCSI_INQ_CMDQUEUE) &&
 					    (pDCB->DevMode & TAG_QUEUING_) &&
 					    (pDCB->DevMode & EN_DISCONNECT_)) {
 						if (pDCB->DevMode &
 						    TAG_QUEUING_) {
 							pDCB->
 							    MaxActiveCommandCnt = 
 							  pACB->TagMaxNum;
 							pDCB->SyncMode |= 
 							  EN_TAG_QUEUING;
 							pDCB->tinfo.disc_tag |=
 							  TRM_CUR_TAGENB;
 						} else {
 							pDCB->SyncMode |= 
 							  EN_ATN_STOP;
 							pDCB->tinfo.disc_tag &=
 							  ~TRM_CUR_TAGENB;
 						}
 					}
 				}
 			}
 			/* pSRB->CmdBlock[0] == INQUIRY */
 		}
 		/* pACB->scan_devices[target_id][target_lun] */
 	}
     	intflag = splcam();
 	/*  ReleaseSRB(pDCB, pSRB); */
 	if (pSRB == pDCB->pGoingSRB)
 		pDCB->pGoingSRB = pSRB->pNextSRB;
 	else {
 		psrb = pDCB->pGoingSRB;
 		while (psrb->pNextSRB != pSRB) {
 			psrb = psrb->pNextSRB;
 		}
 		psrb->pNextSRB = pSRB->pNextSRB;
 		if (pSRB == pDCB->pGoingLastSRB) {
 			pDCB->pGoingLastSRB = psrb;
 		}
 	}
 	pSRB->pNextSRB = pACB->pFreeSRB;
 	pACB->pFreeSRB = pSRB;
 	pDCB->GoingSRBCnt--;
 	trm_DoWaitingSRB(pACB);
 
 	splx(intflag);
 	/*  Notify cmd done */
 	xpt_done (pccb);
 }
 
 static void
 trm_DoingSRB_Done(PACB pACB)
 {
 	PDCB		pDCB, pdcb;
 	PSRB		psrb, psrb2;
 	u_int16_t	cnt, i;
 	union ccb 	*pccb;
 
 	pDCB = pACB->pLinkDCB;
 	if (pDCB == NULL) 
   		return;
 	pdcb = pDCB;
     	do {
 		cnt = pdcb->GoingSRBCnt;
 		psrb = pdcb->pGoingSRB;
 		for (i = 0; i < cnt; i++) {
 			psrb2 = psrb->pNextSRB;
 		    	pccb = psrb->pccb;
 			pccb->ccb_h.status = CAM_SEL_TIMEOUT;
 			/*  ReleaseSRB(pDCB, pSRB); */
 			psrb->pNextSRB = pACB->pFreeSRB;
 			pACB->pFreeSRB = psrb;
 			xpt_done(pccb);
 			psrb  = psrb2;
 		}
 		pdcb->GoingSRBCnt = 0;
 		pdcb->pGoingSRB = NULL;
 		pdcb = pdcb->pNextDCB;
 	}
 	while (pdcb != pDCB);
 }
 
 static void 
 trm_ResetSCSIBus(PACB pACB)
 {
 	int	intflag;
 
 	intflag = splcam();
     	pACB->ACBFlag |= RESET_DEV;
 
 	trm_reg_write16(DO_RSTSCSI,TRMREG_SCSI_CONTROL);
 	while (!(trm_reg_read16(TRMREG_SCSI_INTSTATUS) & INT_SCSIRESET));
 	splx(intflag);
 	return;
 }
 
 static void 
 trm_ScsiRstDetect(PACB pACB)
 {
 	int	intflag;
 	u_long	wlval;
 
 	TRM_DPRINTF("trm_ScsiRstDetect \n");
 	wlval = 1000;
 	while (--wlval)
 		DELAY(1000);
 	intflag = splcam();
     	trm_reg_write8(STOPDMAXFER,TRMREG_DMA_CONTROL);
 
 	trm_reg_write16(DO_CLRFIFO,TRMREG_SCSI_CONTROL);
 
 	if (pACB->ACBFlag & RESET_DEV)
 		pACB->ACBFlag |= RESET_DONE;
 	else {
 		pACB->ACBFlag |= RESET_DETECT;
 		trm_ResetDevParam(pACB);
 		/*	trm_DoingSRB_Done(pACB); ???? */
 		trm_RecoverSRB(pACB);
 		pACB->pActiveDCB = NULL;
 		pACB->ACBFlag = 0;
 		trm_DoWaitingSRB(pACB);
 	}
 	splx(intflag);
     	return;
 }
 
 static void
 trm_RequestSense(PACB pACB, PDCB pDCB, PSRB pSRB)	
 {
 	union ccb		*pccb;
 	struct ccb_scsiio	*pcsio;
 
 	pccb  = pSRB->pccb;
 	pcsio = &pccb->csio;
 
 	pSRB->SRBFlag |= AUTO_REQSENSE;
 	pSRB->Segment0[0] = *((u_long *) &(pSRB->CmdBlock[0]));
 	pSRB->Segment0[1] = *((u_long *) &(pSRB->CmdBlock[4]));
 	pSRB->Segment1[0] = (u_long) ((pSRB->ScsiCmdLen << 8) + 
 	    pSRB->SRBSGCount);
 	pSRB->Segment1[1] = pSRB->SRBTotalXferLength; /* ?????????? */
 
 	/* $$$$$$ Status of initiator/target $$$$$$$$ */
 	pSRB->AdaptStatus = 0;
 	pSRB->TargetStatus = 0;
 	/* $$$$$$ Status of initiator/target $$$$$$$$ */
 	
 	pSRB->SRBTotalXferLength = sizeof(pcsio->sense_data);
 	pSRB->SgSenseTemp.address = pSRB->pSRBSGL->address;
 	pSRB->SgSenseTemp.length  = pSRB->pSRBSGL->length;
 	pSRB->pSRBSGL->address = trm_get_sense_bufaddr(pACB, pSRB);
 	pSRB->pSRBSGL->length = (u_long) sizeof(struct scsi_sense_data);
 	pSRB->SRBSGCount = 1;
 	pSRB->SRBSGIndex = 0;
 	
 	*((u_long *) &(pSRB->CmdBlock[0])) = 0x00000003;
 	pSRB->CmdBlock[1] = pDCB->IdentifyMsg << 5;
 	*((u_int16_t *) &(pSRB->CmdBlock[4])) = pcsio->sense_len;
 	pSRB->ScsiCmdLen = 6;
 	
 	if (trm_StartSCSI(pACB, pDCB, pSRB))
 	   /* 
 	    * If trm_StartSCSI return 1 :
 	    * current interrupt status is interrupt disreenable 
 	    * It's said that SCSI processor has more one SRB need to do
 	    */
 		trm_RewaitSRB(pDCB, pSRB);
 }
 
 static void 
 trm_EnableMsgOutAbort2(PACB pACB, PSRB pSRB)
 {
 
 	pSRB->MsgCnt = 1;
 	trm_reg_write16(DO_SETATN, TRMREG_SCSI_CONTROL);
 }
 
 static void
 trm_EnableMsgOutAbort1(PACB pACB, PSRB pSRB)
 {
   
 	pSRB->MsgOutBuf[0] = MSG_ABORT;
 	trm_EnableMsgOutAbort2(pACB, pSRB);
 }
 
 static void
 trm_initDCB(PACB pACB, PDCB pDCB, u_int16_t unit,u_int32_t i,u_int32_t j)
 {
 	PNVRAMTYPE 	pEEpromBuf;
 	u_int8_t	bval,PeriodIndex;
 	u_int		target_id,target_lun;
 	PDCB		pTempDCB;
 	int		intflag;
     
     	target_id  = i;
 	target_lun = j;
 
 	/*
 	 *  Using the lun 0 device to init other DCB first, if the device 
 	 *  has been initialized.
 	 *  I don't want init sync arguments one by one, it is the same.
 	 */
 	if (target_lun != 0 && 
 	    (pACB->DCBarray[target_id][0].DCBstatus & DS_IN_QUEUE))
 		bcopy(&pACB->DCBarray[target_id][0], pDCB, 
 		    sizeof(TRM_DCB));
 	intflag = splcam();
 	if (pACB->pLinkDCB == 0) {
 		pACB->pLinkDCB = pDCB;
 		/* 
 		 * RunRobin impersonate the role 
 		 * that let each device had good proportion 
 		 * about SCSI command proceeding 
 		 */
 		pACB->pDCBRunRobin = pDCB;
 		pDCB->pNextDCB = pDCB;
 	} else {
 		pTempDCB=pACB->pLinkDCB;
 		/* search the last nod of DCB link */
 		while (pTempDCB->pNextDCB != pACB->pLinkDCB)
 			pTempDCB = pTempDCB->pNextDCB;
 		/* connect current DCB with last DCB tail */
 		pTempDCB->pNextDCB = pDCB;
 		/* connect current DCB tail to this DCB Q head */
 		pDCB->pNextDCB=pACB->pLinkDCB;
 	}
 	splx(intflag);
 
 	pACB->DeviceCnt++;
 	pDCB->TargetID = target_id;
 	pDCB->TargetLUN =  target_lun;
 	pDCB->pWaitingSRB = NULL;
 	pDCB->pGoingSRB = NULL;
 	pDCB->GoingSRBCnt = 0;
 	pDCB->pActiveSRB = NULL;
 	pDCB->MaxActiveCommandCnt = 1;
 	pDCB->DCBFlag = 0;
 	pDCB->DCBstatus |= DS_IN_QUEUE;
 	/* $$$$$$$ */
 	pEEpromBuf = &trm_eepromBuf[unit];
 	pDCB->DevMode = pEEpromBuf->NvramTarget[target_id].NvmTarCfg0;
 	pDCB->AdpMode = pEEpromBuf->NvramChannelCfg;
 	/* $$$$$$$ */
 	/* 
 	 * disconnect enable ?
 	 */
 	if (pDCB->DevMode & NTC_DO_DISCONNECT) {
 		bval = 0xC0;
 		pDCB->tinfo.disc_tag |= TRM_USR_DISCENB ;
 	} else {
 		bval = 0x80;
 		pDCB->tinfo.disc_tag &= ~(TRM_USR_DISCENB);
 	}
 	bval |= target_lun;
 	pDCB->IdentifyMsg = bval;
 	if (target_lun != 0 && 
 	    (pACB->DCBarray[target_id][0].DCBstatus & DS_IN_QUEUE))
 		return;
 	/* $$$$$$$ */
 	/*
 	 * tag Qing enable ?
 	 */
 	if (pDCB->DevMode & TAG_QUEUING_) {
 		pDCB->tinfo.disc_tag |= TRM_USR_TAGENB ;
 	} else
 		pDCB->tinfo.disc_tag &= ~(TRM_USR_TAGENB);
 	/* $$$$$$$ */
 	/*
 	 * wide nego ,sync nego enable ?
 	 */
 	pDCB->SyncPeriod = 0;
 	pDCB->SyncOffset = 0;
 	PeriodIndex = pEEpromBuf->NvramTarget[target_id].NvmTarPeriod & 0x07;
 	if (pACB->AdaptType==1) {/* is U2? */
 	    pDCB->MaxNegoPeriod=dc395u2x_clock_period[ PeriodIndex ];
 	    pDCB->tinfo.user.period=pDCB->MaxNegoPeriod;
 	    pDCB->tinfo.user.offset=(pDCB->SyncMode & SYNC_NEGO_ENABLE) ? 31 : 0;
 	} else {
         pDCB->MaxNegoPeriod=dc395x_clock_period[ PeriodIndex ];
 	    pDCB->tinfo.user.period=pDCB->MaxNegoPeriod;
 	    pDCB->tinfo.user.offset=(pDCB->SyncMode & SYNC_NEGO_ENABLE) ? 15 : 0;
 	}
 	pDCB->SyncMode = 0;
 	if ((pDCB->DevMode & NTC_DO_WIDE_NEGO) && 
 	    (pACB->Config & HCC_WIDE_CARD))
 		pDCB->SyncMode |= WIDE_NEGO_ENABLE;
 	/* enable wide nego */
    	if (pDCB->DevMode & NTC_DO_SYNC_NEGO)
 		pDCB->SyncMode |= SYNC_NEGO_ENABLE;
 	/* enable sync nego */
 	/* $$$$$$$ */
 	/*
 	 *	Fill in tinfo structure.
 	 */
 	pDCB->tinfo.user.width  = (pDCB->SyncMode & WIDE_NEGO_ENABLE) ? 
 	  MSG_EXT_WDTR_BUS_16_BIT : MSG_EXT_WDTR_BUS_8_BIT;
 
 	pDCB->tinfo.current.period = 0;
 	pDCB->tinfo.current.offset = 0;
 	pDCB->tinfo.current.width = MSG_EXT_WDTR_BUS_8_BIT;
 }
 
 static void
 trm_srbmapSG(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 {
 	PSRB pSRB;
 
 	pSRB=(PSRB) arg;
 	pSRB->SRBSGPhyAddr=segs->ds_addr;
 	return;
 }
 
 static void
 trm_destroySRB(PACB pACB)
 {
 	PSRB pSRB;
 
 	pSRB = pACB->pFreeSRB;
 	while (pSRB) {
-		if (pSRB->sg_dmamap) {
+		if (pSRB->SRBSGPhyAddr)
 			bus_dmamap_unload(pACB->sg_dmat, pSRB->sg_dmamap);
+		if (pSRB->pSRBSGL)
 			bus_dmamem_free(pACB->sg_dmat, pSRB->pSRBSGL,
 			    pSRB->sg_dmamap);
-			bus_dmamap_destroy(pACB->sg_dmat, pSRB->sg_dmamap);
-		}
 		if (pSRB->dmamap)
 			bus_dmamap_destroy(pACB->buffer_dmat, pSRB->dmamap);
 		pSRB = pSRB->pNextSRB;
 	}
 }
 
 static int
 trm_initSRB(PACB pACB)
 {
     	u_int16_t    i;
 	PSRB    pSRB;
 	int error;
 
 	for (i = 0; i < TRM_MAX_SRB_CNT; i++) {
 	       	pSRB = (PSRB)&pACB->pFreeSRB[i];
 
 		if (bus_dmamem_alloc(pACB->sg_dmat, (void **)&pSRB->pSRBSGL,
 		    BUS_DMA_NOWAIT, &pSRB->sg_dmamap) !=0 ) {
 			return ENXIO;
 		}
 		bus_dmamap_load(pACB->sg_dmat, pSRB->sg_dmamap, pSRB->pSRBSGL,
 		    TRM_MAX_SG_LISTENTRY * sizeof(SGentry),
 		    trm_srbmapSG, pSRB, /*flags*/0);
 		if (i != TRM_MAX_SRB_CNT - 1) {
 			/*
 			 * link all SRB 
 			 */
 			pSRB->pNextSRB = &pACB->pFreeSRB[i+1];
 		} else {
 			/*
 			 * load NULL to NextSRB of the last SRB
 			 */
 			pSRB->pNextSRB = NULL;
 		}
 		pSRB->TagNumber = i;
 
 		/*
 		 * Create the dmamap.  This is no longer optional!
 		 */
 		if ((error = bus_dmamap_create(pACB->buffer_dmat, 0,
 					       &pSRB->dmamap)) != 0)
 			return (error);
 
 	}
 	return (0);
 }
 
 
 
 
 static void
 trm_initACB(PACB pACB, u_int8_t adaptType, u_int16_t unit)
 {
 	PNVRAMTYPE	pEEpromBuf;
     
 	pEEpromBuf = &trm_eepromBuf[unit];
 	pACB->max_id = 15;
 	
 	if (pEEpromBuf->NvramChannelCfg & NAC_SCANLUN)
   		pACB->max_lun = 7;
 	else
 		pACB->max_lun = 0;
 
 	TRM_DPRINTF("trm: pACB->max_id= %d pACB->max_lun= %d \n",
 	    pACB->max_id, pACB->max_lun);
 	pACB->pLinkDCB = NULL;
 	pACB->pDCBRunRobin = NULL;
 	pACB->pActiveDCB = NULL;
 	pACB->AdapterUnit = (u_int8_t)unit;
 	pACB->AdaptSCSIID = pEEpromBuf->NvramScsiId;
 	pACB->AdaptSCSILUN = 0;
 	pACB->DeviceCnt = 0;
 	pACB->AdaptType = adaptType; 
 	pACB->TagMaxNum = 2 << pEEpromBuf->NvramMaxTag;
 	pACB->ACBFlag = 0;
 	return;
 }
 
 static void
 NVRAM_trm_write_all(PNVRAMTYPE pEEpromBuf,PACB pACB)
 {
 	u_int8_t	*bpEeprom = (u_int8_t *) pEEpromBuf;
 	u_int8_t	bAddr;
 
 	/* Enable SEEPROM */
 	trm_reg_write8((trm_reg_read8(TRMREG_GEN_CONTROL) | EN_EEPROM),
 	    TRMREG_GEN_CONTROL);
 	/*
 	 * Write enable
 	 */
 	NVRAM_trm_write_cmd(pACB, 0x04, 0xFF);
 	trm_reg_write8(0, TRMREG_GEN_NVRAM);
 	NVRAM_trm_wait_30us(pACB);
 	for (bAddr = 0; bAddr < 128; bAddr++, bpEeprom++) { 
 		NVRAM_trm_set_data(pACB, bAddr, *bpEeprom);
 	}
 	/* 
 	 * Write disable
 	 */
 	NVRAM_trm_write_cmd(pACB, 0x04, 0x00);
 	trm_reg_write8(0 , TRMREG_GEN_NVRAM);
 	NVRAM_trm_wait_30us(pACB);
 	/* Disable SEEPROM */
 	trm_reg_write8((trm_reg_read8(TRMREG_GEN_CONTROL) & ~EN_EEPROM),
 	    TRMREG_GEN_CONTROL);
 	return;
 }
 
 static void
 NVRAM_trm_set_data(PACB pACB, u_int8_t bAddr, u_int8_t bData)
 {
 	int		i;
 	u_int8_t	bSendData;
 	/* 
 	 * Send write command & address	
 	 */
 	
 	NVRAM_trm_write_cmd(pACB, 0x05, bAddr);
 	/* 
 	 * Write data 
 	 */
 	for (i = 0; i < 8; i++, bData <<= 1) {
 		bSendData = NVR_SELECT;
 		if (bData & 0x80)
 		  /* Start from bit 7	*/
 			bSendData |= NVR_BITOUT;
 		trm_reg_write8(bSendData , TRMREG_GEN_NVRAM);
 		NVRAM_trm_wait_30us(pACB);
 		trm_reg_write8((bSendData | NVR_CLOCK), TRMREG_GEN_NVRAM);
 		NVRAM_trm_wait_30us(pACB);
 	}
 	trm_reg_write8(NVR_SELECT , TRMREG_GEN_NVRAM);
 	NVRAM_trm_wait_30us(pACB);
 	/*
 	 * Disable chip select 
 	 */
 	trm_reg_write8(0 , TRMREG_GEN_NVRAM);
 	NVRAM_trm_wait_30us(pACB);
 	trm_reg_write8(NVR_SELECT ,TRMREG_GEN_NVRAM);
 	NVRAM_trm_wait_30us(pACB);
 	/* 
 	 * Wait for write ready	
 	 */
 	while (1) {
 		trm_reg_write8((NVR_SELECT | NVR_CLOCK), TRMREG_GEN_NVRAM);
 		NVRAM_trm_wait_30us(pACB);
 		trm_reg_write8(NVR_SELECT, TRMREG_GEN_NVRAM);
 		NVRAM_trm_wait_30us(pACB);
 		if (trm_reg_read8(TRMREG_GEN_NVRAM) & NVR_BITIN) {
 			break;
 		}
 	}
 	/* 
 	 * Disable chip select 
 	 */
 	trm_reg_write8(0, TRMREG_GEN_NVRAM);
 	return;
 }
 
 static void 
 NVRAM_trm_read_all(PNVRAMTYPE pEEpromBuf, PACB pACB)
 {
 	u_int8_t	*bpEeprom = (u_int8_t*) pEEpromBuf;
 	u_int8_t	bAddr;
     
 	/*
 	 * Enable SEEPROM 
 	 */
 	trm_reg_write8((trm_reg_read8(TRMREG_GEN_CONTROL) | EN_EEPROM),
 	    TRMREG_GEN_CONTROL);
 	for (bAddr = 0; bAddr < 128; bAddr++, bpEeprom++)
 		*bpEeprom = NVRAM_trm_get_data(pACB, bAddr);
 	/* 
 	 * Disable SEEPROM 
 	 */
 	trm_reg_write8((trm_reg_read8(TRMREG_GEN_CONTROL) & ~EN_EEPROM),
 	    TRMREG_GEN_CONTROL);
 	return;
 }
 
 static u_int8_t
 NVRAM_trm_get_data(PACB pACB, u_int8_t bAddr)
 {
 	int		i;
 	u_int8_t	bReadData, bData = 0;
 	/* 
 	* Send read command & address
 	*/
 	
 	NVRAM_trm_write_cmd(pACB, 0x06, bAddr);
 				
 	for (i = 0; i < 8; i++) {
 	  /* 
 	   * Read data
 	   */
 		trm_reg_write8((NVR_SELECT | NVR_CLOCK) , TRMREG_GEN_NVRAM);
 		NVRAM_trm_wait_30us(pACB);
 		trm_reg_write8(NVR_SELECT , TRMREG_GEN_NVRAM);
 		/* 
 		 * Get data bit while falling edge 
 		 */
 		bReadData = trm_reg_read8(TRMREG_GEN_NVRAM);
 		bData <<= 1;
 		if (bReadData & NVR_BITIN) {
 			bData |= 1;
 		}
 		NVRAM_trm_wait_30us(pACB);
 	}
 	/* 
 	 * Disable chip select 
 	 */
 	trm_reg_write8(0, TRMREG_GEN_NVRAM);
 	return (bData);
 }
 
 static void
 NVRAM_trm_wait_30us(PACB pACB)
 {
   
 	/*    ScsiPortStallExecution(30);	 wait 30 us	*/
 	trm_reg_write8(5, TRMREG_GEN_TIMER);
 	while (!(trm_reg_read8(TRMREG_GEN_STATUS) & GTIMEOUT));
 	return;
 }
 
 static void
 NVRAM_trm_write_cmd(PACB pACB, u_int8_t bCmd, u_int8_t bAddr)
 {
 	int		i;
 	u_int8_t	bSendData;
 					
     	for (i = 0; i < 3; i++, bCmd <<= 1) {
 	  /* 
    	   * Program SB+OP code		
    	   */
      		bSendData = NVR_SELECT;
 		if (bCmd & 0x04)	
 			bSendData |= NVR_BITOUT;
 		/* start from bit 2 */
 		trm_reg_write8(bSendData, TRMREG_GEN_NVRAM);
 		NVRAM_trm_wait_30us(pACB);
 		trm_reg_write8((bSendData | NVR_CLOCK), TRMREG_GEN_NVRAM);
 		NVRAM_trm_wait_30us(pACB);
 	}	
 	for (i = 0; i < 7; i++, bAddr <<= 1) {
 	  /* 
 	   * Program address		
 	   */
 		bSendData = NVR_SELECT;
 		if (bAddr & 0x40)	
 		  /* Start from bit 6	*/
 			bSendData |= NVR_BITOUT;
 		trm_reg_write8(bSendData , TRMREG_GEN_NVRAM);
 		NVRAM_trm_wait_30us(pACB);
 		trm_reg_write8((bSendData | NVR_CLOCK), TRMREG_GEN_NVRAM);
 		NVRAM_trm_wait_30us(pACB);
 	}
 	trm_reg_write8(NVR_SELECT, TRMREG_GEN_NVRAM);
 	NVRAM_trm_wait_30us(pACB);
 }
 
 static void
 trm_check_eeprom(PNVRAMTYPE pEEpromBuf, PACB pACB)
 {
 	u_int16_t	*wpEeprom = (u_int16_t *) pEEpromBuf;
 	u_int16_t	wAddr, wCheckSum;
 	u_long	dAddr, *dpEeprom;
 
 	NVRAM_trm_read_all(pEEpromBuf,pACB);
 	wCheckSum = 0;
 	for (wAddr = 0, wpEeprom = (u_int16_t *) pEEpromBuf;
 	    wAddr < 64; wAddr++, wpEeprom++) {
 		wCheckSum += *wpEeprom;
 	}
 	if (wCheckSum != 0x1234) {
 	  /* 
    	   * Checksum error, load default	
 	   */
 		pEEpromBuf->NvramSubVendorID[0]	= (u_int8_t) PCI_Vendor_ID_TEKRAM;
 		pEEpromBuf->NvramSubVendorID[1]	=
 		  (u_int8_t) (PCI_Vendor_ID_TEKRAM >> 8);
 		pEEpromBuf->NvramSubSysID[0] = (u_int8_t) PCI_Device_ID_TRM_S1040;
 		pEEpromBuf->NvramSubSysID[1] = 
 		  (u_int8_t) (PCI_Device_ID_TRM_S1040 >> 8);
 		pEEpromBuf->NvramSubClass = 0x00;
 		pEEpromBuf->NvramVendorID[0] = (u_int8_t) PCI_Vendor_ID_TEKRAM;
 		pEEpromBuf->NvramVendorID[1] =
 		  (u_int8_t) (PCI_Vendor_ID_TEKRAM >> 8);
 		pEEpromBuf->NvramDeviceID[0] = (u_int8_t) PCI_Device_ID_TRM_S1040;
 		pEEpromBuf->NvramDeviceID[1] = 
 		  (u_int8_t) (PCI_Device_ID_TRM_S1040 >> 8);
 		pEEpromBuf->NvramReserved = 0x00;
 
 		for (dAddr = 0, dpEeprom = (u_long *) pEEpromBuf->NvramTarget;
 		    dAddr < 16; dAddr++, dpEeprom++) {
 			*dpEeprom = 0x00000077;
 			/* NvmTarCfg3,NvmTarCfg2,NvmTarPeriod,NvmTarCfg0 */
 		}
 
 		*dpEeprom++ = 0x04000F07;
 		/* NvramMaxTag,NvramDelayTime,NvramChannelCfg,NvramScsiId */
 		*dpEeprom++ = 0x00000015;
 		/* NvramReserved1,NvramBootLun,NvramBootTarget,NvramReserved0 */
 		for (dAddr = 0; dAddr < 12; dAddr++, dpEeprom++)
 			*dpEeprom = 0x00;
 		pEEpromBuf->NvramCheckSum = 0x00;
 		for (wAddr = 0, wCheckSum = 0, wpEeprom = (u_int16_t *) pEEpromBuf;
 		    wAddr < 63; wAddr++, wpEeprom++)
 	      		wCheckSum += *wpEeprom;
 		*wpEeprom = 0x1234 - wCheckSum;
 		NVRAM_trm_write_all(pEEpromBuf,pACB);
 	}
 	return;
 }
 static int
 trm_initAdapter(PACB pACB, u_int16_t unit)
 {
 	PNVRAMTYPE	pEEpromBuf;
 	u_int16_t	wval;
 	u_int8_t	bval;
 
 	pEEpromBuf = &trm_eepromBuf[unit];
 
 	/* 250ms selection timeout */
 	trm_reg_write8(SEL_TIMEOUT, TRMREG_SCSI_TIMEOUT);
 	/* Mask all the interrupt */
 	trm_reg_write8(0x00, TRMREG_DMA_INTEN);    
 	trm_reg_write8(0x00, TRMREG_SCSI_INTEN);     
 	/* Reset SCSI module */
 	trm_reg_write16(DO_RSTMODULE, TRMREG_SCSI_CONTROL); 
 	/* program configuration 0 */
 	pACB->Config = HCC_AUTOTERM | HCC_PARITY;
 	if (trm_reg_read8(TRMREG_GEN_STATUS) & WIDESCSI)
 		pACB->Config |= HCC_WIDE_CARD;
 	if (pEEpromBuf->NvramChannelCfg & NAC_POWERON_SCSI_RESET)
 		pACB->Config |= HCC_SCSI_RESET;
 	if (pACB->Config & HCC_PARITY)
 		bval = PHASELATCH | INITIATOR | BLOCKRST | PARITYCHECK;
 	else
 		bval = PHASELATCH | INITIATOR | BLOCKRST ;
 	trm_reg_write8(bval,TRMREG_SCSI_CONFIG0); 
 	/* program configuration 1 */
 	trm_reg_write8(0x13, TRMREG_SCSI_CONFIG1); 
 	/* program Host ID */
 	bval = pEEpromBuf->NvramScsiId;
 	trm_reg_write8(bval, TRMREG_SCSI_HOSTID); 
 	/* set ansynchronous transfer */
 	trm_reg_write8(0x00, TRMREG_SCSI_OFFSET); 
 	/* Trun LED control off*/
 	wval = trm_reg_read16(TRMREG_GEN_CONTROL) & 0x7F;
 	trm_reg_write16(wval, TRMREG_GEN_CONTROL); 
 	/* DMA config */
 	wval = trm_reg_read16(TRMREG_DMA_CONFIG) | DMA_ENHANCE;
 	trm_reg_write16(wval, TRMREG_DMA_CONFIG); 
 	/* Clear pending interrupt status */
 	trm_reg_read8(TRMREG_SCSI_INTSTATUS);
 	/* Enable SCSI interrupt */
 	trm_reg_write8(0x7F, TRMREG_SCSI_INTEN); 
 	trm_reg_write8(EN_SCSIINTR, TRMREG_DMA_INTEN); 
 	return (0);
 }
 
 static void
 trm_mapSRB(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 {
 	PACB pACB;
 
 	pACB = (PACB)arg;
 	pACB->srb_physbase = segs->ds_addr;
 }
 
 static void
 trm_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 {
 	bus_addr_t *baddr;
 
 	baddr = (bus_addr_t *)arg;
 	*baddr = segs->ds_addr;
 }
 
 static PACB
 trm_init(u_int16_t unit, device_t dev)
 {
 	PACB		pACB;
 	int		rid = PCIR_BAR(0), i = 0, j = 0;
 	u_int16_t	adaptType = 0;
     
  	pACB = (PACB) device_get_softc(dev);
    	if (!pACB) {
 		printf("trm%d: cannot allocate ACB !\n", unit);
 		return (NULL);
 	}
 	pACB->iores = bus_alloc_resource_any(dev, SYS_RES_IOPORT, 
 	    &rid, RF_ACTIVE);
     	if (pACB->iores == NULL) {
 		printf("trm_init: bus_alloc_resource failed!\n");
 		return (NULL);
 	}
 	switch (pci_get_devid(dev)) {
 	case PCI_DEVICEID_TRMS1040:
 		adaptType = 0;
 		break;
 	case PCI_DEVICEID_TRMS2080:
 		adaptType = 1;
 		break;
 	default:
 		printf("trm_init %d: unknown adapter type!\n", unit);
 		goto bad;
 	}
 	pACB->dev = dev;
 	pACB->tag = rman_get_bustag(pACB->iores);
 	pACB->bsh = rman_get_bushandle(pACB->iores);
 	if (bus_dma_tag_create(
 	/*parent_dmat*/	bus_get_dma_tag(dev),
 	/*alignment*/	1,
 	/*boundary*/	0,
 	/*lowaddr*/	BUS_SPACE_MAXADDR,
 	/*highaddr*/	BUS_SPACE_MAXADDR,
 	/*filter*/	NULL, 
 	/*filterarg*/	NULL,
 	/*maxsize*/	BUS_SPACE_MAXSIZE_32BIT,
 	/*nsegments*/	BUS_SPACE_UNRESTRICTED,
 	/*maxsegsz*/	BUS_SPACE_MAXSIZE_32BIT,
 	/*flags*/	0,
 	/*lockfunc*/	NULL,
 	/*lockarg*/	NULL,
 	/* dmat */	&pACB->parent_dmat) != 0) 
 		goto bad;
 	if (bus_dma_tag_create(
 	/*parent_dmat*/	pACB->parent_dmat,
 	/*alignment*/	1,
 	/*boundary*/	0,
 	/*lowaddr*/	BUS_SPACE_MAXADDR,
 	/*highaddr*/	BUS_SPACE_MAXADDR,
 	/*filter*/	NULL, 
 	/*filterarg*/	NULL,
 	/*maxsize*/	MAXBSIZE,
 	/*nsegments*/	TRM_NSEG,
 	/*maxsegsz*/	TRM_MAXTRANSFER_SIZE,
 	/*flags*/	BUS_DMA_ALLOCNOW,
 	/*lockfunc*/	busdma_lock_mutex,
 	/*lockarg*/	&Giant,
 	/* dmat */	&pACB->buffer_dmat) != 0) 
 		goto bad;
 	/* DMA tag for our ccb structures */
 	if (bus_dma_tag_create(                                 
 	/*parent_dmat*/pACB->parent_dmat, 
 	/*alignment*/  1, 
 	/*boundary*/   0,
 	/*lowaddr*/    BUS_SPACE_MAXADDR,
 	/*highaddr*/   BUS_SPACE_MAXADDR,
 	/*filter*/     NULL, 
 	/*filterarg*/  NULL,
 	/*maxsize*/    TRM_MAX_SRB_CNT * sizeof(TRM_SRB),
 	/*nsegments*/  1,
 	/*maxsegsz*/   TRM_MAXTRANSFER_SIZE,
 	/*flags*/      0,
 	/*lockfunc*/   busdma_lock_mutex,
 	/*lockarg*/    &Giant,
 	/*dmat*/       &pACB->srb_dmat) != 0) {
 		printf("trm_init %d: bus_dma_tag_create SRB failure\n", unit);
 		goto bad;
 	}
 	if (bus_dmamem_alloc(pACB->srb_dmat, (void **)&pACB->pFreeSRB, 
 	    BUS_DMA_NOWAIT, &pACB->srb_dmamap) != 0) {
 		printf("trm_init %d: bus_dmamem_alloc SRB failure\n", unit);
 		goto bad;
 	}
 	bus_dmamap_load(pACB->srb_dmat, pACB->srb_dmamap, pACB->pFreeSRB,
 	    TRM_MAX_SRB_CNT * sizeof(TRM_SRB), trm_mapSRB, pACB, 
 	    /* flags */0);
 	/* Create, allocate, and map DMA buffers for autosense data */
 	if (bus_dma_tag_create(
 	    /*parent_dmat*/pACB->parent_dmat,
 	    /*alignment*/1,
 	    /*boundary*/0,
 	    /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
 	    /*highaddr*/BUS_SPACE_MAXADDR,
 	    /*filter*/NULL, /*filterarg*/NULL,
 	    sizeof(struct scsi_sense_data) * TRM_MAX_SRB_CNT,
 	    /*nsegments*/1,
 	    /*maxsegsz*/TRM_MAXTRANSFER_SIZE,
 	    /*flags*/0, /*lockfunc*/busdma_lock_mutex,
 	    /*lockarg*/&Giant, &pACB->sense_dmat) != 0) {
 	  if (bootverbose)
 	    device_printf(dev, "cannot create sense buffer dmat\n");
 	  goto bad;
 	}
 
 	if (bus_dmamem_alloc(pACB->sense_dmat, (void **)&pACB->sense_buffers,
 			     BUS_DMA_NOWAIT, &pACB->sense_dmamap) != 0)
 		goto bad;
 
 	bus_dmamap_load(pACB->sense_dmat, pACB->sense_dmamap,
 		       pACB->sense_buffers,
 		       sizeof(struct scsi_sense_data) * TRM_MAX_SRB_CNT,
 		       trm_dmamap_cb, &pACB->sense_busaddr, /*flags*/0);
 
 	trm_check_eeprom(&trm_eepromBuf[unit],pACB);
 	trm_initACB(pACB, adaptType, unit);
 	for (i = 0; i < (pACB->max_id + 1); i++) {   
 		if (pACB->AdaptSCSIID == i)
 			continue;
 		for(j = 0; j < (pACB->max_lun + 1); j++) {
 			pACB->scan_devices[i][j] = 1;
 			/* we assume we need to scan all devices */ 
 			trm_initDCB(pACB, &pACB->DCBarray[i][j], unit, i, j);
 		}
 	}
 	bzero(pACB->pFreeSRB, TRM_MAX_SRB_CNT * sizeof(TRM_SRB));
 	if (bus_dma_tag_create(                    
 		    /*parent_dmat*/pACB->parent_dmat,
 		    /*alignment*/  1,
 		    /*boundary*/   0,
 		    /*lowaddr*/    BUS_SPACE_MAXADDR,
 		    /*highaddr*/   BUS_SPACE_MAXADDR,
 		    /*filter*/     NULL, 
 		    /*filterarg*/  NULL,
 		    /*maxsize*/    TRM_MAX_SG_LISTENTRY * sizeof(SGentry), 
 		    /*nsegments*/  1,
 		    /*maxsegsz*/   TRM_MAXTRANSFER_SIZE,
 		    /*flags*/      0, 
 		    /*lockfunc*/   busdma_lock_mutex,
 		    /*lockarg*/    &Giant,
 		    /*dmat*/       &pACB->sg_dmat) != 0)
 		goto bad;
 
 	if (trm_initSRB(pACB)) {
 		printf("trm_initSRB: error\n");
 		goto bad;
 	}
    	if (trm_initAdapter(pACB, unit)) {
 		printf("trm_initAdapter: initial ERROR\n");
 		goto bad;
 	}
 	return (pACB);
 bad:
 	if (pACB->iores)
 		bus_release_resource(dev, SYS_RES_IOPORT, PCIR_BAR(0),
 		    pACB->iores);
 	if (pACB->sense_dmamap) {
 		bus_dmamap_unload(pACB->sense_dmat, pACB->sense_dmamap);
 		bus_dmamem_free(pACB->sense_dmat, pACB->sense_buffers,
 		    pACB->sense_dmamap);
-		bus_dmamap_destroy(pACB->sense_dmat, pACB->sense_dmamap);
 	}
 	if (pACB->sense_dmat)
 		bus_dma_tag_destroy(pACB->sense_dmat);
 	if (pACB->sg_dmat) {
 		trm_destroySRB(pACB);
 		bus_dma_tag_destroy(pACB->sg_dmat);
 	}
-	if (pACB->srb_dmamap) {
+	if (pACB->pFreeSRB) {
 		bus_dmamap_unload(pACB->srb_dmat, pACB->srb_dmamap);
 		bus_dmamem_free(pACB->srb_dmat, pACB->pFreeSRB, 
 		    pACB->srb_dmamap);
-		bus_dmamap_destroy(pACB->srb_dmat, pACB->srb_dmamap);
 	}
 	if (pACB->srb_dmat)
 		bus_dma_tag_destroy(pACB->srb_dmat);
 	if (pACB->buffer_dmat)
 		bus_dma_tag_destroy(pACB->buffer_dmat);
 	if (pACB->parent_dmat)
 		bus_dma_tag_destroy(pACB->parent_dmat);
 	return (NULL);
 }
 
 static int
 trm_attach(device_t dev)
 {
 	struct	cam_devq *device_Q;
 	u_long	device_id;
 	PACB	pACB = 0;
 	int	rid = 0;
 	int unit = device_get_unit(dev);
 	
 	device_id = pci_get_devid(dev);
 	/*
 	 * These cards do not allow memory mapped accesses
 	 */
 	if ((pACB = trm_init((u_int16_t) unit,
     	    dev)) == NULL) {
 		printf("trm%d: trm_init error!\n",unit);
 		return (ENXIO);
 	}
 	/* After setting up the adapter, map our interrupt */
 	/*  
 	 * Now let the CAM generic SCSI layer find the SCSI devices on the bus
 	 * start queue to reset to the idle loop.
 	 * Create device queue of SIM(s)
 	 * (MAX_START_JOB - 1) : max_sim_transactions
 	 */
 	pACB->irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
 	    RF_SHAREABLE | RF_ACTIVE);
     	if (pACB->irq == NULL ||
 	    bus_setup_intr(dev, pACB->irq, 
 	    INTR_TYPE_CAM, NULL, trm_Interrupt, pACB, &pACB->ih)) {
 		printf("trm%d: register Interrupt handler error!\n", unit);
 		goto bad;
 	}
 	device_Q = cam_simq_alloc(TRM_MAX_START_JOB);
 	if (device_Q == NULL){ 
 		printf("trm%d: device_Q == NULL !\n",unit);
 		goto bad;
 	}
 	/*
 	 * Now tell the generic SCSI layer
 	 * about our bus.
 	 * If this is the xpt layer creating a sim, then it's OK
 	 * to wait for an allocation.
 	 * XXX Should we pass in a flag to indicate that wait is OK?
 	 *
 	 *                    SIM allocation
 	 *
 	 *                 SCSI Interface Modules
 	 * The sim driver creates a sim for each controller.  The sim device
 	 * queue is separately created in order to allow resource sharing betwee
 	 * sims.  For instance, a driver may create one sim for each channel of
 	 * a multi-channel controller and use the same queue for each channel.
 	 * In this way, the queue resources are shared across all the channels
 	 * of the multi-channel controller.
 	 * trm_action     : sim_action_func
 	 * trm_poll       : sim_poll_func
 	 * "trm"        : sim_name ,if sim_name =  "xpt" ..M_DEVBUF,M_WAITOK
 	 * pACB         : *softc    if sim_name <> "xpt" ..M_DEVBUF,M_NOWAIT
 	 * pACB->unit   : unit
 	 * 1            : max_dev_transactions
 	 * MAX_TAGS     : max_tagged_dev_transactions
 	 *
 	 *  *******Construct our first channel SIM entry
 	 */
 	pACB->psim = cam_sim_alloc(trm_action,
 	    trm_poll,
 	    "trm",
 	    pACB,
 	    unit,
 	    &Giant,
 	    1,
 	    TRM_MAX_TAGS_CMD_QUEUE,
 	    device_Q);
 	if (pACB->psim == NULL) {
 		printf("trm%d: SIM allocate fault !\n",unit);
 		cam_simq_free(device_Q);  /* SIM allocate fault*/
 		goto bad;
 	}
 	if (xpt_bus_register(pACB->psim, dev, 0) != CAM_SUCCESS)  {
 		printf("trm%d: xpt_bus_register fault !\n",unit);
 		goto bad;
 	}
 	if (xpt_create_path(&pACB->ppath,
 	      NULL,
 	      cam_sim_path(pACB->psim),
 	      CAM_TARGET_WILDCARD,
 	      CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
 		printf("trm%d: xpt_create_path fault !\n",unit);
 		xpt_bus_deregister(cam_sim_path(pACB->psim));
 		goto bad;
 	}
 	return (0);
 bad:
 	if (pACB->iores)
 		bus_release_resource(dev, SYS_RES_IOPORT, PCIR_BAR(0),
 		    pACB->iores);
 	if (pACB->sg_dmat) {		
 		trm_destroySRB(pACB);
 		bus_dma_tag_destroy(pACB->sg_dmat);
 	}
 	
-	if (pACB->srb_dmamap) {
+	if (pACB->pFreeSRB) {
 		bus_dmamap_unload(pACB->srb_dmat, pACB->srb_dmamap);
 		bus_dmamem_free(pACB->srb_dmat, pACB->pFreeSRB, 
 		    pACB->srb_dmamap);
-		bus_dmamap_destroy(pACB->srb_dmat, pACB->srb_dmamap);
 	}
 	if (pACB->srb_dmat)
 		bus_dma_tag_destroy(pACB->srb_dmat);
-	if (pACB->sense_dmamap) {
+	if (pACB->sense_buffers) {
 	  	  bus_dmamap_unload(pACB->sense_dmat, pACB->sense_dmamap);
 		  bus_dmamem_free(pACB->sense_dmat, pACB->sense_buffers,
 		      pACB->sense_dmamap);
-		  bus_dmamap_destroy(pACB->sense_dmat, pACB->sense_dmamap);
 	}
 	if (pACB->sense_dmat)
 		bus_dma_tag_destroy(pACB->sense_dmat);		
 	if (pACB->buffer_dmat)
 		bus_dma_tag_destroy(pACB->buffer_dmat);
 	if (pACB->ih)
 		bus_teardown_intr(dev, pACB->irq, pACB->ih);
 	if (pACB->irq)
 		bus_release_resource(dev, SYS_RES_IRQ, 0, pACB->irq);
 	if (pACB->psim)
 		cam_sim_free(pACB->psim, TRUE);
 	
 	return (ENXIO);
 	
 }
 
 /*
 *                  pci_device
 *         trm_probe (device_t tag, pcidi_t type)
 *
 */
 static int
 trm_probe(device_t dev)
 {
   	switch (pci_get_devid(dev)) {
 	case PCI_DEVICEID_TRMS1040: 
 		device_set_desc(dev,
 		    "Tekram DC395U/UW/F DC315/U Fast20 Wide SCSI Adapter");
 		return (BUS_PROBE_DEFAULT);
 	case PCI_DEVICEID_TRMS2080:
 		device_set_desc(dev,
 		    "Tekram DC395U2D/U2W Fast40 Wide SCSI Adapter");
 		return (BUS_PROBE_DEFAULT);
 	default:
 		return (ENXIO);
 	}
 }
 
 static int
 trm_detach(device_t dev)
 {
 	PACB pACB = device_get_softc(dev);
 
 	bus_release_resource(dev, SYS_RES_IOPORT, PCIR_BAR(0), pACB->iores);
 	trm_destroySRB(pACB);
 	bus_dma_tag_destroy(pACB->sg_dmat);
 	bus_dmamap_unload(pACB->srb_dmat, pACB->srb_dmamap);
 	bus_dmamem_free(pACB->srb_dmat, pACB->pFreeSRB,
 	    pACB->srb_dmamap);
-	bus_dmamap_destroy(pACB->srb_dmat, pACB->srb_dmamap);
 	bus_dma_tag_destroy(pACB->srb_dmat);	
 	bus_dmamap_unload(pACB->sense_dmat, pACB->sense_dmamap);
 	bus_dmamem_free(pACB->sense_dmat, pACB->sense_buffers,
 	    pACB->sense_dmamap);
-	bus_dmamap_destroy(pACB->sense_dmat, pACB->sense_dmamap);
 	bus_dma_tag_destroy(pACB->sense_dmat);				      
 	bus_dma_tag_destroy(pACB->buffer_dmat);
 	bus_teardown_intr(dev, pACB->irq, pACB->ih);
 	bus_release_resource(dev, SYS_RES_IRQ, 0, pACB->irq);
 	xpt_async(AC_LOST_DEVICE, pACB->ppath, NULL);
 	xpt_free_path(pACB->ppath);
 	xpt_bus_deregister(cam_sim_path(pACB->psim));
 	cam_sim_free(pACB->psim, TRUE);
 	return (0);
 }
 static device_method_t trm_methods[] = {
 	/* Device interface */
 	DEVMETHOD(device_probe,		trm_probe),
 	DEVMETHOD(device_attach,	trm_attach),
 	DEVMETHOD(device_detach,	trm_detach),
 	{ 0, 0 }
 };
 
 static driver_t trm_driver = {
 	"trm", trm_methods, sizeof(struct _ACB)
 };
 
 static devclass_t trm_devclass;
 DRIVER_MODULE(trm, pci, trm_driver, trm_devclass, 0, 0);
 MODULE_DEPEND(trm, pci, 1, 1, 1);
 MODULE_DEPEND(trm, cam, 1, 1, 1);
Index: projects/release-debugdist/sys/dev/tx/if_tx.c
===================================================================
--- projects/release-debugdist/sys/dev/tx/if_tx.c	(revision 267343)
+++ projects/release-debugdist/sys/dev/tx/if_tx.c	(revision 267344)
@@ -1,1856 +1,1853 @@
 /*-
  * Copyright (c) 1997 Semen Ustimenko (semenu@FreeBSD.org)
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 /*
  * EtherPower II 10/100 Fast Ethernet (SMC 9432 serie)
  *
  * These cards are based on SMC83c17x (EPIC) chip and one of the various
  * PHYs (QS6612, AC101 and LXT970 were seen). The media support depends on
  * card model. All cards support 10baseT/UTP and 100baseTX half- and full-
  * duplex (SMB9432TX). SMC9432BTX also supports 10baseT/BNC. SMC9432FTX also
  * supports fibre optics.
  *
  * Thanks are going to Steve Bauer and Jason Wright.
  */
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/sockio.h>
 #include <sys/mbuf.h>
 #include <sys/kernel.h>
 #include <sys/module.h>
 #include <sys/socket.h>
 #include <sys/queue.h>
 
 #include <net/if.h>
 #include <net/if_var.h>
 #include <net/if_arp.h>
 #include <net/ethernet.h>
 #include <net/if_dl.h>
 #include <net/if_media.h>
 #include <net/if_types.h>
 
 #include <net/bpf.h>
 
 #include <net/if_vlan_var.h>
 
 #include <machine/bus.h>
 #include <machine/resource.h>
 #include <sys/bus.h>
 #include <sys/rman.h>
 
 #include <dev/pci/pcireg.h>
 #include <dev/pci/pcivar.h>
 
 #include <dev/mii/mii.h>
 #include <dev/mii/miivar.h>
 #include "miidevs.h"
 
 #include <dev/mii/lxtphyreg.h>
 
 #include "miibus_if.h"
 
 #include <dev/tx/if_txreg.h>
 #include <dev/tx/if_txvar.h>
 
 MODULE_DEPEND(tx, pci, 1, 1, 1);
 MODULE_DEPEND(tx, ether, 1, 1, 1);
 MODULE_DEPEND(tx, miibus, 1, 1, 1);
 
 static int epic_ifioctl(struct ifnet *, u_long, caddr_t);
 static void epic_intr(void *);
 static void epic_tx_underrun(epic_softc_t *);
 static void epic_ifstart(struct ifnet *);
 static void epic_ifstart_locked(struct ifnet *);
 static void epic_timer(void *);
 static void epic_init(void *);
 static void epic_init_locked(epic_softc_t *);
 static void epic_stop(epic_softc_t *);
 static void epic_rx_done(epic_softc_t *);
 static void epic_tx_done(epic_softc_t *);
 static int epic_init_rings(epic_softc_t *);
 static void epic_free_rings(epic_softc_t *);
 static void epic_stop_activity(epic_softc_t *);
 static int epic_queue_last_packet(epic_softc_t *);
 static void epic_start_activity(epic_softc_t *);
 static void epic_set_rx_mode(epic_softc_t *);
 static void epic_set_tx_mode(epic_softc_t *);
 static void epic_set_mc_table(epic_softc_t *);
 static int epic_read_eeprom(epic_softc_t *,u_int16_t);
 static void epic_output_eepromw(epic_softc_t *, u_int16_t);
 static u_int16_t epic_input_eepromw(epic_softc_t *);
 static u_int8_t epic_eeprom_clock(epic_softc_t *,u_int8_t);
 static void epic_write_eepromreg(epic_softc_t *,u_int8_t);
 static u_int8_t epic_read_eepromreg(epic_softc_t *);
 
 static int epic_read_phy_reg(epic_softc_t *, int, int);
 static void epic_write_phy_reg(epic_softc_t *, int, int, int);
 
 static int epic_miibus_readreg(device_t, int, int);
 static int epic_miibus_writereg(device_t, int, int, int);
 static void epic_miibus_statchg(device_t);
 static void epic_miibus_mediainit(device_t);
 
 static int epic_ifmedia_upd(struct ifnet *);
 static int epic_ifmedia_upd_locked(struct ifnet *);
 static void epic_ifmedia_sts(struct ifnet *, struct ifmediareq *);
 
 static int epic_probe(device_t);
 static int epic_attach(device_t);
 static int epic_shutdown(device_t);
 static int epic_detach(device_t);
 static void epic_release(epic_softc_t *);
 static struct epic_type *epic_devtype(device_t);
 
 static device_method_t epic_methods[] = {
 	/* Device interface */
 	DEVMETHOD(device_probe,		epic_probe),
 	DEVMETHOD(device_attach,	epic_attach),
 	DEVMETHOD(device_detach,	epic_detach),
 	DEVMETHOD(device_shutdown,	epic_shutdown),
 
 	/* MII interface */
 	DEVMETHOD(miibus_readreg,	epic_miibus_readreg),
 	DEVMETHOD(miibus_writereg,	epic_miibus_writereg),
 	DEVMETHOD(miibus_statchg,	epic_miibus_statchg),
 	DEVMETHOD(miibus_mediainit,	epic_miibus_mediainit),
 
 	{ 0, 0 }
 };
 
 static driver_t epic_driver = {
 	"tx",
 	epic_methods,
 	sizeof(epic_softc_t)
 };
 
 static devclass_t epic_devclass;
 
 DRIVER_MODULE(tx, pci, epic_driver, epic_devclass, 0, 0);
 DRIVER_MODULE(miibus, tx, miibus_driver, miibus_devclass, 0, 0);
 
 static struct epic_type epic_devs[] = {
 	{ SMC_VENDORID, SMC_DEVICEID_83C170, "SMC EtherPower II 10/100" },
 	{ 0, 0, NULL }
 };
 
 static int
 epic_probe(device_t dev)
 {
 	struct epic_type *t;
 
 	t = epic_devtype(dev);
 
 	if (t != NULL) {
 		device_set_desc(dev, t->name);
 		return (BUS_PROBE_DEFAULT);
 	}
 
 	return (ENXIO);
 }
 
 static struct epic_type *
 epic_devtype(device_t dev)
 {
 	struct epic_type *t;
 
 	t = epic_devs;
 
 	while (t->name != NULL) {
 		if ((pci_get_vendor(dev) == t->ven_id) &&
 		    (pci_get_device(dev) == t->dev_id)) {
 			return (t);
 		}
 		t++;
 	}
 	return (NULL);
 }
 
 #ifdef EPIC_USEIOSPACE
 #define	EPIC_RES	SYS_RES_IOPORT
 #define	EPIC_RID	PCIR_BASEIO
 #else
 #define	EPIC_RES	SYS_RES_MEMORY
 #define	EPIC_RID	PCIR_BASEMEM
 #endif
 
 static void
 epic_dma_map_addr(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 {
 	u_int32_t *addr;
 
 	if (error)
 		return;
 
 	KASSERT(nseg == 1, ("too many DMA segments, %d should be 1", nseg));
 	addr = arg;
 	*addr = segs->ds_addr;
 }
 
 /*
  * Attach routine: map registers, allocate softc, rings and descriptors.
  * Reset to known state.
  */
 static int
 epic_attach(device_t dev)
 {
 	struct ifnet *ifp;
 	epic_softc_t *sc;
 	int error;
 	int i, rid, tmp;
 	u_char eaddr[6];
 
 	sc = device_get_softc(dev);
 
 	/* Preinitialize softc structure. */
 	sc->dev = dev;
 	mtx_init(&sc->lock, device_get_nameunit(dev), MTX_NETWORK_LOCK,
 	    MTX_DEF);
 
 	/* Fill ifnet structure. */
 	ifp = sc->ifp = if_alloc(IFT_ETHER);
 	if (ifp == NULL) {
 		device_printf(dev, "can not if_alloc()\n");
 		error = ENOSPC;
 		goto fail;
 	}
 	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
 	ifp->if_softc = sc;
 	ifp->if_flags = IFF_BROADCAST|IFF_SIMPLEX|IFF_MULTICAST;
 	ifp->if_ioctl = epic_ifioctl;
 	ifp->if_start = epic_ifstart;
 	ifp->if_init = epic_init;
 	IFQ_SET_MAXLEN(&ifp->if_snd, TX_RING_SIZE - 1);
 
 	/* Enable busmastering. */
 	pci_enable_busmaster(dev);
 
 	rid = EPIC_RID;
 	sc->res = bus_alloc_resource_any(dev, EPIC_RES, &rid, RF_ACTIVE);
 	if (sc->res == NULL) {
 		device_printf(dev, "couldn't map ports/memory\n");
 		error = ENXIO;
 		goto fail;
 	}
 
 	/* Allocate interrupt. */
 	rid = 0;
 	sc->irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
 	    RF_SHAREABLE | RF_ACTIVE);
 	if (sc->irq == NULL) {
 		device_printf(dev, "couldn't map interrupt\n");
 		error = ENXIO;
 		goto fail;
 	}
 
 	/* Allocate DMA tags. */
 	error = bus_dma_tag_create(bus_get_dma_tag(dev), 4, 0,
 	    BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL,
 	    MCLBYTES * EPIC_MAX_FRAGS, EPIC_MAX_FRAGS, MCLBYTES, 0, NULL, NULL,
 	    &sc->mtag);
 	if (error) {
 		device_printf(dev, "couldn't allocate dma tag\n");
 		goto fail;
 	}
 
 	error = bus_dma_tag_create(bus_get_dma_tag(dev), 4, 0,
 	    BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL,
 	    sizeof(struct epic_rx_desc) * RX_RING_SIZE,
 	    1, sizeof(struct epic_rx_desc) * RX_RING_SIZE, 0, NULL,
 	    NULL, &sc->rtag);
 	if (error) {
 		device_printf(dev, "couldn't allocate dma tag\n");
 		goto fail;
 	}
 
 	error = bus_dma_tag_create(bus_get_dma_tag(dev), 4, 0,
 	    BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL,
 	    sizeof(struct epic_tx_desc) * TX_RING_SIZE,
 	    1, sizeof(struct epic_tx_desc) * TX_RING_SIZE, 0,
 	    NULL, NULL, &sc->ttag);
 	if (error) {
 		device_printf(dev, "couldn't allocate dma tag\n");
 		goto fail;
 	}
 
 	error = bus_dma_tag_create(bus_get_dma_tag(dev), 4, 0,
 	    BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL,
 	    sizeof(struct epic_frag_list) * TX_RING_SIZE,
 	    1, sizeof(struct epic_frag_list) * TX_RING_SIZE, 0,
 	    NULL, NULL, &sc->ftag);
 	if (error) {
 		device_printf(dev, "couldn't allocate dma tag\n");
 		goto fail;
 	}
 
 	/* Allocate DMA safe memory and get the DMA addresses. */
 	error = bus_dmamem_alloc(sc->ftag, (void **)&sc->tx_flist,
 	    BUS_DMA_NOWAIT | BUS_DMA_ZERO, &sc->fmap);
 	if (error) {
 		device_printf(dev, "couldn't allocate dma memory\n");
 		goto fail;
 	}
 	error = bus_dmamap_load(sc->ftag, sc->fmap, sc->tx_flist,
 	    sizeof(struct epic_frag_list) * TX_RING_SIZE, epic_dma_map_addr,
 	    &sc->frag_addr, 0);
 	if (error) {
 		device_printf(dev, "couldn't map dma memory\n");
 		goto fail;
 	}
 	error = bus_dmamem_alloc(sc->ttag, (void **)&sc->tx_desc,
 	    BUS_DMA_NOWAIT | BUS_DMA_ZERO, &sc->tmap);
 	if (error) {
 		device_printf(dev, "couldn't allocate dma memory\n");
 		goto fail;
 	}
 	error = bus_dmamap_load(sc->ttag, sc->tmap, sc->tx_desc,
 	    sizeof(struct epic_tx_desc) * TX_RING_SIZE, epic_dma_map_addr,
 	    &sc->tx_addr, 0);
 	if (error) {
 		device_printf(dev, "couldn't map dma memory\n");
 		goto fail;
 	}
 	error = bus_dmamem_alloc(sc->rtag, (void **)&sc->rx_desc,
 	    BUS_DMA_NOWAIT | BUS_DMA_ZERO, &sc->rmap);
 	if (error) {
 		device_printf(dev, "couldn't allocate dma memory\n");
 		goto fail;
 	}
 	error = bus_dmamap_load(sc->rtag, sc->rmap, sc->rx_desc,
 	    sizeof(struct epic_rx_desc) * RX_RING_SIZE, epic_dma_map_addr,
 	    &sc->rx_addr, 0);
 	if (error) {
 		device_printf(dev, "couldn't map dma memory\n");
 		goto fail;
 	}
 
 	/* Bring the chip out of low-power mode. */
 	CSR_WRITE_4(sc, GENCTL, GENCTL_SOFT_RESET);
 	DELAY(500);
 
 	/* Workaround for Application Note 7-15. */
 	for (i = 0; i < 16; i++)
 		CSR_WRITE_4(sc, TEST1, TEST1_CLOCK_TEST);
 
 	/* Read MAC address from EEPROM. */
 	for (i = 0; i < ETHER_ADDR_LEN / sizeof(u_int16_t); i++)
 		((u_int16_t *)eaddr)[i] = epic_read_eeprom(sc,i);
 
 	/* Set Non-Volatile Control Register from EEPROM. */
 	CSR_WRITE_4(sc, NVCTL, epic_read_eeprom(sc, EEPROM_NVCTL) & 0x1F);
 
 	/* Set defaults. */
 	sc->tx_threshold = TRANSMIT_THRESHOLD;
 	sc->txcon = TXCON_DEFAULT;
 	sc->miicfg = MIICFG_SMI_ENABLE;
 	sc->phyid = EPIC_UNKN_PHY;
 	sc->serinst = -1;
 
 	/* Fetch card id. */
 	sc->cardvend = pci_read_config(dev, PCIR_SUBVEND_0, 2);
 	sc->cardid = pci_read_config(dev, PCIR_SUBDEV_0, 2);
 
 	if (sc->cardvend != SMC_VENDORID)
 		device_printf(dev, "unknown card vendor %04xh\n", sc->cardvend);
 
 	/* Do ifmedia setup. */
 	error = mii_attach(dev, &sc->miibus, ifp, epic_ifmedia_upd,
 	    epic_ifmedia_sts, BMSR_DEFCAPMASK, MII_PHY_ANY, MII_OFFSET_ANY, 0);
 	if (error != 0) {
 		device_printf(dev, "attaching PHYs failed\n");
 		goto fail;
 	}
 
 	/* board type and ... */
 	printf(" type ");
 	for(i = 0x2c; i < 0x32; i++) {
 		tmp = epic_read_eeprom(sc, i);
 		if (' ' == (u_int8_t)tmp)
 			break;
 		printf("%c", (u_int8_t)tmp);
 		tmp >>= 8;
 		if (' ' == (u_int8_t)tmp)
 			break;
 		printf("%c", (u_int8_t)tmp);
 	}
 	printf("\n");
 
 	/* Initialize rings. */
 	if (epic_init_rings(sc)) {
 		device_printf(dev, "failed to init rings\n");
 		error = ENXIO;
 		goto fail;
 	}
 
 	ifp->if_hdrlen = sizeof(struct ether_vlan_header);
 	ifp->if_capabilities |= IFCAP_VLAN_MTU;
 	ifp->if_capenable |= IFCAP_VLAN_MTU;
 	callout_init_mtx(&sc->timer, &sc->lock, 0);
 
 	/* Attach to OS's managers. */
 	ether_ifattach(ifp, eaddr);
 
 	/* Activate our interrupt handler. */
 	error = bus_setup_intr(dev, sc->irq, INTR_TYPE_NET | INTR_MPSAFE,
 	    NULL, epic_intr, sc, &sc->sc_ih);
 	if (error) {
 		device_printf(dev, "couldn't set up irq\n");
 		ether_ifdetach(ifp);
 		goto fail;
 	}
 
 	return (0);
 fail:
 	epic_release(sc);
 	return (error);
 }
 
 /*
  * Free any resources allocated by the driver.
  */
 static void
 epic_release(epic_softc_t *sc)
 {
 	if (sc->ifp != NULL)
 		if_free(sc->ifp);
 	if (sc->irq)
 		bus_release_resource(sc->dev, SYS_RES_IRQ, 0, sc->irq);
 	if (sc->res)
 		bus_release_resource(sc->dev, EPIC_RES, EPIC_RID, sc->res);
 	epic_free_rings(sc);
 	if (sc->tx_flist) {
 		bus_dmamap_unload(sc->ftag, sc->fmap);
 		bus_dmamem_free(sc->ftag, sc->tx_flist, sc->fmap);
-		bus_dmamap_destroy(sc->ftag, sc->fmap);
 	}
 	if (sc->tx_desc) {
 		bus_dmamap_unload(sc->ttag, sc->tmap);
 		bus_dmamem_free(sc->ttag, sc->tx_desc, sc->tmap);
-		bus_dmamap_destroy(sc->ttag, sc->tmap);
 	}
 	if (sc->rx_desc) {
 		bus_dmamap_unload(sc->rtag, sc->rmap);
 		bus_dmamem_free(sc->rtag, sc->rx_desc, sc->rmap);
-		bus_dmamap_destroy(sc->rtag, sc->rmap);
 	}
 	if (sc->mtag)
 		bus_dma_tag_destroy(sc->mtag);
 	if (sc->ftag)
 		bus_dma_tag_destroy(sc->ftag);
 	if (sc->ttag)
 		bus_dma_tag_destroy(sc->ttag);
 	if (sc->rtag)
 		bus_dma_tag_destroy(sc->rtag);
 	mtx_destroy(&sc->lock);
 }
 
 /*
  * Detach driver and free resources.
  */
 static int
 epic_detach(device_t dev)
 {
 	struct ifnet *ifp;
 	epic_softc_t *sc;
 
 	sc = device_get_softc(dev);
 	ifp = sc->ifp;
 
 	EPIC_LOCK(sc);
 	epic_stop(sc);
 	EPIC_UNLOCK(sc);
 	callout_drain(&sc->timer);
 	ether_ifdetach(ifp);
 	bus_teardown_intr(dev, sc->irq, sc->sc_ih);
 
 	bus_generic_detach(dev);
 	device_delete_child(dev, sc->miibus);
 
 	epic_release(sc);
 	return (0);
 }
 
 #undef	EPIC_RES
 #undef	EPIC_RID
 
 /*
  * Stop all chip I/O so that the kernel's probe routines don't
  * get confused by errant DMAs when rebooting.
  */
 static int
 epic_shutdown(device_t dev)
 {
 	epic_softc_t *sc;
 
 	sc = device_get_softc(dev);
 
 	EPIC_LOCK(sc);
 	epic_stop(sc);
 	EPIC_UNLOCK(sc);
 	return (0);
 }
 
 /*
  * This is if_ioctl handler.
  */
 static int
 epic_ifioctl(struct ifnet *ifp, u_long command, caddr_t data)
 {
 	epic_softc_t *sc = ifp->if_softc;
 	struct mii_data	*mii;
 	struct ifreq *ifr = (struct ifreq *) data;
 	int error = 0;
 
 	switch (command) {
 	case SIOCSIFMTU:
 		if (ifp->if_mtu == ifr->ifr_mtu)
 			break;
 
 		/* XXX Though the datasheet doesn't imply any
 		 * limitations on RX and TX sizes beside max 64Kb
 		 * DMA transfer, seems we can't send more then 1600
 		 * data bytes per ethernet packet (transmitter hangs
 		 * up if more data is sent).
 		 */
 		EPIC_LOCK(sc);
 		if (ifr->ifr_mtu + ifp->if_hdrlen <= EPIC_MAX_MTU) {
 			ifp->if_mtu = ifr->ifr_mtu;
 			epic_stop(sc);
 			epic_init_locked(sc);
 		} else
 			error = EINVAL;
 		EPIC_UNLOCK(sc);
 		break;
 
 	case SIOCSIFFLAGS:
 		/*
 		 * If the interface is marked up and stopped, then start it.
 		 * If it is marked down and running, then stop it.
 		 */
 		EPIC_LOCK(sc);
 		if (ifp->if_flags & IFF_UP) {
 			if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
 				epic_init_locked(sc);
 				EPIC_UNLOCK(sc);
 				break;
 			}
 		} else {
 			if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 				epic_stop(sc);
 				EPIC_UNLOCK(sc);
 				break;
 			}
 		}
 
 		/* Handle IFF_PROMISC and IFF_ALLMULTI flags. */
 		epic_stop_activity(sc);
 		epic_set_mc_table(sc);
 		epic_set_rx_mode(sc);
 		epic_start_activity(sc);
 		EPIC_UNLOCK(sc);
 		break;
 
 	case SIOCADDMULTI:
 	case SIOCDELMULTI:
 		EPIC_LOCK(sc);
 		epic_set_mc_table(sc);
 		EPIC_UNLOCK(sc);
 		error = 0;
 		break;
 
 	case SIOCSIFMEDIA:
 	case SIOCGIFMEDIA:
 		mii = device_get_softc(sc->miibus);
 		error = ifmedia_ioctl(ifp, ifr, &mii->mii_media, command);
 		break;
 
 	default:
 		error = ether_ioctl(ifp, command, data);
 		break;
 	}
 	return (error);
 }
 
 static void
 epic_dma_map_txbuf(void *arg, bus_dma_segment_t *segs, int nseg,
     bus_size_t mapsize, int error)
 {
 	struct epic_frag_list *flist;
 	int i;
 
 	if (error)
 		return;
 
 	KASSERT(nseg <= EPIC_MAX_FRAGS, ("too many DMA segments"));
 	flist = arg;
 	/* Fill fragments list. */
 	for (i = 0; i < nseg; i++) {
 		KASSERT(segs[i].ds_len <= MCLBYTES, ("segment size too large"));
 		flist->frag[i].fraglen = segs[i].ds_len;
 		flist->frag[i].fragaddr = segs[i].ds_addr;
 	}
 	flist->numfrags = nseg;
 }
 
 static void
 epic_dma_map_rxbuf(void *arg, bus_dma_segment_t *segs, int nseg,
     bus_size_t mapsize, int error)
 {
 	struct epic_rx_desc *desc;
 
 	if (error)
 		return;
 
 	KASSERT(nseg == 1, ("too many DMA segments"));
 	desc = arg;
 	desc->bufaddr = segs->ds_addr;
 }
 
 /*
  * This is if_start handler. It takes mbufs from if_snd queue
  * and queue them for transmit, one by one, until TX ring become full
  * or queue become empty.
  */
 static void
 epic_ifstart(struct ifnet * ifp)
 {
 	epic_softc_t *sc = ifp->if_softc;
 
 	EPIC_LOCK(sc);
 	epic_ifstart_locked(ifp);
 	EPIC_UNLOCK(sc);
 }
 
 static void
 epic_ifstart_locked(struct ifnet * ifp)
 {
 	epic_softc_t *sc = ifp->if_softc;
 	struct epic_tx_buffer *buf;
 	struct epic_tx_desc *desc;
 	struct epic_frag_list *flist;
 	struct mbuf *m0, *m;
 	int error;
 
 	while (sc->pending_txs < TX_RING_SIZE) {
 		buf = sc->tx_buffer + sc->cur_tx;
 		desc = sc->tx_desc + sc->cur_tx;
 		flist = sc->tx_flist + sc->cur_tx;
 
 		/* Get next packet to send. */
 		IF_DEQUEUE(&ifp->if_snd, m0);
 
 		/* If nothing to send, return. */
 		if (m0 == NULL)
 			return;
 
 		error = bus_dmamap_load_mbuf(sc->mtag, buf->map, m0,
 		    epic_dma_map_txbuf, flist, 0);
 
 		if (error && error != EFBIG) {
 			m_freem(m0);
 			ifp->if_oerrors++;
 			continue;
 		}
 
 		/*
 		 * If packet was more than EPIC_MAX_FRAGS parts,
 		 * recopy packet to a newly allocated mbuf cluster.
 		 */
 		if (error) {
 			m = m_defrag(m0, M_NOWAIT);
 			if (m == NULL) {
 				m_freem(m0);
 				ifp->if_oerrors++;
 				continue;
 			}
 			m_freem(m0);
 			m0 = m;
 
 			error = bus_dmamap_load_mbuf(sc->mtag, buf->map, m,
 			    epic_dma_map_txbuf, flist, 0);
 			if (error) {
 				m_freem(m);
 				ifp->if_oerrors++;
 				continue;
 			}
 		}
 		bus_dmamap_sync(sc->mtag, buf->map, BUS_DMASYNC_PREWRITE);
 
 		buf->mbuf = m0;
 		sc->pending_txs++;
 		sc->cur_tx = (sc->cur_tx + 1) & TX_RING_MASK;
 		desc->control = 0x01;
 		desc->txlength =
 		    max(m0->m_pkthdr.len, ETHER_MIN_LEN - ETHER_CRC_LEN);
 		desc->status = 0x8000;
 		bus_dmamap_sync(sc->ttag, sc->tmap,
 		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 		bus_dmamap_sync(sc->ftag, sc->fmap, BUS_DMASYNC_PREWRITE);
 		CSR_WRITE_4(sc, COMMAND, COMMAND_TXQUEUED);
 
 		/* Set watchdog timer. */
 		sc->tx_timeout = 8;
 
 		BPF_MTAP(ifp, m0);
 	}
 
 	ifp->if_drv_flags |= IFF_DRV_OACTIVE;
 }
 
 /*
  * Synopsis: Finish all received frames.
  */
 static void
 epic_rx_done(epic_softc_t *sc)
 {
 	struct ifnet *ifp = sc->ifp;
 	u_int16_t len;
 	struct epic_rx_buffer *buf;
 	struct epic_rx_desc *desc;
 	struct mbuf *m;
 	bus_dmamap_t map;
 	int error;
 
 	bus_dmamap_sync(sc->rtag, sc->rmap, BUS_DMASYNC_POSTREAD);
 	while ((sc->rx_desc[sc->cur_rx].status & 0x8000) == 0) {
 		buf = sc->rx_buffer + sc->cur_rx;
 		desc = sc->rx_desc + sc->cur_rx;
 
 		/* Switch to next descriptor. */
 		sc->cur_rx = (sc->cur_rx + 1) & RX_RING_MASK;
 
 		/*
 		 * Check for RX errors. This should only happen if
 		 * SAVE_ERRORED_PACKETS is set. RX errors generate
 		 * RXE interrupt usually.
 		 */
 		if ((desc->status & 1) == 0) {
 			ifp->if_ierrors++;
 			desc->status = 0x8000;
 			continue;
 		}
 
 		/* Save packet length and mbuf contained packet. */
 		bus_dmamap_sync(sc->mtag, buf->map, BUS_DMASYNC_POSTREAD);
 		len = desc->rxlength - ETHER_CRC_LEN;
 		m = buf->mbuf;
 
 		/* Try to get an mbuf cluster. */
 		buf->mbuf = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
 		if (buf->mbuf == NULL) {
 			buf->mbuf = m;
 			desc->status = 0x8000;
 			ifp->if_ierrors++;
 			continue;
 		}
 		buf->mbuf->m_len = buf->mbuf->m_pkthdr.len = MCLBYTES;
 		m_adj(buf->mbuf, ETHER_ALIGN);
 
 		/* Point to new mbuf, and give descriptor to chip. */
 		error = bus_dmamap_load_mbuf(sc->mtag, sc->sparemap, buf->mbuf,
 		    epic_dma_map_rxbuf, desc, 0);
 		if (error) {
 			buf->mbuf = m;
 			desc->status = 0x8000;
 			ifp->if_ierrors++;
 			continue;
 		}
 
 		desc->status = 0x8000;
 		bus_dmamap_unload(sc->mtag, buf->map);
 		map = buf->map;
 		buf->map = sc->sparemap;
 		sc->sparemap = map;
 		bus_dmamap_sync(sc->mtag, buf->map, BUS_DMASYNC_PREREAD);
 
 		/* First mbuf in packet holds the ethernet and packet headers */
 		m->m_pkthdr.rcvif = ifp;
 		m->m_pkthdr.len = m->m_len = len;
 
 		/* Give mbuf to OS. */
 		EPIC_UNLOCK(sc);
 		(*ifp->if_input)(ifp, m);
 		EPIC_LOCK(sc);
 
 		/* Successfuly received frame */
 		ifp->if_ipackets++;
         }
 	bus_dmamap_sync(sc->rtag, sc->rmap,
 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 }
 
 /*
  * Synopsis: Do last phase of transmission. I.e. if desc is
  * transmitted, decrease pending_txs counter, free mbuf contained
  * packet, switch to next descriptor and repeat until no packets
  * are pending or descriptor is not transmitted yet.
  */
 static void
 epic_tx_done(epic_softc_t *sc)
 {
 	struct epic_tx_buffer *buf;
 	struct epic_tx_desc *desc;
 	u_int16_t status;
 
 	bus_dmamap_sync(sc->ttag, sc->tmap, BUS_DMASYNC_POSTREAD);
 	while (sc->pending_txs > 0) {
 		buf = sc->tx_buffer + sc->dirty_tx;
 		desc = sc->tx_desc + sc->dirty_tx;
 		status = desc->status;
 
 		/*
 		 * If packet is not transmitted, thou followed
 		 * packets are not transmitted too.
 		 */
 		if (status & 0x8000)
 			break;
 
 		/* Packet is transmitted. Switch to next and free mbuf. */
 		sc->pending_txs--;
 		sc->dirty_tx = (sc->dirty_tx + 1) & TX_RING_MASK;
 		bus_dmamap_sync(sc->mtag, buf->map, BUS_DMASYNC_POSTWRITE);
 		bus_dmamap_unload(sc->mtag, buf->map);
 		m_freem(buf->mbuf);
 		buf->mbuf = NULL;
 
 		/* Check for errors and collisions. */
 		if (status & 0x0001)
 			sc->ifp->if_opackets++;
 		else
 			sc->ifp->if_oerrors++;
 		sc->ifp->if_collisions += (status >> 8) & 0x1F;
 #ifdef EPIC_DIAG
 		if ((status & 0x1001) == 0x1001)
 			device_printf(sc->dev,
 			    "Tx ERROR: excessive coll. number\n");
 #endif
 	}
 
 	if (sc->pending_txs < TX_RING_SIZE)
 		sc->ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
 	bus_dmamap_sync(sc->ttag, sc->tmap,
 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 }
 
 /*
  * Interrupt function
  */
 static void
 epic_intr(void *arg)
 {
     epic_softc_t *sc;
     int status, i;
 
     sc = arg;
     i = 4;
     EPIC_LOCK(sc);
     while (i-- && ((status = CSR_READ_4(sc, INTSTAT)) & INTSTAT_INT_ACTV)) {
 	CSR_WRITE_4(sc, INTSTAT, status);
 
 	if (status & (INTSTAT_RQE|INTSTAT_RCC|INTSTAT_OVW)) {
 	    epic_rx_done(sc);
 	    if (status & (INTSTAT_RQE|INTSTAT_OVW)) {
 #ifdef EPIC_DIAG
 		if (status & INTSTAT_OVW)
 		    device_printf(sc->dev, "RX buffer overflow\n");
 		if (status & INTSTAT_RQE)
 		    device_printf(sc->dev, "RX FIFO overflow\n");
 #endif
 		if ((CSR_READ_4(sc, COMMAND) & COMMAND_RXQUEUED) == 0)
 		    CSR_WRITE_4(sc, COMMAND, COMMAND_RXQUEUED);
 		sc->ifp->if_ierrors++;
 	    }
 	}
 
 	if (status & (INTSTAT_TXC|INTSTAT_TCC|INTSTAT_TQE)) {
 	    epic_tx_done(sc);
 	    if (sc->ifp->if_snd.ifq_head != NULL)
 		    epic_ifstart_locked(sc->ifp);
 	}
 
 	/* Check for rare errors */
 	if (status & (INTSTAT_FATAL|INTSTAT_PMA|INTSTAT_PTA|
 		      INTSTAT_APE|INTSTAT_DPE|INTSTAT_TXU|INTSTAT_RXE)) {
     	    if (status & (INTSTAT_FATAL|INTSTAT_PMA|INTSTAT_PTA|
 			  INTSTAT_APE|INTSTAT_DPE)) {
 		device_printf(sc->dev, "PCI fatal errors occured: %s%s%s%s\n",
 		    (status & INTSTAT_PMA) ? "PMA " : "",
 		    (status & INTSTAT_PTA) ? "PTA " : "",
 		    (status & INTSTAT_APE) ? "APE " : "",
 		    (status & INTSTAT_DPE) ? "DPE" : "");
 
 		epic_stop(sc);
 		epic_init_locked(sc);
 	    	break;
 	    }
 
 	    if (status & INTSTAT_RXE) {
 #ifdef EPIC_DIAG
 		device_printf(sc->dev, "CRC/Alignment error\n");
 #endif
 		sc->ifp->if_ierrors++;
 	    }
 
 	    if (status & INTSTAT_TXU) {
 		epic_tx_underrun(sc);
 		sc->ifp->if_oerrors++;
 	    }
 	}
     }
 
     /* If no packets are pending, then no timeouts. */
     if (sc->pending_txs == 0)
 	    sc->tx_timeout = 0;
     EPIC_UNLOCK(sc);
 }
 
 /*
  * Handle the TX underrun error: increase the TX threshold
  * and restart the transmitter.
  */
 static void
 epic_tx_underrun(epic_softc_t *sc)
 {
 	if (sc->tx_threshold > TRANSMIT_THRESHOLD_MAX) {
 		sc->txcon &= ~TXCON_EARLY_TRANSMIT_ENABLE;
 #ifdef EPIC_DIAG
 		device_printf(sc->dev, "Tx UNDERRUN: early TX disabled\n");
 #endif
 	} else {
 		sc->tx_threshold += 0x40;
 #ifdef EPIC_DIAG
 		device_printf(sc->dev,
 		    "Tx UNDERRUN: TX threshold increased to %d\n",
 		    sc->tx_threshold);
 #endif
 	}
 
 	/* We must set TXUGO to reset the stuck transmitter. */
 	CSR_WRITE_4(sc, COMMAND, COMMAND_TXUGO);
 
 	/* Update the TX threshold */
 	epic_stop_activity(sc);
 	epic_set_tx_mode(sc);
 	epic_start_activity(sc);
 }
 
 /*
  * This function is called once a second when the interface is running
  * and performs two functions.  First, it provides a timer for the mii
  * to help with autonegotiation.  Second, it checks for transmit
  * timeouts.
  */
 static void
 epic_timer(void *arg)
 {
 	epic_softc_t *sc = arg;
 	struct mii_data *mii;
 	struct ifnet *ifp;
 
 	ifp = sc->ifp;
 	EPIC_ASSERT_LOCKED(sc);
 	if (sc->tx_timeout && --sc->tx_timeout == 0) {
 		device_printf(sc->dev, "device timeout %d packets\n",
 		    sc->pending_txs);
 
 		/* Try to finish queued packets. */
 		epic_tx_done(sc);
 
 		/* If not successful. */
 		if (sc->pending_txs > 0) {
 			ifp->if_oerrors += sc->pending_txs;
 
 			/* Reinitialize board. */
 			device_printf(sc->dev, "reinitialization\n");
 			epic_stop(sc);
 			epic_init_locked(sc);
 		} else
 			device_printf(sc->dev,
 			    "seems we can continue normaly\n");
 
 		/* Start output. */
 		if (ifp->if_snd.ifq_head)
 			epic_ifstart_locked(ifp);
 	}
 
 	mii = device_get_softc(sc->miibus);
 	mii_tick(mii);
 
 	callout_reset(&sc->timer, hz, epic_timer, sc);
 }
 
 /*
  * Set media options.
  */
 static int
 epic_ifmedia_upd(struct ifnet *ifp)
 {
 	epic_softc_t *sc;
 	int error;
 
 	sc = ifp->if_softc;
 	EPIC_LOCK(sc);
 	error = epic_ifmedia_upd_locked(ifp);
 	EPIC_UNLOCK(sc);
 	return (error);
 }
 	
 static int
 epic_ifmedia_upd_locked(struct ifnet *ifp)
 {
 	epic_softc_t *sc;
 	struct mii_data *mii;
 	struct ifmedia *ifm;
 	struct mii_softc *miisc;
 	int cfg, media;
 
 	sc = ifp->if_softc;
 	mii = device_get_softc(sc->miibus);
 	ifm = &mii->mii_media;
 	media = ifm->ifm_cur->ifm_media;
 
 	/* Do not do anything if interface is not up. */
 	if ((ifp->if_flags & IFF_UP) == 0)
 		return (0);
 
 	/*
 	 * Lookup current selected PHY.
 	 */
 	if (IFM_INST(media) == sc->serinst) {
 		sc->phyid = EPIC_SERIAL;
 		sc->physc = NULL;
 	} else {
 		/* If we're not selecting serial interface, select MII mode. */
 		sc->miicfg &= ~MIICFG_SERIAL_ENABLE;
 		CSR_WRITE_4(sc, MIICFG, sc->miicfg);
 
 		/* Default to unknown PHY. */
 		sc->phyid = EPIC_UNKN_PHY;
 
 		/* Lookup selected PHY. */
 		LIST_FOREACH(miisc, &mii->mii_phys, mii_list) {
 			if (IFM_INST(media) == miisc->mii_inst) {
 				sc->physc = miisc;
 				break;
 			}
 		}
 
 		/* Identify selected PHY. */
 		if (sc->physc) {
 			int id1, id2, model, oui;
 
 			id1 = PHY_READ(sc->physc, MII_PHYIDR1);
 			id2 = PHY_READ(sc->physc, MII_PHYIDR2);
 
 			oui = MII_OUI(id1, id2);
 			model = MII_MODEL(id2);
 			switch (oui) {
 			case MII_OUI_xxQUALSEMI:
 				if (model == MII_MODEL_xxQUALSEMI_QS6612)
 					sc->phyid = EPIC_QS6612_PHY;
 				break;
 			case MII_OUI_ALTIMA:
 				if (model == MII_MODEL_ALTIMA_AC101)
 					sc->phyid = EPIC_AC101_PHY;
 				break;
 			case MII_OUI_xxLEVEL1:
 				if (model == MII_MODEL_xxLEVEL1_LXT970)
 					sc->phyid = EPIC_LXT970_PHY;
 				break;
 			}
 		}
 	}
 
 	/*
 	 * Do PHY specific card setup.
 	 */
 
 	/*
 	 * Call this, to isolate all not selected PHYs and
 	 * set up selected.
 	 */
 	mii_mediachg(mii);
 
 	/* Do our own setup. */
 	switch (sc->phyid) {
 	case EPIC_QS6612_PHY:
 		break;
 	case EPIC_AC101_PHY:
 		/* We have to powerup fiber tranceivers. */
 		if (IFM_SUBTYPE(media) == IFM_100_FX)
 			sc->miicfg |= MIICFG_694_ENABLE;
 		else
 			sc->miicfg &= ~MIICFG_694_ENABLE;
 		CSR_WRITE_4(sc, MIICFG, sc->miicfg);
 
 		break;
 	case EPIC_LXT970_PHY:
 		/* We have to powerup fiber tranceivers. */
 		cfg = PHY_READ(sc->physc, MII_LXTPHY_CONFIG);
 		if (IFM_SUBTYPE(media) == IFM_100_FX)
 			cfg |= CONFIG_LEDC1 | CONFIG_LEDC0;
 		else
 			cfg &= ~(CONFIG_LEDC1 | CONFIG_LEDC0);
 		PHY_WRITE(sc->physc, MII_LXTPHY_CONFIG, cfg);
 
 		break;
 	case EPIC_SERIAL:
 		/* Select serial PHY (10base2/BNC usually). */
 		sc->miicfg |= MIICFG_694_ENABLE | MIICFG_SERIAL_ENABLE;
 		CSR_WRITE_4(sc, MIICFG, sc->miicfg);
 
 		/* There is no driver to fill this. */
 		mii->mii_media_active = media;
 		mii->mii_media_status = 0;
 
 		/*
 		 * We need to call this manually as it wasn't called
 		 * in mii_mediachg().
 		 */
 		epic_miibus_statchg(sc->dev);
 		break;
 	default:
 		device_printf(sc->dev, "ERROR! Unknown PHY selected\n");
 		return (EINVAL);
 	}
 
 	return (0);
 }
 
 /*
  * Report current media status.
  */
 static void
 epic_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr)
 {
 	epic_softc_t *sc;
 	struct mii_data *mii;
 
 	sc = ifp->if_softc;
 	mii = device_get_softc(sc->miibus);
 	EPIC_LOCK(sc);
 
 	/* Nothing should be selected if interface is down. */
 	if ((ifp->if_flags & IFF_UP) == 0) {
 		ifmr->ifm_active = IFM_NONE;
 		ifmr->ifm_status = 0;
 		EPIC_UNLOCK(sc);
 		return;
 	}
 
 	/* Call underlying pollstat, if not serial PHY. */
 	if (sc->phyid != EPIC_SERIAL)
 		mii_pollstat(mii);
 
 	/* Simply copy media info. */
 	ifmr->ifm_active = mii->mii_media_active;
 	ifmr->ifm_status = mii->mii_media_status;
 	EPIC_UNLOCK(sc);
 }
 
 /*
  * Callback routine, called on media change.
  */
 static void
 epic_miibus_statchg(device_t dev)
 {
 	epic_softc_t *sc;
 	struct mii_data *mii;
 	int media;
 
 	sc = device_get_softc(dev);
 	mii = device_get_softc(sc->miibus);
 	media = mii->mii_media_active;
 
 	sc->txcon &= ~(TXCON_LOOPBACK_MODE | TXCON_FULL_DUPLEX);
 
 	/*
 	 * If we are in full-duplex mode or loopback operation,
 	 * we need to decouple receiver and transmitter.
 	 */
 	if (IFM_OPTIONS(media) & (IFM_FDX | IFM_LOOP))
  		sc->txcon |= TXCON_FULL_DUPLEX;
 
 	/* On some cards we need manualy set fullduplex led. */
 	if (sc->cardid == SMC9432FTX ||
 	    sc->cardid == SMC9432FTX_SC) {
 		if (IFM_OPTIONS(media) & IFM_FDX)
 			sc->miicfg |= MIICFG_694_ENABLE;
 		else
 			sc->miicfg &= ~MIICFG_694_ENABLE;
 
 		CSR_WRITE_4(sc, MIICFG, sc->miicfg);
 	}
 
 	epic_stop_activity(sc);
 	epic_set_tx_mode(sc);
 	epic_start_activity(sc);
 }
 
 static void
 epic_miibus_mediainit(device_t dev)
 {
 	epic_softc_t *sc;
 	struct mii_data *mii;
 	struct ifmedia *ifm;
 	int media;
 
 	sc = device_get_softc(dev);
 	mii = device_get_softc(sc->miibus);
 	ifm = &mii->mii_media;
 
 	/*
 	 * Add Serial Media Interface if present, this applies to
 	 * SMC9432BTX serie.
 	 */
 	if (CSR_READ_4(sc, MIICFG) & MIICFG_PHY_PRESENT) {
 		/* Store its instance. */
 		sc->serinst = mii->mii_instance++;
 
 		/* Add as 10base2/BNC media. */
 		media = IFM_MAKEWORD(IFM_ETHER, IFM_10_2, 0, sc->serinst);
 		ifmedia_add(ifm, media, 0, NULL);
 
 		/* Report to user. */
 		device_printf(sc->dev, "serial PHY detected (10Base2/BNC)\n");
 	}
 }
 
 /*
  * Reset chip and update media.
  */
 static void
 epic_init(void *xsc)
 {
 	epic_softc_t *sc = xsc;
 
 	EPIC_LOCK(sc);
 	epic_init_locked(sc);
 	EPIC_UNLOCK(sc);
 }
 
 static void
 epic_init_locked(epic_softc_t *sc)
 {
 	struct ifnet *ifp = sc->ifp;
 	int i;
 
 	/* If interface is already running, then we need not do anything. */
 	if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
 		return;
 	}
 
 	/* Soft reset the chip (we have to power up card before). */
 	CSR_WRITE_4(sc, GENCTL, 0);
 	CSR_WRITE_4(sc, GENCTL, GENCTL_SOFT_RESET);
 
 	/*
 	 * Reset takes 15 pci ticks which depends on PCI bus speed.
 	 * Assuming it >= 33000000 hz, we have wait at least 495e-6 sec.
 	 */
 	DELAY(500);
 
 	/* Wake up */
 	CSR_WRITE_4(sc, GENCTL, 0);
 
 	/* Workaround for Application Note 7-15 */
 	for (i = 0; i < 16; i++)
 		CSR_WRITE_4(sc, TEST1, TEST1_CLOCK_TEST);
 
 	/* Give rings to EPIC */
 	CSR_WRITE_4(sc, PRCDAR, sc->rx_addr);
 	CSR_WRITE_4(sc, PTCDAR, sc->tx_addr);
 
 	/* Put node address to EPIC. */
 	CSR_WRITE_4(sc, LAN0, ((u_int16_t *)IF_LLADDR(sc->ifp))[0]);
 	CSR_WRITE_4(sc, LAN1, ((u_int16_t *)IF_LLADDR(sc->ifp))[1]);
 	CSR_WRITE_4(sc, LAN2, ((u_int16_t *)IF_LLADDR(sc->ifp))[2]);
 
 	/* Set tx mode, includeing transmit threshold. */
 	epic_set_tx_mode(sc);
 
 	/* Compute and set RXCON. */
 	epic_set_rx_mode(sc);
 
 	/* Set multicast table. */
 	epic_set_mc_table(sc);
 
 	/* Enable interrupts by setting the interrupt mask. */
 	CSR_WRITE_4(sc, INTMASK,
 		INTSTAT_RCC  | /* INTSTAT_RQE | INTSTAT_OVW | INTSTAT_RXE | */
 		/* INTSTAT_TXC | */ INTSTAT_TCC | INTSTAT_TQE | INTSTAT_TXU |
 		INTSTAT_FATAL);
 
 	/* Acknowledge all pending interrupts. */
 	CSR_WRITE_4(sc, INTSTAT, CSR_READ_4(sc, INTSTAT));
 
 	/* Enable interrupts,  set for PCI read multiple and etc */
 	CSR_WRITE_4(sc, GENCTL,
 		GENCTL_ENABLE_INTERRUPT | GENCTL_MEMORY_READ_MULTIPLE |
 		GENCTL_ONECOPY | GENCTL_RECEIVE_FIFO_THRESHOLD64);
 
 	/* Mark interface running ... */
 	if (ifp->if_flags & IFF_UP)
 		ifp->if_drv_flags |= IFF_DRV_RUNNING;
 	else
 		ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
 
 	/* ... and free */
 	ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
 
 	/* Start Rx process */
 	epic_start_activity(sc);
 
 	/* Set appropriate media */
 	epic_ifmedia_upd_locked(ifp);
 
 	callout_reset(&sc->timer, hz, epic_timer, sc);
 }
 
 /*
  * Synopsis: calculate and set Rx mode. Chip must be in idle state to
  * access RXCON.
  */
 static void
 epic_set_rx_mode(epic_softc_t *sc)
 {
 	u_int32_t flags;
 	u_int32_t rxcon;
 
 	flags = sc->ifp->if_flags;
 	rxcon = RXCON_DEFAULT;
 
 #ifdef EPIC_EARLY_RX
 	rxcon |= RXCON_EARLY_RX;
 #endif
 
 	rxcon |= (flags & IFF_PROMISC) ? RXCON_PROMISCUOUS_MODE : 0;
 
 	CSR_WRITE_4(sc, RXCON, rxcon);
 }
 
 /*
  * Synopsis: Set transmit control register. Chip must be in idle state to
  * access TXCON.
  */
 static void
 epic_set_tx_mode(epic_softc_t *sc)
 {
 
 	if (sc->txcon & TXCON_EARLY_TRANSMIT_ENABLE)
 		CSR_WRITE_4(sc, ETXTHR, sc->tx_threshold);
 
 	CSR_WRITE_4(sc, TXCON, sc->txcon);
 }
 
 /*
  * Synopsis: Program multicast filter honoring IFF_ALLMULTI and IFF_PROMISC
  * flags (note that setting PROMISC bit in EPIC's RXCON will only touch
  * individual frames, multicast filter must be manually programmed).
  *
  * Note: EPIC must be in idle state.
  */
 static void
 epic_set_mc_table(epic_softc_t *sc)
 {
 	struct ifnet *ifp;
 	struct ifmultiaddr *ifma;
 	u_int16_t filter[4];
 	u_int8_t h;
 
 	ifp = sc->ifp;
 	if (ifp->if_flags & (IFF_ALLMULTI | IFF_PROMISC)) {
 		CSR_WRITE_4(sc, MC0, 0xFFFF);
 		CSR_WRITE_4(sc, MC1, 0xFFFF);
 		CSR_WRITE_4(sc, MC2, 0xFFFF);
 		CSR_WRITE_4(sc, MC3, 0xFFFF);
 		return;
 	}
 
 	filter[0] = 0;
 	filter[1] = 0;
 	filter[2] = 0;
 	filter[3] = 0;
 
 	if_maddr_rlock(ifp);
 	TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
 		if (ifma->ifma_addr->sa_family != AF_LINK)
 			continue;
 		h = ether_crc32_be(LLADDR((struct sockaddr_dl *)
 		    ifma->ifma_addr), ETHER_ADDR_LEN) >> 26;
 		filter[h >> 4] |= 1 << (h & 0xF);
 	}
 	if_maddr_runlock(ifp);
 
 	CSR_WRITE_4(sc, MC0, filter[0]);
 	CSR_WRITE_4(sc, MC1, filter[1]);
 	CSR_WRITE_4(sc, MC2, filter[2]);
 	CSR_WRITE_4(sc, MC3, filter[3]);
 }
 
 
 /*
  * Synopsis: Start receive process and transmit one, if they need.
  */
 static void
 epic_start_activity(epic_softc_t *sc)
 {
 
 	/* Start rx process. */
 	CSR_WRITE_4(sc, COMMAND, COMMAND_RXQUEUED | COMMAND_START_RX |
 	    (sc->pending_txs ? COMMAND_TXQUEUED : 0));
 }
 
 /*
  * Synopsis: Completely stop Rx and Tx processes. If TQE is set additional
  * packet needs to be queued to stop Tx DMA.
  */
 static void
 epic_stop_activity(epic_softc_t *sc)
 {
 	int status, i;
 
 	/* Stop Tx and Rx DMA. */
 	CSR_WRITE_4(sc, COMMAND,
 	    COMMAND_STOP_RX | COMMAND_STOP_RDMA | COMMAND_STOP_TDMA);
 
 	/* Wait Rx and Tx DMA to stop (why 1 ms ??? XXX). */
 	for (i = 0; i < 0x1000; i++) {
 		status = CSR_READ_4(sc, INTSTAT) &
 		    (INTSTAT_TXIDLE | INTSTAT_RXIDLE);
 		if (status == (INTSTAT_TXIDLE | INTSTAT_RXIDLE))
 			break;
 		DELAY(1);
 	}
 
 	/* Catch all finished packets. */
 	epic_rx_done(sc);
 	epic_tx_done(sc);
 
 	status = CSR_READ_4(sc, INTSTAT);
 
 	if ((status & INTSTAT_RXIDLE) == 0)
 		device_printf(sc->dev, "ERROR! Can't stop Rx DMA\n");
 
 	if ((status & INTSTAT_TXIDLE) == 0)
 		device_printf(sc->dev, "ERROR! Can't stop Tx DMA\n");
 
 	/*
 	 * May need to queue one more packet if TQE, this is rare
 	 * but existing case.
 	 */
 	if ((status & INTSTAT_TQE) && !(status & INTSTAT_TXIDLE))
 		(void)epic_queue_last_packet(sc);
 }
 
 /*
  * The EPIC transmitter may stuck in TQE state. It will not go IDLE until
  * a packet from current descriptor will be copied to internal RAM. We
  * compose a dummy packet here and queue it for transmission.
  *
  * XXX the packet will then be actually sent over network...
  */
 static int
 epic_queue_last_packet(epic_softc_t *sc)
 {
 	struct epic_tx_desc *desc;
 	struct epic_frag_list *flist;
 	struct epic_tx_buffer *buf;
 	struct mbuf *m0;
 	int error, i;
 
 	device_printf(sc->dev, "queue last packet\n");
 
 	desc = sc->tx_desc + sc->cur_tx;
 	flist = sc->tx_flist + sc->cur_tx;
 	buf = sc->tx_buffer + sc->cur_tx;
 
 	if ((desc->status & 0x8000) || (buf->mbuf != NULL))
 		return (EBUSY);
 
 	MGETHDR(m0, M_NOWAIT, MT_DATA);
 	if (m0 == NULL)
 		return (ENOBUFS);
 
 	/* Prepare mbuf. */
 	m0->m_len = min(MHLEN, ETHER_MIN_LEN - ETHER_CRC_LEN);
 	m0->m_pkthdr.len = m0->m_len;
 	m0->m_pkthdr.rcvif = sc->ifp;
 	bzero(mtod(m0, caddr_t), m0->m_len);
 
 	/* Fill fragments list. */
 	error = bus_dmamap_load_mbuf(sc->mtag, buf->map, m0,
 	    epic_dma_map_txbuf, flist, 0);
 	if (error) {
 		m_freem(m0);
 		return (error);
 	}
 	bus_dmamap_sync(sc->mtag, buf->map, BUS_DMASYNC_PREWRITE);
 
 	/* Fill in descriptor. */
 	buf->mbuf = m0;
 	sc->pending_txs++;
 	sc->cur_tx = (sc->cur_tx + 1) & TX_RING_MASK;
 	desc->control = 0x01;
 	desc->txlength = max(m0->m_pkthdr.len, ETHER_MIN_LEN - ETHER_CRC_LEN);
 	desc->status = 0x8000;
 	bus_dmamap_sync(sc->ttag, sc->tmap,
 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 	bus_dmamap_sync(sc->ftag, sc->fmap, BUS_DMASYNC_PREWRITE);
 
 	/* Launch transmission. */
 	CSR_WRITE_4(sc, COMMAND, COMMAND_STOP_TDMA | COMMAND_TXQUEUED);
 
 	/* Wait Tx DMA to stop (for how long??? XXX) */
 	for (i = 0; i < 1000; i++) {
 		if (CSR_READ_4(sc, INTSTAT) & INTSTAT_TXIDLE)
 			break;
 		DELAY(1);
 	}
 
 	if ((CSR_READ_4(sc, INTSTAT) & INTSTAT_TXIDLE) == 0)
 		device_printf(sc->dev, "ERROR! can't stop Tx DMA (2)\n");
 	else
 		epic_tx_done(sc);
 
 	return (0);
 }
 
 /*
  *  Synopsis: Shut down board and deallocates rings.
  */
 static void
 epic_stop(epic_softc_t *sc)
 {
 
 	EPIC_ASSERT_LOCKED(sc);
 
 	sc->tx_timeout = 0;
 	callout_stop(&sc->timer);
 
 	/* Disable interrupts */
 	CSR_WRITE_4(sc, INTMASK, 0);
 	CSR_WRITE_4(sc, GENCTL, 0);
 
 	/* Try to stop Rx and TX processes */
 	epic_stop_activity(sc);
 
 	/* Reset chip */
 	CSR_WRITE_4(sc, GENCTL, GENCTL_SOFT_RESET);
 	DELAY(1000);
 
 	/* Make chip go to bed */
 	CSR_WRITE_4(sc, GENCTL, GENCTL_POWER_DOWN);
 
 	/* Mark as stopped */
 	sc->ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
 }
 
 /*
  * Synopsis: This function should free all memory allocated for rings.
  */
 static void
 epic_free_rings(epic_softc_t *sc)
 {
 	int i;
 
 	for (i = 0; i < RX_RING_SIZE; i++) {
 		struct epic_rx_buffer *buf = sc->rx_buffer + i;
 		struct epic_rx_desc *desc = sc->rx_desc + i;
 
 		desc->status = 0;
 		desc->buflength = 0;
 		desc->bufaddr = 0;
 
 		if (buf->mbuf) {
 			bus_dmamap_unload(sc->mtag, buf->map);
 			bus_dmamap_destroy(sc->mtag, buf->map);
 			m_freem(buf->mbuf);
 		}
 		buf->mbuf = NULL;
 	}
 
 	if (sc->sparemap != NULL)
 		bus_dmamap_destroy(sc->mtag, sc->sparemap);
 
 	for (i = 0; i < TX_RING_SIZE; i++) {
 		struct epic_tx_buffer *buf = sc->tx_buffer + i;
 		struct epic_tx_desc *desc = sc->tx_desc + i;
 
 		desc->status = 0;
 		desc->buflength = 0;
 		desc->bufaddr = 0;
 
 		if (buf->mbuf) {
 			bus_dmamap_unload(sc->mtag, buf->map);
 			bus_dmamap_destroy(sc->mtag, buf->map);
 			m_freem(buf->mbuf);
 		}
 		buf->mbuf = NULL;
 	}
 }
 
 /*
  * Synopsis:  Allocates mbufs for Rx ring and point Rx descs to them.
  * Point Tx descs to fragment lists. Check that all descs and fraglists
  * are bounded and aligned properly.
  */
 static int
 epic_init_rings(epic_softc_t *sc)
 {
 	int error, i;
 
 	sc->cur_rx = sc->cur_tx = sc->dirty_tx = sc->pending_txs = 0;
 
 	/* Initialize the RX descriptor ring. */
 	for (i = 0; i < RX_RING_SIZE; i++) {
 		struct epic_rx_buffer *buf = sc->rx_buffer + i;
 		struct epic_rx_desc *desc = sc->rx_desc + i;
 
 		desc->status = 0;		/* Owned by driver */
 		desc->next = sc->rx_addr +
 		    ((i + 1) & RX_RING_MASK) * sizeof(struct epic_rx_desc);
 
 		if ((desc->next & 3) ||
 		    ((desc->next & PAGE_MASK) + sizeof *desc) > PAGE_SIZE) {
 			epic_free_rings(sc);
 			return (EFAULT);
 		}
 
 		buf->mbuf = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
 		if (buf->mbuf == NULL) {
 			epic_free_rings(sc);
 			return (ENOBUFS);
 		}
 		buf->mbuf->m_len = buf->mbuf->m_pkthdr.len = MCLBYTES;
 		m_adj(buf->mbuf, ETHER_ALIGN);
 
 		error = bus_dmamap_create(sc->mtag, 0, &buf->map);
 		if (error) {
 			epic_free_rings(sc);
 			return (error);
 		}
 		error = bus_dmamap_load_mbuf(sc->mtag, buf->map, buf->mbuf,
 		    epic_dma_map_rxbuf, desc, 0);
 		if (error) {
 			epic_free_rings(sc);
 			return (error);
 		}
 		bus_dmamap_sync(sc->mtag, buf->map, BUS_DMASYNC_PREREAD);
 
 		desc->buflength = buf->mbuf->m_len; /* Max RX buffer length */
 		desc->status = 0x8000;		/* Set owner bit to NIC */
 	}
 	bus_dmamap_sync(sc->rtag, sc->rmap,
 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 
 	/* Create the spare DMA map. */
 	error = bus_dmamap_create(sc->mtag, 0, &sc->sparemap);
 	if (error) {
 		epic_free_rings(sc);
 		return (error);
 	}
 
 	/* Initialize the TX descriptor ring. */
 	for (i = 0; i < TX_RING_SIZE; i++) {
 		struct epic_tx_buffer *buf = sc->tx_buffer + i;
 		struct epic_tx_desc *desc = sc->tx_desc + i;
 
 		desc->status = 0;
 		desc->next = sc->tx_addr +
 		    ((i + 1) & TX_RING_MASK) * sizeof(struct epic_tx_desc);
 
 		if ((desc->next & 3) ||
 		    ((desc->next & PAGE_MASK) + sizeof *desc) > PAGE_SIZE) {
 			epic_free_rings(sc);
 			return (EFAULT);
 		}
 
 		buf->mbuf = NULL;
 		desc->bufaddr = sc->frag_addr +
 		    i * sizeof(struct epic_frag_list);
 
 		if ((desc->bufaddr & 3) ||
 		    ((desc->bufaddr & PAGE_MASK) +
 		    sizeof(struct epic_frag_list)) > PAGE_SIZE) {
 			epic_free_rings(sc);
 			return (EFAULT);
 		}
 
 		error = bus_dmamap_create(sc->mtag, 0, &buf->map);
 		if (error) {
 			epic_free_rings(sc);
 			return (error);
 		}
 	}
 	bus_dmamap_sync(sc->ttag, sc->tmap,
 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 	bus_dmamap_sync(sc->ftag, sc->fmap, BUS_DMASYNC_PREWRITE);
 
 	return (0);
 }
 
 /*
  * EEPROM operation functions
  */
 static void
 epic_write_eepromreg(epic_softc_t *sc, u_int8_t val)
 {
 	u_int16_t i;
 
 	CSR_WRITE_1(sc, EECTL, val);
 
 	for (i = 0; i < 0xFF; i++) {
 		if ((CSR_READ_1(sc, EECTL) & 0x20) == 0)
 			break;
 	}
 }
 
 static u_int8_t
 epic_read_eepromreg(epic_softc_t *sc)
 {
 
 	return (CSR_READ_1(sc, EECTL));
 }
 
 static u_int8_t
 epic_eeprom_clock(epic_softc_t *sc, u_int8_t val)
 {
 
 	epic_write_eepromreg(sc, val);
 	epic_write_eepromreg(sc, (val | 0x4));
 	epic_write_eepromreg(sc, val);
 
 	return (epic_read_eepromreg(sc));
 }
 
 static void
 epic_output_eepromw(epic_softc_t *sc, u_int16_t val)
 {
 	int i;
 
 	for (i = 0xF; i >= 0; i--) {
 		if (val & (1 << i))
 			epic_eeprom_clock(sc, 0x0B);
 		else
 			epic_eeprom_clock(sc, 0x03);
 	}
 }
 
 static u_int16_t
 epic_input_eepromw(epic_softc_t *sc)
 {
 	u_int16_t retval = 0;
 	int i;
 
 	for (i = 0xF; i >= 0; i--) {
 		if (epic_eeprom_clock(sc, 0x3) & 0x10)
 			retval |= (1 << i);
 	}
 
 	return (retval);
 }
 
 static int
 epic_read_eeprom(epic_softc_t *sc, u_int16_t loc)
 {
 	u_int16_t dataval;
 	u_int16_t read_cmd;
 
 	epic_write_eepromreg(sc, 3);
 
 	if (epic_read_eepromreg(sc) & 0x40)
 		read_cmd = (loc & 0x3F) | 0x180;
 	else
 		read_cmd = (loc & 0xFF) | 0x600;
 
 	epic_output_eepromw(sc, read_cmd);
 
 	dataval = epic_input_eepromw(sc);
 
 	epic_write_eepromreg(sc, 1);
 
 	return (dataval);
 }
 
 /*
  * Here goes MII read/write routines.
  */
 static int
 epic_read_phy_reg(epic_softc_t *sc, int phy, int reg)
 {
 	int i;
 
 	CSR_WRITE_4(sc, MIICTL, ((reg << 4) | (phy << 9) | 0x01));
 
 	for (i = 0; i < 0x100; i++) {
 		if ((CSR_READ_4(sc, MIICTL) & 0x01) == 0)
 			break;
 		DELAY(1);
 	}
 
 	return (CSR_READ_4(sc, MIIDATA));
 }
 
 static void
 epic_write_phy_reg(epic_softc_t *sc, int phy, int reg, int val)
 {
 	int i;
 
 	CSR_WRITE_4(sc, MIIDATA, val);
 	CSR_WRITE_4(sc, MIICTL, ((reg << 4) | (phy << 9) | 0x02));
 
 	for(i = 0; i < 0x100; i++) {
 		if ((CSR_READ_4(sc, MIICTL) & 0x02) == 0)
 			break;
 		DELAY(1);
 	}
 }
 
 static int
 epic_miibus_readreg(device_t dev, int phy, int reg)
 {
 	epic_softc_t *sc;
 
 	sc = device_get_softc(dev);
 
 	return (PHY_READ_2(sc, phy, reg));
 }
 
 static int
 epic_miibus_writereg(device_t dev, int phy, int reg, int data)
 {
 	epic_softc_t *sc;
 
 	sc = device_get_softc(dev);
 
 	PHY_WRITE_2(sc, phy, reg, data);
 
 	return (0);
 }
Index: projects/release-debugdist/sys/dev/ubsec/ubsec.c
===================================================================
--- projects/release-debugdist/sys/dev/ubsec/ubsec.c	(revision 267343)
+++ projects/release-debugdist/sys/dev/ubsec/ubsec.c	(revision 267344)
@@ -1,2869 +1,2858 @@
 /*	$OpenBSD: ubsec.c,v 1.115 2002/09/24 18:33:26 jason Exp $	*/
 
 /*-
  * Copyright (c) 2000 Jason L. Wright (jason@thought.net)
  * Copyright (c) 2000 Theo de Raadt (deraadt@openbsd.org)
  * Copyright (c) 2001 Patrik Lindergren (patrik@ipunplugged.com)
  *
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  * 3. All advertising materials mentioning features or use of this software
  *    must display the following acknowledgement:
  *	This product includes software developed by Jason L. Wright
  * 4. The name of the author may not be used to endorse or promote products
  *    derived from this software without specific prior written permission.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  * DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
  * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
  * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  * POSSIBILITY OF SUCH DAMAGE.
  *
  * Effort sponsored in part by the Defense Advanced Research Projects
  * Agency (DARPA) and Air Force Research Laboratory, Air Force
  * Materiel Command, USAF, under agreement number F30602-01-2-0537.
  */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 /*
  * uBsec 5[56]01, 58xx hardware crypto accelerator
  */
 
 #include "opt_ubsec.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/proc.h>
 #include <sys/errno.h>
 #include <sys/malloc.h>
 #include <sys/kernel.h>
 #include <sys/module.h>
 #include <sys/mbuf.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <sys/sysctl.h>
 #include <sys/endian.h>
 
 #include <vm/vm.h>
 #include <vm/pmap.h>
 
 #include <machine/bus.h>
 #include <machine/resource.h>
 #include <sys/bus.h>
 #include <sys/rman.h>
 
 #include <crypto/sha1.h>
 #include <opencrypto/cryptodev.h>
 #include <opencrypto/cryptosoft.h>
 #include <sys/md5.h>
 #include <sys/random.h>
 #include <sys/kobj.h>
 
 #include "cryptodev_if.h"
 
 #include <dev/pci/pcivar.h>
 #include <dev/pci/pcireg.h>
 
 /* grr, #defines for gratuitous incompatibility in queue.h */
 #define	SIMPLEQ_HEAD		STAILQ_HEAD
 #define	SIMPLEQ_ENTRY		STAILQ_ENTRY
 #define	SIMPLEQ_INIT		STAILQ_INIT
 #define	SIMPLEQ_INSERT_TAIL	STAILQ_INSERT_TAIL
 #define	SIMPLEQ_EMPTY		STAILQ_EMPTY
 #define	SIMPLEQ_FIRST		STAILQ_FIRST
 #define	SIMPLEQ_REMOVE_HEAD	STAILQ_REMOVE_HEAD
 #define	SIMPLEQ_FOREACH		STAILQ_FOREACH
 /* ditto for endian.h */
 #define	letoh16(x)		le16toh(x)
 #define	letoh32(x)		le32toh(x)
 
 #ifdef UBSEC_RNDTEST
 #include <dev/rndtest/rndtest.h>
 #endif
 #include <dev/ubsec/ubsecreg.h>
 #include <dev/ubsec/ubsecvar.h>
 
 /*
  * Prototypes and count for the pci_device structure
  */
 static	int ubsec_probe(device_t);
 static	int ubsec_attach(device_t);
 static	int ubsec_detach(device_t);
 static	int ubsec_suspend(device_t);
 static	int ubsec_resume(device_t);
 static	int ubsec_shutdown(device_t);
 
 static	int ubsec_newsession(device_t, u_int32_t *, struct cryptoini *);
 static	int ubsec_freesession(device_t, u_int64_t);
 static	int ubsec_process(device_t, struct cryptop *, int);
 static	int ubsec_kprocess(device_t, struct cryptkop *, int);
 
 static device_method_t ubsec_methods[] = {
 	/* Device interface */
 	DEVMETHOD(device_probe,		ubsec_probe),
 	DEVMETHOD(device_attach,	ubsec_attach),
 	DEVMETHOD(device_detach,	ubsec_detach),
 	DEVMETHOD(device_suspend,	ubsec_suspend),
 	DEVMETHOD(device_resume,	ubsec_resume),
 	DEVMETHOD(device_shutdown,	ubsec_shutdown),
 
 	/* crypto device methods */
 	DEVMETHOD(cryptodev_newsession,	ubsec_newsession),
 	DEVMETHOD(cryptodev_freesession,ubsec_freesession),
 	DEVMETHOD(cryptodev_process,	ubsec_process),
 	DEVMETHOD(cryptodev_kprocess,	ubsec_kprocess),
 
 	DEVMETHOD_END
 };
 static driver_t ubsec_driver = {
 	"ubsec",
 	ubsec_methods,
 	sizeof (struct ubsec_softc)
 };
 static devclass_t ubsec_devclass;
 
 DRIVER_MODULE(ubsec, pci, ubsec_driver, ubsec_devclass, 0, 0);
 MODULE_DEPEND(ubsec, crypto, 1, 1, 1);
 #ifdef UBSEC_RNDTEST
 MODULE_DEPEND(ubsec, rndtest, 1, 1, 1);
 #endif
 
 static	void ubsec_intr(void *);
 static	void ubsec_callback(struct ubsec_softc *, struct ubsec_q *);
 static	void ubsec_feed(struct ubsec_softc *);
 static	void ubsec_mcopy(struct mbuf *, struct mbuf *, int, int);
 static	void ubsec_callback2(struct ubsec_softc *, struct ubsec_q2 *);
 static	int ubsec_feed2(struct ubsec_softc *);
 static	void ubsec_rng(void *);
 static	int ubsec_dma_malloc(struct ubsec_softc *, bus_size_t,
 			     struct ubsec_dma_alloc *, int);
 #define	ubsec_dma_sync(_dma, _flags) \
 	bus_dmamap_sync((_dma)->dma_tag, (_dma)->dma_map, (_flags))
 static	void ubsec_dma_free(struct ubsec_softc *, struct ubsec_dma_alloc *);
 static	int ubsec_dmamap_aligned(struct ubsec_operand *op);
 
 static	void ubsec_reset_board(struct ubsec_softc *sc);
 static	void ubsec_init_board(struct ubsec_softc *sc);
 static	void ubsec_init_pciregs(device_t dev);
 static	void ubsec_totalreset(struct ubsec_softc *sc);
 
 static	int ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q);
 
 static	int ubsec_kprocess_modexp_hw(struct ubsec_softc *, struct cryptkop *, int);
 static	int ubsec_kprocess_modexp_sw(struct ubsec_softc *, struct cryptkop *, int);
 static	int ubsec_kprocess_rsapriv(struct ubsec_softc *, struct cryptkop *, int);
 static	void ubsec_kfree(struct ubsec_softc *, struct ubsec_q2 *);
 static	int ubsec_ksigbits(struct crparam *);
 static	void ubsec_kshift_r(u_int, u_int8_t *, u_int, u_int8_t *, u_int);
 static	void ubsec_kshift_l(u_int, u_int8_t *, u_int, u_int8_t *, u_int);
 
 static SYSCTL_NODE(_hw, OID_AUTO, ubsec, CTLFLAG_RD, 0,
     "Broadcom driver parameters");
 
 #ifdef UBSEC_DEBUG
 static	void ubsec_dump_pb(volatile struct ubsec_pktbuf *);
 static	void ubsec_dump_mcr(struct ubsec_mcr *);
 static	void ubsec_dump_ctx2(struct ubsec_ctx_keyop *);
 
 static	int ubsec_debug = 0;
 SYSCTL_INT(_hw_ubsec, OID_AUTO, debug, CTLFLAG_RW, &ubsec_debug,
 	    0, "control debugging msgs");
 #endif
 
 #define	READ_REG(sc,r) \
 	bus_space_read_4((sc)->sc_st, (sc)->sc_sh, (r))
 
 #define WRITE_REG(sc,reg,val) \
 	bus_space_write_4((sc)->sc_st, (sc)->sc_sh, reg, val)
 
 #define	SWAP32(x) (x) = htole32(ntohl((x)))
 #define	HTOLE32(x) (x) = htole32(x)
 
 struct ubsec_stats ubsecstats;
 SYSCTL_STRUCT(_hw_ubsec, OID_AUTO, stats, CTLFLAG_RD, &ubsecstats,
 	    ubsec_stats, "driver statistics");
 
 static int
 ubsec_probe(device_t dev)
 {
 	if (pci_get_vendor(dev) == PCI_VENDOR_SUN &&
 	    (pci_get_device(dev) == PCI_PRODUCT_SUN_5821 ||
 	     pci_get_device(dev) == PCI_PRODUCT_SUN_SCA1K))
 		return (BUS_PROBE_DEFAULT);
 	if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL &&
 	    (pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5501 ||
 	     pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601))
 		return (BUS_PROBE_DEFAULT);
 	if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
 	    (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5801 ||
 	     pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5802 ||
 	     pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805 ||
 	     pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820 ||
 	     pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 ||
 	     pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822 ||
 	     pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5823 ||
 	     pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5825
 	     ))
 		return (BUS_PROBE_DEFAULT);
 	return (ENXIO);
 }
 
 static const char*
 ubsec_partname(struct ubsec_softc *sc)
 {
 	/* XXX sprintf numbers when not decoded */
 	switch (pci_get_vendor(sc->sc_dev)) {
 	case PCI_VENDOR_BROADCOM:
 		switch (pci_get_device(sc->sc_dev)) {
 		case PCI_PRODUCT_BROADCOM_5801:	return "Broadcom 5801";
 		case PCI_PRODUCT_BROADCOM_5802:	return "Broadcom 5802";
 		case PCI_PRODUCT_BROADCOM_5805:	return "Broadcom 5805";
 		case PCI_PRODUCT_BROADCOM_5820:	return "Broadcom 5820";
 		case PCI_PRODUCT_BROADCOM_5821:	return "Broadcom 5821";
 		case PCI_PRODUCT_BROADCOM_5822:	return "Broadcom 5822";
 		case PCI_PRODUCT_BROADCOM_5823:	return "Broadcom 5823";
 		case PCI_PRODUCT_BROADCOM_5825:	return "Broadcom 5825";
 		}
 		return "Broadcom unknown-part";
 	case PCI_VENDOR_BLUESTEEL:
 		switch (pci_get_device(sc->sc_dev)) {
 		case PCI_PRODUCT_BLUESTEEL_5601: return "Bluesteel 5601";
 		}
 		return "Bluesteel unknown-part";
 	case PCI_VENDOR_SUN:
 		switch (pci_get_device(sc->sc_dev)) {
 		case PCI_PRODUCT_SUN_5821: return "Sun Crypto 5821";
 		case PCI_PRODUCT_SUN_SCA1K: return "Sun Crypto 1K";
 		}
 		return "Sun unknown-part";
 	}
 	return "Unknown-vendor unknown-part";
 }
 
 static void
 default_harvest(struct rndtest_state *rsp, void *buf, u_int count)
 {
 	random_harvest(buf, count, count*NBBY/2, RANDOM_PURE_UBSEC);
 }
 
 static int
 ubsec_attach(device_t dev)
 {
 	struct ubsec_softc *sc = device_get_softc(dev);
 	struct ubsec_dma *dmap;
 	u_int32_t i;
 	int rid;
 
 	bzero(sc, sizeof (*sc));
 	sc->sc_dev = dev;
 
 	SIMPLEQ_INIT(&sc->sc_queue);
 	SIMPLEQ_INIT(&sc->sc_qchip);
 	SIMPLEQ_INIT(&sc->sc_queue2);
 	SIMPLEQ_INIT(&sc->sc_qchip2);
 	SIMPLEQ_INIT(&sc->sc_q2free);
 
 	/* XXX handle power management */
 
 	sc->sc_statmask = BS_STAT_MCR1_DONE | BS_STAT_DMAERR;
 
 	if (pci_get_vendor(dev) == PCI_VENDOR_BLUESTEEL &&
 	    pci_get_device(dev) == PCI_PRODUCT_BLUESTEEL_5601)
 		sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG;
 
 	if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
 	    (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5802 ||
 	     pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5805))
 		sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG;
 
 	if (pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
 	    pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5820)
 		sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG |
 		    UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY;
 
 	if ((pci_get_vendor(dev) == PCI_VENDOR_BROADCOM &&
 	     (pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5821 ||
 	      pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5822 ||
 	      pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5823 ||
 	      pci_get_device(dev) == PCI_PRODUCT_BROADCOM_5825)) ||
 	    (pci_get_vendor(dev) == PCI_VENDOR_SUN &&
 	     (pci_get_device(dev) == PCI_PRODUCT_SUN_SCA1K ||
 	      pci_get_device(dev) == PCI_PRODUCT_SUN_5821))) {
 		/* NB: the 5821/5822 defines some additional status bits */
 		sc->sc_statmask |= BS_STAT_MCR1_ALLEMPTY |
 		    BS_STAT_MCR2_ALLEMPTY;
 		sc->sc_flags |= UBS_FLAGS_KEY | UBS_FLAGS_RNG |
 		    UBS_FLAGS_LONGCTX | UBS_FLAGS_HWNORM | UBS_FLAGS_BIGKEY;
 	}
 
 	pci_enable_busmaster(dev);
 
 	/*
 	 * Setup memory-mapping of PCI registers.
 	 */
 	rid = BS_BAR;
 	sc->sc_sr = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
 					   RF_ACTIVE);
 	if (sc->sc_sr == NULL) {
 		device_printf(dev, "cannot map register space\n");
 		goto bad;
 	}
 	sc->sc_st = rman_get_bustag(sc->sc_sr);
 	sc->sc_sh = rman_get_bushandle(sc->sc_sr);
 
 	/*
 	 * Arrange interrupt line.
 	 */
 	rid = 0;
 	sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
 					    RF_SHAREABLE|RF_ACTIVE);
 	if (sc->sc_irq == NULL) {
 		device_printf(dev, "could not map interrupt\n");
 		goto bad1;
 	}
 	/*
 	 * NB: Network code assumes we are blocked with splimp()
 	 *     so make sure the IRQ is mapped appropriately.
 	 */
 	if (bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET | INTR_MPSAFE,
 			   NULL, ubsec_intr, sc, &sc->sc_ih)) {
 		device_printf(dev, "could not establish interrupt\n");
 		goto bad2;
 	}
 
 	sc->sc_cid = crypto_get_driverid(dev, CRYPTOCAP_F_HARDWARE);
 	if (sc->sc_cid < 0) {
 		device_printf(dev, "could not get crypto driver id\n");
 		goto bad3;
 	}
 
 	/*
 	 * Setup DMA descriptor area.
 	 */
 	if (bus_dma_tag_create(bus_get_dma_tag(dev),	/* parent */
 			       1, 0,			/* alignment, bounds */
 			       BUS_SPACE_MAXADDR_32BIT,	/* lowaddr */
 			       BUS_SPACE_MAXADDR,	/* highaddr */
 			       NULL, NULL,		/* filter, filterarg */
 			       0x3ffff,			/* maxsize */
 			       UBS_MAX_SCATTER,		/* nsegments */
 			       0xffff,			/* maxsegsize */
 			       BUS_DMA_ALLOCNOW,	/* flags */
 			       NULL, NULL,		/* lockfunc, lockarg */
 			       &sc->sc_dmat)) {
 		device_printf(dev, "cannot allocate DMA tag\n");
 		goto bad4;
 	}
 	SIMPLEQ_INIT(&sc->sc_freequeue);
 	dmap = sc->sc_dmaa;
 	for (i = 0; i < UBS_MAX_NQUEUE; i++, dmap++) {
 		struct ubsec_q *q;
 
 		q = (struct ubsec_q *)malloc(sizeof(struct ubsec_q),
 		    M_DEVBUF, M_NOWAIT);
 		if (q == NULL) {
 			device_printf(dev, "cannot allocate queue buffers\n");
 			break;
 		}
 
 		if (ubsec_dma_malloc(sc, sizeof(struct ubsec_dmachunk),
 		    &dmap->d_alloc, 0)) {
 			device_printf(dev, "cannot allocate dma buffers\n");
 			free(q, M_DEVBUF);
 			break;
 		}
 		dmap->d_dma = (struct ubsec_dmachunk *)dmap->d_alloc.dma_vaddr;
 
 		q->q_dma = dmap;
 		sc->sc_queuea[i] = q;
 
 		SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
 	}
 	mtx_init(&sc->sc_mcr1lock, device_get_nameunit(dev),
 		"mcr1 operations", MTX_DEF);
 	mtx_init(&sc->sc_freeqlock, device_get_nameunit(dev),
 		"mcr1 free q", MTX_DEF);
 
 	device_printf(sc->sc_dev, "%s\n", ubsec_partname(sc));
 
 	crypto_register(sc->sc_cid, CRYPTO_3DES_CBC, 0, 0);
 	crypto_register(sc->sc_cid, CRYPTO_DES_CBC, 0, 0);
 	crypto_register(sc->sc_cid, CRYPTO_MD5_HMAC, 0, 0);
 	crypto_register(sc->sc_cid, CRYPTO_SHA1_HMAC, 0, 0);
 
 	/*
 	 * Reset Broadcom chip
 	 */
 	ubsec_reset_board(sc);
 
 	/*
 	 * Init Broadcom specific PCI settings
 	 */
 	ubsec_init_pciregs(dev);
 
 	/*
 	 * Init Broadcom chip
 	 */
 	ubsec_init_board(sc);
 
 #ifndef UBSEC_NO_RNG
 	if (sc->sc_flags & UBS_FLAGS_RNG) {
 		sc->sc_statmask |= BS_STAT_MCR2_DONE;
 #ifdef UBSEC_RNDTEST
 		sc->sc_rndtest = rndtest_attach(dev);
 		if (sc->sc_rndtest)
 			sc->sc_harvest = rndtest_harvest;
 		else
 			sc->sc_harvest = default_harvest;
 #else
 		sc->sc_harvest = default_harvest;
 #endif
 
 		if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
 		    &sc->sc_rng.rng_q.q_mcr, 0))
 			goto skip_rng;
 
 		if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rngbypass),
 		    &sc->sc_rng.rng_q.q_ctx, 0)) {
 			ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
 			goto skip_rng;
 		}
 
 		if (ubsec_dma_malloc(sc, sizeof(u_int32_t) *
 		    UBSEC_RNG_BUFSIZ, &sc->sc_rng.rng_buf, 0)) {
 			ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx);
 			ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
 			goto skip_rng;
 		}
 
 		if (hz >= 100)
 			sc->sc_rnghz = hz / 100;
 		else
 			sc->sc_rnghz = 1;
 		callout_init(&sc->sc_rngto, CALLOUT_MPSAFE);
 		callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc);
 skip_rng:
 	;
 	}
 #endif /* UBSEC_NO_RNG */
 	mtx_init(&sc->sc_mcr2lock, device_get_nameunit(dev),
 		"mcr2 operations", MTX_DEF);
 
 	if (sc->sc_flags & UBS_FLAGS_KEY) {
 		sc->sc_statmask |= BS_STAT_MCR2_DONE;
 
 		crypto_kregister(sc->sc_cid, CRK_MOD_EXP, 0);
 #if 0
 		crypto_kregister(sc->sc_cid, CRK_MOD_EXP_CRT, 0);
 #endif
 	}
 	return (0);
 bad4:
 	crypto_unregister_all(sc->sc_cid);
 bad3:
 	bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
 bad2:
 	bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
 bad1:
 	bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr);
 bad:
 	return (ENXIO);
 }
 
 /*
  * Detach a device that successfully probed.
  */
 static int
 ubsec_detach(device_t dev)
 {
 	struct ubsec_softc *sc = device_get_softc(dev);
 
 	/* XXX wait/abort active ops */
 
 	/* disable interrupts */
 	WRITE_REG(sc, BS_CTRL, READ_REG(sc, BS_CTRL) &~
 		(BS_CTRL_MCR2INT | BS_CTRL_MCR1INT | BS_CTRL_DMAERR));
 
 	callout_stop(&sc->sc_rngto);
 
 	crypto_unregister_all(sc->sc_cid);
 
 #ifdef UBSEC_RNDTEST
 	if (sc->sc_rndtest)
 		rndtest_detach(sc->sc_rndtest);
 #endif
 
 	while (!SIMPLEQ_EMPTY(&sc->sc_freequeue)) {
 		struct ubsec_q *q;
 
 		q = SIMPLEQ_FIRST(&sc->sc_freequeue);
 		SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q_next);
 		ubsec_dma_free(sc, &q->q_dma->d_alloc);
 		free(q, M_DEVBUF);
 	}
 	mtx_destroy(&sc->sc_mcr1lock);
 	mtx_destroy(&sc->sc_freeqlock);
 #ifndef UBSEC_NO_RNG
 	if (sc->sc_flags & UBS_FLAGS_RNG) {
 		ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_mcr);
 		ubsec_dma_free(sc, &sc->sc_rng.rng_q.q_ctx);
 		ubsec_dma_free(sc, &sc->sc_rng.rng_buf);
 	}
 #endif /* UBSEC_NO_RNG */
 	mtx_destroy(&sc->sc_mcr2lock);
 
 	bus_generic_detach(dev);
 	bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
 	bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq);
 
 	bus_dma_tag_destroy(sc->sc_dmat);
 	bus_release_resource(dev, SYS_RES_MEMORY, BS_BAR, sc->sc_sr);
 
 	return (0);
 }
 
 /*
  * Stop all chip i/o so that the kernel's probe routines don't
  * get confused by errant DMAs when rebooting.
  */
 static int
 ubsec_shutdown(device_t dev)
 {
 #ifdef notyet
 	ubsec_stop(device_get_softc(dev));
 #endif
 	return (0);
 }
 
 /*
  * Device suspend routine.
  */
 static int
 ubsec_suspend(device_t dev)
 {
 	struct ubsec_softc *sc = device_get_softc(dev);
 
 #ifdef notyet
 	/* XXX stop the device and save PCI settings */
 #endif
 	sc->sc_suspended = 1;
 
 	return (0);
 }
 
 static int
 ubsec_resume(device_t dev)
 {
 	struct ubsec_softc *sc = device_get_softc(dev);
 
 #ifdef notyet
 	/* XXX retore PCI settings and start the device */
 #endif
 	sc->sc_suspended = 0;
 	return (0);
 }
 
 /*
  * UBSEC Interrupt routine
  */
 static void
 ubsec_intr(void *arg)
 {
 	struct ubsec_softc *sc = arg;
 	volatile u_int32_t stat;
 	struct ubsec_q *q;
 	struct ubsec_dma *dmap;
 	int npkts = 0, i;
 
 	stat = READ_REG(sc, BS_STAT);
 	stat &= sc->sc_statmask;
 	if (stat == 0)
 		return;
 
 	WRITE_REG(sc, BS_STAT, stat);		/* IACK */
 
 	/*
 	 * Check to see if we have any packets waiting for us
 	 */
 	if ((stat & BS_STAT_MCR1_DONE)) {
 		mtx_lock(&sc->sc_mcr1lock);
 		while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) {
 			q = SIMPLEQ_FIRST(&sc->sc_qchip);
 			dmap = q->q_dma;
 
 			if ((dmap->d_dma->d_mcr.mcr_flags & htole16(UBS_MCR_DONE)) == 0)
 				break;
 
 			SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q_next);
 
 			npkts = q->q_nstacked_mcrs;
 			sc->sc_nqchip -= 1+npkts;
 			/*
 			 * search for further sc_qchip ubsec_q's that share
 			 * the same MCR, and complete them too, they must be
 			 * at the top.
 			 */
 			for (i = 0; i < npkts; i++) {
 				if(q->q_stacked_mcr[i]) {
 					ubsec_callback(sc, q->q_stacked_mcr[i]);
 				} else {
 					break;
 				}
 			}
 			ubsec_callback(sc, q);
 		}
 		/*
 		 * Don't send any more packet to chip if there has been
 		 * a DMAERR.
 		 */
 		if (!(stat & BS_STAT_DMAERR))
 			ubsec_feed(sc);
 		mtx_unlock(&sc->sc_mcr1lock);
 	}
 
 	/*
 	 * Check to see if we have any key setups/rng's waiting for us
 	 */
 	if ((sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG)) &&
 	    (stat & BS_STAT_MCR2_DONE)) {
 		struct ubsec_q2 *q2;
 		struct ubsec_mcr *mcr;
 
 		mtx_lock(&sc->sc_mcr2lock);
 		while (!SIMPLEQ_EMPTY(&sc->sc_qchip2)) {
 			q2 = SIMPLEQ_FIRST(&sc->sc_qchip2);
 
 			ubsec_dma_sync(&q2->q_mcr,
 			    BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
 
 			mcr = (struct ubsec_mcr *)q2->q_mcr.dma_vaddr;
 			if ((mcr->mcr_flags & htole16(UBS_MCR_DONE)) == 0) {
 				ubsec_dma_sync(&q2->q_mcr,
 				    BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
 				break;
 			}
 			SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip2, q_next);
 			ubsec_callback2(sc, q2);
 			/*
 			 * Don't send any more packet to chip if there has been
 			 * a DMAERR.
 			 */
 			if (!(stat & BS_STAT_DMAERR))
 				ubsec_feed2(sc);
 		}
 		mtx_unlock(&sc->sc_mcr2lock);
 	}
 
 	/*
 	 * Check to see if we got any DMA Error
 	 */
 	if (stat & BS_STAT_DMAERR) {
 #ifdef UBSEC_DEBUG
 		if (ubsec_debug) {
 			volatile u_int32_t a = READ_REG(sc, BS_ERR);
 
 			printf("dmaerr %s@%08x\n",
 			    (a & BS_ERR_READ) ? "read" : "write",
 			    a & BS_ERR_ADDR);
 		}
 #endif /* UBSEC_DEBUG */
 		ubsecstats.hst_dmaerr++;
 		mtx_lock(&sc->sc_mcr1lock);
 		ubsec_totalreset(sc);
 		ubsec_feed(sc);
 		mtx_unlock(&sc->sc_mcr1lock);
 	}
 
 	if (sc->sc_needwakeup) {		/* XXX check high watermark */
 		int wakeup;
 
 		mtx_lock(&sc->sc_freeqlock);
 		wakeup = sc->sc_needwakeup & (CRYPTO_SYMQ|CRYPTO_ASYMQ);
 #ifdef UBSEC_DEBUG
 		if (ubsec_debug)
 			device_printf(sc->sc_dev, "wakeup crypto (%x)\n",
 				sc->sc_needwakeup);
 #endif /* UBSEC_DEBUG */
 		sc->sc_needwakeup &= ~wakeup;
 		mtx_unlock(&sc->sc_freeqlock);
 		crypto_unblock(sc->sc_cid, wakeup);
 	}
 }
 
 /*
  * ubsec_feed() - aggregate and post requests to chip
  */
 static void
 ubsec_feed(struct ubsec_softc *sc)
 {
 	struct ubsec_q *q, *q2;
 	int npkts, i;
 	void *v;
 	u_int32_t stat;
 
 	/*
 	 * Decide how many ops to combine in a single MCR.  We cannot
 	 * aggregate more than UBS_MAX_AGGR because this is the number
 	 * of slots defined in the data structure.  Note that
 	 * aggregation only happens if ops are marked batch'able.
 	 * Aggregating ops reduces the number of interrupts to the host
 	 * but also (potentially) increases the latency for processing
 	 * completed ops as we only get an interrupt when all aggregated
 	 * ops have completed.
 	 */
 	if (sc->sc_nqueue == 0)
 		return;
 	if (sc->sc_nqueue > 1) {
 		npkts = 0;
 		SIMPLEQ_FOREACH(q, &sc->sc_queue, q_next) {
 			npkts++;
 			if ((q->q_crp->crp_flags & CRYPTO_F_BATCH) == 0)
 				break;
 		}
 	} else
 		npkts = 1;
 	/*
 	 * Check device status before going any further.
 	 */
 	if ((stat = READ_REG(sc, BS_STAT)) & (BS_STAT_MCR1_FULL | BS_STAT_DMAERR)) {
 		if (stat & BS_STAT_DMAERR) {
 			ubsec_totalreset(sc);
 			ubsecstats.hst_dmaerr++;
 		} else
 			ubsecstats.hst_mcr1full++;
 		return;
 	}
 	if (sc->sc_nqueue > ubsecstats.hst_maxqueue)
 		ubsecstats.hst_maxqueue = sc->sc_nqueue;
 	if (npkts > UBS_MAX_AGGR)
 		npkts = UBS_MAX_AGGR;
 	if (npkts < 2)				/* special case 1 op */
 		goto feed1;
 
 	ubsecstats.hst_totbatch += npkts-1;
 #ifdef UBSEC_DEBUG
 	if (ubsec_debug)
 		printf("merging %d records\n", npkts);
 #endif /* UBSEC_DEBUG */
 
 	q = SIMPLEQ_FIRST(&sc->sc_queue);
 	SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q_next);
 	--sc->sc_nqueue;
 
 	bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_PREWRITE);
 	if (q->q_dst_map != NULL)
 		bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, BUS_DMASYNC_PREREAD);
 
 	q->q_nstacked_mcrs = npkts - 1;		/* Number of packets stacked */
 
 	for (i = 0; i < q->q_nstacked_mcrs; i++) {
 		q2 = SIMPLEQ_FIRST(&sc->sc_queue);
 		bus_dmamap_sync(sc->sc_dmat, q2->q_src_map,
 		    BUS_DMASYNC_PREWRITE);
 		if (q2->q_dst_map != NULL)
 			bus_dmamap_sync(sc->sc_dmat, q2->q_dst_map,
 			    BUS_DMASYNC_PREREAD);
 		SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q_next);
 		--sc->sc_nqueue;
 
 		v = (void*)(((char *)&q2->q_dma->d_dma->d_mcr) + sizeof(struct ubsec_mcr) -
 		    sizeof(struct ubsec_mcr_add));
 		bcopy(v, &q->q_dma->d_dma->d_mcradd[i], sizeof(struct ubsec_mcr_add));
 		q->q_stacked_mcr[i] = q2;
 	}
 	q->q_dma->d_dma->d_mcr.mcr_pkts = htole16(npkts);
 	SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next);
 	sc->sc_nqchip += npkts;
 	if (sc->sc_nqchip > ubsecstats.hst_maxqchip)
 		ubsecstats.hst_maxqchip = sc->sc_nqchip;
 	ubsec_dma_sync(&q->q_dma->d_alloc,
 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 	WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr +
 	    offsetof(struct ubsec_dmachunk, d_mcr));
 	return;
 feed1:
 	q = SIMPLEQ_FIRST(&sc->sc_queue);
 
 	bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_PREWRITE);
 	if (q->q_dst_map != NULL)
 		bus_dmamap_sync(sc->sc_dmat, q->q_dst_map, BUS_DMASYNC_PREREAD);
 	ubsec_dma_sync(&q->q_dma->d_alloc,
 	    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 
 	WRITE_REG(sc, BS_MCR1, q->q_dma->d_alloc.dma_paddr +
 	    offsetof(struct ubsec_dmachunk, d_mcr));
 #ifdef UBSEC_DEBUG
 	if (ubsec_debug)
 		printf("feed1: q->chip %p %08x stat %08x\n",
 		      q, (u_int32_t)vtophys(&q->q_dma->d_dma->d_mcr),
 		      stat);
 #endif /* UBSEC_DEBUG */
 	SIMPLEQ_REMOVE_HEAD(&sc->sc_queue, q_next);
 	--sc->sc_nqueue;
 	SIMPLEQ_INSERT_TAIL(&sc->sc_qchip, q, q_next);
 	sc->sc_nqchip++;
 	if (sc->sc_nqchip > ubsecstats.hst_maxqchip)
 		ubsecstats.hst_maxqchip = sc->sc_nqchip;
 	return;
 }
 
 static void
 ubsec_setup_enckey(struct ubsec_session *ses, int algo, caddr_t key)
 {
 
 	/* Go ahead and compute key in ubsec's byte order */
 	if (algo == CRYPTO_DES_CBC) {
 		bcopy(key, &ses->ses_deskey[0], 8);
 		bcopy(key, &ses->ses_deskey[2], 8);
 		bcopy(key, &ses->ses_deskey[4], 8);
 	} else
 		bcopy(key, ses->ses_deskey, 24);
 
 	SWAP32(ses->ses_deskey[0]);
 	SWAP32(ses->ses_deskey[1]);
 	SWAP32(ses->ses_deskey[2]);
 	SWAP32(ses->ses_deskey[3]);
 	SWAP32(ses->ses_deskey[4]);
 	SWAP32(ses->ses_deskey[5]);
 }
 
 static void
 ubsec_setup_mackey(struct ubsec_session *ses, int algo, caddr_t key, int klen)
 {
 	MD5_CTX md5ctx;
 	SHA1_CTX sha1ctx;
 	int i;
 
 	for (i = 0; i < klen; i++)
 		key[i] ^= HMAC_IPAD_VAL;
 
 	if (algo == CRYPTO_MD5_HMAC) {
 		MD5Init(&md5ctx);
 		MD5Update(&md5ctx, key, klen);
 		MD5Update(&md5ctx, hmac_ipad_buffer, MD5_HMAC_BLOCK_LEN - klen);
 		bcopy(md5ctx.state, ses->ses_hminner, sizeof(md5ctx.state));
 	} else {
 		SHA1Init(&sha1ctx);
 		SHA1Update(&sha1ctx, key, klen);
 		SHA1Update(&sha1ctx, hmac_ipad_buffer,
 		    SHA1_HMAC_BLOCK_LEN - klen);
 		bcopy(sha1ctx.h.b32, ses->ses_hminner, sizeof(sha1ctx.h.b32));
 	}
 
 	for (i = 0; i < klen; i++)
 		key[i] ^= (HMAC_IPAD_VAL ^ HMAC_OPAD_VAL);
 
 	if (algo == CRYPTO_MD5_HMAC) {
 		MD5Init(&md5ctx);
 		MD5Update(&md5ctx, key, klen);
 		MD5Update(&md5ctx, hmac_opad_buffer, MD5_HMAC_BLOCK_LEN - klen);
 		bcopy(md5ctx.state, ses->ses_hmouter, sizeof(md5ctx.state));
 	} else {
 		SHA1Init(&sha1ctx);
 		SHA1Update(&sha1ctx, key, klen);
 		SHA1Update(&sha1ctx, hmac_opad_buffer,
 		    SHA1_HMAC_BLOCK_LEN - klen);
 		bcopy(sha1ctx.h.b32, ses->ses_hmouter, sizeof(sha1ctx.h.b32));
 	}
 
 	for (i = 0; i < klen; i++)
 		key[i] ^= HMAC_OPAD_VAL;
 }
 
 /*
  * Allocate a new 'session' and return an encoded session id.  'sidp'
  * contains our registration id, and should contain an encoded session
  * id on successful allocation.
  */
 static int
 ubsec_newsession(device_t dev, u_int32_t *sidp, struct cryptoini *cri)
 {
 	struct ubsec_softc *sc = device_get_softc(dev);
 	struct cryptoini *c, *encini = NULL, *macini = NULL;
 	struct ubsec_session *ses = NULL;
 	int sesn;
 
 	if (sidp == NULL || cri == NULL || sc == NULL)
 		return (EINVAL);
 
 	for (c = cri; c != NULL; c = c->cri_next) {
 		if (c->cri_alg == CRYPTO_MD5_HMAC ||
 		    c->cri_alg == CRYPTO_SHA1_HMAC) {
 			if (macini)
 				return (EINVAL);
 			macini = c;
 		} else if (c->cri_alg == CRYPTO_DES_CBC ||
 		    c->cri_alg == CRYPTO_3DES_CBC) {
 			if (encini)
 				return (EINVAL);
 			encini = c;
 		} else
 			return (EINVAL);
 	}
 	if (encini == NULL && macini == NULL)
 		return (EINVAL);
 
 	if (sc->sc_sessions == NULL) {
 		ses = sc->sc_sessions = (struct ubsec_session *)malloc(
 		    sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT);
 		if (ses == NULL)
 			return (ENOMEM);
 		sesn = 0;
 		sc->sc_nsessions = 1;
 	} else {
 		for (sesn = 0; sesn < sc->sc_nsessions; sesn++) {
 			if (sc->sc_sessions[sesn].ses_used == 0) {
 				ses = &sc->sc_sessions[sesn];
 				break;
 			}
 		}
 
 		if (ses == NULL) {
 			sesn = sc->sc_nsessions;
 			ses = (struct ubsec_session *)malloc((sesn + 1) *
 			    sizeof(struct ubsec_session), M_DEVBUF, M_NOWAIT);
 			if (ses == NULL)
 				return (ENOMEM);
 			bcopy(sc->sc_sessions, ses, sesn *
 			    sizeof(struct ubsec_session));
 			bzero(sc->sc_sessions, sesn *
 			    sizeof(struct ubsec_session));
 			free(sc->sc_sessions, M_DEVBUF);
 			sc->sc_sessions = ses;
 			ses = &sc->sc_sessions[sesn];
 			sc->sc_nsessions++;
 		}
 	}
 	bzero(ses, sizeof(struct ubsec_session));
 	ses->ses_used = 1;
 
 	if (encini) {
 		/* get an IV, network byte order */
 		/* XXX may read fewer than requested */
 		read_random(ses->ses_iv, sizeof(ses->ses_iv));
 
 		if (encini->cri_key != NULL) {
 			ubsec_setup_enckey(ses, encini->cri_alg,
 			    encini->cri_key);
 		}
 	}
 
 	if (macini) {
 		ses->ses_mlen = macini->cri_mlen;
 		if (ses->ses_mlen == 0) {
 			if (macini->cri_alg == CRYPTO_MD5_HMAC)
 				ses->ses_mlen = MD5_HASH_LEN;
 			else
 				ses->ses_mlen = SHA1_HASH_LEN;
 		}
 
 		if (macini->cri_key != NULL) {
 			ubsec_setup_mackey(ses, macini->cri_alg,
 			    macini->cri_key, macini->cri_klen / 8);
 		}
 	}
 
 	*sidp = UBSEC_SID(device_get_unit(sc->sc_dev), sesn);
 	return (0);
 }
 
 /*
  * Deallocate a session.
  */
 static int
 ubsec_freesession(device_t dev, u_int64_t tid)
 {
 	struct ubsec_softc *sc = device_get_softc(dev);
 	int session, ret;
 	u_int32_t sid = CRYPTO_SESID2LID(tid);
 
 	if (sc == NULL)
 		return (EINVAL);
 
 	session = UBSEC_SESSION(sid);
 	if (session < sc->sc_nsessions) {
 		bzero(&sc->sc_sessions[session],
 			sizeof(sc->sc_sessions[session]));
 		ret = 0;
 	} else
 		ret = EINVAL;
 
 	return (ret);
 }
 
 static void
 ubsec_op_cb(void *arg, bus_dma_segment_t *seg, int nsegs, bus_size_t mapsize, int error)
 {
 	struct ubsec_operand *op = arg;
 
 	KASSERT(nsegs <= UBS_MAX_SCATTER,
 		("Too many DMA segments returned when mapping operand"));
 #ifdef UBSEC_DEBUG
 	if (ubsec_debug)
 		printf("ubsec_op_cb: mapsize %u nsegs %d error %d\n",
 			(u_int) mapsize, nsegs, error);
 #endif
 	if (error != 0)
 		return;
 	op->mapsize = mapsize;
 	op->nsegs = nsegs;
 	bcopy(seg, op->segs, nsegs * sizeof (seg[0]));
 }
 
 static int
 ubsec_process(device_t dev, struct cryptop *crp, int hint)
 {
 	struct ubsec_softc *sc = device_get_softc(dev);
 	struct ubsec_q *q = NULL;
 	int err = 0, i, j, nicealign;
 	struct cryptodesc *crd1, *crd2, *maccrd, *enccrd;
 	int encoffset = 0, macoffset = 0, cpskip, cpoffset;
 	int sskip, dskip, stheend, dtheend;
 	int16_t coffset;
 	struct ubsec_session *ses;
 	struct ubsec_pktctx ctx;
 	struct ubsec_dma *dmap = NULL;
 
 	if (crp == NULL || crp->crp_callback == NULL || sc == NULL) {
 		ubsecstats.hst_invalid++;
 		return (EINVAL);
 	}
 	if (UBSEC_SESSION(crp->crp_sid) >= sc->sc_nsessions) {
 		ubsecstats.hst_badsession++;
 		return (EINVAL);
 	}
 
 	mtx_lock(&sc->sc_freeqlock);
 	if (SIMPLEQ_EMPTY(&sc->sc_freequeue)) {
 		ubsecstats.hst_queuefull++;
 		sc->sc_needwakeup |= CRYPTO_SYMQ;
 		mtx_unlock(&sc->sc_freeqlock);
 		return (ERESTART);
 	}
 	q = SIMPLEQ_FIRST(&sc->sc_freequeue);
 	SIMPLEQ_REMOVE_HEAD(&sc->sc_freequeue, q_next);
 	mtx_unlock(&sc->sc_freeqlock);
 
 	dmap = q->q_dma; /* Save dma pointer */
 	bzero(q, sizeof(struct ubsec_q));
 	bzero(&ctx, sizeof(ctx));
 
 	q->q_sesn = UBSEC_SESSION(crp->crp_sid);
 	q->q_dma = dmap;
 	ses = &sc->sc_sessions[q->q_sesn];
 
 	if (crp->crp_flags & CRYPTO_F_IMBUF) {
 		q->q_src_m = (struct mbuf *)crp->crp_buf;
 		q->q_dst_m = (struct mbuf *)crp->crp_buf;
 	} else if (crp->crp_flags & CRYPTO_F_IOV) {
 		q->q_src_io = (struct uio *)crp->crp_buf;
 		q->q_dst_io = (struct uio *)crp->crp_buf;
 	} else {
 		ubsecstats.hst_badflags++;
 		err = EINVAL;
 		goto errout;	/* XXX we don't handle contiguous blocks! */
 	}
 
 	bzero(&dmap->d_dma->d_mcr, sizeof(struct ubsec_mcr));
 
 	dmap->d_dma->d_mcr.mcr_pkts = htole16(1);
 	dmap->d_dma->d_mcr.mcr_flags = 0;
 	q->q_crp = crp;
 
 	crd1 = crp->crp_desc;
 	if (crd1 == NULL) {
 		ubsecstats.hst_nodesc++;
 		err = EINVAL;
 		goto errout;
 	}
 	crd2 = crd1->crd_next;
 
 	if (crd2 == NULL) {
 		if (crd1->crd_alg == CRYPTO_MD5_HMAC ||
 		    crd1->crd_alg == CRYPTO_SHA1_HMAC) {
 			maccrd = crd1;
 			enccrd = NULL;
 		} else if (crd1->crd_alg == CRYPTO_DES_CBC ||
 		    crd1->crd_alg == CRYPTO_3DES_CBC) {
 			maccrd = NULL;
 			enccrd = crd1;
 		} else {
 			ubsecstats.hst_badalg++;
 			err = EINVAL;
 			goto errout;
 		}
 	} else {
 		if ((crd1->crd_alg == CRYPTO_MD5_HMAC ||
 		    crd1->crd_alg == CRYPTO_SHA1_HMAC) &&
 		    (crd2->crd_alg == CRYPTO_DES_CBC ||
 			crd2->crd_alg == CRYPTO_3DES_CBC) &&
 		    ((crd2->crd_flags & CRD_F_ENCRYPT) == 0)) {
 			maccrd = crd1;
 			enccrd = crd2;
 		} else if ((crd1->crd_alg == CRYPTO_DES_CBC ||
 		    crd1->crd_alg == CRYPTO_3DES_CBC) &&
 		    (crd2->crd_alg == CRYPTO_MD5_HMAC ||
 			crd2->crd_alg == CRYPTO_SHA1_HMAC) &&
 		    (crd1->crd_flags & CRD_F_ENCRYPT)) {
 			enccrd = crd1;
 			maccrd = crd2;
 		} else {
 			/*
 			 * We cannot order the ubsec as requested
 			 */
 			ubsecstats.hst_badalg++;
 			err = EINVAL;
 			goto errout;
 		}
 	}
 
 	if (enccrd) {
 		if (enccrd->crd_flags & CRD_F_KEY_EXPLICIT) {
 			ubsec_setup_enckey(ses, enccrd->crd_alg,
 			    enccrd->crd_key);
 		}
 
 		encoffset = enccrd->crd_skip;
 		ctx.pc_flags |= htole16(UBS_PKTCTX_ENC_3DES);
 
 		if (enccrd->crd_flags & CRD_F_ENCRYPT) {
 			q->q_flags |= UBSEC_QFLAGS_COPYOUTIV;
 
 			if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
 				bcopy(enccrd->crd_iv, ctx.pc_iv, 8);
 			else {
 				ctx.pc_iv[0] = ses->ses_iv[0];
 				ctx.pc_iv[1] = ses->ses_iv[1];
 			}
 
 			if ((enccrd->crd_flags & CRD_F_IV_PRESENT) == 0) {
 				crypto_copyback(crp->crp_flags, crp->crp_buf,
 				    enccrd->crd_inject, 8, (caddr_t)ctx.pc_iv);
 			}
 		} else {
 			ctx.pc_flags |= htole16(UBS_PKTCTX_INBOUND);
 
 			if (enccrd->crd_flags & CRD_F_IV_EXPLICIT)
 				bcopy(enccrd->crd_iv, ctx.pc_iv, 8);
 			else {
 				crypto_copydata(crp->crp_flags, crp->crp_buf,
 				    enccrd->crd_inject, 8, (caddr_t)ctx.pc_iv);
 			}
 		}
 
 		ctx.pc_deskey[0] = ses->ses_deskey[0];
 		ctx.pc_deskey[1] = ses->ses_deskey[1];
 		ctx.pc_deskey[2] = ses->ses_deskey[2];
 		ctx.pc_deskey[3] = ses->ses_deskey[3];
 		ctx.pc_deskey[4] = ses->ses_deskey[4];
 		ctx.pc_deskey[5] = ses->ses_deskey[5];
 		SWAP32(ctx.pc_iv[0]);
 		SWAP32(ctx.pc_iv[1]);
 	}
 
 	if (maccrd) {
 		if (maccrd->crd_flags & CRD_F_KEY_EXPLICIT) {
 			ubsec_setup_mackey(ses, maccrd->crd_alg,
 			    maccrd->crd_key, maccrd->crd_klen / 8);
 		}
 
 		macoffset = maccrd->crd_skip;
 
 		if (maccrd->crd_alg == CRYPTO_MD5_HMAC)
 			ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_MD5);
 		else
 			ctx.pc_flags |= htole16(UBS_PKTCTX_AUTH_SHA1);
 
 		for (i = 0; i < 5; i++) {
 			ctx.pc_hminner[i] = ses->ses_hminner[i];
 			ctx.pc_hmouter[i] = ses->ses_hmouter[i];
 
 			HTOLE32(ctx.pc_hminner[i]);
 			HTOLE32(ctx.pc_hmouter[i]);
 		}
 	}
 
 	if (enccrd && maccrd) {
 		/*
 		 * ubsec cannot handle packets where the end of encryption
 		 * and authentication are not the same, or where the
 		 * encrypted part begins before the authenticated part.
 		 */
 		if ((encoffset + enccrd->crd_len) !=
 		    (macoffset + maccrd->crd_len)) {
 			ubsecstats.hst_lenmismatch++;
 			err = EINVAL;
 			goto errout;
 		}
 		if (enccrd->crd_skip < maccrd->crd_skip) {
 			ubsecstats.hst_skipmismatch++;
 			err = EINVAL;
 			goto errout;
 		}
 		sskip = maccrd->crd_skip;
 		cpskip = dskip = enccrd->crd_skip;
 		stheend = maccrd->crd_len;
 		dtheend = enccrd->crd_len;
 		coffset = enccrd->crd_skip - maccrd->crd_skip;
 		cpoffset = cpskip + dtheend;
 #ifdef UBSEC_DEBUG
 		if (ubsec_debug) {
 			printf("mac: skip %d, len %d, inject %d\n",
 			    maccrd->crd_skip, maccrd->crd_len, maccrd->crd_inject);
 			printf("enc: skip %d, len %d, inject %d\n",
 			    enccrd->crd_skip, enccrd->crd_len, enccrd->crd_inject);
 			printf("src: skip %d, len %d\n", sskip, stheend);
 			printf("dst: skip %d, len %d\n", dskip, dtheend);
 			printf("ubs: coffset %d, pktlen %d, cpskip %d, cpoffset %d\n",
 			    coffset, stheend, cpskip, cpoffset);
 		}
 #endif
 	} else {
 		cpskip = dskip = sskip = macoffset + encoffset;
 		dtheend = stheend = (enccrd)?enccrd->crd_len:maccrd->crd_len;
 		cpoffset = cpskip + dtheend;
 		coffset = 0;
 	}
 	ctx.pc_offset = htole16(coffset >> 2);
 
 	if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT, &q->q_src_map)) {
 		ubsecstats.hst_nomap++;
 		err = ENOMEM;
 		goto errout;
 	}
 	if (crp->crp_flags & CRYPTO_F_IMBUF) {
 		if (bus_dmamap_load_mbuf(sc->sc_dmat, q->q_src_map,
 		    q->q_src_m, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) {
 			bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
 			q->q_src_map = NULL;
 			ubsecstats.hst_noload++;
 			err = ENOMEM;
 			goto errout;
 		}
 	} else if (crp->crp_flags & CRYPTO_F_IOV) {
 		if (bus_dmamap_load_uio(sc->sc_dmat, q->q_src_map,
 		    q->q_src_io, ubsec_op_cb, &q->q_src, BUS_DMA_NOWAIT) != 0) {
 			bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
 			q->q_src_map = NULL;
 			ubsecstats.hst_noload++;
 			err = ENOMEM;
 			goto errout;
 		}
 	}
 	nicealign = ubsec_dmamap_aligned(&q->q_src);
 
 	dmap->d_dma->d_mcr.mcr_pktlen = htole16(stheend);
 
 #ifdef UBSEC_DEBUG
 	if (ubsec_debug)
 		printf("src skip: %d nicealign: %u\n", sskip, nicealign);
 #endif
 	for (i = j = 0; i < q->q_src_nsegs; i++) {
 		struct ubsec_pktbuf *pb;
 		bus_size_t packl = q->q_src_segs[i].ds_len;
 		bus_addr_t packp = q->q_src_segs[i].ds_addr;
 
 		if (sskip >= packl) {
 			sskip -= packl;
 			continue;
 		}
 
 		packl -= sskip;
 		packp += sskip;
 		sskip = 0;
 
 		if (packl > 0xfffc) {
 			err = EIO;
 			goto errout;
 		}
 
 		if (j == 0)
 			pb = &dmap->d_dma->d_mcr.mcr_ipktbuf;
 		else
 			pb = &dmap->d_dma->d_sbuf[j - 1];
 
 		pb->pb_addr = htole32(packp);
 
 		if (stheend) {
 			if (packl > stheend) {
 				pb->pb_len = htole32(stheend);
 				stheend = 0;
 			} else {
 				pb->pb_len = htole32(packl);
 				stheend -= packl;
 			}
 		} else
 			pb->pb_len = htole32(packl);
 
 		if ((i + 1) == q->q_src_nsegs)
 			pb->pb_next = 0;
 		else
 			pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
 			    offsetof(struct ubsec_dmachunk, d_sbuf[j]));
 		j++;
 	}
 
 	if (enccrd == NULL && maccrd != NULL) {
 		dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr = 0;
 		dmap->d_dma->d_mcr.mcr_opktbuf.pb_len = 0;
 		dmap->d_dma->d_mcr.mcr_opktbuf.pb_next = htole32(dmap->d_alloc.dma_paddr +
 		    offsetof(struct ubsec_dmachunk, d_macbuf[0]));
 #ifdef UBSEC_DEBUG
 		if (ubsec_debug)
 			printf("opkt: %x %x %x\n",
 			    dmap->d_dma->d_mcr.mcr_opktbuf.pb_addr,
 			    dmap->d_dma->d_mcr.mcr_opktbuf.pb_len,
 			    dmap->d_dma->d_mcr.mcr_opktbuf.pb_next);
 #endif
 	} else {
 		if (crp->crp_flags & CRYPTO_F_IOV) {
 			if (!nicealign) {
 				ubsecstats.hst_iovmisaligned++;
 				err = EINVAL;
 				goto errout;
 			}
 			if (bus_dmamap_create(sc->sc_dmat, BUS_DMA_NOWAIT,
 			     &q->q_dst_map)) {
 				ubsecstats.hst_nomap++;
 				err = ENOMEM;
 				goto errout;
 			}
 			if (bus_dmamap_load_uio(sc->sc_dmat, q->q_dst_map,
 			    q->q_dst_io, ubsec_op_cb, &q->q_dst, BUS_DMA_NOWAIT) != 0) {
 				bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
 				q->q_dst_map = NULL;
 				ubsecstats.hst_noload++;
 				err = ENOMEM;
 				goto errout;
 			}
 		} else if (crp->crp_flags & CRYPTO_F_IMBUF) {
 			if (nicealign) {
 				q->q_dst = q->q_src;
 			} else {
 				int totlen, len;
 				struct mbuf *m, *top, **mp;
 
 				ubsecstats.hst_unaligned++;
 				totlen = q->q_src_mapsize;
 				if (totlen >= MINCLSIZE) {
 					m = m_getcl(M_NOWAIT, MT_DATA,
 					    q->q_src_m->m_flags & M_PKTHDR);
 					len = MCLBYTES;
 				} else if (q->q_src_m->m_flags & M_PKTHDR) {
 					m = m_gethdr(M_NOWAIT, MT_DATA);
 					len = MHLEN;
 				} else {
 					m = m_get(M_NOWAIT, MT_DATA);
 					len = MLEN;
 				}
 				if (m && q->q_src_m->m_flags & M_PKTHDR &&
 				    !m_dup_pkthdr(m, q->q_src_m, M_NOWAIT)) {
 					m_free(m);
 					m = NULL;
 				}
 				if (m == NULL) {
 					ubsecstats.hst_nombuf++;
 					err = sc->sc_nqueue ? ERESTART : ENOMEM;
 					goto errout;
 				}
 				m->m_len = len = min(totlen, len);
 				totlen -= len;
 				top = m;
 				mp = &top;
 
 				while (totlen > 0) {
 					if (totlen >= MINCLSIZE) {
 						m = m_getcl(M_NOWAIT,
 						    MT_DATA, 0);
 						len = MCLBYTES;
 					} else {
 						m = m_get(M_NOWAIT, MT_DATA);
 						len = MLEN;
 					}
 					if (m == NULL) {
 						m_freem(top);
 						ubsecstats.hst_nombuf++;
 						err = sc->sc_nqueue ? ERESTART : ENOMEM;
 						goto errout;
 					}
 					m->m_len = len = min(totlen, len);
 					totlen -= len;
 					*mp = m;
 					mp = &m->m_next;
 				}
 				q->q_dst_m = top;
 				ubsec_mcopy(q->q_src_m, q->q_dst_m,
 				    cpskip, cpoffset);
 				if (bus_dmamap_create(sc->sc_dmat,
 				    BUS_DMA_NOWAIT, &q->q_dst_map) != 0) {
 					ubsecstats.hst_nomap++;
 					err = ENOMEM;
 					goto errout;
 				}
 				if (bus_dmamap_load_mbuf(sc->sc_dmat,
 				    q->q_dst_map, q->q_dst_m,
 				    ubsec_op_cb, &q->q_dst,
 				    BUS_DMA_NOWAIT) != 0) {
 					bus_dmamap_destroy(sc->sc_dmat,
 					q->q_dst_map);
 					q->q_dst_map = NULL;
 					ubsecstats.hst_noload++;
 					err = ENOMEM;
 					goto errout;
 				}
 			}
 		} else {
 			ubsecstats.hst_badflags++;
 			err = EINVAL;
 			goto errout;
 		}
 
 #ifdef UBSEC_DEBUG
 		if (ubsec_debug)
 			printf("dst skip: %d\n", dskip);
 #endif
 		for (i = j = 0; i < q->q_dst_nsegs; i++) {
 			struct ubsec_pktbuf *pb;
 			bus_size_t packl = q->q_dst_segs[i].ds_len;
 			bus_addr_t packp = q->q_dst_segs[i].ds_addr;
 
 			if (dskip >= packl) {
 				dskip -= packl;
 				continue;
 			}
 
 			packl -= dskip;
 			packp += dskip;
 			dskip = 0;
 
 			if (packl > 0xfffc) {
 				err = EIO;
 				goto errout;
 			}
 
 			if (j == 0)
 				pb = &dmap->d_dma->d_mcr.mcr_opktbuf;
 			else
 				pb = &dmap->d_dma->d_dbuf[j - 1];
 
 			pb->pb_addr = htole32(packp);
 
 			if (dtheend) {
 				if (packl > dtheend) {
 					pb->pb_len = htole32(dtheend);
 					dtheend = 0;
 				} else {
 					pb->pb_len = htole32(packl);
 					dtheend -= packl;
 				}
 			} else
 				pb->pb_len = htole32(packl);
 
 			if ((i + 1) == q->q_dst_nsegs) {
 				if (maccrd)
 					pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
 					    offsetof(struct ubsec_dmachunk, d_macbuf[0]));
 				else
 					pb->pb_next = 0;
 			} else
 				pb->pb_next = htole32(dmap->d_alloc.dma_paddr +
 				    offsetof(struct ubsec_dmachunk, d_dbuf[j]));
 			j++;
 		}
 	}
 
 	dmap->d_dma->d_mcr.mcr_cmdctxp = htole32(dmap->d_alloc.dma_paddr +
 	    offsetof(struct ubsec_dmachunk, d_ctx));
 
 	if (sc->sc_flags & UBS_FLAGS_LONGCTX) {
 		struct ubsec_pktctx_long *ctxl;
 
 		ctxl = (struct ubsec_pktctx_long *)(dmap->d_alloc.dma_vaddr +
 		    offsetof(struct ubsec_dmachunk, d_ctx));
 
 		/* transform small context into long context */
 		ctxl->pc_len = htole16(sizeof(struct ubsec_pktctx_long));
 		ctxl->pc_type = htole16(UBS_PKTCTX_TYPE_IPSEC);
 		ctxl->pc_flags = ctx.pc_flags;
 		ctxl->pc_offset = ctx.pc_offset;
 		for (i = 0; i < 6; i++)
 			ctxl->pc_deskey[i] = ctx.pc_deskey[i];
 		for (i = 0; i < 5; i++)
 			ctxl->pc_hminner[i] = ctx.pc_hminner[i];
 		for (i = 0; i < 5; i++)
 			ctxl->pc_hmouter[i] = ctx.pc_hmouter[i];
 		ctxl->pc_iv[0] = ctx.pc_iv[0];
 		ctxl->pc_iv[1] = ctx.pc_iv[1];
 	} else
 		bcopy(&ctx, dmap->d_alloc.dma_vaddr +
 		    offsetof(struct ubsec_dmachunk, d_ctx),
 		    sizeof(struct ubsec_pktctx));
 
 	mtx_lock(&sc->sc_mcr1lock);
 	SIMPLEQ_INSERT_TAIL(&sc->sc_queue, q, q_next);
 	sc->sc_nqueue++;
 	ubsecstats.hst_ipackets++;
 	ubsecstats.hst_ibytes += dmap->d_alloc.dma_size;
 	if ((hint & CRYPTO_HINT_MORE) == 0 || sc->sc_nqueue >= UBS_MAX_AGGR)
 		ubsec_feed(sc);
 	mtx_unlock(&sc->sc_mcr1lock);
 	return (0);
 
 errout:
 	if (q != NULL) {
 		if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m))
 			m_freem(q->q_dst_m);
 
 		if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) {
 			bus_dmamap_unload(sc->sc_dmat, q->q_dst_map);
 			bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
 		}
 		if (q->q_src_map != NULL) {
 			bus_dmamap_unload(sc->sc_dmat, q->q_src_map);
 			bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
 		}
 	}
 	if (q != NULL || err == ERESTART) {
 		mtx_lock(&sc->sc_freeqlock);
 		if (q != NULL)
 			SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
 		if (err == ERESTART)
 			sc->sc_needwakeup |= CRYPTO_SYMQ;
 		mtx_unlock(&sc->sc_freeqlock);
 	}
 	if (err != ERESTART) {
 		crp->crp_etype = err;
 		crypto_done(crp);
 	}
 	return (err);
 }
 
 static void
 ubsec_callback(struct ubsec_softc *sc, struct ubsec_q *q)
 {
 	struct cryptop *crp = (struct cryptop *)q->q_crp;
 	struct cryptodesc *crd;
 	struct ubsec_dma *dmap = q->q_dma;
 
 	ubsecstats.hst_opackets++;
 	ubsecstats.hst_obytes += dmap->d_alloc.dma_size;
 
 	ubsec_dma_sync(&dmap->d_alloc,
 	    BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
 	if (q->q_dst_map != NULL && q->q_dst_map != q->q_src_map) {
 		bus_dmamap_sync(sc->sc_dmat, q->q_dst_map,
 		    BUS_DMASYNC_POSTREAD);
 		bus_dmamap_unload(sc->sc_dmat, q->q_dst_map);
 		bus_dmamap_destroy(sc->sc_dmat, q->q_dst_map);
 	}
 	bus_dmamap_sync(sc->sc_dmat, q->q_src_map, BUS_DMASYNC_POSTWRITE);
 	bus_dmamap_unload(sc->sc_dmat, q->q_src_map);
 	bus_dmamap_destroy(sc->sc_dmat, q->q_src_map);
 
 	if ((crp->crp_flags & CRYPTO_F_IMBUF) && (q->q_src_m != q->q_dst_m)) {
 		m_freem(q->q_src_m);
 		crp->crp_buf = (caddr_t)q->q_dst_m;
 	}
 
 	/* copy out IV for future use */
 	if (q->q_flags & UBSEC_QFLAGS_COPYOUTIV) {
 		for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
 			if (crd->crd_alg != CRYPTO_DES_CBC &&
 			    crd->crd_alg != CRYPTO_3DES_CBC)
 				continue;
 			crypto_copydata(crp->crp_flags, crp->crp_buf,
 			    crd->crd_skip + crd->crd_len - 8, 8,
 			    (caddr_t)sc->sc_sessions[q->q_sesn].ses_iv);
 			break;
 		}
 	}
 
 	for (crd = crp->crp_desc; crd; crd = crd->crd_next) {
 		if (crd->crd_alg != CRYPTO_MD5_HMAC &&
 		    crd->crd_alg != CRYPTO_SHA1_HMAC)
 			continue;
 		crypto_copyback(crp->crp_flags, crp->crp_buf, crd->crd_inject,
 		    sc->sc_sessions[q->q_sesn].ses_mlen,
 		    (caddr_t)dmap->d_dma->d_macbuf);
 		break;
 	}
 	mtx_lock(&sc->sc_freeqlock);
 	SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
 	mtx_unlock(&sc->sc_freeqlock);
 	crypto_done(crp);
 }
 
 static void
 ubsec_mcopy(struct mbuf *srcm, struct mbuf *dstm, int hoffset, int toffset)
 {
 	int i, j, dlen, slen;
 	caddr_t dptr, sptr;
 
 	j = 0;
 	sptr = srcm->m_data;
 	slen = srcm->m_len;
 	dptr = dstm->m_data;
 	dlen = dstm->m_len;
 
 	while (1) {
 		for (i = 0; i < min(slen, dlen); i++) {
 			if (j < hoffset || j >= toffset)
 				*dptr++ = *sptr++;
 			slen--;
 			dlen--;
 			j++;
 		}
 		if (slen == 0) {
 			srcm = srcm->m_next;
 			if (srcm == NULL)
 				return;
 			sptr = srcm->m_data;
 			slen = srcm->m_len;
 		}
 		if (dlen == 0) {
 			dstm = dstm->m_next;
 			if (dstm == NULL)
 				return;
 			dptr = dstm->m_data;
 			dlen = dstm->m_len;
 		}
 	}
 }
 
 /*
  * feed the key generator, must be called at splimp() or higher.
  */
 static int
 ubsec_feed2(struct ubsec_softc *sc)
 {
 	struct ubsec_q2 *q;
 
 	while (!SIMPLEQ_EMPTY(&sc->sc_queue2)) {
 		if (READ_REG(sc, BS_STAT) & BS_STAT_MCR2_FULL)
 			break;
 		q = SIMPLEQ_FIRST(&sc->sc_queue2);
 
 		ubsec_dma_sync(&q->q_mcr,
 		    BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
 		ubsec_dma_sync(&q->q_ctx, BUS_DMASYNC_PREWRITE);
 
 		WRITE_REG(sc, BS_MCR2, q->q_mcr.dma_paddr);
 		SIMPLEQ_REMOVE_HEAD(&sc->sc_queue2, q_next);
 		--sc->sc_nqueue2;
 		SIMPLEQ_INSERT_TAIL(&sc->sc_qchip2, q, q_next);
 	}
 	return (0);
 }
 
 /*
  * Callback for handling random numbers
  */
 static void
 ubsec_callback2(struct ubsec_softc *sc, struct ubsec_q2 *q)
 {
 	struct cryptkop *krp;
 	struct ubsec_ctx_keyop *ctx;
 
 	ctx = (struct ubsec_ctx_keyop *)q->q_ctx.dma_vaddr;
 	ubsec_dma_sync(&q->q_ctx, BUS_DMASYNC_POSTWRITE);
 
 	switch (q->q_type) {
 #ifndef UBSEC_NO_RNG
 	case UBS_CTXOP_RNGBYPASS: {
 		struct ubsec_q2_rng *rng = (struct ubsec_q2_rng *)q;
 
 		ubsec_dma_sync(&rng->rng_buf, BUS_DMASYNC_POSTREAD);
 		(*sc->sc_harvest)(sc->sc_rndtest,
 			rng->rng_buf.dma_vaddr,
 			UBSEC_RNG_BUFSIZ*sizeof (u_int32_t));
 		rng->rng_used = 0;
 		callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc);
 		break;
 	}
 #endif
 	case UBS_CTXOP_MODEXP: {
 		struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q;
 		u_int rlen, clen;
 
 		krp = me->me_krp;
 		rlen = (me->me_modbits + 7) / 8;
 		clen = (krp->krp_param[krp->krp_iparams].crp_nbits + 7) / 8;
 
 		ubsec_dma_sync(&me->me_M, BUS_DMASYNC_POSTWRITE);
 		ubsec_dma_sync(&me->me_E, BUS_DMASYNC_POSTWRITE);
 		ubsec_dma_sync(&me->me_C, BUS_DMASYNC_POSTREAD);
 		ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_POSTWRITE);
 
 		if (clen < rlen)
 			krp->krp_status = E2BIG;
 		else {
 			if (sc->sc_flags & UBS_FLAGS_HWNORM) {
 				bzero(krp->krp_param[krp->krp_iparams].crp_p,
 				    (krp->krp_param[krp->krp_iparams].crp_nbits
 					+ 7) / 8);
 				bcopy(me->me_C.dma_vaddr,
 				    krp->krp_param[krp->krp_iparams].crp_p,
 				    (me->me_modbits + 7) / 8);
 			} else
 				ubsec_kshift_l(me->me_shiftbits,
 				    me->me_C.dma_vaddr, me->me_normbits,
 				    krp->krp_param[krp->krp_iparams].crp_p,
 				    krp->krp_param[krp->krp_iparams].crp_nbits);
 		}
 
 		crypto_kdone(krp);
 
 		/* bzero all potentially sensitive data */
 		bzero(me->me_E.dma_vaddr, me->me_E.dma_size);
 		bzero(me->me_M.dma_vaddr, me->me_M.dma_size);
 		bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
 		bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size);
 
 		/* Can't free here, so put us on the free list. */
 		SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &me->me_q, q_next);
 		break;
 	}
 	case UBS_CTXOP_RSAPRIV: {
 		struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q;
 		u_int len;
 
 		krp = rp->rpr_krp;
 		ubsec_dma_sync(&rp->rpr_msgin, BUS_DMASYNC_POSTWRITE);
 		ubsec_dma_sync(&rp->rpr_msgout, BUS_DMASYNC_POSTREAD);
 
 		len = (krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_nbits + 7) / 8;
 		bcopy(rp->rpr_msgout.dma_vaddr,
 		    krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT].crp_p, len);
 
 		crypto_kdone(krp);
 
 		bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size);
 		bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size);
 		bzero(rp->rpr_q.q_ctx.dma_vaddr, rp->rpr_q.q_ctx.dma_size);
 
 		/* Can't free here, so put us on the free list. */
 		SIMPLEQ_INSERT_TAIL(&sc->sc_q2free, &rp->rpr_q, q_next);
 		break;
 	}
 	default:
 		device_printf(sc->sc_dev, "unknown ctx op: %x\n",
 		    letoh16(ctx->ctx_op));
 		break;
 	}
 }
 
 #ifndef UBSEC_NO_RNG
 static void
 ubsec_rng(void *vsc)
 {
 	struct ubsec_softc *sc = vsc;
 	struct ubsec_q2_rng *rng = &sc->sc_rng;
 	struct ubsec_mcr *mcr;
 	struct ubsec_ctx_rngbypass *ctx;
 
 	mtx_lock(&sc->sc_mcr2lock);
 	if (rng->rng_used) {
 		mtx_unlock(&sc->sc_mcr2lock);
 		return;
 	}
 	sc->sc_nqueue2++;
 	if (sc->sc_nqueue2 >= UBS_MAX_NQUEUE)
 		goto out;
 
 	mcr = (struct ubsec_mcr *)rng->rng_q.q_mcr.dma_vaddr;
 	ctx = (struct ubsec_ctx_rngbypass *)rng->rng_q.q_ctx.dma_vaddr;
 
 	mcr->mcr_pkts = htole16(1);
 	mcr->mcr_flags = 0;
 	mcr->mcr_cmdctxp = htole32(rng->rng_q.q_ctx.dma_paddr);
 	mcr->mcr_ipktbuf.pb_addr = mcr->mcr_ipktbuf.pb_next = 0;
 	mcr->mcr_ipktbuf.pb_len = 0;
 	mcr->mcr_reserved = mcr->mcr_pktlen = 0;
 	mcr->mcr_opktbuf.pb_addr = htole32(rng->rng_buf.dma_paddr);
 	mcr->mcr_opktbuf.pb_len = htole32(((sizeof(u_int32_t) * UBSEC_RNG_BUFSIZ)) &
 	    UBS_PKTBUF_LEN);
 	mcr->mcr_opktbuf.pb_next = 0;
 
 	ctx->rbp_len = htole16(sizeof(struct ubsec_ctx_rngbypass));
 	ctx->rbp_op = htole16(UBS_CTXOP_RNGBYPASS);
 	rng->rng_q.q_type = UBS_CTXOP_RNGBYPASS;
 
 	ubsec_dma_sync(&rng->rng_buf, BUS_DMASYNC_PREREAD);
 
 	SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rng->rng_q, q_next);
 	rng->rng_used = 1;
 	ubsec_feed2(sc);
 	ubsecstats.hst_rng++;
 	mtx_unlock(&sc->sc_mcr2lock);
 
 	return;
 
 out:
 	/*
 	 * Something weird happened, generate our own call back.
 	 */
 	sc->sc_nqueue2--;
 	mtx_unlock(&sc->sc_mcr2lock);
 	callout_reset(&sc->sc_rngto, sc->sc_rnghz, ubsec_rng, sc);
 }
 #endif /* UBSEC_NO_RNG */
 
 static void
 ubsec_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
 {
 	bus_addr_t *paddr = (bus_addr_t*) arg;
 	*paddr = segs->ds_addr;
 }
 
 static int
 ubsec_dma_malloc(
 	struct ubsec_softc *sc,
 	bus_size_t size,
 	struct ubsec_dma_alloc *dma,
 	int mapflags
 )
 {
 	int r;
 
 	/* XXX could specify sc_dmat as parent but that just adds overhead */
 	r = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev),	/* parent */
 			       1, 0,			/* alignment, bounds */
 			       BUS_SPACE_MAXADDR_32BIT,	/* lowaddr */
 			       BUS_SPACE_MAXADDR,	/* highaddr */
 			       NULL, NULL,		/* filter, filterarg */
 			       size,			/* maxsize */
 			       1,			/* nsegments */
 			       size,			/* maxsegsize */
 			       BUS_DMA_ALLOCNOW,	/* flags */
 			       NULL, NULL,		/* lockfunc, lockarg */
 			       &dma->dma_tag);
 	if (r != 0) {
 		device_printf(sc->sc_dev, "ubsec_dma_malloc: "
 			"bus_dma_tag_create failed; error %u\n", r);
-		goto fail_0;
-	}
-
-	r = bus_dmamap_create(dma->dma_tag, BUS_DMA_NOWAIT, &dma->dma_map);
-	if (r != 0) {
-		device_printf(sc->sc_dev, "ubsec_dma_malloc: "
-			"bus_dmamap_create failed; error %u\n", r);
 		goto fail_1;
 	}
 
 	r = bus_dmamem_alloc(dma->dma_tag, (void**) &dma->dma_vaddr,
 			     BUS_DMA_NOWAIT, &dma->dma_map);
 	if (r != 0) {
 		device_printf(sc->sc_dev, "ubsec_dma_malloc: "
 			"bus_dmammem_alloc failed; size %ju, error %u\n",
 			(intmax_t)size, r);
 		goto fail_2;
 	}
 
 	r = bus_dmamap_load(dma->dma_tag, dma->dma_map, dma->dma_vaddr,
 		            size,
 			    ubsec_dmamap_cb,
 			    &dma->dma_paddr,
 			    mapflags | BUS_DMA_NOWAIT);
 	if (r != 0) {
 		device_printf(sc->sc_dev, "ubsec_dma_malloc: "
 			"bus_dmamap_load failed; error %u\n", r);
 		goto fail_3;
 	}
 
 	dma->dma_size = size;
 	return (0);
 
 fail_3:
 	bus_dmamap_unload(dma->dma_tag, dma->dma_map);
 fail_2:
 	bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
 fail_1:
-	bus_dmamap_destroy(dma->dma_tag, dma->dma_map);
 	bus_dma_tag_destroy(dma->dma_tag);
-fail_0:
-	dma->dma_map = NULL;
 	dma->dma_tag = NULL;
 	return (r);
 }
 
 static void
 ubsec_dma_free(struct ubsec_softc *sc, struct ubsec_dma_alloc *dma)
 {
 	bus_dmamap_unload(dma->dma_tag, dma->dma_map);
 	bus_dmamem_free(dma->dma_tag, dma->dma_vaddr, dma->dma_map);
-	bus_dmamap_destroy(dma->dma_tag, dma->dma_map);
 	bus_dma_tag_destroy(dma->dma_tag);
 }
 
 /*
  * Resets the board.  Values in the regesters are left as is
  * from the reset (i.e. initial values are assigned elsewhere).
  */
 static void
 ubsec_reset_board(struct ubsec_softc *sc)
 {
     volatile u_int32_t ctrl;
 
     ctrl = READ_REG(sc, BS_CTRL);
     ctrl |= BS_CTRL_RESET;
     WRITE_REG(sc, BS_CTRL, ctrl);
 
     /*
      * Wait aprox. 30 PCI clocks = 900 ns = 0.9 us
      */
     DELAY(10);
 }
 
 /*
  * Init Broadcom registers
  */
 static void
 ubsec_init_board(struct ubsec_softc *sc)
 {
 	u_int32_t ctrl;
 
 	ctrl = READ_REG(sc, BS_CTRL);
 	ctrl &= ~(BS_CTRL_BE32 | BS_CTRL_BE64);
 	ctrl |= BS_CTRL_LITTLE_ENDIAN | BS_CTRL_MCR1INT;
 
 	if (sc->sc_flags & (UBS_FLAGS_KEY|UBS_FLAGS_RNG))
 		ctrl |= BS_CTRL_MCR2INT;
 	else
 		ctrl &= ~BS_CTRL_MCR2INT;
 
 	if (sc->sc_flags & UBS_FLAGS_HWNORM)
 		ctrl &= ~BS_CTRL_SWNORM;
 
 	WRITE_REG(sc, BS_CTRL, ctrl);
 }
 
 /*
  * Init Broadcom PCI registers
  */
 static void
 ubsec_init_pciregs(device_t dev)
 {
 #if 0
 	u_int32_t misc;
 
 	misc = pci_conf_read(pc, pa->pa_tag, BS_RTY_TOUT);
 	misc = (misc & ~(UBS_PCI_RTY_MASK << UBS_PCI_RTY_SHIFT))
 	    | ((UBS_DEF_RTY & 0xff) << UBS_PCI_RTY_SHIFT);
 	misc = (misc & ~(UBS_PCI_TOUT_MASK << UBS_PCI_TOUT_SHIFT))
 	    | ((UBS_DEF_TOUT & 0xff) << UBS_PCI_TOUT_SHIFT);
 	pci_conf_write(pc, pa->pa_tag, BS_RTY_TOUT, misc);
 #endif
 
 	/*
 	 * This will set the cache line size to 1, this will
 	 * force the BCM58xx chip just to do burst read/writes.
 	 * Cache line read/writes are to slow
 	 */
 	pci_write_config(dev, PCIR_CACHELNSZ, UBS_DEF_CACHELINE, 1);
 }
 
 /*
  * Clean up after a chip crash.
  * It is assumed that the caller in splimp()
  */
 static void
 ubsec_cleanchip(struct ubsec_softc *sc)
 {
 	struct ubsec_q *q;
 
 	while (!SIMPLEQ_EMPTY(&sc->sc_qchip)) {
 		q = SIMPLEQ_FIRST(&sc->sc_qchip);
 		SIMPLEQ_REMOVE_HEAD(&sc->sc_qchip, q_next);
 		ubsec_free_q(sc, q);
 	}
 	sc->sc_nqchip = 0;
 }
 
 /*
  * free a ubsec_q
  * It is assumed that the caller is within splimp().
  */
 static int
 ubsec_free_q(struct ubsec_softc *sc, struct ubsec_q *q)
 {
 	struct ubsec_q *q2;
 	struct cryptop *crp;
 	int npkts;
 	int i;
 
 	npkts = q->q_nstacked_mcrs;
 
 	for (i = 0; i < npkts; i++) {
 		if(q->q_stacked_mcr[i]) {
 			q2 = q->q_stacked_mcr[i];
 
 			if ((q2->q_dst_m != NULL) && (q2->q_src_m != q2->q_dst_m))
 				m_freem(q2->q_dst_m);
 
 			crp = (struct cryptop *)q2->q_crp;
 
 			SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q2, q_next);
 
 			crp->crp_etype = EFAULT;
 			crypto_done(crp);
 		} else {
 			break;
 		}
 	}
 
 	/*
 	 * Free header MCR
 	 */
 	if ((q->q_dst_m != NULL) && (q->q_src_m != q->q_dst_m))
 		m_freem(q->q_dst_m);
 
 	crp = (struct cryptop *)q->q_crp;
 
 	SIMPLEQ_INSERT_TAIL(&sc->sc_freequeue, q, q_next);
 
 	crp->crp_etype = EFAULT;
 	crypto_done(crp);
 	return(0);
 }
 
 /*
  * Routine to reset the chip and clean up.
  * It is assumed that the caller is in splimp()
  */
 static void
 ubsec_totalreset(struct ubsec_softc *sc)
 {
 	ubsec_reset_board(sc);
 	ubsec_init_board(sc);
 	ubsec_cleanchip(sc);
 }
 
 static int
 ubsec_dmamap_aligned(struct ubsec_operand *op)
 {
 	int i;
 
 	for (i = 0; i < op->nsegs; i++) {
 		if (op->segs[i].ds_addr & 3)
 			return (0);
 		if ((i != (op->nsegs - 1)) &&
 		    (op->segs[i].ds_len & 3))
 			return (0);
 	}
 	return (1);
 }
 
 static void
 ubsec_kfree(struct ubsec_softc *sc, struct ubsec_q2 *q)
 {
 	switch (q->q_type) {
 	case UBS_CTXOP_MODEXP: {
 		struct ubsec_q2_modexp *me = (struct ubsec_q2_modexp *)q;
 
 		ubsec_dma_free(sc, &me->me_q.q_mcr);
 		ubsec_dma_free(sc, &me->me_q.q_ctx);
 		ubsec_dma_free(sc, &me->me_M);
 		ubsec_dma_free(sc, &me->me_E);
 		ubsec_dma_free(sc, &me->me_C);
 		ubsec_dma_free(sc, &me->me_epb);
 		free(me, M_DEVBUF);
 		break;
 	}
 	case UBS_CTXOP_RSAPRIV: {
 		struct ubsec_q2_rsapriv *rp = (struct ubsec_q2_rsapriv *)q;
 
 		ubsec_dma_free(sc, &rp->rpr_q.q_mcr);
 		ubsec_dma_free(sc, &rp->rpr_q.q_ctx);
 		ubsec_dma_free(sc, &rp->rpr_msgin);
 		ubsec_dma_free(sc, &rp->rpr_msgout);
 		free(rp, M_DEVBUF);
 		break;
 	}
 	default:
 		device_printf(sc->sc_dev, "invalid kfree 0x%x\n", q->q_type);
 		break;
 	}
 }
 
 static int
 ubsec_kprocess(device_t dev, struct cryptkop *krp, int hint)
 {
 	struct ubsec_softc *sc = device_get_softc(dev);
 	int r;
 
 	if (krp == NULL || krp->krp_callback == NULL)
 		return (EINVAL);
 
 	while (!SIMPLEQ_EMPTY(&sc->sc_q2free)) {
 		struct ubsec_q2 *q;
 
 		q = SIMPLEQ_FIRST(&sc->sc_q2free);
 		SIMPLEQ_REMOVE_HEAD(&sc->sc_q2free, q_next);
 		ubsec_kfree(sc, q);
 	}
 
 	switch (krp->krp_op) {
 	case CRK_MOD_EXP:
 		if (sc->sc_flags & UBS_FLAGS_HWNORM)
 			r = ubsec_kprocess_modexp_hw(sc, krp, hint);
 		else
 			r = ubsec_kprocess_modexp_sw(sc, krp, hint);
 		break;
 	case CRK_MOD_EXP_CRT:
 		return (ubsec_kprocess_rsapriv(sc, krp, hint));
 	default:
 		device_printf(sc->sc_dev, "kprocess: invalid op 0x%x\n",
 		    krp->krp_op);
 		krp->krp_status = EOPNOTSUPP;
 		crypto_kdone(krp);
 		return (0);
 	}
 	return (0);			/* silence compiler */
 }
 
 /*
  * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (sw normalization)
  */
 static int
 ubsec_kprocess_modexp_sw(struct ubsec_softc *sc, struct cryptkop *krp, int hint)
 {
 	struct ubsec_q2_modexp *me;
 	struct ubsec_mcr *mcr;
 	struct ubsec_ctx_modexp *ctx;
 	struct ubsec_pktbuf *epb;
 	int err = 0;
 	u_int nbits, normbits, mbits, shiftbits, ebits;
 
 	me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT);
 	if (me == NULL) {
 		err = ENOMEM;
 		goto errout;
 	}
 	bzero(me, sizeof *me);
 	me->me_krp = krp;
 	me->me_q.q_type = UBS_CTXOP_MODEXP;
 
 	nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]);
 	if (nbits <= 512)
 		normbits = 512;
 	else if (nbits <= 768)
 		normbits = 768;
 	else if (nbits <= 1024)
 		normbits = 1024;
 	else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536)
 		normbits = 1536;
 	else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048)
 		normbits = 2048;
 	else {
 		err = E2BIG;
 		goto errout;
 	}
 
 	shiftbits = normbits - nbits;
 
 	me->me_modbits = nbits;
 	me->me_shiftbits = shiftbits;
 	me->me_normbits = normbits;
 
 	/* Sanity check: result bits must be >= true modulus bits. */
 	if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) {
 		err = ERANGE;
 		goto errout;
 	}
 
 	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
 	    &me->me_q.q_mcr, 0)) {
 		err = ENOMEM;
 		goto errout;
 	}
 	mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr;
 
 	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp),
 	    &me->me_q.q_ctx, 0)) {
 		err = ENOMEM;
 		goto errout;
 	}
 
 	mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]);
 	if (mbits > nbits) {
 		err = E2BIG;
 		goto errout;
 	}
 	if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) {
 		err = ENOMEM;
 		goto errout;
 	}
 	ubsec_kshift_r(shiftbits,
 	    krp->krp_param[UBS_MODEXP_PAR_M].crp_p, mbits,
 	    me->me_M.dma_vaddr, normbits);
 
 	if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) {
 		err = ENOMEM;
 		goto errout;
 	}
 	bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
 
 	ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]);
 	if (ebits > nbits) {
 		err = E2BIG;
 		goto errout;
 	}
 	if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) {
 		err = ENOMEM;
 		goto errout;
 	}
 	ubsec_kshift_r(shiftbits,
 	    krp->krp_param[UBS_MODEXP_PAR_E].crp_p, ebits,
 	    me->me_E.dma_vaddr, normbits);
 
 	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf),
 	    &me->me_epb, 0)) {
 		err = ENOMEM;
 		goto errout;
 	}
 	epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr;
 	epb->pb_addr = htole32(me->me_E.dma_paddr);
 	epb->pb_next = 0;
 	epb->pb_len = htole32(normbits / 8);
 
 #ifdef UBSEC_DEBUG
 	if (ubsec_debug) {
 		printf("Epb ");
 		ubsec_dump_pb(epb);
 	}
 #endif
 
 	mcr->mcr_pkts = htole16(1);
 	mcr->mcr_flags = 0;
 	mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr);
 	mcr->mcr_reserved = 0;
 	mcr->mcr_pktlen = 0;
 
 	mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr);
 	mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8);
 	mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr);
 
 	mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr);
 	mcr->mcr_opktbuf.pb_next = 0;
 	mcr->mcr_opktbuf.pb_len = htole32(normbits / 8);
 
 #ifdef DIAGNOSTIC
 	/* Misaligned output buffer will hang the chip. */
 	if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0)
 		panic("%s: modexp invalid addr 0x%x\n",
 		    device_get_nameunit(sc->sc_dev),
 		    letoh32(mcr->mcr_opktbuf.pb_addr));
 	if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0)
 		panic("%s: modexp invalid len 0x%x\n",
 		    device_get_nameunit(sc->sc_dev),
 		    letoh32(mcr->mcr_opktbuf.pb_len));
 #endif
 
 	ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr;
 	bzero(ctx, sizeof(*ctx));
 	ubsec_kshift_r(shiftbits,
 	    krp->krp_param[UBS_MODEXP_PAR_N].crp_p, nbits,
 	    ctx->me_N, normbits);
 	ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t)));
 	ctx->me_op = htole16(UBS_CTXOP_MODEXP);
 	ctx->me_E_len = htole16(nbits);
 	ctx->me_N_len = htole16(nbits);
 
 #ifdef UBSEC_DEBUG
 	if (ubsec_debug) {
 		ubsec_dump_mcr(mcr);
 		ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx);
 	}
 #endif
 
 	/*
 	 * ubsec_feed2 will sync mcr and ctx, we just need to sync
 	 * everything else.
 	 */
 	ubsec_dma_sync(&me->me_M, BUS_DMASYNC_PREWRITE);
 	ubsec_dma_sync(&me->me_E, BUS_DMASYNC_PREWRITE);
 	ubsec_dma_sync(&me->me_C, BUS_DMASYNC_PREREAD);
 	ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_PREWRITE);
 
 	/* Enqueue and we're done... */
 	mtx_lock(&sc->sc_mcr2lock);
 	SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next);
 	ubsec_feed2(sc);
 	ubsecstats.hst_modexp++;
 	mtx_unlock(&sc->sc_mcr2lock);
 
 	return (0);
 
 errout:
 	if (me != NULL) {
 		if (me->me_q.q_mcr.dma_map != NULL)
 			ubsec_dma_free(sc, &me->me_q.q_mcr);
 		if (me->me_q.q_ctx.dma_map != NULL) {
 			bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size);
 			ubsec_dma_free(sc, &me->me_q.q_ctx);
 		}
 		if (me->me_M.dma_map != NULL) {
 			bzero(me->me_M.dma_vaddr, me->me_M.dma_size);
 			ubsec_dma_free(sc, &me->me_M);
 		}
 		if (me->me_E.dma_map != NULL) {
 			bzero(me->me_E.dma_vaddr, me->me_E.dma_size);
 			ubsec_dma_free(sc, &me->me_E);
 		}
 		if (me->me_C.dma_map != NULL) {
 			bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
 			ubsec_dma_free(sc, &me->me_C);
 		}
 		if (me->me_epb.dma_map != NULL)
 			ubsec_dma_free(sc, &me->me_epb);
 		free(me, M_DEVBUF);
 	}
 	krp->krp_status = err;
 	crypto_kdone(krp);
 	return (0);
 }
 
 /*
  * Start computation of cr[C] = (cr[M] ^ cr[E]) mod cr[N] (hw normalization)
  */
 static int
 ubsec_kprocess_modexp_hw(struct ubsec_softc *sc, struct cryptkop *krp, int hint)
 {
 	struct ubsec_q2_modexp *me;
 	struct ubsec_mcr *mcr;
 	struct ubsec_ctx_modexp *ctx;
 	struct ubsec_pktbuf *epb;
 	int err = 0;
 	u_int nbits, normbits, mbits, shiftbits, ebits;
 
 	me = (struct ubsec_q2_modexp *)malloc(sizeof *me, M_DEVBUF, M_NOWAIT);
 	if (me == NULL) {
 		err = ENOMEM;
 		goto errout;
 	}
 	bzero(me, sizeof *me);
 	me->me_krp = krp;
 	me->me_q.q_type = UBS_CTXOP_MODEXP;
 
 	nbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_N]);
 	if (nbits <= 512)
 		normbits = 512;
 	else if (nbits <= 768)
 		normbits = 768;
 	else if (nbits <= 1024)
 		normbits = 1024;
 	else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 1536)
 		normbits = 1536;
 	else if (sc->sc_flags & UBS_FLAGS_BIGKEY && nbits <= 2048)
 		normbits = 2048;
 	else {
 		err = E2BIG;
 		goto errout;
 	}
 
 	shiftbits = normbits - nbits;
 
 	/* XXX ??? */
 	me->me_modbits = nbits;
 	me->me_shiftbits = shiftbits;
 	me->me_normbits = normbits;
 
 	/* Sanity check: result bits must be >= true modulus bits. */
 	if (krp->krp_param[krp->krp_iparams].crp_nbits < nbits) {
 		err = ERANGE;
 		goto errout;
 	}
 
 	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
 	    &me->me_q.q_mcr, 0)) {
 		err = ENOMEM;
 		goto errout;
 	}
 	mcr = (struct ubsec_mcr *)me->me_q.q_mcr.dma_vaddr;
 
 	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_modexp),
 	    &me->me_q.q_ctx, 0)) {
 		err = ENOMEM;
 		goto errout;
 	}
 
 	mbits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_M]);
 	if (mbits > nbits) {
 		err = E2BIG;
 		goto errout;
 	}
 	if (ubsec_dma_malloc(sc, normbits / 8, &me->me_M, 0)) {
 		err = ENOMEM;
 		goto errout;
 	}
 	bzero(me->me_M.dma_vaddr, normbits / 8);
 	bcopy(krp->krp_param[UBS_MODEXP_PAR_M].crp_p,
 	    me->me_M.dma_vaddr, (mbits + 7) / 8);
 
 	if (ubsec_dma_malloc(sc, normbits / 8, &me->me_C, 0)) {
 		err = ENOMEM;
 		goto errout;
 	}
 	bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
 
 	ebits = ubsec_ksigbits(&krp->krp_param[UBS_MODEXP_PAR_E]);
 	if (ebits > nbits) {
 		err = E2BIG;
 		goto errout;
 	}
 	if (ubsec_dma_malloc(sc, normbits / 8, &me->me_E, 0)) {
 		err = ENOMEM;
 		goto errout;
 	}
 	bzero(me->me_E.dma_vaddr, normbits / 8);
 	bcopy(krp->krp_param[UBS_MODEXP_PAR_E].crp_p,
 	    me->me_E.dma_vaddr, (ebits + 7) / 8);
 
 	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_pktbuf),
 	    &me->me_epb, 0)) {
 		err = ENOMEM;
 		goto errout;
 	}
 	epb = (struct ubsec_pktbuf *)me->me_epb.dma_vaddr;
 	epb->pb_addr = htole32(me->me_E.dma_paddr);
 	epb->pb_next = 0;
 	epb->pb_len = htole32((ebits + 7) / 8);
 
 #ifdef UBSEC_DEBUG
 	if (ubsec_debug) {
 		printf("Epb ");
 		ubsec_dump_pb(epb);
 	}
 #endif
 
 	mcr->mcr_pkts = htole16(1);
 	mcr->mcr_flags = 0;
 	mcr->mcr_cmdctxp = htole32(me->me_q.q_ctx.dma_paddr);
 	mcr->mcr_reserved = 0;
 	mcr->mcr_pktlen = 0;
 
 	mcr->mcr_ipktbuf.pb_addr = htole32(me->me_M.dma_paddr);
 	mcr->mcr_ipktbuf.pb_len = htole32(normbits / 8);
 	mcr->mcr_ipktbuf.pb_next = htole32(me->me_epb.dma_paddr);
 
 	mcr->mcr_opktbuf.pb_addr = htole32(me->me_C.dma_paddr);
 	mcr->mcr_opktbuf.pb_next = 0;
 	mcr->mcr_opktbuf.pb_len = htole32(normbits / 8);
 
 #ifdef DIAGNOSTIC
 	/* Misaligned output buffer will hang the chip. */
 	if ((letoh32(mcr->mcr_opktbuf.pb_addr) & 3) != 0)
 		panic("%s: modexp invalid addr 0x%x\n",
 		    device_get_nameunit(sc->sc_dev),
 		    letoh32(mcr->mcr_opktbuf.pb_addr));
 	if ((letoh32(mcr->mcr_opktbuf.pb_len) & 3) != 0)
 		panic("%s: modexp invalid len 0x%x\n",
 		    device_get_nameunit(sc->sc_dev),
 		    letoh32(mcr->mcr_opktbuf.pb_len));
 #endif
 
 	ctx = (struct ubsec_ctx_modexp *)me->me_q.q_ctx.dma_vaddr;
 	bzero(ctx, sizeof(*ctx));
 	bcopy(krp->krp_param[UBS_MODEXP_PAR_N].crp_p, ctx->me_N,
 	    (nbits + 7) / 8);
 	ctx->me_len = htole16((normbits / 8) + (4 * sizeof(u_int16_t)));
 	ctx->me_op = htole16(UBS_CTXOP_MODEXP);
 	ctx->me_E_len = htole16(ebits);
 	ctx->me_N_len = htole16(nbits);
 
 #ifdef UBSEC_DEBUG
 	if (ubsec_debug) {
 		ubsec_dump_mcr(mcr);
 		ubsec_dump_ctx2((struct ubsec_ctx_keyop *)ctx);
 	}
 #endif
 
 	/*
 	 * ubsec_feed2 will sync mcr and ctx, we just need to sync
 	 * everything else.
 	 */
 	ubsec_dma_sync(&me->me_M, BUS_DMASYNC_PREWRITE);
 	ubsec_dma_sync(&me->me_E, BUS_DMASYNC_PREWRITE);
 	ubsec_dma_sync(&me->me_C, BUS_DMASYNC_PREREAD);
 	ubsec_dma_sync(&me->me_epb, BUS_DMASYNC_PREWRITE);
 
 	/* Enqueue and we're done... */
 	mtx_lock(&sc->sc_mcr2lock);
 	SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &me->me_q, q_next);
 	ubsec_feed2(sc);
 	mtx_unlock(&sc->sc_mcr2lock);
 
 	return (0);
 
 errout:
 	if (me != NULL) {
 		if (me->me_q.q_mcr.dma_map != NULL)
 			ubsec_dma_free(sc, &me->me_q.q_mcr);
 		if (me->me_q.q_ctx.dma_map != NULL) {
 			bzero(me->me_q.q_ctx.dma_vaddr, me->me_q.q_ctx.dma_size);
 			ubsec_dma_free(sc, &me->me_q.q_ctx);
 		}
 		if (me->me_M.dma_map != NULL) {
 			bzero(me->me_M.dma_vaddr, me->me_M.dma_size);
 			ubsec_dma_free(sc, &me->me_M);
 		}
 		if (me->me_E.dma_map != NULL) {
 			bzero(me->me_E.dma_vaddr, me->me_E.dma_size);
 			ubsec_dma_free(sc, &me->me_E);
 		}
 		if (me->me_C.dma_map != NULL) {
 			bzero(me->me_C.dma_vaddr, me->me_C.dma_size);
 			ubsec_dma_free(sc, &me->me_C);
 		}
 		if (me->me_epb.dma_map != NULL)
 			ubsec_dma_free(sc, &me->me_epb);
 		free(me, M_DEVBUF);
 	}
 	krp->krp_status = err;
 	crypto_kdone(krp);
 	return (0);
 }
 
 static int
 ubsec_kprocess_rsapriv(struct ubsec_softc *sc, struct cryptkop *krp, int hint)
 {
 	struct ubsec_q2_rsapriv *rp = NULL;
 	struct ubsec_mcr *mcr;
 	struct ubsec_ctx_rsapriv *ctx;
 	int err = 0;
 	u_int padlen, msglen;
 
 	msglen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_P]);
 	padlen = ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_Q]);
 	if (msglen > padlen)
 		padlen = msglen;
 
 	if (padlen <= 256)
 		padlen = 256;
 	else if (padlen <= 384)
 		padlen = 384;
 	else if (padlen <= 512)
 		padlen = 512;
 	else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 768)
 		padlen = 768;
 	else if (sc->sc_flags & UBS_FLAGS_BIGKEY && padlen <= 1024)
 		padlen = 1024;
 	else {
 		err = E2BIG;
 		goto errout;
 	}
 
 	if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DP]) > padlen) {
 		err = E2BIG;
 		goto errout;
 	}
 
 	if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_DQ]) > padlen) {
 		err = E2BIG;
 		goto errout;
 	}
 
 	if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_PINV]) > padlen) {
 		err = E2BIG;
 		goto errout;
 	}
 
 	rp = (struct ubsec_q2_rsapriv *)malloc(sizeof *rp, M_DEVBUF, M_NOWAIT);
 	if (rp == NULL)
 		return (ENOMEM);
 	bzero(rp, sizeof *rp);
 	rp->rpr_krp = krp;
 	rp->rpr_q.q_type = UBS_CTXOP_RSAPRIV;
 
 	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_mcr),
 	    &rp->rpr_q.q_mcr, 0)) {
 		err = ENOMEM;
 		goto errout;
 	}
 	mcr = (struct ubsec_mcr *)rp->rpr_q.q_mcr.dma_vaddr;
 
 	if (ubsec_dma_malloc(sc, sizeof(struct ubsec_ctx_rsapriv),
 	    &rp->rpr_q.q_ctx, 0)) {
 		err = ENOMEM;
 		goto errout;
 	}
 	ctx = (struct ubsec_ctx_rsapriv *)rp->rpr_q.q_ctx.dma_vaddr;
 	bzero(ctx, sizeof *ctx);
 
 	/* Copy in p */
 	bcopy(krp->krp_param[UBS_RSAPRIV_PAR_P].crp_p,
 	    &ctx->rpr_buf[0 * (padlen / 8)],
 	    (krp->krp_param[UBS_RSAPRIV_PAR_P].crp_nbits + 7) / 8);
 
 	/* Copy in q */
 	bcopy(krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_p,
 	    &ctx->rpr_buf[1 * (padlen / 8)],
 	    (krp->krp_param[UBS_RSAPRIV_PAR_Q].crp_nbits + 7) / 8);
 
 	/* Copy in dp */
 	bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_p,
 	    &ctx->rpr_buf[2 * (padlen / 8)],
 	    (krp->krp_param[UBS_RSAPRIV_PAR_DP].crp_nbits + 7) / 8);
 
 	/* Copy in dq */
 	bcopy(krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_p,
 	    &ctx->rpr_buf[3 * (padlen / 8)],
 	    (krp->krp_param[UBS_RSAPRIV_PAR_DQ].crp_nbits + 7) / 8);
 
 	/* Copy in pinv */
 	bcopy(krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_p,
 	    &ctx->rpr_buf[4 * (padlen / 8)],
 	    (krp->krp_param[UBS_RSAPRIV_PAR_PINV].crp_nbits + 7) / 8);
 
 	msglen = padlen * 2;
 
 	/* Copy in input message (aligned buffer/length). */
 	if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGIN]) > msglen) {
 		/* Is this likely? */
 		err = E2BIG;
 		goto errout;
 	}
 	if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgin, 0)) {
 		err = ENOMEM;
 		goto errout;
 	}
 	bzero(rp->rpr_msgin.dma_vaddr, (msglen + 7) / 8);
 	bcopy(krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_p,
 	    rp->rpr_msgin.dma_vaddr,
 	    (krp->krp_param[UBS_RSAPRIV_PAR_MSGIN].crp_nbits + 7) / 8);
 
 	/* Prepare space for output message (aligned buffer/length). */
 	if (ubsec_ksigbits(&krp->krp_param[UBS_RSAPRIV_PAR_MSGOUT]) < msglen) {
 		/* Is this likely? */
 		err = E2BIG;
 		goto errout;
 	}
 	if (ubsec_dma_malloc(sc, (msglen + 7) / 8, &rp->rpr_msgout, 0)) {
 		err = ENOMEM;
 		goto errout;
 	}
 	bzero(rp->rpr_msgout.dma_vaddr, (msglen + 7) / 8);
 
 	mcr->mcr_pkts = htole16(1);
 	mcr->mcr_flags = 0;
 	mcr->mcr_cmdctxp = htole32(rp->rpr_q.q_ctx.dma_paddr);
 	mcr->mcr_ipktbuf.pb_addr = htole32(rp->rpr_msgin.dma_paddr);
 	mcr->mcr_ipktbuf.pb_next = 0;
 	mcr->mcr_ipktbuf.pb_len = htole32(rp->rpr_msgin.dma_size);
 	mcr->mcr_reserved = 0;
 	mcr->mcr_pktlen = htole16(msglen);
 	mcr->mcr_opktbuf.pb_addr = htole32(rp->rpr_msgout.dma_paddr);
 	mcr->mcr_opktbuf.pb_next = 0;
 	mcr->mcr_opktbuf.pb_len = htole32(rp->rpr_msgout.dma_size);
 
 #ifdef DIAGNOSTIC
 	if (rp->rpr_msgin.dma_paddr & 3 || rp->rpr_msgin.dma_size & 3) {
 		panic("%s: rsapriv: invalid msgin %x(0x%jx)",
 		    device_get_nameunit(sc->sc_dev),
 		    rp->rpr_msgin.dma_paddr, (uintmax_t)rp->rpr_msgin.dma_size);
 	}
 	if (rp->rpr_msgout.dma_paddr & 3 || rp->rpr_msgout.dma_size & 3) {
 		panic("%s: rsapriv: invalid msgout %x(0x%jx)",
 		    device_get_nameunit(sc->sc_dev),
 		    rp->rpr_msgout.dma_paddr, (uintmax_t)rp->rpr_msgout.dma_size);
 	}
 #endif
 
 	ctx->rpr_len = (sizeof(u_int16_t) * 4) + (5 * (padlen / 8));
 	ctx->rpr_op = htole16(UBS_CTXOP_RSAPRIV);
 	ctx->rpr_q_len = htole16(padlen);
 	ctx->rpr_p_len = htole16(padlen);
 
 	/*
 	 * ubsec_feed2 will sync mcr and ctx, we just need to sync
 	 * everything else.
 	 */
 	ubsec_dma_sync(&rp->rpr_msgin, BUS_DMASYNC_PREWRITE);
 	ubsec_dma_sync(&rp->rpr_msgout, BUS_DMASYNC_PREREAD);
 
 	/* Enqueue and we're done... */
 	mtx_lock(&sc->sc_mcr2lock);
 	SIMPLEQ_INSERT_TAIL(&sc->sc_queue2, &rp->rpr_q, q_next);
 	ubsec_feed2(sc);
 	ubsecstats.hst_modexpcrt++;
 	mtx_unlock(&sc->sc_mcr2lock);
 	return (0);
 
 errout:
 	if (rp != NULL) {
 		if (rp->rpr_q.q_mcr.dma_map != NULL)
 			ubsec_dma_free(sc, &rp->rpr_q.q_mcr);
 		if (rp->rpr_msgin.dma_map != NULL) {
 			bzero(rp->rpr_msgin.dma_vaddr, rp->rpr_msgin.dma_size);
 			ubsec_dma_free(sc, &rp->rpr_msgin);
 		}
 		if (rp->rpr_msgout.dma_map != NULL) {
 			bzero(rp->rpr_msgout.dma_vaddr, rp->rpr_msgout.dma_size);
 			ubsec_dma_free(sc, &rp->rpr_msgout);
 		}
 		free(rp, M_DEVBUF);
 	}
 	krp->krp_status = err;
 	crypto_kdone(krp);
 	return (0);
 }
 
 #ifdef UBSEC_DEBUG
 static void
 ubsec_dump_pb(volatile struct ubsec_pktbuf *pb)
 {
 	printf("addr 0x%x (0x%x) next 0x%x\n",
 	    pb->pb_addr, pb->pb_len, pb->pb_next);
 }
 
 static void
 ubsec_dump_ctx2(struct ubsec_ctx_keyop *c)
 {
 	printf("CTX (0x%x):\n", c->ctx_len);
 	switch (letoh16(c->ctx_op)) {
 	case UBS_CTXOP_RNGBYPASS:
 	case UBS_CTXOP_RNGSHA1:
 		break;
 	case UBS_CTXOP_MODEXP:
 	{
 		struct ubsec_ctx_modexp *cx = (void *)c;
 		int i, len;
 
 		printf(" Elen %u, Nlen %u\n",
 		    letoh16(cx->me_E_len), letoh16(cx->me_N_len));
 		len = (cx->me_N_len + 7)/8;
 		for (i = 0; i < len; i++)
 			printf("%s%02x", (i == 0) ? " N: " : ":", cx->me_N[i]);
 		printf("\n");
 		break;
 	}
 	default:
 		printf("unknown context: %x\n", c->ctx_op);
 	}
 	printf("END CTX\n");
 }
 
 static void
 ubsec_dump_mcr(struct ubsec_mcr *mcr)
 {
 	volatile struct ubsec_mcr_add *ma;
 	int i;
 
 	printf("MCR:\n");
 	printf(" pkts: %u, flags 0x%x\n",
 	    letoh16(mcr->mcr_pkts), letoh16(mcr->mcr_flags));
 	ma = (volatile struct ubsec_mcr_add *)&mcr->mcr_cmdctxp;
 	for (i = 0; i < letoh16(mcr->mcr_pkts); i++) {
 		printf(" %d: ctx 0x%x len 0x%x rsvd 0x%x\n", i,
 		    letoh32(ma->mcr_cmdctxp), letoh16(ma->mcr_pktlen),
 		    letoh16(ma->mcr_reserved));
 		printf(" %d: ipkt ", i);
 		ubsec_dump_pb(&ma->mcr_ipktbuf);
 		printf(" %d: opkt ", i);
 		ubsec_dump_pb(&ma->mcr_opktbuf);
 		ma++;
 	}
 	printf("END MCR\n");
 }
 #endif /* UBSEC_DEBUG */
 
 /*
  * Return the number of significant bits of a big number.
  */
 static int
 ubsec_ksigbits(struct crparam *cr)
 {
 	u_int plen = (cr->crp_nbits + 7) / 8;
 	int i, sig = plen * 8;
 	u_int8_t c, *p = cr->crp_p;
 
 	for (i = plen - 1; i >= 0; i--) {
 		c = p[i];
 		if (c != 0) {
 			while ((c & 0x80) == 0) {
 				sig--;
 				c <<= 1;
 			}
 			break;
 		}
 		sig -= 8;
 	}
 	return (sig);
 }
 
 static void
 ubsec_kshift_r(
 	u_int shiftbits,
 	u_int8_t *src, u_int srcbits,
 	u_int8_t *dst, u_int dstbits)
 {
 	u_int slen, dlen;
 	int i, si, di, n;
 
 	slen = (srcbits + 7) / 8;
 	dlen = (dstbits + 7) / 8;
 
 	for (i = 0; i < slen; i++)
 		dst[i] = src[i];
 	for (i = 0; i < dlen - slen; i++)
 		dst[slen + i] = 0;
 
 	n = shiftbits / 8;
 	if (n != 0) {
 		si = dlen - n - 1;
 		di = dlen - 1;
 		while (si >= 0)
 			dst[di--] = dst[si--];
 		while (di >= 0)
 			dst[di--] = 0;
 	}
 
 	n = shiftbits % 8;
 	if (n != 0) {
 		for (i = dlen - 1; i > 0; i--)
 			dst[i] = (dst[i] << n) |
 			    (dst[i - 1] >> (8 - n));
 		dst[0] = dst[0] << n;
 	}
 }
 
 static void
 ubsec_kshift_l(
 	u_int shiftbits,
 	u_int8_t *src, u_int srcbits,
 	u_int8_t *dst, u_int dstbits)
 {
 	int slen, dlen, i, n;
 
 	slen = (srcbits + 7) / 8;
 	dlen = (dstbits + 7) / 8;
 
 	n = shiftbits / 8;
 	for (i = 0; i < slen; i++)
 		dst[i] = src[i + n];
 	for (i = 0; i < dlen - slen; i++)
 		dst[slen + i] = 0;
 
 	n = shiftbits % 8;
 	if (n != 0) {
 		for (i = 0; i < (dlen - 1); i++)
 			dst[i] = (dst[i] >> n) | (dst[i + 1] << (8 - n));
 		dst[dlen - 1] = dst[dlen - 1] >> n;
 	}
 }
Index: projects/release-debugdist/sys/netgraph/bluetooth/socket/ng_btsocket_hci_raw.c
===================================================================
--- projects/release-debugdist/sys/netgraph/bluetooth/socket/ng_btsocket_hci_raw.c	(revision 267343)
+++ projects/release-debugdist/sys/netgraph/bluetooth/socket/ng_btsocket_hci_raw.c	(revision 267344)
@@ -1,1640 +1,1647 @@
 /*
  * ng_btsocket_hci_raw.c
  */
 
 /*-
  * Copyright (c) 2001-2002 Maksim Yevmenkin <m_evmenkin@yahoo.com>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
  * $Id: ng_btsocket_hci_raw.c,v 1.14 2003/09/14 23:29:06 max Exp $
  * $FreeBSD$
  */
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/bitstring.h>
 #include <sys/domain.h>
 #include <sys/endian.h>
 #include <sys/errno.h>
 #include <sys/filedesc.h>
 #include <sys/ioccom.h>
 #include <sys/kernel.h>
 #include <sys/lock.h>
 #include <sys/malloc.h>
 #include <sys/mbuf.h>
 #include <sys/mutex.h>
 #include <sys/priv.h>
 #include <sys/protosw.h>
 #include <sys/queue.h>
 #include <sys/socket.h>
 #include <sys/socketvar.h>
 #include <sys/sysctl.h>
 #include <sys/taskqueue.h>
+
+#include <net/vnet.h>
+
 #include <netgraph/ng_message.h>
 #include <netgraph/netgraph.h>
 #include <netgraph/bluetooth/include/ng_bluetooth.h>
 #include <netgraph/bluetooth/include/ng_hci.h>
 #include <netgraph/bluetooth/include/ng_l2cap.h>
 #include <netgraph/bluetooth/include/ng_btsocket.h>
 #include <netgraph/bluetooth/include/ng_btsocket_hci_raw.h>
 
 /* MALLOC define */
 #ifdef NG_SEPARATE_MALLOC
 static MALLOC_DEFINE(M_NETGRAPH_BTSOCKET_HCI_RAW, "netgraph_btsocks_hci_raw",
 	"Netgraph Bluetooth raw HCI sockets");
 #else
 #define M_NETGRAPH_BTSOCKET_HCI_RAW M_NETGRAPH
 #endif /* NG_SEPARATE_MALLOC */
 
 /* Netgraph node methods */
 static ng_constructor_t	ng_btsocket_hci_raw_node_constructor;
 static ng_rcvmsg_t	ng_btsocket_hci_raw_node_rcvmsg;
 static ng_shutdown_t	ng_btsocket_hci_raw_node_shutdown;
 static ng_newhook_t	ng_btsocket_hci_raw_node_newhook;
 static ng_connect_t	ng_btsocket_hci_raw_node_connect;
 static ng_rcvdata_t	ng_btsocket_hci_raw_node_rcvdata;
 static ng_disconnect_t	ng_btsocket_hci_raw_node_disconnect;
 
 static void 		ng_btsocket_hci_raw_input (void *, int);
 static void 		ng_btsocket_hci_raw_output(node_p, hook_p, void *, int);
 static void		ng_btsocket_hci_raw_savctl(ng_btsocket_hci_raw_pcb_p, 
 						   struct mbuf **,
 						   struct mbuf *); 
 static int		ng_btsocket_hci_raw_filter(ng_btsocket_hci_raw_pcb_p,
 						   struct mbuf *, int);
 
 #define ng_btsocket_hci_raw_wakeup_input_task() \
 	taskqueue_enqueue(taskqueue_swi, &ng_btsocket_hci_raw_task)
 
 /* Security filter */
 struct ng_btsocket_hci_raw_sec_filter {
 	bitstr_t	bit_decl(events, 0xff);
 	bitstr_t	bit_decl(commands[0x3f], 0x3ff);
 };
 
 /* Netgraph type descriptor */
 static struct ng_type typestruct = {
 	.version =	NG_ABI_VERSION,
 	.name =		NG_BTSOCKET_HCI_RAW_NODE_TYPE,
 	.constructor =	ng_btsocket_hci_raw_node_constructor,
 	.rcvmsg =	ng_btsocket_hci_raw_node_rcvmsg,
 	.shutdown =	ng_btsocket_hci_raw_node_shutdown,
 	.newhook =	ng_btsocket_hci_raw_node_newhook,
 	.connect =	ng_btsocket_hci_raw_node_connect,
 	.rcvdata =	ng_btsocket_hci_raw_node_rcvdata,
 	.disconnect =	ng_btsocket_hci_raw_node_disconnect,
 };
 
 /* Globals */
 static u_int32_t				ng_btsocket_hci_raw_debug_level;
 static u_int32_t				ng_btsocket_hci_raw_ioctl_timeout;
 static node_p					ng_btsocket_hci_raw_node;
 static struct ng_bt_itemq			ng_btsocket_hci_raw_queue;
 static struct mtx				ng_btsocket_hci_raw_queue_mtx;
 static struct task				ng_btsocket_hci_raw_task;
 static LIST_HEAD(, ng_btsocket_hci_raw_pcb)	ng_btsocket_hci_raw_sockets;
 static struct mtx				ng_btsocket_hci_raw_sockets_mtx;
 static u_int32_t				ng_btsocket_hci_raw_token;
 static struct mtx				ng_btsocket_hci_raw_token_mtx;
 static struct ng_btsocket_hci_raw_sec_filter	*ng_btsocket_hci_raw_sec_filter;
 static struct timeval				ng_btsocket_hci_raw_lasttime;
 static int					ng_btsocket_hci_raw_curpps;
  
 /* Sysctl tree */
 SYSCTL_DECL(_net_bluetooth_hci_sockets);
 static SYSCTL_NODE(_net_bluetooth_hci_sockets, OID_AUTO, raw, CTLFLAG_RW,
         0, "Bluetooth raw HCI sockets family");
 SYSCTL_UINT(_net_bluetooth_hci_sockets_raw, OID_AUTO, debug_level, CTLFLAG_RW,
         &ng_btsocket_hci_raw_debug_level, NG_BTSOCKET_WARN_LEVEL,
 	"Bluetooth raw HCI sockets debug level");
 SYSCTL_UINT(_net_bluetooth_hci_sockets_raw, OID_AUTO, ioctl_timeout, CTLFLAG_RW,
         &ng_btsocket_hci_raw_ioctl_timeout, 5,
 	"Bluetooth raw HCI sockets ioctl timeout");
 SYSCTL_UINT(_net_bluetooth_hci_sockets_raw, OID_AUTO, queue_len, CTLFLAG_RD,
         &ng_btsocket_hci_raw_queue.len, 0,
         "Bluetooth raw HCI sockets input queue length");
 SYSCTL_UINT(_net_bluetooth_hci_sockets_raw, OID_AUTO, queue_maxlen, CTLFLAG_RD,
         &ng_btsocket_hci_raw_queue.maxlen, 0,
         "Bluetooth raw HCI sockets input queue max. length");
 SYSCTL_UINT(_net_bluetooth_hci_sockets_raw, OID_AUTO, queue_drops, CTLFLAG_RD,
         &ng_btsocket_hci_raw_queue.drops, 0,
         "Bluetooth raw HCI sockets input queue drops");
 
 /* Debug */
 #define NG_BTSOCKET_HCI_RAW_INFO \
 	if (ng_btsocket_hci_raw_debug_level >= NG_BTSOCKET_INFO_LEVEL && \
 	    ppsratecheck(&ng_btsocket_hci_raw_lasttime, &ng_btsocket_hci_raw_curpps, 1)) \
 		printf
 
 #define NG_BTSOCKET_HCI_RAW_WARN \
 	if (ng_btsocket_hci_raw_debug_level >= NG_BTSOCKET_WARN_LEVEL && \
 	    ppsratecheck(&ng_btsocket_hci_raw_lasttime, &ng_btsocket_hci_raw_curpps, 1)) \
 		printf
 
 #define NG_BTSOCKET_HCI_RAW_ERR \
 	if (ng_btsocket_hci_raw_debug_level >= NG_BTSOCKET_ERR_LEVEL && \
 	    ppsratecheck(&ng_btsocket_hci_raw_lasttime, &ng_btsocket_hci_raw_curpps, 1)) \
 		printf
 
 #define NG_BTSOCKET_HCI_RAW_ALERT \
 	if (ng_btsocket_hci_raw_debug_level >= NG_BTSOCKET_ALERT_LEVEL && \
 	    ppsratecheck(&ng_btsocket_hci_raw_lasttime, &ng_btsocket_hci_raw_curpps, 1)) \
 		printf
 
 /****************************************************************************
  ****************************************************************************
  **                          Netgraph specific
  ****************************************************************************
  ****************************************************************************/
 
 /*
  * Netgraph node constructor. Do not allow to create node of this type.
  */
 
 static int
 ng_btsocket_hci_raw_node_constructor(node_p node)
 {
 	return (EINVAL);
 } /* ng_btsocket_hci_raw_node_constructor */
 
 /*
  * Netgraph node destructor. Just let old node go and create new fresh one.
  */
 
 static int
 ng_btsocket_hci_raw_node_shutdown(node_p node)
 {
 	int	error = 0;
 
 	NG_NODE_UNREF(node);
 
 	error = ng_make_node_common(&typestruct, &ng_btsocket_hci_raw_node);
 	if (error  != 0) {
 		NG_BTSOCKET_HCI_RAW_ALERT(
 "%s: Could not create Netgraph node, error=%d\n", __func__, error);
 
 		ng_btsocket_hci_raw_node = NULL;
 
 		return (ENOMEM);
         }
 
 	error = ng_name_node(ng_btsocket_hci_raw_node,
 				NG_BTSOCKET_HCI_RAW_NODE_TYPE);
 	if (error != 0) {
 		NG_BTSOCKET_HCI_RAW_ALERT(
 "%s: Could not name Netgraph node, error=%d\n", __func__, error);
 
 		NG_NODE_UNREF(ng_btsocket_hci_raw_node);
 		ng_btsocket_hci_raw_node = NULL;
 
 		return (EINVAL);
 	}
 
 	return (0);
 } /* ng_btsocket_hci_raw_node_shutdown */
 
 /*
  * Create new hook. Just say "yes"
  */
 
 static int
 ng_btsocket_hci_raw_node_newhook(node_p node, hook_p hook, char const *name)
 {
 	return (0);
 } /* ng_btsocket_hci_raw_node_newhook */
 
 /*
  * Connect hook. Just say "yes"
  */
 
 static int
 ng_btsocket_hci_raw_node_connect(hook_p hook)
 {
 	return (0);
 } /* ng_btsocket_hci_raw_node_connect */
 
 /*
  * Disconnect hook
  */
 
 static int
 ng_btsocket_hci_raw_node_disconnect(hook_p hook)
 {
 	return (0);
 } /* ng_btsocket_hci_raw_node_disconnect */
 
 /*
  * Receive control message.
  * Make sure it is a message from HCI node and it is a response.
  * Enqueue item and schedule input task.
  */
 
 static int
 ng_btsocket_hci_raw_node_rcvmsg(node_p node, item_p item, hook_p lasthook) 
 {
 	struct ng_mesg	*msg = NGI_MSG(item); /* item still has message */
 	int		 error = 0;
 
 	/*
 	 * Check for empty sockets list creates LOR when both sender and
 	 * receiver device are connected to the same host, so remove it
 	 * for now
 	 */
 
 	if (msg != NULL &&
 	    (msg->header.typecookie == NGM_HCI_COOKIE ||
 	     msg->header.typecookie == NGM_GENERIC_COOKIE) &&
 	    msg->header.flags & NGF_RESP) {
 		if (msg->header.token == 0) {
 			NG_FREE_ITEM(item);
 			return (0);
 		}
 
 		mtx_lock(&ng_btsocket_hci_raw_queue_mtx);
 		if (NG_BT_ITEMQ_FULL(&ng_btsocket_hci_raw_queue)) {
 			NG_BTSOCKET_HCI_RAW_ERR(
 "%s: Input queue is full\n", __func__);
 
 			NG_BT_ITEMQ_DROP(&ng_btsocket_hci_raw_queue);
 			NG_FREE_ITEM(item);
 			error = ENOBUFS;
 		} else {
 			NG_BT_ITEMQ_ENQUEUE(&ng_btsocket_hci_raw_queue, item);
 			error = ng_btsocket_hci_raw_wakeup_input_task();
 		}
 		mtx_unlock(&ng_btsocket_hci_raw_queue_mtx);
 	} else {
 		NG_FREE_ITEM(item);
 		error = EINVAL;
 	}
 
 	return (error);
 } /* ng_btsocket_hci_raw_node_rcvmsg */
 
 /*
  * Receive packet from the one of our hook.
  * Prepend every packet with sockaddr_hci and record sender's node name.
  * Enqueue item and schedule input task.
  */
 
 static int
 ng_btsocket_hci_raw_node_rcvdata(hook_p hook, item_p item)
 {
 	struct mbuf	*nam = NULL;
 	int		 error;
 
 	/*
 	 * Check for empty sockets list creates LOR when both sender and
 	 * receiver device are connected to the same host, so remove it
 	 * for now
 	 */
 
 	MGET(nam, M_NOWAIT, MT_SONAME);
 	if (nam != NULL) {
 		struct sockaddr_hci	*sa = mtod(nam, struct sockaddr_hci *);
 
 		nam->m_len = sizeof(struct sockaddr_hci);
 
 		sa->hci_len = sizeof(*sa);
 		sa->hci_family = AF_BLUETOOTH;
 		strlcpy(sa->hci_node, NG_PEER_NODE_NAME(hook),
 			sizeof(sa->hci_node));
 
 		NGI_GET_M(item, nam->m_next);
 		NGI_M(item) = nam;
 
 		mtx_lock(&ng_btsocket_hci_raw_queue_mtx);
 		if (NG_BT_ITEMQ_FULL(&ng_btsocket_hci_raw_queue)) {
 			NG_BTSOCKET_HCI_RAW_ERR(
 "%s: Input queue is full\n", __func__);
 
 			NG_BT_ITEMQ_DROP(&ng_btsocket_hci_raw_queue);
 			NG_FREE_ITEM(item);
 			error = ENOBUFS;
 		} else {
 			NG_BT_ITEMQ_ENQUEUE(&ng_btsocket_hci_raw_queue, item);
 			error = ng_btsocket_hci_raw_wakeup_input_task();
 		}
 		mtx_unlock(&ng_btsocket_hci_raw_queue_mtx);
 	} else {
 		NG_BTSOCKET_HCI_RAW_ERR(
 "%s: Failed to allocate address mbuf\n", __func__);
 
 		NG_FREE_ITEM(item);
 		error = ENOBUFS;
 	}
 
 	return (error);
 } /* ng_btsocket_hci_raw_node_rcvdata */
 
 /****************************************************************************
  ****************************************************************************
  **                              Sockets specific
  ****************************************************************************
  ****************************************************************************/
 
 /*
  * Get next token. We need token to avoid theoretical race where process
  * submits ioctl() message then interrupts ioctl() and re-submits another
  * ioctl() on the same socket *before* first ioctl() complete.
  */
  
 static void
 ng_btsocket_hci_raw_get_token(u_int32_t *token)
 {
 	mtx_lock(&ng_btsocket_hci_raw_token_mtx);
   
 	if (++ ng_btsocket_hci_raw_token == 0)
 		ng_btsocket_hci_raw_token = 1;
  
 	*token = ng_btsocket_hci_raw_token;
  
 	mtx_unlock(&ng_btsocket_hci_raw_token_mtx);
 } /* ng_btsocket_hci_raw_get_token */
 
 /*
  * Send Netgraph message to the node - do not expect reply
  */
 
 static int
 ng_btsocket_hci_raw_send_ngmsg(char *path, int cmd, void *arg, int arglen)
 {
 	struct ng_mesg	*msg = NULL;
 	int		 error = 0;
 
 	NG_MKMESSAGE(msg, NGM_HCI_COOKIE, cmd, arglen, M_NOWAIT);
 	if (msg == NULL)
 		return (ENOMEM);
 
 	if (arg != NULL && arglen > 0)
 		bcopy(arg, msg->data, arglen);
 
 	NG_SEND_MSG_PATH(error, ng_btsocket_hci_raw_node, msg, path, 0);
 
 	return (error);
 } /* ng_btsocket_hci_raw_send_ngmsg */
 
 /*
  * Send Netgraph message to the node (no data) and wait for reply 
  */
 
 static int
 ng_btsocket_hci_raw_send_sync_ngmsg(ng_btsocket_hci_raw_pcb_p pcb, char *path,
 		int cmd, void *rsp, int rsplen)
 {
 	struct ng_mesg	*msg = NULL;
 	int		 error = 0;
 
 	mtx_assert(&pcb->pcb_mtx, MA_OWNED);
 
 	NG_MKMESSAGE(msg, NGM_HCI_COOKIE, cmd, 0, M_NOWAIT);
 	if (msg == NULL)
 		return (ENOMEM);
 
 	ng_btsocket_hci_raw_get_token(&msg->header.token);
 	pcb->token = msg->header.token;
 	pcb->msg = NULL;
 
 	NG_SEND_MSG_PATH(error, ng_btsocket_hci_raw_node, msg, path, 0);
 	if (error != 0) {
 		pcb->token = 0;
 		return (error);
 	}
 
 	error = msleep(&pcb->msg, &pcb->pcb_mtx, PZERO|PCATCH, "hcictl", 
 			ng_btsocket_hci_raw_ioctl_timeout * hz);
 	pcb->token = 0;
 
 	if (error != 0)
 		return (error);
 
 	if (pcb->msg != NULL && pcb->msg->header.cmd == cmd)
 		bcopy(pcb->msg->data, rsp, rsplen);
 	else
 		error = EINVAL;
 
 	NG_FREE_MSG(pcb->msg); /* checks for != NULL */
 
 	return (0);
 } /* ng_btsocket_hci_raw_send_sync_ngmsg */
 
 /*
  * Create control information for the packet
  */
 
 static void
 ng_btsocket_hci_raw_savctl(ng_btsocket_hci_raw_pcb_p pcb, struct mbuf **ctl,
 		struct mbuf *m) 
 {
 	int		dir;
 	struct timeval	tv;
 
 	mtx_assert(&pcb->pcb_mtx, MA_OWNED);
 
 	if (pcb->flags & NG_BTSOCKET_HCI_RAW_DIRECTION) {
 		dir = (m->m_flags & M_PROTO1)? 1 : 0;
 		*ctl = sbcreatecontrol((caddr_t) &dir, sizeof(dir),
 					SCM_HCI_RAW_DIRECTION, SOL_HCI_RAW);
 		if (*ctl != NULL)
 			ctl = &((*ctl)->m_next);
 	}
 
 	if (pcb->so->so_options & SO_TIMESTAMP) {
 		microtime(&tv);
 		*ctl = sbcreatecontrol((caddr_t) &tv, sizeof(tv),
 					SCM_TIMESTAMP, SOL_SOCKET);
 		if (*ctl != NULL)
 			ctl = &((*ctl)->m_next);
 	}
 } /* ng_btsocket_hci_raw_savctl */
 
 /*
  * Raw HCI sockets data input routine
  */
 
 static void
 ng_btsocket_hci_raw_data_input(struct mbuf *nam)
 {
 	ng_btsocket_hci_raw_pcb_p	 pcb = NULL;
 	struct mbuf			*m0 = NULL, *m = NULL;
 	struct sockaddr_hci		*sa = NULL;
 
 	m0 = nam->m_next;
 	nam->m_next = NULL;
 
 	KASSERT((nam->m_type == MT_SONAME),
 		("%s: m_type=%d\n", __func__, nam->m_type));
 	KASSERT((m0->m_flags & M_PKTHDR),
 		("%s: m_flags=%#x\n", __func__, m0->m_flags));
 
 	sa = mtod(nam, struct sockaddr_hci *);
 
 	mtx_lock(&ng_btsocket_hci_raw_sockets_mtx);
 
 	LIST_FOREACH(pcb, &ng_btsocket_hci_raw_sockets, next) {
 
 		mtx_lock(&pcb->pcb_mtx);
 
 		/*
 		 * If socket was bound then check address and
 		 *  make sure it matches.
 		 */
 
 		if (pcb->addr.hci_node[0] != 0 &&
 		    strcmp(sa->hci_node, pcb->addr.hci_node) != 0)
 			goto next;
 
 		/*
 		 * Check packet against filters
 		 * XXX do we have to call m_pullup() here?
 		 */
 
 		if (ng_btsocket_hci_raw_filter(pcb, m0, 1) != 0)
 			goto next;
 
 		/*
 		 * Make a copy of the packet, append to the socket's
 		 * receive queue and wakeup socket. sbappendaddr()
 		 * will check if socket has enough buffer space.
 		 */
 
 		m = m_dup(m0, M_NOWAIT);
 		if (m != NULL) {
 			struct mbuf	*ctl = NULL;
 
 			ng_btsocket_hci_raw_savctl(pcb, &ctl, m);
 
 			if (sbappendaddr(&pcb->so->so_rcv, 
 					(struct sockaddr *) sa, m, ctl))
 				sorwakeup(pcb->so);
 			else {
 				NG_BTSOCKET_HCI_RAW_INFO(
 "%s: sbappendaddr() failed\n", __func__);
 
 				NG_FREE_M(m);
 				NG_FREE_M(ctl);
 			}
 		}
 next:
 		mtx_unlock(&pcb->pcb_mtx);
 	}
 
 	mtx_unlock(&ng_btsocket_hci_raw_sockets_mtx);
 
 	NG_FREE_M(nam);
 	NG_FREE_M(m0);
 } /* ng_btsocket_hci_raw_data_input */ 
 
 /*
  * Raw HCI sockets message input routine
  */
 
 static void
 ng_btsocket_hci_raw_msg_input(struct ng_mesg *msg)
 {
 	ng_btsocket_hci_raw_pcb_p	pcb = NULL;
 
 	mtx_lock(&ng_btsocket_hci_raw_sockets_mtx);
 
 	LIST_FOREACH(pcb, &ng_btsocket_hci_raw_sockets, next) {
 		mtx_lock(&pcb->pcb_mtx);
 
 		if (msg->header.token == pcb->token) {
 			pcb->msg = msg;
 			wakeup(&pcb->msg);
 
 			mtx_unlock(&pcb->pcb_mtx);
 			mtx_unlock(&ng_btsocket_hci_raw_sockets_mtx);
 
 			return;
 		}
 
 		mtx_unlock(&pcb->pcb_mtx);
 	}
 
 	mtx_unlock(&ng_btsocket_hci_raw_sockets_mtx);
 
 	NG_FREE_MSG(msg); /* checks for != NULL */
 } /* ng_btsocket_hci_raw_msg_input */
 
 /*
  * Raw HCI sockets input routines
  */
 
 static void
 ng_btsocket_hci_raw_input(void *context, int pending)
 {
 	item_p	item = NULL;
 
 	for (;;) {
 		mtx_lock(&ng_btsocket_hci_raw_queue_mtx);
 		NG_BT_ITEMQ_DEQUEUE(&ng_btsocket_hci_raw_queue, item);
 		mtx_unlock(&ng_btsocket_hci_raw_queue_mtx);
 
 		if (item == NULL)
 			break;
 
 		switch(item->el_flags & NGQF_TYPE) {
 		case NGQF_DATA: {
 			struct mbuf	*m = NULL;
 
 			NGI_GET_M(item, m);
 			ng_btsocket_hci_raw_data_input(m);
 			} break;
 
 		case NGQF_MESG: {
 			struct ng_mesg	*msg = NULL;
 
 			NGI_GET_MSG(item, msg);
 			ng_btsocket_hci_raw_msg_input(msg);
 			} break;
 
 		default:
 			KASSERT(0, 
 ("%s: invalid item type=%ld\n", __func__, (item->el_flags & NGQF_TYPE)));
 			break;
 		}
 
 		NG_FREE_ITEM(item);
 	}
 } /* ng_btsocket_hci_raw_input */
 
 /*
  * Raw HCI sockets output routine
  */
 
 static void
 ng_btsocket_hci_raw_output(node_p node, hook_p hook, void *arg1, int arg2)
 {
 	struct mbuf		*nam = (struct mbuf *) arg1, *m = NULL;
 	struct sockaddr_hci	*sa = NULL;
 	int			 error;
 
 	m = nam->m_next;
 	nam->m_next = NULL;
 
 	KASSERT((nam->m_type == MT_SONAME),
 		("%s: m_type=%d\n", __func__, nam->m_type));
 	KASSERT((m->m_flags & M_PKTHDR),
 		("%s: m_flags=%#x\n", __func__, m->m_flags));
 
 	sa = mtod(nam, struct sockaddr_hci *);
 
 	/*
 	 * Find downstream hook
 	 * XXX For now access node hook list directly. Should be safe because
 	 * we used ng_send_fn() and we should have exclusive lock on the node.
 	 */
 
 	LIST_FOREACH(hook, &node->nd_hooks, hk_hooks) {
 		if (hook == NULL || NG_HOOK_NOT_VALID(hook) || 
 		    NG_NODE_NOT_VALID(NG_PEER_NODE(hook)))
 			continue;
 
 		if (strcmp(sa->hci_node, NG_PEER_NODE_NAME(hook)) == 0) {
 			NG_SEND_DATA_ONLY(error, hook, m); /* sets m to NULL */
 			break;
 		}
 	}
 
 	NG_FREE_M(nam); /* check for != NULL */
 	NG_FREE_M(m);
 } /* ng_btsocket_hci_raw_output */
 
 /*
  * Check frame against security and socket filters. 
  * d (direction bit) == 1 means incoming frame.
  */
 
 static int
 ng_btsocket_hci_raw_filter(ng_btsocket_hci_raw_pcb_p pcb, struct mbuf *m, int d)
 {
 	int	type, event, opcode;
 
 	mtx_assert(&pcb->pcb_mtx, MA_OWNED);
 
 	switch ((type = *mtod(m, u_int8_t *))) {
 	case NG_HCI_CMD_PKT:
 		if (!(pcb->flags & NG_BTSOCKET_HCI_RAW_PRIVILEGED)) {
 			opcode = le16toh(mtod(m, ng_hci_cmd_pkt_t *)->opcode);
 		
 			if (!bit_test(
 ng_btsocket_hci_raw_sec_filter->commands[NG_HCI_OGF(opcode) - 1],
 NG_HCI_OCF(opcode) - 1))
 				return (EPERM);
 		}
 
 		if (d && !bit_test(pcb->filter.packet_mask, NG_HCI_CMD_PKT - 1))
 			return (EPERM);
 		break;
 
 	case NG_HCI_ACL_DATA_PKT:
 	case NG_HCI_SCO_DATA_PKT:
 		if (!(pcb->flags & NG_BTSOCKET_HCI_RAW_PRIVILEGED) ||
 		    !bit_test(pcb->filter.packet_mask, type - 1) ||
 		    !d)
 			return (EPERM);
 		break;
 
 	case NG_HCI_EVENT_PKT:
 		if (!d)
 			return (EINVAL);
 
 		event = mtod(m, ng_hci_event_pkt_t *)->event - 1;
 
 		if (!(pcb->flags & NG_BTSOCKET_HCI_RAW_PRIVILEGED))
 			if (!bit_test(ng_btsocket_hci_raw_sec_filter->events, event))
 				return (EPERM);
 
 		if (!bit_test(pcb->filter.event_mask, event))
 			return (EPERM);
 		break;
 
 	default:
 		return (EINVAL);
 	}
 
 	return (0);
 } /* ng_btsocket_hci_raw_filter */
 
 /*
  * Initialize everything
  */
 
 void
 ng_btsocket_hci_raw_init(void)
 {
 	bitstr_t	*f = NULL;
 	int		 error = 0;
+
+	/* Skip initialization of globals for non-default instances. */
+	if (!IS_DEFAULT_VNET(curvnet))
+		return;
 
 	ng_btsocket_hci_raw_node = NULL;
 	ng_btsocket_hci_raw_debug_level = NG_BTSOCKET_WARN_LEVEL;
 	ng_btsocket_hci_raw_ioctl_timeout = 5;
 
 	/* Register Netgraph node type */
 	error = ng_newtype(&typestruct);
 	if (error != 0) {
 		NG_BTSOCKET_HCI_RAW_ALERT(
 "%s: Could not register Netgraph node type, error=%d\n", __func__, error);
 
 		return;
 	}
 
 	/* Create Netgrapg node */
 	error = ng_make_node_common(&typestruct, &ng_btsocket_hci_raw_node);
 	if (error != 0) {
 		NG_BTSOCKET_HCI_RAW_ALERT(
 "%s: Could not create Netgraph node, error=%d\n", __func__, error);
 
 		ng_btsocket_hci_raw_node = NULL;
 
 		return;
         }
 
 	error = ng_name_node(ng_btsocket_hci_raw_node,
 				NG_BTSOCKET_HCI_RAW_NODE_TYPE);
 	if (error != 0) {
 		NG_BTSOCKET_HCI_RAW_ALERT(
 "%s: Could not name Netgraph node, error=%d\n", __func__, error);
 
 		NG_NODE_UNREF(ng_btsocket_hci_raw_node);
 		ng_btsocket_hci_raw_node = NULL;
 
 		return;
 	}
 
 	/* Create input queue */
 	NG_BT_ITEMQ_INIT(&ng_btsocket_hci_raw_queue, 300);
 	mtx_init(&ng_btsocket_hci_raw_queue_mtx,
 		"btsocks_hci_raw_queue_mtx", NULL, MTX_DEF);
 	TASK_INIT(&ng_btsocket_hci_raw_task, 0,
 		ng_btsocket_hci_raw_input, NULL);
 
 	/* Create list of sockets */
 	LIST_INIT(&ng_btsocket_hci_raw_sockets);
 	mtx_init(&ng_btsocket_hci_raw_sockets_mtx,
 		"btsocks_hci_raw_sockets_mtx", NULL, MTX_DEF);
 
 	/* Tokens */
 	ng_btsocket_hci_raw_token = 0;
 	mtx_init(&ng_btsocket_hci_raw_token_mtx,
 		"btsocks_hci_raw_token_mtx", NULL, MTX_DEF);
 
 	/* 
 	 * Security filter
 	 * XXX never free()ed
 	 */
 	ng_btsocket_hci_raw_sec_filter =
 	    malloc(sizeof(struct ng_btsocket_hci_raw_sec_filter), 
 		M_NETGRAPH_BTSOCKET_HCI_RAW, M_NOWAIT|M_ZERO);
 	if (ng_btsocket_hci_raw_sec_filter == NULL) {
 		printf("%s: Could not allocate security filter!\n", __func__);
 		return;
 	}
 
 	/*
 	 * XXX How paranoid can we get? 
 	 *
 	 * Initialize security filter. If bit is set in the mask then
 	 * unprivileged socket is allowed to send (receive) this command
 	 * (event).
 	 */
 
 	/* Enable all events */
 	memset(&ng_btsocket_hci_raw_sec_filter->events, 0xff,
 		sizeof(ng_btsocket_hci_raw_sec_filter->events)/
 			sizeof(ng_btsocket_hci_raw_sec_filter->events[0]));
 
 	/* Disable some critical events */
 	f = ng_btsocket_hci_raw_sec_filter->events;
 	bit_clear(f, NG_HCI_EVENT_RETURN_LINK_KEYS - 1);
 	bit_clear(f, NG_HCI_EVENT_LINK_KEY_NOTIFICATION - 1);
 	bit_clear(f, NG_HCI_EVENT_VENDOR - 1);
 
 	/* Commands - Link control */
 	f = ng_btsocket_hci_raw_sec_filter->commands[NG_HCI_OGF_LINK_CONTROL-1];
 	bit_set(f, NG_HCI_OCF_INQUIRY - 1);
 	bit_set(f, NG_HCI_OCF_INQUIRY_CANCEL - 1);
 	bit_set(f, NG_HCI_OCF_PERIODIC_INQUIRY - 1);
 	bit_set(f, NG_HCI_OCF_EXIT_PERIODIC_INQUIRY - 1);
 	bit_set(f, NG_HCI_OCF_REMOTE_NAME_REQ - 1);
 	bit_set(f, NG_HCI_OCF_READ_REMOTE_FEATURES - 1);
 	bit_set(f, NG_HCI_OCF_READ_REMOTE_VER_INFO - 1);
 	bit_set(f, NG_HCI_OCF_READ_CLOCK_OFFSET - 1);
 
 	/* Commands - Link policy */
 	f = ng_btsocket_hci_raw_sec_filter->commands[NG_HCI_OGF_LINK_POLICY-1];
 	bit_set(f, NG_HCI_OCF_ROLE_DISCOVERY - 1);
 	bit_set(f, NG_HCI_OCF_READ_LINK_POLICY_SETTINGS - 1);
 
 	/* Commands - Host controller and baseband */
 	f = ng_btsocket_hci_raw_sec_filter->commands[NG_HCI_OGF_HC_BASEBAND-1];
 	bit_set(f, NG_HCI_OCF_READ_PIN_TYPE - 1);
 	bit_set(f, NG_HCI_OCF_READ_LOCAL_NAME - 1);
 	bit_set(f, NG_HCI_OCF_READ_CON_ACCEPT_TIMO - 1);
 	bit_set(f, NG_HCI_OCF_READ_PAGE_TIMO - 1);
 	bit_set(f, NG_HCI_OCF_READ_SCAN_ENABLE - 1);
 	bit_set(f, NG_HCI_OCF_READ_PAGE_SCAN_ACTIVITY - 1);
 	bit_set(f, NG_HCI_OCF_READ_INQUIRY_SCAN_ACTIVITY - 1);
 	bit_set(f, NG_HCI_OCF_READ_AUTH_ENABLE - 1);
 	bit_set(f, NG_HCI_OCF_READ_ENCRYPTION_MODE - 1);
 	bit_set(f, NG_HCI_OCF_READ_UNIT_CLASS - 1);
 	bit_set(f, NG_HCI_OCF_READ_VOICE_SETTINGS - 1);
 	bit_set(f, NG_HCI_OCF_READ_AUTO_FLUSH_TIMO - 1);
 	bit_set(f, NG_HCI_OCF_READ_NUM_BROADCAST_RETRANS - 1);
 	bit_set(f, NG_HCI_OCF_READ_HOLD_MODE_ACTIVITY - 1);
 	bit_set(f, NG_HCI_OCF_READ_XMIT_LEVEL - 1);
 	bit_set(f, NG_HCI_OCF_READ_SCO_FLOW_CONTROL - 1);
 	bit_set(f, NG_HCI_OCF_READ_LINK_SUPERVISION_TIMO - 1);
 	bit_set(f, NG_HCI_OCF_READ_SUPPORTED_IAC_NUM - 1);
 	bit_set(f, NG_HCI_OCF_READ_IAC_LAP - 1);
 	bit_set(f, NG_HCI_OCF_READ_PAGE_SCAN_PERIOD - 1);
 	bit_set(f, NG_HCI_OCF_READ_PAGE_SCAN - 1);
 
 	/* Commands - Informational */
 	f = ng_btsocket_hci_raw_sec_filter->commands[NG_HCI_OGF_INFO - 1];
 	bit_set(f, NG_HCI_OCF_READ_LOCAL_VER - 1);
 	bit_set(f, NG_HCI_OCF_READ_LOCAL_FEATURES - 1);
 	bit_set(f, NG_HCI_OCF_READ_BUFFER_SIZE - 1);
 	bit_set(f, NG_HCI_OCF_READ_COUNTRY_CODE - 1);
 	bit_set(f, NG_HCI_OCF_READ_BDADDR - 1);
 
 	/* Commands - Status */
 	f = ng_btsocket_hci_raw_sec_filter->commands[NG_HCI_OGF_STATUS - 1];
 	bit_set(f, NG_HCI_OCF_READ_FAILED_CONTACT_CNTR - 1);
 	bit_set(f, NG_HCI_OCF_GET_LINK_QUALITY - 1);
 	bit_set(f, NG_HCI_OCF_READ_RSSI - 1);
 
 	/* Commands - Testing */
 	f = ng_btsocket_hci_raw_sec_filter->commands[NG_HCI_OGF_TESTING - 1];
 	bit_set(f, NG_HCI_OCF_READ_LOOPBACK_MODE - 1);
 } /* ng_btsocket_hci_raw_init */
 
 /*
  * Abort connection on socket
  */
 
 void
 ng_btsocket_hci_raw_abort(struct socket *so)
 {
 } /* ng_btsocket_hci_raw_abort */
 
 void
 ng_btsocket_hci_raw_close(struct socket *so)
 {
 } /* ng_btsocket_hci_raw_close */
 
 /*
  * Create new raw HCI socket
  */
 
 int
 ng_btsocket_hci_raw_attach(struct socket *so, int proto, struct thread *td)
 {
 	ng_btsocket_hci_raw_pcb_p	pcb = so2hci_raw_pcb(so);
 	int				error = 0;
 
 	if (pcb != NULL)
 		return (EISCONN);
 
 	if (ng_btsocket_hci_raw_node == NULL)
 		return (EPROTONOSUPPORT);
 	if (proto != BLUETOOTH_PROTO_HCI)
 		return (EPROTONOSUPPORT);
 	if (so->so_type != SOCK_RAW)
 		return (ESOCKTNOSUPPORT);
 
 	error = soreserve(so, NG_BTSOCKET_HCI_RAW_SENDSPACE,
 				NG_BTSOCKET_HCI_RAW_RECVSPACE);
 	if (error != 0)
 		return (error);
 
 	pcb = malloc(sizeof(*pcb), 
 		M_NETGRAPH_BTSOCKET_HCI_RAW, M_NOWAIT|M_ZERO);
 	if (pcb == NULL)
 		return (ENOMEM);
 
 	so->so_pcb = (caddr_t) pcb;
 	pcb->so = so;
 
 	if (priv_check(td, PRIV_NETBLUETOOTH_RAW) == 0)
 		pcb->flags |= NG_BTSOCKET_HCI_RAW_PRIVILEGED;
 
 	/*
 	 * Set default socket filter. By default socket only accepts HCI
 	 * Command_Complete and Command_Status event packets.
 	 */
 
 	bit_set(pcb->filter.event_mask, NG_HCI_EVENT_COMMAND_COMPL - 1);
 	bit_set(pcb->filter.event_mask, NG_HCI_EVENT_COMMAND_STATUS - 1);
 
 	mtx_init(&pcb->pcb_mtx, "btsocks_hci_raw_pcb_mtx", NULL, MTX_DEF);
 
 	mtx_lock(&ng_btsocket_hci_raw_sockets_mtx);
 	LIST_INSERT_HEAD(&ng_btsocket_hci_raw_sockets, pcb, next);
 	mtx_unlock(&ng_btsocket_hci_raw_sockets_mtx);
 
 	return (0);
 } /* ng_btsocket_hci_raw_attach */
 
 /*
  * Bind raw HCI socket
  */
 
 int
 ng_btsocket_hci_raw_bind(struct socket *so, struct sockaddr *nam,
 		struct thread *td)
 {
 	ng_btsocket_hci_raw_pcb_p	 pcb = so2hci_raw_pcb(so);
 	struct sockaddr_hci		*sa = (struct sockaddr_hci *) nam;
 
 	if (pcb == NULL)
 		return (EINVAL);
 	if (ng_btsocket_hci_raw_node == NULL)
 		return (EINVAL);
 
 	if (sa == NULL)
 		return (EINVAL);
 	if (sa->hci_family != AF_BLUETOOTH)
 		return (EAFNOSUPPORT);
 	if (sa->hci_len != sizeof(*sa))
 		return (EINVAL);
 	if (sa->hci_node[0] == 0)
 		return (EINVAL);
 
 	mtx_lock(&pcb->pcb_mtx);
 	bcopy(sa, &pcb->addr, sizeof(pcb->addr));
 	mtx_unlock(&pcb->pcb_mtx);
 
 	return (0);
 } /* ng_btsocket_hci_raw_bind */
 
 /*
  * Connect raw HCI socket
  */
 
 int
 ng_btsocket_hci_raw_connect(struct socket *so, struct sockaddr *nam,
 		struct thread *td)
 {
 	ng_btsocket_hci_raw_pcb_p	 pcb = so2hci_raw_pcb(so);
 	struct sockaddr_hci		*sa = (struct sockaddr_hci *) nam;
 
 	if (pcb == NULL)
 		return (EINVAL);
 	if (ng_btsocket_hci_raw_node == NULL)
 		return (EINVAL);
 
 	if (sa == NULL)
 		return (EINVAL);
 	if (sa->hci_family != AF_BLUETOOTH)
 		return (EAFNOSUPPORT);
 	if (sa->hci_len != sizeof(*sa))
 		return (EINVAL);
 	if (sa->hci_node[0] == 0)
 		return (EDESTADDRREQ);
 
 	mtx_lock(&pcb->pcb_mtx);
 
 	if (bcmp(sa, &pcb->addr, sizeof(pcb->addr)) != 0) {
 		mtx_unlock(&pcb->pcb_mtx);
 		return (EADDRNOTAVAIL);
 	}
 
 	soisconnected(so);
 
 	mtx_unlock(&pcb->pcb_mtx);
 
 	return (0);
 } /* ng_btsocket_hci_raw_connect */
 
 /*
  * Process ioctl on socket
  */
 
 int
 ng_btsocket_hci_raw_control(struct socket *so, u_long cmd, caddr_t data,
 		struct ifnet *ifp, struct thread *td)
 {
 	ng_btsocket_hci_raw_pcb_p	 pcb = so2hci_raw_pcb(so);
 	char				 path[NG_NODESIZ + 1];
 	struct ng_mesg			*msg = NULL;
 	int				 error = 0;
 
 	if (pcb == NULL)
 		return (EINVAL);
 	if (ng_btsocket_hci_raw_node == NULL)
 		return (EINVAL);
 
 	mtx_lock(&pcb->pcb_mtx);
 
 	/* Check if we have device name */
 	if (pcb->addr.hci_node[0] == 0) {
 		mtx_unlock(&pcb->pcb_mtx);
 		return (EHOSTUNREACH);
 	}
 
 	/* Check if we have pending ioctl() */
 	if (pcb->token != 0) {
 		mtx_unlock(&pcb->pcb_mtx);
 		return (EBUSY);
 	}
 
 	snprintf(path, sizeof(path), "%s:", pcb->addr.hci_node);
 
 	switch (cmd) {
 	case SIOC_HCI_RAW_NODE_GET_STATE: {
 		struct ng_btsocket_hci_raw_node_state	*p =
 			(struct ng_btsocket_hci_raw_node_state *) data;
 
 		error = ng_btsocket_hci_raw_send_sync_ngmsg(pcb, path, 
 				NGM_HCI_NODE_GET_STATE,
 				&p->state, sizeof(p->state));
 		} break;
 
 	case SIOC_HCI_RAW_NODE_INIT:
 		if (pcb->flags & NG_BTSOCKET_HCI_RAW_PRIVILEGED)
 			error = ng_btsocket_hci_raw_send_ngmsg(path,
 					NGM_HCI_NODE_INIT, NULL, 0);
 		else
 			error = EPERM;
 		break;
 
 	case SIOC_HCI_RAW_NODE_GET_DEBUG: {
 		struct ng_btsocket_hci_raw_node_debug	*p = 
 			(struct ng_btsocket_hci_raw_node_debug *) data;
 
 		error = ng_btsocket_hci_raw_send_sync_ngmsg(pcb, path,
 				NGM_HCI_NODE_GET_DEBUG,
 				&p->debug, sizeof(p->debug));
 		} break;
 
 	case SIOC_HCI_RAW_NODE_SET_DEBUG: {
 		struct ng_btsocket_hci_raw_node_debug	*p = 
 			(struct ng_btsocket_hci_raw_node_debug *) data;
 
 		if (pcb->flags & NG_BTSOCKET_HCI_RAW_PRIVILEGED)
 			error = ng_btsocket_hci_raw_send_ngmsg(path,
 					NGM_HCI_NODE_SET_DEBUG, &p->debug,
 					sizeof(p->debug));
 		else
 			error = EPERM;
 		} break;
 
 	case SIOC_HCI_RAW_NODE_GET_BUFFER: {
 		struct ng_btsocket_hci_raw_node_buffer	*p = 
 			(struct ng_btsocket_hci_raw_node_buffer *) data;
 
 		error = ng_btsocket_hci_raw_send_sync_ngmsg(pcb, path,
 				NGM_HCI_NODE_GET_BUFFER,
 				&p->buffer, sizeof(p->buffer));
 		} break;
 
 	case SIOC_HCI_RAW_NODE_GET_BDADDR: {
 		struct ng_btsocket_hci_raw_node_bdaddr	*p = 
 			(struct ng_btsocket_hci_raw_node_bdaddr *) data;
 
 		error = ng_btsocket_hci_raw_send_sync_ngmsg(pcb, path,
 				NGM_HCI_NODE_GET_BDADDR,
 				&p->bdaddr, sizeof(p->bdaddr));
 		} break;
 
 	case SIOC_HCI_RAW_NODE_GET_FEATURES: {
 		struct ng_btsocket_hci_raw_node_features	*p = 
 			(struct ng_btsocket_hci_raw_node_features *) data;
 
 		error = ng_btsocket_hci_raw_send_sync_ngmsg(pcb, path,
 				NGM_HCI_NODE_GET_FEATURES,
 				&p->features, sizeof(p->features));
 		} break;
 
 	case SIOC_HCI_RAW_NODE_GET_STAT: {
 		struct ng_btsocket_hci_raw_node_stat	*p = 
 			(struct ng_btsocket_hci_raw_node_stat *) data;
 
 		error = ng_btsocket_hci_raw_send_sync_ngmsg(pcb, path,
 				NGM_HCI_NODE_GET_STAT,
 				&p->stat, sizeof(p->stat));
 		} break;
 
 	case SIOC_HCI_RAW_NODE_RESET_STAT:
 		if (pcb->flags & NG_BTSOCKET_HCI_RAW_PRIVILEGED)
 			error = ng_btsocket_hci_raw_send_ngmsg(path,
 					NGM_HCI_NODE_RESET_STAT, NULL, 0);
 		else
 			error = EPERM;
 		break;
 
 	case SIOC_HCI_RAW_NODE_FLUSH_NEIGHBOR_CACHE:
 		if (pcb->flags & NG_BTSOCKET_HCI_RAW_PRIVILEGED)
 			error = ng_btsocket_hci_raw_send_ngmsg(path,
 					NGM_HCI_NODE_FLUSH_NEIGHBOR_CACHE,
 					NULL, 0);
 		else
 			error = EPERM;
 		break;
 
 	case SIOC_HCI_RAW_NODE_GET_NEIGHBOR_CACHE:  {
 		struct ng_btsocket_hci_raw_node_neighbor_cache	*p = 
 			(struct ng_btsocket_hci_raw_node_neighbor_cache *) data;
 		ng_hci_node_get_neighbor_cache_ep		*p1 = NULL;
 		ng_hci_node_neighbor_cache_entry_ep		*p2 = NULL;
 
 		if (p->num_entries <= 0 || 
 		    p->num_entries > NG_HCI_MAX_NEIGHBOR_NUM ||
 		    p->entries == NULL) {
 			error = EINVAL;
 			break;
 		}
 
 		NG_MKMESSAGE(msg, NGM_HCI_COOKIE,
 			NGM_HCI_NODE_GET_NEIGHBOR_CACHE, 0, M_NOWAIT);
 		if (msg == NULL) {
 			error = ENOMEM;
 			break;
 		}
 		ng_btsocket_hci_raw_get_token(&msg->header.token);
 		pcb->token = msg->header.token;
 		pcb->msg = NULL;
 
 		NG_SEND_MSG_PATH(error, ng_btsocket_hci_raw_node, msg, path, 0);
 		if (error != 0) {
 			pcb->token = 0;
 			break;
 		}
 
 		error = msleep(&pcb->msg, &pcb->pcb_mtx,
 				PZERO|PCATCH, "hcictl", 
 				ng_btsocket_hci_raw_ioctl_timeout * hz);
 		pcb->token = 0;
 
 		if (error != 0)
 			break;
 
 		if (pcb->msg != NULL &&
 		    pcb->msg->header.cmd == NGM_HCI_NODE_GET_NEIGHBOR_CACHE) {
 			/* Return data back to user space */
 			p1 = (ng_hci_node_get_neighbor_cache_ep *)
 				(pcb->msg->data);
 			p2 = (ng_hci_node_neighbor_cache_entry_ep *)
 				(p1 + 1);
 
 			p->num_entries = min(p->num_entries, p1->num_entries);
 			if (p->num_entries > 0)
 				error = copyout((caddr_t) p2, 
 						(caddr_t) p->entries,
 						p->num_entries * sizeof(*p2));
 		} else
 			error = EINVAL;
 
 		NG_FREE_MSG(pcb->msg); /* checks for != NULL */
 		}break;
 
 	case SIOC_HCI_RAW_NODE_GET_CON_LIST: {
 		struct ng_btsocket_hci_raw_con_list	*p = 
 			(struct ng_btsocket_hci_raw_con_list *) data;
 		ng_hci_node_con_list_ep			*p1 = NULL;
 		ng_hci_node_con_ep			*p2 = NULL;
 
 		if (p->num_connections == 0 ||
 		    p->num_connections > NG_HCI_MAX_CON_NUM ||
 		    p->connections == NULL) {
 			error = EINVAL;
 			break;
 		}
 
 		NG_MKMESSAGE(msg, NGM_HCI_COOKIE, NGM_HCI_NODE_GET_CON_LIST,
 			0, M_NOWAIT);
 		if (msg == NULL) {
 			error = ENOMEM;
 			break;
 		}
 		ng_btsocket_hci_raw_get_token(&msg->header.token);
 		pcb->token = msg->header.token;
 		pcb->msg = NULL;
 
 		NG_SEND_MSG_PATH(error, ng_btsocket_hci_raw_node, msg, path, 0);
 		if (error != 0) {
 			pcb->token = 0;
 			break;
 		}
 
 		error = msleep(&pcb->msg, &pcb->pcb_mtx,
 				PZERO|PCATCH, "hcictl", 
 				ng_btsocket_hci_raw_ioctl_timeout * hz);
 		pcb->token = 0;
 
 		if (error != 0)
 			break;
 
 		if (pcb->msg != NULL &&
 		    pcb->msg->header.cmd == NGM_HCI_NODE_GET_CON_LIST) {
 			/* Return data back to user space */
 			p1 = (ng_hci_node_con_list_ep *)(pcb->msg->data);
 			p2 = (ng_hci_node_con_ep *)(p1 + 1);
 
 			p->num_connections = min(p->num_connections,
 						p1->num_connections);
 			if (p->num_connections > 0)
 				error = copyout((caddr_t) p2, 
 					(caddr_t) p->connections,
 					p->num_connections * sizeof(*p2));
 		} else
 			error = EINVAL;
 
 		NG_FREE_MSG(pcb->msg); /* checks for != NULL */
 		} break;
 
 	case SIOC_HCI_RAW_NODE_GET_LINK_POLICY_MASK: {
 		struct ng_btsocket_hci_raw_node_link_policy_mask	*p = 
 			(struct ng_btsocket_hci_raw_node_link_policy_mask *) 
 				data;
 
 		error = ng_btsocket_hci_raw_send_sync_ngmsg(pcb, path,
 				NGM_HCI_NODE_GET_LINK_POLICY_SETTINGS_MASK,
 				&p->policy_mask, sizeof(p->policy_mask));
 		} break;
 
 	case SIOC_HCI_RAW_NODE_SET_LINK_POLICY_MASK: {
 		struct ng_btsocket_hci_raw_node_link_policy_mask	*p = 
 			(struct ng_btsocket_hci_raw_node_link_policy_mask *) 
 				data;
 
 		if (pcb->flags & NG_BTSOCKET_HCI_RAW_PRIVILEGED)
 			error = ng_btsocket_hci_raw_send_ngmsg(path,
 					NGM_HCI_NODE_SET_LINK_POLICY_SETTINGS_MASK,
 					&p->policy_mask,
 					sizeof(p->policy_mask));
 		else
 			error = EPERM;
 		} break;
 
 	case SIOC_HCI_RAW_NODE_GET_PACKET_MASK: {
 		struct ng_btsocket_hci_raw_node_packet_mask	*p = 
 			(struct ng_btsocket_hci_raw_node_packet_mask *) data;
 
 		error = ng_btsocket_hci_raw_send_sync_ngmsg(pcb, path,
 				NGM_HCI_NODE_GET_PACKET_MASK,
 				&p->packet_mask, sizeof(p->packet_mask));
 		} break;
 
 	case SIOC_HCI_RAW_NODE_SET_PACKET_MASK: {
 		struct ng_btsocket_hci_raw_node_packet_mask	*p = 
 			(struct ng_btsocket_hci_raw_node_packet_mask *) data;
 
 		if (pcb->flags & NG_BTSOCKET_HCI_RAW_PRIVILEGED)
 			error = ng_btsocket_hci_raw_send_ngmsg(path,
 					NGM_HCI_NODE_SET_PACKET_MASK,
 					&p->packet_mask,
 					sizeof(p->packet_mask));
 		else
 			error = EPERM;
 		} break;
 
 	case SIOC_HCI_RAW_NODE_GET_ROLE_SWITCH: {
 		struct ng_btsocket_hci_raw_node_role_switch	*p = 
 			(struct ng_btsocket_hci_raw_node_role_switch *) data;
 
 		error = ng_btsocket_hci_raw_send_sync_ngmsg(pcb, path,
 				NGM_HCI_NODE_GET_ROLE_SWITCH,
 				&p->role_switch, sizeof(p->role_switch));
 		} break;
 
 	case SIOC_HCI_RAW_NODE_SET_ROLE_SWITCH: {
 		struct ng_btsocket_hci_raw_node_role_switch	*p = 
 			(struct ng_btsocket_hci_raw_node_role_switch *) data;
 
 		if (pcb->flags & NG_BTSOCKET_HCI_RAW_PRIVILEGED)
 			error = ng_btsocket_hci_raw_send_ngmsg(path,
 					NGM_HCI_NODE_SET_ROLE_SWITCH,
 					&p->role_switch,
 					sizeof(p->role_switch));
 		else
 			error = EPERM;
 		} break;
 
 	case SIOC_HCI_RAW_NODE_LIST_NAMES: {
 		struct ng_btsocket_hci_raw_node_list_names	*nl =
 			(struct ng_btsocket_hci_raw_node_list_names *) data;
 		struct nodeinfo					*ni = nl->names;
 
 		if (nl->num_names == 0) {
 			error = EINVAL;
 			break;
 		}
 
 		NG_MKMESSAGE(msg, NGM_GENERIC_COOKIE, NGM_LISTNAMES,
 			0, M_NOWAIT);
 		if (msg == NULL) {
 			error = ENOMEM;
 			break;
 		}
 		ng_btsocket_hci_raw_get_token(&msg->header.token);
 		pcb->token = msg->header.token;
 		pcb->msg = NULL;
 
 		NG_SEND_MSG_PATH(error, ng_btsocket_hci_raw_node, msg, ".:", 0);
 		if (error != 0) {
 			pcb->token = 0;
 			break;
 		}
 
 		error = msleep(&pcb->msg, &pcb->pcb_mtx,
 				PZERO|PCATCH, "hcictl",
 				ng_btsocket_hci_raw_ioctl_timeout * hz);
 		pcb->token = 0;
 
 		if (error != 0)
 			break;
 
 		if (pcb->msg != NULL && pcb->msg->header.cmd == NGM_LISTNAMES) {
 			/* Return data back to user space */
 			struct namelist	*nl1 = (struct namelist *) pcb->msg->data;
 			struct nodeinfo	*ni1 = &nl1->nodeinfo[0];
 
 			while (nl->num_names > 0 && nl1->numnames > 0) {
 				if (strcmp(ni1->type, NG_HCI_NODE_TYPE) == 0) {
 					error = copyout((caddr_t) ni1,
 							(caddr_t) ni,
 							sizeof(*ni));
 					if (error != 0)
 						break;
 
 					nl->num_names --;
 					ni ++;
 				}
 
 				nl1->numnames --;
 				ni1 ++;
 			}
 
 			nl->num_names = ni - nl->names;
 		} else
 			error = EINVAL;
 
 		NG_FREE_MSG(pcb->msg); /* checks for != NULL */
 		} break;
 
 	default:
 		error = EINVAL;
 		break;
 	}
 
 	mtx_unlock(&pcb->pcb_mtx);
 
 	return (error);
 } /* ng_btsocket_hci_raw_control */
 
 /*
  * Process getsockopt/setsockopt system calls
  */
 
 int
 ng_btsocket_hci_raw_ctloutput(struct socket *so, struct sockopt *sopt)
 {
 	ng_btsocket_hci_raw_pcb_p		pcb = so2hci_raw_pcb(so);
 	struct ng_btsocket_hci_raw_filter	filter;
 	int					error = 0, dir;
 
 	if (pcb == NULL)
 		return (EINVAL);
 	if (ng_btsocket_hci_raw_node == NULL)
 		return (EINVAL);
 
 	if (sopt->sopt_level != SOL_HCI_RAW)
 		return (0);
 
 	mtx_lock(&pcb->pcb_mtx);
 
 	switch (sopt->sopt_dir) {
 	case SOPT_GET:
 		switch (sopt->sopt_name) {
 		case SO_HCI_RAW_FILTER:
 			error = sooptcopyout(sopt, &pcb->filter,
 						sizeof(pcb->filter));
 			break;
 
 		case SO_HCI_RAW_DIRECTION:
 			dir = (pcb->flags & NG_BTSOCKET_HCI_RAW_DIRECTION)?1:0;
 			error = sooptcopyout(sopt, &dir, sizeof(dir));
 			break;
 
 		default:
 			error = EINVAL;
 			break;
 		}
 		break;
 
 	case SOPT_SET:
 		switch (sopt->sopt_name) {
 		case SO_HCI_RAW_FILTER:
 			error = sooptcopyin(sopt, &filter, sizeof(filter),
 						sizeof(filter));
 			if (error == 0)
 				bcopy(&filter, &pcb->filter,
 						sizeof(pcb->filter));
 			break;
 
 		case SO_HCI_RAW_DIRECTION:
 			error = sooptcopyin(sopt, &dir, sizeof(dir),
 						sizeof(dir));
 			if (error != 0)
 				break;
 
 			if (dir)
 				pcb->flags |= NG_BTSOCKET_HCI_RAW_DIRECTION;
 			else
 				pcb->flags &= ~NG_BTSOCKET_HCI_RAW_DIRECTION;
 			break;
 
 		default:
 			error = EINVAL;
 			break;
 		}
 		break;
 
 	default:
 		error = EINVAL;
 		break;
 	}
 
 	mtx_unlock(&pcb->pcb_mtx);
 	
 	return (error);
 } /* ng_btsocket_hci_raw_ctloutput */
 
 /*
  * Detach raw HCI socket
  */
 
 void
 ng_btsocket_hci_raw_detach(struct socket *so)
 {
 	ng_btsocket_hci_raw_pcb_p	pcb = so2hci_raw_pcb(so);
 
 	KASSERT(pcb != NULL, ("ng_btsocket_hci_raw_detach: pcb == NULL"));
 
 	if (ng_btsocket_hci_raw_node == NULL)
 		return;
 
 	mtx_lock(&ng_btsocket_hci_raw_sockets_mtx);
 	mtx_lock(&pcb->pcb_mtx);
 
 	LIST_REMOVE(pcb, next);
 
 	mtx_unlock(&pcb->pcb_mtx);
 	mtx_unlock(&ng_btsocket_hci_raw_sockets_mtx);
 
 	mtx_destroy(&pcb->pcb_mtx);
 
 	bzero(pcb, sizeof(*pcb));
 	free(pcb, M_NETGRAPH_BTSOCKET_HCI_RAW);
 
 	so->so_pcb = NULL;
 } /* ng_btsocket_hci_raw_detach */
 
 /*
  * Disconnect raw HCI socket
  */
 
 int
 ng_btsocket_hci_raw_disconnect(struct socket *so)
 {
 	ng_btsocket_hci_raw_pcb_p	 pcb = so2hci_raw_pcb(so);
 
 	if (pcb == NULL)
 		return (EINVAL);
 	if (ng_btsocket_hci_raw_node == NULL)
 		return (EINVAL);
 
 	mtx_lock(&pcb->pcb_mtx);
 	soisdisconnected(so);
 	mtx_unlock(&pcb->pcb_mtx);
 
 	return (0);
 } /* ng_btsocket_hci_raw_disconnect */
 
 /*
  * Get socket peer's address
  */
 
 int
 ng_btsocket_hci_raw_peeraddr(struct socket *so, struct sockaddr **nam)
 {
 	return (ng_btsocket_hci_raw_sockaddr(so, nam));
 } /* ng_btsocket_hci_raw_peeraddr */
 
 /*
  * Send data
  */
 
 int
 ng_btsocket_hci_raw_send(struct socket *so, int flags, struct mbuf *m,
 		struct sockaddr *sa, struct mbuf *control, struct thread *td)
 {
 	ng_btsocket_hci_raw_pcb_p	 pcb = so2hci_raw_pcb(so);
 	struct mbuf			*nam = NULL;
 	int				 error = 0;
 
 	if (ng_btsocket_hci_raw_node == NULL) {
 		error = ENETDOWN;
 		goto drop;
 	}
 	if (pcb == NULL) {
 		error = EINVAL;
 		goto drop;
 	}
 	if (control != NULL) {
 		error = EINVAL;
 		goto drop;
 	}
 
 	if (m->m_pkthdr.len < sizeof(ng_hci_cmd_pkt_t) ||
 	    m->m_pkthdr.len > sizeof(ng_hci_cmd_pkt_t) + NG_HCI_CMD_PKT_SIZE) {
 		error = EMSGSIZE;
 		goto drop;
 	}
 
 	if (m->m_len < sizeof(ng_hci_cmd_pkt_t)) {
 		if ((m = m_pullup(m, sizeof(ng_hci_cmd_pkt_t))) == NULL) {
 			error = ENOBUFS;
 			goto drop;
 		}
 	}
 	if (*mtod(m, u_int8_t *) != NG_HCI_CMD_PKT) {
 		error = ENOTSUP;
 		goto drop;
 	}
 
 	mtx_lock(&pcb->pcb_mtx);
 
 	error = ng_btsocket_hci_raw_filter(pcb, m, 0);
 	if (error != 0) {
 		mtx_unlock(&pcb->pcb_mtx);
 		goto drop;
 	}
 
 	if (sa == NULL) {
 		if (pcb->addr.hci_node[0] == 0) {
 			mtx_unlock(&pcb->pcb_mtx);
 			error = EDESTADDRREQ;
 			goto drop;
 		}
 
 		sa = (struct sockaddr *) &pcb->addr;
 	}
 
 	MGET(nam, M_NOWAIT, MT_SONAME);
 	if (nam == NULL) {
 		mtx_unlock(&pcb->pcb_mtx);
 		error = ENOBUFS;
 		goto drop;
 	}
 
 	nam->m_len = sizeof(struct sockaddr_hci);
 	bcopy(sa,mtod(nam, struct sockaddr_hci *),sizeof(struct sockaddr_hci));
 
 	nam->m_next = m;
 	m = NULL;
 
 	mtx_unlock(&pcb->pcb_mtx);
 
 	return (ng_send_fn(ng_btsocket_hci_raw_node, NULL, 
 				ng_btsocket_hci_raw_output, nam, 0));
 drop:
 	NG_FREE_M(control); /* NG_FREE_M checks for != NULL */
 	NG_FREE_M(nam);
 	NG_FREE_M(m);
 	
 	return (error);
 } /* ng_btsocket_hci_raw_send */
 
 /*
  * Get socket address
  */
 
 int
 ng_btsocket_hci_raw_sockaddr(struct socket *so, struct sockaddr **nam)
 {
 	ng_btsocket_hci_raw_pcb_p	pcb = so2hci_raw_pcb(so);
 	struct sockaddr_hci		sa;
 
 	if (pcb == NULL)
 		return (EINVAL);
 	if (ng_btsocket_hci_raw_node == NULL)
 		return (EINVAL);
 
 	bzero(&sa, sizeof(sa));
 	sa.hci_len = sizeof(sa);
 	sa.hci_family = AF_BLUETOOTH;
 
 	mtx_lock(&pcb->pcb_mtx);
 	strlcpy(sa.hci_node, pcb->addr.hci_node, sizeof(sa.hci_node));
 	mtx_unlock(&pcb->pcb_mtx);
 
 	*nam = sodupsockaddr((struct sockaddr *) &sa, M_NOWAIT);
 
 	return ((*nam == NULL)? ENOMEM : 0);
 } /* ng_btsocket_hci_raw_sockaddr */
 
Index: projects/release-debugdist/sys/netgraph/bluetooth/socket/ng_btsocket_l2cap.c
===================================================================
--- projects/release-debugdist/sys/netgraph/bluetooth/socket/ng_btsocket_l2cap.c	(revision 267343)
+++ projects/release-debugdist/sys/netgraph/bluetooth/socket/ng_btsocket_l2cap.c	(revision 267344)
@@ -1,2826 +1,2830 @@
 /*
  * ng_btsocket_l2cap.c
  */
 
 /*-
  * Copyright (c) 2001-2002 Maksim Yevmenkin <m_evmenkin@yahoo.com>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
  * $Id: ng_btsocket_l2cap.c,v 1.16 2003/09/14 23:29:06 max Exp $
  * $FreeBSD$
  */
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/bitstring.h>
 #include <sys/domain.h>
 #include <sys/endian.h>
 #include <sys/errno.h>
 #include <sys/filedesc.h>
 #include <sys/ioccom.h>
 #include <sys/kernel.h>
 #include <sys/lock.h>
 #include <sys/malloc.h>
 #include <sys/mbuf.h>
 #include <sys/mutex.h>
 #include <sys/protosw.h>
 #include <sys/queue.h>
 #include <sys/socket.h>
 #include <sys/socketvar.h>
 #include <sys/sysctl.h>
 #include <sys/taskqueue.h>
 
 #include <net/vnet.h>
 
 #include <netgraph/ng_message.h>
 #include <netgraph/netgraph.h>
 #include <netgraph/bluetooth/include/ng_bluetooth.h>
 #include <netgraph/bluetooth/include/ng_hci.h>
 #include <netgraph/bluetooth/include/ng_l2cap.h>
 #include <netgraph/bluetooth/include/ng_btsocket.h>
 #include <netgraph/bluetooth/include/ng_btsocket_l2cap.h>
 
 /* MALLOC define */
 #ifdef NG_SEPARATE_MALLOC
 static MALLOC_DEFINE(M_NETGRAPH_BTSOCKET_L2CAP, "netgraph_btsocks_l2cap",
 		"Netgraph Bluetooth L2CAP sockets");
 #else
 #define M_NETGRAPH_BTSOCKET_L2CAP M_NETGRAPH
 #endif /* NG_SEPARATE_MALLOC */
 
 /* Netgraph node methods */
 static ng_constructor_t	ng_btsocket_l2cap_node_constructor;
 static ng_rcvmsg_t	ng_btsocket_l2cap_node_rcvmsg;
 static ng_shutdown_t	ng_btsocket_l2cap_node_shutdown;
 static ng_newhook_t	ng_btsocket_l2cap_node_newhook;
 static ng_connect_t	ng_btsocket_l2cap_node_connect;
 static ng_rcvdata_t	ng_btsocket_l2cap_node_rcvdata;
 static ng_disconnect_t	ng_btsocket_l2cap_node_disconnect;
 
 static void		ng_btsocket_l2cap_input   (void *, int);
 static void		ng_btsocket_l2cap_rtclean (void *, int);
 
 /* Netgraph type descriptor */
 static struct ng_type	typestruct = {
 	.version =	NG_ABI_VERSION,
 	.name =		NG_BTSOCKET_L2CAP_NODE_TYPE,
 	.constructor =	ng_btsocket_l2cap_node_constructor,
 	.rcvmsg =	ng_btsocket_l2cap_node_rcvmsg,
 	.shutdown =	ng_btsocket_l2cap_node_shutdown,
 	.newhook =	ng_btsocket_l2cap_node_newhook,
 	.connect =	ng_btsocket_l2cap_node_connect,
 	.rcvdata =	ng_btsocket_l2cap_node_rcvdata,
 	.disconnect =	ng_btsocket_l2cap_node_disconnect,
 };
 
 /* Globals */
 extern int					ifqmaxlen;
 static u_int32_t				ng_btsocket_l2cap_debug_level;
 static node_p					ng_btsocket_l2cap_node;
 static struct ng_bt_itemq			ng_btsocket_l2cap_queue;
 static struct mtx				ng_btsocket_l2cap_queue_mtx;
 static struct task				ng_btsocket_l2cap_queue_task;
 static LIST_HEAD(, ng_btsocket_l2cap_pcb)	ng_btsocket_l2cap_sockets;
 static struct mtx				ng_btsocket_l2cap_sockets_mtx;
 static LIST_HEAD(, ng_btsocket_l2cap_rtentry)	ng_btsocket_l2cap_rt;
 static struct mtx				ng_btsocket_l2cap_rt_mtx;
 static struct task				ng_btsocket_l2cap_rt_task;
 static struct timeval				ng_btsocket_l2cap_lasttime;
 static int					ng_btsocket_l2cap_curpps;
 
 /* Sysctl tree */
 SYSCTL_DECL(_net_bluetooth_l2cap_sockets);
 static SYSCTL_NODE(_net_bluetooth_l2cap_sockets, OID_AUTO, seq, CTLFLAG_RW,
 	0, "Bluetooth SEQPACKET L2CAP sockets family");
 SYSCTL_UINT(_net_bluetooth_l2cap_sockets_seq, OID_AUTO, debug_level,
 	CTLFLAG_RW,
 	&ng_btsocket_l2cap_debug_level, NG_BTSOCKET_WARN_LEVEL,
 	"Bluetooth SEQPACKET L2CAP sockets debug level");
 SYSCTL_UINT(_net_bluetooth_l2cap_sockets_seq, OID_AUTO, queue_len,
 	CTLFLAG_RD,
 	&ng_btsocket_l2cap_queue.len, 0,
 	"Bluetooth SEQPACKET L2CAP sockets input queue length");
 SYSCTL_UINT(_net_bluetooth_l2cap_sockets_seq, OID_AUTO, queue_maxlen,
 	CTLFLAG_RD,
 	&ng_btsocket_l2cap_queue.maxlen, 0,
 	"Bluetooth SEQPACKET L2CAP sockets input queue max. length");
 SYSCTL_UINT(_net_bluetooth_l2cap_sockets_seq, OID_AUTO, queue_drops,
 	CTLFLAG_RD,
 	&ng_btsocket_l2cap_queue.drops, 0,
 	"Bluetooth SEQPACKET L2CAP sockets input queue drops");
 
 /* Debug */
 #define NG_BTSOCKET_L2CAP_INFO \
 	if (ng_btsocket_l2cap_debug_level >= NG_BTSOCKET_INFO_LEVEL && \
 	    ppsratecheck(&ng_btsocket_l2cap_lasttime, &ng_btsocket_l2cap_curpps, 1)) \
 		printf
 
 #define NG_BTSOCKET_L2CAP_WARN \
 	if (ng_btsocket_l2cap_debug_level >= NG_BTSOCKET_WARN_LEVEL && \
 	    ppsratecheck(&ng_btsocket_l2cap_lasttime, &ng_btsocket_l2cap_curpps, 1)) \
 		printf
 
 #define NG_BTSOCKET_L2CAP_ERR \
 	if (ng_btsocket_l2cap_debug_level >= NG_BTSOCKET_ERR_LEVEL && \
 	    ppsratecheck(&ng_btsocket_l2cap_lasttime, &ng_btsocket_l2cap_curpps, 1)) \
 		printf
 
 #define NG_BTSOCKET_L2CAP_ALERT \
 	if (ng_btsocket_l2cap_debug_level >= NG_BTSOCKET_ALERT_LEVEL && \
 	    ppsratecheck(&ng_btsocket_l2cap_lasttime, &ng_btsocket_l2cap_curpps, 1)) \
 		printf
 
 /* 
  * Netgraph message processing routines
  */
 
 static int ng_btsocket_l2cap_process_l2ca_con_req_rsp
 	(struct ng_mesg *, ng_btsocket_l2cap_rtentry_p);
 static int ng_btsocket_l2cap_process_l2ca_con_rsp_rsp
 	(struct ng_mesg *, ng_btsocket_l2cap_rtentry_p);
 static int ng_btsocket_l2cap_process_l2ca_con_ind
 	(struct ng_mesg *, ng_btsocket_l2cap_rtentry_p);
 
 static int ng_btsocket_l2cap_process_l2ca_cfg_req_rsp
 	(struct ng_mesg *, ng_btsocket_l2cap_rtentry_p);
 static int ng_btsocket_l2cap_process_l2ca_cfg_rsp_rsp
 	(struct ng_mesg *, ng_btsocket_l2cap_rtentry_p);
 static int ng_btsocket_l2cap_process_l2ca_cfg_ind
 	(struct ng_mesg *, ng_btsocket_l2cap_rtentry_p);
 
 static int ng_btsocket_l2cap_process_l2ca_discon_rsp
 	(struct ng_mesg *, ng_btsocket_l2cap_rtentry_p);
 static int ng_btsocket_l2cap_process_l2ca_discon_ind
 	(struct ng_mesg *, ng_btsocket_l2cap_rtentry_p);
 
 static int ng_btsocket_l2cap_process_l2ca_write_rsp
 	(struct ng_mesg *, ng_btsocket_l2cap_rtentry_p);
 
 /*
  * Send L2CA_xxx messages to the lower layer
  */
 
 static int  ng_btsocket_l2cap_send_l2ca_con_req
 	(ng_btsocket_l2cap_pcb_p);
 static int  ng_btsocket_l2cap_send_l2ca_con_rsp_req
 	(u_int32_t, ng_btsocket_l2cap_rtentry_p, bdaddr_p, int, int, int);
 static int  ng_btsocket_l2cap_send_l2ca_cfg_req
 	(ng_btsocket_l2cap_pcb_p);
 static int  ng_btsocket_l2cap_send_l2ca_cfg_rsp
 	(ng_btsocket_l2cap_pcb_p);
 static int  ng_btsocket_l2cap_send_l2ca_discon_req
 	(u_int32_t, ng_btsocket_l2cap_pcb_p);
 
 static int ng_btsocket_l2cap_send2
 	(ng_btsocket_l2cap_pcb_p);
 
 /* 
  * Timeout processing routines
  */
 
 static void ng_btsocket_l2cap_timeout         (ng_btsocket_l2cap_pcb_p);
 static void ng_btsocket_l2cap_untimeout       (ng_btsocket_l2cap_pcb_p);
 static void ng_btsocket_l2cap_process_timeout (void *);
 
 /* 
  * Other stuff 
  */
 
 static ng_btsocket_l2cap_pcb_p     ng_btsocket_l2cap_pcb_by_addr(bdaddr_p, int);
 static ng_btsocket_l2cap_pcb_p     ng_btsocket_l2cap_pcb_by_token(u_int32_t);
 static ng_btsocket_l2cap_pcb_p     ng_btsocket_l2cap_pcb_by_cid (bdaddr_p, int);
 static int                         ng_btsocket_l2cap_result2errno(int);
 
 #define ng_btsocket_l2cap_wakeup_input_task() \
 	taskqueue_enqueue(taskqueue_swi_giant, &ng_btsocket_l2cap_queue_task)
 
 #define ng_btsocket_l2cap_wakeup_route_task() \
 	taskqueue_enqueue(taskqueue_swi_giant, &ng_btsocket_l2cap_rt_task)
 
 /*****************************************************************************
  *****************************************************************************
  **                        Netgraph node interface
  *****************************************************************************
  *****************************************************************************/
 
 /*
  * Netgraph node constructor. Do not allow to create node of this type.
  */
 
 static int
 ng_btsocket_l2cap_node_constructor(node_p node)
 {
 	return (EINVAL);
 } /* ng_btsocket_l2cap_node_constructor */
 
 /*
  * Do local shutdown processing. Let old node go and create new fresh one.
  */
 
 static int
 ng_btsocket_l2cap_node_shutdown(node_p node)
 {
 	int	error = 0;
 
 	NG_NODE_UNREF(node);
 
 	/* Create new node */
 	error = ng_make_node_common(&typestruct, &ng_btsocket_l2cap_node);
 	if (error != 0) {
 		NG_BTSOCKET_L2CAP_ALERT(
 "%s: Could not create Netgraph node, error=%d\n", __func__, error);
 
 		ng_btsocket_l2cap_node = NULL;
 
 		return (error);
 	}
 
 	error = ng_name_node(ng_btsocket_l2cap_node,
 				NG_BTSOCKET_L2CAP_NODE_TYPE);
 	if (error != 0) {
 		NG_BTSOCKET_L2CAP_ALERT(
 "%s: Could not name Netgraph node, error=%d\n", __func__, error);
 
 		NG_NODE_UNREF(ng_btsocket_l2cap_node);
 		ng_btsocket_l2cap_node = NULL;
 
 		return (error);
 	}
 		
 	return (0);
 } /* ng_btsocket_l2cap_node_shutdown */
 
 /*
  * We allow any hook to be connected to the node.
  */
 
 static int
 ng_btsocket_l2cap_node_newhook(node_p node, hook_p hook, char const *name)
 {
 	return (0);
 } /* ng_btsocket_l2cap_node_newhook */
 
 /* 
  * Just say "YEP, that's OK by me!"
  */
 
 static int
 ng_btsocket_l2cap_node_connect(hook_p hook)
 {
 	NG_HOOK_SET_PRIVATE(hook, NULL);
 	NG_HOOK_REF(hook); /* Keep extra reference to the hook */
 
 #if 0
 	NG_HOOK_FORCE_QUEUE(NG_HOOK_PEER(hook));
 	NG_HOOK_FORCE_QUEUE(hook);
 #endif
 
 	return (0);
 } /* ng_btsocket_l2cap_node_connect */
 
 /*
  * Hook disconnection. Schedule route cleanup task
  */
 
 static int
 ng_btsocket_l2cap_node_disconnect(hook_p hook)
 {
 	/*
 	 * If hook has private information than we must have this hook in
 	 * the routing table and must schedule cleaning for the routing table.
 	 * Otherwise hook was connected but we never got "hook_info" message,
 	 * so we have never added this hook to the routing table and it save
 	 * to just delete it.
 	 */
 
 	if (NG_HOOK_PRIVATE(hook) != NULL)
 		return (ng_btsocket_l2cap_wakeup_route_task());
 
 	NG_HOOK_UNREF(hook); /* Remove extra reference */
 
 	return (0);
 } /* ng_btsocket_l2cap_node_disconnect */
 
 /*
  * Process incoming messages 
  */
 
 static int
 ng_btsocket_l2cap_node_rcvmsg(node_p node, item_p item, hook_p hook)
 {
 	struct ng_mesg	*msg = NGI_MSG(item); /* item still has message */
 	int		 error = 0;
 
 	if (msg != NULL && msg->header.typecookie == NGM_L2CAP_COOKIE) {
 		mtx_lock(&ng_btsocket_l2cap_queue_mtx);
 		if (NG_BT_ITEMQ_FULL(&ng_btsocket_l2cap_queue)) {
 			NG_BTSOCKET_L2CAP_ERR(
 "%s: Input queue is full (msg)\n", __func__);
 
 			NG_BT_ITEMQ_DROP(&ng_btsocket_l2cap_queue);
 			NG_FREE_ITEM(item);
 			error = ENOBUFS;
 		} else {
 			if (hook != NULL) {
 				NG_HOOK_REF(hook);
 				NGI_SET_HOOK(item, hook);
 			}
 
 			NG_BT_ITEMQ_ENQUEUE(&ng_btsocket_l2cap_queue, item);
 			error = ng_btsocket_l2cap_wakeup_input_task();
 		}
 		mtx_unlock(&ng_btsocket_l2cap_queue_mtx);
 	} else {
 		NG_FREE_ITEM(item);
 		error = EINVAL;
 	}
 
 	return (error);
 } /* ng_btsocket_l2cap_node_rcvmsg */
 
 /*
  * Receive data on a hook
  */
 
 static int
 ng_btsocket_l2cap_node_rcvdata(hook_p hook, item_p item)
 {
 	int	error = 0;
 
 	mtx_lock(&ng_btsocket_l2cap_queue_mtx);
 	if (NG_BT_ITEMQ_FULL(&ng_btsocket_l2cap_queue)) {
 		NG_BTSOCKET_L2CAP_ERR(
 "%s: Input queue is full (data)\n", __func__);
 
 		NG_BT_ITEMQ_DROP(&ng_btsocket_l2cap_queue);
 		NG_FREE_ITEM(item);
 		error = ENOBUFS;
 	} else {
 		NG_HOOK_REF(hook);
 		NGI_SET_HOOK(item, hook);
 
 		NG_BT_ITEMQ_ENQUEUE(&ng_btsocket_l2cap_queue, item);
 		error = ng_btsocket_l2cap_wakeup_input_task();
 	}
 	mtx_unlock(&ng_btsocket_l2cap_queue_mtx);
 
 	return (error);
 } /* ng_btsocket_l2cap_node_rcvdata */
 
 /*
  * Process L2CA_Connect respose. Socket layer must have initiated connection,
  * so we have to have a socket associated with message token.
  */
 
 static int
 ng_btsocket_l2cap_process_l2ca_con_req_rsp(struct ng_mesg *msg,
 		ng_btsocket_l2cap_rtentry_p rt)
 {
 	ng_l2cap_l2ca_con_op	*op = NULL;
 	ng_btsocket_l2cap_pcb_t	*pcb = NULL;
 	int			 error = 0;
 
 	if (msg->header.arglen != sizeof(*op))
 		return (EMSGSIZE);
 
 	op = (ng_l2cap_l2ca_con_op *)(msg->data);
 
 	mtx_lock(&ng_btsocket_l2cap_sockets_mtx);
 
 	/* Look for the socket with the token */
 	pcb = ng_btsocket_l2cap_pcb_by_token(msg->header.token);
 	if (pcb == NULL) {
 		mtx_unlock(&ng_btsocket_l2cap_sockets_mtx);
 		return (ENOENT);
 	}
 
 	mtx_lock(&pcb->pcb_mtx);
 
 	NG_BTSOCKET_L2CAP_INFO(
 "%s: Got L2CA_Connect response, token=%d, src bdaddr=%x:%x:%x:%x:%x:%x, " \
 "dst bdaddr=%x:%x:%x:%x:%x:%x, psm=%d, lcid=%d, result=%d, status=%d, " \
 "state=%d\n",	__func__, msg->header.token,
 		pcb->src.b[5], pcb->src.b[4], pcb->src.b[3],
 		pcb->src.b[2], pcb->src.b[1], pcb->src.b[0],
 		pcb->dst.b[5], pcb->dst.b[4], pcb->dst.b[3],
 		pcb->dst.b[2], pcb->dst.b[1], pcb->dst.b[0],
 		pcb->psm, op->lcid, op->result, op->status,
 		pcb->state);
 
 	if (pcb->state != NG_BTSOCKET_L2CAP_CONNECTING) {
 		mtx_unlock(&pcb->pcb_mtx);
 		mtx_unlock(&ng_btsocket_l2cap_sockets_mtx);
 
 		return (ENOENT);
 	}
 
 	ng_btsocket_l2cap_untimeout(pcb);
 
 	if (op->result == NG_L2CAP_PENDING) {
 		ng_btsocket_l2cap_timeout(pcb);
 		mtx_unlock(&pcb->pcb_mtx);
 		mtx_unlock(&ng_btsocket_l2cap_sockets_mtx);
 
 		return (0);
 	}
 
 	if (op->result == NG_L2CAP_SUCCESS) {
 		/*
 		 * Channel is now open, so update local channel ID and 
 		 * start configuration process. Source and destination
 		 * addresses as well as route must be already set.
 		 */
 
 		pcb->cid = op->lcid;
 
 		error = ng_btsocket_l2cap_send_l2ca_cfg_req(pcb);
 		if (error != 0) {
 			/* Send disconnect request with "zero" token */
 			ng_btsocket_l2cap_send_l2ca_discon_req(0, pcb);
 
 			/* ... and close the socket */
 			pcb->state = NG_BTSOCKET_L2CAP_CLOSED;
 			soisdisconnected(pcb->so);
 		} else {
 			pcb->cfg_state = NG_BTSOCKET_L2CAP_CFG_IN_SENT;
 			pcb->state = NG_BTSOCKET_L2CAP_CONFIGURING;
 
 			ng_btsocket_l2cap_timeout(pcb);
 		}
 	} else {
 		/*
 		 * We have failed to open connection, so convert result
 		 * code to "errno" code and disconnect the socket. Channel
 		 * already has been closed.
 		 */
 
 		pcb->so->so_error = ng_btsocket_l2cap_result2errno(op->result);
 		pcb->state = NG_BTSOCKET_L2CAP_CLOSED;
 		soisdisconnected(pcb->so); 
 	}
 
 	mtx_unlock(&pcb->pcb_mtx);
 	mtx_unlock(&ng_btsocket_l2cap_sockets_mtx);
 
 	return (error);
 } /* ng_btsocket_l2cap_process_l2ca_con_req_rsp */
 
 /*
  * Process L2CA_ConnectRsp response
  */
 
 static int
 ng_btsocket_l2cap_process_l2ca_con_rsp_rsp(struct ng_mesg *msg,
 		ng_btsocket_l2cap_rtentry_p rt)
 {
 	ng_l2cap_l2ca_con_rsp_op	*op = NULL;
 	ng_btsocket_l2cap_pcb_t		*pcb = NULL;
 
 	if (msg->header.arglen != sizeof(*op)) 
 		return (EMSGSIZE);
 
 	op = (ng_l2cap_l2ca_con_rsp_op *)(msg->data);
 
 	mtx_lock(&ng_btsocket_l2cap_sockets_mtx);
 
 	/* Look for the socket with the token */
 	pcb = ng_btsocket_l2cap_pcb_by_token(msg->header.token);
 	if (pcb == NULL) {
 		mtx_unlock(&ng_btsocket_l2cap_sockets_mtx);
 		return (ENOENT);
 	}
 
 	mtx_lock(&pcb->pcb_mtx);
 
 	NG_BTSOCKET_L2CAP_INFO(
 "%s: Got L2CA_ConnectRsp response, token=%d, src bdaddr=%x:%x:%x:%x:%x:%x, " \
 "dst bdaddr=%x:%x:%x:%x:%x:%x, psm=%d, lcid=%d, result=%d, state=%d\n",
 		__func__, msg->header.token,
 		pcb->src.b[5], pcb->src.b[4], pcb->src.b[3],
 		pcb->src.b[2], pcb->src.b[1], pcb->src.b[0],
 		pcb->dst.b[5], pcb->dst.b[4], pcb->dst.b[3],
 		pcb->dst.b[2], pcb->dst.b[1], pcb->dst.b[0],
 		pcb->psm, pcb->cid, op->result, pcb->state);
 
 	if (pcb->state != NG_BTSOCKET_L2CAP_CONNECTING) {
 		mtx_unlock(&pcb->pcb_mtx);
 		mtx_unlock(&ng_btsocket_l2cap_sockets_mtx);
 
 		return (ENOENT);
 	}
 
 	ng_btsocket_l2cap_untimeout(pcb);
 
 	/* Check the result and disconnect the socket on failure */
 	if (op->result != NG_L2CAP_SUCCESS) {
 		/* Close the socket - channel already closed */
 		pcb->so->so_error = ng_btsocket_l2cap_result2errno(op->result);
 		pcb->state = NG_BTSOCKET_L2CAP_CLOSED;
 		soisdisconnected(pcb->so);
 	} else {
 		/* Move to CONFIGURING state and wait for CONFIG_IND */
 		pcb->cfg_state = 0;
 		pcb->state = NG_BTSOCKET_L2CAP_CONFIGURING;
 		ng_btsocket_l2cap_timeout(pcb);
 	}
 
 	mtx_unlock(&pcb->pcb_mtx);
 	mtx_unlock(&ng_btsocket_l2cap_sockets_mtx);
 
 	return (0);
 } /* ng_btsocket_process_l2ca_con_rsp_rsp */
 
 /*
  * Process L2CA_Connect indicator. Find socket that listens on address 
  * and PSM. Find exact or closest match. Create new socket and initiate 
  * connection.
  */
 
 static int
 ng_btsocket_l2cap_process_l2ca_con_ind(struct ng_mesg *msg,
 		ng_btsocket_l2cap_rtentry_p rt)
 {
 	ng_l2cap_l2ca_con_ind_ip	*ip = NULL;
 	ng_btsocket_l2cap_pcb_t		*pcb = NULL, *pcb1 = NULL;
 	int				 error = 0;
 	u_int32_t			 token = 0;
 	u_int16_t			 result = 0;
 
 	if (msg->header.arglen != sizeof(*ip))
 		return (EMSGSIZE);
 
 	ip = (ng_l2cap_l2ca_con_ind_ip *)(msg->data);
 
 	NG_BTSOCKET_L2CAP_INFO(
 "%s: Got L2CA_Connect indicator, src bdaddr=%x:%x:%x:%x:%x:%x, " \
 "dst bdaddr=%x:%x:%x:%x:%x:%x, psm=%d, lcid=%d, ident=%d\n",
 		__func__,
 		rt->src.b[5], rt->src.b[4], rt->src.b[3],
 		rt->src.b[2], rt->src.b[1], rt->src.b[0],
 		ip->bdaddr.b[5], ip->bdaddr.b[4], ip->bdaddr.b[3],
 		ip->bdaddr.b[2], ip->bdaddr.b[1], ip->bdaddr.b[0],
 		ip->psm, ip->lcid, ip->ident);
 
 	mtx_lock(&ng_btsocket_l2cap_sockets_mtx);
 	
 	pcb = ng_btsocket_l2cap_pcb_by_addr(&rt->src, ip->psm);
 	if (pcb != NULL) {
 		struct socket	*so1 = NULL;
 
 		mtx_lock(&pcb->pcb_mtx);
 
 		/*
 		 * First check the pending connections queue and if we have
 		 * space then create new socket and set proper source address.
 		 */
 
 		if (pcb->so->so_qlen <= pcb->so->so_qlimit) {
 			CURVNET_SET(pcb->so->so_vnet);
 			so1 = sonewconn(pcb->so, 0);
 			CURVNET_RESTORE();
 		}
 
 		if (so1 == NULL) {
 			result = NG_L2CAP_NO_RESOURCES;
 			goto respond;
 		}
 
 		/*
 		 * If we got here than we have created new socket. So complete 
 		 * connection. If we we listening on specific address then copy 
 		 * source address from listening socket, otherwise copy source 
 		 * address from hook's routing information.
 		 */
 
 		pcb1 = so2l2cap_pcb(so1);
 		KASSERT((pcb1 != NULL),
 ("%s: pcb1 == NULL\n", __func__));
 
  		mtx_lock(&pcb1->pcb_mtx);
 
 		if (bcmp(&pcb->src, NG_HCI_BDADDR_ANY, sizeof(pcb->src)) != 0)
 			bcopy(&pcb->src, &pcb1->src, sizeof(pcb1->src));
 		else
 			bcopy(&rt->src, &pcb1->src, sizeof(pcb1->src));
 
 		pcb1->flags &= ~NG_BTSOCKET_L2CAP_CLIENT;
 
 		bcopy(&ip->bdaddr, &pcb1->dst, sizeof(pcb1->dst));
 		pcb1->psm = ip->psm;
 		pcb1->cid = ip->lcid;
 		pcb1->rt = rt;
 
 		/* Copy socket settings */
 		pcb1->imtu = pcb->imtu;
 		bcopy(&pcb->oflow, &pcb1->oflow, sizeof(pcb1->oflow));
 		pcb1->flush_timo = pcb->flush_timo;
 
 		token = pcb1->token;
 	} else
 		/* Nobody listens on requested BDADDR/PSM */
 		result = NG_L2CAP_PSM_NOT_SUPPORTED;
 
 respond:
 	error = ng_btsocket_l2cap_send_l2ca_con_rsp_req(token, rt,
 			&ip->bdaddr, ip->ident, ip->lcid, result);
 	if (pcb1 != NULL) {
 		if (error != 0) {
 			pcb1->so->so_error = error;
 			pcb1->state = NG_BTSOCKET_L2CAP_CLOSED;
 			soisdisconnected(pcb1->so);
 		} else {
 			pcb1->state = NG_BTSOCKET_L2CAP_CONNECTING;
 			soisconnecting(pcb1->so);
 
 			ng_btsocket_l2cap_timeout(pcb1);
 		}
 
 		mtx_unlock(&pcb1->pcb_mtx);
 	}
 
 	if (pcb != NULL)
 		mtx_unlock(&pcb->pcb_mtx);
 
 	mtx_unlock(&ng_btsocket_l2cap_sockets_mtx);
 
 	return (error);
 } /* ng_btsocket_l2cap_process_l2ca_con_ind */
 
 /*
  * Process L2CA_Config response
  */
 
 static int
 ng_btsocket_l2cap_process_l2ca_cfg_req_rsp(struct ng_mesg *msg,
 		ng_btsocket_l2cap_rtentry_p rt)
 {
 	ng_l2cap_l2ca_cfg_op	*op = NULL;
 	ng_btsocket_l2cap_pcb_p	 pcb = NULL;
 
 	if (msg->header.arglen != sizeof(*op))
 		return (EMSGSIZE);
 
 	op = (ng_l2cap_l2ca_cfg_op *)(msg->data);
 
 	mtx_lock(&ng_btsocket_l2cap_sockets_mtx);
 
 	/* 
 	 * Socket must have issued a Configure request, so we must have a 
 	 * socket that wants to be configured. Use Netgraph message token 
 	 * to find it
 	 */
 
 	pcb = ng_btsocket_l2cap_pcb_by_token(msg->header.token);
 	if (pcb == NULL) {
 		/*
 		 * XXX FIXME what to do here? We could not find a
 		 * socket with requested token. We even can not send
 		 * Disconnect, because we do not know channel ID
 		 */
 
 		mtx_unlock(&ng_btsocket_l2cap_sockets_mtx);
 		return (ENOENT);
 	}
 
 	mtx_lock(&pcb->pcb_mtx);
 
         NG_BTSOCKET_L2CAP_INFO(
 "%s: Got L2CA_Config response, token=%d, src bdaddr=%x:%x:%x:%x:%x:%x, " \
 "dst bdaddr=%x:%x:%x:%x:%x:%x, psm=%d, lcid=%d, result=%d, state=%d, " \
 "cfg_state=%x\n",
 		__func__, msg->header.token,
 		pcb->src.b[5], pcb->src.b[4], pcb->src.b[3],
 		pcb->src.b[2], pcb->src.b[1], pcb->src.b[0],
 		pcb->dst.b[5], pcb->dst.b[4], pcb->dst.b[3],
 		pcb->dst.b[2], pcb->dst.b[1], pcb->dst.b[0],
 		pcb->psm, pcb->cid, op->result, pcb->state, pcb->cfg_state);
 
 	if (pcb->state != NG_BTSOCKET_L2CAP_CONFIGURING) {
 		mtx_unlock(&pcb->pcb_mtx);
 		mtx_unlock(&ng_btsocket_l2cap_sockets_mtx);
 
 		return (ENOENT);
 	}
 
 	if (op->result == NG_L2CAP_SUCCESS) {
 		/*
 		 * XXX FIXME Actually set flush and link timeout.
 		 * Set QoS here if required. Resolve conficts (flush_timo). 
 		 * Save incoming MTU (peer's outgoing MTU) and outgoing flow 
 		 * spec.
 		 */
 
 		pcb->imtu = op->imtu;
 		bcopy(&op->oflow, &pcb->oflow, sizeof(pcb->oflow));
 		pcb->flush_timo = op->flush_timo;
 
 		/*
 		 * We have configured incoming side, so record it and check 
 		 * if configuration is complete. If complete then mark socket
 		 * as connected, otherwise wait for the peer.
 		 */
 
 		pcb->cfg_state &= ~NG_BTSOCKET_L2CAP_CFG_IN_SENT;
 		pcb->cfg_state |= NG_BTSOCKET_L2CAP_CFG_IN;
 
 		if (pcb->cfg_state == NG_BTSOCKET_L2CAP_CFG_BOTH) {
 			/* Configuration complete - mark socket as open */
 			ng_btsocket_l2cap_untimeout(pcb);
 			pcb->state = NG_BTSOCKET_L2CAP_OPEN;
 			soisconnected(pcb->so); 
 		} 
 	} else {
 		/*
 		 * Something went wrong. Could be unacceptable parameters,
 		 * reject or unknown option. That's too bad, but we will
 		 * not negotiate. Send Disconnect and close the channel.
 		 */
 
 		ng_btsocket_l2cap_untimeout(pcb);
 
 		switch (op->result) {
 		case NG_L2CAP_UNACCEPTABLE_PARAMS:
 		case NG_L2CAP_UNKNOWN_OPTION:
 			pcb->so->so_error = EINVAL;
 			break;
 
 		default:
 			pcb->so->so_error = ECONNRESET;
 			break;
 		}
 
 		/* Send disconnect with "zero" token */
 		ng_btsocket_l2cap_send_l2ca_discon_req(0, pcb);
 
 		/* ... and close the socket */
 		pcb->state = NG_BTSOCKET_L2CAP_CLOSED;
 		soisdisconnected(pcb->so);
 	}
 
 	mtx_unlock(&pcb->pcb_mtx);
 	mtx_unlock(&ng_btsocket_l2cap_sockets_mtx);
 
 	return (0);
 } /* ng_btsocket_l2cap_process_l2ca_cfg_req_rsp */
 
 /*
  * Process L2CA_ConfigRsp response
  */
 
 static int
 ng_btsocket_l2cap_process_l2ca_cfg_rsp_rsp(struct ng_mesg *msg,
 		ng_btsocket_l2cap_rtentry_p rt)
 {
 	ng_l2cap_l2ca_cfg_rsp_op	*op = NULL;
 	ng_btsocket_l2cap_pcb_t		*pcb = NULL;
 	int				 error = 0;
 
 	if (msg->header.arglen != sizeof(*op))
 		return (EMSGSIZE);
 
 	op = (ng_l2cap_l2ca_cfg_rsp_op *)(msg->data);
 
 	mtx_lock(&ng_btsocket_l2cap_sockets_mtx);
 
 	/* Look for the socket with the token */
 	pcb = ng_btsocket_l2cap_pcb_by_token(msg->header.token);
 	if (pcb == NULL) {
 		mtx_unlock(&ng_btsocket_l2cap_sockets_mtx);
 		return (ENOENT);
 	}
 
 	mtx_lock(&pcb->pcb_mtx);
 
         NG_BTSOCKET_L2CAP_INFO(
 "%s: Got L2CA_ConfigRsp response, token=%d, src bdaddr=%x:%x:%x:%x:%x:%x, " \
 "dst bdaddr=%x:%x:%x:%x:%x:%x, psm=%d, lcid=%d, result=%d, state=%d, " \
 "cfg_state=%x\n",
 		__func__, msg->header.token,
 		pcb->src.b[5], pcb->src.b[4], pcb->src.b[3],
 		pcb->src.b[2], pcb->src.b[1], pcb->src.b[0],
 		pcb->dst.b[5], pcb->dst.b[4], pcb->dst.b[3],
 		pcb->dst.b[2], pcb->dst.b[1], pcb->dst.b[0],
 		pcb->psm, pcb->cid, op->result, pcb->state, pcb->cfg_state);
 
 	if (pcb->state != NG_BTSOCKET_L2CAP_CONFIGURING) {
 		mtx_unlock(&pcb->pcb_mtx);
 		mtx_unlock(&ng_btsocket_l2cap_sockets_mtx);
 
 		return (ENOENT);
 	}
 
 	/* Check the result and disconnect socket of failure */
 	if (op->result != NG_L2CAP_SUCCESS)
 		goto disconnect;
 
 	/*
 	 * Now we done with remote side configuration. Configure local 
 	 * side if we have not done it yet.
 	 */
 
 	pcb->cfg_state &= ~NG_BTSOCKET_L2CAP_CFG_OUT_SENT;
 	pcb->cfg_state |= NG_BTSOCKET_L2CAP_CFG_OUT;
 
 	if (pcb->cfg_state == NG_BTSOCKET_L2CAP_CFG_BOTH) {
 		/* Configuration complete - mask socket as open */
 		ng_btsocket_l2cap_untimeout(pcb);
 		pcb->state = NG_BTSOCKET_L2CAP_OPEN;
 		soisconnected(pcb->so);
 	} else {
 		if (!(pcb->cfg_state & NG_BTSOCKET_L2CAP_CFG_IN_SENT)) {
 			/* Send L2CA_Config request - incoming path */
 			error = ng_btsocket_l2cap_send_l2ca_cfg_req(pcb);
 			if (error != 0)
 				goto disconnect;
 
 			pcb->cfg_state |= NG_BTSOCKET_L2CAP_CFG_IN_SENT;
 		}
 	}
 
 	mtx_unlock(&pcb->pcb_mtx);
 	mtx_unlock(&ng_btsocket_l2cap_sockets_mtx);
 
 	return (error);
 
 disconnect:
 	ng_btsocket_l2cap_untimeout(pcb);
 
 	/* Send disconnect with "zero" token */
 	ng_btsocket_l2cap_send_l2ca_discon_req(0, pcb);
 
 	/* ... and close the socket */
 	pcb->state = NG_BTSOCKET_L2CAP_CLOSED;
 	soisdisconnected(pcb->so);
 
 	mtx_unlock(&pcb->pcb_mtx);
 	mtx_unlock(&ng_btsocket_l2cap_sockets_mtx);
 
 	return (error);
 } /* ng_btsocket_l2cap_process_l2ca_cfg_rsp_rsp */
 
 /*
  * Process L2CA_Config indicator
  */
 
 static int
 ng_btsocket_l2cap_process_l2ca_cfg_ind(struct ng_mesg *msg,
 		ng_btsocket_l2cap_rtentry_p rt)
 {
 	ng_l2cap_l2ca_cfg_ind_ip	*ip = NULL;
 	ng_btsocket_l2cap_pcb_t		*pcb = NULL;
 	int				 error = 0;
 
 	if (msg->header.arglen != sizeof(*ip))
 		return (EMSGSIZE);
 
 	ip = (ng_l2cap_l2ca_cfg_ind_ip *)(msg->data);
 
 	mtx_lock(&ng_btsocket_l2cap_sockets_mtx);
 
 	/* Check for the open socket that has given channel ID */
 	pcb = ng_btsocket_l2cap_pcb_by_cid(&rt->src, ip->lcid);
 	if (pcb == NULL) {
 		mtx_unlock(&ng_btsocket_l2cap_sockets_mtx);
 		return (ENOENT);
 	}
 
 	mtx_lock(&pcb->pcb_mtx);
 
         NG_BTSOCKET_L2CAP_INFO(
 "%s: Got L2CA_Config indicator, src bdaddr=%x:%x:%x:%x:%x:%x, " \
 "dst bdaddr=%x:%x:%x:%x:%x:%x, psm=%d, lcid=%d, state=%d, cfg_state=%x\n",
 		__func__,
 		pcb->src.b[5], pcb->src.b[4], pcb->src.b[3],
 		pcb->src.b[2], pcb->src.b[1], pcb->src.b[0],
 		pcb->dst.b[5], pcb->dst.b[4], pcb->dst.b[3],
 		pcb->dst.b[2], pcb->dst.b[1], pcb->dst.b[0],
 		pcb->psm, pcb->cid, pcb->state, pcb->cfg_state);
 
 	/* XXX FIXME re-configuration on open socket */
  	if (pcb->state != NG_BTSOCKET_L2CAP_CONFIGURING) {
 		mtx_unlock(&pcb->pcb_mtx);
 		mtx_unlock(&ng_btsocket_l2cap_sockets_mtx);
 
 		return (ENOENT);
 	}
 
 	/*
 	 * XXX FIXME Actually set flush and link timeout. Set QoS here if
 	 * required. Resolve conficts (flush_timo). Note outgoing MTU (peer's 
 	 * incoming MTU) and incoming flow spec.
 	 */
 
 	pcb->omtu = ip->omtu;
 	bcopy(&ip->iflow, &pcb->iflow, sizeof(pcb->iflow));
 	pcb->flush_timo = ip->flush_timo;
 
 	/*
 	 * Send L2CA_Config response to our peer and check for the errors, 
 	 * if any send disconnect to close the channel. 
 	 */
 
 	if (!(pcb->cfg_state & NG_BTSOCKET_L2CAP_CFG_OUT_SENT)) {
 		error = ng_btsocket_l2cap_send_l2ca_cfg_rsp(pcb);
 		if (error != 0) {
 			ng_btsocket_l2cap_untimeout(pcb);
 
 			pcb->so->so_error = error;
 
 			/* Send disconnect with "zero" token */
 			ng_btsocket_l2cap_send_l2ca_discon_req(0, pcb);
 
 			/* ... and close the socket */
 			pcb->state = NG_BTSOCKET_L2CAP_CLOSED;
 			soisdisconnected(pcb->so);
 		} else
 			pcb->cfg_state |= NG_BTSOCKET_L2CAP_CFG_OUT_SENT;
 	}
 
 	mtx_unlock(&pcb->pcb_mtx);
 	mtx_unlock(&ng_btsocket_l2cap_sockets_mtx);
 
 	return (error);
 } /* ng_btsocket_l2cap_process_l2cap_cfg_ind */
 
 /*
  * Process L2CA_Disconnect response
  */
 
 static int
 ng_btsocket_l2cap_process_l2ca_discon_rsp(struct ng_mesg *msg,
 		ng_btsocket_l2cap_rtentry_p rt)
 {
 	ng_l2cap_l2ca_discon_op	*op = NULL;
 	ng_btsocket_l2cap_pcb_t	*pcb = NULL;
 
 	/* Check message */
 	if (msg->header.arglen != sizeof(*op))
 		return (EMSGSIZE);
 
 	op = (ng_l2cap_l2ca_discon_op *)(msg->data);
 
 	mtx_lock(&ng_btsocket_l2cap_sockets_mtx);
 
 	/*
 	 * Socket layer must have issued L2CA_Disconnect request, so there 
 	 * must be a socket that wants to be disconnected. Use Netgraph 
 	 * message token to find it.
 	 */
 
 	pcb = ng_btsocket_l2cap_pcb_by_token(msg->header.token);
 	if (pcb == NULL) {
 		mtx_unlock(&ng_btsocket_l2cap_sockets_mtx);
 		return (0);
 	}
 
 	mtx_lock(&pcb->pcb_mtx);
 
 	/* XXX Close socket no matter what op->result says */
 	if (pcb->state != NG_BTSOCKET_L2CAP_CLOSED) {
        		NG_BTSOCKET_L2CAP_INFO(
 "%s: Got L2CA_Disconnect response, token=%d, src bdaddr=%x:%x:%x:%x:%x:%x, " \
 "dst bdaddr=%x:%x:%x:%x:%x:%x, psm=%d, lcid=%d, result=%d, state=%d\n",
 			__func__, msg->header.token,
 			pcb->src.b[5], pcb->src.b[4], pcb->src.b[3],
 			pcb->src.b[2], pcb->src.b[1], pcb->src.b[0],
 			pcb->dst.b[5], pcb->dst.b[4], pcb->dst.b[3],
 			pcb->dst.b[2], pcb->dst.b[1], pcb->dst.b[0],
 			pcb->psm, pcb->cid, op->result, pcb->state);
 
 		ng_btsocket_l2cap_untimeout(pcb);
 
 		pcb->state = NG_BTSOCKET_L2CAP_CLOSED;
 		soisdisconnected(pcb->so);
 	}
 
 	mtx_unlock(&pcb->pcb_mtx);
 	mtx_unlock(&ng_btsocket_l2cap_sockets_mtx);
 
 	return (0);
 } /* ng_btsocket_l2cap_process_l2ca_discon_rsp */
 
 /*
  * Process L2CA_Disconnect indicator
  */
 
 static int
 ng_btsocket_l2cap_process_l2ca_discon_ind(struct ng_mesg *msg,
 		ng_btsocket_l2cap_rtentry_p rt)
 {
 	ng_l2cap_l2ca_discon_ind_ip	*ip = NULL;
 	ng_btsocket_l2cap_pcb_t		*pcb = NULL;
 
 	/* Check message */
 	if (msg->header.arglen != sizeof(*ip))
 		return (EMSGSIZE);
 
 	ip = (ng_l2cap_l2ca_discon_ind_ip *)(msg->data);
 
 	mtx_lock(&ng_btsocket_l2cap_sockets_mtx);
 
 	/* Look for the socket with given channel ID */
 	pcb = ng_btsocket_l2cap_pcb_by_cid(&rt->src, ip->lcid);
 	if (pcb == NULL) {
 		mtx_unlock(&ng_btsocket_l2cap_sockets_mtx);
 		return (0);
 	}
 
 	/*
 	 * Channel has already been destroyed, so disconnect the socket 
 	 * and be done with it. If there was any pending request we can
 	 * not do anything here anyway.
 	 */
 
 	mtx_lock(&pcb->pcb_mtx);
 
        	NG_BTSOCKET_L2CAP_INFO(
 "%s: Got L2CA_Disconnect indicator, src bdaddr=%x:%x:%x:%x:%x:%x, " \
 "dst bdaddr=%x:%x:%x:%x:%x:%x, psm=%d, lcid=%d, state=%d\n",
 		__func__,
 		pcb->src.b[5], pcb->src.b[4], pcb->src.b[3],
 		pcb->src.b[2], pcb->src.b[1], pcb->src.b[0],
 		pcb->dst.b[5], pcb->dst.b[4], pcb->dst.b[3],
 		pcb->dst.b[2], pcb->dst.b[1], pcb->dst.b[0],
 		pcb->psm, pcb->cid, pcb->state);
 
 	if (pcb->flags & NG_BTSOCKET_L2CAP_TIMO)
 		ng_btsocket_l2cap_untimeout(pcb);
 
 	pcb->state = NG_BTSOCKET_L2CAP_CLOSED;
 	soisdisconnected(pcb->so);
 
 	mtx_unlock(&pcb->pcb_mtx);
 	mtx_unlock(&ng_btsocket_l2cap_sockets_mtx);
 
 	return (0);
 } /* ng_btsocket_l2cap_process_l2ca_discon_ind */
 
 /*
  * Process L2CA_Write response
  */
 
 static int 
 ng_btsocket_l2cap_process_l2ca_write_rsp(struct ng_mesg *msg,
 		ng_btsocket_l2cap_rtentry_p rt)
 {
 	ng_l2cap_l2ca_write_op	*op = NULL;
 	ng_btsocket_l2cap_pcb_t	*pcb = NULL;
 
 	/* Check message */
 	if (msg->header.arglen != sizeof(*op))
 		return (EMSGSIZE);
 
 	op = (ng_l2cap_l2ca_write_op *)(msg->data);
 
 	mtx_lock(&ng_btsocket_l2cap_sockets_mtx);
 
 	/* Look for the socket with given token */
 	pcb = ng_btsocket_l2cap_pcb_by_token(msg->header.token);
 	if (pcb == NULL) {
 		mtx_unlock(&ng_btsocket_l2cap_sockets_mtx);
 		return (ENOENT);
 	}
 
 	mtx_lock(&pcb->pcb_mtx);
 
        	NG_BTSOCKET_L2CAP_INFO(
 "%s: Got L2CA_Write response, src bdaddr=%x:%x:%x:%x:%x:%x, " \
 "dst bdaddr=%x:%x:%x:%x:%x:%x, psm=%d, lcid=%d, result=%d, length=%d, " \
 "state=%d\n",		__func__,
 			pcb->src.b[5], pcb->src.b[4], pcb->src.b[3],
 			pcb->src.b[2], pcb->src.b[1], pcb->src.b[0],
 			pcb->dst.b[5], pcb->dst.b[4], pcb->dst.b[3],
 			pcb->dst.b[2], pcb->dst.b[1], pcb->dst.b[0],
 			pcb->psm, pcb->cid, op->result, op->length,
 			pcb->state);
 
 	if (pcb->state != NG_BTSOCKET_L2CAP_OPEN) {
 		mtx_unlock(&pcb->pcb_mtx);
 		mtx_unlock(&ng_btsocket_l2cap_sockets_mtx);
 
 		return (ENOENT);
 	}
 	
 	ng_btsocket_l2cap_untimeout(pcb);
 
 	/*
  	 * Check if we have more data to send
  	 */
 
 	sbdroprecord(&pcb->so->so_snd);
 	if (pcb->so->so_snd.sb_cc > 0) {
 		if (ng_btsocket_l2cap_send2(pcb) == 0)
 			ng_btsocket_l2cap_timeout(pcb);
 		else
 			sbdroprecord(&pcb->so->so_snd); /* XXX */
 	}
 
 	/*
 	 * Now set the result, drop packet from the socket send queue and 
 	 * ask for more (wakeup sender)
 	 */
 
 	pcb->so->so_error = ng_btsocket_l2cap_result2errno(op->result);
 	sowwakeup(pcb->so);
 
 	mtx_unlock(&pcb->pcb_mtx);
 	mtx_unlock(&ng_btsocket_l2cap_sockets_mtx);
 
 	return (0);
 } /* ng_btsocket_l2cap_process_l2ca_write_rsp */
 
 /*
  * Send L2CA_Connect request
  */
 
 static int
 ng_btsocket_l2cap_send_l2ca_con_req(ng_btsocket_l2cap_pcb_p pcb)
 {
 	struct ng_mesg		*msg = NULL;
 	ng_l2cap_l2ca_con_ip	*ip = NULL;
 	int			 error = 0;
 
 	mtx_assert(&pcb->pcb_mtx, MA_OWNED);
 
 	if (pcb->rt == NULL || 
 	    pcb->rt->hook == NULL || NG_HOOK_NOT_VALID(pcb->rt->hook))
 		return (ENETDOWN); 
 
 	NG_MKMESSAGE(msg, NGM_L2CAP_COOKIE, NGM_L2CAP_L2CA_CON,
 		sizeof(*ip), M_NOWAIT);
 	if (msg == NULL)
 		return (ENOMEM);
 
 	msg->header.token = pcb->token;
 
 	ip = (ng_l2cap_l2ca_con_ip *)(msg->data);
 	bcopy(&pcb->dst, &ip->bdaddr, sizeof(ip->bdaddr));
 	ip->psm = pcb->psm;
 
 	NG_SEND_MSG_HOOK(error, ng_btsocket_l2cap_node, msg,pcb->rt->hook, 0);
 
 	return (error);
 } /* ng_btsocket_l2cap_send_l2ca_con_req */
 
 /*
  * Send L2CA_Connect response
  */
 
 static int
 ng_btsocket_l2cap_send_l2ca_con_rsp_req(u_int32_t token,
 		ng_btsocket_l2cap_rtentry_p rt, bdaddr_p dst, int ident, 
 		int lcid, int result)
 {
 	struct ng_mesg			*msg = NULL;
 	ng_l2cap_l2ca_con_rsp_ip	*ip = NULL;
 	int				 error = 0;
 
 	if (rt == NULL || rt->hook == NULL || NG_HOOK_NOT_VALID(rt->hook))
 		return (ENETDOWN); 
 
 	NG_MKMESSAGE(msg, NGM_L2CAP_COOKIE, NGM_L2CAP_L2CA_CON_RSP,
 		sizeof(*ip), M_NOWAIT);
 	if (msg == NULL)
 		return (ENOMEM);
 
 	msg->header.token = token;
 
 	ip = (ng_l2cap_l2ca_con_rsp_ip *)(msg->data);
 	bcopy(dst, &ip->bdaddr, sizeof(ip->bdaddr));
 	ip->ident = ident;
 	ip->lcid = lcid;
 	ip->result = result;
 	ip->status = 0;
 
 	NG_SEND_MSG_HOOK(error, ng_btsocket_l2cap_node, msg, rt->hook, 0);
 
 	return (error);
 } /* ng_btsocket_l2cap_send_l2ca_con_rsp_req */
 
 /*
  * Send L2CA_Config request
  */
 
 static int
 ng_btsocket_l2cap_send_l2ca_cfg_req(ng_btsocket_l2cap_pcb_p pcb)
 {
 	struct ng_mesg		*msg = NULL;
 	ng_l2cap_l2ca_cfg_ip	*ip = NULL;
 	int			 error = 0;
 
 	mtx_assert(&pcb->pcb_mtx, MA_OWNED);
 
 	if (pcb->rt == NULL || 
 	    pcb->rt->hook == NULL || NG_HOOK_NOT_VALID(pcb->rt->hook))
 		return (ENETDOWN); 
 
 	NG_MKMESSAGE(msg, NGM_L2CAP_COOKIE, NGM_L2CAP_L2CA_CFG,
 		sizeof(*ip), M_NOWAIT);
 	if (msg == NULL)
 		return (ENOMEM);
 
 	msg->header.token = pcb->token;
 
 	ip = (ng_l2cap_l2ca_cfg_ip *)(msg->data);
 	ip->lcid = pcb->cid;
 	ip->imtu = pcb->imtu;
 	bcopy(&pcb->oflow, &ip->oflow, sizeof(ip->oflow));
 	ip->flush_timo = pcb->flush_timo;
 	ip->link_timo = pcb->link_timo;
 
 	NG_SEND_MSG_HOOK(error, ng_btsocket_l2cap_node, msg,pcb->rt->hook, 0);
 
 	return (error);
 } /* ng_btsocket_l2cap_send_l2ca_cfg_req */
 
 /*
  * Send L2CA_Config response
  */
 
 static int
 ng_btsocket_l2cap_send_l2ca_cfg_rsp(ng_btsocket_l2cap_pcb_p pcb)
 {
 	struct ng_mesg			*msg = NULL;
 	ng_l2cap_l2ca_cfg_rsp_ip	*ip = NULL;
 	int				 error = 0;
 
 	mtx_assert(&pcb->pcb_mtx, MA_OWNED);
 
 	if (pcb->rt == NULL || 
 	    pcb->rt->hook == NULL || NG_HOOK_NOT_VALID(pcb->rt->hook))
 		return (ENETDOWN); 
 
 	NG_MKMESSAGE(msg, NGM_L2CAP_COOKIE, NGM_L2CAP_L2CA_CFG_RSP,
 		sizeof(*ip), M_NOWAIT);
 	if (msg == NULL)
 		return (ENOMEM);
 
 	msg->header.token = pcb->token;
 
 	ip = (ng_l2cap_l2ca_cfg_rsp_ip *)(msg->data);
 	ip->lcid = pcb->cid;
 	ip->omtu = pcb->omtu;
 	bcopy(&pcb->iflow, &ip->iflow, sizeof(ip->iflow));
 
 	NG_SEND_MSG_HOOK(error, ng_btsocket_l2cap_node, msg, pcb->rt->hook, 0);
 
 	return (error);
 } /* ng_btsocket_l2cap_send_l2ca_cfg_rsp */
 
 /*
  * Send L2CA_Disconnect request
  */
 
 static int
 ng_btsocket_l2cap_send_l2ca_discon_req(u_int32_t token,
 		ng_btsocket_l2cap_pcb_p pcb)
 {
 	struct ng_mesg		*msg = NULL;
 	ng_l2cap_l2ca_discon_ip	*ip = NULL;
 	int			 error = 0;
 
 	mtx_assert(&pcb->pcb_mtx, MA_OWNED);
 
 	if (pcb->rt == NULL || 
 	    pcb->rt->hook == NULL || NG_HOOK_NOT_VALID(pcb->rt->hook))
 		return (ENETDOWN); 
 
 	NG_MKMESSAGE(msg, NGM_L2CAP_COOKIE, NGM_L2CAP_L2CA_DISCON,
 		sizeof(*ip), M_NOWAIT);
 	if (msg == NULL)
 		return (ENOMEM);
 
 	msg->header.token = token;
 
 	ip = (ng_l2cap_l2ca_discon_ip *)(msg->data);
 	ip->lcid = pcb->cid;
 
 	NG_SEND_MSG_HOOK(error, ng_btsocket_l2cap_node, msg,pcb->rt->hook, 0);
 
 	return (error);
 } /* ng_btsocket_l2cap_send_l2ca_discon_req */
 
 /*****************************************************************************
  *****************************************************************************
  **                              Socket interface
  *****************************************************************************
  *****************************************************************************/
 
 /*
  * L2CAP sockets data input routine
  */
 
 static void
 ng_btsocket_l2cap_data_input(struct mbuf *m, hook_p hook)
 {
 	ng_l2cap_hdr_t			*hdr = NULL;
 	ng_l2cap_clt_hdr_t		*clt_hdr = NULL;
 	ng_btsocket_l2cap_pcb_t		*pcb = NULL;
 	ng_btsocket_l2cap_rtentry_t	*rt = NULL;
 
 	if (hook == NULL) {
 		NG_BTSOCKET_L2CAP_ALERT(
 "%s: Invalid source hook for L2CAP data packet\n", __func__);
 		goto drop;
 	}
 
 	rt = (ng_btsocket_l2cap_rtentry_t *) NG_HOOK_PRIVATE(hook);
 	if (rt == NULL) {
 		NG_BTSOCKET_L2CAP_ALERT(
 "%s: Could not find out source bdaddr for L2CAP data packet\n", __func__);
 		goto drop;
 	}
 
 	/* Make sure we can access header */
 	if (m->m_pkthdr.len < sizeof(*hdr)) {
 		NG_BTSOCKET_L2CAP_ERR(
 "%s: L2CAP data packet too small, len=%d\n", __func__, m->m_pkthdr.len);
 		goto drop;
 	}
 
 	if (m->m_len < sizeof(*hdr)) { 
 		m = m_pullup(m, sizeof(*hdr));
 		if (m == NULL)
 			goto drop;
 	}
 
 	/* Strip L2CAP packet header and verify packet length */
 	hdr = mtod(m, ng_l2cap_hdr_t *);
 	m_adj(m, sizeof(*hdr));
 
 	if (hdr->length != m->m_pkthdr.len) {
 		NG_BTSOCKET_L2CAP_ERR(
 "%s: Bad L2CAP data packet length, len=%d, length=%d\n",
 			__func__, m->m_pkthdr.len, hdr->length);
 		goto drop;
 	}
 
 	/*
 	 * Now process packet. Two cases:
 	 *
 	 * 1) Normal packet (cid != 2) then find connected socket and append
 	 *    mbuf to the socket queue. Wakeup socket.
 	 *
 	 * 2) Broadcast packet (cid == 2) then find all sockets that connected
 	 *    to the given PSM and have SO_BROADCAST bit set and append mbuf
 	 *    to the socket queue. Wakeup socket.
 	 */
 
 	NG_BTSOCKET_L2CAP_INFO(
 "%s: Received L2CAP data packet: src bdaddr=%x:%x:%x:%x:%x:%x, " \
 "dcid=%d, length=%d\n",
 		__func__, 
 		rt->src.b[5], rt->src.b[4], rt->src.b[3],
 		rt->src.b[2], rt->src.b[1], rt->src.b[0],
 		hdr->dcid, hdr->length);
 
 	if (hdr->dcid >= NG_L2CAP_FIRST_CID) {
 
 		mtx_lock(&ng_btsocket_l2cap_sockets_mtx);
 
 		/* Normal packet: find connected socket */
 		pcb = ng_btsocket_l2cap_pcb_by_cid(&rt->src, hdr->dcid);
 		if (pcb == NULL) {
 			mtx_unlock(&ng_btsocket_l2cap_sockets_mtx);
 			goto drop;
 		}
 
 		mtx_lock(&pcb->pcb_mtx);
 
 		if (pcb->state != NG_BTSOCKET_L2CAP_OPEN) {
 			NG_BTSOCKET_L2CAP_ERR(
 "%s: No connected socket found, src bdaddr=%x:%x:%x:%x:%x:%x, dcid=%d, " \
 "state=%d\n",			__func__,
 				rt->src.b[5], rt->src.b[4], rt->src.b[3],
 				rt->src.b[2], rt->src.b[1], rt->src.b[0],
 				hdr->dcid, pcb->state);
 
 			mtx_unlock(&pcb->pcb_mtx);
 			mtx_unlock(&ng_btsocket_l2cap_sockets_mtx);
 			goto drop;
 		}
 
 		/* Check packet size against socket's incoming MTU */
 		if (hdr->length > pcb->imtu) {
 			NG_BTSOCKET_L2CAP_ERR(
 "%s: L2CAP data packet too big, src bdaddr=%x:%x:%x:%x:%x:%x, " \
 "dcid=%d, length=%d, imtu=%d\n",
 				__func__, 
 				rt->src.b[5], rt->src.b[4], rt->src.b[3],
 				rt->src.b[2], rt->src.b[1], rt->src.b[0],
 				hdr->dcid, hdr->length, pcb->imtu);
 
 			mtx_unlock(&pcb->pcb_mtx);
 			mtx_unlock(&ng_btsocket_l2cap_sockets_mtx);
 			goto drop;
 		}
 
 		/* Check if we have enough space in socket receive queue */
 		if (m->m_pkthdr.len > sbspace(&pcb->so->so_rcv)) {
 
 			/* 
 			 * This is really bad. Receive queue on socket does
 			 * not have enough space for the packet. We do not 
 			 * have any other choice but drop the packet. L2CAP 
 			 * does not provide any flow control.
 			 */
 
 			NG_BTSOCKET_L2CAP_ERR(
 "%s: Not enough space in socket receive queue. Dropping L2CAP data packet, " \
 "src bdaddr=%x:%x:%x:%x:%x:%x, dcid=%d, len=%d, space=%ld\n",
 				__func__,
 				rt->src.b[5], rt->src.b[4], rt->src.b[3],
 				rt->src.b[2], rt->src.b[1], rt->src.b[0],
 				hdr->dcid, m->m_pkthdr.len,
 				sbspace(&pcb->so->so_rcv));
 
 			mtx_unlock(&pcb->pcb_mtx);
 			mtx_unlock(&ng_btsocket_l2cap_sockets_mtx);
 			goto drop;
 		}
 
 		/* Append packet to the socket receive queue and wakeup */
 		sbappendrecord(&pcb->so->so_rcv, m);
 		m = NULL;
 
 		sorwakeup(pcb->so);
 
 		mtx_unlock(&pcb->pcb_mtx);
 		mtx_unlock(&ng_btsocket_l2cap_sockets_mtx);
 	} else if (hdr->dcid == NG_L2CAP_CLT_CID) {
 		/* Broadcast packet: give packet to all sockets  */
 
 		/* Check packet size against connectionless MTU */
 		if (hdr->length > NG_L2CAP_MTU_DEFAULT) {
 			NG_BTSOCKET_L2CAP_ERR(
 "%s: Connectionless L2CAP data packet too big, " \
 "src bdaddr=%x:%x:%x:%x:%x:%x, length=%d\n",
 				__func__,
 				rt->src.b[5], rt->src.b[4], rt->src.b[3],
 				rt->src.b[2], rt->src.b[1], rt->src.b[0],
 				hdr->length);
 			goto drop;
 		}
 
 		/* Make sure we can access connectionless header */
 		if (m->m_pkthdr.len < sizeof(*clt_hdr)) {
 			NG_BTSOCKET_L2CAP_ERR(
 "%s: Can not get L2CAP connectionless packet header, " \
 "src bdaddr=%x:%x:%x:%x:%x:%x, length=%d\n",
 				__func__,
 				rt->src.b[5], rt->src.b[4], rt->src.b[3],
 				rt->src.b[2], rt->src.b[1], rt->src.b[0],
 				hdr->length);
 			goto drop;
 		}
 
 		if (m->m_len < sizeof(*clt_hdr)) {
 			m = m_pullup(m, sizeof(*clt_hdr));
 			if (m == NULL)
 				goto drop;
 		}
 
 		/* Strip connectionless header and deliver packet */
 		clt_hdr = mtod(m, ng_l2cap_clt_hdr_t *);
 		m_adj(m, sizeof(*clt_hdr));
 
 		NG_BTSOCKET_L2CAP_INFO(
 "%s: Got L2CAP connectionless data packet, " \
 "src bdaddr=%x:%x:%x:%x:%x:%x, psm=%d, length=%d\n",
 			__func__,
 			rt->src.b[5], rt->src.b[4], rt->src.b[3],
 			rt->src.b[2], rt->src.b[1], rt->src.b[0],
 			clt_hdr->psm, hdr->length);
 
 		mtx_lock(&ng_btsocket_l2cap_sockets_mtx);
 
 		LIST_FOREACH(pcb, &ng_btsocket_l2cap_sockets, next) {
 			struct mbuf	*copy = NULL;
 
 			mtx_lock(&pcb->pcb_mtx);
 
 			if (bcmp(&rt->src, &pcb->src, sizeof(pcb->src)) != 0 ||
 			    pcb->psm != clt_hdr->psm || 
 			    pcb->state != NG_BTSOCKET_L2CAP_OPEN || 
 			    (pcb->so->so_options & SO_BROADCAST) == 0 || 
 			    m->m_pkthdr.len > sbspace(&pcb->so->so_rcv))
 				goto next;
 
 			/*
 			 * Create a copy of the packet and append it to the 
 			 * socket's queue. If m_dup() failed - no big deal
 			 * it is a broadcast traffic after all
 			 */
 
 			copy = m_dup(m, M_NOWAIT);
 			if (copy != NULL) {
 				sbappendrecord(&pcb->so->so_rcv, copy);
 				sorwakeup(pcb->so);
 			}
 next:
 			mtx_unlock(&pcb->pcb_mtx);
 		}
 
 		mtx_unlock(&ng_btsocket_l2cap_sockets_mtx);
 	}
 drop:
 	NG_FREE_M(m); /* checks for m != NULL */
 } /* ng_btsocket_l2cap_data_input */
 
 /*
  * L2CAP sockets default message input routine
  */
 
 static void
 ng_btsocket_l2cap_default_msg_input(struct ng_mesg *msg, hook_p hook)
 {
 	switch (msg->header.cmd) {
 	case NGM_L2CAP_NODE_HOOK_INFO: {
 		ng_btsocket_l2cap_rtentry_t	*rt = NULL;
 
 		if (hook == NULL || msg->header.arglen != sizeof(bdaddr_t))
 			break;
 
 		if (bcmp(msg->data, NG_HCI_BDADDR_ANY, sizeof(bdaddr_t)) == 0)
 			break;
 
 		mtx_lock(&ng_btsocket_l2cap_rt_mtx);
 
 		rt = (ng_btsocket_l2cap_rtentry_t *) NG_HOOK_PRIVATE(hook);
 		if (rt == NULL) {
 			rt = malloc(sizeof(*rt),
 				M_NETGRAPH_BTSOCKET_L2CAP, M_NOWAIT|M_ZERO);
 			if (rt == NULL) {
 				mtx_unlock(&ng_btsocket_l2cap_rt_mtx);
 				break;
 			}
 
 			LIST_INSERT_HEAD(&ng_btsocket_l2cap_rt, rt, next);
 
 			NG_HOOK_SET_PRIVATE(hook, rt);
 		}
 
 		bcopy(msg->data, &rt->src, sizeof(rt->src));
 		rt->hook = hook;
 
 		mtx_unlock(&ng_btsocket_l2cap_rt_mtx);
 
 		NG_BTSOCKET_L2CAP_INFO(
 "%s: Updating hook \"%s\", src bdaddr=%x:%x:%x:%x:%x:%x\n",
 			__func__, NG_HOOK_NAME(hook), 
 			rt->src.b[5], rt->src.b[4], rt->src.b[3], 
 			rt->src.b[2], rt->src.b[1], rt->src.b[0]);
 		} break;
 
 	default:
 		NG_BTSOCKET_L2CAP_WARN(
 "%s: Unknown message, cmd=%d\n", __func__, msg->header.cmd);
 		break;
 	}
 
 	NG_FREE_MSG(msg); /* Checks for msg != NULL */
 } /* ng_btsocket_l2cap_default_msg_input */
 
 /*
  * L2CAP sockets L2CA message input routine
  */
 
 static void
 ng_btsocket_l2cap_l2ca_msg_input(struct ng_mesg *msg, hook_p hook)
 {
 	ng_btsocket_l2cap_rtentry_p	rt = NULL;
 
 	if (hook == NULL) {
 		NG_BTSOCKET_L2CAP_ALERT(
 "%s: Invalid source hook for L2CA message\n", __func__);
 		goto drop;
 	}
 
 	rt = (ng_btsocket_l2cap_rtentry_p) NG_HOOK_PRIVATE(hook);
 	if (rt == NULL) {
 		NG_BTSOCKET_L2CAP_ALERT(
 "%s: Could not find out source bdaddr for L2CA message\n", __func__);
 		goto drop;
 	}
 
 	switch (msg->header.cmd) {
 	case NGM_L2CAP_L2CA_CON: /* L2CA_Connect response */
 		ng_btsocket_l2cap_process_l2ca_con_req_rsp(msg, rt);
 		break;
 
 	case NGM_L2CAP_L2CA_CON_RSP: /* L2CA_ConnectRsp response */
 		ng_btsocket_l2cap_process_l2ca_con_rsp_rsp(msg, rt);
 		break;
 
 	case NGM_L2CAP_L2CA_CON_IND: /* L2CA_Connect indicator */
 		ng_btsocket_l2cap_process_l2ca_con_ind(msg, rt);
 		break;
 
 	case NGM_L2CAP_L2CA_CFG: /* L2CA_Config response */
 		ng_btsocket_l2cap_process_l2ca_cfg_req_rsp(msg, rt);
 		break;
 
 	case NGM_L2CAP_L2CA_CFG_RSP: /* L2CA_ConfigRsp response */
 		ng_btsocket_l2cap_process_l2ca_cfg_rsp_rsp(msg, rt);
 		break;
 
 	case NGM_L2CAP_L2CA_CFG_IND: /* L2CA_Config indicator */
 		ng_btsocket_l2cap_process_l2ca_cfg_ind(msg, rt);
 		break;
 
 	case NGM_L2CAP_L2CA_DISCON: /* L2CA_Disconnect response */
 		ng_btsocket_l2cap_process_l2ca_discon_rsp(msg, rt);
 		break;
 
 	case NGM_L2CAP_L2CA_DISCON_IND: /* L2CA_Disconnect indicator */
 		ng_btsocket_l2cap_process_l2ca_discon_ind(msg, rt);
 		break;
 
 	case NGM_L2CAP_L2CA_WRITE: /* L2CA_Write response */
 		ng_btsocket_l2cap_process_l2ca_write_rsp(msg, rt);
 		break;
 
 	/* XXX FIXME add other L2CA messages */
 
 	default:
 		NG_BTSOCKET_L2CAP_WARN(
 "%s: Unknown L2CA message, cmd=%d\n", __func__, msg->header.cmd);
 		break;
 	}
 drop:
 	NG_FREE_MSG(msg);
 } /* ng_btsocket_l2cap_l2ca_msg_input */
 
 /*
  * L2CAP sockets input routine
  */
 
 static void
 ng_btsocket_l2cap_input(void *context, int pending)
 {
 	item_p	item = NULL;
 	hook_p	hook = NULL;
 
 	for (;;) {
 		mtx_lock(&ng_btsocket_l2cap_queue_mtx);
 		NG_BT_ITEMQ_DEQUEUE(&ng_btsocket_l2cap_queue, item);
 		mtx_unlock(&ng_btsocket_l2cap_queue_mtx);
 
 		if (item == NULL)
 			break;
 
 		NGI_GET_HOOK(item, hook);
 		if (hook != NULL && NG_HOOK_NOT_VALID(hook))
 			goto drop;
 
 		switch(item->el_flags & NGQF_TYPE) {
 		case NGQF_DATA: {
 			struct mbuf     *m = NULL;
 
 			NGI_GET_M(item, m);
 			ng_btsocket_l2cap_data_input(m, hook);
 			} break;
 
 		case NGQF_MESG: {
 			struct ng_mesg  *msg = NULL;
 
 			NGI_GET_MSG(item, msg);
 
 			switch (msg->header.cmd) {
 			case NGM_L2CAP_L2CA_CON:
 			case NGM_L2CAP_L2CA_CON_RSP:
 			case NGM_L2CAP_L2CA_CON_IND:
 			case NGM_L2CAP_L2CA_CFG:
 			case NGM_L2CAP_L2CA_CFG_RSP:
 			case NGM_L2CAP_L2CA_CFG_IND: 
 			case NGM_L2CAP_L2CA_DISCON:
 			case NGM_L2CAP_L2CA_DISCON_IND:
 			case NGM_L2CAP_L2CA_WRITE:
 			/* XXX FIXME add other L2CA messages */
 				ng_btsocket_l2cap_l2ca_msg_input(msg, hook);
 				break;
 
 			default:
 				ng_btsocket_l2cap_default_msg_input(msg, hook);
 				break;
 			}
 			} break;
 
 		default:
 			KASSERT(0,
 ("%s: invalid item type=%ld\n", __func__, (item->el_flags & NGQF_TYPE)));
 			break;
 		}
 drop:
 		if (hook != NULL)
 			NG_HOOK_UNREF(hook);
 
 		NG_FREE_ITEM(item);
 	}
 } /* ng_btsocket_l2cap_input */
 
 /*
  * Route cleanup task. Gets scheduled when hook is disconnected. Here we 
  * will find all sockets that use "invalid" hook and disconnect them.
  */
 
 static void
 ng_btsocket_l2cap_rtclean(void *context, int pending)
 {
 	ng_btsocket_l2cap_pcb_p		pcb = NULL, pcb_next = NULL;
 	ng_btsocket_l2cap_rtentry_p	rt = NULL;
 
 	mtx_lock(&ng_btsocket_l2cap_rt_mtx);
 	mtx_lock(&ng_btsocket_l2cap_sockets_mtx);
 
 	/*
 	 * First disconnect all sockets that use "invalid" hook
 	 */
 
 	for (pcb = LIST_FIRST(&ng_btsocket_l2cap_sockets); pcb != NULL; ) {
 		mtx_lock(&pcb->pcb_mtx);
 		pcb_next = LIST_NEXT(pcb, next);
 
 		if (pcb->rt != NULL &&
 		    pcb->rt->hook != NULL && NG_HOOK_NOT_VALID(pcb->rt->hook)) {
 			if (pcb->flags & NG_BTSOCKET_L2CAP_TIMO)
 				ng_btsocket_l2cap_untimeout(pcb);
 
 			pcb->so->so_error = ENETDOWN;
 			pcb->state = NG_BTSOCKET_L2CAP_CLOSED;
 			soisdisconnected(pcb->so);
 
 			pcb->token = 0;
 			pcb->cid = 0;
 			pcb->rt = NULL;
 		}
 
 		mtx_unlock(&pcb->pcb_mtx);
 		pcb = pcb_next;
 	}
 
 	/*
 	 * Now cleanup routing table
 	 */
 
 	for (rt = LIST_FIRST(&ng_btsocket_l2cap_rt); rt != NULL; ) {
 		ng_btsocket_l2cap_rtentry_p	rt_next = LIST_NEXT(rt, next);
 
 		if (rt->hook != NULL && NG_HOOK_NOT_VALID(rt->hook)) {
 			LIST_REMOVE(rt, next);
 
 			NG_HOOK_SET_PRIVATE(rt->hook, NULL);
 			NG_HOOK_UNREF(rt->hook); /* Remove extra reference */
 
 			bzero(rt, sizeof(*rt));
 			free(rt, M_NETGRAPH_BTSOCKET_L2CAP);
 		}
 
 		rt = rt_next;
 	}
 
 	mtx_unlock(&ng_btsocket_l2cap_sockets_mtx);
 	mtx_unlock(&ng_btsocket_l2cap_rt_mtx);
 } /* ng_btsocket_l2cap_rtclean */
 
 /*
  * Initialize everything
  */
 
 void
 ng_btsocket_l2cap_init(void)
 {
 	int	error = 0;
 
+	/* Skip initialization of globals for non-default instances. */
+	if (!IS_DEFAULT_VNET(curvnet))
+		return;
+
 	ng_btsocket_l2cap_node = NULL;
 	ng_btsocket_l2cap_debug_level = NG_BTSOCKET_WARN_LEVEL;
 
 	/* Register Netgraph node type */
 	error = ng_newtype(&typestruct);
 	if (error != 0) {
 		NG_BTSOCKET_L2CAP_ALERT(
 "%s: Could not register Netgraph node type, error=%d\n", __func__, error);
 
                 return;
 	}
 
 	/* Create Netgrapg node */
 	error = ng_make_node_common(&typestruct, &ng_btsocket_l2cap_node);
 	if (error != 0) {
 		NG_BTSOCKET_L2CAP_ALERT(
 "%s: Could not create Netgraph node, error=%d\n", __func__, error);
 
 		ng_btsocket_l2cap_node = NULL;
 
 		return;
 	}
 
 	error = ng_name_node(ng_btsocket_l2cap_node,
 				NG_BTSOCKET_L2CAP_NODE_TYPE);
 	if (error != 0) {
 		NG_BTSOCKET_L2CAP_ALERT(
 "%s: Could not name Netgraph node, error=%d\n", __func__, error);
 
 		NG_NODE_UNREF(ng_btsocket_l2cap_node);
 		ng_btsocket_l2cap_node = NULL;
 
 		return;
 	}
 
 	/* Create input queue */
 	NG_BT_ITEMQ_INIT(&ng_btsocket_l2cap_queue, ifqmaxlen);
 	mtx_init(&ng_btsocket_l2cap_queue_mtx,
 		"btsocks_l2cap_queue_mtx", NULL, MTX_DEF);
 	TASK_INIT(&ng_btsocket_l2cap_queue_task, 0,
 		ng_btsocket_l2cap_input, NULL);
 
 	/* Create list of sockets */
 	LIST_INIT(&ng_btsocket_l2cap_sockets);
 	mtx_init(&ng_btsocket_l2cap_sockets_mtx,
 		"btsocks_l2cap_sockets_mtx", NULL, MTX_DEF);
 
 	/* Routing table */
 	LIST_INIT(&ng_btsocket_l2cap_rt);
 	mtx_init(&ng_btsocket_l2cap_rt_mtx,
 		"btsocks_l2cap_rt_mtx", NULL, MTX_DEF);
 	TASK_INIT(&ng_btsocket_l2cap_rt_task, 0,
 		ng_btsocket_l2cap_rtclean, NULL);
 } /* ng_btsocket_l2cap_init */
 
 /*
  * Abort connection on socket
  */
 
 void
 ng_btsocket_l2cap_abort(struct socket *so)
 {
 	so->so_error = ECONNABORTED;
 
 	(void)ng_btsocket_l2cap_disconnect(so);
 } /* ng_btsocket_l2cap_abort */
 
 void
 ng_btsocket_l2cap_close(struct socket *so)
 {
 
 	(void)ng_btsocket_l2cap_disconnect(so);
 } /* ng_btsocket_l2cap_close */
 
 /*
  * Accept connection on socket. Nothing to do here, socket must be connected
  * and ready, so just return peer address and be done with it.
  */
 
 int
 ng_btsocket_l2cap_accept(struct socket *so, struct sockaddr **nam)
 {
 	if (ng_btsocket_l2cap_node == NULL) 
 		return (EINVAL);
 
 	return (ng_btsocket_l2cap_peeraddr(so, nam));
 } /* ng_btsocket_l2cap_accept */
 
 /*
  * Create and attach new socket
  */
 
 int
 ng_btsocket_l2cap_attach(struct socket *so, int proto, struct thread *td)
 {
 	static u_int32_t	token = 0;
 	ng_btsocket_l2cap_pcb_p	pcb = so2l2cap_pcb(so);
 	int			error;
 
 	/* Check socket and protocol */
 	if (ng_btsocket_l2cap_node == NULL) 
 		return (EPROTONOSUPPORT);
 	if (so->so_type != SOCK_SEQPACKET)
 		return (ESOCKTNOSUPPORT);
 
 #if 0 /* XXX sonewconn() calls "pru_attach" with proto == 0 */
 	if (proto != 0) 
 		if (proto != BLUETOOTH_PROTO_L2CAP)
 			return (EPROTONOSUPPORT);
 #endif /* XXX */
 
 	if (pcb != NULL)
 		return (EISCONN);
 
 	/* Reserve send and receive space if it is not reserved yet */
 	if ((so->so_snd.sb_hiwat == 0) || (so->so_rcv.sb_hiwat == 0)) {
 		error = soreserve(so, NG_BTSOCKET_L2CAP_SENDSPACE,
 					NG_BTSOCKET_L2CAP_RECVSPACE);
 		if (error != 0)
 			return (error);
 	}
 
 	/* Allocate the PCB */
         pcb = malloc(sizeof(*pcb),
 		M_NETGRAPH_BTSOCKET_L2CAP, M_NOWAIT | M_ZERO);
         if (pcb == NULL)
                 return (ENOMEM);
 
 	/* Link the PCB and the socket */
 	so->so_pcb = (caddr_t) pcb;
 	pcb->so = so;
 	pcb->state = NG_BTSOCKET_L2CAP_CLOSED;
 
 	/* Initialize PCB */
 	pcb->imtu = pcb->omtu = NG_L2CAP_MTU_DEFAULT;
 
 	/* Default flow */
 	pcb->iflow.flags = 0x0;
 	pcb->iflow.service_type = NG_HCI_SERVICE_TYPE_BEST_EFFORT;
 	pcb->iflow.token_rate = 0xffffffff; /* maximum */
 	pcb->iflow.token_bucket_size = 0xffffffff; /* maximum */
 	pcb->iflow.peak_bandwidth = 0x00000000; /* maximum */
 	pcb->iflow.latency = 0xffffffff; /* don't care */
 	pcb->iflow.delay_variation = 0xffffffff; /* don't care */
 
 	bcopy(&pcb->iflow, &pcb->oflow, sizeof(pcb->oflow));
 
 	pcb->flush_timo = NG_L2CAP_FLUSH_TIMO_DEFAULT;
 	pcb->link_timo = NG_L2CAP_LINK_TIMO_DEFAULT;
 
 	callout_handle_init(&pcb->timo);
 
 	/*
 	 * XXX Mark PCB mutex as DUPOK to prevent "duplicated lock of
 	 * the same type" message. When accepting new L2CAP connection 
 	 * ng_btsocket_l2cap_process_l2ca_con_ind() holds both PCB mutexes 
 	 * for "old" (accepting) PCB and "new" (created) PCB.
 	 */
 		
 	mtx_init(&pcb->pcb_mtx, "btsocks_l2cap_pcb_mtx", NULL,
 		MTX_DEF|MTX_DUPOK);
 
         /*
 	 * Add the PCB to the list
 	 * 
 	 * XXX FIXME VERY IMPORTANT!
 	 *
 	 * This is totally FUBAR. We could get here in two cases:
 	 *
 	 * 1) When user calls socket()
 	 * 2) When we need to accept new incomming connection and call 
 	 *    sonewconn()
 	 *
 	 * In the first case we must acquire ng_btsocket_l2cap_sockets_mtx.
 	 * In the second case we hold ng_btsocket_l2cap_sockets_mtx already.
 	 * So we now need to distinguish between these cases. From reading
 	 * /sys/kern/uipc_socket.c we can find out that sonewconn() calls
 	 * pru_attach with proto == 0 and td == NULL. For now use this fact
 	 * to figure out if we were called from socket() or from sonewconn().
 	 */
 
 	if (td != NULL)
 		mtx_lock(&ng_btsocket_l2cap_sockets_mtx);
 	else
 		mtx_assert(&ng_btsocket_l2cap_sockets_mtx, MA_OWNED);
 	
 	/* Set PCB token. Use ng_btsocket_l2cap_sockets_mtx for protection */
 	if (++ token == 0)
 		token ++;
 
 	pcb->token = token;
 
 	LIST_INSERT_HEAD(&ng_btsocket_l2cap_sockets, pcb, next);
 
 	if (td != NULL)
 		mtx_unlock(&ng_btsocket_l2cap_sockets_mtx);
 
         return (0);
 } /* ng_btsocket_l2cap_attach */
 
 /*
  * Bind socket
  */
 
 int
 ng_btsocket_l2cap_bind(struct socket *so, struct sockaddr *nam, 
 		struct thread *td)
 {
 	ng_btsocket_l2cap_pcb_t	*pcb = NULL;
 	struct sockaddr_l2cap	*sa = (struct sockaddr_l2cap *) nam;
 	int			 psm, error = 0;
 
 	if (ng_btsocket_l2cap_node == NULL) 
 		return (EINVAL);
 
 	/* Verify address */
 	if (sa == NULL)
 		return (EINVAL);
 	if (sa->l2cap_family != AF_BLUETOOTH)
 		return (EAFNOSUPPORT);
 	if (sa->l2cap_len != sizeof(*sa))
 		return (EINVAL);
 
 	psm = le16toh(sa->l2cap_psm);
 
 	/* 
 	 * Check if other socket has this address already (look for exact
 	 * match PSM and bdaddr) and assign socket address if it's available.
 	 *
 	 * Note: socket can be bound to ANY PSM (zero) thus allowing several
 	 * channels with the same PSM between the same pair of BD_ADDR'es.
 	 */
 
 	mtx_lock(&ng_btsocket_l2cap_sockets_mtx);
 
 	LIST_FOREACH(pcb, &ng_btsocket_l2cap_sockets, next)
 		if (psm != 0 && psm == pcb->psm &&
 		    bcmp(&pcb->src, &sa->l2cap_bdaddr, sizeof(bdaddr_t)) == 0)
 			break;
 
 	if (pcb == NULL) {
 		/* Set socket address */
 		pcb = so2l2cap_pcb(so);
 		if (pcb != NULL) {
 			bcopy(&sa->l2cap_bdaddr, &pcb->src, sizeof(pcb->src));
 			pcb->psm = psm;
 		} else
 			error = EINVAL;
 	} else
 		error = EADDRINUSE;
 
 	mtx_unlock(&ng_btsocket_l2cap_sockets_mtx);
 
 	return (error);
 } /* ng_btsocket_l2cap_bind */
 
 /*
  * Connect socket
  */
 
 int
 ng_btsocket_l2cap_connect(struct socket *so, struct sockaddr *nam, 
 		struct thread *td)
 {
 	ng_btsocket_l2cap_pcb_t		*pcb = so2l2cap_pcb(so);
 	struct sockaddr_l2cap		*sa = (struct sockaddr_l2cap *) nam;
 	ng_btsocket_l2cap_rtentry_t	*rt = NULL;
 	int				 have_src, error = 0;
 
 	/* Check socket */
 	if (pcb == NULL)
 		return (EINVAL);
 	if (ng_btsocket_l2cap_node == NULL) 
 		return (EINVAL);
 	if (pcb->state == NG_BTSOCKET_L2CAP_CONNECTING)
 		return (EINPROGRESS);
 
 	/* Verify address */
 	if (sa == NULL)
 		return (EINVAL);
 	if (sa->l2cap_family != AF_BLUETOOTH)
 		return (EAFNOSUPPORT);
 	if (sa->l2cap_len != sizeof(*sa))
 		return (EINVAL);
 	if (sa->l2cap_psm == 0 ||
 	    bcmp(&sa->l2cap_bdaddr, NG_HCI_BDADDR_ANY, sizeof(bdaddr_t)) == 0)
 		return (EDESTADDRREQ);
 	if (pcb->psm != 0 && pcb->psm != le16toh(sa->l2cap_psm))
 		return (EINVAL);
 
 	/*
 	 * Routing. Socket should be bound to some source address. The source
 	 * address can be ANY. Destination address must be set and it must not
 	 * be ANY. If source address is ANY then find first rtentry that has
 	 * src != dst.
 	 */
 
 	mtx_lock(&ng_btsocket_l2cap_rt_mtx);
 	mtx_lock(&ng_btsocket_l2cap_sockets_mtx);
 	mtx_lock(&pcb->pcb_mtx);
 
 	/* Send destination address and PSM */
 	bcopy(&sa->l2cap_bdaddr, &pcb->dst, sizeof(pcb->dst));
 	pcb->psm = le16toh(sa->l2cap_psm);
 
 	pcb->rt = NULL;
 	have_src = bcmp(&pcb->src, NG_HCI_BDADDR_ANY, sizeof(pcb->src));
 
 	LIST_FOREACH(rt, &ng_btsocket_l2cap_rt, next) {
 		if (rt->hook == NULL || NG_HOOK_NOT_VALID(rt->hook))
 			continue;
 
 		/* Match src and dst */
 		if (have_src) {
 			if (bcmp(&pcb->src, &rt->src, sizeof(rt->src)) == 0)
 				break;
 		} else {
 			if (bcmp(&pcb->dst, &rt->src, sizeof(rt->src)) != 0)
 				break;
 		}
 	}
 
 	if (rt != NULL) {
 		pcb->rt = rt;
 
 		if (!have_src)
 			bcopy(&rt->src, &pcb->src, sizeof(pcb->src));
 	} else
 		error = EHOSTUNREACH;
 
 	/*
 	 * Send L2CA_Connect request 
 	 */
 
 	if (error == 0) {	
 		error = ng_btsocket_l2cap_send_l2ca_con_req(pcb);
 		if (error == 0) {
 			pcb->flags |= NG_BTSOCKET_L2CAP_CLIENT;
 			pcb->state = NG_BTSOCKET_L2CAP_CONNECTING;
 			soisconnecting(pcb->so);
 
 			ng_btsocket_l2cap_timeout(pcb);
 		}
 	}
 
 	mtx_unlock(&pcb->pcb_mtx);
 	mtx_unlock(&ng_btsocket_l2cap_sockets_mtx);
 	mtx_unlock(&ng_btsocket_l2cap_rt_mtx);
 
 	return (error);
 } /* ng_btsocket_l2cap_connect */
 
 /*
  * Process ioctl's calls on socket
  */
 
 int
 ng_btsocket_l2cap_control(struct socket *so, u_long cmd, caddr_t data,
 		struct ifnet *ifp, struct thread *td)
 {
 	return (EINVAL);
 } /* ng_btsocket_l2cap_control */
 
 /*
  * Process getsockopt/setsockopt system calls
  */
 
 int
 ng_btsocket_l2cap_ctloutput(struct socket *so, struct sockopt *sopt)
 {
 	ng_btsocket_l2cap_pcb_p	pcb = so2l2cap_pcb(so);
 	int			error = 0;
 	ng_l2cap_cfg_opt_val_t	v;
 
 	if (pcb == NULL)
 		return (EINVAL);
 	if (ng_btsocket_l2cap_node == NULL) 
 		return (EINVAL);
 
 	if (sopt->sopt_level != SOL_L2CAP)
 		return (0);
 
 	mtx_lock(&pcb->pcb_mtx);
 
 	switch (sopt->sopt_dir) {
 	case SOPT_GET:
 		switch (sopt->sopt_name) {
 		case SO_L2CAP_IMTU: /* get incoming MTU */
 			error = sooptcopyout(sopt, &pcb->imtu,
 						sizeof(pcb->imtu));
 			break;
 
 		case SO_L2CAP_OMTU: /* get outgoing (peer incoming) MTU */
 			error = sooptcopyout(sopt, &pcb->omtu,
 						sizeof(pcb->omtu));
 			break;
 
 		case SO_L2CAP_IFLOW: /* get incoming flow spec. */
 			error = sooptcopyout(sopt, &pcb->iflow,
 						sizeof(pcb->iflow));
 			break;
 
 		case SO_L2CAP_OFLOW: /* get outgoing flow spec. */
 			error = sooptcopyout(sopt, &pcb->oflow,
 						sizeof(pcb->oflow));
 			break;
 
 		case SO_L2CAP_FLUSH: /* get flush timeout */
 			error = sooptcopyout(sopt, &pcb->flush_timo,
 						sizeof(pcb->flush_timo));
 			break;
 
 		default:
 			error = ENOPROTOOPT;
 			break;
 		}
 		break;
 
 	case SOPT_SET:
 		/*
 		 * XXX
 		 * We do not allow to change these parameters while socket is 
 		 * connected or we are in the process of creating a connection.
 		 * May be this should indicate re-configuration of the open 
 		 * channel?
 		 */
 
 		if (pcb->state != NG_BTSOCKET_L2CAP_CLOSED) {
 			error = EACCES;
 			break;
 		}
 
 		switch (sopt->sopt_name) {
 		case SO_L2CAP_IMTU: /* set incoming MTU */
 			error = sooptcopyin(sopt, &v, sizeof(v), sizeof(v.mtu));
 			if (error == 0)
 				pcb->imtu = v.mtu;
 			break;
 
 		case SO_L2CAP_OFLOW: /* set outgoing flow spec. */
 			error = sooptcopyin(sopt, &v, sizeof(v),sizeof(v.flow));
 			if (error == 0)
 				bcopy(&v.flow, &pcb->oflow, sizeof(pcb->oflow));
 			break;
 
 		case SO_L2CAP_FLUSH: /* set flush timeout */
 			error = sooptcopyin(sopt, &v, sizeof(v),
 						sizeof(v.flush_timo));
 			if (error == 0)
 				pcb->flush_timo = v.flush_timo;
 			break;
 
 		default:
 			error = ENOPROTOOPT;
 			break;
 		}
 		break;
 
 	default:
 		error = EINVAL;
 		break;
 	}
 
 	mtx_unlock(&pcb->pcb_mtx);
 
 	return (error);
 } /* ng_btsocket_l2cap_ctloutput */
 
 /*
  * Detach and destroy socket
  */
 
 void
 ng_btsocket_l2cap_detach(struct socket *so)
 {
 	ng_btsocket_l2cap_pcb_p	pcb = so2l2cap_pcb(so);
 
 	KASSERT(pcb != NULL, ("ng_btsocket_l2cap_detach: pcb == NULL"));
 
 	if (ng_btsocket_l2cap_node == NULL) 
 		return;
 
 	mtx_lock(&ng_btsocket_l2cap_sockets_mtx);
 	mtx_lock(&pcb->pcb_mtx);
 
 	/* XXX what to do with pending request? */
 	if (pcb->flags & NG_BTSOCKET_L2CAP_TIMO)
 		ng_btsocket_l2cap_untimeout(pcb);
 
 	if (pcb->state != NG_BTSOCKET_L2CAP_CLOSED &&
 	    pcb->state != NG_BTSOCKET_L2CAP_DISCONNECTING)
 		/* Send disconnect request with "zero" token */
 		ng_btsocket_l2cap_send_l2ca_discon_req(0, pcb);
 
 	pcb->state = NG_BTSOCKET_L2CAP_CLOSED;
 
 	LIST_REMOVE(pcb, next);
 
 	mtx_unlock(&pcb->pcb_mtx);
 	mtx_unlock(&ng_btsocket_l2cap_sockets_mtx);
 
 	mtx_destroy(&pcb->pcb_mtx);
 	bzero(pcb, sizeof(*pcb));
 	free(pcb, M_NETGRAPH_BTSOCKET_L2CAP);
 
 	soisdisconnected(so);
 	so->so_pcb = NULL;
 } /* ng_btsocket_l2cap_detach */
 
 /*
  * Disconnect socket
  */
 
 int
 ng_btsocket_l2cap_disconnect(struct socket *so)
 {
 	ng_btsocket_l2cap_pcb_p	pcb = so2l2cap_pcb(so);
 	int			error = 0;
 
 	if (pcb == NULL)
 		return (EINVAL);
 	if (ng_btsocket_l2cap_node == NULL) 
 		return (EINVAL);
 
 	mtx_lock(&pcb->pcb_mtx);
 
 	if (pcb->state == NG_BTSOCKET_L2CAP_DISCONNECTING) {
 		mtx_unlock(&pcb->pcb_mtx);
 		return (EINPROGRESS);
 	}
 
 	if (pcb->state != NG_BTSOCKET_L2CAP_CLOSED) {
 		/* XXX FIXME what to do with pending request? */
 		if (pcb->flags & NG_BTSOCKET_L2CAP_TIMO)
 			ng_btsocket_l2cap_untimeout(pcb);
 
 		error = ng_btsocket_l2cap_send_l2ca_discon_req(pcb->token, pcb);
 		if (error == 0) {
 			pcb->state = NG_BTSOCKET_L2CAP_DISCONNECTING;
 			soisdisconnecting(so);
 
 			ng_btsocket_l2cap_timeout(pcb);
 		}
 
 		/* XXX FIXME what to do if error != 0 */
 	}
 
 	mtx_unlock(&pcb->pcb_mtx);
 
 	return (error);
 } /* ng_btsocket_l2cap_disconnect */
 
 /*
  * Listen on socket
  */
 
 int
 ng_btsocket_l2cap_listen(struct socket *so, int backlog, struct thread *td)
 {
 	ng_btsocket_l2cap_pcb_p	pcb = so2l2cap_pcb(so);
 	int error;
 
 	SOCK_LOCK(so);
 	error = solisten_proto_check(so);
 	if (error != 0)
 		goto out;
 	if (pcb == NULL) {
 		error = EINVAL;
 		goto out;
 	}
 	if (ng_btsocket_l2cap_node == NULL) {
 		error = EINVAL;
 		goto out;
 	}
 	if (pcb->psm == 0) {
 		error = EADDRNOTAVAIL;
 		goto out;
 	}
 	solisten_proto(so, backlog);
 out:
 	SOCK_UNLOCK(so);
 	return (error);
 } /* ng_btsocket_listen */
 
 /*
  * Get peer address
  */
 
 int
 ng_btsocket_l2cap_peeraddr(struct socket *so, struct sockaddr **nam)
 {
 	ng_btsocket_l2cap_pcb_p	pcb = so2l2cap_pcb(so);
 	struct sockaddr_l2cap	sa;
 
 	if (pcb == NULL)
 		return (EINVAL);
 	if (ng_btsocket_l2cap_node == NULL) 
 		return (EINVAL);
 
 	bcopy(&pcb->dst, &sa.l2cap_bdaddr, sizeof(sa.l2cap_bdaddr));
 	sa.l2cap_psm = htole16(pcb->psm);
 	sa.l2cap_len = sizeof(sa);
 	sa.l2cap_family = AF_BLUETOOTH;
 
 	*nam = sodupsockaddr((struct sockaddr *) &sa, M_NOWAIT);
 
 	return ((*nam == NULL)? ENOMEM : 0);
 } /* ng_btsocket_l2cap_peeraddr */
 
 /*
  * Send data to socket
  */
 
 int
 ng_btsocket_l2cap_send(struct socket *so, int flags, struct mbuf *m,
 		struct sockaddr *nam, struct mbuf *control, struct thread *td)
 {
 	ng_btsocket_l2cap_pcb_t	*pcb = so2l2cap_pcb(so);
 	int			 error = 0;
 
 	if (ng_btsocket_l2cap_node == NULL) {
 		error = ENETDOWN;
 		goto drop;
 	}
 
 	/* Check socket and input */
 	if (pcb == NULL || m == NULL || control != NULL) {
 		error = EINVAL;
 		goto drop;
 	}
 
 	mtx_lock(&pcb->pcb_mtx);
 
 	/* Make sure socket is connected */
 	if (pcb->state != NG_BTSOCKET_L2CAP_OPEN) {
 		mtx_unlock(&pcb->pcb_mtx);
 		error = ENOTCONN;
 		goto drop;
 	}
 
 	/* Check route */
 	if (pcb->rt == NULL ||
 	    pcb->rt->hook == NULL || NG_HOOK_NOT_VALID(pcb->rt->hook)) {
 		mtx_unlock(&pcb->pcb_mtx);
 		error = ENETDOWN;
 		goto drop;
 	}
 
 	/* Check packet size agains outgoing (peer's incoming) MTU) */
 	if (m->m_pkthdr.len > pcb->omtu) {
 		NG_BTSOCKET_L2CAP_ERR(
 "%s: Packet too big, len=%d, omtu=%d\n", __func__, m->m_pkthdr.len, pcb->omtu);
 
 		mtx_unlock(&pcb->pcb_mtx);
 		error = EMSGSIZE;
 		goto drop;
 	}
 
 	/*
 	 * First put packet on socket send queue. Then check if we have
 	 * pending timeout. If we do not have timeout then we must send
 	 * packet and schedule timeout. Otherwise do nothing and wait for
 	 * L2CA_WRITE_RSP.
 	 */
 
 	sbappendrecord(&pcb->so->so_snd, m);
 	m = NULL;
 
 	if (!(pcb->flags & NG_BTSOCKET_L2CAP_TIMO)) {
 		error = ng_btsocket_l2cap_send2(pcb);
 		if (error == 0)
 			ng_btsocket_l2cap_timeout(pcb);
 		else
 			sbdroprecord(&pcb->so->so_snd); /* XXX */
 	}
 
 	mtx_unlock(&pcb->pcb_mtx);
 drop:
 	NG_FREE_M(m); /* checks for != NULL */
 	NG_FREE_M(control);
 
 	return (error);
 } /* ng_btsocket_l2cap_send */
 
 /*
  * Send first packet in the socket queue to the L2CAP layer
  */
 
 static int
 ng_btsocket_l2cap_send2(ng_btsocket_l2cap_pcb_p pcb)
 {
 	struct	mbuf		*m = NULL;
 	ng_l2cap_l2ca_hdr_t	*hdr = NULL;
 	int			 error = 0;
 	
 	mtx_assert(&pcb->pcb_mtx, MA_OWNED);
 
 	if (pcb->so->so_snd.sb_cc == 0)
 		return (EINVAL); /* XXX */
 
 	m = m_dup(pcb->so->so_snd.sb_mb, M_NOWAIT);
 	if (m == NULL)
 		return (ENOBUFS);
 
 	/* Create L2CA packet header */
 	M_PREPEND(m, sizeof(*hdr), M_NOWAIT);
 	if (m != NULL)
 		if (m->m_len < sizeof(*hdr))
 			m = m_pullup(m, sizeof(*hdr));
 
 	if (m == NULL) {
 		NG_BTSOCKET_L2CAP_ERR(
 "%s: Failed to create L2CA packet header\n", __func__);
 
 		return (ENOBUFS);
 	}
 
 	hdr = mtod(m, ng_l2cap_l2ca_hdr_t *);
 	hdr->token = pcb->token;
 	hdr->length = m->m_pkthdr.len - sizeof(*hdr);
 	hdr->lcid = pcb->cid;
 
 	NG_BTSOCKET_L2CAP_INFO(
 "%s: Sending packet: len=%d, length=%d, lcid=%d, token=%d, state=%d\n",
 		__func__, m->m_pkthdr.len, hdr->length, hdr->lcid, 
 		hdr->token, pcb->state);
 
 	/*
 	 * If we got here than we have successfuly creates new L2CAP 
 	 * data packet and now we can send it to the L2CAP layer
 	 */
 
 	NG_SEND_DATA_ONLY(error, pcb->rt->hook, m);
 
 	return (error);
 } /* ng_btsocket_l2cap_send2 */
 
 /*
  * Get socket address
  */
 
 int
 ng_btsocket_l2cap_sockaddr(struct socket *so, struct sockaddr **nam)
 {
 	ng_btsocket_l2cap_pcb_p	pcb = so2l2cap_pcb(so);
 	struct sockaddr_l2cap	sa;
 
 	if (pcb == NULL)
 		return (EINVAL);
 	if (ng_btsocket_l2cap_node == NULL) 
 		return (EINVAL);
 
 	bcopy(&pcb->src, &sa.l2cap_bdaddr, sizeof(sa.l2cap_bdaddr));
 	sa.l2cap_psm = htole16(pcb->psm);
 	sa.l2cap_len = sizeof(sa);
 	sa.l2cap_family = AF_BLUETOOTH;
 
 	*nam = sodupsockaddr((struct sockaddr *) &sa, M_NOWAIT);
 
 	return ((*nam == NULL)? ENOMEM : 0);
 } /* ng_btsocket_l2cap_sockaddr */
 
 /*****************************************************************************
  *****************************************************************************
  **                              Misc. functions
  *****************************************************************************
  *****************************************************************************/
 
 /*
  * Look for the socket that listens on given PSM and bdaddr. Returns exact or
  * close match (if any). Caller must hold ng_btsocket_l2cap_sockets_mtx.
  */
 
 static ng_btsocket_l2cap_pcb_p
 ng_btsocket_l2cap_pcb_by_addr(bdaddr_p bdaddr, int psm)
 {
 	ng_btsocket_l2cap_pcb_p	p = NULL, p1 = NULL;
 
 	mtx_assert(&ng_btsocket_l2cap_sockets_mtx, MA_OWNED);
 
 	LIST_FOREACH(p, &ng_btsocket_l2cap_sockets, next) {
 		if (p->so == NULL || !(p->so->so_options & SO_ACCEPTCONN) || 
 		    p->psm != psm) 
 			continue;
 
 		if (bcmp(&p->src, bdaddr, sizeof(p->src)) == 0)
 			break;
 
 		if (bcmp(&p->src, NG_HCI_BDADDR_ANY, sizeof(p->src)) == 0)
 			p1 = p;
 	}
 
 	return ((p != NULL)? p : p1);
 } /* ng_btsocket_l2cap_pcb_by_addr */
 
 /*
  * Look for the socket that has given token.
  * Caller must hold ng_btsocket_l2cap_sockets_mtx.
  */
 
 static ng_btsocket_l2cap_pcb_p
 ng_btsocket_l2cap_pcb_by_token(u_int32_t token)
 {
 	ng_btsocket_l2cap_pcb_p	p = NULL;
 
 	if (token == 0)
 		return (NULL);
 
 	mtx_assert(&ng_btsocket_l2cap_sockets_mtx, MA_OWNED);
 
 	LIST_FOREACH(p, &ng_btsocket_l2cap_sockets, next)
 		if (p->token == token)
 			break;
 
 	return (p);
 } /* ng_btsocket_l2cap_pcb_by_token */
 
 /*
  * Look for the socket that assigned to given source address and channel ID.
  * Caller must hold ng_btsocket_l2cap_sockets_mtx
  */
 
 static ng_btsocket_l2cap_pcb_p
 ng_btsocket_l2cap_pcb_by_cid(bdaddr_p src, int cid)
 {
 	ng_btsocket_l2cap_pcb_p	p = NULL;
 
 	mtx_assert(&ng_btsocket_l2cap_sockets_mtx, MA_OWNED);
 
 	LIST_FOREACH(p, &ng_btsocket_l2cap_sockets, next)
 		if (p->cid == cid && bcmp(src, &p->src, sizeof(p->src)) == 0)
 			break;
 
 	return (p);
 } /* ng_btsocket_l2cap_pcb_by_cid */
 
 /*
  * Set timeout on socket
  */
 
 static void
 ng_btsocket_l2cap_timeout(ng_btsocket_l2cap_pcb_p pcb)
 {
 	mtx_assert(&pcb->pcb_mtx, MA_OWNED);
 
 	if (!(pcb->flags & NG_BTSOCKET_L2CAP_TIMO)) {
 		pcb->flags |= NG_BTSOCKET_L2CAP_TIMO;
 		pcb->timo = timeout(ng_btsocket_l2cap_process_timeout, pcb,
 					bluetooth_l2cap_ertx_timeout());
 	} else
 		KASSERT(0,
 ("%s: Duplicated socket timeout?!\n", __func__));
 } /* ng_btsocket_l2cap_timeout */
 
 /*
  * Unset timeout on socket
  */
 
 static void
 ng_btsocket_l2cap_untimeout(ng_btsocket_l2cap_pcb_p pcb)
 {
 	mtx_assert(&pcb->pcb_mtx, MA_OWNED);
 
 	if (pcb->flags & NG_BTSOCKET_L2CAP_TIMO) {
 		untimeout(ng_btsocket_l2cap_process_timeout, pcb, pcb->timo);
 		pcb->flags &= ~NG_BTSOCKET_L2CAP_TIMO;
 	} else
 		KASSERT(0,
 ("%s: No socket timeout?!\n", __func__));
 } /* ng_btsocket_l2cap_untimeout */
 
 /*
  * Process timeout on socket
  */
 
 static void
 ng_btsocket_l2cap_process_timeout(void *xpcb)
 {
 	ng_btsocket_l2cap_pcb_p	pcb = (ng_btsocket_l2cap_pcb_p) xpcb;
 
 	mtx_lock(&pcb->pcb_mtx);
 
 	pcb->flags &= ~NG_BTSOCKET_L2CAP_TIMO;
 	pcb->so->so_error = ETIMEDOUT;
 
 	switch (pcb->state) {
 	case NG_BTSOCKET_L2CAP_CONNECTING:
 	case NG_BTSOCKET_L2CAP_CONFIGURING:
 		/* Send disconnect request with "zero" token */
 		if (pcb->cid != 0)
 			ng_btsocket_l2cap_send_l2ca_discon_req(0, pcb);
 
 		/* ... and close the socket */
 		pcb->state = NG_BTSOCKET_L2CAP_CLOSED;
 		soisdisconnected(pcb->so);
 		break;
 
 	case NG_BTSOCKET_L2CAP_OPEN:
 		/* Send timeout - drop packet and wakeup sender */
 		sbdroprecord(&pcb->so->so_snd);
 		sowwakeup(pcb->so);
 		break;
 
 	case NG_BTSOCKET_L2CAP_DISCONNECTING:
 		/* Disconnect timeout - disconnect the socket anyway */
 		pcb->state = NG_BTSOCKET_L2CAP_CLOSED;
 		soisdisconnected(pcb->so);
 		break;
 
 	default:
 		NG_BTSOCKET_L2CAP_ERR(
 "%s: Invalid socket state=%d\n", __func__, pcb->state);
 		break;
 	}
 
 	mtx_unlock(&pcb->pcb_mtx);
 } /* ng_btsocket_l2cap_process_timeout */
 
 /*
  * Translate HCI/L2CAP error code into "errno" code
  * XXX Note: Some L2CAP and HCI error codes have the same value, but 
  *     different meaning
  */
 
 static int
 ng_btsocket_l2cap_result2errno(int result)
 {
 	switch (result) {
 	case 0x00: /* No error */ 
 		return (0);
 
 	case 0x01: /* Unknown HCI command */
 		return (ENODEV);
 
 	case 0x02: /* No connection */
 		return (ENOTCONN);
 
 	case 0x03: /* Hardware failure */
 		return (EIO);
 
 	case 0x04: /* Page timeout */
 		return (EHOSTDOWN);
 
 	case 0x05: /* Authentication failure */
 	case 0x06: /* Key missing */
 	case 0x18: /* Pairing not allowed */
 	case 0x21: /* Role change not allowed */
 	case 0x24: /* LMP PSU not allowed */
 	case 0x25: /* Encryption mode not acceptable */
 	case 0x26: /* Unit key used */
 		return (EACCES);
 
 	case 0x07: /* Memory full */
 		return (ENOMEM);
 
 	case 0x08:   /* Connection timeout */
 	case 0x10:   /* Host timeout */
 	case 0x22:   /* LMP response timeout */
 	case 0xee:   /* HCI timeout */
 	case 0xeeee: /* L2CAP timeout */
 		return (ETIMEDOUT);
 
 	case 0x09: /* Max number of connections */
 	case 0x0a: /* Max number of SCO connections to a unit */
 		return (EMLINK);
 
 	case 0x0b: /* ACL connection already exists */
 		return (EEXIST);
 
 	case 0x0c: /* Command disallowed */
 		return (EBUSY);
 
 	case 0x0d: /* Host rejected due to limited resources */
 	case 0x0e: /* Host rejected due to securiity reasons */
 	case 0x0f: /* Host rejected due to remote unit is a personal unit */
 	case 0x1b: /* SCO offset rejected */
 	case 0x1c: /* SCO interval rejected */
 	case 0x1d: /* SCO air mode rejected */
 		return (ECONNREFUSED);
 
 	case 0x11: /* Unsupported feature or parameter value */
 	case 0x19: /* Unknown LMP PDU */
 	case 0x1a: /* Unsupported remote feature */
 	case 0x20: /* Unsupported LMP parameter value */
 	case 0x27: /* QoS is not supported */
 	case 0x29: /* Paring with unit key not supported */
 		return (EOPNOTSUPP);
 
 	case 0x12: /* Invalid HCI command parameter */
 	case 0x1e: /* Invalid LMP parameters */
 		return (EINVAL);
 
 	case 0x13: /* Other end terminated connection: User ended connection */
 	case 0x14: /* Other end terminated connection: Low resources */
 	case 0x15: /* Other end terminated connection: About to power off */
 		return (ECONNRESET);
 
 	case 0x16: /* Connection terminated by local host */
 		return (ECONNABORTED);
 
 #if 0 /* XXX not yet */
 	case 0x17: /* Repeated attempts */
 	case 0x1f: /* Unspecified error */
 	case 0x23: /* LMP error transaction collision */
 	case 0x28: /* Instant passed */
 #endif
 	}
 
 	return (ENOSYS);
 } /* ng_btsocket_l2cap_result2errno */
 
Index: projects/release-debugdist/sys/netgraph/bluetooth/socket/ng_btsocket_l2cap_raw.c
===================================================================
--- projects/release-debugdist/sys/netgraph/bluetooth/socket/ng_btsocket_l2cap_raw.c	(revision 267343)
+++ projects/release-debugdist/sys/netgraph/bluetooth/socket/ng_btsocket_l2cap_raw.c	(revision 267344)
@@ -1,1306 +1,1313 @@
 /*
  * ng_btsocket_l2cap_raw.c
  */
 
 /*-
  * Copyright (c) 2001-2002 Maksim Yevmenkin <m_evmenkin@yahoo.com>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
  * $Id: ng_btsocket_l2cap_raw.c,v 1.12 2003/09/14 23:29:06 max Exp $
  * $FreeBSD$
  */
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/bitstring.h>
 #include <sys/domain.h>
 #include <sys/errno.h>
 #include <sys/filedesc.h>
 #include <sys/ioccom.h>
 #include <sys/kernel.h>
 #include <sys/lock.h>
 #include <sys/malloc.h>
 #include <sys/mbuf.h>
 #include <sys/mutex.h>
 #include <sys/priv.h>
 #include <sys/protosw.h>
 #include <sys/queue.h>
 #include <sys/socket.h>
 #include <sys/socketvar.h>
 #include <sys/sysctl.h>
 #include <sys/taskqueue.h>
+
+#include <net/vnet.h>
+
 #include <netgraph/ng_message.h>
 #include <netgraph/netgraph.h>
 #include <netgraph/bluetooth/include/ng_bluetooth.h>
 #include <netgraph/bluetooth/include/ng_hci.h>
 #include <netgraph/bluetooth/include/ng_l2cap.h>
 #include <netgraph/bluetooth/include/ng_btsocket.h>
 #include <netgraph/bluetooth/include/ng_btsocket_l2cap.h>
 
 /* MALLOC define */
 #ifdef NG_SEPARATE_MALLOC
 static MALLOC_DEFINE(M_NETGRAPH_BTSOCKET_L2CAP_RAW,
     "netgraph_btsocks_l2cap_raw", "Netgraph Bluetooth raw L2CAP sockets");
 #else
 #define M_NETGRAPH_BTSOCKET_L2CAP_RAW M_NETGRAPH
 #endif /* NG_SEPARATE_MALLOC */
 
 /* Netgraph node methods */
 static ng_constructor_t	ng_btsocket_l2cap_raw_node_constructor;
 static ng_rcvmsg_t	ng_btsocket_l2cap_raw_node_rcvmsg;
 static ng_shutdown_t	ng_btsocket_l2cap_raw_node_shutdown;
 static ng_newhook_t	ng_btsocket_l2cap_raw_node_newhook;
 static ng_connect_t	ng_btsocket_l2cap_raw_node_connect;
 static ng_rcvdata_t	ng_btsocket_l2cap_raw_node_rcvdata;
 static ng_disconnect_t	ng_btsocket_l2cap_raw_node_disconnect;
 
 static void		ng_btsocket_l2cap_raw_input     (void *, int);
 static void		ng_btsocket_l2cap_raw_rtclean   (void *, int);
 static void		ng_btsocket_l2cap_raw_get_token (u_int32_t *);
 
 static int		ng_btsocket_l2cap_raw_send_ngmsg
 				(hook_p, int, void *, int);
 static int		ng_btsocket_l2cap_raw_send_sync_ngmsg
 				(ng_btsocket_l2cap_raw_pcb_p, int, void *, int);
 
 #define ng_btsocket_l2cap_raw_wakeup_input_task() \
 	taskqueue_enqueue(taskqueue_swi, &ng_btsocket_l2cap_raw_queue_task)
 
 #define ng_btsocket_l2cap_raw_wakeup_route_task() \
 	taskqueue_enqueue(taskqueue_swi, &ng_btsocket_l2cap_raw_rt_task)
 
 /* Netgraph type descriptor */
 static struct ng_type	typestruct = {
 	.version =	NG_ABI_VERSION,
 	.name =		NG_BTSOCKET_L2CAP_RAW_NODE_TYPE,
 	.constructor =	ng_btsocket_l2cap_raw_node_constructor,
 	.rcvmsg =	ng_btsocket_l2cap_raw_node_rcvmsg,
 	.shutdown =	ng_btsocket_l2cap_raw_node_shutdown,
 	.newhook =	ng_btsocket_l2cap_raw_node_newhook,
 	.connect =	ng_btsocket_l2cap_raw_node_connect,
 	.rcvdata =	ng_btsocket_l2cap_raw_node_rcvdata,
 	.disconnect =	ng_btsocket_l2cap_raw_node_disconnect,
 };
 
 /* Globals */
 extern int					ifqmaxlen;
 static u_int32_t				ng_btsocket_l2cap_raw_debug_level;
 static u_int32_t				ng_btsocket_l2cap_raw_ioctl_timeout;
 static node_p					ng_btsocket_l2cap_raw_node;
 static struct ng_bt_itemq			ng_btsocket_l2cap_raw_queue;
 static struct mtx				ng_btsocket_l2cap_raw_queue_mtx;
 static struct task				ng_btsocket_l2cap_raw_queue_task;
 static LIST_HEAD(, ng_btsocket_l2cap_raw_pcb)	ng_btsocket_l2cap_raw_sockets;
 static struct mtx				ng_btsocket_l2cap_raw_sockets_mtx;
 static u_int32_t				ng_btsocket_l2cap_raw_token;
 static struct mtx				ng_btsocket_l2cap_raw_token_mtx;
 static LIST_HEAD(, ng_btsocket_l2cap_rtentry)	ng_btsocket_l2cap_raw_rt;
 static struct mtx				ng_btsocket_l2cap_raw_rt_mtx;
 static struct task				ng_btsocket_l2cap_raw_rt_task;
 static struct timeval				ng_btsocket_l2cap_raw_lasttime;
 static int					ng_btsocket_l2cap_raw_curpps;
 
 /* Sysctl tree */
 SYSCTL_DECL(_net_bluetooth_l2cap_sockets);
 static SYSCTL_NODE(_net_bluetooth_l2cap_sockets, OID_AUTO, raw, CTLFLAG_RW,
 	0, "Bluetooth raw L2CAP sockets family");
 SYSCTL_UINT(_net_bluetooth_l2cap_sockets_raw, OID_AUTO, debug_level,
 	CTLFLAG_RW,
 	&ng_btsocket_l2cap_raw_debug_level, NG_BTSOCKET_WARN_LEVEL,
 	"Bluetooth raw L2CAP sockets debug level");
 SYSCTL_UINT(_net_bluetooth_l2cap_sockets_raw, OID_AUTO, ioctl_timeout,
 	CTLFLAG_RW,
 	&ng_btsocket_l2cap_raw_ioctl_timeout, 5,
 	"Bluetooth raw L2CAP sockets ioctl timeout");
 SYSCTL_UINT(_net_bluetooth_l2cap_sockets_raw, OID_AUTO, queue_len,
 	CTLFLAG_RD,
 	&ng_btsocket_l2cap_raw_queue.len, 0,
 	"Bluetooth raw L2CAP sockets input queue length");
 SYSCTL_UINT(_net_bluetooth_l2cap_sockets_raw, OID_AUTO, queue_maxlen,
 	CTLFLAG_RD,
 	&ng_btsocket_l2cap_raw_queue.maxlen, 0,
 	"Bluetooth raw L2CAP sockets input queue max. length");
 SYSCTL_UINT(_net_bluetooth_l2cap_sockets_raw, OID_AUTO, queue_drops,
 	CTLFLAG_RD,
 	&ng_btsocket_l2cap_raw_queue.drops, 0,
 	"Bluetooth raw L2CAP sockets input queue drops");
 
 /* Debug */
 #define NG_BTSOCKET_L2CAP_RAW_INFO \
 	if (ng_btsocket_l2cap_raw_debug_level >= NG_BTSOCKET_INFO_LEVEL && \
 	    ppsratecheck(&ng_btsocket_l2cap_raw_lasttime, &ng_btsocket_l2cap_raw_curpps, 1)) \
 		printf
 
 #define NG_BTSOCKET_L2CAP_RAW_WARN \
 	if (ng_btsocket_l2cap_raw_debug_level >= NG_BTSOCKET_WARN_LEVEL && \
 	    ppsratecheck(&ng_btsocket_l2cap_raw_lasttime, &ng_btsocket_l2cap_raw_curpps, 1)) \
 		printf
 
 #define NG_BTSOCKET_L2CAP_RAW_ERR \
 	if (ng_btsocket_l2cap_raw_debug_level >= NG_BTSOCKET_ERR_LEVEL && \
 	    ppsratecheck(&ng_btsocket_l2cap_raw_lasttime, &ng_btsocket_l2cap_raw_curpps, 1)) \
 		printf
 
 #define NG_BTSOCKET_L2CAP_RAW_ALERT \
 	if (ng_btsocket_l2cap_raw_debug_level >= NG_BTSOCKET_ALERT_LEVEL && \
 	    ppsratecheck(&ng_btsocket_l2cap_raw_lasttime, &ng_btsocket_l2cap_raw_curpps, 1)) \
 		printf
 
 /*****************************************************************************
  *****************************************************************************
  **                        Netgraph node interface
  *****************************************************************************
  *****************************************************************************/
 
 /*
  * Netgraph node constructor. Do not allow to create node of this type.
  */
 
 static int
 ng_btsocket_l2cap_raw_node_constructor(node_p node)
 {
 	return (EINVAL);
 } /* ng_btsocket_l2cap_raw_node_constructor */
 
 /*
  * Do local shutdown processing. Let old node go and create new fresh one.
  */
 
 static int
 ng_btsocket_l2cap_raw_node_shutdown(node_p node)
 {
 	int	error = 0;
 
 	NG_NODE_UNREF(node);
 
 	/* Create new node */
 	error = ng_make_node_common(&typestruct, &ng_btsocket_l2cap_raw_node);
 	if (error != 0) {
 		NG_BTSOCKET_L2CAP_RAW_ALERT(
 "%s: Could not create Netgraph node, error=%d\n", __func__, error);
 
 		ng_btsocket_l2cap_raw_node = NULL;
 
 		return (error);
 	}
 
 	error = ng_name_node(ng_btsocket_l2cap_raw_node,
 				NG_BTSOCKET_L2CAP_RAW_NODE_TYPE);
 	if (error != 0) {
 		NG_BTSOCKET_L2CAP_RAW_ALERT(
 "%s: Could not name Netgraph node, error=%d\n", __func__, error);
 
 		NG_NODE_UNREF(ng_btsocket_l2cap_raw_node);
 		ng_btsocket_l2cap_raw_node = NULL;
 
 		return (error);
 	}
 		
 	return (0);
 } /* ng_btsocket_l2cap_raw_node_shutdown */
 
 /*
  * We allow any hook to be connected to the node.
  */
 
 static int
 ng_btsocket_l2cap_raw_node_newhook(node_p node, hook_p hook, char const *name)
 {
 	return (0);
 } /* ng_btsocket_l2cap_raw_node_newhook */
 
 /* 
  * Just say "YEP, that's OK by me!"
  */
 
 static int
 ng_btsocket_l2cap_raw_node_connect(hook_p hook)
 {
 	NG_HOOK_SET_PRIVATE(hook, NULL);
 	NG_HOOK_REF(hook); /* Keep extra reference to the hook */
 
 	return (0);
 } /* ng_btsocket_l2cap_raw_node_connect */
 
 /*
  * Hook disconnection. Schedule route cleanup task
  */
 
 static int
 ng_btsocket_l2cap_raw_node_disconnect(hook_p hook)
 {
 	/*
 	 * If hook has private information than we must have this hook in
 	 * the routing table and must schedule cleaning for the routing table.
 	 * Otherwise hook was connected but we never got "hook_info" message,
 	 * so we have never added this hook to the routing table and it save
 	 * to just delete it.
 	 */
 
 	if (NG_HOOK_PRIVATE(hook) != NULL)
 		return (ng_btsocket_l2cap_raw_wakeup_route_task());
 
 	NG_HOOK_UNREF(hook); /* Remove extra reference */
 
 	return (0);
 } /* ng_btsocket_l2cap_raw_node_disconnect */
 
 /*
  * Process incoming messages 
  */
 
 static int
 ng_btsocket_l2cap_raw_node_rcvmsg(node_p node, item_p item, hook_p hook)
 {
 	struct ng_mesg	*msg = NGI_MSG(item); /* item still has message */
 	int		 error = 0;
 
 	if (msg != NULL && msg->header.typecookie == NGM_L2CAP_COOKIE) {
 
 		/*
 		 * NGM_L2CAP_NODE_HOOK_INFO is special message initiated by
 		 * L2CAP layer. Ignore all other messages if they are not
 		 * replies or token is zero 
 		 */
 
 		if (msg->header.cmd != NGM_L2CAP_NODE_HOOK_INFO) {
 			if (msg->header.token == 0 ||
 			    !(msg->header.flags & NGF_RESP)) {
 				NG_FREE_ITEM(item);
 				return (0);
 			}
 		}
 
 		mtx_lock(&ng_btsocket_l2cap_raw_queue_mtx);
 		if (NG_BT_ITEMQ_FULL(&ng_btsocket_l2cap_raw_queue)) {
 			NG_BTSOCKET_L2CAP_RAW_ERR(
 "%s: Input queue is full\n", __func__);
 
 			NG_BT_ITEMQ_DROP(&ng_btsocket_l2cap_raw_queue);
 			NG_FREE_ITEM(item);
 			error = ENOBUFS;
 		} else {
 			if (hook != NULL) {
 				NG_HOOK_REF(hook);
 				NGI_SET_HOOK(item, hook);
 			}
 
 			NG_BT_ITEMQ_ENQUEUE(&ng_btsocket_l2cap_raw_queue, item);
 			error = ng_btsocket_l2cap_raw_wakeup_input_task();
 		}
 		mtx_unlock(&ng_btsocket_l2cap_raw_queue_mtx);
 	} else {
 		NG_FREE_ITEM(item);
 		error = EINVAL;
 	}
 
 	return (error);
 } /* ng_btsocket_l2cap_raw_node_rcvmsg */
 
 /*
  * Receive data on a hook
  */
 
 static int
 ng_btsocket_l2cap_raw_node_rcvdata(hook_p hook, item_p item)
 {
 	NG_FREE_ITEM(item);
 
 	return (EINVAL);
 } /* ng_btsocket_l2cap_raw_node_rcvdata */
 
 /*****************************************************************************
  *****************************************************************************
  **                              Socket interface
  *****************************************************************************
  *****************************************************************************/
 
 /*
  * L2CAP sockets input routine
  */
 
 static void
 ng_btsocket_l2cap_raw_input(void *context, int pending)
 {
 	item_p		 item = NULL;
 	hook_p		 hook = NULL;
 	struct ng_mesg  *msg = NULL;
 
 	for (;;) {
 		mtx_lock(&ng_btsocket_l2cap_raw_queue_mtx);
 		NG_BT_ITEMQ_DEQUEUE(&ng_btsocket_l2cap_raw_queue, item);
 		mtx_unlock(&ng_btsocket_l2cap_raw_queue_mtx);
 
 		if (item == NULL)
 			break;
 
 		KASSERT((item->el_flags & NGQF_TYPE) == NGQF_MESG,
 ("%s: invalid item type=%ld\n", __func__, (item->el_flags & NGQF_TYPE)));
 
 		NGI_GET_MSG(item, msg);
 		NGI_GET_HOOK(item, hook);
 		NG_FREE_ITEM(item);
 
 		switch (msg->header.cmd) {
 		case NGM_L2CAP_NODE_HOOK_INFO: {
 			ng_btsocket_l2cap_rtentry_t	*rt = NULL;
 
 			if (hook == NULL || NG_HOOK_NOT_VALID(hook) ||
 			    msg->header.arglen != sizeof(bdaddr_t))
 				break;
 
 			if (bcmp(msg->data, NG_HCI_BDADDR_ANY,
 					sizeof(bdaddr_t)) == 0)
 				break;
 
 			rt = (ng_btsocket_l2cap_rtentry_t *) 
 				NG_HOOK_PRIVATE(hook);
 			if (rt == NULL) {
 				rt = malloc(sizeof(*rt),
 					M_NETGRAPH_BTSOCKET_L2CAP_RAW,
 					M_NOWAIT|M_ZERO);
 				if (rt == NULL)
 					break;
 
 				NG_HOOK_SET_PRIVATE(hook, rt);
 
 				mtx_lock(&ng_btsocket_l2cap_raw_rt_mtx);
 
 				LIST_INSERT_HEAD(&ng_btsocket_l2cap_raw_rt, 
 					rt, next);
 			} else
 				mtx_lock(&ng_btsocket_l2cap_raw_rt_mtx);
 		
 			bcopy(msg->data, &rt->src, sizeof(rt->src));
 			rt->hook = hook;
 
 			NG_BTSOCKET_L2CAP_RAW_INFO(
 "%s: Updating hook \"%s\", src bdaddr=%x:%x:%x:%x:%x:%x\n",
 				__func__, NG_HOOK_NAME(hook), 
 				rt->src.b[5], rt->src.b[4], rt->src.b[3],
 				rt->src.b[2], rt->src.b[1], rt->src.b[0]);
 
 			mtx_unlock(&ng_btsocket_l2cap_raw_rt_mtx);
 			} break;
 
 		case NGM_L2CAP_NODE_GET_FLAGS:
 		case NGM_L2CAP_NODE_GET_DEBUG:
 		case NGM_L2CAP_NODE_GET_CON_LIST:
 		case NGM_L2CAP_NODE_GET_CHAN_LIST:
 		case NGM_L2CAP_NODE_GET_AUTO_DISCON_TIMO:
 		case NGM_L2CAP_L2CA_PING:
 		case NGM_L2CAP_L2CA_GET_INFO: {
 			ng_btsocket_l2cap_raw_pcb_p	pcb = NULL;
 
 			mtx_lock(&ng_btsocket_l2cap_raw_sockets_mtx);
 
 			LIST_FOREACH(pcb,&ng_btsocket_l2cap_raw_sockets,next) {
 				mtx_lock(&pcb->pcb_mtx);
 
 				if (pcb->token == msg->header.token) {
 					pcb->msg = msg;
 					msg = NULL;
 					wakeup(&pcb->msg);
 					mtx_unlock(&pcb->pcb_mtx);
 					break;
 				}
 
 				mtx_unlock(&pcb->pcb_mtx);
 			}
 
 			mtx_unlock(&ng_btsocket_l2cap_raw_sockets_mtx);
 			} break;
 
 		default:
 			NG_BTSOCKET_L2CAP_RAW_WARN(
 "%s: Unknown message, cmd=%d\n", __func__, msg->header.cmd);
 			break;
 		}
 
 		if (hook != NULL)
 			NG_HOOK_UNREF(hook); /* remove extra reference */
 
 		NG_FREE_MSG(msg); /* Checks for msg != NULL */
 	}
 } /* ng_btsocket_l2cap_raw_input */
 
 /*
  * Route cleanup task. Gets scheduled when hook is disconnected. Here we 
  * will find all sockets that use "invalid" hook and disconnect them.
  */
 
 static void
 ng_btsocket_l2cap_raw_rtclean(void *context, int pending)
 {
 	ng_btsocket_l2cap_raw_pcb_p	pcb = NULL;
 	ng_btsocket_l2cap_rtentry_p	rt = NULL;
 
 	/*
 	 * First disconnect all sockets that use "invalid" hook
 	 */
 
 	mtx_lock(&ng_btsocket_l2cap_raw_sockets_mtx);
 
 	LIST_FOREACH(pcb, &ng_btsocket_l2cap_raw_sockets, next) {
 		mtx_lock(&pcb->pcb_mtx);
 
 		if (pcb->rt != NULL &&
 		    pcb->rt->hook != NULL && NG_HOOK_NOT_VALID(pcb->rt->hook)) {
 			if (pcb->so != NULL &&
 			    pcb->so->so_state & SS_ISCONNECTED)
 				soisdisconnected(pcb->so);
 
 			pcb->rt = NULL;
 		}
 
 		mtx_unlock(&pcb->pcb_mtx);
 	}
 
 	mtx_unlock(&ng_btsocket_l2cap_raw_sockets_mtx);
 
 	/*
 	 * Now cleanup routing table
 	 */
 
 	mtx_lock(&ng_btsocket_l2cap_raw_rt_mtx);
 
 	for (rt = LIST_FIRST(&ng_btsocket_l2cap_raw_rt); rt != NULL; ) {
 		ng_btsocket_l2cap_rtentry_p	rt_next = LIST_NEXT(rt, next);
 
 		if (rt->hook != NULL && NG_HOOK_NOT_VALID(rt->hook)) {
 			LIST_REMOVE(rt, next);
 
 			NG_HOOK_SET_PRIVATE(rt->hook, NULL);
 			NG_HOOK_UNREF(rt->hook); /* Remove extra reference */
 
 			bzero(rt, sizeof(*rt));
 			free(rt, M_NETGRAPH_BTSOCKET_L2CAP_RAW);
 		}
 
 		rt = rt_next;
 	}
 
 	mtx_unlock(&ng_btsocket_l2cap_raw_rt_mtx);
 } /* ng_btsocket_l2cap_raw_rtclean */
 
 /*
  * Initialize everything
  */
 
 void
 ng_btsocket_l2cap_raw_init(void)
 {
 	int	error = 0;
+
+	/* Skip initialization of globals for non-default instances. */
+	if (!IS_DEFAULT_VNET(curvnet))
+		return;
 
 	ng_btsocket_l2cap_raw_node = NULL;
 	ng_btsocket_l2cap_raw_debug_level = NG_BTSOCKET_WARN_LEVEL;
 	ng_btsocket_l2cap_raw_ioctl_timeout = 5;
 
 	/* Register Netgraph node type */
 	error = ng_newtype(&typestruct);
 	if (error != 0) {
 		NG_BTSOCKET_L2CAP_RAW_ALERT(
 "%s: Could not register Netgraph node type, error=%d\n", __func__, error);
 
                 return;
 	}
 
 	/* Create Netgrapg node */
 	error = ng_make_node_common(&typestruct, &ng_btsocket_l2cap_raw_node);
 	if (error != 0) {
 		NG_BTSOCKET_L2CAP_RAW_ALERT(
 "%s: Could not create Netgraph node, error=%d\n", __func__, error);
 
 		ng_btsocket_l2cap_raw_node = NULL;
 
 		return;
 	}
 
 	error = ng_name_node(ng_btsocket_l2cap_raw_node,
 				NG_BTSOCKET_L2CAP_RAW_NODE_TYPE);
 	if (error != 0) {
 		NG_BTSOCKET_L2CAP_RAW_ALERT(
 "%s: Could not name Netgraph node, error=%d\n", __func__, error);
 
 		NG_NODE_UNREF(ng_btsocket_l2cap_raw_node);
 		ng_btsocket_l2cap_raw_node = NULL;
 
 		return;
 	}
 
 	/* Create input queue */
 	NG_BT_ITEMQ_INIT(&ng_btsocket_l2cap_raw_queue, ifqmaxlen);
 	mtx_init(&ng_btsocket_l2cap_raw_queue_mtx,
 		"btsocks_l2cap_raw_queue_mtx", NULL, MTX_DEF);
 	TASK_INIT(&ng_btsocket_l2cap_raw_queue_task, 0,
 		ng_btsocket_l2cap_raw_input, NULL);
 
 	/* Create list of sockets */
 	LIST_INIT(&ng_btsocket_l2cap_raw_sockets);
 	mtx_init(&ng_btsocket_l2cap_raw_sockets_mtx,
 		"btsocks_l2cap_raw_sockets_mtx", NULL, MTX_DEF);
 
 	/* Tokens */
 	ng_btsocket_l2cap_raw_token = 0;
 	mtx_init(&ng_btsocket_l2cap_raw_token_mtx,
 		"btsocks_l2cap_raw_token_mtx", NULL, MTX_DEF);
 
 	/* Routing table */
 	LIST_INIT(&ng_btsocket_l2cap_raw_rt);
 	mtx_init(&ng_btsocket_l2cap_raw_rt_mtx,
 		"btsocks_l2cap_raw_rt_mtx", NULL, MTX_DEF);
 	TASK_INIT(&ng_btsocket_l2cap_raw_rt_task, 0,
 		ng_btsocket_l2cap_raw_rtclean, NULL);
 } /* ng_btsocket_l2cap_raw_init */
 
 /*
  * Abort connection on socket
  */
 
 void
 ng_btsocket_l2cap_raw_abort(struct socket *so)
 {
 
 	(void)ng_btsocket_l2cap_raw_disconnect(so);
 } /* ng_btsocket_l2cap_raw_abort */
 
 void
 ng_btsocket_l2cap_raw_close(struct socket *so)
 {
 
 	(void)ng_btsocket_l2cap_raw_disconnect(so);
 } /* ng_btsocket_l2cap_raw_close */
 
 /*
  * Create and attach new socket
  */
 
 int
 ng_btsocket_l2cap_raw_attach(struct socket *so, int proto, struct thread *td)
 {
 	ng_btsocket_l2cap_raw_pcb_p	pcb = so2l2cap_raw_pcb(so);
 	int				error;
 
 	if (pcb != NULL)
 		return (EISCONN);
 
 	if (ng_btsocket_l2cap_raw_node == NULL) 
 		return (EPROTONOSUPPORT);
 	if (so->so_type != SOCK_RAW)
 		return (ESOCKTNOSUPPORT);
 
 	/* Reserve send and receive space if it is not reserved yet */
 	error = soreserve(so, NG_BTSOCKET_L2CAP_RAW_SENDSPACE,
 			NG_BTSOCKET_L2CAP_RAW_RECVSPACE);
 	if (error != 0)
 		return (error);
 
 	/* Allocate the PCB */
         pcb = malloc(sizeof(*pcb),
 		M_NETGRAPH_BTSOCKET_L2CAP_RAW, M_NOWAIT|M_ZERO);
         if (pcb == NULL)
                 return (ENOMEM);
 
 	/* Link the PCB and the socket */
 	so->so_pcb = (caddr_t) pcb;
 	pcb->so = so;
 
 	if (priv_check(td, PRIV_NETBLUETOOTH_RAW) == 0)
 		pcb->flags |= NG_BTSOCKET_L2CAP_RAW_PRIVILEGED;
 
 	mtx_init(&pcb->pcb_mtx, "btsocks_l2cap_raw_pcb_mtx", NULL, MTX_DEF);
 
         /* Add the PCB to the list */
 	mtx_lock(&ng_btsocket_l2cap_raw_sockets_mtx);
 	LIST_INSERT_HEAD(&ng_btsocket_l2cap_raw_sockets, pcb, next);
 	mtx_unlock(&ng_btsocket_l2cap_raw_sockets_mtx);
 
         return (0);
 } /* ng_btsocket_l2cap_raw_attach */
 
 /*
  * Bind socket
  */
 
 int
 ng_btsocket_l2cap_raw_bind(struct socket *so, struct sockaddr *nam, 
 		struct thread *td)
 {
 	ng_btsocket_l2cap_raw_pcb_t	*pcb = so2l2cap_raw_pcb(so);
 	struct sockaddr_l2cap		*sa = (struct sockaddr_l2cap *) nam;
 	ng_btsocket_l2cap_rtentry_t	*rt = NULL;
 
 	if (pcb == NULL)
 		return (EINVAL);
 	if (ng_btsocket_l2cap_raw_node == NULL) 
 		return (EINVAL);
 
 	if (sa == NULL)
 		return (EINVAL);
 	if (sa->l2cap_family != AF_BLUETOOTH)
 		return (EAFNOSUPPORT);
 	if (sa->l2cap_len != sizeof(*sa))
 		return (EINVAL);
 
 	if (bcmp(&sa->l2cap_bdaddr, NG_HCI_BDADDR_ANY,
 				sizeof(sa->l2cap_bdaddr)) != 0) {
 		mtx_lock(&ng_btsocket_l2cap_raw_rt_mtx);
 
 		LIST_FOREACH(rt, &ng_btsocket_l2cap_raw_rt, next) {
 			if (rt->hook == NULL || NG_HOOK_NOT_VALID(rt->hook))
 				continue;
 
 			if (bcmp(&sa->l2cap_bdaddr, &rt->src,
 					sizeof(rt->src)) == 0)
 				break;
 		}
 
 		mtx_unlock(&ng_btsocket_l2cap_raw_rt_mtx);
 
 		if (rt == NULL)
 			return (ENETDOWN);
 	} else
 		rt = NULL;
 
 	mtx_lock(&pcb->pcb_mtx);
 	bcopy(&sa->l2cap_bdaddr, &pcb->src, sizeof(pcb->src));
 	pcb->rt = rt;
 	mtx_unlock(&pcb->pcb_mtx);
 
 	return (0);
 } /* ng_btsocket_l2cap_raw_bind */
 
 /*
  * Connect socket
  */
 
 int
 ng_btsocket_l2cap_raw_connect(struct socket *so, struct sockaddr *nam, 
 		struct thread *td)
 {
 	ng_btsocket_l2cap_raw_pcb_t	*pcb = so2l2cap_raw_pcb(so);
 	struct sockaddr_l2cap		*sa = (struct sockaddr_l2cap *) nam;
 	ng_btsocket_l2cap_rtentry_t	*rt = NULL;
 	int				 error;
 
 	if (pcb == NULL)
 		return (EINVAL);
 	if (ng_btsocket_l2cap_raw_node == NULL) 
 		return (EINVAL);
 
 	if (sa == NULL)
 		return (EINVAL);
 	if (sa->l2cap_family != AF_BLUETOOTH)
 		return (EAFNOSUPPORT);
 	if (sa->l2cap_len != sizeof(*sa))
 		return (EINVAL);
 	if (bcmp(&sa->l2cap_bdaddr, NG_HCI_BDADDR_ANY, sizeof(bdaddr_t)) == 0)
 		return (EINVAL);
 
 	mtx_lock(&pcb->pcb_mtx);
 
 	bcopy(&sa->l2cap_bdaddr, &pcb->dst, sizeof(pcb->dst));
 
 	if (bcmp(&pcb->src, &pcb->dst, sizeof(pcb->src)) == 0) {
 		mtx_unlock(&pcb->pcb_mtx);
 
 		return (EADDRNOTAVAIL);
 	}
 
 	/*
 	 * If there is route already - use it
 	 */
 
 	if (pcb->rt != NULL) {
 		soisconnected(so);
 		mtx_unlock(&pcb->pcb_mtx);
 
 		return (0);
 	}
 	
 	/*
 	 * Find the first hook that does not match specified destination address
 	 */
 
 	mtx_lock(&ng_btsocket_l2cap_raw_rt_mtx);
 
 	LIST_FOREACH(rt, &ng_btsocket_l2cap_raw_rt, next) {
 		if (rt->hook == NULL || NG_HOOK_NOT_VALID(rt->hook))
 			continue;
 
 		if (bcmp(&pcb->dst, &rt->src, sizeof(rt->src)) != 0)
 			break;
 	}
 
 	if (rt != NULL) {
 		soisconnected(so);
 
 		pcb->rt = rt;
 		bcopy(&rt->src, &pcb->src, sizeof(pcb->src));
 
 		error = 0;
 	} else
 		error = ENETDOWN;
 
 	mtx_unlock(&ng_btsocket_l2cap_raw_rt_mtx);
 	mtx_unlock(&pcb->pcb_mtx);
 
 	return  (error);
 } /* ng_btsocket_l2cap_raw_connect */
 
 /*
  * Process ioctl's calls on socket
  */
 
 int
 ng_btsocket_l2cap_raw_control(struct socket *so, u_long cmd, caddr_t data,
 		struct ifnet *ifp, struct thread *td)
 {
 	ng_btsocket_l2cap_raw_pcb_p	 pcb = so2l2cap_raw_pcb(so);
 	struct ng_mesg			*msg = NULL;
 	int				 error = 0;
 
 	if (pcb == NULL)
 		return (EINVAL);
 	if (ng_btsocket_l2cap_raw_node == NULL)
 		return (EINVAL);
 
 	mtx_lock(&pcb->pcb_mtx);
 
 	/* Check if we route info */
 	if (pcb->rt == NULL) {
 		mtx_unlock(&pcb->pcb_mtx);
 		return (EHOSTUNREACH);
 	}
 
 	/* Check if we have pending ioctl() */
 	if (pcb->token != 0) {
 		mtx_unlock(&pcb->pcb_mtx);
 		return (EBUSY);
 	}  
 
 	switch (cmd) {
 	case SIOC_L2CAP_NODE_GET_FLAGS: {
 		struct ng_btsocket_l2cap_raw_node_flags	*p =
 			(struct ng_btsocket_l2cap_raw_node_flags *) data;
 
 		error = ng_btsocket_l2cap_raw_send_sync_ngmsg(pcb,
 				NGM_L2CAP_NODE_GET_FLAGS,
 				&p->flags, sizeof(p->flags));
 		} break;
 
 	case SIOC_L2CAP_NODE_GET_DEBUG: {
 		struct ng_btsocket_l2cap_raw_node_debug	*p =
 			(struct ng_btsocket_l2cap_raw_node_debug *) data;
 
 		error = ng_btsocket_l2cap_raw_send_sync_ngmsg(pcb,
 				NGM_L2CAP_NODE_GET_DEBUG,
 				&p->debug, sizeof(p->debug));
 		} break;
 
 	case SIOC_L2CAP_NODE_SET_DEBUG: {
 		struct ng_btsocket_l2cap_raw_node_debug	*p = 
 			(struct ng_btsocket_l2cap_raw_node_debug *) data;
 
 		if (pcb->flags & NG_BTSOCKET_L2CAP_RAW_PRIVILEGED)
 			error = ng_btsocket_l2cap_raw_send_ngmsg(pcb->rt->hook,
 					NGM_L2CAP_NODE_SET_DEBUG,
 					&p->debug, sizeof(p->debug));
 		else
 			error = EPERM;
 		} break;
 
 	case SIOC_L2CAP_NODE_GET_CON_LIST: {
 		struct ng_btsocket_l2cap_raw_con_list	*p =
 			(struct ng_btsocket_l2cap_raw_con_list *) data;
 		ng_l2cap_node_con_list_ep		*p1 = NULL;
                 ng_l2cap_node_con_ep			*p2 = NULL;
  
 		if (p->num_connections == 0 ||
 		    p->num_connections > NG_L2CAP_MAX_CON_NUM ||
 		    p->connections == NULL) {
 			error = EINVAL;
 			break;
 		}
 
 		NG_MKMESSAGE(msg, NGM_L2CAP_COOKIE, NGM_L2CAP_NODE_GET_CON_LIST,
 			0, M_NOWAIT);
 		if (msg == NULL) {
 			error = ENOMEM;
 			break;
 		}
 		ng_btsocket_l2cap_raw_get_token(&msg->header.token);
 		pcb->token = msg->header.token;
 		pcb->msg = NULL;
 
 		NG_SEND_MSG_HOOK(error, ng_btsocket_l2cap_raw_node, msg,
 			pcb->rt->hook, 0);
 		if (error != 0) {
 			pcb->token = 0;
 			break;
 		}
 
 		error = msleep(&pcb->msg, &pcb->pcb_mtx, PZERO|PCATCH, "l2ctl",
 				ng_btsocket_l2cap_raw_ioctl_timeout * hz);
 		pcb->token = 0;
 
 		if (error != 0)
 			break;
 
 		if (pcb->msg != NULL &&
 		    pcb->msg->header.cmd == NGM_L2CAP_NODE_GET_CON_LIST) {
 			/* Return data back to user space */
 			p1 = (ng_l2cap_node_con_list_ep *)(pcb->msg->data);
 			p2 = (ng_l2cap_node_con_ep *)(p1 + 1);
 
 			p->num_connections = min(p->num_connections,
 						p1->num_connections);
 			if (p->num_connections > 0)
 				error = copyout((caddr_t) p2, 
 					(caddr_t) p->connections,
 					p->num_connections * sizeof(*p2));
 		} else
 			error = EINVAL;
 
 		NG_FREE_MSG(pcb->msg); /* checks for != NULL */
 		} break;
 
 	case SIOC_L2CAP_NODE_GET_CHAN_LIST: {
 		struct ng_btsocket_l2cap_raw_chan_list	*p =
 			(struct ng_btsocket_l2cap_raw_chan_list *) data;
 		ng_l2cap_node_chan_list_ep		*p1 = NULL;
                 ng_l2cap_node_chan_ep			*p2 = NULL;
  
 		if (p->num_channels == 0 ||
 		    p->num_channels > NG_L2CAP_MAX_CHAN_NUM ||
 		    p->channels == NULL) {
 			error = EINVAL;
 			break;
 		}
  
 		NG_MKMESSAGE(msg, NGM_L2CAP_COOKIE,
 			NGM_L2CAP_NODE_GET_CHAN_LIST, 0, M_NOWAIT);
 		if (msg == NULL) {
 			error = ENOMEM;
 			break;
 		}
 		ng_btsocket_l2cap_raw_get_token(&msg->header.token);
 		pcb->token = msg->header.token;
 		pcb->msg = NULL;
 
 		NG_SEND_MSG_HOOK(error, ng_btsocket_l2cap_raw_node, msg,
 			pcb->rt->hook, 0);
 		if (error != 0) {
 			pcb->token = 0;
 			break;
 		}
 
 		error = msleep(&pcb->msg, &pcb->pcb_mtx, PZERO|PCATCH, "l2ctl",
 				ng_btsocket_l2cap_raw_ioctl_timeout * hz);
 		pcb->token = 0;
 
 		if (error != 0)
 			break;
 
 		if (pcb->msg != NULL &&
 		    pcb->msg->header.cmd == NGM_L2CAP_NODE_GET_CHAN_LIST) {
 			/* Return data back to user space */
 			p1 = (ng_l2cap_node_chan_list_ep *)(pcb->msg->data);
 			p2 = (ng_l2cap_node_chan_ep *)(p1 + 1);
 
 			p->num_channels = min(p->num_channels, 
 						p1->num_channels);
 			if (p->num_channels > 0)
 				error = copyout((caddr_t) p2, 
 						(caddr_t) p->channels,
 						p->num_channels * sizeof(*p2));
 		} else
 			error = EINVAL;
 
 		NG_FREE_MSG(pcb->msg); /* checks for != NULL */
 		} break;
 
 	case SIOC_L2CAP_L2CA_PING: {
 		struct ng_btsocket_l2cap_raw_ping	*p = 
 			(struct ng_btsocket_l2cap_raw_ping *) data;
 		ng_l2cap_l2ca_ping_ip			*ip = NULL;
 		ng_l2cap_l2ca_ping_op			*op = NULL;
 
 		if ((p->echo_size != 0 && p->echo_data == NULL) ||
 		     p->echo_size > NG_L2CAP_MAX_ECHO_SIZE) {
 			error = EINVAL;
 			break;
 		}
 
 		NG_MKMESSAGE(msg, NGM_L2CAP_COOKIE,
 			NGM_L2CAP_L2CA_PING, sizeof(*ip) + p->echo_size,
 			M_NOWAIT);
 		if (msg == NULL) {
 			error = ENOMEM;
 			break;
 		}
 		ng_btsocket_l2cap_raw_get_token(&msg->header.token);
 		pcb->token = msg->header.token;
 		pcb->msg = NULL;
 
 		ip = (ng_l2cap_l2ca_ping_ip *)(msg->data);
 		bcopy(&pcb->dst, &ip->bdaddr, sizeof(ip->bdaddr));
 		ip->echo_size = p->echo_size;
 
 		if (ip->echo_size > 0) {
 			error = copyin(p->echo_data, ip + 1, p->echo_size);
 			if (error != 0) {
 				NG_FREE_MSG(msg);
 				pcb->token = 0;
 				break;
 			}
 		}
 
 		NG_SEND_MSG_HOOK(error, ng_btsocket_l2cap_raw_node, msg,
 			pcb->rt->hook, 0);
 		if (error != 0) {
 			pcb->token = 0;
 			break;
 		}
 
 		error = msleep(&pcb->msg, &pcb->pcb_mtx, PZERO|PCATCH, "l2ctl",
 				bluetooth_l2cap_rtx_timeout());
 		pcb->token = 0;
 
 		if (error != 0)
 			break;
 
 		if (pcb->msg != NULL &&
 		    pcb->msg->header.cmd == NGM_L2CAP_L2CA_PING) {
 			/* Return data back to the user space */
 			op = (ng_l2cap_l2ca_ping_op *)(pcb->msg->data);
 			p->result = op->result;
 			p->echo_size = min(p->echo_size, op->echo_size);
 
 			if (p->echo_size > 0)
 				error = copyout(op + 1, p->echo_data, 
 						p->echo_size);
 		} else
 			error = EINVAL;
 
 		NG_FREE_MSG(pcb->msg); /* checks for != NULL */
 		} break;
 
 	case SIOC_L2CAP_L2CA_GET_INFO: {
 		struct ng_btsocket_l2cap_raw_get_info	*p = 
 			(struct ng_btsocket_l2cap_raw_get_info *) data;
 		ng_l2cap_l2ca_get_info_ip		*ip = NULL;
 		ng_l2cap_l2ca_get_info_op		*op = NULL;
 
 		if (!(pcb->flags & NG_BTSOCKET_L2CAP_RAW_PRIVILEGED)) {
 			error = EPERM;
 			break;
 		}
 
 		if (p->info_size != 0 && p->info_data == NULL) {
 			error = EINVAL;
 			break;
 		}
 
 		NG_MKMESSAGE(msg, NGM_L2CAP_COOKIE,
 			NGM_L2CAP_L2CA_GET_INFO, sizeof(*ip) + p->info_size,
 			M_NOWAIT);
 		if (msg == NULL) {
 			error = ENOMEM;
 			break;
 		}
 		ng_btsocket_l2cap_raw_get_token(&msg->header.token);
 		pcb->token = msg->header.token;
 		pcb->msg = NULL;
 
 		ip = (ng_l2cap_l2ca_get_info_ip *)(msg->data);
 		bcopy(&pcb->dst, &ip->bdaddr, sizeof(ip->bdaddr));
 		ip->info_type = p->info_type;
 
 		NG_SEND_MSG_HOOK(error, ng_btsocket_l2cap_raw_node, msg,
 			pcb->rt->hook, 0);
 		if (error != 0) {
 			pcb->token = 0;
 			break;
 		}
 
 		error = msleep(&pcb->msg, &pcb->pcb_mtx, PZERO|PCATCH, "l2ctl",
 				bluetooth_l2cap_rtx_timeout());
 		pcb->token = 0;
 
 		if (error != 0)
 			break;
 
 		if (pcb->msg != NULL &&
 		    pcb->msg->header.cmd == NGM_L2CAP_L2CA_GET_INFO) {
 			/* Return data back to the user space */
 			op = (ng_l2cap_l2ca_get_info_op *)(pcb->msg->data);
 			p->result = op->result;
 			p->info_size = min(p->info_size, op->info_size);
 
 			if (p->info_size > 0)
 				error = copyout(op + 1, p->info_data, 
 						p->info_size);
 		} else
 			error = EINVAL;
 
 		NG_FREE_MSG(pcb->msg); /* checks for != NULL */
 		} break;
 
 	case SIOC_L2CAP_NODE_GET_AUTO_DISCON_TIMO: {
 		struct ng_btsocket_l2cap_raw_auto_discon_timo	*p =
 			(struct ng_btsocket_l2cap_raw_auto_discon_timo *) data;
 
 		error = ng_btsocket_l2cap_raw_send_sync_ngmsg(pcb,
 				NGM_L2CAP_NODE_GET_AUTO_DISCON_TIMO,
 				&p->timeout, sizeof(p->timeout));
 		} break;
 
 	case SIOC_L2CAP_NODE_SET_AUTO_DISCON_TIMO: {
 		struct ng_btsocket_l2cap_raw_auto_discon_timo	*p =
 			(struct ng_btsocket_l2cap_raw_auto_discon_timo *) data;
 
 		if (pcb->flags & NG_BTSOCKET_L2CAP_RAW_PRIVILEGED)
 			error = ng_btsocket_l2cap_raw_send_ngmsg(pcb->rt->hook,
 					NGM_L2CAP_NODE_SET_AUTO_DISCON_TIMO,
 					&p->timeout, sizeof(p->timeout));
 		else
 			error = EPERM;
 		} break;
 
 	default:
 		error = EINVAL;
 		break;
 	}
 
 	mtx_unlock(&pcb->pcb_mtx);
 
 	return (error);
 } /* ng_btsocket_l2cap_raw_control */
 
 /*
  * Detach and destroy socket
  */
 
 void
 ng_btsocket_l2cap_raw_detach(struct socket *so)
 {
 	ng_btsocket_l2cap_raw_pcb_p	pcb = so2l2cap_raw_pcb(so);
 
 	KASSERT(pcb != NULL, ("nt_btsocket_l2cap_raw_detach: pcb == NULL"));
 	if (ng_btsocket_l2cap_raw_node == NULL) 
 		return;
 
 	mtx_lock(&ng_btsocket_l2cap_raw_sockets_mtx);
 	mtx_lock(&pcb->pcb_mtx);
 
 	LIST_REMOVE(pcb, next);
 
 	mtx_unlock(&pcb->pcb_mtx);
 	mtx_unlock(&ng_btsocket_l2cap_raw_sockets_mtx);
 
 	mtx_destroy(&pcb->pcb_mtx);
 
 	bzero(pcb, sizeof(*pcb));
 	free(pcb, M_NETGRAPH_BTSOCKET_L2CAP_RAW);
 
 	so->so_pcb = NULL;
 } /* ng_btsocket_l2cap_raw_detach */
 
 /*
  * Disconnect socket
  */
 
 int
 ng_btsocket_l2cap_raw_disconnect(struct socket *so)
 {
 	ng_btsocket_l2cap_raw_pcb_p	pcb = so2l2cap_raw_pcb(so);
 
 	if (pcb == NULL)
 		return (EINVAL);
 	if (ng_btsocket_l2cap_raw_node == NULL)
 		return (EINVAL);
 
 	mtx_lock(&pcb->pcb_mtx);
 	pcb->rt = NULL;
 	soisdisconnected(so);
 	mtx_unlock(&pcb->pcb_mtx);
 
 	return (0);
 } /* ng_btsocket_l2cap_raw_disconnect */
 
 /*
  * Get peer address
  */
 
 int
 ng_btsocket_l2cap_raw_peeraddr(struct socket *so, struct sockaddr **nam)
 {
 	ng_btsocket_l2cap_raw_pcb_p	pcb = so2l2cap_raw_pcb(so);
 	struct sockaddr_l2cap		sa;
 
 	if (pcb == NULL)
 		return (EINVAL);
 	if (ng_btsocket_l2cap_raw_node == NULL) 
 		return (EINVAL);
 
 	mtx_lock(&pcb->pcb_mtx);
 	bcopy(&pcb->dst, &sa.l2cap_bdaddr, sizeof(sa.l2cap_bdaddr));
 	mtx_unlock(&pcb->pcb_mtx);
 
 	sa.l2cap_psm = 0;
 	sa.l2cap_len = sizeof(sa);
 	sa.l2cap_family = AF_BLUETOOTH;
 
 	*nam = sodupsockaddr((struct sockaddr *) &sa, M_NOWAIT);
 
 	return ((*nam == NULL)? ENOMEM : 0);
 } /* ng_btsocket_l2cap_raw_peeraddr */
 
 /*
  * Send data to socket
  */
 
 int
 ng_btsocket_l2cap_raw_send(struct socket *so, int flags, struct mbuf *m,
 		struct sockaddr *nam, struct mbuf *control, struct thread *td)
 {
 	NG_FREE_M(m); /* Checks for m != NULL */
 	NG_FREE_M(control);
 
 	return (EOPNOTSUPP);
 } /* ng_btsocket_l2cap_raw_send */
 
 /*
  * Get socket address
  */
 
 int
 ng_btsocket_l2cap_raw_sockaddr(struct socket *so, struct sockaddr **nam)
 {
 	ng_btsocket_l2cap_raw_pcb_p	pcb = so2l2cap_raw_pcb(so);
 	struct sockaddr_l2cap		sa;
 
 	if (pcb == NULL)
 		return (EINVAL);
 	if (ng_btsocket_l2cap_raw_node == NULL) 
 		return (EINVAL);
 
 	mtx_lock(&pcb->pcb_mtx);
 	bcopy(&pcb->src, &sa.l2cap_bdaddr, sizeof(sa.l2cap_bdaddr));
 	mtx_unlock(&pcb->pcb_mtx);
 
 	sa.l2cap_psm = 0;
 	sa.l2cap_len = sizeof(sa);
 	sa.l2cap_family = AF_BLUETOOTH;
 
 	*nam = sodupsockaddr((struct sockaddr *) &sa, M_NOWAIT);
 
 	return ((*nam == NULL)? ENOMEM : 0);
 } /* ng_btsocket_l2cap_raw_sockaddr */
 
 /*
  * Get next token
  */
 
 static void
 ng_btsocket_l2cap_raw_get_token(u_int32_t *token)
 {
 	mtx_lock(&ng_btsocket_l2cap_raw_token_mtx);
   
 	if (++ ng_btsocket_l2cap_raw_token == 0)
 		ng_btsocket_l2cap_raw_token = 1;
  
 	*token = ng_btsocket_l2cap_raw_token;
  
 	mtx_unlock(&ng_btsocket_l2cap_raw_token_mtx);
 } /* ng_btsocket_l2cap_raw_get_token */
 
 /*
  * Send Netgraph message to the node - do not expect reply
  */
 
 static int
 ng_btsocket_l2cap_raw_send_ngmsg(hook_p hook, int cmd, void *arg, int arglen)
 {
 	struct ng_mesg	*msg = NULL;
 	int		 error = 0;
 
 	NG_MKMESSAGE(msg, NGM_L2CAP_COOKIE, cmd, arglen, M_NOWAIT);
 	if (msg == NULL)
 		return (ENOMEM);
 
 	if (arg != NULL && arglen > 0)
 		bcopy(arg, msg->data, arglen);
 
 	NG_SEND_MSG_HOOK(error, ng_btsocket_l2cap_raw_node, msg, hook, 0);
 
 	return (error);
 } /* ng_btsocket_l2cap_raw_send_ngmsg */
 
 /*
  * Send Netgraph message to the node (no data) and wait for reply
  */
 
 static int
 ng_btsocket_l2cap_raw_send_sync_ngmsg(ng_btsocket_l2cap_raw_pcb_p pcb,
 		int cmd, void *rsp, int rsplen)
 {
 	struct ng_mesg	*msg = NULL;
 	int		 error = 0;
 
 	mtx_assert(&pcb->pcb_mtx, MA_OWNED);
 
 	NG_MKMESSAGE(msg, NGM_L2CAP_COOKIE, cmd, 0, M_NOWAIT);
 	if (msg == NULL)
 		return (ENOMEM);
 
 	ng_btsocket_l2cap_raw_get_token(&msg->header.token);
 	pcb->token = msg->header.token;
 	pcb->msg = NULL;
 
 	NG_SEND_MSG_HOOK(error, ng_btsocket_l2cap_raw_node, msg,
 		pcb->rt->hook, 0);
 	if (error != 0) {
 		pcb->token = 0;
 		return (error);
 	}
 
 	error = msleep(&pcb->msg, &pcb->pcb_mtx, PZERO|PCATCH, "l2ctl",
 			ng_btsocket_l2cap_raw_ioctl_timeout * hz);
 	pcb->token = 0;
 
 	if (error != 0)
 		return (error);
 
 	if (pcb->msg != NULL && pcb->msg->header.cmd == cmd)
 		bcopy(pcb->msg->data, rsp, rsplen);
 	else
 		error = EINVAL;
 
 	NG_FREE_MSG(pcb->msg); /* checks for != NULL */
 
 	return (0);
 } /* ng_btsocket_l2cap_raw_send_sync_ngmsg */
 
Index: projects/release-debugdist/sys/netgraph/bluetooth/socket/ng_btsocket_rfcomm.c
===================================================================
--- projects/release-debugdist/sys/netgraph/bluetooth/socket/ng_btsocket_rfcomm.c	(revision 267343)
+++ projects/release-debugdist/sys/netgraph/bluetooth/socket/ng_btsocket_rfcomm.c	(revision 267344)
@@ -1,3575 +1,3580 @@
 /*
  * ng_btsocket_rfcomm.c
  */
 
 /*-
  * Copyright (c) 2001-2003 Maksim Yevmenkin <m_evmenkin@yahoo.com>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
  * $Id: ng_btsocket_rfcomm.c,v 1.28 2003/09/14 23:29:06 max Exp $
  * $FreeBSD$
  */
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/bitstring.h>
 #include <sys/domain.h>
 #include <sys/endian.h>
 #include <sys/errno.h>
 #include <sys/filedesc.h>
 #include <sys/ioccom.h>
 #include <sys/kernel.h>
 #include <sys/lock.h>
 #include <sys/malloc.h>
 #include <sys/mbuf.h>
 #include <sys/mutex.h>
 #include <sys/proc.h>
 #include <sys/protosw.h>
 #include <sys/queue.h>
 #include <sys/socket.h>
 #include <sys/socketvar.h>
 #include <sys/sysctl.h>
 #include <sys/taskqueue.h>
 #include <sys/uio.h>
 
 #include <net/vnet.h>
 
 #include <netgraph/ng_message.h>
 #include <netgraph/netgraph.h>
 #include <netgraph/bluetooth/include/ng_bluetooth.h>
 #include <netgraph/bluetooth/include/ng_hci.h>
 #include <netgraph/bluetooth/include/ng_l2cap.h>
 #include <netgraph/bluetooth/include/ng_btsocket.h>
 #include <netgraph/bluetooth/include/ng_btsocket_l2cap.h>
 #include <netgraph/bluetooth/include/ng_btsocket_rfcomm.h>
 
 /* MALLOC define */
 #ifdef NG_SEPARATE_MALLOC
 static MALLOC_DEFINE(M_NETGRAPH_BTSOCKET_RFCOMM, "netgraph_btsocks_rfcomm",
 		"Netgraph Bluetooth RFCOMM sockets");
 #else
 #define M_NETGRAPH_BTSOCKET_RFCOMM M_NETGRAPH
 #endif /* NG_SEPARATE_MALLOC */
 
 /* Debug */
 #define NG_BTSOCKET_RFCOMM_INFO \
 	if (ng_btsocket_rfcomm_debug_level >= NG_BTSOCKET_INFO_LEVEL && \
 	    ppsratecheck(&ng_btsocket_rfcomm_lasttime, &ng_btsocket_rfcomm_curpps, 1)) \
 		printf
 
 #define NG_BTSOCKET_RFCOMM_WARN \
 	if (ng_btsocket_rfcomm_debug_level >= NG_BTSOCKET_WARN_LEVEL && \
 	    ppsratecheck(&ng_btsocket_rfcomm_lasttime, &ng_btsocket_rfcomm_curpps, 1)) \
 		printf
 
 #define NG_BTSOCKET_RFCOMM_ERR \
 	if (ng_btsocket_rfcomm_debug_level >= NG_BTSOCKET_ERR_LEVEL && \
 	    ppsratecheck(&ng_btsocket_rfcomm_lasttime, &ng_btsocket_rfcomm_curpps, 1)) \
 		printf
 
 #define NG_BTSOCKET_RFCOMM_ALERT \
 	if (ng_btsocket_rfcomm_debug_level >= NG_BTSOCKET_ALERT_LEVEL && \
 	    ppsratecheck(&ng_btsocket_rfcomm_lasttime, &ng_btsocket_rfcomm_curpps, 1)) \
 		printf
 
 #define	ALOT	0x7fff
 
 /* Local prototypes */
 static int ng_btsocket_rfcomm_upcall
 	(struct socket *so, void *arg, int waitflag);
 static void ng_btsocket_rfcomm_sessions_task
 	(void *ctx, int pending);
 static void ng_btsocket_rfcomm_session_task
 	(ng_btsocket_rfcomm_session_p s);
 #define ng_btsocket_rfcomm_task_wakeup() \
 	taskqueue_enqueue(taskqueue_swi_giant, &ng_btsocket_rfcomm_task)
 
 static ng_btsocket_rfcomm_pcb_p ng_btsocket_rfcomm_connect_ind
 	(ng_btsocket_rfcomm_session_p s, int channel);
 static void ng_btsocket_rfcomm_connect_cfm
 	(ng_btsocket_rfcomm_session_p s);
 
 static int ng_btsocket_rfcomm_session_create
 	(ng_btsocket_rfcomm_session_p *sp, struct socket *l2so,
 	 bdaddr_p src, bdaddr_p dst, struct thread *td);
 static int ng_btsocket_rfcomm_session_accept
 	(ng_btsocket_rfcomm_session_p s0);
 static int ng_btsocket_rfcomm_session_connect
 	(ng_btsocket_rfcomm_session_p s);
 static int ng_btsocket_rfcomm_session_receive
 	(ng_btsocket_rfcomm_session_p s);
 static int ng_btsocket_rfcomm_session_send
 	(ng_btsocket_rfcomm_session_p s);
 static void ng_btsocket_rfcomm_session_clean
 	(ng_btsocket_rfcomm_session_p s);
 static void ng_btsocket_rfcomm_session_process_pcb
 	(ng_btsocket_rfcomm_session_p s);
 static ng_btsocket_rfcomm_session_p ng_btsocket_rfcomm_session_by_addr
 	(bdaddr_p src, bdaddr_p dst);
 
 static int ng_btsocket_rfcomm_receive_frame
 	(ng_btsocket_rfcomm_session_p s, struct mbuf *m0);
 static int ng_btsocket_rfcomm_receive_sabm
 	(ng_btsocket_rfcomm_session_p s, int dlci);
 static int ng_btsocket_rfcomm_receive_disc
 	(ng_btsocket_rfcomm_session_p s, int dlci);
 static int ng_btsocket_rfcomm_receive_ua
 	(ng_btsocket_rfcomm_session_p s, int dlci);
 static int ng_btsocket_rfcomm_receive_dm
 	(ng_btsocket_rfcomm_session_p s, int dlci);
 static int ng_btsocket_rfcomm_receive_uih
 	(ng_btsocket_rfcomm_session_p s, int dlci, int pf, struct mbuf *m0);
 static int ng_btsocket_rfcomm_receive_mcc
 	(ng_btsocket_rfcomm_session_p s, struct mbuf *m0);
 static int ng_btsocket_rfcomm_receive_test
 	(ng_btsocket_rfcomm_session_p s, struct mbuf *m0);
 static int ng_btsocket_rfcomm_receive_fc
 	(ng_btsocket_rfcomm_session_p s, struct mbuf *m0);
 static int ng_btsocket_rfcomm_receive_msc
 	(ng_btsocket_rfcomm_session_p s, struct mbuf *m0);
 static int ng_btsocket_rfcomm_receive_rpn
 	(ng_btsocket_rfcomm_session_p s, struct mbuf *m0);
 static int ng_btsocket_rfcomm_receive_rls
 	(ng_btsocket_rfcomm_session_p s, struct mbuf *m0);
 static int ng_btsocket_rfcomm_receive_pn
 	(ng_btsocket_rfcomm_session_p s, struct mbuf *m0);
 static void ng_btsocket_rfcomm_set_pn
 	(ng_btsocket_rfcomm_pcb_p pcb, u_int8_t cr, u_int8_t flow_control, 
 	 u_int8_t credits, u_int16_t mtu);
 
 static int ng_btsocket_rfcomm_send_command
 	(ng_btsocket_rfcomm_session_p s, u_int8_t type, u_int8_t dlci);
 static int ng_btsocket_rfcomm_send_uih
 	(ng_btsocket_rfcomm_session_p s, u_int8_t address, u_int8_t pf, 
 	 u_int8_t credits, struct mbuf *data);
 static int ng_btsocket_rfcomm_send_msc
 	(ng_btsocket_rfcomm_pcb_p pcb);
 static int ng_btsocket_rfcomm_send_pn
 	(ng_btsocket_rfcomm_pcb_p pcb);
 static int ng_btsocket_rfcomm_send_credits
 	(ng_btsocket_rfcomm_pcb_p pcb);
 
 static int ng_btsocket_rfcomm_pcb_send
 	(ng_btsocket_rfcomm_pcb_p pcb, int limit);
 static void ng_btsocket_rfcomm_pcb_kill
 	(ng_btsocket_rfcomm_pcb_p pcb, int error);
 static ng_btsocket_rfcomm_pcb_p ng_btsocket_rfcomm_pcb_by_dlci
 	(ng_btsocket_rfcomm_session_p s, int dlci);
 static ng_btsocket_rfcomm_pcb_p ng_btsocket_rfcomm_pcb_listener
 	(bdaddr_p src, int channel);
 
 static void ng_btsocket_rfcomm_timeout
 	(ng_btsocket_rfcomm_pcb_p pcb);
 static void ng_btsocket_rfcomm_untimeout
 	(ng_btsocket_rfcomm_pcb_p pcb);
 static void ng_btsocket_rfcomm_process_timeout
 	(void *xpcb);
 
 static struct mbuf * ng_btsocket_rfcomm_prepare_packet
 	(struct sockbuf *sb, int length);
 
 /* Globals */
 extern int					ifqmaxlen;
 static u_int32_t				ng_btsocket_rfcomm_debug_level;
 static u_int32_t				ng_btsocket_rfcomm_timo;
 struct task					ng_btsocket_rfcomm_task;
 static LIST_HEAD(, ng_btsocket_rfcomm_session)	ng_btsocket_rfcomm_sessions;
 static struct mtx				ng_btsocket_rfcomm_sessions_mtx;
 static LIST_HEAD(, ng_btsocket_rfcomm_pcb)	ng_btsocket_rfcomm_sockets;
 static struct mtx				ng_btsocket_rfcomm_sockets_mtx;
 static struct timeval				ng_btsocket_rfcomm_lasttime;
 static int					ng_btsocket_rfcomm_curpps;
 
 /* Sysctl tree */
 SYSCTL_DECL(_net_bluetooth_rfcomm_sockets);
 static SYSCTL_NODE(_net_bluetooth_rfcomm_sockets, OID_AUTO, stream, CTLFLAG_RW,
 	0, "Bluetooth STREAM RFCOMM sockets family");
 SYSCTL_UINT(_net_bluetooth_rfcomm_sockets_stream, OID_AUTO, debug_level,
 	CTLFLAG_RW,
 	&ng_btsocket_rfcomm_debug_level, NG_BTSOCKET_INFO_LEVEL,
 	"Bluetooth STREAM RFCOMM sockets debug level");
 SYSCTL_UINT(_net_bluetooth_rfcomm_sockets_stream, OID_AUTO, timeout,
 	CTLFLAG_RW,
 	&ng_btsocket_rfcomm_timo, 60,
 	"Bluetooth STREAM RFCOMM sockets timeout");
 
 /*****************************************************************************
  *****************************************************************************
  **                              RFCOMM CRC
  *****************************************************************************
  *****************************************************************************/
 
 static u_int8_t	ng_btsocket_rfcomm_crc_table[256] = {
 	0x00, 0x91, 0xe3, 0x72, 0x07, 0x96, 0xe4, 0x75,
 	0x0e, 0x9f, 0xed, 0x7c, 0x09, 0x98, 0xea, 0x7b,
 	0x1c, 0x8d, 0xff, 0x6e, 0x1b, 0x8a, 0xf8, 0x69,
 	0x12, 0x83, 0xf1, 0x60, 0x15, 0x84, 0xf6, 0x67,
 
 	0x38, 0xa9, 0xdb, 0x4a, 0x3f, 0xae, 0xdc, 0x4d,
 	0x36, 0xa7, 0xd5, 0x44, 0x31, 0xa0, 0xd2, 0x43,
 	0x24, 0xb5, 0xc7, 0x56, 0x23, 0xb2, 0xc0, 0x51,
 	0x2a, 0xbb, 0xc9, 0x58, 0x2d, 0xbc, 0xce, 0x5f,
 
 	0x70, 0xe1, 0x93, 0x02, 0x77, 0xe6, 0x94, 0x05,
 	0x7e, 0xef, 0x9d, 0x0c, 0x79, 0xe8, 0x9a, 0x0b,
 	0x6c, 0xfd, 0x8f, 0x1e, 0x6b, 0xfa, 0x88, 0x19,
 	0x62, 0xf3, 0x81, 0x10, 0x65, 0xf4, 0x86, 0x17,
 
 	0x48, 0xd9, 0xab, 0x3a, 0x4f, 0xde, 0xac, 0x3d,
 	0x46, 0xd7, 0xa5, 0x34, 0x41, 0xd0, 0xa2, 0x33,
 	0x54, 0xc5, 0xb7, 0x26, 0x53, 0xc2, 0xb0, 0x21,
 	0x5a, 0xcb, 0xb9, 0x28, 0x5d, 0xcc, 0xbe, 0x2f,
 
 	0xe0, 0x71, 0x03, 0x92, 0xe7, 0x76, 0x04, 0x95,
 	0xee, 0x7f, 0x0d, 0x9c, 0xe9, 0x78, 0x0a, 0x9b,
 	0xfc, 0x6d, 0x1f, 0x8e, 0xfb, 0x6a, 0x18, 0x89,
 	0xf2, 0x63, 0x11, 0x80, 0xf5, 0x64, 0x16, 0x87,
 
 	0xd8, 0x49, 0x3b, 0xaa, 0xdf, 0x4e, 0x3c, 0xad,
 	0xd6, 0x47, 0x35, 0xa4, 0xd1, 0x40, 0x32, 0xa3,
 	0xc4, 0x55, 0x27, 0xb6, 0xc3, 0x52, 0x20, 0xb1,
 	0xca, 0x5b, 0x29, 0xb8, 0xcd, 0x5c, 0x2e, 0xbf,
 
 	0x90, 0x01, 0x73, 0xe2, 0x97, 0x06, 0x74, 0xe5,
 	0x9e, 0x0f, 0x7d, 0xec, 0x99, 0x08, 0x7a, 0xeb,
 	0x8c, 0x1d, 0x6f, 0xfe, 0x8b, 0x1a, 0x68, 0xf9,
 	0x82, 0x13, 0x61, 0xf0, 0x85, 0x14, 0x66, 0xf7,
 
 	0xa8, 0x39, 0x4b, 0xda, 0xaf, 0x3e, 0x4c, 0xdd,
 	0xa6, 0x37, 0x45, 0xd4, 0xa1, 0x30, 0x42, 0xd3,
 	0xb4, 0x25, 0x57, 0xc6, 0xb3, 0x22, 0x50, 0xc1,
 	0xba, 0x2b, 0x59, 0xc8, 0xbd, 0x2c, 0x5e, 0xcf
 };
 
 /* CRC */
 static u_int8_t
 ng_btsocket_rfcomm_crc(u_int8_t *data, int length)
 {
 	u_int8_t	crc = 0xff;
 
 	while (length --)
 		crc = ng_btsocket_rfcomm_crc_table[crc ^ *data++];
 
 	return (crc);
 } /* ng_btsocket_rfcomm_crc */
 
 /* FCS on 2 bytes */
 static u_int8_t
 ng_btsocket_rfcomm_fcs2(u_int8_t *data)
 {
 	return (0xff - ng_btsocket_rfcomm_crc(data, 2));
 } /* ng_btsocket_rfcomm_fcs2 */
   
 /* FCS on 3 bytes */
 static u_int8_t
 ng_btsocket_rfcomm_fcs3(u_int8_t *data)
 {
 	return (0xff - ng_btsocket_rfcomm_crc(data, 3));
 } /* ng_btsocket_rfcomm_fcs3 */
 
 /* 
  * Check FCS
  *
  * From Bluetooth spec
  *
  * "... In 07.10, the frame check sequence (FCS) is calculated on different 
  * sets of fields for different frame types. These are the fields that the 
  * FCS are calculated on:
  *
  * For SABM, DISC, UA, DM frames: on Address, Control and length field.
  * For UIH frames: on Address and Control field.
  *
  * (This is stated here for clarification, and to set the standard for RFCOMM;
  * the fields included in FCS calculation have actually changed in version
  * 7.0.0 of TS 07.10, but RFCOMM will not change the FCS calculation scheme
  * from the one above.) ..."
  */
 
 static int
 ng_btsocket_rfcomm_check_fcs(u_int8_t *data, int type, u_int8_t fcs)
 {
 	if (type != RFCOMM_FRAME_UIH)
 		return (ng_btsocket_rfcomm_fcs3(data) != fcs);
 
 	return (ng_btsocket_rfcomm_fcs2(data) != fcs);
 } /* ng_btsocket_rfcomm_check_fcs */
 
 /*****************************************************************************
  *****************************************************************************
  **                              Socket interface
  *****************************************************************************
  *****************************************************************************/
 
 /* 
  * Initialize everything
  */
 
 void
 ng_btsocket_rfcomm_init(void)
 {
+
+	/* Skip initialization of globals for non-default instances. */
+	if (!IS_DEFAULT_VNET(curvnet))
+		return;
+
 	ng_btsocket_rfcomm_debug_level = NG_BTSOCKET_WARN_LEVEL;
 	ng_btsocket_rfcomm_timo = 60;
 
 	/* RFCOMM task */
 	TASK_INIT(&ng_btsocket_rfcomm_task, 0,
 		ng_btsocket_rfcomm_sessions_task, NULL);
 
 	/* RFCOMM sessions list */
 	LIST_INIT(&ng_btsocket_rfcomm_sessions);
 	mtx_init(&ng_btsocket_rfcomm_sessions_mtx,
 		"btsocks_rfcomm_sessions_mtx", NULL, MTX_DEF);
 
 	/* RFCOMM sockets list */
 	LIST_INIT(&ng_btsocket_rfcomm_sockets);
 	mtx_init(&ng_btsocket_rfcomm_sockets_mtx,
 		"btsocks_rfcomm_sockets_mtx", NULL, MTX_DEF);
 } /* ng_btsocket_rfcomm_init */
 
 /*
  * Abort connection on socket
  */
 
 void
 ng_btsocket_rfcomm_abort(struct socket *so)
 {
 
 	so->so_error = ECONNABORTED;
 	(void)ng_btsocket_rfcomm_disconnect(so);
 } /* ng_btsocket_rfcomm_abort */
 
 void
 ng_btsocket_rfcomm_close(struct socket *so)
 {
 
 	(void)ng_btsocket_rfcomm_disconnect(so);
 } /* ng_btsocket_rfcomm_close */
 
 /*
  * Accept connection on socket. Nothing to do here, socket must be connected
  * and ready, so just return peer address and be done with it.
  */
 
 int
 ng_btsocket_rfcomm_accept(struct socket *so, struct sockaddr **nam)
 {
 	return (ng_btsocket_rfcomm_peeraddr(so, nam));
 } /* ng_btsocket_rfcomm_accept */
 
 /*
  * Create and attach new socket
  */
 
 int
 ng_btsocket_rfcomm_attach(struct socket *so, int proto, struct thread *td)
 {
 	ng_btsocket_rfcomm_pcb_p	pcb = so2rfcomm_pcb(so);
 	int				error;
 
 	/* Check socket and protocol */
 	if (so->so_type != SOCK_STREAM)
 		return (ESOCKTNOSUPPORT);
 
 #if 0 /* XXX sonewconn() calls "pru_attach" with proto == 0 */
 	if (proto != 0) 
 		if (proto != BLUETOOTH_PROTO_RFCOMM)
 			return (EPROTONOSUPPORT);
 #endif /* XXX */
 
 	if (pcb != NULL)
 		return (EISCONN);
 
 	/* Reserve send and receive space if it is not reserved yet */
 	if ((so->so_snd.sb_hiwat == 0) || (so->so_rcv.sb_hiwat == 0)) {
 		error = soreserve(so, NG_BTSOCKET_RFCOMM_SENDSPACE,
 					NG_BTSOCKET_RFCOMM_RECVSPACE);
 		if (error != 0)
 			return (error);
 	}
 
 	/* Allocate the PCB */
         pcb = malloc(sizeof(*pcb),
 		M_NETGRAPH_BTSOCKET_RFCOMM, M_NOWAIT | M_ZERO);
         if (pcb == NULL)
                 return (ENOMEM);
 
 	/* Link the PCB and the socket */
 	so->so_pcb = (caddr_t) pcb;
 	pcb->so = so;
 
 	/* Initialize PCB */
 	pcb->state = NG_BTSOCKET_RFCOMM_DLC_CLOSED;
 	pcb->flags = NG_BTSOCKET_RFCOMM_DLC_CFC;
 
 	pcb->lmodem =
 	pcb->rmodem = (RFCOMM_MODEM_RTC | RFCOMM_MODEM_RTR | RFCOMM_MODEM_DV);
 
 	pcb->mtu = RFCOMM_DEFAULT_MTU;
 	pcb->tx_cred = 0;
 	pcb->rx_cred = RFCOMM_DEFAULT_CREDITS;
 
 	mtx_init(&pcb->pcb_mtx, "btsocks_rfcomm_pcb_mtx", NULL, MTX_DEF);
 	callout_handle_init(&pcb->timo);
 
 	/* Add the PCB to the list */
 	mtx_lock(&ng_btsocket_rfcomm_sockets_mtx);
 	LIST_INSERT_HEAD(&ng_btsocket_rfcomm_sockets, pcb, next);
 	mtx_unlock(&ng_btsocket_rfcomm_sockets_mtx);
 
         return (0);
 } /* ng_btsocket_rfcomm_attach */
 
 /*
  * Bind socket
  */
 
 int
 ng_btsocket_rfcomm_bind(struct socket *so, struct sockaddr *nam, 
 		struct thread *td)
 {
 	ng_btsocket_rfcomm_pcb_t	*pcb = so2rfcomm_pcb(so), *pcb1;
 	struct sockaddr_rfcomm		*sa = (struct sockaddr_rfcomm *) nam;
 
 	if (pcb == NULL)
 		return (EINVAL);
 
 	/* Verify address */
 	if (sa == NULL)
 		return (EINVAL);
 	if (sa->rfcomm_family != AF_BLUETOOTH)
 		return (EAFNOSUPPORT);
 	if (sa->rfcomm_len != sizeof(*sa))
 		return (EINVAL);
 	if (sa->rfcomm_channel > 30)
 		return (EINVAL);
 
 	mtx_lock(&pcb->pcb_mtx);
 
 	if (sa->rfcomm_channel != 0) {
 		mtx_lock(&ng_btsocket_rfcomm_sockets_mtx);
 
 		LIST_FOREACH(pcb1, &ng_btsocket_rfcomm_sockets, next) {
 			if (pcb1->channel == sa->rfcomm_channel &&
 			    bcmp(&pcb1->src, &sa->rfcomm_bdaddr,
 					sizeof(pcb1->src)) == 0) {
 				mtx_unlock(&ng_btsocket_rfcomm_sockets_mtx);
 				mtx_unlock(&pcb->pcb_mtx);
 
 				return (EADDRINUSE);
 			}
 		}
 
 		mtx_unlock(&ng_btsocket_rfcomm_sockets_mtx);
 	}
 
 	bcopy(&sa->rfcomm_bdaddr, &pcb->src, sizeof(pcb->src));
 	pcb->channel = sa->rfcomm_channel;
 
 	mtx_unlock(&pcb->pcb_mtx);
 
 	return (0);
 } /* ng_btsocket_rfcomm_bind */
 
 /*
  * Connect socket
  */
 
 int
 ng_btsocket_rfcomm_connect(struct socket *so, struct sockaddr *nam, 
 		struct thread *td)
 {
 	ng_btsocket_rfcomm_pcb_t	*pcb = so2rfcomm_pcb(so);
 	struct sockaddr_rfcomm		*sa = (struct sockaddr_rfcomm *) nam;
 	ng_btsocket_rfcomm_session_t	*s = NULL;
 	struct socket			*l2so = NULL;
 	int				 dlci, error = 0;
 
 	if (pcb == NULL)
 		return (EINVAL);
 
 	/* Verify address */
 	if (sa == NULL)
 		return (EINVAL);
 	if (sa->rfcomm_family != AF_BLUETOOTH)
 		return (EAFNOSUPPORT);
 	if (sa->rfcomm_len != sizeof(*sa))
 		return (EINVAL);
 	if (sa->rfcomm_channel > 30)
 		return (EINVAL);
 	if (sa->rfcomm_channel == 0 ||
 	    bcmp(&sa->rfcomm_bdaddr, NG_HCI_BDADDR_ANY, sizeof(bdaddr_t)) == 0)
 		return (EDESTADDRREQ);
 
 	/*
 	 * Note that we will not check for errors in socreate() because
 	 * if we failed to create L2CAP socket at this point we still
 	 * might have already open session.
 	 */
 
 	error = socreate(PF_BLUETOOTH, &l2so, SOCK_SEQPACKET,
 			BLUETOOTH_PROTO_L2CAP, td->td_ucred, td);
 
 	/* 
 	 * Look for session between "pcb->src" and "sa->rfcomm_bdaddr" (dst)
 	 */
 
 	mtx_lock(&ng_btsocket_rfcomm_sessions_mtx);
 
 	s = ng_btsocket_rfcomm_session_by_addr(&pcb->src, &sa->rfcomm_bdaddr);
 	if (s == NULL) {
 		/*
 		 * We need to create new RFCOMM session. Check if we have L2CAP
 		 * socket. If l2so == NULL then error has the error code from
 		 * socreate()
 		 */
 
 		if (l2so == NULL) {
 			mtx_unlock(&ng_btsocket_rfcomm_sessions_mtx);
 			return (error);
 		}
 
 		error = ng_btsocket_rfcomm_session_create(&s, l2so,
 				&pcb->src, &sa->rfcomm_bdaddr, td);
 		if (error != 0) {
 			mtx_unlock(&ng_btsocket_rfcomm_sessions_mtx);
 			soclose(l2so);
 
 			return (error);
 		}
 	} else if (l2so != NULL)
 		soclose(l2so); /* we don't need new L2CAP socket */
 
 	/*
 	 * Check if we already have the same DLCI the same session
 	 */
 
 	mtx_lock(&s->session_mtx);
 	mtx_lock(&pcb->pcb_mtx);
 
 	dlci = RFCOMM_MKDLCI(!INITIATOR(s), sa->rfcomm_channel);
 
 	if (ng_btsocket_rfcomm_pcb_by_dlci(s, dlci) != NULL) {
 		mtx_unlock(&pcb->pcb_mtx);
 		mtx_unlock(&s->session_mtx);
 		mtx_unlock(&ng_btsocket_rfcomm_sessions_mtx);
 
 		return (EBUSY);
 	}
 
 	/*
 	 * Check session state and if its not acceptable then refuse connection
 	 */
 
 	switch (s->state) {
 	case NG_BTSOCKET_RFCOMM_SESSION_CONNECTING:
 	case NG_BTSOCKET_RFCOMM_SESSION_CONNECTED:
 	case NG_BTSOCKET_RFCOMM_SESSION_OPEN:
 		/*
 		 * Update destination address and channel and attach 
 		 * DLC to the session
 		 */
 
 		bcopy(&sa->rfcomm_bdaddr, &pcb->dst, sizeof(pcb->dst));
 		pcb->channel = sa->rfcomm_channel;
 		pcb->dlci = dlci;
 
 		LIST_INSERT_HEAD(&s->dlcs, pcb, session_next);
 		pcb->session = s;
 
 		ng_btsocket_rfcomm_timeout(pcb);
 		soisconnecting(pcb->so);
 
 		if (s->state == NG_BTSOCKET_RFCOMM_SESSION_OPEN) {
 			pcb->mtu = s->mtu;
 			bcopy(&so2l2cap_pcb(s->l2so)->src, &pcb->src,
 				sizeof(pcb->src));
 
 			pcb->state = NG_BTSOCKET_RFCOMM_DLC_CONFIGURING;
 
 			error = ng_btsocket_rfcomm_send_pn(pcb);
 			if (error == 0)
 				error = ng_btsocket_rfcomm_task_wakeup();
 		} else
 			pcb->state = NG_BTSOCKET_RFCOMM_DLC_W4_CONNECT;
 		break;
 
 	default:
 		error = ECONNRESET;
 		break;
 	}
 
 	mtx_unlock(&pcb->pcb_mtx);
 	mtx_unlock(&s->session_mtx);
 	mtx_unlock(&ng_btsocket_rfcomm_sessions_mtx);
 
 	return (error);
 } /* ng_btsocket_rfcomm_connect */
 
 /*
  * Process ioctl's calls on socket.
  * XXX FIXME this should provide interface to the RFCOMM multiplexor channel
  */
 
 int
 ng_btsocket_rfcomm_control(struct socket *so, u_long cmd, caddr_t data,
 		struct ifnet *ifp, struct thread *td)
 {
 	return (EINVAL);
 } /* ng_btsocket_rfcomm_control */
 
 /*
  * Process getsockopt/setsockopt system calls
  */
 
 int
 ng_btsocket_rfcomm_ctloutput(struct socket *so, struct sockopt *sopt)
 {
 	ng_btsocket_rfcomm_pcb_p		pcb = so2rfcomm_pcb(so);
 	struct ng_btsocket_rfcomm_fc_info	fcinfo;
 	int					error = 0;
 
 	if (pcb == NULL)
 		return (EINVAL);
 	if (sopt->sopt_level != SOL_RFCOMM)
 		return (0);
 
 	mtx_lock(&pcb->pcb_mtx);
 
 	switch (sopt->sopt_dir) {
 	case SOPT_GET:
 		switch (sopt->sopt_name) {
 		case SO_RFCOMM_MTU:
 			error = sooptcopyout(sopt, &pcb->mtu, sizeof(pcb->mtu));
 			break;
 
 		case SO_RFCOMM_FC_INFO:
 			fcinfo.lmodem = pcb->lmodem;
 			fcinfo.rmodem = pcb->rmodem;
 			fcinfo.tx_cred = pcb->tx_cred;
 			fcinfo.rx_cred = pcb->rx_cred;
 			fcinfo.cfc = (pcb->flags & NG_BTSOCKET_RFCOMM_DLC_CFC)?
 				1 : 0;
 			fcinfo.reserved = 0;
 
 			error = sooptcopyout(sopt, &fcinfo, sizeof(fcinfo));
 			break;
 
 		default:
 			error = ENOPROTOOPT;
 			break;
 		}
 		break;
 
 	case SOPT_SET:
 		switch (sopt->sopt_name) {
 		default:
 			error = ENOPROTOOPT;
 			break;
 		}
 		break;
 
 	default:
 		error = EINVAL;
 		break;
 	}
 
 	mtx_unlock(&pcb->pcb_mtx);
 
 	return (error);
 } /* ng_btsocket_rfcomm_ctloutput */
 
 /*
  * Detach and destroy socket
  */
 
 void
 ng_btsocket_rfcomm_detach(struct socket *so)
 {
 	ng_btsocket_rfcomm_pcb_p	pcb = so2rfcomm_pcb(so);
 
 	KASSERT(pcb != NULL, ("ng_btsocket_rfcomm_detach: pcb == NULL"));
 
 	mtx_lock(&pcb->pcb_mtx);
 
 	switch (pcb->state) {
 	case NG_BTSOCKET_RFCOMM_DLC_W4_CONNECT:
 	case NG_BTSOCKET_RFCOMM_DLC_CONFIGURING:
 	case NG_BTSOCKET_RFCOMM_DLC_CONNECTING:
 	case NG_BTSOCKET_RFCOMM_DLC_CONNECTED:
 		/* XXX What to do with pending request? */
 		if (pcb->flags & NG_BTSOCKET_RFCOMM_DLC_TIMO)
 			ng_btsocket_rfcomm_untimeout(pcb);
 
 		if (pcb->state == NG_BTSOCKET_RFCOMM_DLC_W4_CONNECT)
 			pcb->flags |= NG_BTSOCKET_RFCOMM_DLC_DETACHED;
 		else
 			pcb->state = NG_BTSOCKET_RFCOMM_DLC_DISCONNECTING;
 
 		ng_btsocket_rfcomm_task_wakeup();
 		break;
 
 	case NG_BTSOCKET_RFCOMM_DLC_DISCONNECTING:
 		ng_btsocket_rfcomm_task_wakeup();
 		break;
 	}
 	
 	while (pcb->state != NG_BTSOCKET_RFCOMM_DLC_CLOSED)
 		msleep(&pcb->state, &pcb->pcb_mtx, PZERO, "rf_det", 0);
 
 	if (pcb->session != NULL)
 		panic("%s: pcb->session != NULL\n", __func__);
 	if (pcb->flags & NG_BTSOCKET_RFCOMM_DLC_TIMO)
 		panic("%s: timeout on closed DLC, flags=%#x\n",
 			__func__, pcb->flags);
 
 	mtx_lock(&ng_btsocket_rfcomm_sockets_mtx);
 	LIST_REMOVE(pcb, next);
 	mtx_unlock(&ng_btsocket_rfcomm_sockets_mtx);
 
 	mtx_unlock(&pcb->pcb_mtx);
 
 	mtx_destroy(&pcb->pcb_mtx);
 	bzero(pcb, sizeof(*pcb));
 	free(pcb, M_NETGRAPH_BTSOCKET_RFCOMM);
 
 	soisdisconnected(so);
 	so->so_pcb = NULL;
 } /* ng_btsocket_rfcomm_detach */
 
 /*
  * Disconnect socket
  */
 
 int
 ng_btsocket_rfcomm_disconnect(struct socket *so)
 {
 	ng_btsocket_rfcomm_pcb_p	pcb = so2rfcomm_pcb(so);
 
 	if (pcb == NULL)
 		return (EINVAL);
 
 	mtx_lock(&pcb->pcb_mtx);
 
 	if (pcb->state == NG_BTSOCKET_RFCOMM_DLC_DISCONNECTING) {
 		mtx_unlock(&pcb->pcb_mtx);
 		return (EINPROGRESS);
 	}
 
 	/* XXX What to do with pending request? */
 	if (pcb->flags & NG_BTSOCKET_RFCOMM_DLC_TIMO)
 		ng_btsocket_rfcomm_untimeout(pcb);
 
 	switch (pcb->state) {
 	case NG_BTSOCKET_RFCOMM_DLC_CONFIGURING: /* XXX can we get here? */
 	case NG_BTSOCKET_RFCOMM_DLC_CONNECTING: /* XXX can we get here? */
 	case NG_BTSOCKET_RFCOMM_DLC_CONNECTED:
 
 		/*
 		 * Just change DLC state and enqueue RFCOMM task. It will
 		 * queue and send DISC on the DLC.
 		 */ 
 
 		pcb->state = NG_BTSOCKET_RFCOMM_DLC_DISCONNECTING;
 		soisdisconnecting(so);
 
 		ng_btsocket_rfcomm_task_wakeup();
 		break;
 
 	case NG_BTSOCKET_RFCOMM_DLC_CLOSED:
 	case NG_BTSOCKET_RFCOMM_DLC_W4_CONNECT:
 		break;
 
 	default:
 		panic("%s: Invalid DLC state=%d, flags=%#x\n",
 			__func__, pcb->state, pcb->flags);
 		break;
 	}
 
 	mtx_unlock(&pcb->pcb_mtx);
 
 	return (0);
 } /* ng_btsocket_rfcomm_disconnect */
 
 /*
  * Listen on socket. First call to listen() will create listening RFCOMM session
  */
 
 int
 ng_btsocket_rfcomm_listen(struct socket *so, int backlog, struct thread *td)
 {
 	ng_btsocket_rfcomm_pcb_p	 pcb = so2rfcomm_pcb(so), pcb1;
 	ng_btsocket_rfcomm_session_p	 s = NULL;
 	struct socket			*l2so = NULL;
 	int				 error, socreate_error, usedchannels;
 
 	if (pcb == NULL)
 		return (EINVAL);
 	if (pcb->channel > 30)
 		return (EADDRNOTAVAIL);
 
 	usedchannels = 0;
 
 	mtx_lock(&pcb->pcb_mtx);
 
 	if (pcb->channel == 0) {
 		mtx_lock(&ng_btsocket_rfcomm_sockets_mtx);
 
 		LIST_FOREACH(pcb1, &ng_btsocket_rfcomm_sockets, next)
 			if (pcb1->channel != 0 &&
 			    bcmp(&pcb1->src, &pcb->src, sizeof(pcb->src)) == 0)
 				usedchannels |= (1 << (pcb1->channel - 1));
 
 		for (pcb->channel = 30; pcb->channel > 0; pcb->channel --)
 			if (!(usedchannels & (1 << (pcb->channel - 1))))
 				break;
 
 		if (pcb->channel == 0) {
 			mtx_unlock(&ng_btsocket_rfcomm_sockets_mtx);
 			mtx_unlock(&pcb->pcb_mtx);
 
 			return (EADDRNOTAVAIL);
 		}
 
 		mtx_unlock(&ng_btsocket_rfcomm_sockets_mtx);
 	}
 
 	mtx_unlock(&pcb->pcb_mtx);
 
 	/*
 	 * Note that we will not check for errors in socreate() because
 	 * if we failed to create L2CAP socket at this point we still
 	 * might have already open session.
 	 */
 
 	socreate_error = socreate(PF_BLUETOOTH, &l2so, SOCK_SEQPACKET,
 			BLUETOOTH_PROTO_L2CAP, td->td_ucred, td);
 
 	/*
 	 * Transition the socket and session into the LISTENING state.  Check
 	 * for collisions first, as there can only be one.
 	 */
 	mtx_lock(&ng_btsocket_rfcomm_sessions_mtx);
 	SOCK_LOCK(so);
 	error = solisten_proto_check(so);
 	SOCK_UNLOCK(so);
 	if (error != 0)
 		goto out;
 
 	LIST_FOREACH(s, &ng_btsocket_rfcomm_sessions, next)
 		if (s->state == NG_BTSOCKET_RFCOMM_SESSION_LISTENING)
 			break;
 
 	if (s == NULL) {
 		/*
 		 * We need to create default RFCOMM session. Check if we have 
 		 * L2CAP socket. If l2so == NULL then error has the error code 
 		 * from socreate()
 		 */
 		if (l2so == NULL) {
 			error = socreate_error;
 			goto out;
 		}
 
 		/* 
 		 * Create default listen RFCOMM session. The default RFCOMM 
 		 * session will listen on ANY address.
 		 *
 		 * XXX FIXME Note that currently there is no way to adjust MTU
 		 * for the default session.
 		 */
 		error = ng_btsocket_rfcomm_session_create(&s, l2so,
 					NG_HCI_BDADDR_ANY, NULL, td);
 		if (error != 0)
 			goto out;
 		l2so = NULL;
 	}
 	SOCK_LOCK(so);
 	solisten_proto(so, backlog);
 	SOCK_UNLOCK(so);
 out:
 	mtx_unlock(&ng_btsocket_rfcomm_sessions_mtx);
 	/*
 	 * If we still have an l2so reference here, it's unneeded, so release
 	 * it.
 	 */
 	if (l2so != NULL)
 		soclose(l2so);
 	return (error);
 } /* ng_btsocket_listen */
 
 /*
  * Get peer address
  */
 
 int
 ng_btsocket_rfcomm_peeraddr(struct socket *so, struct sockaddr **nam)
 {
 	ng_btsocket_rfcomm_pcb_p	pcb = so2rfcomm_pcb(so);
 	struct sockaddr_rfcomm		sa;
 
 	if (pcb == NULL)
 		return (EINVAL);
 
 	bcopy(&pcb->dst, &sa.rfcomm_bdaddr, sizeof(sa.rfcomm_bdaddr));
 	sa.rfcomm_channel = pcb->channel;
 	sa.rfcomm_len = sizeof(sa);
 	sa.rfcomm_family = AF_BLUETOOTH;
 
 	*nam = sodupsockaddr((struct sockaddr *) &sa, M_NOWAIT);
 
 	return ((*nam == NULL)? ENOMEM : 0);
 } /* ng_btsocket_rfcomm_peeraddr */
 
 /*
  * Send data to socket
  */
 
 int
 ng_btsocket_rfcomm_send(struct socket *so, int flags, struct mbuf *m,
 		struct sockaddr *nam, struct mbuf *control, struct thread *td)
 {
 	ng_btsocket_rfcomm_pcb_t	*pcb = so2rfcomm_pcb(so);
 	int				 error = 0;
 
 	/* Check socket and input */
 	if (pcb == NULL || m == NULL || control != NULL) {
 		error = EINVAL;
 		goto drop;
 	}
 
 	mtx_lock(&pcb->pcb_mtx);
 
 	/* Make sure DLC is connected */
 	if (pcb->state != NG_BTSOCKET_RFCOMM_DLC_CONNECTED) {
 		mtx_unlock(&pcb->pcb_mtx);
 		error = ENOTCONN;
 		goto drop;
 	}
 
 	/* Put the packet on the socket's send queue and wakeup RFCOMM task */
 	sbappend(&pcb->so->so_snd, m);
 	m = NULL;
 	
 	if (!(pcb->flags & NG_BTSOCKET_RFCOMM_DLC_SENDING)) {
 		pcb->flags |= NG_BTSOCKET_RFCOMM_DLC_SENDING;
 		error = ng_btsocket_rfcomm_task_wakeup();
 	}
 
 	mtx_unlock(&pcb->pcb_mtx);
 drop:
 	NG_FREE_M(m); /* checks for != NULL */
 	NG_FREE_M(control);
 
 	return (error);
 } /* ng_btsocket_rfcomm_send */
 
 /*
  * Get socket address
  */
 
 int
 ng_btsocket_rfcomm_sockaddr(struct socket *so, struct sockaddr **nam)
 {
 	ng_btsocket_rfcomm_pcb_p	pcb = so2rfcomm_pcb(so);
 	struct sockaddr_rfcomm		sa;
 
 	if (pcb == NULL)
 		return (EINVAL);
 
 	bcopy(&pcb->src, &sa.rfcomm_bdaddr, sizeof(sa.rfcomm_bdaddr));
 	sa.rfcomm_channel = pcb->channel;
 	sa.rfcomm_len = sizeof(sa);
 	sa.rfcomm_family = AF_BLUETOOTH;
 
 	*nam = sodupsockaddr((struct sockaddr *) &sa, M_NOWAIT);
 
 	return ((*nam == NULL)? ENOMEM : 0);
 } /* ng_btsocket_rfcomm_sockaddr */
 
 /*
  * Upcall function for L2CAP sockets. Enqueue RFCOMM task.
  */
 
 static int
 ng_btsocket_rfcomm_upcall(struct socket *so, void *arg, int waitflag)
 {
 	int	error;
 
 	if (so == NULL)
 		panic("%s: so == NULL\n", __func__);
 
 	if ((error = ng_btsocket_rfcomm_task_wakeup()) != 0)
 		NG_BTSOCKET_RFCOMM_ALERT(
 "%s: Could not enqueue RFCOMM task, error=%d\n", __func__, error);
 	return (SU_OK);
 } /* ng_btsocket_rfcomm_upcall */
 
 /*
  * RFCOMM task. Will handle all RFCOMM sessions in one pass.
  * XXX FIXME does not scale very well
  */
 
 static void
 ng_btsocket_rfcomm_sessions_task(void *ctx, int pending)
 {
 	ng_btsocket_rfcomm_session_p	s = NULL, s_next = NULL;
 
 	mtx_lock(&ng_btsocket_rfcomm_sessions_mtx);
 
 	for (s = LIST_FIRST(&ng_btsocket_rfcomm_sessions); s != NULL; ) {
 		mtx_lock(&s->session_mtx);
 		s_next = LIST_NEXT(s, next);
 
 		ng_btsocket_rfcomm_session_task(s);
 
 		if (s->state == NG_BTSOCKET_RFCOMM_SESSION_CLOSED) {
 			/* Unlink and clean the session */
 			LIST_REMOVE(s, next);
 
 			NG_BT_MBUFQ_DRAIN(&s->outq);
 			if (!LIST_EMPTY(&s->dlcs))
 				panic("%s: DLC list is not empty\n", __func__);
 
 			/* Close L2CAP socket */
 			SOCKBUF_LOCK(&s->l2so->so_rcv);
 			soupcall_clear(s->l2so, SO_RCV);
 			SOCKBUF_UNLOCK(&s->l2so->so_rcv);
 			SOCKBUF_LOCK(&s->l2so->so_snd);
 			soupcall_clear(s->l2so, SO_SND);
 			SOCKBUF_UNLOCK(&s->l2so->so_snd);
 			soclose(s->l2so);
 
 			mtx_unlock(&s->session_mtx);
 
 			mtx_destroy(&s->session_mtx);
 			bzero(s, sizeof(*s));
 			free(s, M_NETGRAPH_BTSOCKET_RFCOMM);
 		} else
 			mtx_unlock(&s->session_mtx);
 
 		s = s_next;
 	}
 
 	mtx_unlock(&ng_btsocket_rfcomm_sessions_mtx);
 } /* ng_btsocket_rfcomm_sessions_task */
 
 /*
  * Process RFCOMM session. Will handle all RFCOMM sockets in one pass.
  */
 
 static void
 ng_btsocket_rfcomm_session_task(ng_btsocket_rfcomm_session_p s)
 {
 	mtx_assert(&s->session_mtx, MA_OWNED);
 
 	if (s->l2so->so_rcv.sb_state & SBS_CANTRCVMORE) {
 		NG_BTSOCKET_RFCOMM_INFO(
 "%s: L2CAP connection has been terminated, so=%p, so_state=%#x, so_count=%d, " \
 "state=%d, flags=%#x\n", __func__, s->l2so, s->l2so->so_state, 
 			s->l2so->so_count, s->state, s->flags);
 
 		s->state = NG_BTSOCKET_RFCOMM_SESSION_CLOSED;
 		ng_btsocket_rfcomm_session_clean(s);
 	}
 
 	/* Now process upcall */
 	switch (s->state) {
 	/* Try to accept new L2CAP connection(s) */
 	case NG_BTSOCKET_RFCOMM_SESSION_LISTENING:
 		while (ng_btsocket_rfcomm_session_accept(s) == 0)
 			;
 		break;
 
 	/* Process the results of the L2CAP connect */
 	case NG_BTSOCKET_RFCOMM_SESSION_CONNECTING:
 		ng_btsocket_rfcomm_session_process_pcb(s);
 
 		if (ng_btsocket_rfcomm_session_connect(s) != 0) {
 			s->state = NG_BTSOCKET_RFCOMM_SESSION_CLOSED;
 			ng_btsocket_rfcomm_session_clean(s);
 		} 
 		break;
 
 	/* Try to receive/send more data */
 	case NG_BTSOCKET_RFCOMM_SESSION_CONNECTED:
 	case NG_BTSOCKET_RFCOMM_SESSION_OPEN:
 	case NG_BTSOCKET_RFCOMM_SESSION_DISCONNECTING:
 		ng_btsocket_rfcomm_session_process_pcb(s);
 
 		if (ng_btsocket_rfcomm_session_receive(s) != 0) {
 			s->state = NG_BTSOCKET_RFCOMM_SESSION_CLOSED;
 			ng_btsocket_rfcomm_session_clean(s);
 		} else if (ng_btsocket_rfcomm_session_send(s) != 0) {
 			s->state = NG_BTSOCKET_RFCOMM_SESSION_CLOSED;
 			ng_btsocket_rfcomm_session_clean(s);
 		}
 		break;
 
 	case NG_BTSOCKET_RFCOMM_SESSION_CLOSED:
 		break;
 
 	default:
 		panic("%s: Invalid session state=%d, flags=%#x\n",
 			__func__, s->state, s->flags);
 		break;
 	}
 } /* ng_btsocket_rfcomm_session_task */
 
 /*
  * Process RFCOMM connection indicator. Caller must hold s->session_mtx
  */
 
 static ng_btsocket_rfcomm_pcb_p
 ng_btsocket_rfcomm_connect_ind(ng_btsocket_rfcomm_session_p s, int channel)
 {
 	ng_btsocket_rfcomm_pcb_p	 pcb = NULL, pcb1 = NULL;
 	ng_btsocket_l2cap_pcb_p		 l2pcb = NULL;
 	struct socket			*so1 = NULL;
 
 	mtx_assert(&s->session_mtx, MA_OWNED);
 
 	/*
 	 * Try to find RFCOMM socket that listens on given source address 
 	 * and channel. This will return the best possible match.
 	 */
 
 	l2pcb = so2l2cap_pcb(s->l2so);
 	pcb = ng_btsocket_rfcomm_pcb_listener(&l2pcb->src, channel);
 	if (pcb == NULL)
 		return (NULL);
 
 	/*
 	 * Check the pending connections queue and if we have space then 
 	 * create new socket and set proper source and destination address,
 	 * and channel.
 	 */
 
 	mtx_lock(&pcb->pcb_mtx);
 
 	if (pcb->so->so_qlen <= pcb->so->so_qlimit) {
 		CURVNET_SET(pcb->so->so_vnet);
 		so1 = sonewconn(pcb->so, 0);
 		CURVNET_RESTORE();
 	}
 
 	mtx_unlock(&pcb->pcb_mtx);
 
 	if (so1 == NULL)
 		return (NULL);
 
 	/*
 	 * If we got here than we have created new socket. So complete the 
 	 * connection. Set source and destination address from the session.
 	 */
 
 	pcb1 = so2rfcomm_pcb(so1);
 	if (pcb1 == NULL)
 		panic("%s: pcb1 == NULL\n", __func__);
 
 	mtx_lock(&pcb1->pcb_mtx);
 
 	bcopy(&l2pcb->src, &pcb1->src, sizeof(pcb1->src));
 	bcopy(&l2pcb->dst, &pcb1->dst, sizeof(pcb1->dst));
 	pcb1->channel = channel;
 
 	/* Link new DLC to the session. We already hold s->session_mtx */
 	LIST_INSERT_HEAD(&s->dlcs, pcb1, session_next);
 	pcb1->session = s;
 			
 	mtx_unlock(&pcb1->pcb_mtx);
 
 	return (pcb1);
 } /* ng_btsocket_rfcomm_connect_ind */
 
 /*
  * Process RFCOMM connect confirmation. Caller must hold s->session_mtx.
  */
 
 static void
 ng_btsocket_rfcomm_connect_cfm(ng_btsocket_rfcomm_session_p s)
 {
 	ng_btsocket_rfcomm_pcb_p	pcb = NULL, pcb_next = NULL;
 	int				error;
 
 	mtx_assert(&s->session_mtx, MA_OWNED);
 
 	/*
 	 * Wake up all waiting sockets and send PN request for each of them. 
 	 * Note that timeout already been set in ng_btsocket_rfcomm_connect()
 	 *
 	 * Note: cannot use LIST_FOREACH because ng_btsocket_rfcomm_pcb_kill
 	 * will unlink DLC from the session
 	 */
 
 	for (pcb = LIST_FIRST(&s->dlcs); pcb != NULL; ) {
 		mtx_lock(&pcb->pcb_mtx);
 		pcb_next = LIST_NEXT(pcb, session_next);
 
 		if (pcb->state == NG_BTSOCKET_RFCOMM_DLC_W4_CONNECT) {
 			pcb->mtu = s->mtu;
 			bcopy(&so2l2cap_pcb(s->l2so)->src, &pcb->src,
 				sizeof(pcb->src));
 
 			error = ng_btsocket_rfcomm_send_pn(pcb);
 			if (error == 0)
 				pcb->state = NG_BTSOCKET_RFCOMM_DLC_CONFIGURING;
 			else
 				ng_btsocket_rfcomm_pcb_kill(pcb, error);
 		}
 
 		mtx_unlock(&pcb->pcb_mtx);
 		pcb = pcb_next;
 	}
 } /* ng_btsocket_rfcomm_connect_cfm */
 
 /*****************************************************************************
  *****************************************************************************
  **                              RFCOMM sessions
  *****************************************************************************
  *****************************************************************************/
 
 /*
  * Create new RFCOMM session. That function WILL NOT take ownership over l2so.
  * Caller MUST free l2so if function failed.
  */
 
 static int
 ng_btsocket_rfcomm_session_create(ng_btsocket_rfcomm_session_p *sp,
 		struct socket *l2so, bdaddr_p src, bdaddr_p dst,
 		struct thread *td)
 {
 	ng_btsocket_rfcomm_session_p	s = NULL;
 	struct sockaddr_l2cap		l2sa;
 	struct sockopt			l2sopt;
 	int				error;
 	u_int16_t			mtu;
 
 	mtx_assert(&ng_btsocket_rfcomm_sessions_mtx, MA_OWNED);
 
 	/* Allocate the RFCOMM session */
         s = malloc(sizeof(*s),
 		M_NETGRAPH_BTSOCKET_RFCOMM, M_NOWAIT | M_ZERO);
         if (s == NULL)
                 return (ENOMEM);
 
 	/* Set defaults */
 	s->mtu = RFCOMM_DEFAULT_MTU;
 	s->flags = 0;
 	s->state = NG_BTSOCKET_RFCOMM_SESSION_CLOSED;
 	NG_BT_MBUFQ_INIT(&s->outq, ifqmaxlen);
 
 	/*
 	 * XXX Mark session mutex as DUPOK to prevent "duplicated lock of 
 	 * the same type" message. When accepting new L2CAP connection
 	 * ng_btsocket_rfcomm_session_accept() holds both session mutexes 
 	 * for "old" (accepting) session and "new" (created) session.
 	 */
 
 	mtx_init(&s->session_mtx, "btsocks_rfcomm_session_mtx", NULL,
 		MTX_DEF|MTX_DUPOK);
 
 	LIST_INIT(&s->dlcs);
 
 	/* Prepare L2CAP socket */
 	SOCKBUF_LOCK(&l2so->so_rcv);
 	soupcall_set(l2so, SO_RCV, ng_btsocket_rfcomm_upcall, NULL);
 	SOCKBUF_UNLOCK(&l2so->so_rcv);
 	SOCKBUF_LOCK(&l2so->so_snd);
 	soupcall_set(l2so, SO_SND, ng_btsocket_rfcomm_upcall, NULL);
 	SOCKBUF_UNLOCK(&l2so->so_snd);
 	l2so->so_state |= SS_NBIO;
 	s->l2so = l2so;
 
 	mtx_lock(&s->session_mtx);
 
 	/*
 	 * "src" == NULL and "dst" == NULL means just create session.
 	 * caller must do the rest
 	 */
 
 	if (src == NULL && dst == NULL)
 		goto done;
 
 	/*
 	 * Set incoming MTU on L2CAP socket. It is RFCOMM session default MTU 
 	 * plus 5 bytes: RFCOMM frame header, one extra byte for length and one
 	 * extra byte for credits.
 	 */
 
 	mtu = s->mtu + sizeof(struct rfcomm_frame_hdr) + 1 + 1;
 
 	l2sopt.sopt_dir = SOPT_SET;
 	l2sopt.sopt_level = SOL_L2CAP;
 	l2sopt.sopt_name = SO_L2CAP_IMTU;
 	l2sopt.sopt_val = (void *) &mtu;
 	l2sopt.sopt_valsize = sizeof(mtu);
 	l2sopt.sopt_td = NULL;
 
 	error = sosetopt(s->l2so, &l2sopt);
 	if (error != 0)
 		goto bad;
 
 	/* Bind socket to "src" address */
 	l2sa.l2cap_len = sizeof(l2sa);
 	l2sa.l2cap_family = AF_BLUETOOTH;
 	l2sa.l2cap_psm = (dst == NULL)? htole16(NG_L2CAP_PSM_RFCOMM) : 0;
 	bcopy(src, &l2sa.l2cap_bdaddr, sizeof(l2sa.l2cap_bdaddr));
 
 	error = sobind(s->l2so, (struct sockaddr *) &l2sa, td);
 	if (error != 0)
 		goto bad;
 
 	/* If "dst" is not NULL then initiate connect(), otherwise listen() */
 	if (dst == NULL) {
 		s->flags = 0;
 		s->state = NG_BTSOCKET_RFCOMM_SESSION_LISTENING;
 
 		error = solisten(s->l2so, 10, td);
 		if (error != 0)
 			goto bad;
 	} else {
 		s->flags = NG_BTSOCKET_RFCOMM_SESSION_INITIATOR;
 		s->state = NG_BTSOCKET_RFCOMM_SESSION_CONNECTING;
 
 		l2sa.l2cap_len = sizeof(l2sa);   
 		l2sa.l2cap_family = AF_BLUETOOTH;
 		l2sa.l2cap_psm = htole16(NG_L2CAP_PSM_RFCOMM);
 	        bcopy(dst, &l2sa.l2cap_bdaddr, sizeof(l2sa.l2cap_bdaddr));
 
 		error = soconnect(s->l2so, (struct sockaddr *) &l2sa, td);
 		if (error != 0)
 			goto bad;
 	}
 
 done:
 	LIST_INSERT_HEAD(&ng_btsocket_rfcomm_sessions, s, next);
 	*sp = s;
 
 	mtx_unlock(&s->session_mtx);
 
 	return (0);
 
 bad:
 	mtx_unlock(&s->session_mtx);
 
 	/* Return L2CAP socket back to its original state */
 	SOCKBUF_LOCK(&l2so->so_rcv);
 	soupcall_clear(s->l2so, SO_RCV);
 	SOCKBUF_UNLOCK(&l2so->so_rcv);
 	SOCKBUF_LOCK(&l2so->so_snd);
 	soupcall_clear(s->l2so, SO_SND);
 	SOCKBUF_UNLOCK(&l2so->so_snd);
 	l2so->so_state &= ~SS_NBIO;
 
 	mtx_destroy(&s->session_mtx);
 	bzero(s, sizeof(*s));
 	free(s, M_NETGRAPH_BTSOCKET_RFCOMM);
 
 	return (error);
 } /* ng_btsocket_rfcomm_session_create */
 
 /*
  * Process accept() on RFCOMM session
  * XXX FIXME locking for "l2so"?
  */
 
 static int
 ng_btsocket_rfcomm_session_accept(ng_btsocket_rfcomm_session_p s0)
 {
 	struct socket			*l2so = NULL;
 	struct sockaddr_l2cap		*l2sa = NULL;
 	ng_btsocket_l2cap_pcb_t		*l2pcb = NULL;
 	ng_btsocket_rfcomm_session_p	 s = NULL;
 	int				 error = 0;
 
 	mtx_assert(&ng_btsocket_rfcomm_sessions_mtx, MA_OWNED);
 	mtx_assert(&s0->session_mtx, MA_OWNED);
 
 	/* Check if there is a complete L2CAP connection in the queue */
 	if ((error = s0->l2so->so_error) != 0) {
 		NG_BTSOCKET_RFCOMM_ERR(
 "%s: Could not accept connection on L2CAP socket, error=%d\n", __func__, error);
 		s0->l2so->so_error = 0;
 
 		return (error);
 	}
 
 	ACCEPT_LOCK();
 	if (TAILQ_EMPTY(&s0->l2so->so_comp)) {
 		ACCEPT_UNLOCK();
 		if (s0->l2so->so_rcv.sb_state & SBS_CANTRCVMORE)
 			return (ECONNABORTED);
 		return (EWOULDBLOCK);
 	}
 
 	/* Accept incoming L2CAP connection */
 	l2so = TAILQ_FIRST(&s0->l2so->so_comp);
 	if (l2so == NULL)
 		panic("%s: l2so == NULL\n", __func__);
 
 	TAILQ_REMOVE(&s0->l2so->so_comp, l2so, so_list);
 	s0->l2so->so_qlen --;
 	l2so->so_qstate &= ~SQ_COMP;
 	l2so->so_head = NULL;
 	SOCK_LOCK(l2so);
 	soref(l2so);
 	l2so->so_state |= SS_NBIO;
 	SOCK_UNLOCK(l2so);
 	ACCEPT_UNLOCK();
 
 	error = soaccept(l2so, (struct sockaddr **) &l2sa);
 	if (error != 0) {
 		NG_BTSOCKET_RFCOMM_ERR(
 "%s: soaccept() on L2CAP socket failed, error=%d\n", __func__, error);
 		soclose(l2so);
 
 		return (error);
 	}
 
 	/*
 	 * Check if there is already active RFCOMM session between two devices.
 	 * If so then close L2CAP connection. We only support one RFCOMM session
 	 * between each pair of devices. Note that here we assume session in any
 	 * state. The session even could be in the middle of disconnecting.
 	 */
 
 	l2pcb = so2l2cap_pcb(l2so);
 	s = ng_btsocket_rfcomm_session_by_addr(&l2pcb->src, &l2pcb->dst);
 	if (s == NULL) {
 		/* Create a new RFCOMM session */
 		error = ng_btsocket_rfcomm_session_create(&s, l2so, NULL, NULL,
 				curthread /* XXX */);
 		if (error == 0) {
 			mtx_lock(&s->session_mtx);
 
 			s->flags = 0;
 			s->state = NG_BTSOCKET_RFCOMM_SESSION_CONNECTED;
 
 			/*
 			 * Adjust MTU on incomming connection. Reserve 5 bytes:
 			 * RFCOMM frame header, one extra byte for length and 
 			 * one extra byte for credits.
 			 */
 
 			s->mtu = min(l2pcb->imtu, l2pcb->omtu) -
 					sizeof(struct rfcomm_frame_hdr) - 1 - 1;
 
 			mtx_unlock(&s->session_mtx);
 		} else {
 			NG_BTSOCKET_RFCOMM_ALERT(
 "%s: Failed to create new RFCOMM session, error=%d\n", __func__, error);
 
 			soclose(l2so);
 		}
 	} else {
 		NG_BTSOCKET_RFCOMM_WARN(
 "%s: Rejecting duplicating RFCOMM session between src=%x:%x:%x:%x:%x:%x and " \
 "dst=%x:%x:%x:%x:%x:%x, state=%d, flags=%#x\n",	__func__,
 			l2pcb->src.b[5], l2pcb->src.b[4], l2pcb->src.b[3],
 			l2pcb->src.b[2], l2pcb->src.b[1], l2pcb->src.b[0],
 			l2pcb->dst.b[5], l2pcb->dst.b[4], l2pcb->dst.b[3],
 			l2pcb->dst.b[2], l2pcb->dst.b[1], l2pcb->dst.b[0],
 			s->state, s->flags);
 
 		error = EBUSY;
 		soclose(l2so);
 	}
 
 	return (error);
 } /* ng_btsocket_rfcomm_session_accept */
 
 /*
  * Process connect() on RFCOMM session
  * XXX FIXME locking for "l2so"?
  */
 
 static int
 ng_btsocket_rfcomm_session_connect(ng_btsocket_rfcomm_session_p s)
 {
 	ng_btsocket_l2cap_pcb_p	l2pcb = so2l2cap_pcb(s->l2so);
 	int			error;
 
 	mtx_assert(&s->session_mtx, MA_OWNED);
 
 	/* First check if connection has failed */
 	if ((error = s->l2so->so_error) != 0) {
 		s->l2so->so_error = 0;
 
 		NG_BTSOCKET_RFCOMM_ERR(
 "%s: Could not connect RFCOMM session, error=%d, state=%d, flags=%#x\n",
 			__func__, error, s->state, s->flags);
 
 		return (error);
 	}
 
 	/* Is connection still in progress? */
 	if (s->l2so->so_state & SS_ISCONNECTING)
 		return (0); 
 
 	/* 
 	 * If we got here then we are connected. Send SABM on DLCI 0 to 
 	 * open multiplexor channel.
 	 */
 
 	if (error == 0) {
 		s->state = NG_BTSOCKET_RFCOMM_SESSION_CONNECTED;
 
 		/*
 		 * Adjust MTU on outgoing connection. Reserve 5 bytes: RFCOMM 
 		 * frame header, one extra byte for length and one extra byte 
 		 * for credits.
 		 */
 
 		s->mtu = min(l2pcb->imtu, l2pcb->omtu) -
 				sizeof(struct rfcomm_frame_hdr) - 1 - 1;
 
 		error = ng_btsocket_rfcomm_send_command(s,RFCOMM_FRAME_SABM,0);
 		if (error == 0)
 			error = ng_btsocket_rfcomm_task_wakeup();
 	}
 
 	return (error);
 }/* ng_btsocket_rfcomm_session_connect */
 
 /*
  * Receive data on RFCOMM session
  * XXX FIXME locking for "l2so"?
  */
 
 static int
 ng_btsocket_rfcomm_session_receive(ng_btsocket_rfcomm_session_p s)
 {
 	struct mbuf	*m = NULL;
 	struct uio	 uio;
 	int		 more, flags, error;
 
 	mtx_assert(&s->session_mtx, MA_OWNED);
 
 	/* Can we read from the L2CAP socket? */
 	if (!soreadable(s->l2so))
 		return (0);
 
 	/* First check for error on L2CAP socket */
 	if ((error = s->l2so->so_error) != 0) {
 		s->l2so->so_error = 0;
 
 		NG_BTSOCKET_RFCOMM_ERR(
 "%s: Could not receive data from L2CAP socket, error=%d, state=%d, flags=%#x\n",
 			__func__, error, s->state, s->flags);
 
 		return (error);
 	}
 
 	/*
 	 * Read all packets from the L2CAP socket. 
 	 * XXX FIXME/VERIFY is that correct? For now use m->m_nextpkt as
 	 * indication that there is more packets on the socket's buffer.
 	 * Also what should we use in uio.uio_resid?
 	 * May be s->mtu + sizeof(struct rfcomm_frame_hdr) + 1 + 1?
 	 */
 
 	for (more = 1; more; ) {
 		/* Try to get next packet from socket */
 		bzero(&uio, sizeof(uio));
 /*		uio.uio_td = NULL; */
 		uio.uio_resid = 1000000000;
 		flags = MSG_DONTWAIT;
 
 		m = NULL;
 		error = soreceive(s->l2so, NULL, &uio, &m,
 		    (struct mbuf **) NULL, &flags);
 		if (error != 0) {
 			if (error == EWOULDBLOCK)
 				return (0); /* XXX can happen? */
 
 			NG_BTSOCKET_RFCOMM_ERR(
 "%s: Could not receive data from L2CAP socket, error=%d\n", __func__, error);
 
 			return (error);
 		}
 	
 		more = (m->m_nextpkt != NULL);
 		m->m_nextpkt = NULL;
 
 		ng_btsocket_rfcomm_receive_frame(s, m);
 	}
 
 	return (0);
 } /* ng_btsocket_rfcomm_session_receive */
 
 /*
  * Send data on RFCOMM session
  * XXX FIXME locking for "l2so"?
  */
 
 static int
 ng_btsocket_rfcomm_session_send(ng_btsocket_rfcomm_session_p s)
 {
 	struct mbuf	*m = NULL;
 	int		 error;
 
 	mtx_assert(&s->session_mtx, MA_OWNED);
 
 	/* Send as much as we can from the session queue */
 	while (sowriteable(s->l2so)) {
 		/* Check if socket still OK */
 		if ((error = s->l2so->so_error) != 0) {
 			s->l2so->so_error = 0;
 
 			NG_BTSOCKET_RFCOMM_ERR(
 "%s: Detected error=%d on L2CAP socket, state=%d, flags=%#x\n",
 				__func__, error, s->state, s->flags);
 
 			return (error);
 		}
 
 		NG_BT_MBUFQ_DEQUEUE(&s->outq, m);
 		if (m == NULL)
 			return (0); /* we are done */
 
 		/* Call send function on the L2CAP socket */
 		error = (*s->l2so->so_proto->pr_usrreqs->pru_send)(s->l2so,
 				0, m, NULL, NULL, curthread /* XXX */);
 		if (error != 0) {
 			NG_BTSOCKET_RFCOMM_ERR(
 "%s: Could not send data to L2CAP socket, error=%d\n", __func__, error);
 
 			return (error);
 		}
 	}
 
 	return (0);
 } /* ng_btsocket_rfcomm_session_send */
 
 /*
  * Close and disconnect all DLCs for the given session. Caller must hold 
  * s->sesson_mtx. Will wakeup session.
  */
 
 static void
 ng_btsocket_rfcomm_session_clean(ng_btsocket_rfcomm_session_p s)
 {
 	ng_btsocket_rfcomm_pcb_p	pcb = NULL, pcb_next = NULL;
 	int				error;
 
 	mtx_assert(&s->session_mtx, MA_OWNED);
 
 	/*
 	 * Note: cannot use LIST_FOREACH because ng_btsocket_rfcomm_pcb_kill
 	 * will unlink DLC from the session
 	 */
 
 	for (pcb = LIST_FIRST(&s->dlcs); pcb != NULL; ) {
 		mtx_lock(&pcb->pcb_mtx);
 		pcb_next = LIST_NEXT(pcb, session_next);
 
 		NG_BTSOCKET_RFCOMM_INFO(
 "%s: Disconnecting dlci=%d, state=%d, flags=%#x\n",
 			__func__, pcb->dlci, pcb->state, pcb->flags);
 
 		if (pcb->state == NG_BTSOCKET_RFCOMM_DLC_CONNECTED)
 			error = ECONNRESET;
 		else
 			error = ECONNREFUSED;
 
 		ng_btsocket_rfcomm_pcb_kill(pcb, error);
 
 		mtx_unlock(&pcb->pcb_mtx);
 		pcb = pcb_next;
 	}
 } /* ng_btsocket_rfcomm_session_clean */
 
 /*
  * Process all DLCs on the session. Caller MUST hold s->session_mtx.
  */
 
 static void
 ng_btsocket_rfcomm_session_process_pcb(ng_btsocket_rfcomm_session_p s)
 {
 	ng_btsocket_rfcomm_pcb_p	pcb = NULL, pcb_next = NULL;
 	int				error;
 
 	mtx_assert(&s->session_mtx, MA_OWNED);
 
 	/*
 	 * Note: cannot use LIST_FOREACH because ng_btsocket_rfcomm_pcb_kill
 	 * will unlink DLC from the session
 	 */
 
 	for (pcb = LIST_FIRST(&s->dlcs); pcb != NULL; ) {
 		mtx_lock(&pcb->pcb_mtx);
 		pcb_next = LIST_NEXT(pcb, session_next);
 
 		switch (pcb->state) {
 
 		/*
 		 * If DLC in W4_CONNECT state then we should check for both
 		 * timeout and detach.
 		 */
 
 		case NG_BTSOCKET_RFCOMM_DLC_W4_CONNECT:
 			if (pcb->flags & NG_BTSOCKET_RFCOMM_DLC_DETACHED)
 				ng_btsocket_rfcomm_pcb_kill(pcb, 0);
 			else if (pcb->flags & NG_BTSOCKET_RFCOMM_DLC_TIMEDOUT)
 				ng_btsocket_rfcomm_pcb_kill(pcb, ETIMEDOUT);
 			break;
 
 		/*
 		 * If DLC in CONFIGURING or CONNECTING state then we only
 		 * should check for timeout. If detach() was called then
 		 * DLC will be moved into DISCONNECTING state.
 		 */
 
 		case NG_BTSOCKET_RFCOMM_DLC_CONFIGURING:
 		case NG_BTSOCKET_RFCOMM_DLC_CONNECTING:
 			if (pcb->flags & NG_BTSOCKET_RFCOMM_DLC_TIMEDOUT)
 				ng_btsocket_rfcomm_pcb_kill(pcb, ETIMEDOUT);
 			break;
 
 		/*
 		 * If DLC in CONNECTED state then we need to send data (if any)
 		 * from the socket's send queue. Note that we will send data
 		 * from either all sockets or none. This may overload session's
 		 * outgoing queue (but we do not check for that).
 		 *
  		 * XXX FIXME need scheduler for RFCOMM sockets
 		 */
 
 		case NG_BTSOCKET_RFCOMM_DLC_CONNECTED:
 			error = ng_btsocket_rfcomm_pcb_send(pcb, ALOT);
 			if (error != 0)
 				ng_btsocket_rfcomm_pcb_kill(pcb, error);
 			break;
 
 		/*
 		 * If DLC in DISCONNECTING state then we must send DISC frame.
 		 * Note that if DLC has timeout set then we do not need to 
 		 * resend DISC frame.
 		 *
 		 * XXX FIXME need to drain all data from the socket's queue
 		 * if LINGER option was set
 		 */
 
 		case NG_BTSOCKET_RFCOMM_DLC_DISCONNECTING:
 			if (!(pcb->flags & NG_BTSOCKET_RFCOMM_DLC_TIMO)) {
 				error = ng_btsocket_rfcomm_send_command(
 						pcb->session, RFCOMM_FRAME_DISC,
 						pcb->dlci);
 				if (error == 0)
 					ng_btsocket_rfcomm_timeout(pcb);
 				else
 					ng_btsocket_rfcomm_pcb_kill(pcb, error);
 			} else if (pcb->flags & NG_BTSOCKET_RFCOMM_DLC_TIMEDOUT)
 				ng_btsocket_rfcomm_pcb_kill(pcb, ETIMEDOUT);
 			break;
 		
 /*		case NG_BTSOCKET_RFCOMM_DLC_CLOSED: */
 		default:
 			panic("%s: Invalid DLC state=%d, flags=%#x\n",
 				__func__, pcb->state, pcb->flags);
 			break;
 		}
 
 		mtx_unlock(&pcb->pcb_mtx);
 		pcb = pcb_next;
 	}
 } /* ng_btsocket_rfcomm_session_process_pcb */
 
 /*
  * Find RFCOMM session between "src" and "dst".
  * Caller MUST hold ng_btsocket_rfcomm_sessions_mtx.
  */
 
 static ng_btsocket_rfcomm_session_p
 ng_btsocket_rfcomm_session_by_addr(bdaddr_p src, bdaddr_p dst)
 {
 	ng_btsocket_rfcomm_session_p	s = NULL;
 	ng_btsocket_l2cap_pcb_p		l2pcb = NULL;
 	int				any_src;
 
 	mtx_assert(&ng_btsocket_rfcomm_sessions_mtx, MA_OWNED);
 
 	any_src = (bcmp(src, NG_HCI_BDADDR_ANY, sizeof(*src)) == 0);
 
 	LIST_FOREACH(s, &ng_btsocket_rfcomm_sessions, next) {
 		l2pcb = so2l2cap_pcb(s->l2so);
 
 		if ((any_src || bcmp(&l2pcb->src, src, sizeof(*src)) == 0) &&
 		    bcmp(&l2pcb->dst, dst, sizeof(*dst)) == 0)
 			break;
 	}
 
 	return (s);
 } /* ng_btsocket_rfcomm_session_by_addr */
 
 /*****************************************************************************
  *****************************************************************************
  **                                  RFCOMM 
  *****************************************************************************
  *****************************************************************************/
 
 /*
  * Process incoming RFCOMM frame. Caller must hold s->session_mtx.
  * XXX FIXME check frame length
  */
 
 static int
 ng_btsocket_rfcomm_receive_frame(ng_btsocket_rfcomm_session_p s,
 		struct mbuf *m0)
 {
 	struct rfcomm_frame_hdr	*hdr = NULL;
 	struct mbuf		*m = NULL;
 	u_int16_t		 length;
 	u_int8_t		 dlci, type;
 	int			 error = 0;
 
 	mtx_assert(&s->session_mtx, MA_OWNED);
 
 	/* Pullup as much as we can into first mbuf (for direct access) */
 	length = min(m0->m_pkthdr.len, MHLEN);
 	if (m0->m_len < length) {
 		if ((m0 = m_pullup(m0, length)) == NULL) {
 			NG_BTSOCKET_RFCOMM_ALERT(
 "%s: m_pullup(%d) failed\n", __func__, length);
 
 			return (ENOBUFS);
 		}
 	}
 
 	hdr = mtod(m0, struct rfcomm_frame_hdr *);
 	dlci = RFCOMM_DLCI(hdr->address);
 	type = RFCOMM_TYPE(hdr->control);
 
 	/* Test EA bit in length. If not set then we have 2 bytes of length */
 	if (!RFCOMM_EA(hdr->length)) {
 		bcopy(&hdr->length, &length, sizeof(length));
 		length = le16toh(length) >> 1;
 		m_adj(m0, sizeof(*hdr) + 1);
 	} else {
 		length = hdr->length >> 1;
 		m_adj(m0, sizeof(*hdr));
 	}
 
 	NG_BTSOCKET_RFCOMM_INFO(
 "%s: Got frame type=%#x, dlci=%d, length=%d, cr=%d, pf=%d, len=%d\n",
 		__func__, type, dlci, length, RFCOMM_CR(hdr->address),
 		RFCOMM_PF(hdr->control), m0->m_pkthdr.len);
 
 	/*
 	 * Get FCS (the last byte in the frame)
 	 * XXX this will not work if mbuf chain ends with empty mbuf.
 	 * XXX let's hope it never happens :)
 	 */
 
 	for (m = m0; m->m_next != NULL; m = m->m_next)
 		;
 	if (m->m_len <= 0)
 		panic("%s: Empty mbuf at the end of the chain, len=%d\n",
 			__func__, m->m_len);
 
 	/*
 	 * Check FCS. We only need to calculate FCS on first 2 or 3 bytes
 	 * and already m_pullup'ed mbuf chain, so it should be safe.
 	 */
 
 	if (ng_btsocket_rfcomm_check_fcs((u_int8_t *) hdr, type, m->m_data[m->m_len - 1])) {
 		NG_BTSOCKET_RFCOMM_ERR(
 "%s: Invalid RFCOMM packet. Bad checksum\n", __func__);
 		NG_FREE_M(m0);
 
 		return (EINVAL);
 	}
 
 	m_adj(m0, -1); /* Trim FCS byte */
 
 	/*
 	 * Process RFCOMM frame.
 	 *
 	 * From TS 07.10 spec
 	 * 
 	 * "... In the case where a SABM or DISC command with the P bit set
 	 * to 0 is received then the received frame shall be discarded..."
  	 *
 	 * "... If a unsolicited DM response is received then the frame shall
 	 * be processed irrespective of the P/F setting... "
 	 *
 	 * "... The station may transmit response frames with the F bit set 
 	 * to 0 at any opportunity on an asynchronous basis. However, in the 
 	 * case where a UA response is received with the F bit set to 0 then 
 	 * the received frame shall be discarded..."
 	 *
 	 * From Bluetooth spec
 	 *
 	 * "... When credit based flow control is being used, the meaning of
 	 * the P/F bit in the control field of the RFCOMM header is redefined
 	 * for UIH frames..."
 	 */
 
 	switch (type) {
 	case RFCOMM_FRAME_SABM:
 		if (RFCOMM_PF(hdr->control))
 			error = ng_btsocket_rfcomm_receive_sabm(s, dlci);
 		break;
 
 	case RFCOMM_FRAME_DISC:
 		if (RFCOMM_PF(hdr->control))
 			error = ng_btsocket_rfcomm_receive_disc(s, dlci);
 		break;
 
 	case RFCOMM_FRAME_UA:
 		if (RFCOMM_PF(hdr->control))
 			error = ng_btsocket_rfcomm_receive_ua(s, dlci);
 		break;
 
 	case RFCOMM_FRAME_DM:
 		error = ng_btsocket_rfcomm_receive_dm(s, dlci);
 		break;
 
 	case RFCOMM_FRAME_UIH:
 		if (dlci == 0)
 			error = ng_btsocket_rfcomm_receive_mcc(s, m0);
 		else
 			error = ng_btsocket_rfcomm_receive_uih(s, dlci,
 					RFCOMM_PF(hdr->control), m0);
 
 		return (error);
 		/* NOT REACHED */
 
 	default:
 		NG_BTSOCKET_RFCOMM_ERR(
 "%s: Invalid RFCOMM packet. Unknown type=%#x\n", __func__, type);
 		error = EINVAL;
 		break;
 	}
 
 	NG_FREE_M(m0);
 
 	return (error);
 } /* ng_btsocket_rfcomm_receive_frame */
 
 /*
  * Process RFCOMM SABM frame
  */
 
 static int
 ng_btsocket_rfcomm_receive_sabm(ng_btsocket_rfcomm_session_p s, int dlci)
 {
 	ng_btsocket_rfcomm_pcb_p	pcb = NULL;
 	int				error = 0;
 
 	mtx_assert(&s->session_mtx, MA_OWNED);
 
 	NG_BTSOCKET_RFCOMM_INFO(
 "%s: Got SABM, session state=%d, flags=%#x, mtu=%d, dlci=%d\n",
 		__func__, s->state, s->flags, s->mtu, dlci);
 
 	/* DLCI == 0 means open multiplexor channel */
 	if (dlci == 0) {
 		switch (s->state) {
 		case NG_BTSOCKET_RFCOMM_SESSION_CONNECTED:
 		case NG_BTSOCKET_RFCOMM_SESSION_OPEN:
 			error = ng_btsocket_rfcomm_send_command(s,
 					RFCOMM_FRAME_UA, dlci);
 			if (error == 0) {
 				s->state = NG_BTSOCKET_RFCOMM_SESSION_OPEN;
 				ng_btsocket_rfcomm_connect_cfm(s);
 			} else {
 				s->state = NG_BTSOCKET_RFCOMM_SESSION_CLOSED;
 				ng_btsocket_rfcomm_session_clean(s);
 			}
 			break;
 
 		default:
 			NG_BTSOCKET_RFCOMM_WARN(
 "%s: Got SABM for session in invalid state state=%d, flags=%#x\n",
 				__func__, s->state, s->flags);
 			error = EINVAL;
 			break;
 		}
 
 		return (error);
 	}
 
 	/* Make sure multiplexor channel is open */
 	if (s->state != NG_BTSOCKET_RFCOMM_SESSION_OPEN) {
 		NG_BTSOCKET_RFCOMM_ERR(
 "%s: Got SABM for dlci=%d with mulitplexor channel closed, state=%d, " \
 "flags=%#x\n",		__func__, dlci, s->state, s->flags);
 
 		return (EINVAL);
 	}
 
 	/*
 	 * Check if we have this DLCI. This might happen when remote
 	 * peer uses PN command before actual open (SABM) happens.
 	 */
 
 	pcb = ng_btsocket_rfcomm_pcb_by_dlci(s, dlci);
 	if (pcb != NULL) {
 		mtx_lock(&pcb->pcb_mtx);
 
 		if (pcb->state != NG_BTSOCKET_RFCOMM_DLC_CONNECTING) {
 			NG_BTSOCKET_RFCOMM_ERR(
 "%s: Got SABM for dlci=%d in invalid state=%d, flags=%#x\n",
 				__func__, dlci, pcb->state, pcb->flags);
 			mtx_unlock(&pcb->pcb_mtx);
 
 			return (ENOENT);
 		}
 
 		ng_btsocket_rfcomm_untimeout(pcb);
 
 		error = ng_btsocket_rfcomm_send_command(s,RFCOMM_FRAME_UA,dlci);
 		if (error == 0)
 			error = ng_btsocket_rfcomm_send_msc(pcb);
 
 		if (error == 0) {
 			pcb->state = NG_BTSOCKET_RFCOMM_DLC_CONNECTED;
 			soisconnected(pcb->so);
 		} else
 			ng_btsocket_rfcomm_pcb_kill(pcb, error);
 
 		mtx_unlock(&pcb->pcb_mtx);
 
 		return (error);
 	}
 
 	/*
 	 * We do not have requested DLCI, so it must be an incoming connection
 	 * with default parameters. Try to accept it.
 	 */ 
 
 	pcb = ng_btsocket_rfcomm_connect_ind(s, RFCOMM_SRVCHANNEL(dlci));
 	if (pcb != NULL) {
 		mtx_lock(&pcb->pcb_mtx);
 
 		pcb->dlci = dlci;
 
 		error = ng_btsocket_rfcomm_send_command(s,RFCOMM_FRAME_UA,dlci);
 		if (error == 0)
 			error = ng_btsocket_rfcomm_send_msc(pcb);
 
 		if (error == 0) {
 			pcb->state = NG_BTSOCKET_RFCOMM_DLC_CONNECTED;
 			soisconnected(pcb->so);
 		} else
 			ng_btsocket_rfcomm_pcb_kill(pcb, error);
 
 		mtx_unlock(&pcb->pcb_mtx);
 	} else
 		/* Nobody is listen()ing on the requested DLCI */
 		error = ng_btsocket_rfcomm_send_command(s,RFCOMM_FRAME_DM,dlci);
 
 	return (error);
 } /* ng_btsocket_rfcomm_receive_sabm */
 
 /*
  * Process RFCOMM DISC frame
  */
 
 static int
 ng_btsocket_rfcomm_receive_disc(ng_btsocket_rfcomm_session_p s, int dlci)
 {
 	ng_btsocket_rfcomm_pcb_p	pcb = NULL;
 	int				error = 0;
 
 	mtx_assert(&s->session_mtx, MA_OWNED);
 
 	NG_BTSOCKET_RFCOMM_INFO(
 "%s: Got DISC, session state=%d, flags=%#x, mtu=%d, dlci=%d\n",
 		__func__, s->state, s->flags, s->mtu, dlci);
 
 	/* DLCI == 0 means close multiplexor channel */
 	if (dlci == 0) {
 		/* XXX FIXME assume that remote side will close the socket */
 		error = ng_btsocket_rfcomm_send_command(s, RFCOMM_FRAME_UA, 0);
 		if (error == 0) {
 			if (s->state == NG_BTSOCKET_RFCOMM_SESSION_DISCONNECTING)
 				s->state = NG_BTSOCKET_RFCOMM_SESSION_CLOSED; /* XXX */
 			else
 				s->state = NG_BTSOCKET_RFCOMM_SESSION_DISCONNECTING;
 		} else
 			s->state = NG_BTSOCKET_RFCOMM_SESSION_CLOSED; /* XXX */
 
 		ng_btsocket_rfcomm_session_clean(s);
 	} else {
 		pcb = ng_btsocket_rfcomm_pcb_by_dlci(s, dlci);
 		if (pcb != NULL) {
 			int	err;
 
 			mtx_lock(&pcb->pcb_mtx);
 
 			NG_BTSOCKET_RFCOMM_INFO(
 "%s: Got DISC for dlci=%d, state=%d, flags=%#x\n",
 				__func__, dlci, pcb->state, pcb->flags);
 
 			error = ng_btsocket_rfcomm_send_command(s,
 					RFCOMM_FRAME_UA, dlci);
 
 			if (pcb->state == NG_BTSOCKET_RFCOMM_DLC_CONNECTED)
 				err = 0;
 			else
 				err = ECONNREFUSED;
 
 			ng_btsocket_rfcomm_pcb_kill(pcb, err);
 
 			mtx_unlock(&pcb->pcb_mtx);
 		} else {
 			NG_BTSOCKET_RFCOMM_WARN(
 "%s: Got DISC for non-existing dlci=%d\n", __func__, dlci);
 
 			error = ng_btsocket_rfcomm_send_command(s,
 					RFCOMM_FRAME_DM, dlci);
 		}
 	}
 
 	return (error);
 } /* ng_btsocket_rfcomm_receive_disc */
 
 /*
  * Process RFCOMM UA frame
  */
 
 static int
 ng_btsocket_rfcomm_receive_ua(ng_btsocket_rfcomm_session_p s, int dlci)
 {
 	ng_btsocket_rfcomm_pcb_p	pcb = NULL;
 	int				error = 0;
 
 	mtx_assert(&s->session_mtx, MA_OWNED);
 
 	NG_BTSOCKET_RFCOMM_INFO(
 "%s: Got UA, session state=%d, flags=%#x, mtu=%d, dlci=%d\n",
 		__func__, s->state, s->flags, s->mtu, dlci);
 
 	/* dlci == 0 means multiplexor channel */
 	if (dlci == 0) {
 		switch (s->state) {
 		case NG_BTSOCKET_RFCOMM_SESSION_CONNECTED:
 			s->state = NG_BTSOCKET_RFCOMM_SESSION_OPEN;
 			ng_btsocket_rfcomm_connect_cfm(s);
 			break;
 
 		case NG_BTSOCKET_RFCOMM_SESSION_DISCONNECTING:
 			s->state = NG_BTSOCKET_RFCOMM_SESSION_CLOSED;
 			ng_btsocket_rfcomm_session_clean(s);
 			break;
 
 		default:
 			NG_BTSOCKET_RFCOMM_WARN(
 "%s: Got UA for session in invalid state=%d(%d), flags=%#x, mtu=%d\n",
 				__func__, s->state, INITIATOR(s), s->flags,
 				s->mtu);
 			error = ENOENT;
 			break;
 		}
 
 		return (error);
 	}
 
 	/* Check if we have this DLCI */
 	pcb = ng_btsocket_rfcomm_pcb_by_dlci(s, dlci);
 	if (pcb != NULL) {
 		mtx_lock(&pcb->pcb_mtx);
 
 		NG_BTSOCKET_RFCOMM_INFO(
 "%s: Got UA for dlci=%d, state=%d, flags=%#x\n",
 			__func__, dlci, pcb->state, pcb->flags);
 
 		switch (pcb->state) {
 		case NG_BTSOCKET_RFCOMM_DLC_CONNECTING:
 			ng_btsocket_rfcomm_untimeout(pcb);
 
 			error = ng_btsocket_rfcomm_send_msc(pcb);
 			if (error == 0) {
 				pcb->state = NG_BTSOCKET_RFCOMM_DLC_CONNECTED;
 				soisconnected(pcb->so);
 			}
 			break;
 
 		case NG_BTSOCKET_RFCOMM_DLC_DISCONNECTING:
 			ng_btsocket_rfcomm_pcb_kill(pcb, 0);
 			break;
 
 		default:
 			NG_BTSOCKET_RFCOMM_WARN(
 "%s: Got UA for dlci=%d in invalid state=%d, flags=%#x\n",
 				__func__, dlci, pcb->state, pcb->flags);
 			error = ENOENT;
 			break;
 		}
 
 		mtx_unlock(&pcb->pcb_mtx);
 	} else {
 		NG_BTSOCKET_RFCOMM_WARN(
 "%s: Got UA for non-existing dlci=%d\n", __func__, dlci);
 
 		error = ng_btsocket_rfcomm_send_command(s,RFCOMM_FRAME_DM,dlci);
 	}
 
 	return (error);
 } /* ng_btsocket_rfcomm_receive_ua */
 
 /*
  * Process RFCOMM DM frame
  */
 
 static int
 ng_btsocket_rfcomm_receive_dm(ng_btsocket_rfcomm_session_p s, int dlci)
 {
 	ng_btsocket_rfcomm_pcb_p	pcb = NULL;
 	int				error;
 
 	mtx_assert(&s->session_mtx, MA_OWNED);
 
 	NG_BTSOCKET_RFCOMM_INFO(
 "%s: Got DM, session state=%d, flags=%#x, mtu=%d, dlci=%d\n",
 		__func__, s->state, s->flags, s->mtu, dlci);
 
 	/* DLCI == 0 means multiplexor channel */
 	if (dlci == 0) {
 		/* Disconnect all dlc's on the session */
 		s->state = NG_BTSOCKET_RFCOMM_SESSION_CLOSED;
 		ng_btsocket_rfcomm_session_clean(s);
 	} else {
 		pcb = ng_btsocket_rfcomm_pcb_by_dlci(s, dlci);
 		if (pcb != NULL) {
 			mtx_lock(&pcb->pcb_mtx);
 
 			NG_BTSOCKET_RFCOMM_INFO(
 "%s: Got DM for dlci=%d, state=%d, flags=%#x\n",
 				__func__, dlci, pcb->state, pcb->flags);
 
 			if (pcb->state == NG_BTSOCKET_RFCOMM_DLC_CONNECTED)
 				error = ECONNRESET;
 			else
 				error = ECONNREFUSED;
 
 			ng_btsocket_rfcomm_pcb_kill(pcb, error);
 
 			mtx_unlock(&pcb->pcb_mtx);
 		} else
 			NG_BTSOCKET_RFCOMM_WARN(
 "%s: Got DM for non-existing dlci=%d\n", __func__, dlci);
 	}
 
 	return (0);
 } /* ng_btsocket_rfcomm_receive_dm */
 
 /*
  * Process RFCOMM UIH frame (data)
  */
 
 static int
 ng_btsocket_rfcomm_receive_uih(ng_btsocket_rfcomm_session_p s, int dlci,
 		int pf, struct mbuf *m0)
 {
 	ng_btsocket_rfcomm_pcb_p	pcb = NULL;
 	int				error = 0;
 
 	mtx_assert(&s->session_mtx, MA_OWNED);
 
 	NG_BTSOCKET_RFCOMM_INFO(
 "%s: Got UIH, session state=%d, flags=%#x, mtu=%d, dlci=%d, pf=%d, len=%d\n",
 		__func__, s->state, s->flags, s->mtu, dlci, pf,
 		m0->m_pkthdr.len);
 
 	/* XXX should we do it here? Check for session flow control */
 	if (s->flags & NG_BTSOCKET_RFCOMM_SESSION_LFC) {
 		NG_BTSOCKET_RFCOMM_WARN(
 "%s: Got UIH with session flow control asserted, state=%d, flags=%#x\n",
 			__func__, s->state, s->flags);
 		goto drop;
 	}
 
 	/* Check if we have this dlci */
 	pcb = ng_btsocket_rfcomm_pcb_by_dlci(s, dlci);
 	if (pcb == NULL) {
 		NG_BTSOCKET_RFCOMM_WARN(
 "%s: Got UIH for non-existing dlci=%d\n", __func__, dlci);
 		error = ng_btsocket_rfcomm_send_command(s,RFCOMM_FRAME_DM,dlci);
 		goto drop;
 	}
 
 	mtx_lock(&pcb->pcb_mtx);
 
 	/* Check dlci state */	
 	if (pcb->state != NG_BTSOCKET_RFCOMM_DLC_CONNECTED) {
 		NG_BTSOCKET_RFCOMM_WARN(
 "%s: Got UIH for dlci=%d in invalid state=%d, flags=%#x\n",
 			__func__, dlci, pcb->state, pcb->flags);
 		error = EINVAL;
 		goto drop1;
 	}
 
 	/* Check dlci flow control */
 	if (((pcb->flags & NG_BTSOCKET_RFCOMM_DLC_CFC) && pcb->rx_cred <= 0) ||
 	     (pcb->lmodem & RFCOMM_MODEM_FC)) {
 		NG_BTSOCKET_RFCOMM_ERR(
 "%s: Got UIH for dlci=%d with asserted flow control, state=%d, " \
 "flags=%#x, rx_cred=%d, lmodem=%#x\n",
 			__func__, dlci, pcb->state, pcb->flags,
 			pcb->rx_cred, pcb->lmodem);
 		goto drop1;
 	}
 
 	/* Did we get any credits? */
 	if ((pcb->flags & NG_BTSOCKET_RFCOMM_DLC_CFC) && pf) {
 		NG_BTSOCKET_RFCOMM_INFO(
 "%s: Got %d more credits for dlci=%d, state=%d, flags=%#x, " \
 "rx_cred=%d, tx_cred=%d\n",
 			__func__, *mtod(m0, u_int8_t *), dlci, pcb->state, 
 			pcb->flags, pcb->rx_cred, pcb->tx_cred);
 
 		pcb->tx_cred += *mtod(m0, u_int8_t *);
 		m_adj(m0, 1);
 
 		/* Send more from the DLC. XXX check for errors? */
 		ng_btsocket_rfcomm_pcb_send(pcb, ALOT);
 	} 
 
 	/* OK the of the rest of the mbuf is the data */
 	if (m0->m_pkthdr.len > 0) {
 		/* If we are using credit flow control decrease rx_cred here */
 		if (pcb->flags & NG_BTSOCKET_RFCOMM_DLC_CFC) {
 			/* Give remote peer more credits (if needed) */
 			if (-- pcb->rx_cred <= RFCOMM_MAX_CREDITS / 2)
 				ng_btsocket_rfcomm_send_credits(pcb);
 			else
 				NG_BTSOCKET_RFCOMM_INFO(
 "%s: Remote side still has credits, dlci=%d, state=%d, flags=%#x, " \
 "rx_cred=%d, tx_cred=%d\n",		__func__, dlci, pcb->state, pcb->flags,
 					pcb->rx_cred, pcb->tx_cred);
 		}
 		
 		/* Check packet against mtu on dlci */
 		if (m0->m_pkthdr.len > pcb->mtu) {
 			NG_BTSOCKET_RFCOMM_ERR(
 "%s: Got oversized UIH for dlci=%d, state=%d, flags=%#x, mtu=%d, len=%d\n",
 				__func__, dlci, pcb->state, pcb->flags,
 				pcb->mtu, m0->m_pkthdr.len);
 
 			error = EMSGSIZE;
 		} else if (m0->m_pkthdr.len > sbspace(&pcb->so->so_rcv)) {
  
 			/*
 			 * This is really bad. Receive queue on socket does
 			 * not have enough space for the packet. We do not
 			 * have any other choice but drop the packet. 
 			 */
  
 			NG_BTSOCKET_RFCOMM_ERR(
 "%s: Not enough space in socket receive queue. Dropping UIH for dlci=%d, " \
 "state=%d, flags=%#x, len=%d, space=%ld\n",
 				__func__, dlci, pcb->state, pcb->flags,
 				m0->m_pkthdr.len, sbspace(&pcb->so->so_rcv));
 
 			error = ENOBUFS;
 		} else {
 			/* Append packet to the socket receive queue */
 			sbappend(&pcb->so->so_rcv, m0);
 			m0 = NULL;
 
 			sorwakeup(pcb->so);
 		}
 	}
 drop1:
 	mtx_unlock(&pcb->pcb_mtx);
 drop:
 	NG_FREE_M(m0); /* checks for != NULL */
 
 	return (error);
 } /* ng_btsocket_rfcomm_receive_uih */
 
 /*
  * Process RFCOMM MCC command (Multiplexor)
  * 
  * From TS 07.10 spec
  *
  * "5.4.3.1 Information Data
  * 
  *  ...The frames (UIH) sent by the initiating station have the C/R bit set 
  *  to 1 and those sent by the responding station have the C/R bit set to 0..."
  *
  * "5.4.6.2 Operating procedures
  *
  *  Messages always exist in pairs; a command message and a corresponding 
  *  response message. If the C/R bit is set to 1 the message is a command, 
  *  if it is set to 0 the message is a response...
  *
  *  ...
  * 
  *  NOTE: Notice that when UIH frames are used to convey information on DLCI 0
  *  there are at least two different fields that contain a C/R bit, and the 
  *  bits are set of different form. The C/R bit in the Type field shall be set
  *  as it is stated above, while the C/R bit in the Address field (see subclause
  *  5.2.1.2) shall be set as it is described in subclause 5.4.3.1."
  */
 
 static int
 ng_btsocket_rfcomm_receive_mcc(ng_btsocket_rfcomm_session_p s, struct mbuf *m0)
 {
 	struct rfcomm_mcc_hdr	*hdr = NULL;
 	u_int8_t		 cr, type, length;
 
 	mtx_assert(&s->session_mtx, MA_OWNED);
 
 	/*
 	 * We can access data directly in the first mbuf, because we have
 	 * m_pullup()'ed mbuf chain in ng_btsocket_rfcomm_receive_frame().
 	 * All MCC commands should fit into single mbuf (except probably TEST).
 	 */
 
 	hdr = mtod(m0, struct rfcomm_mcc_hdr *);
 	cr = RFCOMM_CR(hdr->type);
 	type = RFCOMM_MCC_TYPE(hdr->type);
 	length = RFCOMM_MCC_LENGTH(hdr->length);
 
 	/* Check MCC frame length */
 	if (sizeof(*hdr) + length != m0->m_pkthdr.len) {
 		NG_BTSOCKET_RFCOMM_ERR(
 "%s: Invalid MCC frame length=%d, len=%d\n",
 			__func__, length, m0->m_pkthdr.len);
 		NG_FREE_M(m0);
 
 		return (EMSGSIZE);
 	}
 
 	switch (type) {
 	case RFCOMM_MCC_TEST:
 		return (ng_btsocket_rfcomm_receive_test(s, m0));
 		/* NOT REACHED */
 
 	case RFCOMM_MCC_FCON:
 	case RFCOMM_MCC_FCOFF:
 		return (ng_btsocket_rfcomm_receive_fc(s, m0));
 		/* NOT REACHED */
 
 	case RFCOMM_MCC_MSC:
 		return (ng_btsocket_rfcomm_receive_msc(s, m0));
 		/* NOT REACHED */
 
 	case RFCOMM_MCC_RPN:
 		return (ng_btsocket_rfcomm_receive_rpn(s, m0));
 		/* NOT REACHED */
 
 	case RFCOMM_MCC_RLS:
 		return (ng_btsocket_rfcomm_receive_rls(s, m0));
 		/* NOT REACHED */
 
 	case RFCOMM_MCC_PN:
 		return (ng_btsocket_rfcomm_receive_pn(s, m0));
 		/* NOT REACHED */
 
 	case RFCOMM_MCC_NSC:
 		NG_BTSOCKET_RFCOMM_ERR(
 "%s: Got MCC NSC, type=%#x, cr=%d, length=%d, session state=%d, flags=%#x, " \
 "mtu=%d, len=%d\n",	__func__, RFCOMM_MCC_TYPE(*((u_int8_t *)(hdr + 1))), cr,
 			 length, s->state, s->flags, s->mtu, m0->m_pkthdr.len);
 		NG_FREE_M(m0);
 		break;
 
 	default:
 		NG_BTSOCKET_RFCOMM_ERR(
 "%s: Got unknown MCC, type=%#x, cr=%d, length=%d, session state=%d, " \
 "flags=%#x, mtu=%d, len=%d\n",
 			__func__, type, cr, length, s->state, s->flags,
 			s->mtu, m0->m_pkthdr.len);
 
 		/* Reuse mbuf to send NSC */
 		hdr = mtod(m0, struct rfcomm_mcc_hdr *);
 		m0->m_pkthdr.len = m0->m_len = sizeof(*hdr);
 
 		/* Create MCC NSC header */
 		hdr->type = RFCOMM_MKMCC_TYPE(0, RFCOMM_MCC_NSC);
 		hdr->length = RFCOMM_MKLEN8(1);
 
 		/* Put back MCC command type we did not like */
 		m0->m_data[m0->m_len] = RFCOMM_MKMCC_TYPE(cr, type);
 		m0->m_pkthdr.len ++;
 		m0->m_len ++;
 
 		/* Send UIH frame */
 		return (ng_btsocket_rfcomm_send_uih(s,
 				RFCOMM_MKADDRESS(INITIATOR(s), 0), 0, 0, m0));
 		/* NOT REACHED */
 	}
 
 	return (0);
 } /* ng_btsocket_rfcomm_receive_mcc */
 
 /*
  * Receive RFCOMM TEST MCC command
  */
 
 static int
 ng_btsocket_rfcomm_receive_test(ng_btsocket_rfcomm_session_p s, struct mbuf *m0)
 {
 	struct rfcomm_mcc_hdr	*hdr = mtod(m0, struct rfcomm_mcc_hdr *);
 	int			 error = 0;
 
 	mtx_assert(&s->session_mtx, MA_OWNED);
 
 	NG_BTSOCKET_RFCOMM_INFO(
 "%s: Got MCC TEST, cr=%d, length=%d, session state=%d, flags=%#x, mtu=%d, " \
 "len=%d\n",	__func__, RFCOMM_CR(hdr->type), RFCOMM_MCC_LENGTH(hdr->length),
 		s->state, s->flags, s->mtu, m0->m_pkthdr.len);
 
 	if (RFCOMM_CR(hdr->type)) {
 		hdr->type = RFCOMM_MKMCC_TYPE(0, RFCOMM_MCC_TEST);
 		error = ng_btsocket_rfcomm_send_uih(s,
 				RFCOMM_MKADDRESS(INITIATOR(s), 0), 0, 0, m0);
 	} else
 		NG_FREE_M(m0); /* XXX ignore response */
 
 	return (error);
 } /* ng_btsocket_rfcomm_receive_test */
 
 /*
  * Receive RFCOMM FCON/FCOFF MCC command
  */
 
 static int
 ng_btsocket_rfcomm_receive_fc(ng_btsocket_rfcomm_session_p s, struct mbuf *m0)
 {
 	struct rfcomm_mcc_hdr	*hdr = mtod(m0, struct rfcomm_mcc_hdr *);
 	u_int8_t		 type = RFCOMM_MCC_TYPE(hdr->type);
 	int			 error = 0;
 
 	mtx_assert(&s->session_mtx, MA_OWNED);
 
 	/*
 	 * Turn ON/OFF aggregate flow on the entire session. When remote peer 
 	 * asserted flow control no transmission shall occur except on dlci 0
 	 * (control channel).
 	 */
 
 	NG_BTSOCKET_RFCOMM_INFO(
 "%s: Got MCC FC%s, cr=%d, length=%d, session state=%d, flags=%#x, mtu=%d, " \
 "len=%d\n",	__func__, (type == RFCOMM_MCC_FCON)? "ON" : "OFF",
 		RFCOMM_CR(hdr->type), RFCOMM_MCC_LENGTH(hdr->length),
 		s->state, s->flags, s->mtu, m0->m_pkthdr.len);
 
 	if (RFCOMM_CR(hdr->type)) {
 		if (type == RFCOMM_MCC_FCON)
 			s->flags &= ~NG_BTSOCKET_RFCOMM_SESSION_RFC;
 		else
 			s->flags |= NG_BTSOCKET_RFCOMM_SESSION_RFC;
 
 		hdr->type = RFCOMM_MKMCC_TYPE(0, type);
 		error = ng_btsocket_rfcomm_send_uih(s,
 				RFCOMM_MKADDRESS(INITIATOR(s), 0), 0, 0, m0);
 	} else
 		NG_FREE_M(m0); /* XXX ignore response */
 
 	return (error);
 } /* ng_btsocket_rfcomm_receive_fc  */
 
 /*
  * Receive RFCOMM MSC MCC command
  */
 
 static int
 ng_btsocket_rfcomm_receive_msc(ng_btsocket_rfcomm_session_p s, struct mbuf *m0)
 {
 	struct rfcomm_mcc_hdr		*hdr = mtod(m0, struct rfcomm_mcc_hdr*);
 	struct rfcomm_mcc_msc		*msc = (struct rfcomm_mcc_msc *)(hdr+1);
 	ng_btsocket_rfcomm_pcb_t	*pcb = NULL;
 	int				 error = 0;
 
 	mtx_assert(&s->session_mtx, MA_OWNED);
 
 	NG_BTSOCKET_RFCOMM_INFO(
 "%s: Got MCC MSC, dlci=%d, cr=%d, length=%d, session state=%d, flags=%#x, " \
 "mtu=%d, len=%d\n",
 		__func__,  RFCOMM_DLCI(msc->address), RFCOMM_CR(hdr->type),
 		RFCOMM_MCC_LENGTH(hdr->length), s->state, s->flags,
 		s->mtu, m0->m_pkthdr.len);
 
 	if (RFCOMM_CR(hdr->type)) {
 		pcb = ng_btsocket_rfcomm_pcb_by_dlci(s, RFCOMM_DLCI(msc->address));
 		if (pcb == NULL) {
 			NG_BTSOCKET_RFCOMM_WARN(
 "%s: Got MSC command for non-existing dlci=%d\n",
 				__func__, RFCOMM_DLCI(msc->address));
 			NG_FREE_M(m0);
 
 			return (ENOENT);
 		}
 
 		mtx_lock(&pcb->pcb_mtx);
 
 		if (pcb->state != NG_BTSOCKET_RFCOMM_DLC_CONNECTING &&
 		    pcb->state != NG_BTSOCKET_RFCOMM_DLC_CONNECTED) {
 			NG_BTSOCKET_RFCOMM_WARN(
 "%s: Got MSC on dlci=%d in invalid state=%d\n",
 				__func__, RFCOMM_DLCI(msc->address),
 				pcb->state);
 
 			mtx_unlock(&pcb->pcb_mtx);
 			NG_FREE_M(m0);
 
 			return (EINVAL);
 		}
 
 		pcb->rmodem = msc->modem; /* Update remote port signals */
 
 		hdr->type = RFCOMM_MKMCC_TYPE(0, RFCOMM_MCC_MSC);
 		error = ng_btsocket_rfcomm_send_uih(s,
 				RFCOMM_MKADDRESS(INITIATOR(s), 0), 0, 0, m0);
 
 #if 0 /* YYY */
 		/* Send more data from DLC. XXX check for errors? */
 		if (!(pcb->rmodem & RFCOMM_MODEM_FC) &&
 		    !(pcb->flags & NG_BTSOCKET_RFCOMM_DLC_CFC))
 			ng_btsocket_rfcomm_pcb_send(pcb, ALOT);
 #endif /* YYY */
 
 		mtx_unlock(&pcb->pcb_mtx);
 	} else
 		NG_FREE_M(m0); /* XXX ignore response */
 
 	return (error);
 } /* ng_btsocket_rfcomm_receive_msc */
 
 /*
  * Receive RFCOMM RPN MCC command
  * XXX FIXME do we need htole16/le16toh for RPN param_mask?
  */
 
 static int
 ng_btsocket_rfcomm_receive_rpn(ng_btsocket_rfcomm_session_p s, struct mbuf *m0)
 {
 	struct rfcomm_mcc_hdr	*hdr = mtod(m0, struct rfcomm_mcc_hdr *);
 	struct rfcomm_mcc_rpn	*rpn = (struct rfcomm_mcc_rpn *)(hdr + 1);
 	int			 error = 0;
 	u_int16_t		 param_mask;
 	u_int8_t		 bit_rate, data_bits, stop_bits, parity,
 				 flow_control, xon_char, xoff_char;
 
 	mtx_assert(&s->session_mtx, MA_OWNED);
 
 	NG_BTSOCKET_RFCOMM_INFO(
 "%s: Got MCC RPN, dlci=%d, cr=%d, length=%d, session state=%d, flags=%#x, " \
 "mtu=%d, len=%d\n",
 		__func__, RFCOMM_DLCI(rpn->dlci), RFCOMM_CR(hdr->type),
 		RFCOMM_MCC_LENGTH(hdr->length), s->state, s->flags,
 		s->mtu, m0->m_pkthdr.len);
 
 	if (RFCOMM_CR(hdr->type)) {
 		param_mask = RFCOMM_RPN_PM_ALL;
 
 		if (RFCOMM_MCC_LENGTH(hdr->length) == 1) {
 			/* Request - return default setting */
 			bit_rate = RFCOMM_RPN_BR_115200;
 			data_bits = RFCOMM_RPN_DATA_8;
 			stop_bits = RFCOMM_RPN_STOP_1;
 			parity = RFCOMM_RPN_PARITY_NONE;
 			flow_control = RFCOMM_RPN_FLOW_NONE;
 			xon_char = RFCOMM_RPN_XON_CHAR;
 			xoff_char = RFCOMM_RPN_XOFF_CHAR;
                 } else {
 			/*
 			 * Ignore/accept bit_rate, 8 bits, 1 stop bit, no 
 			 * parity, no flow control lines, default XON/XOFF 
 			 * chars.
 			 */
 
 			bit_rate = rpn->bit_rate;
 			rpn->param_mask = le16toh(rpn->param_mask); /* XXX */
 
 			data_bits = RFCOMM_RPN_DATA_BITS(rpn->line_settings);
 			if (rpn->param_mask & RFCOMM_RPN_PM_DATA &&
 			    data_bits != RFCOMM_RPN_DATA_8) {
 				data_bits = RFCOMM_RPN_DATA_8;
 				param_mask ^= RFCOMM_RPN_PM_DATA;
 			}
 
 			stop_bits = RFCOMM_RPN_STOP_BITS(rpn->line_settings);
 			if (rpn->param_mask & RFCOMM_RPN_PM_STOP &&
 			    stop_bits != RFCOMM_RPN_STOP_1) {
 				stop_bits = RFCOMM_RPN_STOP_1;
 				param_mask ^= RFCOMM_RPN_PM_STOP;
 			}
 
 			parity = RFCOMM_RPN_PARITY(rpn->line_settings);
 			if (rpn->param_mask & RFCOMM_RPN_PM_PARITY &&
 			    parity != RFCOMM_RPN_PARITY_NONE) {
 				parity = RFCOMM_RPN_PARITY_NONE;
 				param_mask ^= RFCOMM_RPN_PM_PARITY;
 			}
 
 			flow_control = rpn->flow_control;
 			if (rpn->param_mask & RFCOMM_RPN_PM_FLOW &&
 			    flow_control != RFCOMM_RPN_FLOW_NONE) {
 				flow_control = RFCOMM_RPN_FLOW_NONE;
 				param_mask ^= RFCOMM_RPN_PM_FLOW;
 			}
 
 			xon_char = rpn->xon_char;
 			if (rpn->param_mask & RFCOMM_RPN_PM_XON &&
 			    xon_char != RFCOMM_RPN_XON_CHAR) {
 				xon_char = RFCOMM_RPN_XON_CHAR;
 				param_mask ^= RFCOMM_RPN_PM_XON;
 			}
 
 			xoff_char = rpn->xoff_char;
 			if (rpn->param_mask & RFCOMM_RPN_PM_XOFF &&
 			    xoff_char != RFCOMM_RPN_XOFF_CHAR) {
 				xoff_char = RFCOMM_RPN_XOFF_CHAR;
 				param_mask ^= RFCOMM_RPN_PM_XOFF;
 			}
 		}
 
 		rpn->bit_rate = bit_rate;
 		rpn->line_settings = RFCOMM_MKRPN_LINE_SETTINGS(data_bits, 
 						stop_bits, parity);
 		rpn->flow_control = flow_control;
 		rpn->xon_char = xon_char;
 		rpn->xoff_char = xoff_char;
 		rpn->param_mask = htole16(param_mask); /* XXX */
 
 		m0->m_pkthdr.len = m0->m_len = sizeof(*hdr) + sizeof(*rpn);
 
 		hdr->type = RFCOMM_MKMCC_TYPE(0, RFCOMM_MCC_RPN);
 		error = ng_btsocket_rfcomm_send_uih(s,
 				RFCOMM_MKADDRESS(INITIATOR(s), 0), 0, 0, m0);
 	} else
 		NG_FREE_M(m0); /* XXX ignore response */
 
 	return (error);
 } /* ng_btsocket_rfcomm_receive_rpn */
 
 /*
  * Receive RFCOMM RLS MCC command
  */
 
 static int
 ng_btsocket_rfcomm_receive_rls(ng_btsocket_rfcomm_session_p s, struct mbuf *m0)
 {
 	struct rfcomm_mcc_hdr	*hdr = mtod(m0, struct rfcomm_mcc_hdr *);
 	struct rfcomm_mcc_rls	*rls = (struct rfcomm_mcc_rls *)(hdr + 1);
 	int			 error = 0;
 
 	mtx_assert(&s->session_mtx, MA_OWNED);
 
 	/*
 	 * XXX FIXME Do we have to do anything else here? Remote peer tries to 
 	 * tell us something about DLCI. Just report what we have received and
 	 * return back received values as required by TS 07.10 spec.
 	 */
 
 	NG_BTSOCKET_RFCOMM_INFO(
 "%s: Got MCC RLS, dlci=%d, status=%#x, cr=%d, length=%d, session state=%d, " \
 "flags=%#x, mtu=%d, len=%d\n",
 		__func__, RFCOMM_DLCI(rls->address), rls->status,
 		RFCOMM_CR(hdr->type), RFCOMM_MCC_LENGTH(hdr->length),
 		s->state, s->flags, s->mtu, m0->m_pkthdr.len);
 
 	if (RFCOMM_CR(hdr->type)) {
 		if (rls->status & 0x1)
 			NG_BTSOCKET_RFCOMM_ERR(
 "%s: Got RLS dlci=%d, error=%#x\n", __func__, RFCOMM_DLCI(rls->address),
 				rls->status >> 1);
 
 		hdr->type = RFCOMM_MKMCC_TYPE(0, RFCOMM_MCC_RLS);
 		error = ng_btsocket_rfcomm_send_uih(s,
 				RFCOMM_MKADDRESS(INITIATOR(s), 0), 0, 0, m0);
 	} else
 		NG_FREE_M(m0); /* XXX ignore responses */
 
 	return (error);
 } /* ng_btsocket_rfcomm_receive_rls */
 
 /*
  * Receive RFCOMM PN MCC command
  */
 
 static int
 ng_btsocket_rfcomm_receive_pn(ng_btsocket_rfcomm_session_p s, struct mbuf *m0)
 {
 	struct rfcomm_mcc_hdr		*hdr = mtod(m0, struct rfcomm_mcc_hdr*);
 	struct rfcomm_mcc_pn		*pn = (struct rfcomm_mcc_pn *)(hdr+1);
 	ng_btsocket_rfcomm_pcb_t	*pcb = NULL;
 	int				 error = 0;
 
 	mtx_assert(&s->session_mtx, MA_OWNED);
 
 	NG_BTSOCKET_RFCOMM_INFO(
 "%s: Got MCC PN, dlci=%d, cr=%d, length=%d, flow_control=%#x, priority=%d, " \
 "ack_timer=%d, mtu=%d, max_retrans=%d, credits=%d, session state=%d, " \
 "flags=%#x, session mtu=%d, len=%d\n",
 		__func__, pn->dlci, RFCOMM_CR(hdr->type),
 		RFCOMM_MCC_LENGTH(hdr->length), pn->flow_control, pn->priority,
 		pn->ack_timer, le16toh(pn->mtu), pn->max_retrans, pn->credits,
 		s->state, s->flags, s->mtu, m0->m_pkthdr.len);
 
 	if (pn->dlci == 0) {
 		NG_BTSOCKET_RFCOMM_ERR("%s: Zero dlci in MCC PN\n", __func__);
 		NG_FREE_M(m0);
 
 		return (EINVAL);
 	}
 
 	/* Check if we have this dlci */
 	pcb = ng_btsocket_rfcomm_pcb_by_dlci(s, pn->dlci);
 	if (pcb != NULL) {
 		mtx_lock(&pcb->pcb_mtx);
 
 		if (RFCOMM_CR(hdr->type)) {
 			/* PN Request */
 			ng_btsocket_rfcomm_set_pn(pcb, 1, pn->flow_control,
 				pn->credits, pn->mtu);
 
 			if (pcb->flags & NG_BTSOCKET_RFCOMM_DLC_CFC) {
 				pn->flow_control = 0xe0;
 				pn->credits = RFCOMM_DEFAULT_CREDITS;
 			} else {
 				pn->flow_control = 0;
 				pn->credits = 0;
 			}
 
 			hdr->type = RFCOMM_MKMCC_TYPE(0, RFCOMM_MCC_PN);
 			error = ng_btsocket_rfcomm_send_uih(s, 
 					RFCOMM_MKADDRESS(INITIATOR(s), 0),
 					0, 0, m0);
 		} else {
 			/* PN Response - proceed with SABM. Timeout still set */
 			if (pcb->state == NG_BTSOCKET_RFCOMM_DLC_CONFIGURING) {
 				ng_btsocket_rfcomm_set_pn(pcb, 0,
 					pn->flow_control, pn->credits, pn->mtu);
 
 				pcb->state = NG_BTSOCKET_RFCOMM_DLC_CONNECTING;
 				error = ng_btsocket_rfcomm_send_command(s,
 						RFCOMM_FRAME_SABM, pn->dlci);
 			} else
 				NG_BTSOCKET_RFCOMM_WARN(
 "%s: Got PN response for dlci=%d in invalid state=%d\n",
 					__func__, pn->dlci, pcb->state);
 
 			NG_FREE_M(m0);
 		}
 
 		mtx_unlock(&pcb->pcb_mtx);
 	} else if (RFCOMM_CR(hdr->type)) {
 		/* PN request to non-existing dlci - incomming connection */
 		pcb = ng_btsocket_rfcomm_connect_ind(s,
 				RFCOMM_SRVCHANNEL(pn->dlci));
 		if (pcb != NULL) {
 			mtx_lock(&pcb->pcb_mtx);
 
 			pcb->dlci = pn->dlci;
 
 			ng_btsocket_rfcomm_set_pn(pcb, 1, pn->flow_control,
 				pn->credits, pn->mtu);
 
 			if (pcb->flags & NG_BTSOCKET_RFCOMM_DLC_CFC) {
 				pn->flow_control = 0xe0;
 				pn->credits = RFCOMM_DEFAULT_CREDITS;
 			} else {
 				pn->flow_control = 0;
 				pn->credits = 0;
 			}
 
 			hdr->type = RFCOMM_MKMCC_TYPE(0, RFCOMM_MCC_PN);
 			error = ng_btsocket_rfcomm_send_uih(s, 
 					RFCOMM_MKADDRESS(INITIATOR(s), 0),
 					0, 0, m0);
 
 			if (error == 0) {
 				ng_btsocket_rfcomm_timeout(pcb);
 				pcb->state = NG_BTSOCKET_RFCOMM_DLC_CONNECTING;
 				soisconnecting(pcb->so);
 			} else
 				ng_btsocket_rfcomm_pcb_kill(pcb, error);
 
 			mtx_unlock(&pcb->pcb_mtx);
 		} else {
 			/* Nobody is listen()ing on this channel */
 			error = ng_btsocket_rfcomm_send_command(s,
 					RFCOMM_FRAME_DM, pn->dlci);
 			NG_FREE_M(m0);
 		}
 	} else
 		NG_FREE_M(m0); /* XXX ignore response to non-existing dlci */
 
 	return (error);
 } /* ng_btsocket_rfcomm_receive_pn */
 
 /*
  * Set PN parameters for dlci. Caller must hold pcb->pcb_mtx.
  * 
  * From Bluetooth spec.
  * 
  * "... The CL1 - CL4 field is completely redefined. (In TS07.10 this defines 
  *  the convergence layer to use, which is not applicable to RFCOMM. In RFCOMM,
  *  in Bluetooth versions up to 1.0B, this field was forced to 0).
  *
  *  In the PN request sent prior to a DLC establishment, this field must contain
  *  the value 15 (0xF), indicating support of credit based flow control in the 
  *  sender. See Table 5.3 below. If the PN response contains any other value 
  *  than 14 (0xE) in this field, it is inferred that the peer RFCOMM entity is 
  *  not supporting the credit based flow control feature. (This is only possible
  *  if the peer RFCOMM implementation is only conforming to Bluetooth version 
  *  1.0B.) If a PN request is sent on an already open DLC, then this field must
  *  contain the value zero; it is not possible to set initial credits  more 
  *  than once per DLC activation. A responding implementation must set this 
  *  field in the PN response to 14 (0xE), if (and only if) the value in the PN 
  *  request was 15..."
  */
 
 static void
 ng_btsocket_rfcomm_set_pn(ng_btsocket_rfcomm_pcb_p pcb, u_int8_t cr,
 		u_int8_t flow_control, u_int8_t credits, u_int16_t mtu)
 {
 	mtx_assert(&pcb->pcb_mtx, MA_OWNED);
 
 	pcb->mtu = le16toh(mtu);
 
 	if (cr) {
 		if (flow_control == 0xf0) {
 			pcb->flags |= NG_BTSOCKET_RFCOMM_DLC_CFC;
 			pcb->tx_cred = credits;
 		} else {
 			pcb->flags &= ~NG_BTSOCKET_RFCOMM_DLC_CFC;
 			pcb->tx_cred = 0;
 		}
 	} else {
 		if (flow_control == 0xe0) {
 			pcb->flags |= NG_BTSOCKET_RFCOMM_DLC_CFC;
 			pcb->tx_cred = credits;
 		} else {
 			pcb->flags &= ~NG_BTSOCKET_RFCOMM_DLC_CFC;
 			pcb->tx_cred = 0;
 		}
 	}
 
 	NG_BTSOCKET_RFCOMM_INFO(
 "%s: cr=%d, dlci=%d, state=%d, flags=%#x, mtu=%d, rx_cred=%d, tx_cred=%d\n",
 		__func__, cr, pcb->dlci, pcb->state, pcb->flags, pcb->mtu,
 		pcb->rx_cred, pcb->tx_cred);
 } /* ng_btsocket_rfcomm_set_pn */
 
 /*
  * Send RFCOMM SABM/DISC/UA/DM frames. Caller must hold s->session_mtx
  */
 
 static int
 ng_btsocket_rfcomm_send_command(ng_btsocket_rfcomm_session_p s,
 		u_int8_t type, u_int8_t dlci)
 {
 	struct rfcomm_cmd_hdr	*hdr = NULL;
 	struct mbuf		*m = NULL;
 	int			 cr;
 
 	mtx_assert(&s->session_mtx, MA_OWNED);
 
 	NG_BTSOCKET_RFCOMM_INFO(
 "%s: Sending command type %#x, session state=%d, flags=%#x, mtu=%d, dlci=%d\n",
 		__func__, type, s->state, s->flags, s->mtu, dlci);
 
 	switch (type) {
 	case RFCOMM_FRAME_SABM:
 	case RFCOMM_FRAME_DISC:
 		cr = INITIATOR(s);
 		break;
 
 	case RFCOMM_FRAME_UA:
 	case RFCOMM_FRAME_DM:
 		cr = !INITIATOR(s);
 		break;
 
 	default:
 		panic("%s: Invalid frame type=%#x\n", __func__, type);
 		return (EINVAL);
 		/* NOT REACHED */
 	}
 
 	MGETHDR(m, M_NOWAIT, MT_DATA);
 	if (m == NULL)
 		return (ENOBUFS);
 
 	m->m_pkthdr.len = m->m_len = sizeof(*hdr);
 
 	hdr = mtod(m, struct rfcomm_cmd_hdr *);
 	hdr->address = RFCOMM_MKADDRESS(cr, dlci);
 	hdr->control = RFCOMM_MKCONTROL(type, 1);
 	hdr->length = RFCOMM_MKLEN8(0);
 	hdr->fcs = ng_btsocket_rfcomm_fcs3((u_int8_t *) hdr);
 
 	NG_BT_MBUFQ_ENQUEUE(&s->outq, m);
 
 	return (0);
 } /* ng_btsocket_rfcomm_send_command */
 
 /*
  * Send RFCOMM UIH frame. Caller must hold s->session_mtx
  */
 
 static int
 ng_btsocket_rfcomm_send_uih(ng_btsocket_rfcomm_session_p s, u_int8_t address,
 		u_int8_t pf, u_int8_t credits, struct mbuf *data)
 {
 	struct rfcomm_frame_hdr	*hdr = NULL;
 	struct mbuf		*m = NULL, *mcrc = NULL;
 	u_int16_t		 length;
 
 	mtx_assert(&s->session_mtx, MA_OWNED);
 
 	MGETHDR(m, M_NOWAIT, MT_DATA);
 	if (m == NULL) {
 		NG_FREE_M(data);
 		return (ENOBUFS);
 	}
 	m->m_pkthdr.len = m->m_len = sizeof(*hdr);
 
 	MGET(mcrc, M_NOWAIT, MT_DATA);
 	if (mcrc == NULL) {
 		NG_FREE_M(data);
 		return (ENOBUFS);
 	}
 	mcrc->m_len = 1;
 
 	/* Fill UIH frame header */
 	hdr = mtod(m, struct rfcomm_frame_hdr *);
 	hdr->address = address;
 	hdr->control = RFCOMM_MKCONTROL(RFCOMM_FRAME_UIH, pf);
 
 	/* Calculate FCS */
 	mcrc->m_data[0] = ng_btsocket_rfcomm_fcs2((u_int8_t *) hdr);
 
 	/* Put length back */
 	length = (data != NULL)? data->m_pkthdr.len : 0;
 	if (length > 127) {
 		u_int16_t	l = htole16(RFCOMM_MKLEN16(length));
 
 		bcopy(&l, &hdr->length, sizeof(l));
 		m->m_pkthdr.len ++;
 		m->m_len ++;
 	} else
 		hdr->length = RFCOMM_MKLEN8(length);
 
 	if (pf) {
 		m->m_data[m->m_len] = credits;
 		m->m_pkthdr.len ++;
 		m->m_len ++;
 	}
 
 	/* Add payload */
 	if (data != NULL) {
 		m_cat(m, data);
 		m->m_pkthdr.len += length;
 	}
 
 	/* Put FCS back */
 	m_cat(m, mcrc);
 	m->m_pkthdr.len ++;
 
 	NG_BTSOCKET_RFCOMM_INFO(
 "%s: Sending UIH state=%d, flags=%#x, address=%d, length=%d, pf=%d, " \
 "credits=%d, len=%d\n",
 		__func__, s->state, s->flags, address, length, pf, credits,
 		m->m_pkthdr.len);
 
 	NG_BT_MBUFQ_ENQUEUE(&s->outq, m);
 
 	return (0);
 } /* ng_btsocket_rfcomm_send_uih */
 
 /*
  * Send MSC request. Caller must hold pcb->pcb_mtx and pcb->session->session_mtx
  */
 
 static int
 ng_btsocket_rfcomm_send_msc(ng_btsocket_rfcomm_pcb_p pcb)
 {
 	struct mbuf		*m = NULL;
 	struct rfcomm_mcc_hdr	*hdr = NULL;
 	struct rfcomm_mcc_msc	*msc = NULL;
 
 	mtx_assert(&pcb->session->session_mtx, MA_OWNED);
 	mtx_assert(&pcb->pcb_mtx, MA_OWNED);
 
 	MGETHDR(m, M_NOWAIT, MT_DATA);
 	if (m == NULL)
 		return (ENOBUFS);
 
 	m->m_pkthdr.len = m->m_len = sizeof(*hdr) + sizeof(*msc);
 
 	hdr = mtod(m, struct rfcomm_mcc_hdr *);
 	msc = (struct rfcomm_mcc_msc *)(hdr + 1);
 
 	hdr->type = RFCOMM_MKMCC_TYPE(1, RFCOMM_MCC_MSC);
 	hdr->length = RFCOMM_MKLEN8(sizeof(*msc));
 
 	msc->address = RFCOMM_MKADDRESS(1, pcb->dlci);
 	msc->modem = pcb->lmodem;
 
 	NG_BTSOCKET_RFCOMM_INFO(
 "%s: Sending MSC dlci=%d, state=%d, flags=%#x, address=%d, modem=%#x\n",
 		__func__, pcb->dlci, pcb->state, pcb->flags, msc->address,
 		msc->modem);
 
 	return (ng_btsocket_rfcomm_send_uih(pcb->session,
 			RFCOMM_MKADDRESS(INITIATOR(pcb->session), 0), 0, 0, m));
 } /* ng_btsocket_rfcomm_send_msc */
 
 /*
  * Send PN request. Caller must hold pcb->pcb_mtx and pcb->session->session_mtx
  */
 
 static int
 ng_btsocket_rfcomm_send_pn(ng_btsocket_rfcomm_pcb_p pcb)
 {
 	struct mbuf		*m = NULL;
 	struct rfcomm_mcc_hdr	*hdr = NULL;
 	struct rfcomm_mcc_pn	*pn = NULL;
 
 	mtx_assert(&pcb->session->session_mtx, MA_OWNED);
 	mtx_assert(&pcb->pcb_mtx, MA_OWNED);
 
 	MGETHDR(m, M_NOWAIT, MT_DATA);
 	if (m == NULL)
 		return (ENOBUFS);
 
 	m->m_pkthdr.len = m->m_len = sizeof(*hdr) + sizeof(*pn);
 
 	hdr = mtod(m, struct rfcomm_mcc_hdr *);
 	pn = (struct rfcomm_mcc_pn *)(hdr + 1);
 
 	hdr->type = RFCOMM_MKMCC_TYPE(1, RFCOMM_MCC_PN);
 	hdr->length = RFCOMM_MKLEN8(sizeof(*pn));
 
 	pn->dlci = pcb->dlci;
 
 	/*
 	 * Set default DLCI priority as described in GSM 07.10
 	 * (ETSI TS 101 369) clause 5.6 page 42
 	 */
 
 	pn->priority = (pcb->dlci < 56)? (((pcb->dlci >> 3) << 3) + 7) : 61;
 	pn->ack_timer = 0;
 	pn->mtu = htole16(pcb->mtu);
 	pn->max_retrans = 0;
 
 	if (pcb->flags & NG_BTSOCKET_RFCOMM_DLC_CFC) {
 		pn->flow_control = 0xf0;
 		pn->credits = pcb->rx_cred;
 	} else {
 		pn->flow_control = 0;
 		pn->credits = 0;
 	}
 
 	NG_BTSOCKET_RFCOMM_INFO(
 "%s: Sending PN dlci=%d, state=%d, flags=%#x, mtu=%d, flow_control=%#x, " \
 "credits=%d\n",	__func__, pcb->dlci, pcb->state, pcb->flags, pcb->mtu,
 		pn->flow_control, pn->credits);
 
 	return (ng_btsocket_rfcomm_send_uih(pcb->session,
 			RFCOMM_MKADDRESS(INITIATOR(pcb->session), 0), 0, 0, m));
 } /* ng_btsocket_rfcomm_send_pn */
 
 /*
  * Calculate and send credits based on available space in receive buffer
  */
 
 static int
 ng_btsocket_rfcomm_send_credits(ng_btsocket_rfcomm_pcb_p pcb)
 {
 	int		error = 0;
 	u_int8_t	credits;
 
 	mtx_assert(&pcb->pcb_mtx, MA_OWNED);
 	mtx_assert(&pcb->session->session_mtx, MA_OWNED);
 
 	NG_BTSOCKET_RFCOMM_INFO(
 "%s: Sending more credits, dlci=%d, state=%d, flags=%#x, mtu=%d, " \
 "space=%ld, tx_cred=%d, rx_cred=%d\n",
 		__func__, pcb->dlci, pcb->state, pcb->flags, pcb->mtu,
 		sbspace(&pcb->so->so_rcv), pcb->tx_cred, pcb->rx_cred);
 
 	credits = sbspace(&pcb->so->so_rcv) / pcb->mtu;
 	if (credits > 0) {
 		if (pcb->rx_cred + credits > RFCOMM_MAX_CREDITS)
 			credits = RFCOMM_MAX_CREDITS - pcb->rx_cred;
 
 		error = ng_btsocket_rfcomm_send_uih(
 				pcb->session,
 				RFCOMM_MKADDRESS(INITIATOR(pcb->session),
 					pcb->dlci), 1, credits, NULL);
 		if (error == 0) {
 			pcb->rx_cred += credits;
 
 			NG_BTSOCKET_RFCOMM_INFO(
 "%s: Gave remote side %d more credits, dlci=%d, state=%d, flags=%#x, " \
 "rx_cred=%d, tx_cred=%d\n",	__func__, credits, pcb->dlci, pcb->state,
 				pcb->flags, pcb->rx_cred, pcb->tx_cred);
 		} else
 			NG_BTSOCKET_RFCOMM_ERR(
 "%s: Could not send credits, error=%d, dlci=%d, state=%d, flags=%#x, " \
 "mtu=%d, space=%ld, tx_cred=%d, rx_cred=%d\n",
 				__func__, error, pcb->dlci, pcb->state,
 				pcb->flags, pcb->mtu, sbspace(&pcb->so->so_rcv),
 				pcb->tx_cred, pcb->rx_cred);
 	}
 
 	return (error);
 } /* ng_btsocket_rfcomm_send_credits */
 
 /*****************************************************************************
  *****************************************************************************
  **                              RFCOMM DLCs
  *****************************************************************************
  *****************************************************************************/
 
 /*
  * Send data from socket send buffer
  * Caller must hold pcb->pcb_mtx and pcb->session->session_mtx
  */
 
 static int
 ng_btsocket_rfcomm_pcb_send(ng_btsocket_rfcomm_pcb_p pcb, int limit)
 {
 	struct mbuf	*m = NULL;
 	int		 sent, length, error;
 
 	mtx_assert(&pcb->session->session_mtx, MA_OWNED);
 	mtx_assert(&pcb->pcb_mtx, MA_OWNED);
 
 	if (pcb->flags & NG_BTSOCKET_RFCOMM_DLC_CFC)
 		limit = min(limit, pcb->tx_cred);
 	else if (!(pcb->rmodem & RFCOMM_MODEM_FC))
 		limit = min(limit, RFCOMM_MAX_CREDITS); /* XXX ??? */
 	else
 		limit = 0;
 
 	if (limit == 0) {
 		NG_BTSOCKET_RFCOMM_INFO(
 "%s: Could not send - remote flow control asserted, dlci=%d, flags=%#x, " \
 "rmodem=%#x, tx_cred=%d\n",
 			__func__, pcb->dlci, pcb->flags, pcb->rmodem,
 			pcb->tx_cred);
 
 		return (0);
 	}
 
 	for (error = 0, sent = 0; sent < limit; sent ++) { 
 		length = min(pcb->mtu, pcb->so->so_snd.sb_cc);
 		if (length == 0)
 			break;
 
 		/* Get the chunk from the socket's send buffer */
 		m = ng_btsocket_rfcomm_prepare_packet(&pcb->so->so_snd, length);
 		if (m == NULL) {
 			error = ENOBUFS;
 			break;
 		}
 
 		sbdrop(&pcb->so->so_snd, length);
 
 		error = ng_btsocket_rfcomm_send_uih(pcb->session,
 				RFCOMM_MKADDRESS(INITIATOR(pcb->session),
 					pcb->dlci), 0, 0, m);
 		if (error != 0)
 			break;
 	}
 
 	if (pcb->flags & NG_BTSOCKET_RFCOMM_DLC_CFC)
 		pcb->tx_cred -= sent;
 
 	if (error == 0 && sent > 0) {
 		pcb->flags &= ~NG_BTSOCKET_RFCOMM_DLC_SENDING;
 		sowwakeup(pcb->so);
 	}
 
 	return (error);
 } /* ng_btsocket_rfcomm_pcb_send */
 
 /*
  * Unlink and disconnect DLC. If ng_btsocket_rfcomm_pcb_kill() returns
  * non zero value than socket has no reference and has to be detached.
  * Caller must hold pcb->pcb_mtx and pcb->session->session_mtx
  */
 
 static void
 ng_btsocket_rfcomm_pcb_kill(ng_btsocket_rfcomm_pcb_p pcb, int error)
 {
 	ng_btsocket_rfcomm_session_p	s = pcb->session;
 
 	NG_BTSOCKET_RFCOMM_INFO(
 "%s: Killing DLC, so=%p, dlci=%d, state=%d, flags=%#x, error=%d\n",
 		__func__, pcb->so, pcb->dlci, pcb->state, pcb->flags, error);
 
 	if (pcb->session == NULL)
 		panic("%s: DLC without session, pcb=%p, state=%d, flags=%#x\n",
 			__func__, pcb, pcb->state, pcb->flags);
 
 	mtx_assert(&pcb->session->session_mtx, MA_OWNED);
 	mtx_assert(&pcb->pcb_mtx, MA_OWNED);
 
 	if (pcb->flags & NG_BTSOCKET_RFCOMM_DLC_TIMO)
 		ng_btsocket_rfcomm_untimeout(pcb);
 
 	/* Detach DLC from the session. Does not matter which state DLC in */
 	LIST_REMOVE(pcb, session_next);
 	pcb->session = NULL;
 
 	/* Change DLC state and wakeup all sleepers */
 	pcb->state = NG_BTSOCKET_RFCOMM_DLC_CLOSED;
 	pcb->so->so_error = error;
 	soisdisconnected(pcb->so);
 	wakeup(&pcb->state);
 
 	/* Check if we have any DLCs left on the session */
 	if (LIST_EMPTY(&s->dlcs) && INITIATOR(s)) {
 		NG_BTSOCKET_RFCOMM_INFO(
 "%s: Disconnecting session, state=%d, flags=%#x, mtu=%d\n",
 			__func__, s->state, s->flags, s->mtu);
 
 		switch (s->state) {
 		case NG_BTSOCKET_RFCOMM_SESSION_CLOSED:
 		case NG_BTSOCKET_RFCOMM_SESSION_DISCONNECTING:
 			/*
 			 * Do not have to do anything here. We can get here
 			 * when L2CAP connection was terminated or we have 
 			 * received DISC on multiplexor channel
 			 */
 			break;
 
 		case NG_BTSOCKET_RFCOMM_SESSION_OPEN:
 			/* Send DISC on multiplexor channel */
 			error = ng_btsocket_rfcomm_send_command(s,
 					RFCOMM_FRAME_DISC, 0);
 			if (error == 0) {
 				s->state = NG_BTSOCKET_RFCOMM_SESSION_DISCONNECTING;
 				break;
 			}
 			/* FALL THROUGH */
 
 		case NG_BTSOCKET_RFCOMM_SESSION_CONNECTING:
 		case NG_BTSOCKET_RFCOMM_SESSION_CONNECTED:
 			s->state = NG_BTSOCKET_RFCOMM_SESSION_CLOSED;
 			break;
 
 /*		case NG_BTSOCKET_RFCOMM_SESSION_LISTENING: */
 		default:
 			panic("%s: Invalid session state=%d, flags=%#x\n",
 				__func__, s->state, s->flags);
 			break;
 		}
 
 		ng_btsocket_rfcomm_task_wakeup();
 	}
 } /* ng_btsocket_rfcomm_pcb_kill */
 
 /*
  * Look for given dlci for given RFCOMM session. Caller must hold s->session_mtx
  */
 
 static ng_btsocket_rfcomm_pcb_p
 ng_btsocket_rfcomm_pcb_by_dlci(ng_btsocket_rfcomm_session_p s, int dlci)
 {
 	ng_btsocket_rfcomm_pcb_p	pcb = NULL;
 
 	mtx_assert(&s->session_mtx, MA_OWNED);
 
 	LIST_FOREACH(pcb, &s->dlcs, session_next)
 		if (pcb->dlci == dlci)
 			break;
 
 	return (pcb);
 } /* ng_btsocket_rfcomm_pcb_by_dlci */
 
 /*
  * Look for socket that listens on given src address and given channel
  */
 
 static ng_btsocket_rfcomm_pcb_p
 ng_btsocket_rfcomm_pcb_listener(bdaddr_p src, int channel)
 {
 	ng_btsocket_rfcomm_pcb_p	pcb = NULL, pcb1 = NULL;
 
 	mtx_lock(&ng_btsocket_rfcomm_sockets_mtx);
 
 	LIST_FOREACH(pcb, &ng_btsocket_rfcomm_sockets, next) {
 		if (pcb->channel != channel ||
 		    !(pcb->so->so_options & SO_ACCEPTCONN))
 			continue;
 
 		if (bcmp(&pcb->src, src, sizeof(*src)) == 0)
 			break;
 
 		if (bcmp(&pcb->src, NG_HCI_BDADDR_ANY, sizeof(bdaddr_t)) == 0)
 			pcb1 = pcb;
 	}
 
 	mtx_unlock(&ng_btsocket_rfcomm_sockets_mtx);
 
 	return ((pcb != NULL)? pcb : pcb1);
 } /* ng_btsocket_rfcomm_pcb_listener */
 
 /*****************************************************************************
  *****************************************************************************
  **                              Misc. functions 
  *****************************************************************************
  *****************************************************************************/
 
 /*
  *  Set timeout. Caller MUST hold pcb_mtx
  */
 
 static void
 ng_btsocket_rfcomm_timeout(ng_btsocket_rfcomm_pcb_p pcb)
 {
 	mtx_assert(&pcb->pcb_mtx, MA_OWNED);
 
 	if (!(pcb->flags & NG_BTSOCKET_RFCOMM_DLC_TIMO)) {
 		pcb->flags |= NG_BTSOCKET_RFCOMM_DLC_TIMO;
 		pcb->flags &= ~NG_BTSOCKET_RFCOMM_DLC_TIMEDOUT;
 		pcb->timo = timeout(ng_btsocket_rfcomm_process_timeout, pcb,
 					ng_btsocket_rfcomm_timo * hz);
 	} else
 		panic("%s: Duplicated socket timeout?!\n", __func__);
 } /* ng_btsocket_rfcomm_timeout */
 
 /*
  *  Unset pcb timeout. Caller MUST hold pcb_mtx
  */
 
 static void
 ng_btsocket_rfcomm_untimeout(ng_btsocket_rfcomm_pcb_p pcb)
 {
 	mtx_assert(&pcb->pcb_mtx, MA_OWNED);
 
 	if (pcb->flags & NG_BTSOCKET_RFCOMM_DLC_TIMO) {
 		untimeout(ng_btsocket_rfcomm_process_timeout, pcb, pcb->timo);
 		pcb->flags &= ~NG_BTSOCKET_RFCOMM_DLC_TIMO;
 		pcb->flags &= ~NG_BTSOCKET_RFCOMM_DLC_TIMEDOUT;
 	} else
 		panic("%s: No socket timeout?!\n", __func__);
 } /* ng_btsocket_rfcomm_timeout */
 
 /*
  * Process pcb timeout
  */
 
 static void
 ng_btsocket_rfcomm_process_timeout(void *xpcb)
 {
 	ng_btsocket_rfcomm_pcb_p	pcb = (ng_btsocket_rfcomm_pcb_p) xpcb;
 
 	mtx_lock(&pcb->pcb_mtx);
 
 	NG_BTSOCKET_RFCOMM_INFO(
 "%s: Timeout, so=%p, dlci=%d, state=%d, flags=%#x\n",
 		__func__, pcb->so, pcb->dlci, pcb->state, pcb->flags);
 
 	pcb->flags &= ~NG_BTSOCKET_RFCOMM_DLC_TIMO;
 	pcb->flags |= NG_BTSOCKET_RFCOMM_DLC_TIMEDOUT;
 
 	switch (pcb->state) {
 	case NG_BTSOCKET_RFCOMM_DLC_CONFIGURING:
 	case NG_BTSOCKET_RFCOMM_DLC_CONNECTING:
 		pcb->state = NG_BTSOCKET_RFCOMM_DLC_DISCONNECTING;
 		break;
 
 	case NG_BTSOCKET_RFCOMM_DLC_W4_CONNECT:
 	case NG_BTSOCKET_RFCOMM_DLC_DISCONNECTING:
 		break;
 
 	default:
 		panic(
 "%s: DLC timeout in invalid state, dlci=%d, state=%d, flags=%#x\n",
 			__func__, pcb->dlci, pcb->state, pcb->flags);
 		break;
 	}
 
 	ng_btsocket_rfcomm_task_wakeup();
 
 	mtx_unlock(&pcb->pcb_mtx);
 } /* ng_btsocket_rfcomm_process_timeout */
 
 /*
  * Get up to length bytes from the socket buffer
  */
 
 static struct mbuf *
 ng_btsocket_rfcomm_prepare_packet(struct sockbuf *sb, int length)
 {
 	struct mbuf	*top = NULL, *m = NULL, *n = NULL, *nextpkt = NULL;
 	int		 mlen, noff, len;
 
 	MGETHDR(top, M_NOWAIT, MT_DATA);
 	if (top == NULL)
 		return (NULL);
 
 	top->m_pkthdr.len = length;
 	top->m_len = 0;
 	mlen = MHLEN;
 
 	m = top;
 	n = sb->sb_mb;
 	nextpkt = n->m_nextpkt;
 	noff = 0;
 
 	while (length > 0 && n != NULL) {
 		len = min(mlen - m->m_len, n->m_len - noff);
 		if (len > length)
 			len = length;
 
 		bcopy(mtod(n, caddr_t)+noff, mtod(m, caddr_t)+m->m_len, len);
 		m->m_len += len;
 		noff += len;
 		length -= len;
 
 		if (length > 0 && m->m_len == mlen) {
 			MGET(m->m_next, M_NOWAIT, MT_DATA);
 			if (m->m_next == NULL) {
 				NG_FREE_M(top);
 				return (NULL);
 			}
 
 			m = m->m_next;
 			m->m_len = 0;
 			mlen = MLEN;
 		}
 
 		if (noff == n->m_len) {
 			noff = 0;
 			n = n->m_next;
 
 			if (n == NULL)
 				n = nextpkt;
 
 			nextpkt = (n != NULL)? n->m_nextpkt : NULL;
 		}
 	}
 
 	if (length < 0)
 		panic("%s: length=%d\n", __func__, length);
 	if (length > 0 && n == NULL)
 		panic("%s: bogus length=%d, n=%p\n", __func__, length, n);
 
 	return (top);
 } /* ng_btsocket_rfcomm_prepare_packet */
 
Index: projects/release-debugdist/sys/netgraph/bluetooth/socket/ng_btsocket_sco.c
===================================================================
--- projects/release-debugdist/sys/netgraph/bluetooth/socket/ng_btsocket_sco.c	(revision 267343)
+++ projects/release-debugdist/sys/netgraph/bluetooth/socket/ng_btsocket_sco.c	(revision 267344)
@@ -1,1983 +1,1987 @@
 /*
  * ng_btsocket_sco.c
  */
 
 /*-
  * Copyright (c) 2001-2002 Maksim Yevmenkin <m_evmenkin@yahoo.com>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
  * $Id: ng_btsocket_sco.c,v 1.2 2005/10/31 18:08:51 max Exp $
  * $FreeBSD$
  */
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/bitstring.h>
 #include <sys/domain.h>
 #include <sys/endian.h>
 #include <sys/errno.h>
 #include <sys/filedesc.h>
 #include <sys/ioccom.h>
 #include <sys/kernel.h>
 #include <sys/lock.h>
 #include <sys/malloc.h>
 #include <sys/mbuf.h>
 #include <sys/mutex.h>
 #include <sys/protosw.h>
 #include <sys/queue.h>
 #include <sys/socket.h>
 #include <sys/socketvar.h>
 #include <sys/sysctl.h>
 #include <sys/taskqueue.h>
 
 #include <net/vnet.h>
 
 #include <netgraph/ng_message.h>
 #include <netgraph/netgraph.h>
 #include <netgraph/bluetooth/include/ng_bluetooth.h>
 #include <netgraph/bluetooth/include/ng_hci.h>
 #include <netgraph/bluetooth/include/ng_l2cap.h>
 #include <netgraph/bluetooth/include/ng_btsocket.h>
 #include <netgraph/bluetooth/include/ng_btsocket_sco.h>
 
 /* MALLOC define */
 #ifdef NG_SEPARATE_MALLOC
 static MALLOC_DEFINE(M_NETGRAPH_BTSOCKET_SCO, "netgraph_btsocks_sco",
 		"Netgraph Bluetooth SCO sockets");
 #else
 #define M_NETGRAPH_BTSOCKET_SCO M_NETGRAPH
 #endif /* NG_SEPARATE_MALLOC */
 
 /* Netgraph node methods */
 static ng_constructor_t	ng_btsocket_sco_node_constructor;
 static ng_rcvmsg_t	ng_btsocket_sco_node_rcvmsg;
 static ng_shutdown_t	ng_btsocket_sco_node_shutdown;
 static ng_newhook_t	ng_btsocket_sco_node_newhook;
 static ng_connect_t	ng_btsocket_sco_node_connect;
 static ng_rcvdata_t	ng_btsocket_sco_node_rcvdata;
 static ng_disconnect_t	ng_btsocket_sco_node_disconnect;
 
 static void		ng_btsocket_sco_input   (void *, int);
 static void		ng_btsocket_sco_rtclean (void *, int);
 
 /* Netgraph type descriptor */
 static struct ng_type	typestruct = {
 	.version =	NG_ABI_VERSION,
 	.name =		NG_BTSOCKET_SCO_NODE_TYPE,
 	.constructor =	ng_btsocket_sco_node_constructor,
 	.rcvmsg =	ng_btsocket_sco_node_rcvmsg,
 	.shutdown =	ng_btsocket_sco_node_shutdown,
 	.newhook =	ng_btsocket_sco_node_newhook,
 	.connect =	ng_btsocket_sco_node_connect,
 	.rcvdata =	ng_btsocket_sco_node_rcvdata,
 	.disconnect =	ng_btsocket_sco_node_disconnect,
 };
 
 /* Globals */
 static u_int32_t				ng_btsocket_sco_debug_level;
 static node_p					ng_btsocket_sco_node;
 static struct ng_bt_itemq			ng_btsocket_sco_queue;
 static struct mtx				ng_btsocket_sco_queue_mtx;
 static struct task				ng_btsocket_sco_queue_task;
 static struct mtx				ng_btsocket_sco_sockets_mtx;
 static LIST_HEAD(, ng_btsocket_sco_pcb)		ng_btsocket_sco_sockets;
 static LIST_HEAD(, ng_btsocket_sco_rtentry)	ng_btsocket_sco_rt;
 static struct mtx				ng_btsocket_sco_rt_mtx;
 static struct task				ng_btsocket_sco_rt_task;
 static struct timeval				ng_btsocket_sco_lasttime;
 static int					ng_btsocket_sco_curpps;
 
 /* Sysctl tree */
 SYSCTL_DECL(_net_bluetooth_sco_sockets);
 static SYSCTL_NODE(_net_bluetooth_sco_sockets, OID_AUTO, seq, CTLFLAG_RW,
 	0, "Bluetooth SEQPACKET SCO sockets family");
 SYSCTL_UINT(_net_bluetooth_sco_sockets_seq, OID_AUTO, debug_level,
 	CTLFLAG_RW,
 	&ng_btsocket_sco_debug_level, NG_BTSOCKET_WARN_LEVEL,
 	"Bluetooth SEQPACKET SCO sockets debug level");
 SYSCTL_UINT(_net_bluetooth_sco_sockets_seq, OID_AUTO, queue_len,
 	CTLFLAG_RD,
 	&ng_btsocket_sco_queue.len, 0,
 	"Bluetooth SEQPACKET SCO sockets input queue length");
 SYSCTL_UINT(_net_bluetooth_sco_sockets_seq, OID_AUTO, queue_maxlen,
 	CTLFLAG_RD,
 	&ng_btsocket_sco_queue.maxlen, 0,
 	"Bluetooth SEQPACKET SCO sockets input queue max. length");
 SYSCTL_UINT(_net_bluetooth_sco_sockets_seq, OID_AUTO, queue_drops,
 	CTLFLAG_RD,
 	&ng_btsocket_sco_queue.drops, 0,
 	"Bluetooth SEQPACKET SCO sockets input queue drops");
 
 /* Debug */
 #define NG_BTSOCKET_SCO_INFO \
 	if (ng_btsocket_sco_debug_level >= NG_BTSOCKET_INFO_LEVEL && \
 	    ppsratecheck(&ng_btsocket_sco_lasttime, &ng_btsocket_sco_curpps, 1)) \
 		printf
 
 #define NG_BTSOCKET_SCO_WARN \
 	if (ng_btsocket_sco_debug_level >= NG_BTSOCKET_WARN_LEVEL && \
 	    ppsratecheck(&ng_btsocket_sco_lasttime, &ng_btsocket_sco_curpps, 1)) \
 		printf
 
 #define NG_BTSOCKET_SCO_ERR \
 	if (ng_btsocket_sco_debug_level >= NG_BTSOCKET_ERR_LEVEL && \
 	    ppsratecheck(&ng_btsocket_sco_lasttime, &ng_btsocket_sco_curpps, 1)) \
 		printf
 
 #define NG_BTSOCKET_SCO_ALERT \
 	if (ng_btsocket_sco_debug_level >= NG_BTSOCKET_ALERT_LEVEL && \
 	    ppsratecheck(&ng_btsocket_sco_lasttime, &ng_btsocket_sco_curpps, 1)) \
 		printf
 
 /* 
  * Netgraph message processing routines
  */
 
 static int ng_btsocket_sco_process_lp_con_cfm
 	(struct ng_mesg *, ng_btsocket_sco_rtentry_p);
 static int ng_btsocket_sco_process_lp_con_ind
 	(struct ng_mesg *, ng_btsocket_sco_rtentry_p);
 static int ng_btsocket_sco_process_lp_discon_ind
 	(struct ng_mesg *, ng_btsocket_sco_rtentry_p);
 
 /*
  * Send LP messages to the lower layer
  */
 
 static int  ng_btsocket_sco_send_lp_con_req
 	(ng_btsocket_sco_pcb_p);
 static int  ng_btsocket_sco_send_lp_con_rsp
 	(ng_btsocket_sco_rtentry_p, bdaddr_p, int);
 static int  ng_btsocket_sco_send_lp_discon_req
 	(ng_btsocket_sco_pcb_p);
 
 static int ng_btsocket_sco_send2
 	(ng_btsocket_sco_pcb_p);
 
 /* 
  * Timeout processing routines
  */
 
 static void ng_btsocket_sco_timeout         (ng_btsocket_sco_pcb_p);
 static void ng_btsocket_sco_untimeout       (ng_btsocket_sco_pcb_p);
 static void ng_btsocket_sco_process_timeout (void *);
 
 /* 
  * Other stuff 
  */
 
 static ng_btsocket_sco_pcb_p	ng_btsocket_sco_pcb_by_addr(bdaddr_p);
 static ng_btsocket_sco_pcb_p	ng_btsocket_sco_pcb_by_handle(bdaddr_p, int);
 static ng_btsocket_sco_pcb_p	ng_btsocket_sco_pcb_by_addrs(bdaddr_p, bdaddr_p);
 
 #define ng_btsocket_sco_wakeup_input_task() \
 	taskqueue_enqueue(taskqueue_swi, &ng_btsocket_sco_queue_task)
 
 #define ng_btsocket_sco_wakeup_route_task() \
 	taskqueue_enqueue(taskqueue_swi, &ng_btsocket_sco_rt_task)
 
 /*****************************************************************************
  *****************************************************************************
  **                        Netgraph node interface
  *****************************************************************************
  *****************************************************************************/
 
 /*
  * Netgraph node constructor. Do not allow to create node of this type.
  */
 
 static int
 ng_btsocket_sco_node_constructor(node_p node)
 {
 	return (EINVAL);
 } /* ng_btsocket_sco_node_constructor */
 
 /*
  * Do local shutdown processing. Let old node go and create new fresh one.
  */
 
 static int
 ng_btsocket_sco_node_shutdown(node_p node)
 {
 	int	error = 0;
 
 	NG_NODE_UNREF(node);
 
 	/* Create new node */
 	error = ng_make_node_common(&typestruct, &ng_btsocket_sco_node);
 	if (error != 0) {
 		NG_BTSOCKET_SCO_ALERT(
 "%s: Could not create Netgraph node, error=%d\n", __func__, error);
 
 		ng_btsocket_sco_node = NULL;
 
 		return (error);
 	}
 
 	error = ng_name_node(ng_btsocket_sco_node,
 				NG_BTSOCKET_SCO_NODE_TYPE);
 	if (error != 0) {
 		NG_BTSOCKET_SCO_ALERT(
 "%s: Could not name Netgraph node, error=%d\n", __func__, error);
 
 		NG_NODE_UNREF(ng_btsocket_sco_node);
 		ng_btsocket_sco_node = NULL;
 
 		return (error);
 	}
 		
 	return (0);
 } /* ng_btsocket_sco_node_shutdown */
 
 /*
  * We allow any hook to be connected to the node.
  */
 
 static int
 ng_btsocket_sco_node_newhook(node_p node, hook_p hook, char const *name)
 {
 	return (0);
 } /* ng_btsocket_sco_node_newhook */
 
 /* 
  * Just say "YEP, that's OK by me!"
  */
 
 static int
 ng_btsocket_sco_node_connect(hook_p hook)
 {
 	NG_HOOK_SET_PRIVATE(hook, NULL);
 	NG_HOOK_REF(hook); /* Keep extra reference to the hook */
 
 #if 0
 	NG_HOOK_FORCE_QUEUE(NG_HOOK_PEER(hook));
 	NG_HOOK_FORCE_QUEUE(hook);
 #endif
 
 	return (0);
 } /* ng_btsocket_sco_node_connect */
 
 /*
  * Hook disconnection. Schedule route cleanup task
  */
 
 static int
 ng_btsocket_sco_node_disconnect(hook_p hook)
 {
 	/*
 	 * If hook has private information than we must have this hook in
 	 * the routing table and must schedule cleaning for the routing table.
 	 * Otherwise hook was connected but we never got "hook_info" message,
 	 * so we have never added this hook to the routing table and it save
 	 * to just delete it.
 	 */
 
 	if (NG_HOOK_PRIVATE(hook) != NULL)
 		return (ng_btsocket_sco_wakeup_route_task());
 
 	NG_HOOK_UNREF(hook); /* Remove extra reference */
 
 	return (0);
 } /* ng_btsocket_sco_node_disconnect */
 
 /*
  * Process incoming messages 
  */
 
 static int
 ng_btsocket_sco_node_rcvmsg(node_p node, item_p item, hook_p hook)
 {
 	struct ng_mesg	*msg = NGI_MSG(item); /* item still has message */
 	int		 error = 0;
 
 	if (msg != NULL && msg->header.typecookie == NGM_HCI_COOKIE) {
 		mtx_lock(&ng_btsocket_sco_queue_mtx);
 		if (NG_BT_ITEMQ_FULL(&ng_btsocket_sco_queue)) {
 			NG_BTSOCKET_SCO_ERR(
 "%s: Input queue is full (msg)\n", __func__);
 
 			NG_BT_ITEMQ_DROP(&ng_btsocket_sco_queue);
 			NG_FREE_ITEM(item);
 			error = ENOBUFS;
 		} else {
 			if (hook != NULL) {
 				NG_HOOK_REF(hook);
 				NGI_SET_HOOK(item, hook);
 			}
 
 			NG_BT_ITEMQ_ENQUEUE(&ng_btsocket_sco_queue, item);
 			error = ng_btsocket_sco_wakeup_input_task();
 		}
 		mtx_unlock(&ng_btsocket_sco_queue_mtx);
 	} else {
 		NG_FREE_ITEM(item);
 		error = EINVAL;
 	}
 
 	return (error);
 } /* ng_btsocket_sco_node_rcvmsg */
 
 /*
  * Receive data on a hook
  */
 
 static int
 ng_btsocket_sco_node_rcvdata(hook_p hook, item_p item)
 {
 	int	error = 0;
 
 	mtx_lock(&ng_btsocket_sco_queue_mtx);
 	if (NG_BT_ITEMQ_FULL(&ng_btsocket_sco_queue)) {
 		NG_BTSOCKET_SCO_ERR(
 "%s: Input queue is full (data)\n", __func__);
 
 		NG_BT_ITEMQ_DROP(&ng_btsocket_sco_queue);
 		NG_FREE_ITEM(item);
 		error = ENOBUFS;
 	} else {
 		NG_HOOK_REF(hook);
 		NGI_SET_HOOK(item, hook);
 
 		NG_BT_ITEMQ_ENQUEUE(&ng_btsocket_sco_queue, item);
 		error = ng_btsocket_sco_wakeup_input_task();
 	}
 	mtx_unlock(&ng_btsocket_sco_queue_mtx);
 
 	return (error);
 } /* ng_btsocket_sco_node_rcvdata */
 
 /*
  * Process LP_ConnectCfm event from the lower layer protocol
  */
 
 static int
 ng_btsocket_sco_process_lp_con_cfm(struct ng_mesg *msg,
 		ng_btsocket_sco_rtentry_p rt)
 {
 	ng_hci_lp_con_cfm_ep	*ep = NULL;
 	ng_btsocket_sco_pcb_t	*pcb = NULL;
 	int			 error = 0;
 
 	if (msg->header.arglen != sizeof(*ep))
 		return (EMSGSIZE);
 
 	ep = (ng_hci_lp_con_cfm_ep *)(msg->data);
 
 	mtx_lock(&ng_btsocket_sco_sockets_mtx);
 
 	/* Look for the socket with the token */
 	pcb = ng_btsocket_sco_pcb_by_addrs(&rt->src, &ep->bdaddr);
 	if (pcb == NULL) {
 		mtx_unlock(&ng_btsocket_sco_sockets_mtx);
 		return (ENOENT);
 	}
 
 	/* pcb is locked */
 
 	NG_BTSOCKET_SCO_INFO(
 "%s: Got LP_ConnectCfm response, src bdaddr=%x:%x:%x:%x:%x:%x, " \
 "dst bdaddr=%x:%x:%x:%x:%x:%x, status=%d, handle=%d, state=%d\n",
 		__func__,
 		pcb->src.b[5], pcb->src.b[4], pcb->src.b[3],
 		pcb->src.b[2], pcb->src.b[1], pcb->src.b[0],
 		pcb->dst.b[5], pcb->dst.b[4], pcb->dst.b[3],
 		pcb->dst.b[2], pcb->dst.b[1], pcb->dst.b[0],
 		ep->status, ep->con_handle, pcb->state);
 
 	if (pcb->state != NG_BTSOCKET_SCO_CONNECTING) {
 		mtx_unlock(&pcb->pcb_mtx);
 		mtx_unlock(&ng_btsocket_sco_sockets_mtx);
 
 		return (ENOENT);
 	}
 
 	ng_btsocket_sco_untimeout(pcb);
 
 	if (ep->status == 0) {
 		/*
 		 * Connection is open. Update connection handle and
 		 * socket state
 		 */
 
 		pcb->con_handle = ep->con_handle; 
 		pcb->state = NG_BTSOCKET_SCO_OPEN;
 		soisconnected(pcb->so); 
 	} else {
 		/*
 		 * We have failed to open connection, so disconnect the socket
 		 */
 
 		pcb->so->so_error = ECONNREFUSED; /* XXX convert status ??? */
 		pcb->state = NG_BTSOCKET_SCO_CLOSED;
 		soisdisconnected(pcb->so); 
 	}
 
 	mtx_unlock(&pcb->pcb_mtx);
 	mtx_unlock(&ng_btsocket_sco_sockets_mtx);
 
 	return (error);
 } /* ng_btsocket_sco_process_lp_con_cfm */
 
 /*
  * Process LP_ConnectInd indicator. Find socket that listens on address.
  * Find exact or closest match.
  */
 
 static int
 ng_btsocket_sco_process_lp_con_ind(struct ng_mesg *msg,
 		ng_btsocket_sco_rtentry_p rt)
 {
 	ng_hci_lp_con_ind_ep	*ep = NULL;
 	ng_btsocket_sco_pcb_t	*pcb = NULL, *pcb1 = NULL;
 	int			 error = 0;
 	u_int16_t		 status = 0;
 
 	if (msg->header.arglen != sizeof(*ep))
 		return (EMSGSIZE);
 
 	ep = (ng_hci_lp_con_ind_ep *)(msg->data);
 
 	NG_BTSOCKET_SCO_INFO(
 "%s: Got LP_ConnectInd indicator, src bdaddr=%x:%x:%x:%x:%x:%x, " \
 "dst bdaddr=%x:%x:%x:%x:%x:%x\n",
 		__func__,
 		rt->src.b[5], rt->src.b[4], rt->src.b[3],
 		rt->src.b[2], rt->src.b[1], rt->src.b[0],
 		ep->bdaddr.b[5], ep->bdaddr.b[4], ep->bdaddr.b[3],
 		ep->bdaddr.b[2], ep->bdaddr.b[1], ep->bdaddr.b[0]);
 
 	mtx_lock(&ng_btsocket_sco_sockets_mtx);
 
 	pcb = ng_btsocket_sco_pcb_by_addr(&rt->src);
 	if (pcb != NULL) {
 		struct socket	*so1 = NULL;
 
 		/* pcb is locked */
 
 		/*
 		 * First check the pending connections queue and if we have
 		 * space then create new socket and set proper source address.
 		 */
 
 		if (pcb->so->so_qlen <= pcb->so->so_qlimit) {
 			CURVNET_SET(pcb->so->so_vnet);
 			so1 = sonewconn(pcb->so, 0);
 			CURVNET_RESTORE();
 		}
 
 		if (so1 == NULL) {
 			status = 0x0d; /* Rejected due to limited resources */
 			goto respond;
 		}
 
 		/*
 		 * If we got here than we have created new socket. So complete 
 		 * connection. If we we listening on specific address then copy 
 		 * source address from listening socket, otherwise copy source 
 		 * address from hook's routing information.
 		 */
 
 		pcb1 = so2sco_pcb(so1);
 		KASSERT((pcb1 != NULL),
 ("%s: pcb1 == NULL\n", __func__));
 
  		mtx_lock(&pcb1->pcb_mtx);
 
 		if (bcmp(&pcb->src, NG_HCI_BDADDR_ANY, sizeof(pcb->src)) != 0)
 			bcopy(&pcb->src, &pcb1->src, sizeof(pcb1->src));
 		else
 			bcopy(&rt->src, &pcb1->src, sizeof(pcb1->src));
 
 		pcb1->flags &= ~NG_BTSOCKET_SCO_CLIENT;
 
 		bcopy(&ep->bdaddr, &pcb1->dst, sizeof(pcb1->dst));
 		pcb1->rt = rt;
 	} else
 		/* Nobody listens on requested BDADDR */
 		status = 0x1f; /* Unspecified Error */
 
 respond:
 	error = ng_btsocket_sco_send_lp_con_rsp(rt, &ep->bdaddr, status);
 	if (pcb1 != NULL) {
 		if (error != 0) {
 			pcb1->so->so_error = error;
 			pcb1->state = NG_BTSOCKET_SCO_CLOSED;
 			soisdisconnected(pcb1->so);
 		} else {
 			pcb1->state = NG_BTSOCKET_SCO_CONNECTING;
 			soisconnecting(pcb1->so);
 
 			ng_btsocket_sco_timeout(pcb1);
 		}
 
 		mtx_unlock(&pcb1->pcb_mtx);
 	}
 
 	if (pcb != NULL)
 		mtx_unlock(&pcb->pcb_mtx);
 
 	mtx_unlock(&ng_btsocket_sco_sockets_mtx);
 
 	return (error);
 } /* ng_btsocket_sco_process_lp_con_ind */
 
 /*
  * Process LP_DisconnectInd indicator
  */
 
 static int
 ng_btsocket_sco_process_lp_discon_ind(struct ng_mesg *msg,
 		ng_btsocket_sco_rtentry_p rt)
 {
 	ng_hci_lp_discon_ind_ep	*ep = NULL;
 	ng_btsocket_sco_pcb_t	*pcb = NULL;
 
 	/* Check message */
 	if (msg->header.arglen != sizeof(*ep))
 		return (EMSGSIZE);
 
 	ep = (ng_hci_lp_discon_ind_ep *)(msg->data);
 
 	mtx_lock(&ng_btsocket_sco_sockets_mtx);
 
 	/* Look for the socket with given channel ID */
 	pcb = ng_btsocket_sco_pcb_by_handle(&rt->src, ep->con_handle);
 	if (pcb == NULL) {
 		mtx_unlock(&ng_btsocket_sco_sockets_mtx);
 		return (0);
 	}
 
 	/*
 	 * Disconnect the socket. If there was any pending request we can
 	 * not do anything here anyway.
 	 */
 
 	/* pcb is locked */
 
        	NG_BTSOCKET_SCO_INFO(
 "%s: Got LP_DisconnectInd indicator, src bdaddr=%x:%x:%x:%x:%x:%x, " \
 "dst bdaddr=%x:%x:%x:%x:%x:%x, handle=%d, state=%d\n",
 		__func__,
 		pcb->src.b[5], pcb->src.b[4], pcb->src.b[3],
 		pcb->src.b[2], pcb->src.b[1], pcb->src.b[0],
 		pcb->dst.b[5], pcb->dst.b[4], pcb->dst.b[3],
 		pcb->dst.b[2], pcb->dst.b[1], pcb->dst.b[0],
 		pcb->con_handle, pcb->state);
 
 	if (pcb->flags & NG_BTSOCKET_SCO_TIMO)
 		ng_btsocket_sco_untimeout(pcb);
 
 	pcb->state = NG_BTSOCKET_SCO_CLOSED;
 	soisdisconnected(pcb->so);
 
 	mtx_unlock(&pcb->pcb_mtx);
 	mtx_unlock(&ng_btsocket_sco_sockets_mtx);
 
 	return (0);
 } /* ng_btsocket_sco_process_lp_discon_ind */
 
 /*
  * Send LP_ConnectReq request
  */
 
 static int
 ng_btsocket_sco_send_lp_con_req(ng_btsocket_sco_pcb_p pcb)
 {
 	struct ng_mesg		*msg = NULL;
 	ng_hci_lp_con_req_ep	*ep = NULL;
 	int			 error = 0;
 
 	mtx_assert(&pcb->pcb_mtx, MA_OWNED);
 
 	if (pcb->rt == NULL || 
 	    pcb->rt->hook == NULL || NG_HOOK_NOT_VALID(pcb->rt->hook))
 		return (ENETDOWN); 
 
 	NG_MKMESSAGE(msg, NGM_HCI_COOKIE, NGM_HCI_LP_CON_REQ,
 		sizeof(*ep), M_NOWAIT);
 	if (msg == NULL)
 		return (ENOMEM);
 
 	ep = (ng_hci_lp_con_req_ep *)(msg->data);
 	ep->link_type = NG_HCI_LINK_SCO;
 	bcopy(&pcb->dst, &ep->bdaddr, sizeof(ep->bdaddr));
 
 	NG_SEND_MSG_HOOK(error, ng_btsocket_sco_node, msg, pcb->rt->hook, 0);
 
 	return (error);
 } /* ng_btsocket_sco_send_lp_con_req */
 
 /*
  * Send LP_ConnectRsp response
  */
 
 static int
 ng_btsocket_sco_send_lp_con_rsp(ng_btsocket_sco_rtentry_p rt, bdaddr_p dst, int status)
 {
 	struct ng_mesg		*msg = NULL;
 	ng_hci_lp_con_rsp_ep	*ep = NULL;
 	int			 error = 0;
 
 	if (rt == NULL || rt->hook == NULL || NG_HOOK_NOT_VALID(rt->hook))
 		return (ENETDOWN); 
 
 	NG_MKMESSAGE(msg, NGM_HCI_COOKIE, NGM_HCI_LP_CON_RSP,
 		sizeof(*ep), M_NOWAIT);
 	if (msg == NULL)
 		return (ENOMEM);
 
 	ep = (ng_hci_lp_con_rsp_ep *)(msg->data);
 	ep->status = status;
 	ep->link_type = NG_HCI_LINK_SCO;
 	bcopy(dst, &ep->bdaddr, sizeof(ep->bdaddr));
 
 	NG_SEND_MSG_HOOK(error, ng_btsocket_sco_node, msg, rt->hook, 0);
 
 	return (error);
 } /* ng_btsocket_sco_send_lp_con_rsp */
 
 /*
  * Send LP_DisconReq request
  */
 
 static int
 ng_btsocket_sco_send_lp_discon_req(ng_btsocket_sco_pcb_p pcb)
 {
 	struct ng_mesg		*msg = NULL;
 	ng_hci_lp_discon_req_ep	*ep = NULL;
 	int			 error = 0;
 
 	mtx_assert(&pcb->pcb_mtx, MA_OWNED);
 
 	if (pcb->rt == NULL || 
 	    pcb->rt->hook == NULL || NG_HOOK_NOT_VALID(pcb->rt->hook))
 		return (ENETDOWN); 
 
 	NG_MKMESSAGE(msg, NGM_HCI_COOKIE, NGM_HCI_LP_DISCON_REQ,
 		sizeof(*ep), M_NOWAIT);
 	if (msg == NULL)
 		return (ENOMEM);
 
 	ep = (ng_hci_lp_discon_req_ep *)(msg->data);
 	ep->con_handle = pcb->con_handle;
 	ep->reason = 0x13; /* User Ended Connection */
 
 	NG_SEND_MSG_HOOK(error, ng_btsocket_sco_node, msg, pcb->rt->hook, 0);
 
 	return (error);
 } /* ng_btsocket_sco_send_lp_discon_req */
 
 /*****************************************************************************
  *****************************************************************************
  **                              Socket interface
  *****************************************************************************
  *****************************************************************************/
 
 /*
  * SCO sockets data input routine
  */
 
 static void
 ng_btsocket_sco_data_input(struct mbuf *m, hook_p hook)
 {
 	ng_hci_scodata_pkt_t		*hdr = NULL;
 	ng_btsocket_sco_pcb_t		*pcb = NULL;
 	ng_btsocket_sco_rtentry_t	*rt = NULL;
 	u_int16_t			 con_handle;
 
 	if (hook == NULL) {
 		NG_BTSOCKET_SCO_ALERT(
 "%s: Invalid source hook for SCO data packet\n", __func__);
 		goto drop;
 	}
 
 	rt = (ng_btsocket_sco_rtentry_t *) NG_HOOK_PRIVATE(hook);
 	if (rt == NULL) {
 		NG_BTSOCKET_SCO_ALERT(
 "%s: Could not find out source bdaddr for SCO data packet\n", __func__);
 		goto drop;
 	}
 
 	/* Make sure we can access header */
 	if (m->m_pkthdr.len < sizeof(*hdr)) {
 		NG_BTSOCKET_SCO_ERR(
 "%s: SCO data packet too small, len=%d\n", __func__, m->m_pkthdr.len);
 		goto drop;
 	}
 
 	if (m->m_len < sizeof(*hdr)) { 
 		m = m_pullup(m, sizeof(*hdr));
 		if (m == NULL)
 			goto drop;
 	}
 
 	/* Strip SCO packet header and verify packet length */
 	hdr = mtod(m, ng_hci_scodata_pkt_t *);
 	m_adj(m, sizeof(*hdr));
 
 	if (hdr->length != m->m_pkthdr.len) {
 		NG_BTSOCKET_SCO_ERR(
 "%s: Bad SCO data packet length, len=%d, length=%d\n",
 			__func__, m->m_pkthdr.len, hdr->length);
 		goto drop;
 	}
 
 	/*
 	 * Now process packet
 	 */
 
 	con_handle = NG_HCI_CON_HANDLE(le16toh(hdr->con_handle));
 
 	NG_BTSOCKET_SCO_INFO(
 "%s: Received SCO data packet: src bdaddr=%x:%x:%x:%x:%x:%x, handle=%d, " \
 "length=%d\n",	__func__,
 		rt->src.b[5], rt->src.b[4], rt->src.b[3],
 		rt->src.b[2], rt->src.b[1], rt->src.b[0],
 		con_handle, hdr->length);
 
 	mtx_lock(&ng_btsocket_sco_sockets_mtx);
 
 	/* Find socket */
 	pcb = ng_btsocket_sco_pcb_by_handle(&rt->src, con_handle);
 	if (pcb == NULL) {
 		mtx_unlock(&ng_btsocket_sco_sockets_mtx);
 		goto drop;
 	}
 
 	/* pcb is locked */
 
 	if (pcb->state != NG_BTSOCKET_SCO_OPEN) {
 		NG_BTSOCKET_SCO_ERR(
 "%s: No connected socket found, src bdaddr=%x:%x:%x:%x:%x:%x, state=%d\n",
 			__func__,
 			rt->src.b[5], rt->src.b[4], rt->src.b[3],
 			rt->src.b[2], rt->src.b[1], rt->src.b[0],
 			pcb->state);
 
 		mtx_unlock(&pcb->pcb_mtx);
 		mtx_unlock(&ng_btsocket_sco_sockets_mtx);
 		goto drop;
 	}
 
 	/* Check if we have enough space in socket receive queue */
 	if (m->m_pkthdr.len > sbspace(&pcb->so->so_rcv)) {
 		NG_BTSOCKET_SCO_ERR(
 "%s: Not enough space in socket receive queue. Dropping SCO data packet, " \
 "src bdaddr=%x:%x:%x:%x:%x:%x, len=%d, space=%ld\n",
 			__func__,
 			rt->src.b[5], rt->src.b[4], rt->src.b[3],
 			rt->src.b[2], rt->src.b[1], rt->src.b[0],
 			m->m_pkthdr.len,
 			sbspace(&pcb->so->so_rcv));
 
 		mtx_unlock(&pcb->pcb_mtx);
 		mtx_unlock(&ng_btsocket_sco_sockets_mtx);
 		goto drop;
 	}
 
 	/* Append packet to the socket receive queue and wakeup */
 	sbappendrecord(&pcb->so->so_rcv, m);
 	m = NULL;
 
 	sorwakeup(pcb->so);
 
 	mtx_unlock(&pcb->pcb_mtx);
 	mtx_unlock(&ng_btsocket_sco_sockets_mtx);
 drop:
 	NG_FREE_M(m); /* checks for m != NULL */
 } /* ng_btsocket_sco_data_input */
 
 /*
  * SCO sockets default message input routine
  */
 
 static void
 ng_btsocket_sco_default_msg_input(struct ng_mesg *msg, hook_p hook)
 {
 	ng_btsocket_sco_rtentry_t	*rt = NULL;
 
 	if (hook == NULL || NG_HOOK_NOT_VALID(hook))
 		return;
 
 	rt = (ng_btsocket_sco_rtentry_t *) NG_HOOK_PRIVATE(hook);
 
 	switch (msg->header.cmd) {
 	case NGM_HCI_NODE_UP: {
 		ng_hci_node_up_ep	*ep = NULL;
 
 		if (msg->header.arglen != sizeof(*ep))
 			break;
 
 		ep = (ng_hci_node_up_ep *)(msg->data);
 		if (bcmp(&ep->bdaddr, NG_HCI_BDADDR_ANY, sizeof(bdaddr_t)) == 0)
 			break;
 
 		if (rt == NULL) {
 			rt = malloc(sizeof(*rt),
 				M_NETGRAPH_BTSOCKET_SCO, M_NOWAIT|M_ZERO);
 			if (rt == NULL)
 				break;
 
 			NG_HOOK_SET_PRIVATE(hook, rt);
 
 			mtx_lock(&ng_btsocket_sco_rt_mtx);
 
 			LIST_INSERT_HEAD(&ng_btsocket_sco_rt, rt, next);
 		} else
 			mtx_lock(&ng_btsocket_sco_rt_mtx);
 
 		bcopy(&ep->bdaddr, &rt->src, sizeof(rt->src));
 		rt->pkt_size = (ep->pkt_size == 0)? 60 : ep->pkt_size;
 		rt->num_pkts = ep->num_pkts;
 		rt->hook = hook;
 
 		mtx_unlock(&ng_btsocket_sco_rt_mtx);
 
 		NG_BTSOCKET_SCO_INFO(
 "%s: Updating hook \"%s\", src bdaddr=%x:%x:%x:%x:%x:%x, pkt_size=%d, " \
 "num_pkts=%d\n",	__func__, NG_HOOK_NAME(hook), 
 			rt->src.b[5], rt->src.b[4], rt->src.b[3], 
 			rt->src.b[2], rt->src.b[1], rt->src.b[0],
 			rt->pkt_size, rt->num_pkts);
 		} break;
 
 	case NGM_HCI_SYNC_CON_QUEUE: {
 		ng_hci_sync_con_queue_ep	*ep = NULL;
 		ng_btsocket_sco_pcb_t		*pcb = NULL;
 
 		if (rt == NULL || msg->header.arglen != sizeof(*ep))
 			break;
 
 		ep = (ng_hci_sync_con_queue_ep *)(msg->data);
 
 		rt->pending -= ep->completed;
 		if (rt->pending < 0) {
 			NG_BTSOCKET_SCO_WARN(
 "%s: Pending packet counter is out of sync! bdaddr=%x:%x:%x:%x:%x:%x, " \
 "handle=%d, pending=%d, completed=%d\n",
 				__func__,
 				rt->src.b[5], rt->src.b[4], rt->src.b[3],
 				rt->src.b[2], rt->src.b[1], rt->src.b[0],
 				ep->con_handle, rt->pending,
 				ep->completed);
 
 			rt->pending = 0;
 		}
 
 		mtx_lock(&ng_btsocket_sco_sockets_mtx);
 
 		/* Find socket */
 		pcb = ng_btsocket_sco_pcb_by_handle(&rt->src, ep->con_handle);
 		if (pcb == NULL) {
 			mtx_unlock(&ng_btsocket_sco_sockets_mtx);
 			break;
 		}
 
 		/* pcb is locked */
 
 		/* Check state */
 		if (pcb->state == NG_BTSOCKET_SCO_OPEN) {
 			/* Remove timeout */
 			ng_btsocket_sco_untimeout(pcb);
 			
 			/* Drop completed packets from the send queue */
 			for (; ep->completed > 0; ep->completed --)
 				sbdroprecord(&pcb->so->so_snd);
 
 			/* Send more if we have any */
 			if (pcb->so->so_snd.sb_cc > 0)
 				if (ng_btsocket_sco_send2(pcb) == 0)
 					ng_btsocket_sco_timeout(pcb);
 
 			/* Wake up writers */
 			sowwakeup(pcb->so);
 		}
 
 		mtx_unlock(&pcb->pcb_mtx);
 		mtx_unlock(&ng_btsocket_sco_sockets_mtx);
 	} break;
 
 	default:
 		NG_BTSOCKET_SCO_WARN(
 "%s: Unknown message, cmd=%d\n", __func__, msg->header.cmd);
 		break;
 	}
 
 	NG_FREE_MSG(msg); /* Checks for msg != NULL */
 } /* ng_btsocket_sco_default_msg_input */
 
 /*
  * SCO sockets LP message input routine
  */
 
 static void
 ng_btsocket_sco_lp_msg_input(struct ng_mesg *msg, hook_p hook)
 {
 	ng_btsocket_sco_rtentry_p	 rt = NULL;
 
 	if (hook == NULL) {
 		NG_BTSOCKET_SCO_ALERT(
 "%s: Invalid source hook for LP message\n", __func__);
 		goto drop;
 	}
 
 	rt = (ng_btsocket_sco_rtentry_p) NG_HOOK_PRIVATE(hook);
 	if (rt == NULL) {
 		NG_BTSOCKET_SCO_ALERT(
 "%s: Could not find out source bdaddr for LP message\n", __func__);
 		goto drop;
 	}
 
 	switch (msg->header.cmd) {
 	case NGM_HCI_LP_CON_CFM: /* Connection Confirmation Event */
 		ng_btsocket_sco_process_lp_con_cfm(msg, rt);
 		break;
 
 	case NGM_HCI_LP_CON_IND: /* Connection Indication Event */
 		ng_btsocket_sco_process_lp_con_ind(msg, rt);
 		break;
 
 	case NGM_HCI_LP_DISCON_IND: /* Disconnection Indication Event */
 		ng_btsocket_sco_process_lp_discon_ind(msg, rt);
 		break;
 
 	/* XXX FIXME add other LP messages */
 
 	default:
 		NG_BTSOCKET_SCO_WARN(
 "%s: Unknown LP message, cmd=%d\n", __func__, msg->header.cmd);
 		break;
 	}
 drop:
 	NG_FREE_MSG(msg);
 } /* ng_btsocket_sco_lp_msg_input */
 
 /*
  * SCO sockets input routine
  */
 
 static void
 ng_btsocket_sco_input(void *context, int pending)
 {
 	item_p	item = NULL;
 	hook_p	hook = NULL;
 
 	for (;;) {
 		mtx_lock(&ng_btsocket_sco_queue_mtx);
 		NG_BT_ITEMQ_DEQUEUE(&ng_btsocket_sco_queue, item);
 		mtx_unlock(&ng_btsocket_sco_queue_mtx);
 
 		if (item == NULL)
 			break;
 
 		NGI_GET_HOOK(item, hook);
 		if (hook != NULL && NG_HOOK_NOT_VALID(hook))
 			goto drop;
 
 		switch(item->el_flags & NGQF_TYPE) {
 		case NGQF_DATA: {
 			struct mbuf     *m = NULL;
 
 			NGI_GET_M(item, m);
 			ng_btsocket_sco_data_input(m, hook);
 			} break;
 
 		case NGQF_MESG: {
 			struct ng_mesg  *msg = NULL;
 
 			NGI_GET_MSG(item, msg);
 
 			switch (msg->header.cmd) {
 			case NGM_HCI_LP_CON_CFM:
 			case NGM_HCI_LP_CON_IND:
 			case NGM_HCI_LP_DISCON_IND:
 			/* XXX FIXME add other LP messages */
 				ng_btsocket_sco_lp_msg_input(msg, hook);
 				break;
 
 			default:
 				ng_btsocket_sco_default_msg_input(msg, hook);
 				break;
 			}
 			} break;
 
 		default:
 			KASSERT(0,
 ("%s: invalid item type=%ld\n", __func__, (item->el_flags & NGQF_TYPE)));
 			break;
 		}
 drop:
 		if (hook != NULL)
 			NG_HOOK_UNREF(hook);
 
 		NG_FREE_ITEM(item);
 	}
 } /* ng_btsocket_sco_input */
 
 /*
  * Route cleanup task. Gets scheduled when hook is disconnected. Here we 
  * will find all sockets that use "invalid" hook and disconnect them.
  */
 
 static void
 ng_btsocket_sco_rtclean(void *context, int pending)
 {
 	ng_btsocket_sco_pcb_p		pcb = NULL, pcb_next = NULL;
 	ng_btsocket_sco_rtentry_p	rt = NULL;
 
 	/*
 	 * First disconnect all sockets that use "invalid" hook
 	 */
 
 	mtx_lock(&ng_btsocket_sco_sockets_mtx);
 
 	for(pcb = LIST_FIRST(&ng_btsocket_sco_sockets); pcb != NULL; ) {
 		mtx_lock(&pcb->pcb_mtx);
 		pcb_next = LIST_NEXT(pcb, next);
 
 		if (pcb->rt != NULL &&
 		    pcb->rt->hook != NULL && NG_HOOK_NOT_VALID(pcb->rt->hook)) {
 			if (pcb->flags & NG_BTSOCKET_SCO_TIMO)
 				ng_btsocket_sco_untimeout(pcb);
 
 			pcb->rt = NULL;
 			pcb->so->so_error = ENETDOWN;
 			pcb->state = NG_BTSOCKET_SCO_CLOSED;
 			soisdisconnected(pcb->so);
 		}
 
 		mtx_unlock(&pcb->pcb_mtx);
 		pcb = pcb_next;
 	}
 
 	mtx_unlock(&ng_btsocket_sco_sockets_mtx);
 
 	/*
 	 * Now cleanup routing table
 	 */
 
 	mtx_lock(&ng_btsocket_sco_rt_mtx);
 
 	for (rt = LIST_FIRST(&ng_btsocket_sco_rt); rt != NULL; ) {
 		ng_btsocket_sco_rtentry_p	rt_next = LIST_NEXT(rt, next);
 
 		if (rt->hook != NULL && NG_HOOK_NOT_VALID(rt->hook)) {
 			LIST_REMOVE(rt, next);
 
 			NG_HOOK_SET_PRIVATE(rt->hook, NULL);
 			NG_HOOK_UNREF(rt->hook); /* Remove extra reference */
 
 			bzero(rt, sizeof(*rt));
 			free(rt, M_NETGRAPH_BTSOCKET_SCO);
 		}
 
 		rt = rt_next;
 	}
 
 	mtx_unlock(&ng_btsocket_sco_rt_mtx);
 } /* ng_btsocket_sco_rtclean */
 
 /*
  * Initialize everything
  */
 
 void
 ng_btsocket_sco_init(void)
 {
 	int	error = 0;
 
+	/* Skip initialization of globals for non-default instances. */
+	if (!IS_DEFAULT_VNET(curvnet))
+		return;
+
 	ng_btsocket_sco_node = NULL;
 	ng_btsocket_sco_debug_level = NG_BTSOCKET_WARN_LEVEL;
 
 	/* Register Netgraph node type */
 	error = ng_newtype(&typestruct);
 	if (error != 0) {
 		NG_BTSOCKET_SCO_ALERT(
 "%s: Could not register Netgraph node type, error=%d\n", __func__, error);
 
                 return;
 	}
 
 	/* Create Netgrapg node */
 	error = ng_make_node_common(&typestruct, &ng_btsocket_sco_node);
 	if (error != 0) {
 		NG_BTSOCKET_SCO_ALERT(
 "%s: Could not create Netgraph node, error=%d\n", __func__, error);
 
 		ng_btsocket_sco_node = NULL;
 
 		return;
 	}
 
 	error = ng_name_node(ng_btsocket_sco_node, NG_BTSOCKET_SCO_NODE_TYPE);
 	if (error != 0) {
 		NG_BTSOCKET_SCO_ALERT(
 "%s: Could not name Netgraph node, error=%d\n", __func__, error);
 
 		NG_NODE_UNREF(ng_btsocket_sco_node);
 		ng_btsocket_sco_node = NULL;
 
 		return;
 	}
 
 	/* Create input queue */
 	NG_BT_ITEMQ_INIT(&ng_btsocket_sco_queue, 300);
 	mtx_init(&ng_btsocket_sco_queue_mtx,
 		"btsocks_sco_queue_mtx", NULL, MTX_DEF);
 	TASK_INIT(&ng_btsocket_sco_queue_task, 0,
 		ng_btsocket_sco_input, NULL);
 
 	/* Create list of sockets */
 	LIST_INIT(&ng_btsocket_sco_sockets);
 	mtx_init(&ng_btsocket_sco_sockets_mtx,
 		"btsocks_sco_sockets_mtx", NULL, MTX_DEF);
 
 	/* Routing table */
 	LIST_INIT(&ng_btsocket_sco_rt);
 	mtx_init(&ng_btsocket_sco_rt_mtx,
 		"btsocks_sco_rt_mtx", NULL, MTX_DEF);
 	TASK_INIT(&ng_btsocket_sco_rt_task, 0,
 		ng_btsocket_sco_rtclean, NULL);
 } /* ng_btsocket_sco_init */
 
 /*
  * Abort connection on socket
  */
 
 void
 ng_btsocket_sco_abort(struct socket *so)
 {
 	so->so_error = ECONNABORTED;
 
 	(void) ng_btsocket_sco_disconnect(so);
 } /* ng_btsocket_sco_abort */
 
 void
 ng_btsocket_sco_close(struct socket *so)
 {
 	(void) ng_btsocket_sco_disconnect(so);
 } /* ng_btsocket_sco_close */
 
 /*
  * Accept connection on socket. Nothing to do here, socket must be connected
  * and ready, so just return peer address and be done with it.
  */
 
 int
 ng_btsocket_sco_accept(struct socket *so, struct sockaddr **nam)
 {
 	if (ng_btsocket_sco_node == NULL) 
 		return (EINVAL);
 
 	return (ng_btsocket_sco_peeraddr(so, nam));
 } /* ng_btsocket_sco_accept */
 
 /*
  * Create and attach new socket
  */
 
 int
 ng_btsocket_sco_attach(struct socket *so, int proto, struct thread *td)
 {
 	ng_btsocket_sco_pcb_p	pcb = so2sco_pcb(so);
 	int			error;
 
 	/* Check socket and protocol */
 	if (ng_btsocket_sco_node == NULL) 
 		return (EPROTONOSUPPORT);
 	if (so->so_type != SOCK_SEQPACKET)
 		return (ESOCKTNOSUPPORT);
 
 #if 0 /* XXX sonewconn() calls "pru_attach" with proto == 0 */
 	if (proto != 0) 
 		if (proto != BLUETOOTH_PROTO_SCO)
 			return (EPROTONOSUPPORT);
 #endif /* XXX */
 
 	if (pcb != NULL)
 		return (EISCONN);
 
 	/* Reserve send and receive space if it is not reserved yet */
 	if ((so->so_snd.sb_hiwat == 0) || (so->so_rcv.sb_hiwat == 0)) {
 		error = soreserve(so, NG_BTSOCKET_SCO_SENDSPACE,
 					NG_BTSOCKET_SCO_RECVSPACE);
 		if (error != 0)
 			return (error);
 	}
 
 	/* Allocate the PCB */
         pcb = malloc(sizeof(*pcb),
 		M_NETGRAPH_BTSOCKET_SCO, M_NOWAIT | M_ZERO);
         if (pcb == NULL)
                 return (ENOMEM);
 
 	/* Link the PCB and the socket */
 	so->so_pcb = (caddr_t) pcb;
 	pcb->so = so;
 	pcb->state = NG_BTSOCKET_SCO_CLOSED;
 
 	callout_init(&pcb->timo, 1);
 
 	/*
 	 * Mark PCB mutex as DUPOK to prevent "duplicated lock of
 	 * the same type" message. When accepting new SCO connection 
 	 * ng_btsocket_sco_process_lp_con_ind() holds both PCB mutexes 
 	 * for "old" (accepting) PCB and "new" (created) PCB.
 	 */
 		
 	mtx_init(&pcb->pcb_mtx, "btsocks_sco_pcb_mtx", NULL,
 		MTX_DEF|MTX_DUPOK);
 
 	/*
 	 * Add the PCB to the list
 	 *
 	 * XXX FIXME VERY IMPORTANT!
 	 *
 	 * This is totally FUBAR. We could get here in two cases:
 	 *
 	 * 1) When user calls socket()
 	 * 2) When we need to accept new incomming connection and call
 	 *    sonewconn()
 	 *
 	 * In the first case we must aquire ng_btsocket_sco_sockets_mtx.
 	 * In the second case we hold ng_btsocket_sco_sockets_mtx already.
 	 * So we now need to distinguish between these cases. From reading
 	 * /sys/kern/uipc_socket2.c we can find out that sonewconn() calls
 	 * pru_attach with proto == 0 and td == NULL. For now use this fact
 	 * to figure out if we were called from socket() or from sonewconn().
 	 */
 
 	if (td != NULL)
 		mtx_lock(&ng_btsocket_sco_sockets_mtx);
 	else
 		mtx_assert(&ng_btsocket_sco_sockets_mtx, MA_OWNED);
 
 	LIST_INSERT_HEAD(&ng_btsocket_sco_sockets, pcb, next);
 
 	if (td != NULL)
 		mtx_unlock(&ng_btsocket_sco_sockets_mtx);
 
         return (0);
 } /* ng_btsocket_sco_attach */
 
 /*
  * Bind socket
  */
 
 int
 ng_btsocket_sco_bind(struct socket *so, struct sockaddr *nam, 
 		struct thread *td)
 {
 	ng_btsocket_sco_pcb_t	*pcb = NULL;
 	struct sockaddr_sco	*sa = (struct sockaddr_sco *) nam;
 
 	if (ng_btsocket_sco_node == NULL) 
 		return (EINVAL);
 
 	/* Verify address */
 	if (sa == NULL)
 		return (EINVAL);
 	if (sa->sco_family != AF_BLUETOOTH)
 		return (EAFNOSUPPORT);
 	if (sa->sco_len != sizeof(*sa))
 		return (EINVAL);
 
 	mtx_lock(&ng_btsocket_sco_sockets_mtx);
 
 	/* 
 	 * Check if other socket has this address already (look for exact
 	 * match in bdaddr) and assign socket address if it's available.
 	 */
 
 	if (bcmp(&sa->sco_bdaddr, NG_HCI_BDADDR_ANY, sizeof(sa->sco_bdaddr)) != 0) {
  		LIST_FOREACH(pcb, &ng_btsocket_sco_sockets, next) {
 			mtx_lock(&pcb->pcb_mtx);
 
 			if (bcmp(&pcb->src, &sa->sco_bdaddr, sizeof(bdaddr_t)) == 0) {
 				mtx_unlock(&pcb->pcb_mtx);
 				mtx_unlock(&ng_btsocket_sco_sockets_mtx);
 
 				return (EADDRINUSE);
 			}
 
 			mtx_unlock(&pcb->pcb_mtx);
 		}
 
 	}
 
 	pcb = so2sco_pcb(so);
 	if (pcb == NULL) {
 		mtx_unlock(&ng_btsocket_sco_sockets_mtx);
 		return (EINVAL);
 	}
 
 	mtx_lock(&pcb->pcb_mtx);
 	bcopy(&sa->sco_bdaddr, &pcb->src, sizeof(pcb->src));
 	mtx_unlock(&pcb->pcb_mtx);
 
 	mtx_unlock(&ng_btsocket_sco_sockets_mtx);
 
 	return (0);
 } /* ng_btsocket_sco_bind */
 
 /*
  * Connect socket
  */
 
 int
 ng_btsocket_sco_connect(struct socket *so, struct sockaddr *nam, 
 		struct thread *td)
 {
 	ng_btsocket_sco_pcb_t		*pcb = so2sco_pcb(so);
 	struct sockaddr_sco		*sa = (struct sockaddr_sco *) nam;
 	ng_btsocket_sco_rtentry_t	*rt = NULL;
 	int				 have_src, error = 0;
 
 	/* Check socket */
 	if (pcb == NULL)
 		return (EINVAL);
 	if (ng_btsocket_sco_node == NULL) 
 		return (EINVAL);
 
 	/* Verify address */
 	if (sa == NULL)
 		return (EINVAL);
 	if (sa->sco_family != AF_BLUETOOTH)
 		return (EAFNOSUPPORT);
 	if (sa->sco_len != sizeof(*sa))
 		return (EINVAL);
 	if (bcmp(&sa->sco_bdaddr, NG_HCI_BDADDR_ANY, sizeof(bdaddr_t)) == 0)
 		return (EDESTADDRREQ);
 
 	/*
 	 * Routing. Socket should be bound to some source address. The source
 	 * address can be ANY. Destination address must be set and it must not
 	 * be ANY. If source address is ANY then find first rtentry that has
 	 * src != dst.
 	 */
 
 	mtx_lock(&ng_btsocket_sco_rt_mtx);
 	mtx_lock(&pcb->pcb_mtx);
 
 	if (pcb->state == NG_BTSOCKET_SCO_CONNECTING) {
 		mtx_unlock(&pcb->pcb_mtx);
 		mtx_unlock(&ng_btsocket_sco_rt_mtx);
 
 		return (EINPROGRESS);
 	}
 
 	if (bcmp(&sa->sco_bdaddr, &pcb->src, sizeof(pcb->src)) == 0) {
 		mtx_unlock(&pcb->pcb_mtx);
 		mtx_unlock(&ng_btsocket_sco_rt_mtx);
 
 		return (EINVAL);
 	}
 
 	/* Send destination address and PSM */
 	bcopy(&sa->sco_bdaddr, &pcb->dst, sizeof(pcb->dst));
 
 	pcb->rt = NULL;
 	have_src = bcmp(&pcb->src, NG_HCI_BDADDR_ANY, sizeof(pcb->src));
 
 	LIST_FOREACH(rt, &ng_btsocket_sco_rt, next) {
 		if (rt->hook == NULL || NG_HOOK_NOT_VALID(rt->hook))
 			continue;
 
 		/* Match src and dst */
 		if (have_src) {
 			if (bcmp(&pcb->src, &rt->src, sizeof(rt->src)) == 0)
 				break;
 		} else {
 			if (bcmp(&pcb->dst, &rt->src, sizeof(rt->src)) != 0)
 				break;
 		}
 	}
 
 	if (rt != NULL) {
 		pcb->rt = rt;
 
 		if (!have_src)
 			bcopy(&rt->src, &pcb->src, sizeof(pcb->src));
 	} else
 		error = EHOSTUNREACH;
 
 	/*
 	 * Send LP_Connect request 
 	 */
 
 	if (error == 0) {	
 		error = ng_btsocket_sco_send_lp_con_req(pcb);
 		if (error == 0) {
 			pcb->flags |= NG_BTSOCKET_SCO_CLIENT;
 			pcb->state = NG_BTSOCKET_SCO_CONNECTING;
 			soisconnecting(pcb->so);
 
 			ng_btsocket_sco_timeout(pcb);
 		}
 	}
 
 	mtx_unlock(&pcb->pcb_mtx);
 	mtx_unlock(&ng_btsocket_sco_rt_mtx);
 
 	return (error);
 } /* ng_btsocket_sco_connect */
 
 /*
  * Process ioctl's calls on socket
  */
 
 int
 ng_btsocket_sco_control(struct socket *so, u_long cmd, caddr_t data,
 		struct ifnet *ifp, struct thread *td)
 {
 	return (EINVAL);
 } /* ng_btsocket_sco_control */
 
 /*
  * Process getsockopt/setsockopt system calls
  */
 
 int
 ng_btsocket_sco_ctloutput(struct socket *so, struct sockopt *sopt)
 {
 	ng_btsocket_sco_pcb_p	pcb = so2sco_pcb(so);
         int			error, tmp;
 
 	if (ng_btsocket_sco_node == NULL) 
 		return (EINVAL);
 	if (pcb == NULL)
 		return (EINVAL);
 
 	if (sopt->sopt_level != SOL_SCO)
 		return (0);
 
 	mtx_lock(&pcb->pcb_mtx);
 
 	switch (sopt->sopt_dir) {
 	case SOPT_GET:
 		if (pcb->state != NG_BTSOCKET_SCO_OPEN) {
 			error = ENOTCONN;
 			break;
 		}
 		
 		switch (sopt->sopt_name) {
 		case SO_SCO_MTU:
 			tmp = pcb->rt->pkt_size;
 			error = sooptcopyout(sopt, &tmp, sizeof(tmp));
 			break;
 
 		case SO_SCO_CONNINFO:
 			tmp = pcb->con_handle;
 			error = sooptcopyout(sopt, &tmp, sizeof(tmp));
 			break;
 
 		default:
 			error = EINVAL;
 			break;
 		}
 		break;
 
 	case SOPT_SET:
 		error = ENOPROTOOPT;
 		break;
 
 	default:
 		error = EINVAL;
 		break;
 	}
 
 	mtx_unlock(&pcb->pcb_mtx);
 	
 	return (error);
 } /* ng_btsocket_sco_ctloutput */
 
 /*
  * Detach and destroy socket
  */
 
 void
 ng_btsocket_sco_detach(struct socket *so)
 {
 	ng_btsocket_sco_pcb_p	pcb = so2sco_pcb(so);
 
 	KASSERT(pcb != NULL, ("ng_btsocket_sco_detach: pcb == NULL"));
 
 	if (ng_btsocket_sco_node == NULL) 
 		return;
 
 	mtx_lock(&ng_btsocket_sco_sockets_mtx);
 	mtx_lock(&pcb->pcb_mtx);
 
 	if (pcb->flags & NG_BTSOCKET_SCO_TIMO)
 		ng_btsocket_sco_untimeout(pcb);
 
 	if (pcb->state == NG_BTSOCKET_SCO_OPEN)
 		ng_btsocket_sco_send_lp_discon_req(pcb);
 
 	pcb->state = NG_BTSOCKET_SCO_CLOSED;
 
 	LIST_REMOVE(pcb, next);
 
 	mtx_unlock(&pcb->pcb_mtx);
 	mtx_unlock(&ng_btsocket_sco_sockets_mtx);
 
 	mtx_destroy(&pcb->pcb_mtx);
 	bzero(pcb, sizeof(*pcb));
 	free(pcb, M_NETGRAPH_BTSOCKET_SCO);
 
 	soisdisconnected(so);
 	so->so_pcb = NULL;
 } /* ng_btsocket_sco_detach */
 
 /*
  * Disconnect socket
  */
 
 int
 ng_btsocket_sco_disconnect(struct socket *so)
 {
 	ng_btsocket_sco_pcb_p	pcb = so2sco_pcb(so);
 
 	if (pcb == NULL)
 		return (EINVAL);
 	if (ng_btsocket_sco_node == NULL) 
 		return (EINVAL);
 
 	mtx_lock(&pcb->pcb_mtx);
 
 	if (pcb->state == NG_BTSOCKET_SCO_DISCONNECTING) {
 		mtx_unlock(&pcb->pcb_mtx);
 
 		return (EINPROGRESS);
 	}
 
 	if (pcb->flags & NG_BTSOCKET_SCO_TIMO)
 		ng_btsocket_sco_untimeout(pcb);
 
 	if (pcb->state == NG_BTSOCKET_SCO_OPEN) {
 		ng_btsocket_sco_send_lp_discon_req(pcb);
 
 		pcb->state = NG_BTSOCKET_SCO_DISCONNECTING;
 		soisdisconnecting(so);
 
 		ng_btsocket_sco_timeout(pcb);
 	} else {
 		pcb->state = NG_BTSOCKET_SCO_CLOSED;
 		soisdisconnected(so);
 	}
 
 	mtx_unlock(&pcb->pcb_mtx);
 
 	return (0);
 } /* ng_btsocket_sco_disconnect */
 
 /*
  * Listen on socket
  */
 
 int
 ng_btsocket_sco_listen(struct socket *so, int backlog, struct thread *td)
 {
 	ng_btsocket_sco_pcb_p	pcb = so2sco_pcb(so);
 	int			error;
 
 	if (pcb == NULL)
 		return (EINVAL);
 	if (ng_btsocket_sco_node == NULL)
 		return (EINVAL);
 
 	SOCK_LOCK(so);
 	mtx_lock(&pcb->pcb_mtx);
 
 	error = solisten_proto_check(so);
 	if (error != 0)
 		goto out;
 #if 0
 	if (bcmp(&pcb->src, NG_HCI_BDADDR_ANY, sizeof(bdaddr_t)) == 0) {
 		error = EDESTADDRREQ;
 		goto out;
 	}
 #endif
 	solisten_proto(so, backlog);
 out:
 	mtx_unlock(&pcb->pcb_mtx);
 	SOCK_UNLOCK(so);
 
 	return (error);
 } /* ng_btsocket_listen */
 
 /*
  * Get peer address
  */
 
 int
 ng_btsocket_sco_peeraddr(struct socket *so, struct sockaddr **nam)
 {
 	ng_btsocket_sco_pcb_p	pcb = so2sco_pcb(so);
 	struct sockaddr_sco	sa;
 
 	if (pcb == NULL)
 		return (EINVAL);
 	if (ng_btsocket_sco_node == NULL) 
 		return (EINVAL);
 
 	mtx_lock(&pcb->pcb_mtx);
 	bcopy(&pcb->dst, &sa.sco_bdaddr, sizeof(sa.sco_bdaddr));
 	mtx_unlock(&pcb->pcb_mtx);
 
 	sa.sco_len = sizeof(sa);
 	sa.sco_family = AF_BLUETOOTH;
 
 	*nam = sodupsockaddr((struct sockaddr *) &sa, M_NOWAIT);
 
 	return ((*nam == NULL)? ENOMEM : 0);
 } /* ng_btsocket_sco_peeraddr */
 
 /*
  * Send data to socket
  */
 
 int
 ng_btsocket_sco_send(struct socket *so, int flags, struct mbuf *m,
 		struct sockaddr *nam, struct mbuf *control, struct thread *td)
 {
 	ng_btsocket_sco_pcb_t	*pcb = so2sco_pcb(so);
 	int			 error = 0;
                         
 	if (ng_btsocket_sco_node == NULL) {
 		error = ENETDOWN;
 		goto drop;
 	}
 
 	/* Check socket and input */
 	if (pcb == NULL || m == NULL || control != NULL) {
 		error = EINVAL;
 		goto drop;
 	}
                  
 	mtx_lock(&pcb->pcb_mtx);
                   
 	/* Make sure socket is connected */
 	if (pcb->state != NG_BTSOCKET_SCO_OPEN) {
 		mtx_unlock(&pcb->pcb_mtx); 
 		error = ENOTCONN;
 		goto drop;
 	}
 
 	/* Check route */
 	if (pcb->rt == NULL ||
 	    pcb->rt->hook == NULL || NG_HOOK_NOT_VALID(pcb->rt->hook)) {
 		mtx_unlock(&pcb->pcb_mtx);
 		error = ENETDOWN;
 		goto drop;
 	}
 
 	/* Check packet size */
 	if (m->m_pkthdr.len > pcb->rt->pkt_size) {
 		NG_BTSOCKET_SCO_ERR(
 "%s: Packet too big, len=%d, pkt_size=%d\n",
 			__func__, m->m_pkthdr.len, pcb->rt->pkt_size);
 
 		mtx_unlock(&pcb->pcb_mtx);
 		error = EMSGSIZE;
 		goto drop;
 	}
 
 	/*
 	 * First put packet on socket send queue. Then check if we have
 	 * pending timeout. If we do not have timeout then we must send
 	 * packet and schedule timeout. Otherwise do nothing and wait for
 	 * NGM_HCI_SYNC_CON_QUEUE message.
 	 */
 
 	sbappendrecord(&pcb->so->so_snd, m);
 	m = NULL;
 
 	if (!(pcb->flags & NG_BTSOCKET_SCO_TIMO)) {
 		error = ng_btsocket_sco_send2(pcb);
 		if (error == 0)
 			ng_btsocket_sco_timeout(pcb);
 		else
 			sbdroprecord(&pcb->so->so_snd); /* XXX */
 	}
 
 	mtx_unlock(&pcb->pcb_mtx);
 drop:
 	NG_FREE_M(m); /* checks for != NULL */
 	NG_FREE_M(control);
 
 	return (error);
 } /* ng_btsocket_sco_send */
 
 /*
  * Send first packet in the socket queue to the SCO layer
  */
 
 static int
 ng_btsocket_sco_send2(ng_btsocket_sco_pcb_p pcb)
 {
 	struct  mbuf		*m = NULL;
 	ng_hci_scodata_pkt_t	*hdr = NULL;
 	int			 error = 0;
 
 	mtx_assert(&pcb->pcb_mtx, MA_OWNED);
 
 	while (pcb->rt->pending < pcb->rt->num_pkts &&
 	       pcb->so->so_snd.sb_cc > 0) {
 		/* Get a copy of the first packet on send queue */
 		m = m_dup(pcb->so->so_snd.sb_mb, M_NOWAIT);
 		if (m == NULL) {
 			error = ENOBUFS;
 			break;
 		}
 
 		/* Create SCO packet header */
 		M_PREPEND(m, sizeof(*hdr), M_NOWAIT);
 		if (m != NULL)
 			if (m->m_len < sizeof(*hdr))
 				m = m_pullup(m, sizeof(*hdr));
 
 		if (m == NULL) {
 			error = ENOBUFS;
 			break;
 		}
 
 		/* Fill in the header */
 		hdr = mtod(m, ng_hci_scodata_pkt_t *);
 		hdr->type = NG_HCI_SCO_DATA_PKT;
 		hdr->con_handle = htole16(NG_HCI_MK_CON_HANDLE(pcb->con_handle, 0, 0));
 		hdr->length = m->m_pkthdr.len - sizeof(*hdr);
 
 		/* Send packet */
 		NG_SEND_DATA_ONLY(error, pcb->rt->hook, m);
 		if (error != 0)
 			break;
 
 		pcb->rt->pending ++;
 	}
 
 	return ((pcb->rt->pending > 0)? 0 : error);
 } /* ng_btsocket_sco_send2 */
 
 /*
  * Get socket address
  */
 
 int
 ng_btsocket_sco_sockaddr(struct socket *so, struct sockaddr **nam)
 {
 	ng_btsocket_sco_pcb_p	pcb = so2sco_pcb(so);
 	struct sockaddr_sco	sa;
 
 	if (pcb == NULL)
 		return (EINVAL);
 	if (ng_btsocket_sco_node == NULL) 
 		return (EINVAL);
 
 	mtx_lock(&pcb->pcb_mtx);
 	bcopy(&pcb->src, &sa.sco_bdaddr, sizeof(sa.sco_bdaddr));
 	mtx_unlock(&pcb->pcb_mtx);
 
 	sa.sco_len = sizeof(sa);
 	sa.sco_family = AF_BLUETOOTH;
 
 	*nam = sodupsockaddr((struct sockaddr *) &sa, M_NOWAIT);
 
 	return ((*nam == NULL)? ENOMEM : 0);
 } /* ng_btsocket_sco_sockaddr */
 
 /*****************************************************************************
  *****************************************************************************
  **                              Misc. functions
  *****************************************************************************
  *****************************************************************************/
 
 /*
  * Look for the socket that listens on given bdaddr.
  * Returns exact or close match (if any).
  * Caller must hold ng_btsocket_sco_sockets_mtx.
  * Returns with locked pcb.
  */
 
 static ng_btsocket_sco_pcb_p
 ng_btsocket_sco_pcb_by_addr(bdaddr_p bdaddr)
 {
 	ng_btsocket_sco_pcb_p	p = NULL, p1 = NULL;
 
 	mtx_assert(&ng_btsocket_sco_sockets_mtx, MA_OWNED);
 
 	LIST_FOREACH(p, &ng_btsocket_sco_sockets, next) {
 		mtx_lock(&p->pcb_mtx);
 
 		if (p->so == NULL || !(p->so->so_options & SO_ACCEPTCONN)) {
 			mtx_unlock(&p->pcb_mtx);
 			continue;
 		}
 
 		if (bcmp(&p->src, bdaddr, sizeof(p->src)) == 0)
 			return (p); /* return with locked pcb */
 
 		if (bcmp(&p->src, NG_HCI_BDADDR_ANY, sizeof(p->src)) == 0)
 			p1 = p;
 
 		mtx_unlock(&p->pcb_mtx);
 	}
 
 	if (p1 != NULL)
 		mtx_lock(&p1->pcb_mtx);
 
 	return (p1);
 } /* ng_btsocket_sco_pcb_by_addr */
 
 /*
  * Look for the socket that assigned to given source address and handle.
  * Caller must hold ng_btsocket_sco_sockets_mtx.
  * Returns with locked pcb.
  */
 
 static ng_btsocket_sco_pcb_p
 ng_btsocket_sco_pcb_by_handle(bdaddr_p src, int con_handle)
 {
 	ng_btsocket_sco_pcb_p	p = NULL;
 
 	mtx_assert(&ng_btsocket_sco_sockets_mtx, MA_OWNED);
 
 	LIST_FOREACH(p, &ng_btsocket_sco_sockets, next) {
 		mtx_lock(&p->pcb_mtx);
 
 		if (p->con_handle == con_handle &&
 		    bcmp(src, &p->src, sizeof(p->src)) == 0)
 			return (p); /* return with locked pcb */
 
 		mtx_unlock(&p->pcb_mtx);
 	}
 
 	return (NULL);
 } /* ng_btsocket_sco_pcb_by_handle */
 
 /*
  * Look for the socket in CONNECTING state with given source and destination
  * addresses. Caller must hold ng_btsocket_sco_sockets_mtx.
  * Returns with locked pcb.
  */
 
 static ng_btsocket_sco_pcb_p
 ng_btsocket_sco_pcb_by_addrs(bdaddr_p src, bdaddr_p dst)
 {
 	ng_btsocket_sco_pcb_p	p = NULL;
 
 	mtx_assert(&ng_btsocket_sco_sockets_mtx, MA_OWNED);
 
 	LIST_FOREACH(p, &ng_btsocket_sco_sockets, next) {
 		mtx_lock(&p->pcb_mtx);
 
 		if (p->state == NG_BTSOCKET_SCO_CONNECTING &&
 		    bcmp(src, &p->src, sizeof(p->src)) == 0 &&
 		    bcmp(dst, &p->dst, sizeof(p->dst)) == 0)
 			return (p); /* return with locked pcb */
 
 		mtx_unlock(&p->pcb_mtx);
 	}
 
 	return (NULL);
 } /* ng_btsocket_sco_pcb_by_addrs */
 
 /*
  * Set timeout on socket
  */
 
 static void
 ng_btsocket_sco_timeout(ng_btsocket_sco_pcb_p pcb)
 {
 	mtx_assert(&pcb->pcb_mtx, MA_OWNED);
 
 	if (!(pcb->flags & NG_BTSOCKET_SCO_TIMO)) {
 		pcb->flags |= NG_BTSOCKET_SCO_TIMO;
 		callout_reset(&pcb->timo, bluetooth_sco_rtx_timeout(),
 					ng_btsocket_sco_process_timeout, pcb);
 	} else
 		KASSERT(0,
 ("%s: Duplicated socket timeout?!\n", __func__));
 } /* ng_btsocket_sco_timeout */
 
 /*
  * Unset timeout on socket
  */
 
 static void
 ng_btsocket_sco_untimeout(ng_btsocket_sco_pcb_p pcb)
 {
 	mtx_assert(&pcb->pcb_mtx, MA_OWNED);
 
 	if (pcb->flags & NG_BTSOCKET_SCO_TIMO) {
 		callout_stop(&pcb->timo);
 		pcb->flags &= ~NG_BTSOCKET_SCO_TIMO;
 	} else
 		KASSERT(0,
 ("%s: No socket timeout?!\n", __func__));
 } /* ng_btsocket_sco_untimeout */
 
 /*
  * Process timeout on socket
  */
 
 static void
 ng_btsocket_sco_process_timeout(void *xpcb)
 {
 	ng_btsocket_sco_pcb_p	 pcb = (ng_btsocket_sco_pcb_p) xpcb;
 
 	mtx_lock(&pcb->pcb_mtx);
 
 	pcb->flags &= ~NG_BTSOCKET_SCO_TIMO;
 	pcb->so->so_error = ETIMEDOUT;
 
 	switch (pcb->state) {
 	case NG_BTSOCKET_SCO_CONNECTING:
 		/* Connect timeout - close the socket */
 		pcb->state = NG_BTSOCKET_SCO_CLOSED;
 		soisdisconnected(pcb->so);
 		break;
 
 	case NG_BTSOCKET_SCO_OPEN:
 		/* Send timeout - did not get NGM_HCI_SYNC_CON_QUEUE */
 		sbdroprecord(&pcb->so->so_snd);
 		sowwakeup(pcb->so);
 		/* XXX FIXME what to do with pcb->rt->pending??? */
 		break;
 
 	case NG_BTSOCKET_SCO_DISCONNECTING:
 		/* Disconnect timeout - disconnect the socket anyway */
 		pcb->state = NG_BTSOCKET_SCO_CLOSED;
 		soisdisconnected(pcb->so);
 		break;
 
 	default:
 		NG_BTSOCKET_SCO_ERR(
 "%s: Invalid socket state=%d\n", __func__, pcb->state);
 		break;
 	}
 
 	mtx_unlock(&pcb->pcb_mtx);
 } /* ng_btsocket_sco_process_timeout */
 
Index: projects/release-debugdist/sys
===================================================================
--- projects/release-debugdist/sys	(revision 267343)
+++ projects/release-debugdist/sys	(revision 267344)

Property changes on: projects/release-debugdist/sys
___________________________________________________________________
Modified: svn:mergeinfo
## -0,0 +0,1 ##
   Merged /head/sys:r267325-267343
Index: projects/release-debugdist/tools/tools/vt/fontcvt/fontcvt.c
===================================================================
--- projects/release-debugdist/tools/tools/vt/fontcvt/fontcvt.c	(revision 267343)
+++ projects/release-debugdist/tools/tools/vt/fontcvt/fontcvt.c	(nonexistent)
@@ -1,568 +0,0 @@
-/*-
- * Copyright (c) 2009, 2014 The FreeBSD Foundation
- * All rights reserved.
- *
- * This software was developed by Ed Schouten under sponsorship from the
- * FreeBSD Foundation.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/types.h>
-#include <sys/fnv_hash.h>
-#include <sys/endian.h>
-#include <sys/param.h>
-#include <sys/queue.h>
-
-#include <assert.h>
-#include <err.h>
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-
-#define VFNT_MAPS 4
-#define VFNT_MAP_NORMAL 0
-#define VFNT_MAP_NORMAL_RH 1
-#define VFNT_MAP_BOLD 2
-#define VFNT_MAP_BOLD_RH 3
-
-static unsigned int width = 8, wbytes, height = 16;
-
-struct glyph {
-	TAILQ_ENTRY(glyph)	 g_list;
-	SLIST_ENTRY(glyph)	 g_hash;
-	uint8_t			*g_data;
-	unsigned int		 g_index;
-};
-
-#define	FONTCVT_NHASH 4096
-TAILQ_HEAD(glyph_list, glyph);
-static SLIST_HEAD(, glyph) glyph_hash[FONTCVT_NHASH];
-static struct glyph_list glyphs[VFNT_MAPS] = {
-    TAILQ_HEAD_INITIALIZER(glyphs[0]),
-    TAILQ_HEAD_INITIALIZER(glyphs[1]),
-    TAILQ_HEAD_INITIALIZER(glyphs[2]),
-    TAILQ_HEAD_INITIALIZER(glyphs[3]),
-};
-static unsigned int glyph_total, glyph_count[4], glyph_unique, glyph_dupe;
-
-struct mapping {
-	TAILQ_ENTRY(mapping)	 m_list;
-	unsigned int		 m_char;
-	unsigned int		 m_length;
-	struct glyph		*m_glyph;
-};
-
-TAILQ_HEAD(mapping_list, mapping);
-static struct mapping_list maps[VFNT_MAPS] = {
-    TAILQ_HEAD_INITIALIZER(maps[0]),
-    TAILQ_HEAD_INITIALIZER(maps[1]),
-    TAILQ_HEAD_INITIALIZER(maps[2]),
-    TAILQ_HEAD_INITIALIZER(maps[3]),
-};
-static unsigned int mapping_total, map_count[4], map_folded_count[4],
-    mapping_unique, mapping_dupe;
-
-static void
-usage(void)
-{
-
-	errx(1,
-"usage: fontcvt [-w width] [-h height] [-v] normal.bdf [bold.bdf] out.fnt\n");
-	exit(1);
-}
-
-static int
-add_mapping(struct glyph *gl, unsigned int c, unsigned int map_idx)
-{
-	struct mapping *mp;
-	struct mapping_list *ml;
-
-	mapping_total++;
-
-	mp = malloc(sizeof *mp);
-	mp->m_char = c;
-	mp->m_glyph = gl;
-	mp->m_length = 0;
-
-	ml = &maps[map_idx];
-	if (TAILQ_LAST(ml, mapping_list) != NULL &&
-	    TAILQ_LAST(ml, mapping_list)->m_char >= c) {
-		errx(1, "Bad ordering at character %u\n", c);
-		return (1);
-	}
-	TAILQ_INSERT_TAIL(ml, mp, m_list);
-
-	map_count[map_idx]++;
-	mapping_unique++;
-
-	return (0);
-}
-
-static int
-dedup_mapping(unsigned int map_idx)
-{
-	struct mapping *mp_bold, *mp_normal, *mp_temp;
-	unsigned normal_map_idx = map_idx - VFNT_MAP_BOLD;
-
-	assert(map_idx == VFNT_MAP_BOLD || map_idx == VFNT_MAP_BOLD_RH);
-	mp_normal = TAILQ_FIRST(&maps[normal_map_idx]);
-	TAILQ_FOREACH_SAFE(mp_bold, &maps[map_idx], m_list, mp_temp) {
-		while (mp_normal->m_char < mp_bold->m_char)
-			mp_normal = TAILQ_NEXT(mp_normal, m_list);
-		if (mp_bold->m_char != mp_normal->m_char) {
-			errx(1, "Character %u not in normal font!\n",
-			    mp_bold->m_char);
-			return (1);
-		}
-		if (mp_bold->m_glyph != mp_normal->m_glyph)
-			continue;
-
-		/* No mapping is needed if it's equal to the normal mapping. */
-		TAILQ_REMOVE(&maps[map_idx], mp_bold, m_list);
-		free(mp_bold);
-		mapping_dupe++;
-	}
-	return (0);
-}
-
-static struct glyph *
-add_glyph(const uint8_t *bytes, unsigned int map_idx, int fallback)
-{
-	struct glyph *gl;
-	int hash;
-
-	glyph_total++;
-	glyph_count[map_idx]++;
-
-	hash = fnv_32_buf(bytes, wbytes * height, FNV1_32_INIT) % FONTCVT_NHASH;
-	SLIST_FOREACH(gl, &glyph_hash[hash], g_hash) {
-		if (memcmp(gl->g_data, bytes, wbytes * height) == 0) {
-			glyph_dupe++;
-			return (gl);
-		}
-	}
-
-	gl = malloc(sizeof *gl);
-	gl->g_data = malloc(wbytes * height);
-	memcpy(gl->g_data, bytes, wbytes * height);
-	if (fallback)
-		TAILQ_INSERT_HEAD(&glyphs[map_idx], gl, g_list);
-	else
-		TAILQ_INSERT_TAIL(&glyphs[map_idx], gl, g_list);
-	SLIST_INSERT_HEAD(&glyph_hash[hash], gl, g_hash);
-
-	glyph_unique++;
-	return (gl);
-}
-
-static int
-add_char(unsigned curchar, unsigned map_idx, uint8_t *bytes, uint8_t *bytes_r)
-{
-	struct glyph *gl;
-
-	/* Prevent adding two glyphs for 0xFFFD */
-	if (curchar == 0xFFFD) {
-		if (map_idx < VFNT_MAP_BOLD)
-			gl = add_glyph(bytes, 0, 1);
-	} else if (curchar >= 0x20) {
-		gl = add_glyph(bytes, map_idx, 0);
-		if (add_mapping(gl, curchar, map_idx) != 0)
-			return (1);
-		if (bytes_r != NULL) {
-			gl = add_glyph(bytes_r, map_idx + 1, 0);
-			if (add_mapping(gl, curchar,
-			    map_idx + 1) != 0)
-				return (1);
-		}
-	}
-	return (0);
-}
-
-
-static int
-parse_bitmap_line(uint8_t *left, uint8_t *right, unsigned int line,
-    unsigned int dwidth)
-{
-	uint8_t *p;
-	unsigned int i, subline;
-
-	if (dwidth != width && dwidth != width * 2) {
-		errx(1,
-		    "Bitmap with unsupported width %u!\n", dwidth);
-		return (1);
-	}
-
-	/* Move pixel data right to simplify splitting double characters. */
-	line >>= (howmany(dwidth, 8) * 8) - dwidth;
-
-	for (i = dwidth / width; i > 0; i--) {
-		p = (i == 2) ? right : left;
-
-		subline = line & ((1 << width) - 1);
-		subline <<= (howmany(width, 8) * 8) - width;
-
-		if (wbytes == 1) {
-			*p = subline;
-		} else if (wbytes == 2) {
-			*p++ = subline >> 8;
-			*p = subline;
-		} else {
-			errx(1,
-			    "Unsupported wbytes %u!\n", wbytes);
-			return (1);
-		}
-
-		line >>= width;
-	}
-	
-	return (0);
-}
-
-static int
-parse_bdf(FILE *fp, unsigned int map_idx)
-{
-	char *ln;
-	size_t length;
-	uint8_t bytes[wbytes * height], bytes_r[wbytes * height];
-	unsigned int curchar = 0, dwidth = 0, i, line;
-
-	while ((ln = fgetln(fp, &length)) != NULL) {
-		ln[length - 1] = '\0';
-
-		if (strncmp(ln, "ENCODING ", 9) == 0) {
-			curchar = atoi(ln + 9);
-		}
-
-		if (strncmp(ln, "DWIDTH ", 7) == 0) {
-			dwidth = atoi(ln + 7);
-		}
-
-		if (strncmp(ln, "BITMAP", 6) == 0 &&
-		    (ln[6] == ' ' || ln[6] == '\0')) {
-			for (i = 0; i < height; i++) {
-				if ((ln = fgetln(fp, &length)) == NULL) {
-					errx(1, "Unexpected EOF!\n");
-					return (1);
-				}
-				ln[length - 1] = '\0';
-				sscanf(ln, "%x", &line);
-				if (parse_bitmap_line(bytes + i * wbytes,
-				     bytes_r + i * wbytes, line, dwidth) != 0)
-					return (1);
-			}
-
-			if (add_char(curchar, map_idx, bytes,
-			    dwidth == width * 2 ? bytes_r : NULL) != 0)
-				return (1);
-		}
-	}
-
-	return (0);
-}
-
-static int
-parse_hex(FILE *fp, unsigned int map_idx)
-{
-	char *ln, *p;
-	char fmt_str[8];
-	size_t length;
-	uint8_t bytes[wbytes * height], bytes_r[wbytes * height];
-	unsigned curchar = 0, i, line, chars_per_row, dwidth;
-
-	while ((ln = fgetln(fp, &length)) != NULL) {
-		ln[length - 1] = '\0';
-
-		if (strncmp(ln, "# Height: ", 10) == 0) {
-			height = atoi(ln + 10);
-		} else if (strncmp(ln, "# Width: ", 9) == 0) {
-			width = atoi(ln + 9);
-		} else if (sscanf(ln, "%4x:", &curchar)) {
-			p = ln + 5;
-			chars_per_row = strlen(p) / height;
-			dwidth = width;
-			if (chars_per_row / 2 > width / 8)
-				dwidth *= 2; /* Double-width character. */
-			snprintf(fmt_str, sizeof(fmt_str), "%%%ux",
-			    chars_per_row);
-			
-			for (i = 0; i < height; i++) {
-				sscanf(p, fmt_str, &line);
-				p += chars_per_row;
-				if (parse_bitmap_line(bytes + i * wbytes,
-				    bytes_r + i * wbytes, line, dwidth) != 0)
-					return (1);
-			}
-
-			if (add_char(curchar, map_idx, bytes,
-			    dwidth == width * 2 ? bytes_r : NULL) != 0)
-				return (1);
-		}
-	}
-	return (0);
-}
-
-static int
-parse_file(const char *filename, unsigned int map_idx)
-{
-	FILE *fp;
-	size_t len;
-
-	fp = fopen(filename, "r");
-	if (fp == NULL) {
-		perror(filename);
-		return (1);
-	}
-	len = strlen(filename);
-	if (len > 4 && strcasecmp(filename + len - 4, ".hex") == 0)
-		return parse_hex(fp, map_idx);
-	return parse_bdf(fp, map_idx);
-}
-
-static void
-number_glyphs(void)
-{
-	struct glyph *gl;
-	unsigned int i, idx = 0;
-
-	for (i = 0; i < VFNT_MAPS; i++)
-		TAILQ_FOREACH(gl, &glyphs[i], g_list)
-			gl->g_index = idx++;
-}
-
-static int
-write_glyphs(FILE *fp)
-{
-	struct glyph *gl;
-	unsigned int i;
-
-	for (i = 0; i < VFNT_MAPS; i++) {
-		TAILQ_FOREACH(gl, &glyphs[i], g_list)
-			if (fwrite(gl->g_data, wbytes * height, 1, fp) != 1)
-				return (1);
-	}
-	return (0);
-}
-
-static void
-fold_mappings(unsigned int map_idx)
-{
-	struct mapping_list *ml = &maps[map_idx];
-	struct mapping *mn, *mp, *mbase;
-
-	mp = mbase = TAILQ_FIRST(ml);
-	for (mp = mbase = TAILQ_FIRST(ml); mp != NULL; mp = mn) {
-		mn = TAILQ_NEXT(mp, m_list);
-		if (mn != NULL && mn->m_char == mp->m_char + 1 &&
-		    mn->m_glyph->g_index == mp->m_glyph->g_index + 1)
-			continue;
-		mbase->m_length = mp->m_char - mbase->m_char + 1;
-		mbase = mp = mn;
-		map_folded_count[map_idx]++;
-	}
-}
-
-struct file_mapping {
-	uint32_t	source;
-	uint16_t	destination;
-	uint16_t	length;
-} __packed;
-
-static int
-write_mappings(FILE *fp, unsigned int map_idx)
-{
-	struct mapping_list *ml = &maps[map_idx];
-	struct mapping *mp;
-	struct file_mapping fm;
-	unsigned int i = 0, j = 0;
-
-	TAILQ_FOREACH(mp, ml, m_list) {
-		j++;
-		if (mp->m_length > 0) {
-			i += mp->m_length;
-			fm.source = htobe32(mp->m_char);
-			fm.destination = htobe16(mp->m_glyph->g_index);
-			fm.length = htobe16(mp->m_length - 1);
-			if (fwrite(&fm, sizeof fm, 1, fp) != 1)
-				return (1);
-		}
-	}
-	assert(i == j);
-	return (0);
-}
-
-struct file_header {
-	uint8_t		magic[8];
-	uint8_t		width;
-	uint8_t		height;
-	uint16_t	pad;
-	uint32_t	glyph_count;
-	uint32_t	map_count[4];
-} __packed;
-
-static int
-write_fnt(const char *filename)
-{
-	FILE *fp;
-	struct file_header fh = {
-		.magic = "VFNT0002",
-	};
-
-	fp = fopen(filename, "wb");
-	if (fp == NULL) {
-		perror(filename);
-		return (1);
-	}
-
-	fh.width = width;
-	fh.height = height;
-	fh.glyph_count = htobe32(glyph_unique);
-	fh.map_count[0] = htobe32(map_folded_count[0]);
-	fh.map_count[1] = htobe32(map_folded_count[1]);
-	fh.map_count[2] = htobe32(map_folded_count[2]);
-	fh.map_count[3] = htobe32(map_folded_count[3]);
-	if (fwrite(&fh, sizeof fh, 1, fp) != 1) {
-		perror(filename);
-		return (1);
-	}
-	
-	if (write_glyphs(fp) != 0 ||
-	    write_mappings(fp, VFNT_MAP_NORMAL) != 0 ||
-	    write_mappings(fp, 1) != 0 ||
-	    write_mappings(fp, VFNT_MAP_BOLD) != 0 ||
-	    write_mappings(fp, 3) != 0) {
-		perror(filename);
-		return (1);
-	}
-
-	return (0);
-}
-
-static void
-print_font_info(void)
-{
-	printf(
-"Statistics:\n"
-"- glyph_total:                 %5u\n"
-"- glyph_normal:                %5u\n"
-"- glyph_normal_right:          %5u\n"
-"- glyph_bold:                  %5u\n"
-"- glyph_bold_right:            %5u\n"
-"- glyph_unique:                %5u\n"
-"- glyph_dupe:                  %5u\n"
-"- mapping_total:               %5u\n"
-"- mapping_normal:              %5u\n"
-"- mapping_normal_folded:       %5u\n"
-"- mapping_normal_right:        %5u\n"
-"- mapping_normal_right_folded: %5u\n"
-"- mapping_bold:                %5u\n"
-"- mapping_bold_folded:         %5u\n"
-"- mapping_bold_right:          %5u\n"
-"- mapping_bold_right_folded:   %5u\n"
-"- mapping_unique:              %5u\n"
-"- mapping_dupe:                %5u\n",
-	    glyph_total,
-	    glyph_count[0],
-	    glyph_count[1],
-	    glyph_count[2],
-	    glyph_count[3],
-	    glyph_unique, glyph_dupe,
-	    mapping_total,
-	    map_count[0], map_folded_count[0],
-	    map_count[1], map_folded_count[1],
-	    map_count[2], map_folded_count[2],
-	    map_count[3], map_folded_count[3],
-	    mapping_unique, mapping_dupe);
-}
-
-int
-main(int argc, char *argv[])
-{
-	int ch, val, verbose = 0;
-
-	assert(sizeof(struct file_header) == 32);
-	assert(sizeof(struct file_mapping) == 8);
-
-	while ((ch = getopt(argc, argv, "h:w:")) != -1) {
-		switch (ch) {
-		case 'h':
-			val = atoi(optarg);
-			if (val <= 0 || val > 128) {
-				errx(1, "Invalid height %d", val);
-				return (1);
-			}
-			height = val;
-			break;
-		case 'v':
-			verbose = 1;
-			break;
-		case 'w':
-			val = atoi(optarg);
-			if (val <= 0 || val > 128) {
-				errx(1, "Invalid width %d", val);
-				return (1);
-			}
-			width = val;
-			break;
-		case '?':
-		default:
-			usage();
-		}
-	}
-	argc -= optind;
-	argv += optind;
-
-	if (argc < 2 || argc > 3)
-		usage();
-
-	wbytes = howmany(width, 8);
-
-	if (parse_file(argv[0], VFNT_MAP_NORMAL) != 0)
-		return (1);
-	argc--;
-	argv++;
-	if (argc == 2) {
-		if (parse_file(argv[0], VFNT_MAP_BOLD) != 0)
-			return (1);
-		argc--;
-		argv++;
-	}
-	number_glyphs();
-	dedup_mapping(VFNT_MAP_BOLD);
-	dedup_mapping(VFNT_MAP_BOLD_RH);
-	fold_mappings(0);
-	fold_mappings(1);
-	fold_mappings(2);
-	fold_mappings(3);
-	if (write_fnt(argv[0]) != 0)
-		return (1);
-
-	if (verbose)
-		print_font_info();
-
-	return (0);
-}

Property changes on: projects/release-debugdist/tools/tools/vt/fontcvt/fontcvt.c
___________________________________________________________________
Deleted: svn:keywords
## -1 +0,0 ##
-FreeBSD=%H
\ No newline at end of property
Index: projects/release-debugdist/tools/tools/vt/fontcvt/Makefile
===================================================================
--- projects/release-debugdist/tools/tools/vt/fontcvt/Makefile	(revision 267343)
+++ projects/release-debugdist/tools/tools/vt/fontcvt/Makefile	(nonexistent)
@@ -1,6 +0,0 @@
-PROG=	fontcvt
-MAN1=
-
-WARNS?=	6
-
-.include <bsd.prog.mk>

Property changes on: projects/release-debugdist/tools/tools/vt/fontcvt/Makefile
___________________________________________________________________
Deleted: svn:keywords
## -1 +0,0 ##
-FreeBSD=%H
\ No newline at end of property
Index: projects/release-debugdist/tools/tools/vt/fontcvt/terminus.sh
===================================================================
--- projects/release-debugdist/tools/tools/vt/fontcvt/terminus.sh	(revision 267343)
+++ projects/release-debugdist/tools/tools/vt/fontcvt/terminus.sh	(revision 267344)
@@ -1,14 +1,14 @@
 #!/bin/sh
 # $FreeBSD$
 
 for i in 6:12 8:14 8:16 10:18 10:20 11:22 12:24 14:28 16:32
 do
 	C=`echo $i | cut -f 1 -d :`
 	R=`echo $i | cut -f 2 -d :`
-	./fontcvt \
+	./vtfontcvt \
 		-w $C -h $R \
 		~/terminus-font-4.36/ter-u${R}n.bdf \
 		~/terminus-font-4.36/ter-u${R}b.bdf \
 		terminus-u${R}.vfnt
 	gzip -9nf terminus-u${R}.vfnt
 done
Index: projects/release-debugdist/usr.bin/Makefile
===================================================================
--- projects/release-debugdist/usr.bin/Makefile	(revision 267343)
+++ projects/release-debugdist/usr.bin/Makefile	(revision 267344)
@@ -1,386 +1,390 @@
 #	From: @(#)Makefile	8.3 (Berkeley) 1/7/94
 # $FreeBSD$
 
 .include <src.opts.mk>
 
 # XXX MISSING:		deroff diction graph learn plot
 #			spell spline struct xsend
 # XXX Use GNU versions: diff ld patch
 # Moved to secure: bdes
 #
 
 SUBDIR=	alias \
 	apply \
 	asa \
 	awk \
 	banner \
 	basename \
 	brandelf \
 	bsdiff \
 	bzip2 \
 	bzip2recover \
 	cap_mkdb \
 	chat \
 	chpass \
 	cksum \
 	${_clang} \
 	cmp \
 	col \
 	colldef \
 	colrm \
 	column \
 	comm \
 	compress \
 	cpuset \
 	csplit \
 	ctlstat \
 	cut \
 	dirname \
 	du \
 	ee \
 	elf2aout \
 	elfdump \
 	enigma \
 	env \
 	expand \
 	false \
 	fetch \
 	file \
 	find \
 	finger \
 	fmt \
 	fold \
 	fstat \
 	fsync \
 	ftp \
 	gcore \
 	gencat \
 	getconf \
 	getent \
 	getopt \
 	grep \
 	gzip \
 	head \
 	hexdump \
 	${_iconv} \
 	id \
 	ipcrm \
 	ipcs \
 	iscsictl \
 	join \
 	jot \
 	${_kdump} \
 	keylogin \
 	keylogout \
 	killall \
 	ktrace \
 	ktrdump \
 	lam \
 	lastcomm \
 	ldd \
 	leave \
 	less \
 	lessecho \
 	lesskey \
 	limits \
 	locale \
 	lock \
 	lockf \
 	logger \
 	login \
 	logins \
 	logname \
 	look \
 	lorder \
 	lsvfs \
 	lzmainfo \
 	m4 \
 	${_makewhatis} \
 	${_man} \
 	mandoc \
 	mesg \
 	minigzip \
 	ministat \
 	${_mkcsmapper} \
 	mkdep \
 	${_mkesdb} \
 	mkfifo \
 	mkimg \
 	mklocale \
 	mktemp \
 	mkulzma \
 	mkuzip \
 	mt \
 	ncal \
 	netstat \
 	newgrp \
 	nfsstat \
 	nice \
 	nl \
 	nohup \
 	opieinfo \
 	opiekey \
 	opiepasswd \
 	pagesize \
 	passwd \
 	paste \
 	patch \
 	pathchk \
 	perror \
 	pr \
 	printenv \
 	printf \
 	procstat \
 	protect \
 	rctl \
 	renice \
 	rev \
 	revoke \
 	rpcinfo \
 	rs \
 	rup \
 	rusers \
 	rwall \
 	script \
 	sed \
 	seq \
 	shar \
 	showmount \
 	sockstat \
 	sort \
 	split \
 	stat \
 	stdbuf \
 	su \
 	systat \
 	tabs \
 	tail \
 	talk \
 	tar \
 	tcopy \
 	tee \
 	${_tests} \
 	tftp \
 	time \
 	tip \
 	top \
 	touch \
 	tput \
 	tr \
 	true \
 	truncate \
 	${_truss} \
 	tset \
 	tsort \
 	tty \
 	uname \
 	unexpand \
 	uniq \
 	unzip \
 	units \
 	unvis \
 	uudecode \
 	uuencode \
 	vis \
 	vmstat \
 	w \
 	wall \
 	wc \
 	what \
 	whereis \
 	which \
 	whois \
 	write \
 	xargs \
 	xinstall \
 	${_xlint} \
 	${_xstr} \
 	xz \
 	xzdec \
 	${_yacc} \
 	yes \
 	${_ypcat} \
 	${_ypmatch} \
 	${_ypwhich}
 
 # NB: keep these sorted by MK_* knobs
 
 .if ${MK_AT} != "no"
 SUBDIR+=	at
 .endif
 
 .if ${MK_ATM} != "no"
 SUBDIR+=	atm
 .endif
 
 .if ${MK_BLUETOOTH} != "no"
 SUBDIR+=	bluetooth
 .endif
 
 .if ${MK_BSD_CPIO} != "no"
 SUBDIR+=	cpio
 .endif
 
 .if ${MK_CALENDAR} != "no"
 SUBDIR+=	calendar
 .endif
 
 .if ${MK_CLANG} != "no"
 _clang=		clang
 .endif
 
 .if ${MK_FMAKE} != "no"
 SUBDIR+=	make
 .endif
 
 .if ${MK_GPL_DTC} != "yes"
 SUBDIR+=	dtc
 .endif
 
 .if ${MK_GROFF} != "no"
 SUBDIR+=	vgrind
 .endif
 
 .if ${MK_HESIOD} != "no"
 SUBDIR+=	hesinfo
 .endif
 
 .if ${MK_ICONV} != "no"
 _iconv=		iconv
 _mkcsmapper=	mkcsmapper
 _mkesdb=	mkesdb
 .endif
 
 .if ${MK_KDUMP} != "no"
 SUBDIR+=        kdump
 SUBDIR+=        truss
 .endif
 
 .if ${MK_KERBEROS_SUPPORT} != "no"
 SUBDIR+=	compile_et
 .endif
 
 .if ${MK_LDNS_UTILS} != "no"
 SUBDIR+=	drill
 SUBDIR+=	host
 .endif
 
 .if ${MK_LIBTHR} != "no"
 SUBDIR+=	csup
 .endif
 
 .if ${MK_LOCATE} != "no"
 SUBDIR+=	locate
 .endif
 
 # XXX msgs?
 .if ${MK_MAIL} != "no"
 SUBDIR+=	biff
 SUBDIR+=	from
 SUBDIR+=	mail
 SUBDIR+=	msgs
 .endif
 
 .if ${MK_MAKE} != "no"
 SUBDIR+=	bmake
 .endif
 
 .if ${MK_MAN_UTILS} != "no"
 SUBDIR+=	catman
 _makewhatis=	makewhatis
 _man=		man
 .endif
 
 .if ${MK_NETCAT} != "no"
 SUBDIR+=	nc
 .endif
 
 .if ${MK_NIS} != "no"
 SUBDIR+=	ypcat
 SUBDIR+=	ypmatch
 SUBDIR+=	ypwhich
 .endif
 
 .if ${MK_OPENSSH} != "no"
 SUBDIR+=	ssh-copy-id
 .endif
 
 .if ${MK_OPENSSL} != "no"
 SUBDIR+=	bc
 SUBDIR+=	chkey
 SUBDIR+=	dc
 SUBDIR+=	newkey
 .endif
 
 .if ${MK_QUOTAS} != "no"
 SUBDIR+=	quota
 .endif
 
 .if ${MK_RCMDS} != "no"
 SUBDIR+=	rlogin
 SUBDIR+=	rsh
 SUBDIR+=	ruptime
 SUBDIR+=	rwho
 .endif
 
 .if ${MK_SENDMAIL} != "no"
 SUBDIR+=	vacation
 .endif
 
 .if ${MK_TELNET} != "no"
 SUBDIR+=	telnet
 .endif
 
 .if ${MK_TESTS} != "no"
 _tests=		tests
 .endif
 
 .if ${MK_TEXTPROC} != "no"
 SUBDIR+=	checknr
 SUBDIR+=	colcrt
 SUBDIR+=	ul
 .endif
 
 .if ${MK_TOOLCHAIN} != "no"
 SUBDIR+=	ar
 SUBDIR+=	c89
 SUBDIR+=	c99
 SUBDIR+=	ctags
 SUBDIR+=	file2c
 SUBDIR+=	gprof
 SUBDIR+=	indent
 SUBDIR+=	lex
 SUBDIR+=	mkstr
 SUBDIR+=	rpcgen
 SUBDIR+=	unifdef
 SUBDIR+=	xlint
 SUBDIR+=	xstr
 SUBDIR+=	yacc
 .endif
 
 .if ${MK_VI} != "no"
 SUBDIR+=	vi
 .endif
 
+.if ${MK_VT_SUPPORT} != "no"
+SUBDIR+=	vtfontcvt
+.endif
+
 .if ${MK_USB} != "no"
 SUBDIR+=	usbhidaction
 SUBDIR+=	usbhidctl
 .endif
 
 .if ${MK_UTMPX} != "no"
 SUBDIR+=	last
 SUBDIR+=	users
 SUBDIR+=	who
 .endif
 
 .if ${MK_SVN} == "yes" || ${MK_SVNLITE} == "yes"
 SUBDIR+=	svn
 .endif
 
 .include <bsd.arch.inc.mk>
 
 SUBDIR:=	${SUBDIR:O}
 
 SUBDIR_PARALLEL=
 
 .include <bsd.subdir.mk>
Index: projects/release-debugdist/usr.bin/vtfontcvt/Makefile
===================================================================
--- projects/release-debugdist/usr.bin/vtfontcvt/Makefile	(nonexistent)
+++ projects/release-debugdist/usr.bin/vtfontcvt/Makefile	(revision 267344)
@@ -0,0 +1,8 @@
+# $FreeBSD$
+
+PROG=	vtfontcvt
+MAN8=	vtfontcvt.8
+
+WARNS?=	6
+
+.include <bsd.prog.mk>

Property changes on: projects/release-debugdist/usr.bin/vtfontcvt/Makefile
___________________________________________________________________
Added: svn:keywords
## -0,0 +1 ##
+FreeBSD=%H
\ No newline at end of property
Index: projects/release-debugdist/usr.bin/vtfontcvt/vtfontcvt.8
===================================================================
--- projects/release-debugdist/usr.bin/vtfontcvt/vtfontcvt.8	(nonexistent)
+++ projects/release-debugdist/usr.bin/vtfontcvt/vtfontcvt.8	(revision 267344)
@@ -0,0 +1,74 @@
+.\" Copyright (c) 2014 The FreeBSD Foundation.  All rights reserved.
+.\"
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd June 9, 2014
+.Dt VTFONTCVT 8
+.Os
+.Sh NAME
+.Nm vtfontcvt
+.Nd "convert font files for use by the video console"
+.Sh SYNOPSIS
+.Nm
+.Op Fl h Ar height
+.Op Fl v
+.Op Fl w Ar width
+.Ar normal_font
+.Op Ar bold_font
+.Ar output_file
+.Sh DESCRIPTION
+The
+.Nm
+utility reads source font files in either BDF or Unifont HEX format and
+outputs a binary font file for use by
+.Xr vt 4 .
+HEX format files must have the file extension
+.Pa .hex .
+.Pp
+The following options are available:
+.Bl -tag -width "12345678"
+.It Fl h Ar height
+Set font height.
+The default is 16.
+Font height is set automatically for HEX files that have a
+.Ql # Height: Ar height
+comment before any font data.
+.It Fl v
+Display verbose statistics about the converted font.
+.It Fl w Ar width
+Set font width.
+The default is 8.
+Font width is set automatically for HEX files that have a
+.Ql # Width: Ar width
+comment before any font data.
+.El
+.Sh SEE ALSO
+.Xr vidcontrol 1 ,
+.Xr vt 4
+.Sh HISTORY
+The
+.Nm
+utility first appeared in
+.Fx 11.0 .

Property changes on: projects/release-debugdist/usr.bin/vtfontcvt/vtfontcvt.8
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Added: svn:keywords
## -0,0 +1 ##
+FreeBSD=%H
\ No newline at end of property
Added: svn:mime-type
## -0,0 +1 ##
+text/plain
\ No newline at end of property
Index: projects/release-debugdist/usr.bin/vtfontcvt/vtfontcvt.c
===================================================================
--- projects/release-debugdist/usr.bin/vtfontcvt/vtfontcvt.c	(nonexistent)
+++ projects/release-debugdist/usr.bin/vtfontcvt/vtfontcvt.c	(revision 267344)
@@ -0,0 +1,568 @@
+/*-
+ * Copyright (c) 2009, 2014 The FreeBSD Foundation
+ * All rights reserved.
+ *
+ * This software was developed by Ed Schouten under sponsorship from the
+ * FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/types.h>
+#include <sys/fnv_hash.h>
+#include <sys/endian.h>
+#include <sys/param.h>
+#include <sys/queue.h>
+
+#include <assert.h>
+#include <err.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#define VFNT_MAPS 4
+#define VFNT_MAP_NORMAL 0
+#define VFNT_MAP_NORMAL_RH 1
+#define VFNT_MAP_BOLD 2
+#define VFNT_MAP_BOLD_RH 3
+
+static unsigned int width = 8, wbytes, height = 16;
+
+struct glyph {
+	TAILQ_ENTRY(glyph)	 g_list;
+	SLIST_ENTRY(glyph)	 g_hash;
+	uint8_t			*g_data;
+	unsigned int		 g_index;
+};
+
+#define	FONTCVT_NHASH 4096
+TAILQ_HEAD(glyph_list, glyph);
+static SLIST_HEAD(, glyph) glyph_hash[FONTCVT_NHASH];
+static struct glyph_list glyphs[VFNT_MAPS] = {
+    TAILQ_HEAD_INITIALIZER(glyphs[0]),
+    TAILQ_HEAD_INITIALIZER(glyphs[1]),
+    TAILQ_HEAD_INITIALIZER(glyphs[2]),
+    TAILQ_HEAD_INITIALIZER(glyphs[3]),
+};
+static unsigned int glyph_total, glyph_count[4], glyph_unique, glyph_dupe;
+
+struct mapping {
+	TAILQ_ENTRY(mapping)	 m_list;
+	unsigned int		 m_char;
+	unsigned int		 m_length;
+	struct glyph		*m_glyph;
+};
+
+TAILQ_HEAD(mapping_list, mapping);
+static struct mapping_list maps[VFNT_MAPS] = {
+    TAILQ_HEAD_INITIALIZER(maps[0]),
+    TAILQ_HEAD_INITIALIZER(maps[1]),
+    TAILQ_HEAD_INITIALIZER(maps[2]),
+    TAILQ_HEAD_INITIALIZER(maps[3]),
+};
+static unsigned int mapping_total, map_count[4], map_folded_count[4],
+    mapping_unique, mapping_dupe;
+
+static void
+usage(void)
+{
+
+	errx(1,
+"usage: vtfontcvt [-w width] [-h height] [-v] normal.bdf [bold.bdf] out.fnt\n");
+	exit(1);
+}
+
+static int
+add_mapping(struct glyph *gl, unsigned int c, unsigned int map_idx)
+{
+	struct mapping *mp;
+	struct mapping_list *ml;
+
+	mapping_total++;
+
+	mp = malloc(sizeof *mp);
+	mp->m_char = c;
+	mp->m_glyph = gl;
+	mp->m_length = 0;
+
+	ml = &maps[map_idx];
+	if (TAILQ_LAST(ml, mapping_list) != NULL &&
+	    TAILQ_LAST(ml, mapping_list)->m_char >= c) {
+		errx(1, "Bad ordering at character %u\n", c);
+		return (1);
+	}
+	TAILQ_INSERT_TAIL(ml, mp, m_list);
+
+	map_count[map_idx]++;
+	mapping_unique++;
+
+	return (0);
+}
+
+static int
+dedup_mapping(unsigned int map_idx)
+{
+	struct mapping *mp_bold, *mp_normal, *mp_temp;
+	unsigned normal_map_idx = map_idx - VFNT_MAP_BOLD;
+
+	assert(map_idx == VFNT_MAP_BOLD || map_idx == VFNT_MAP_BOLD_RH);
+	mp_normal = TAILQ_FIRST(&maps[normal_map_idx]);
+	TAILQ_FOREACH_SAFE(mp_bold, &maps[map_idx], m_list, mp_temp) {
+		while (mp_normal->m_char < mp_bold->m_char)
+			mp_normal = TAILQ_NEXT(mp_normal, m_list);
+		if (mp_bold->m_char != mp_normal->m_char) {
+			errx(1, "Character %u not in normal font!\n",
+			    mp_bold->m_char);
+			return (1);
+		}
+		if (mp_bold->m_glyph != mp_normal->m_glyph)
+			continue;
+
+		/* No mapping is needed if it's equal to the normal mapping. */
+		TAILQ_REMOVE(&maps[map_idx], mp_bold, m_list);
+		free(mp_bold);
+		mapping_dupe++;
+	}
+	return (0);
+}
+
+static struct glyph *
+add_glyph(const uint8_t *bytes, unsigned int map_idx, int fallback)
+{
+	struct glyph *gl;
+	int hash;
+
+	glyph_total++;
+	glyph_count[map_idx]++;
+
+	hash = fnv_32_buf(bytes, wbytes * height, FNV1_32_INIT) % FONTCVT_NHASH;
+	SLIST_FOREACH(gl, &glyph_hash[hash], g_hash) {
+		if (memcmp(gl->g_data, bytes, wbytes * height) == 0) {
+			glyph_dupe++;
+			return (gl);
+		}
+	}
+
+	gl = malloc(sizeof *gl);
+	gl->g_data = malloc(wbytes * height);
+	memcpy(gl->g_data, bytes, wbytes * height);
+	if (fallback)
+		TAILQ_INSERT_HEAD(&glyphs[map_idx], gl, g_list);
+	else
+		TAILQ_INSERT_TAIL(&glyphs[map_idx], gl, g_list);
+	SLIST_INSERT_HEAD(&glyph_hash[hash], gl, g_hash);
+
+	glyph_unique++;
+	return (gl);
+}
+
+static int
+add_char(unsigned curchar, unsigned map_idx, uint8_t *bytes, uint8_t *bytes_r)
+{
+	struct glyph *gl;
+
+	/* Prevent adding two glyphs for 0xFFFD */
+	if (curchar == 0xFFFD) {
+		if (map_idx < VFNT_MAP_BOLD)
+			gl = add_glyph(bytes, 0, 1);
+	} else if (curchar >= 0x20) {
+		gl = add_glyph(bytes, map_idx, 0);
+		if (add_mapping(gl, curchar, map_idx) != 0)
+			return (1);
+		if (bytes_r != NULL) {
+			gl = add_glyph(bytes_r, map_idx + 1, 0);
+			if (add_mapping(gl, curchar,
+			    map_idx + 1) != 0)
+				return (1);
+		}
+	}
+	return (0);
+}
+
+
+static int
+parse_bitmap_line(uint8_t *left, uint8_t *right, unsigned int line,
+    unsigned int dwidth)
+{
+	uint8_t *p;
+	unsigned int i, subline;
+
+	if (dwidth != width && dwidth != width * 2) {
+		errx(1,
+		    "Bitmap with unsupported width %u!\n", dwidth);
+		return (1);
+	}
+
+	/* Move pixel data right to simplify splitting double characters. */
+	line >>= (howmany(dwidth, 8) * 8) - dwidth;
+
+	for (i = dwidth / width; i > 0; i--) {
+		p = (i == 2) ? right : left;
+
+		subline = line & ((1 << width) - 1);
+		subline <<= (howmany(width, 8) * 8) - width;
+
+		if (wbytes == 1) {
+			*p = subline;
+		} else if (wbytes == 2) {
+			*p++ = subline >> 8;
+			*p = subline;
+		} else {
+			errx(1,
+			    "Unsupported wbytes %u!\n", wbytes);
+			return (1);
+		}
+
+		line >>= width;
+	}
+
+	return (0);
+}
+
+static int
+parse_bdf(FILE *fp, unsigned int map_idx)
+{
+	char *ln;
+	size_t length;
+	uint8_t bytes[wbytes * height], bytes_r[wbytes * height];
+	unsigned int curchar = 0, dwidth = 0, i, line;
+
+	while ((ln = fgetln(fp, &length)) != NULL) {
+		ln[length - 1] = '\0';
+
+		if (strncmp(ln, "ENCODING ", 9) == 0) {
+			curchar = atoi(ln + 9);
+		}
+
+		if (strncmp(ln, "DWIDTH ", 7) == 0) {
+			dwidth = atoi(ln + 7);
+		}
+
+		if (strncmp(ln, "BITMAP", 6) == 0 &&
+		    (ln[6] == ' ' || ln[6] == '\0')) {
+			for (i = 0; i < height; i++) {
+				if ((ln = fgetln(fp, &length)) == NULL) {
+					errx(1, "Unexpected EOF!\n");
+					return (1);
+				}
+				ln[length - 1] = '\0';
+				sscanf(ln, "%x", &line);
+				if (parse_bitmap_line(bytes + i * wbytes,
+				     bytes_r + i * wbytes, line, dwidth) != 0)
+					return (1);
+			}
+
+			if (add_char(curchar, map_idx, bytes,
+			    dwidth == width * 2 ? bytes_r : NULL) != 0)
+				return (1);
+		}
+	}
+
+	return (0);
+}
+
+static int
+parse_hex(FILE *fp, unsigned int map_idx)
+{
+	char *ln, *p;
+	char fmt_str[8];
+	size_t length;
+	uint8_t bytes[wbytes * height], bytes_r[wbytes * height];
+	unsigned curchar = 0, i, line, chars_per_row, dwidth;
+
+	while ((ln = fgetln(fp, &length)) != NULL) {
+		ln[length - 1] = '\0';
+
+		if (strncmp(ln, "# Height: ", 10) == 0) {
+			height = atoi(ln + 10);
+		} else if (strncmp(ln, "# Width: ", 9) == 0) {
+			width = atoi(ln + 9);
+		} else if (sscanf(ln, "%4x:", &curchar)) {
+			p = ln + 5;
+			chars_per_row = strlen(p) / height;
+			dwidth = width;
+			if (chars_per_row / 2 > width / 8)
+				dwidth *= 2; /* Double-width character. */
+			snprintf(fmt_str, sizeof(fmt_str), "%%%ux",
+			    chars_per_row);
+
+			for (i = 0; i < height; i++) {
+				sscanf(p, fmt_str, &line);
+				p += chars_per_row;
+				if (parse_bitmap_line(bytes + i * wbytes,
+				    bytes_r + i * wbytes, line, dwidth) != 0)
+					return (1);
+			}
+
+			if (add_char(curchar, map_idx, bytes,
+			    dwidth == width * 2 ? bytes_r : NULL) != 0)
+				return (1);
+		}
+	}
+	return (0);
+}
+
+static int
+parse_file(const char *filename, unsigned int map_idx)
+{
+	FILE *fp;
+	size_t len;
+
+	fp = fopen(filename, "r");
+	if (fp == NULL) {
+		perror(filename);
+		return (1);
+	}
+	len = strlen(filename);
+	if (len > 4 && strcasecmp(filename + len - 4, ".hex") == 0)
+		return parse_hex(fp, map_idx);
+	return parse_bdf(fp, map_idx);
+}
+
+static void
+number_glyphs(void)
+{
+	struct glyph *gl;
+	unsigned int i, idx = 0;
+
+	for (i = 0; i < VFNT_MAPS; i++)
+		TAILQ_FOREACH(gl, &glyphs[i], g_list)
+			gl->g_index = idx++;
+}
+
+static int
+write_glyphs(FILE *fp)
+{
+	struct glyph *gl;
+	unsigned int i;
+
+	for (i = 0; i < VFNT_MAPS; i++) {
+		TAILQ_FOREACH(gl, &glyphs[i], g_list)
+			if (fwrite(gl->g_data, wbytes * height, 1, fp) != 1)
+				return (1);
+	}
+	return (0);
+}
+
+static void
+fold_mappings(unsigned int map_idx)
+{
+	struct mapping_list *ml = &maps[map_idx];
+	struct mapping *mn, *mp, *mbase;
+
+	mp = mbase = TAILQ_FIRST(ml);
+	for (mp = mbase = TAILQ_FIRST(ml); mp != NULL; mp = mn) {
+		mn = TAILQ_NEXT(mp, m_list);
+		if (mn != NULL && mn->m_char == mp->m_char + 1 &&
+		    mn->m_glyph->g_index == mp->m_glyph->g_index + 1)
+			continue;
+		mbase->m_length = mp->m_char - mbase->m_char + 1;
+		mbase = mp = mn;
+		map_folded_count[map_idx]++;
+	}
+}
+
+struct file_mapping {
+	uint32_t	source;
+	uint16_t	destination;
+	uint16_t	length;
+} __packed;
+
+static int
+write_mappings(FILE *fp, unsigned int map_idx)
+{
+	struct mapping_list *ml = &maps[map_idx];
+	struct mapping *mp;
+	struct file_mapping fm;
+	unsigned int i = 0, j = 0;
+
+	TAILQ_FOREACH(mp, ml, m_list) {
+		j++;
+		if (mp->m_length > 0) {
+			i += mp->m_length;
+			fm.source = htobe32(mp->m_char);
+			fm.destination = htobe16(mp->m_glyph->g_index);
+			fm.length = htobe16(mp->m_length - 1);
+			if (fwrite(&fm, sizeof fm, 1, fp) != 1)
+				return (1);
+		}
+	}
+	assert(i == j);
+	return (0);
+}
+
+struct file_header {
+	uint8_t		magic[8];
+	uint8_t		width;
+	uint8_t		height;
+	uint16_t	pad;
+	uint32_t	glyph_count;
+	uint32_t	map_count[4];
+} __packed;
+
+static int
+write_fnt(const char *filename)
+{
+	FILE *fp;
+	struct file_header fh = {
+		.magic = "VFNT0002",
+	};
+
+	fp = fopen(filename, "wb");
+	if (fp == NULL) {
+		perror(filename);
+		return (1);
+	}
+
+	fh.width = width;
+	fh.height = height;
+	fh.glyph_count = htobe32(glyph_unique);
+	fh.map_count[0] = htobe32(map_folded_count[0]);
+	fh.map_count[1] = htobe32(map_folded_count[1]);
+	fh.map_count[2] = htobe32(map_folded_count[2]);
+	fh.map_count[3] = htobe32(map_folded_count[3]);
+	if (fwrite(&fh, sizeof fh, 1, fp) != 1) {
+		perror(filename);
+		return (1);
+	}
+
+	if (write_glyphs(fp) != 0 ||
+	    write_mappings(fp, VFNT_MAP_NORMAL) != 0 ||
+	    write_mappings(fp, 1) != 0 ||
+	    write_mappings(fp, VFNT_MAP_BOLD) != 0 ||
+	    write_mappings(fp, 3) != 0) {
+		perror(filename);
+		return (1);
+	}
+
+	return (0);
+}
+
+static void
+print_font_info(void)
+{
+	printf(
+"Statistics:\n"
+"- glyph_total:                 %5u\n"
+"- glyph_normal:                %5u\n"
+"- glyph_normal_right:          %5u\n"
+"- glyph_bold:                  %5u\n"
+"- glyph_bold_right:            %5u\n"
+"- glyph_unique:                %5u\n"
+"- glyph_dupe:                  %5u\n"
+"- mapping_total:               %5u\n"
+"- mapping_normal:              %5u\n"
+"- mapping_normal_folded:       %5u\n"
+"- mapping_normal_right:        %5u\n"
+"- mapping_normal_right_folded: %5u\n"
+"- mapping_bold:                %5u\n"
+"- mapping_bold_folded:         %5u\n"
+"- mapping_bold_right:          %5u\n"
+"- mapping_bold_right_folded:   %5u\n"
+"- mapping_unique:              %5u\n"
+"- mapping_dupe:                %5u\n",
+	    glyph_total,
+	    glyph_count[0],
+	    glyph_count[1],
+	    glyph_count[2],
+	    glyph_count[3],
+	    glyph_unique, glyph_dupe,
+	    mapping_total,
+	    map_count[0], map_folded_count[0],
+	    map_count[1], map_folded_count[1],
+	    map_count[2], map_folded_count[2],
+	    map_count[3], map_folded_count[3],
+	    mapping_unique, mapping_dupe);
+}
+
+int
+main(int argc, char *argv[])
+{
+	int ch, val, verbose = 0;
+
+	assert(sizeof(struct file_header) == 32);
+	assert(sizeof(struct file_mapping) == 8);
+
+	while ((ch = getopt(argc, argv, "h:vw:")) != -1) {
+		switch (ch) {
+		case 'h':
+			val = atoi(optarg);
+			if (val <= 0 || val > 128) {
+				errx(1, "Invalid height %d", val);
+				return (1);
+			}
+			height = val;
+			break;
+		case 'v':
+			verbose = 1;
+			break;
+		case 'w':
+			val = atoi(optarg);
+			if (val <= 0 || val > 128) {
+				errx(1, "Invalid width %d", val);
+				return (1);
+			}
+			width = val;
+			break;
+		case '?':
+		default:
+			usage();
+		}
+	}
+	argc -= optind;
+	argv += optind;
+
+	if (argc < 2 || argc > 3)
+		usage();
+
+	wbytes = howmany(width, 8);
+
+	if (parse_file(argv[0], VFNT_MAP_NORMAL) != 0)
+		return (1);
+	argc--;
+	argv++;
+	if (argc == 2) {
+		if (parse_file(argv[0], VFNT_MAP_BOLD) != 0)
+			return (1);
+		argc--;
+		argv++;
+	}
+	number_glyphs();
+	dedup_mapping(VFNT_MAP_BOLD);
+	dedup_mapping(VFNT_MAP_BOLD_RH);
+	fold_mappings(0);
+	fold_mappings(1);
+	fold_mappings(2);
+	fold_mappings(3);
+	if (write_fnt(argv[0]) != 0)
+		return (1);
+
+	if (verbose)
+		print_font_info();
+
+	return (0);
+}

Property changes on: projects/release-debugdist/usr.bin/vtfontcvt/vtfontcvt.c
___________________________________________________________________
Added: svn:keywords
## -0,0 +1 ##
+FreeBSD=%H
\ No newline at end of property
Index: projects/release-debugdist
===================================================================
--- projects/release-debugdist	(revision 267343)
+++ projects/release-debugdist	(revision 267344)

Property changes on: projects/release-debugdist
___________________________________________________________________
Modified: svn:mergeinfo
## -0,0 +0,1 ##
   Merged /head:r267325-267343