Index: user/ngie/release-pkg-fix-tests/Makefile.inc1 =================================================================== --- user/ngie/release-pkg-fix-tests/Makefile.inc1 (revision 299090) +++ user/ngie/release-pkg-fix-tests/Makefile.inc1 (revision 299091) @@ -1,2486 +1,2488 @@ # # $FreeBSD$ # # Make command line options: # -DNO_CLEANDIR run ${MAKE} clean, instead of ${MAKE} cleandir # -DNO_CLEAN do not clean at all # -DDB_FROM_SRC use the user/group databases in src/etc instead of # the system database when installing. # -DNO_SHARE do not go into share subdir # -DKERNFAST define NO_KERNEL{CONFIG,CLEAN,OBJ} # -DNO_KERNELCONFIG do not run config in ${MAKE} buildkernel # -DNO_KERNELCLEAN do not run ${MAKE} clean in ${MAKE} buildkernel # -DNO_KERNELOBJ do not run ${MAKE} obj in ${MAKE} buildkernel # -DNO_PORTSUPDATE do not update ports in ${MAKE} update # -DNO_ROOT install without using root privilege # -DNO_DOCUPDATE do not update doc in ${MAKE} update # -DWITHOUT_CTF do not run the DTrace CTF conversion tools on built objects # LOCAL_DIRS="list of dirs" to add additional dirs to the SUBDIR list # LOCAL_ITOOLS="list of tools" to add additional tools to the ITOOLS list # LOCAL_LIB_DIRS="list of dirs" to add additional dirs to libraries target # LOCAL_MTREE="list of mtree files" to process to allow local directories # to be created before files are installed # LOCAL_TOOL_DIRS="list of dirs" to add additional dirs to the build-tools # list # METALOG="path to metadata log" to write permission and ownership # when NO_ROOT is set. (default: ${DESTDIR}/METALOG) # TARGET="machine" to crossbuild world for a different machine type # TARGET_ARCH= may be required when a TARGET supports multiple endians # BUILDENV_SHELL= shell to launch for the buildenv target (def:${SHELL}) # WORLD_FLAGS= additional flags to pass to make(1) during buildworld # KERNEL_FLAGS= additional flags to pass to make(1) during buildkernel # SUBDIR_OVERRIDE="list of dirs" to build rather than everything. # All libraries and includes, and some build tools will still build. # # The intended user-driven targets are: # buildworld - rebuild *everything*, including glue to help do upgrades # installworld- install everything built by "buildworld" # checkworld - run test suite on installed world # doxygen - build API documentation of the kernel # update - convenient way to update your source tree (eg: svn/svnup) # # Standard targets (not defined here) are documented in the makefiles in # /usr/share/mk. These include: # obj depend all install clean cleandepend cleanobj .if !defined(TARGET) || !defined(TARGET_ARCH) .error "Both TARGET and TARGET_ARCH must be defined." .endif LOCALBASE?= /usr/local # Cross toolchain changes must be in effect before bsd.compiler.mk # so that gets the right CC, and pass CROSS_TOOLCHAIN to submakes. .if defined(CROSS_TOOLCHAIN) .include "${LOCALBASE}/share/toolchains/${CROSS_TOOLCHAIN}.mk" CROSSENV+=CROSS_TOOLCHAIN="${CROSS_TOOLCHAIN}" .endif .include # don't depend on src.opts.mk doing it .include "share/mk/src.opts.mk" # We must do lib/ and libexec/ before bin/ in case of a mid-install error to # keep the users system reasonably usable. For static->dynamic root upgrades, # we don't want to install a dynamic binary without rtld and the needed # libraries. More commonly, for dynamic root, we don't want to install a # binary that requires a newer library version that hasn't been installed yet. # This ordering is not a guarantee though. The only guarantee of a working # system here would require fine-grained ordering of all components based # on their dependencies. SRCDIR?= ${.CURDIR} .if !empty(SUBDIR_OVERRIDE) SUBDIR= ${SUBDIR_OVERRIDE} .else SUBDIR= lib libexec .if !defined(NO_ROOT) && (make(installworld) || make(install)) # Ensure libraries are installed before progressing. SUBDIR+=.WAIT .endif SUBDIR+=bin .if ${MK_CDDL} != "no" SUBDIR+=cddl .endif SUBDIR+=gnu include .if ${MK_KERBEROS} != "no" SUBDIR+=kerberos5 .endif .if ${MK_RESCUE} != "no" SUBDIR+=rescue .endif SUBDIR+=sbin .if ${MK_CRYPT} != "no" SUBDIR+=secure .endif .if !defined(NO_SHARE) SUBDIR+=share .endif SUBDIR+=sys usr.bin usr.sbin .if ${MK_TESTS} != "no" SUBDIR+= tests .endif .if ${MK_OFED} != "no" SUBDIR+=contrib/ofed .endif # Local directories are last, since it is nice to at least get the base # system rebuilt before you do them. .for _DIR in ${LOCAL_DIRS} .if exists(${.CURDIR}/${_DIR}/Makefile) SUBDIR+= ${_DIR} .endif .endfor # Add LOCAL_LIB_DIRS, but only if they will not be picked up as a SUBDIR # of a LOCAL_DIRS directory. This allows LOCAL_DIRS=foo and # LOCAL_LIB_DIRS=foo/lib to behave as expected. .for _DIR in ${LOCAL_DIRS:M*/} ${LOCAL_DIRS:N*/:S|$|/|} _REDUNDENT_LIB_DIRS+= ${LOCAL_LIB_DIRS:M${_DIR}*} .endfor .for _DIR in ${LOCAL_LIB_DIRS} .if empty(_REDUNDENT_LIB_DIRS:M${_DIR}) && exists(${.CURDIR}/${_DIR}/Makefile) SUBDIR+= ${_DIR} .else .warning ${_DIR} not added to SUBDIR list. See UPDATING 20141121. .endif .endfor # We must do etc/ last as it hooks into building the man whatis file # by calling 'makedb' in share/man. This is only relevant for # install/distribute so they build the whatis file after every manpage is # installed. .if make(installworld) || make(install) SUBDIR+=.WAIT .endif SUBDIR+=etc .endif # !empty(SUBDIR_OVERRIDE) .if defined(NOCLEAN) .warning NOCLEAN option is deprecated. Use NO_CLEAN instead. NO_CLEAN= ${NOCLEAN} .endif .if defined(NO_CLEANDIR) CLEANDIR= clean cleandepend .else CLEANDIR= cleandir .endif .if ${MK_META_MODE} == "yes" # If filemon is used then we can rely on the build being incremental-safe. # The .meta files will also track the build command and rebuild should # it change. .if empty(.MAKE.MODE:Mnofilemon) NO_CLEAN= t .endif .endif LOCAL_TOOL_DIRS?= PACKAGEDIR?= ${DESTDIR}/${DISTDIR} .if empty(SHELL:M*csh*) BUILDENV_SHELL?=${SHELL} .else BUILDENV_SHELL?=/bin/sh .endif .if !defined(SVN) || empty(SVN) . for _P in /usr/bin /usr/local/bin . for _S in svn svnlite . if exists(${_P}/${_S}) SVN= ${_P}/${_S} . endif . endfor . endfor .endif SVNFLAGS?= -r HEAD MAKEOBJDIRPREFIX?= /usr/obj .if !defined(OSRELDATE) .if exists(/usr/include/osreldate.h) OSRELDATE!= awk '/^\#define[[:space:]]*__FreeBSD_version/ { print $$3 }' \ /usr/include/osreldate.h .else OSRELDATE= 0 .endif .export OSRELDATE .endif # Set VERSION for CTFMERGE to use via the default CTFFLAGS=-L VERSION. .if !defined(_REVISION) _REVISION!= MK_AUTO_OBJ=no ${MAKE} -C ${SRCDIR}/release -V REVISION .export _REVISION .endif .if !defined(_BRANCH) _BRANCH!= MK_AUTO_OBJ=no ${MAKE} -C ${SRCDIR}/release -V BRANCH .export _BRANCH .endif .if !defined(SRCRELDATE) SRCRELDATE!= awk '/^\#define[[:space:]]*__FreeBSD_version/ { print $$3 }' \ ${SRCDIR}/sys/sys/param.h .export SRCRELDATE .endif .if !defined(VERSION) VERSION= FreeBSD ${_REVISION}-${_BRANCH:C/-p[0-9]+$//} ${TARGET_ARCH} ${SRCRELDATE} .export VERSION .endif .if !defined(PKG_VERSION) .if ${_BRANCH:MSTABLE*} || ${_BRANCH:MCURRENT*} TIMENOW= %Y%m%d%H%M%S EXTRA_REVISION= .s${TIMENOW:gmtime} .endif .if ${_BRANCH:M*-p*} EXTRA_REVISION= _${_BRANCH:C/.*-p([0-9]+$)/\1/} .endif PKG_VERSION= ${_REVISION}${EXTRA_REVISION} .endif KNOWN_ARCHES?= aarch64/arm64 \ amd64 \ arm \ armeb/arm \ armv6/arm \ armv6hf/arm \ i386 \ i386/pc98 \ mips \ mipsel/mips \ mips64el/mips \ mips64/mips \ mipsn32el/mips \ mipsn32/mips \ powerpc \ powerpc64/powerpc \ riscv64/riscv \ sparc64 .if ${TARGET} == ${TARGET_ARCH} _t= ${TARGET} .else _t= ${TARGET_ARCH}/${TARGET} .endif .for _t in ${_t} .if empty(KNOWN_ARCHES:M${_t}) .error Unknown target ${TARGET_ARCH}:${TARGET}. .endif .endfor .if ${TARGET} == ${MACHINE} TARGET_CPUTYPE?=${CPUTYPE} .else TARGET_CPUTYPE?= .endif .if !empty(TARGET_CPUTYPE) _TARGET_CPUTYPE=${TARGET_CPUTYPE} .else _TARGET_CPUTYPE=dummy .endif _CPUTYPE!= MK_AUTO_OBJ=no MAKEFLAGS= CPUTYPE=${_TARGET_CPUTYPE} ${MAKE} \ -f /dev/null -m ${.CURDIR}/share/mk -V CPUTYPE .if ${_CPUTYPE} != ${_TARGET_CPUTYPE} .error CPUTYPE global should be set with ?=. .endif .if make(buildworld) BUILD_ARCH!= uname -p .if ${MACHINE_ARCH} != ${BUILD_ARCH} .error To cross-build, set TARGET_ARCH. .endif .endif .if ${MACHINE} == ${TARGET} && ${MACHINE_ARCH} == ${TARGET_ARCH} && !defined(CROSS_BUILD_TESTING) OBJTREE= ${MAKEOBJDIRPREFIX} .else OBJTREE= ${MAKEOBJDIRPREFIX}/${TARGET}.${TARGET_ARCH} .endif WORLDTMP= ${OBJTREE}${.CURDIR}/tmp BPATH= ${WORLDTMP}/legacy/usr/sbin:${WORLDTMP}/legacy/usr/bin:${WORLDTMP}/legacy/bin XPATH= ${WORLDTMP}/usr/sbin:${WORLDTMP}/usr/bin STRICTTMPPATH= ${BPATH}:${XPATH} TMPPATH= ${STRICTTMPPATH}:${PATH} # # Avoid running mktemp(1) unless actually needed. # It may not be functional, e.g., due to new ABI # when in the middle of installing over this system. # .if make(distributeworld) || make(installworld) || make(stageworld) INSTALLTMP!= /usr/bin/mktemp -d -u -t install .endif .if make(stagekernel) || make(distributekernel) TAGS+= kernel PACKAGE= kernel .endif # # Building a world goes through the following stages # # 1. legacy stage [BMAKE] # This stage is responsible for creating compatibility # shims that are needed by the bootstrap-tools, # build-tools and cross-tools stages. These are generally # APIs that tools from one of those three stages need to # build that aren't present on the host. # 1. bootstrap-tools stage [BMAKE] # This stage is responsible for creating programs that # are needed for backward compatibility reasons. They # are not built as cross-tools. # 2. build-tools stage [TMAKE] # This stage is responsible for creating the object # tree and building any tools that are needed during # the build process. Some programs are listed during # this phase because they build binaries to generate # files needed to build these programs. This stage also # builds the 'build-tools' target rather than 'all'. # 3. cross-tools stage [XMAKE] # This stage is responsible for creating any tools that # are needed for building the system. A cross-compiler is one # of them. This differs from build tools in two ways: # 1. the 'all' target is built rather than 'build-tools' # 2. these tools are installed into TMPPATH for stage 4. # 4. world stage [WMAKE] # This stage actually builds the world. # 5. install stage (optional) [IMAKE] # This stage installs a previously built world. # BOOTSTRAPPING?= 0 # Common environment for world related stages CROSSENV+= MAKEOBJDIRPREFIX=${OBJTREE} \ MACHINE_ARCH=${TARGET_ARCH} \ MACHINE=${TARGET} \ CPUTYPE=${TARGET_CPUTYPE} .if ${MK_META_MODE} != "no" # Don't rebuild build-tools targets during normal build. CROSSENV+= BUILD_TOOLS_META=.NOMETA_CMP .endif .if ${MK_GROFF} != "no" CROSSENV+= GROFF_BIN_PATH=${WORLDTMP}/legacy/usr/bin \ GROFF_FONT_PATH=${WORLDTMP}/legacy/usr/share/groff_font \ GROFF_TMAC_PATH=${WORLDTMP}/legacy/usr/share/tmac .endif .if defined(TARGET_CFLAGS) CROSSENV+= ${TARGET_CFLAGS} .endif # bootstrap-tools stage BMAKEENV= INSTALL="sh ${.CURDIR}/tools/install.sh" \ TOOLS_PREFIX=${WORLDTMP} \ PATH=${BPATH}:${PATH} \ WORLDTMP=${WORLDTMP} \ MAKEFLAGS="-m ${.CURDIR}/tools/build/mk ${.MAKEFLAGS}" # need to keep this in sync with targets/pseudo/bootstrap-tools/Makefile BSARGS= DESTDIR= \ BOOTSTRAPPING=${OSRELDATE} \ SSP_CFLAGS= \ MK_HTML=no NO_LINT=yes MK_MAN=no \ -DNO_PIC MK_PROFILE=no -DNO_SHARED \ -DNO_CPU_CFLAGS MK_WARNS=no MK_CTF=no \ MK_CLANG_EXTRAS=no MK_CLANG_FULL=no \ MK_LLDB=no MK_TESTS=no \ MK_INCLUDES=yes BMAKE= MAKEOBJDIRPREFIX=${WORLDTMP} \ ${BMAKEENV} ${MAKE} ${WORLD_FLAGS} -f Makefile.inc1 \ ${BSARGS} # build-tools stage TMAKE= MAKEOBJDIRPREFIX=${OBJTREE} \ ${BMAKEENV} ${MAKE} ${WORLD_FLAGS} -f Makefile.inc1 \ TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \ DESTDIR= \ BOOTSTRAPPING=${OSRELDATE} \ SSP_CFLAGS= \ -DNO_LINT \ -DNO_CPU_CFLAGS MK_WARNS=no MK_CTF=no \ MK_CLANG_EXTRAS=no MK_CLANG_FULL=no \ MK_LLDB=no MK_TESTS=no # cross-tools stage XMAKE= TOOLS_PREFIX=${WORLDTMP} ${BMAKE} \ TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \ MK_GDB=no MK_TESTS=no # kernel-tools stage KTMAKEENV= INSTALL="sh ${.CURDIR}/tools/install.sh" \ PATH=${BPATH}:${PATH} \ WORLDTMP=${WORLDTMP} KTMAKE= TOOLS_PREFIX=${WORLDTMP} MAKEOBJDIRPREFIX=${WORLDTMP} \ ${KTMAKEENV} ${MAKE} ${WORLD_FLAGS} -f Makefile.inc1 \ DESTDIR= \ BOOTSTRAPPING=${OSRELDATE} \ SSP_CFLAGS= \ MK_HTML=no -DNO_LINT MK_MAN=no \ -DNO_PIC MK_PROFILE=no -DNO_SHARED \ -DNO_CPU_CFLAGS MK_WARNS=no MK_CTF=no # world stage WMAKEENV= ${CROSSENV} \ INSTALL="sh ${.CURDIR}/tools/install.sh" \ PATH=${TMPPATH} # make hierarchy HMAKE= PATH=${TMPPATH} ${MAKE} LOCAL_MTREE=${LOCAL_MTREE:Q} .if defined(NO_ROOT) HMAKE+= PATH=${TMPPATH} METALOG=${METALOG} -DNO_ROOT .endif .if defined(CROSS_TOOLCHAIN_PREFIX) CROSS_COMPILER_PREFIX?=${CROSS_TOOLCHAIN_PREFIX} CROSS_BINUTILS_PREFIX?=${CROSS_TOOLCHAIN_PREFIX} .endif # If we do not have a bootstrap binutils (because the in-tree one does not # support the target architecture), provide a default cross-binutils prefix. # This allows aarch64 builds, for example, to automatically use the # aarch64-binutils port or package. .if !make(showconfig) .if !empty(BROKEN_OPTIONS:MBINUTILS_BOOTSTRAP) && \ !defined(CROSS_BINUTILS_PREFIX) CROSS_BINUTILS_PREFIX=/usr/local/${TARGET_ARCH}-freebsd/bin/ .if !exists(${CROSS_BINUTILS_PREFIX}) .error In-tree binutils does not support the ${TARGET_ARCH} architecture. Install the ${TARGET_ARCH}-binutils port or package or set CROSS_BINUTILS_PREFIX. .endif .endif .endif XCOMPILERS= CC CXX CPP .for COMPILER in ${XCOMPILERS} .if defined(CROSS_COMPILER_PREFIX) X${COMPILER}?= ${CROSS_COMPILER_PREFIX}${${COMPILER}} .else X${COMPILER}?= ${${COMPILER}} .endif .endfor XBINUTILS= AS AR LD NM OBJCOPY OBJDUMP RANLIB SIZE STRINGS .for BINUTIL in ${XBINUTILS} .if defined(CROSS_BINUTILS_PREFIX) && \ exists(${CROSS_BINUTILS_PREFIX}${${BINUTIL}}) X${BINUTIL}?= ${CROSS_BINUTILS_PREFIX}${${BINUTIL}} .else X${BINUTIL}?= ${${BINUTIL}} .endif .endfor CROSSENV+= CC="${XCC} ${XCFLAGS}" CXX="${XCXX} ${XCFLAGS} ${XCXXFLAGS}" \ CPP="${XCPP} ${XCFLAGS}" \ AS="${XAS}" AR="${XAR}" LD="${XLD}" NM=${XNM} \ OBJDUMP=${XOBJDUMP} OBJCOPY="${XOBJCOPY}" \ RANLIB=${XRANLIB} STRINGS=${XSTRINGS} \ SIZE="${XSIZE}" .if defined(CROSS_BINUTILS_PREFIX) && exists(${CROSS_BINUTILS_PREFIX}) # In the case of xdev-build tools, CROSS_BINUTILS_PREFIX won't be a # directory, but the compiler will look in the right place for its # tools so we don't need to tell it where to look. BFLAGS+= -B${CROSS_BINUTILS_PREFIX} .endif # External compiler needs sysroot and target flags. .if ${XCC:N${CCACHE_BIN}:M/*} || ${MK_CROSS_COMPILER} == "no" .if !defined(CROSS_BINUTILS_PREFIX) || !exists(${CROSS_BINUTILS_PREFIX}) BFLAGS+= -B${WORLDTMP}/usr/bin .endif .if ${TARGET} == "arm" .if ${TARGET_ARCH:M*hf*} != "" TARGET_ABI= gnueabihf .else TARGET_ABI= gnueabi .endif .endif .if defined(X_COMPILER_TYPE) && ${X_COMPILER_TYPE} == gcc # GCC requires -isystem and -L when using a cross-compiler. XCFLAGS+= -isystem ${WORLDTMP}/usr/include -L${WORLDTMP}/usr/lib # Force using libc++ for external GCC. XCXXFLAGS+= -isystem ${WORLDTMP}/usr/include/c++/v1 -std=c++11 \ -nostdinc++ -L${WORLDTMP}/../lib/libc++ .else TARGET_ABI?= unknown TARGET_TRIPLE?= ${TARGET_ARCH:C/amd64/x86_64/}-${TARGET_ABI}-freebsd11.0 XCFLAGS+= -target ${TARGET_TRIPLE} .endif XCFLAGS+= --sysroot=${WORLDTMP} .else .endif # ${XCC:M/*} || ${MK_CROSS_COMPILER} == "no" .if !empty(BFLAGS) XCFLAGS+= ${BFLAGS} .endif .if ${MK_LIB32} != "no" && (${TARGET_ARCH} == "amd64" || \ ${TARGET_ARCH} == "powerpc64") LIBCOMPAT= 32 .include "Makefile.libcompat" .elif ${MK_LIBSOFT} != "no" && ${TARGET_ARCH} == "armv6" LIBCOMPAT= SOFT .include "Makefile.libcompat" .endif WMAKE= ${WMAKEENV} ${MAKE} ${WORLD_FLAGS} -f Makefile.inc1 DESTDIR=${WORLDTMP} IMAKEENV= ${CROSSENV} IMAKE= ${IMAKEENV} ${MAKE} -f Makefile.inc1 \ ${IMAKE_INSTALL} ${IMAKE_MTREE} .if empty(.MAKEFLAGS:M-n) IMAKEENV+= PATH=${STRICTTMPPATH}:${INSTALLTMP} \ LD_LIBRARY_PATH=${INSTALLTMP} \ PATH_LOCALE=${INSTALLTMP}/locale IMAKE+= __MAKE_SHELL=${INSTALLTMP}/sh .else IMAKEENV+= PATH=${TMPPATH}:${INSTALLTMP} .endif .if defined(DB_FROM_SRC) INSTALLFLAGS+= -N ${.CURDIR}/etc MTREEFLAGS+= -N ${.CURDIR}/etc .endif _INSTALL_DDIR= ${DESTDIR}/${DISTDIR} INSTALL_DDIR= ${_INSTALL_DDIR:S://:/:g:C:/$::} .if defined(NO_ROOT) METALOG?= ${DESTDIR}/${DISTDIR}/METALOG IMAKE+= -DNO_ROOT METALOG=${METALOG} INSTALLFLAGS+= -U -M ${METALOG} -D ${INSTALL_DDIR} MTREEFLAGS+= -W .endif .if defined(BUILD_PKGS) INSTALLFLAGS+= -h sha256 .endif .if defined(DB_FROM_SRC) || defined(NO_ROOT) IMAKE_INSTALL= INSTALL="install ${INSTALLFLAGS}" IMAKE_MTREE= MTREE_CMD="mtree ${MTREEFLAGS}" .endif # kernel stage KMAKEENV= ${WMAKEENV} KMAKE= ${KMAKEENV} ${MAKE} ${.MAKEFLAGS} ${KERNEL_FLAGS} KERNEL=${INSTKERNNAME} .if ${MK_META_MODE} == "yes" # meta mode normally is disallowed when building from curdir==objdir, but we # want to allow it for the kernel build. KMAKE+= .MAKE.MODE="${.MAKE.MODE} curdirOk=yes" .endif # # buildworld # # Attempt to rebuild the entire system, with reasonable chance of # success, regardless of how old your existing system is. # _worldtmp: .PHONY .if ${.CURDIR:C/[^,]//g} != "" # The m4 build of sendmail files doesn't like it if ',' is used # anywhere in the path of it's files. @echo @echo "*** Error: path to source tree contains a comma ','" @echo false .endif @echo @echo "--------------------------------------------------------------" @echo ">>> Rebuilding the temporary build tree" @echo "--------------------------------------------------------------" .if !defined(NO_CLEAN) rm -rf ${WORLDTMP} .if defined(LIBCOMPAT) rm -rf ${LIBCOMPATTMP} .endif .else rm -rf ${WORLDTMP}/legacy/usr/include # XXX - These can depend on any header file. rm -f ${OBJTREE}${.CURDIR}/lib/libsysdecode/ioctl.c rm -f ${OBJTREE}${.CURDIR}/usr.bin/kdump/kdump_subr.c .endif .for _dir in \ lib lib/casper usr legacy/bin legacy/usr mkdir -p ${WORLDTMP}/${_dir} .endfor mtree -deU -f ${.CURDIR}/etc/mtree/BSD.usr.dist \ -p ${WORLDTMP}/legacy/usr >/dev/null .if ${MK_GROFF} != "no" mtree -deU -f ${.CURDIR}/etc/mtree/BSD.groff.dist \ -p ${WORLDTMP}/legacy/usr >/dev/null .endif mtree -deU -f ${.CURDIR}/etc/mtree/BSD.usr.dist \ -p ${WORLDTMP}/usr >/dev/null mtree -deU -f ${.CURDIR}/etc/mtree/BSD.include.dist \ -p ${WORLDTMP}/usr/include >/dev/null ln -sf ${.CURDIR}/sys ${WORLDTMP} .if ${MK_DEBUG_FILES} != "no" # We could instead disable debug files for these build stages mtree -deU -f ${.CURDIR}/etc/mtree/BSD.debug.dist \ -p ${WORLDTMP}/legacy/usr/lib >/dev/null mtree -deU -f ${.CURDIR}/etc/mtree/BSD.debug.dist \ -p ${WORLDTMP}/usr/lib >/dev/null .endif .if defined(LIBCOMPAT) mtree -deU -f ${.CURDIR}/etc/mtree/BSD.lib${libcompat}.dist \ -p ${WORLDTMP}/usr >/dev/null .if ${MK_DEBUG_FILES} != "no" mtree -deU -f ${.CURDIR}/etc/mtree/BSD.lib${libcompat}.dist \ -p ${WORLDTMP}/legacy/usr/lib/debug/usr >/dev/null mtree -deU -f ${.CURDIR}/etc/mtree/BSD.lib${libcompat}.dist \ -p ${WORLDTMP}/usr/lib/debug/usr >/dev/null .endif .endif .if ${MK_TESTS} != "no" mkdir -p ${WORLDTMP}${TESTSBASE} mtree -deU -f ${.CURDIR}/etc/mtree/BSD.tests.dist \ -p ${WORLDTMP}${TESTSBASE} >/dev/null .if ${MK_DEBUG_FILES} != "no" mkdir -p ${WORLDTMP}/usr/lib/debug/${TESTSBASE} mtree -deU -f ${.CURDIR}/etc/mtree/BSD.tests.dist \ -p ${WORLDTMP}/usr/lib/debug/${TESTSBASE} >/dev/null .endif .endif .for _mtree in ${LOCAL_MTREE} mtree -deU -f ${.CURDIR}/${_mtree} -p ${WORLDTMP} > /dev/null .endfor _legacy: @echo @echo "--------------------------------------------------------------" @echo ">>> stage 1.1: legacy release compatibility shims" @echo "--------------------------------------------------------------" ${_+_}cd ${.CURDIR}; ${BMAKE} legacy _bootstrap-tools: @echo @echo "--------------------------------------------------------------" @echo ">>> stage 1.2: bootstrap tools" @echo "--------------------------------------------------------------" ${_+_}cd ${.CURDIR}; ${BMAKE} bootstrap-tools _cleanobj: .if !defined(NO_CLEAN) @echo @echo "--------------------------------------------------------------" @echo ">>> stage 2.1: cleaning up the object tree" @echo "--------------------------------------------------------------" ${_+_}cd ${.CURDIR}; ${WMAKE} ${CLEANDIR} .if defined(LIBCOMPAT) ${_+_}cd ${.CURDIR}; ${LIBCOMPATWMAKE} -f Makefile.inc1 ${CLEANDIR} .endif .endif _obj: @echo @echo "--------------------------------------------------------------" @echo ">>> stage 2.2: rebuilding the object tree" @echo "--------------------------------------------------------------" ${_+_}cd ${.CURDIR}; ${WMAKE} obj _build-tools: @echo @echo "--------------------------------------------------------------" @echo ">>> stage 2.3: build tools" @echo "--------------------------------------------------------------" ${_+_}cd ${.CURDIR}; ${TMAKE} build-tools _cross-tools: @echo @echo "--------------------------------------------------------------" @echo ">>> stage 3: cross tools" @echo "--------------------------------------------------------------" ${_+_}cd ${.CURDIR}; ${XMAKE} cross-tools ${_+_}cd ${.CURDIR}; ${XMAKE} kernel-tools _includes: @echo @echo "--------------------------------------------------------------" @echo ">>> stage 4.1: building includes" @echo "--------------------------------------------------------------" # Special handling for SUBDIR_OVERRIDE in buildworld as they most likely need # headers from default SUBDIR. Do SUBDIR_OVERRIDE includes last. ${_+_}cd ${.CURDIR}; ${WMAKE} SUBDIR_OVERRIDE= SHARED=symlinks \ MK_INCLUDES=yes includes .if !empty(SUBDIR_OVERRIDE) && make(buildworld) ${_+_}cd ${.CURDIR}; ${WMAKE} MK_INCLUDES=yes SHARED=symlinks includes .endif _libraries: @echo @echo "--------------------------------------------------------------" @echo ">>> stage 4.2: building libraries" @echo "--------------------------------------------------------------" ${_+_}cd ${.CURDIR}; \ ${WMAKE} -DNO_FSCHG MK_HTML=no -DNO_LINT MK_MAN=no \ MK_PROFILE=no MK_TESTS=no MK_TESTS_SUPPORT=${MK_TESTS} libraries everything: .PHONY @echo @echo "--------------------------------------------------------------" @echo ">>> stage 4.3: building everything" @echo "--------------------------------------------------------------" ${_+_}cd ${.CURDIR}; _PARALLEL_SUBDIR_OK=1 ${WMAKE} all WMAKE_TGTS= WMAKE_TGTS+= _worldtmp _legacy .if empty(SUBDIR_OVERRIDE) WMAKE_TGTS+= _bootstrap-tools .endif WMAKE_TGTS+= _cleanobj _obj _build-tools _cross-tools WMAKE_TGTS+= _includes _libraries WMAKE_TGTS+= everything .if defined(LIBCOMPAT) && empty(SUBDIR_OVERRIDE) WMAKE_TGTS+= build${libcompat} .endif buildworld: buildworld_prologue ${WMAKE_TGTS} buildworld_epilogue .PHONY .ORDER: buildworld_prologue ${WMAKE_TGTS} buildworld_epilogue buildworld_prologue: .PHONY @echo "--------------------------------------------------------------" @echo ">>> World build started on `LC_ALL=C date`" @echo "--------------------------------------------------------------" buildworld_epilogue: .PHONY @echo @echo "--------------------------------------------------------------" @echo ">>> World build completed on `LC_ALL=C date`" @echo "--------------------------------------------------------------" # # We need to have this as a target because the indirection between Makefile # and Makefile.inc1 causes the correct PATH to be used, rather than a # modification of the current environment's PATH. In addition, we need # to quote multiword values. # buildenvvars: .PHONY @echo ${WMAKEENV:Q} ${.MAKE.EXPORTED:@v@$v=\"${$v}\"@} .if ${.TARGETS:Mbuildenv} .if ${.MAKEFLAGS:M-j} .error The buildenv target is incompatible with -j .endif .endif BUILDENV_DIR?= ${.CURDIR} buildenv: .PHONY @echo Entering world for ${TARGET_ARCH}:${TARGET} .if ${BUILDENV_SHELL:M*zsh*} @echo For ZSH you must run: export CPUTYPE=${TARGET_CPUTYPE} .endif @cd ${BUILDENV_DIR} && env ${WMAKEENV} BUILDENV=1 ${BUILDENV_SHELL} \ || true TOOLCHAIN_TGTS= ${WMAKE_TGTS:Neverything:Nbuild${libcompat}} toolchain: ${TOOLCHAIN_TGTS} .PHONY kernel-toolchain: ${TOOLCHAIN_TGTS:N_includes:N_libraries} .PHONY # # installcheck # # Checks to be sure system is ready for installworld/installkernel. # installcheck: _installcheck_world _installcheck_kernel .PHONY _installcheck_world: .PHONY _installcheck_kernel: .PHONY # # Require DESTDIR to be set if installing for a different architecture or # using the user/group database in the source tree. # .if ${TARGET_ARCH} != ${MACHINE_ARCH} || ${TARGET} != ${MACHINE} || \ defined(DB_FROM_SRC) .if !make(distributeworld) _installcheck_world: __installcheck_DESTDIR _installcheck_kernel: __installcheck_DESTDIR __installcheck_DESTDIR: .PHONY .if !defined(DESTDIR) || empty(DESTDIR) @echo "ERROR: Please set DESTDIR!"; \ false .endif .endif .endif .if !defined(DB_FROM_SRC) # # Check for missing UIDs/GIDs. # CHECK_UIDS= auditdistd CHECK_GIDS= audit .if ${MK_SENDMAIL} != "no" CHECK_UIDS+= smmsp CHECK_GIDS+= smmsp .endif .if ${MK_PF} != "no" CHECK_UIDS+= proxy CHECK_GIDS+= proxy authpf .endif .if ${MK_UNBOUND} != "no" CHECK_UIDS+= unbound CHECK_GIDS+= unbound .endif _installcheck_world: __installcheck_UGID __installcheck_UGID: .PHONY .for uid in ${CHECK_UIDS} @if ! `id -u ${uid} >/dev/null 2>&1`; then \ echo "ERROR: Required ${uid} user is missing, see /usr/src/UPDATING."; \ false; \ fi .endfor .for gid in ${CHECK_GIDS} @if ! `find / -prune -group ${gid} >/dev/null 2>&1`; then \ echo "ERROR: Required ${gid} group is missing, see /usr/src/UPDATING."; \ false; \ fi .endfor .endif # # Required install tools to be saved in a scratch dir for safety. # .if ${MK_ZONEINFO} != "no" _zoneinfo= zic tzsetup .endif ITOOLS= [ awk cap_mkdb cat chflags chmod chown cmp cp \ date echo egrep find grep id install ${_install-info} \ ln make mkdir mtree mv pwd_mkdb \ rm sed services_mkdb sh strip sysctl test true uname wc ${_zoneinfo} \ ${LOCAL_ITOOLS} # Needed for share/man .if ${MK_MAN} != "no" ITOOLS+=makewhatis .endif # # distributeworld # # Distributes everything compiled by a `buildworld'. # # installworld # # Installs everything compiled by a 'buildworld'. # # Non-base distributions produced by the base system EXTRA_DISTRIBUTIONS= doc .if defined(LIBCOMPAT) EXTRA_DISTRIBUTIONS+= lib${libcompat} .endif .if ${MK_TESTS} != "no" EXTRA_DISTRIBUTIONS+= tests .endif DEBUG_DISTRIBUTIONS= .if ${MK_DEBUG_FILES} != "no" DEBUG_DISTRIBUTIONS+= base ${EXTRA_DISTRIBUTIONS:S,doc,,:S,tests,,} .endif MTREE_MAGIC?= mtree 2.0 distributeworld installworld stageworld: _installcheck_world .PHONY mkdir -p ${INSTALLTMP} progs=$$(for prog in ${ITOOLS}; do \ if progpath=`which $$prog`; then \ echo $$progpath; \ else \ echo "Required tool $$prog not found in PATH." >&2; \ exit 1; \ fi; \ done); \ libs=$$(ldd -f "%o %p\n" -f "%o %p\n" $$progs 2>/dev/null | sort -u | \ while read line; do \ set -- $$line; \ if [ "$$2 $$3" != "not found" ]; then \ echo $$2; \ else \ echo "Required library $$1 not found." >&2; \ exit 1; \ fi; \ done); \ cp $$libs $$progs ${INSTALLTMP} cp -R $${PATH_LOCALE:-"/usr/share/locale"} ${INSTALLTMP}/locale .if defined(NO_ROOT) -mkdir -p ${METALOG:H} echo "#${MTREE_MAGIC}" > ${METALOG} .endif .if make(distributeworld) .for dist in ${EXTRA_DISTRIBUTIONS} -mkdir ${DESTDIR}/${DISTDIR}/${dist} mtree -deU -f ${.CURDIR}/etc/mtree/BSD.root.dist \ -p ${DESTDIR}/${DISTDIR}/${dist} >/dev/null mtree -deU -f ${.CURDIR}/etc/mtree/BSD.usr.dist \ -p ${DESTDIR}/${DISTDIR}/${dist}/usr >/dev/null mtree -deU -f ${.CURDIR}/etc/mtree/BSD.include.dist \ -p ${DESTDIR}/${DISTDIR}/${dist}/usr/include >/dev/null .if ${MK_DEBUG_FILES} != "no" mtree -deU -f ${.CURDIR}/etc/mtree/BSD.debug.dist \ -p ${DESTDIR}/${DISTDIR}/${dist}/usr/lib >/dev/null .endif .if defined(LIBCOMPAT) mtree -deU -f ${.CURDIR}/etc/mtree/BSD.lib${libcompat}.dist \ -p ${DESTDIR}/${DISTDIR}/${dist}/usr >/dev/null .if ${MK_DEBUG_FILES} != "no" mtree -deU -f ${.CURDIR}/etc/mtree/BSD.lib${libcompat}.dist \ -p ${DESTDIR}/${DISTDIR}/${dist}/usr/lib/debug/usr >/dev/null .endif .endif .if ${MK_TESTS} != "no" && ${dist} == "tests" -mkdir -p ${DESTDIR}/${DISTDIR}/${dist}${TESTSBASE} mtree -deU -f ${.CURDIR}/etc/mtree/BSD.tests.dist \ -p ${DESTDIR}/${DISTDIR}/${dist}${TESTSBASE} >/dev/null .if ${MK_DEBUG_FILES} != "no" mtree -deU -f ${.CURDIR}/etc/mtree/BSD.tests.dist \ -p ${DESTDIR}/${DISTDIR}/${dist}/usr/lib/debug/${TESTSBASE} >/dev/null .endif .endif .if defined(NO_ROOT) ${IMAKEENV} mtree -C -f ${.CURDIR}/etc/mtree/BSD.root.dist | \ sed -e 's#^\./#./${dist}/#' >> ${METALOG} ${IMAKEENV} mtree -C -f ${.CURDIR}/etc/mtree/BSD.usr.dist | \ sed -e 's#^\./#./${dist}/usr/#' >> ${METALOG} ${IMAKEENV} mtree -C -f ${.CURDIR}/etc/mtree/BSD.include.dist | \ sed -e 's#^\./#./${dist}/usr/include/#' >> ${METALOG} .if defined(LIBCOMPAT) ${IMAKEENV} mtree -C -f ${.CURDIR}/etc/mtree/BSD.lib${libcompat}.dist | \ sed -e 's#^\./#./${dist}/usr/#' >> ${METALOG} .endif .endif .endfor -mkdir ${DESTDIR}/${DISTDIR}/base ${_+_}cd ${.CURDIR}/etc; ${CROSSENV} PATH=${TMPPATH} ${MAKE} \ METALOG=${METALOG} ${IMAKE_INSTALL} ${IMAKE_MTREE} \ DISTBASE=/base DESTDIR=${DESTDIR}/${DISTDIR}/base \ LOCAL_MTREE=${LOCAL_MTREE:Q} distrib-dirs .endif ${_+_}cd ${.CURDIR}; ${IMAKE} re${.TARGET:S/world$//}; \ ${IMAKEENV} rm -rf ${INSTALLTMP} .if make(distributeworld) .for dist in ${EXTRA_DISTRIBUTIONS} find ${DESTDIR}/${DISTDIR}/${dist} -mindepth 1 -empty -delete .endfor .if defined(NO_ROOT) .for dist in base ${EXTRA_DISTRIBUTIONS} @# For each file that exists in this dist, print the corresponding @# line from the METALOG. This relies on the fact that @# a line containing only the filename will sort immediately before @# the relevant mtree line. cd ${DESTDIR}/${DISTDIR}; \ find ./${dist} | sort -u ${METALOG} - | \ awk 'BEGIN { print "#${MTREE_MAGIC}" } !/ type=/ { file = $$1 } / type=/ { if ($$1 == file) { sub(/^\.\/${dist}\//, "./"); print } }' > \ ${DESTDIR}/${DISTDIR}/${dist}.meta .endfor .for dist in ${DEBUG_DISTRIBUTIONS} @# For each file that exists in this dist, print the corresponding @# line from the METALOG. This relies on the fact that @# a line containing only the filename will sort immediately before @# the relevant mtree line. cd ${DESTDIR}/${DISTDIR}; \ find ./${dist}/usr/lib/debug | sort -u ${METALOG} - | \ awk 'BEGIN { print "#${MTREE_MAGIC}" } !/ type=/ { file = $$1 } / type=/ { if ($$1 == file) { sub(/^\.\/${dist}\//, "./"); print } }' > \ ${DESTDIR}/${DISTDIR}/${dist}.debug.meta .endfor .endif .endif packageworld: .PHONY .for dist in base ${EXTRA_DISTRIBUTIONS} .if defined(NO_ROOT) ${_+_}cd ${DESTDIR}/${DISTDIR}/${dist}; \ tar cvf - --exclude usr/lib/debug \ @${DESTDIR}/${DISTDIR}/${dist}.meta | \ ${XZ_CMD} > ${PACKAGEDIR}/${dist}.txz .else ${_+_}cd ${DESTDIR}/${DISTDIR}/${dist}; \ tar cvf - --exclude usr/lib/debug . | \ ${XZ_CMD} > ${PACKAGEDIR}/${dist}.txz .endif .endfor .for dist in ${DEBUG_DISTRIBUTIONS} . if defined(NO_ROOT) ${_+_}cd ${DESTDIR}/${DISTDIR}/${dist}; \ tar cvf - @${DESTDIR}/${DISTDIR}/${dist}.debug.meta | \ ${XZ_CMD} > ${PACKAGEDIR}/${dist}-dbg.txz . else ${_+_}cd ${DESTDIR}/${DISTDIR}/${dist}; \ tar cvLf - usr/lib/debug | \ ${XZ_CMD} > ${PACKAGEDIR}/${dist}-dbg.txz . endif .endfor # # reinstall # # If you have a build server, you can NFS mount the source and obj directories # and do a 'make reinstall' on the *client* to install new binaries from the # most recent server build. # reinstall: .MAKE .PHONY @echo "--------------------------------------------------------------" @echo ">>> Making hierarchy" @echo "--------------------------------------------------------------" ${_+_}cd ${.CURDIR}; ${MAKE} -f Makefile.inc1 \ LOCAL_MTREE=${LOCAL_MTREE:Q} hierarchy @echo @echo "--------------------------------------------------------------" @echo ">>> Installing everything" @echo "--------------------------------------------------------------" ${_+_}cd ${.CURDIR}; ${MAKE} -f Makefile.inc1 install .if defined(LIBCOMPAT) ${_+_}cd ${.CURDIR}; ${MAKE} -f Makefile.inc1 install${libcompat} .endif restage: .MAKE .PHONY @echo "--------------------------------------------------------------" @echo ">>> Making hierarchy" @echo "--------------------------------------------------------------" ${_+_}cd ${.CURDIR}; ${MAKE} -f Makefile.inc1 \ LOCAL_MTREE=${LOCAL_MTREE:Q} hierarchy distribution @echo @echo "--------------------------------------------------------------" @echo ">>> Installing everything" @echo "--------------------------------------------------------------" ${_+_}cd ${.CURDIR}; ${MAKE} -f Makefile.inc1 install .if defined(LIB32TMP) && ${MK_LIB32} != "no" ${_+_}cd ${.CURDIR}; ${MAKE} -f Makefile.inc1 install32 .endif redistribute: .MAKE .PHONY @echo "--------------------------------------------------------------" @echo ">>> Distributing everything" @echo "--------------------------------------------------------------" ${_+_}cd ${.CURDIR}; ${MAKE} -f Makefile.inc1 distribute .if defined(LIBCOMPAT) ${_+_}cd ${.CURDIR}; ${MAKE} -f Makefile.inc1 distribute${libcompat} \ DISTRIBUTION=lib${libcompat} .endif distrib-dirs distribution: .MAKE .PHONY ${_+_}cd ${.CURDIR}/etc; ${CROSSENV} PATH=${TMPPATH} ${MAKE} \ ${IMAKE_INSTALL} ${IMAKE_MTREE} METALOG=${METALOG} ${.TARGET} .if make(distribution) ${_+_}cd ${.CURDIR}; ${CROSSENV} PATH=${TMPPATH} \ ${MAKE} -f Makefile.inc1 ${IMAKE_INSTALL} \ METALOG=${METALOG} MK_TESTS=no installconfig .endif # # buildkernel and installkernel # # Which kernels to build and/or install is specified by setting # KERNCONF. If not defined a GENERIC kernel is built/installed. # Only the existing (depending TARGET) config files are used # for building kernels and only the first of these is designated # as the one being installed. # # Note that we have to use TARGET instead of TARGET_ARCH when # we're in kernel-land. Since only TARGET_ARCH is (expected) to # be set to cross-build, we have to make sure TARGET is set # properly. .if defined(KERNFAST) NO_KERNELCLEAN= t NO_KERNELCONFIG= t NO_KERNELOBJ= t # Shortcut for KERNCONF=Blah -DKERNFAST is now KERNFAST=Blah .if !defined(KERNCONF) && ${KERNFAST} != "1" KERNCONF=${KERNFAST} .endif .endif .if ${TARGET_ARCH} == "powerpc64" KERNCONF?= GENERIC64 .else KERNCONF?= GENERIC .endif INSTKERNNAME?= kernel KERNSRCDIR?= ${.CURDIR}/sys KRNLCONFDIR= ${KERNSRCDIR}/${TARGET}/conf KRNLOBJDIR= ${OBJTREE}${KERNSRCDIR} KERNCONFDIR?= ${KRNLCONFDIR} BUILDKERNELS= INSTALLKERNEL= .if defined(NO_INSTALLKERNEL) # All of the BUILDKERNELS loops start at index 1. BUILDKERNELS+= dummy .endif .for _kernel in ${KERNCONF} .if exists(${KERNCONFDIR}/${_kernel}) BUILDKERNELS+= ${_kernel} .if empty(INSTALLKERNEL) && !defined(NO_INSTALLKERNEL) INSTALLKERNEL= ${_kernel} .endif .endif .endfor ${WMAKE_TGTS:N_worldtmp:Nbuild${libcompat}} ${.ALLTARGETS:M_*:N_worldtmp}: .MAKE .PHONY # # buildkernel # # Builds all kernels defined by BUILDKERNELS. # buildkernel: .MAKE .PHONY .if empty(BUILDKERNELS:Ndummy) @echo "ERROR: Missing kernel configuration file(s) (${KERNCONF})."; \ false .endif @echo .for _kernel in ${BUILDKERNELS:Ndummy} @echo "--------------------------------------------------------------" @echo ">>> Kernel build for ${_kernel} started on `LC_ALL=C date`" @echo "--------------------------------------------------------------" @echo "===> ${_kernel}" mkdir -p ${KRNLOBJDIR} .if !defined(NO_KERNELCONFIG) @echo @echo "--------------------------------------------------------------" @echo ">>> stage 1: configuring the kernel" @echo "--------------------------------------------------------------" cd ${KRNLCONFDIR}; \ PATH=${TMPPATH} \ config ${CONFIGARGS} -d ${KRNLOBJDIR}/${_kernel} \ -I '${KERNCONFDIR}' '${KERNCONFDIR}/${_kernel}' .endif .if !defined(NO_CLEAN) && !defined(NO_KERNELCLEAN) @echo @echo "--------------------------------------------------------------" @echo ">>> stage 2.1: cleaning up the object tree" @echo "--------------------------------------------------------------" ${_+_}cd ${KRNLOBJDIR}/${_kernel}; ${KMAKE} ${CLEANDIR} .endif .if !defined(NO_KERNELOBJ) @echo @echo "--------------------------------------------------------------" @echo ">>> stage 2.2: rebuilding the object tree" @echo "--------------------------------------------------------------" ${_+_}cd ${KRNLOBJDIR}/${_kernel}; ${KMAKE} obj .endif @echo @echo "--------------------------------------------------------------" @echo ">>> stage 2.3: build tools" @echo "--------------------------------------------------------------" ${_+_}cd ${.CURDIR}; ${KTMAKE} kernel-tools @echo @echo "--------------------------------------------------------------" @echo ">>> stage 3.1: building everything" @echo "--------------------------------------------------------------" ${_+_}cd ${KRNLOBJDIR}/${_kernel}; ${KMAKE} all -DNO_MODULES_OBJ @echo "--------------------------------------------------------------" @echo ">>> Kernel build for ${_kernel} completed on `LC_ALL=C date`" @echo "--------------------------------------------------------------" .endfor +NO_INSTALLEXTRAKERNELS?= yes + # # installkernel, etc. # # Install the kernel defined by INSTALLKERNEL # installkernel installkernel.debug \ reinstallkernel reinstallkernel.debug: _installcheck_kernel .PHONY .if !defined(NO_INSTALLKERNEL) .if empty(INSTALLKERNEL) @echo "ERROR: No kernel \"${KERNCONF}\" to install."; \ false .endif @echo "--------------------------------------------------------------" @echo ">>> Installing kernel ${INSTALLKERNEL}" @echo "--------------------------------------------------------------" cd ${KRNLOBJDIR}/${INSTALLKERNEL}; \ ${CROSSENV} PATH=${TMPPATH} \ ${MAKE} ${IMAKE_INSTALL} KERNEL=${INSTKERNNAME} ${.TARGET:S/kernel//} .endif -.if ${BUILDKERNELS:[#]} > 1 && !defined(NO_INSTALLEXTRAKERNELS) +.if ${BUILDKERNELS:[#]} > 1 && ${NO_INSTALLEXTRAKERNELS} != "yes" .for _kernel in ${BUILDKERNELS:[2..-1]} @echo "--------------------------------------------------------------" @echo ">>> Installing kernel ${_kernel}" @echo "--------------------------------------------------------------" cd ${KRNLOBJDIR}/${_kernel}; \ ${CROSSENV} PATH=${TMPPATH} \ ${MAKE} ${IMAKE_INSTALL} KERNEL=${INSTKERNNAME}.${_kernel} ${.TARGET:S/kernel//} .endfor .endif distributekernel distributekernel.debug: .PHONY .if !defined(NO_INSTALLKERNEL) .if empty(INSTALLKERNEL) @echo "ERROR: No kernel \"${KERNCONF}\" to install."; \ false .endif mkdir -p ${DESTDIR}/${DISTDIR} .if defined(NO_ROOT) @echo "#${MTREE_MAGIC}" > ${DESTDIR}/${DISTDIR}/kernel.premeta .endif cd ${KRNLOBJDIR}/${INSTALLKERNEL}; \ ${IMAKEENV} ${IMAKE_INSTALL:S/METALOG/kernel.premeta/} \ ${IMAKE_MTREE} PATH=${TMPPATH} ${MAKE} KERNEL=${INSTKERNNAME} \ DESTDIR=${INSTALL_DDIR}/kernel \ ${.TARGET:S/distributekernel/install/} .if defined(NO_ROOT) @sed -e 's|^./kernel|.|' ${DESTDIR}/${DISTDIR}/kernel.premeta > \ ${DESTDIR}/${DISTDIR}/kernel.meta .endif .endif -.if ${BUILDKERNELS:[#]} > 1 && !defined(NO_INSTALLEXTRAKERNELS) +.if ${BUILDKERNELS:[#]} > 1 && ${NO_INSTALLEXTRAKERNELS} != "yes" .for _kernel in ${BUILDKERNELS:[2..-1]} .if defined(NO_ROOT) @echo "#${MTREE_MAGIC}" > ${DESTDIR}/${DISTDIR}/kernel.${_kernel}.premeta .endif cd ${KRNLOBJDIR}/${_kernel}; \ ${IMAKEENV} ${IMAKE_INSTALL:S/METALOG/kernel.${_kernel}.premeta/} \ ${IMAKE_MTREE} PATH=${TMPPATH} ${MAKE} \ KERNEL=${INSTKERNNAME}.${_kernel} \ DESTDIR=${INSTALL_DDIR}/kernel.${_kernel} \ ${.TARGET:S/distributekernel/install/} .if defined(NO_ROOT) @sed -e "s|^./kernel.${_kernel}|.|" \ ${DESTDIR}/${DISTDIR}/kernel.${_kernel}.premeta > \ ${DESTDIR}/${DISTDIR}/kernel.${_kernel}.meta .endif .endfor .endif packagekernel: .PHONY .if defined(NO_ROOT) .if !defined(NO_INSTALLKERNEL) cd ${DESTDIR}/${DISTDIR}/kernel; \ tar cvf - --exclude '*.debug' \ @${DESTDIR}/${DISTDIR}/kernel.meta | \ ${XZ_CMD} > ${PACKAGEDIR}/kernel.txz .endif cd ${DESTDIR}/${DISTDIR}/kernel; \ tar cvf - --include '*/*/*.debug' \ @${DESTDIR}/${DISTDIR}/kernel.meta | \ ${XZ_CMD} > ${DESTDIR}/${DISTDIR}/kernel-dbg.txz -.if ${BUILDKERNELS:[#]} > 1 && !defined(NO_INSTALLEXTRAKERNELS) +.if ${BUILDKERNELS:[#]} > 1 && ${NO_INSTALLEXTRAKERNELS} != "yes" .for _kernel in ${BUILDKERNELS:[2..-1]} cd ${DESTDIR}/${DISTDIR}/kernel.${_kernel}; \ tar cvf - --exclude '*.debug' \ @${DESTDIR}/${DISTDIR}/kernel.${_kernel}.meta | \ ${XZ_CMD} > ${PACKAGEDIR}/kernel.${_kernel}.txz cd ${DESTDIR}/${DISTDIR}/kernel.${_kernel}; \ tar cvf - --include '*/*/*.debug' \ @${DESTDIR}/${DISTDIR}/kernel.${_kernel}.meta | \ ${XZ_CMD} > ${DESTDIR}/${DISTDIR}/kernel.${_kernel}-dbg.txz .endfor .endif .else .if !defined(NO_INSTALLKERNEL) cd ${DESTDIR}/${DISTDIR}/kernel; \ tar cvf - --exclude '*.debug' . | \ ${XZ_CMD} > ${PACKAGEDIR}/kernel.txz .endif cd ${DESTDIR}/${DISTDIR}/kernel; \ tar cvf - --include '*/*/*.debug' $$(eval find .) | \ ${XZ_CMD} > ${DESTDIR}/${DISTDIR}/kernel-dbg.txz -.if ${BUILDKERNELS:[#]} > 1 && !defined(NO_INSTALLEXTRAKERNELS) +.if ${BUILDKERNELS:[#]} > 1 && ${NO_INSTALLEXTRAKERNELS} != "yes" .for _kernel in ${BUILDKERNELS:[2..-1]} cd ${DESTDIR}/${DISTDIR}/kernel.${_kernel}; \ tar cvf - --exclude '*.debug' . | \ ${XZ_CMD} > ${PACKAGEDIR}/kernel.${_kernel}.txz cd ${DESTDIR}/${DISTDIR}/kernel.${_kernel}; \ tar cvf - --include '*/*/*.debug' $$(eval find .) | \ ${XZ_CMD} > ${DESTDIR}/${DISTDIR}/kernel.${_kernel}-dbg.txz .endfor .endif .endif stagekernel: .PHONY ${_+_}${MAKE} -C ${.CURDIR} ${.MAKEFLAGS} distributekernel PORTSDIR?= /usr/ports WSTAGEDIR?= ${MAKEOBJDIRPREFIX}${.CURDIR}/${TARGET}.${TARGET_ARCH}/worldstage KSTAGEDIR?= ${MAKEOBJDIRPREFIX}${.CURDIR}/${TARGET}.${TARGET_ARCH}/kernelstage REPODIR?= ${MAKEOBJDIRPREFIX}${.CURDIR}/repo PKGSIGNKEY?= # empty .ORDER: stage-packages create-packages .ORDER: create-packages create-world-packages .ORDER: create-packages create-kernel-packages .ORDER: create-packages sign-packages _pkgbootstrap: .PHONY .if !exists(${LOCALBASE}/sbin/pkg) @env ASSUME_ALWAYS_YES=YES pkg bootstrap .endif packages: .PHONY ${_+_}${MAKE} -C ${.CURDIR} PKG_VERSION=${PKG_VERSION} real-packages package-pkg: .PHONY rm -rf /tmp/ports.${TARGET} || : env ${WMAKEENV:Q} SRCDIR=${.CURDIR} PORTSDIR=${PORTSDIR} REVISION=${_REVISION} \ PKG_VERSION=${PKG_VERSION} REPODIR=${REPODIR} WSTAGEDIR=${WSTAGEDIR} \ sh ${.CURDIR}/release/scripts/make-pkg-package.sh real-packages: stage-packages create-packages sign-packages .PHONY stage-packages: .PHONY @mkdir -p ${REPODIR} ${WSTAGEDIR} ${KSTAGEDIR} ${_+_}@cd ${.CURDIR}; \ ${MAKE} DESTDIR=${WSTAGEDIR} -DNO_ROOT -B stageworld ; \ ${MAKE} DESTDIR=${KSTAGEDIR} -DNO_ROOT -B stagekernel create-packages: _pkgbootstrap .PHONY @mkdir -p ${REPODIR} ${_+_}@cd ${.CURDIR}; \ ${MAKE} DESTDIR=${WSTAGEDIR} \ PKG_VERSION=${PKG_VERSION} create-world-packages ; \ ${MAKE} DESTDIR=${KSTAGEDIR} \ PKG_VERSION=${PKG_VERSION} DISTDIR=kernel \ create-kernel-packages create-world-packages: _pkgbootstrap .PHONY @rm -f ${WSTAGEDIR}/*.plist 2>/dev/null || : @cd ${WSTAGEDIR} ; \ awk -f ${SRCDIR}/release/scripts/mtree-to-plist.awk \ ${WSTAGEDIR}/METALOG @for plist in ${WSTAGEDIR}/*.plist; do \ plist=$${plist##*/} ; \ pkgname=$${plist%.plist} ; \ sh ${SRCDIR}/release/packages/generate-ucl.sh -o $${pkgname} \ -s ${SRCDIR} -u ${WSTAGEDIR}/$${pkgname}.ucl ; \ done @for plist in ${WSTAGEDIR}/*.plist; do \ plist=$${plist##*/} ; \ pkgname=$${plist%.plist} ; \ awk -F\" ' \ /^name/ { printf("===> Creating %s-", $$2); next } \ /^version/ { print $$2; next } \ ' ${WSTAGEDIR}/$${pkgname}.ucl ; \ pkg -o ABI_FILE=${WSTAGEDIR}/bin/sh -o ALLOW_BASE_SHLIBS=yes \ create -M ${WSTAGEDIR}/$${pkgname}.ucl \ -p ${WSTAGEDIR}/$${pkgname}.plist \ -r ${WSTAGEDIR} \ -o ${REPODIR}/$$(pkg -o ABI_FILE=${WSTAGEDIR}/bin/sh config ABI)/${PKG_VERSION} ; \ done create-kernel-packages: _pkgbootstrap .PHONY .if exists(${KSTAGEDIR}/kernel.meta) .for flavor in "" -debug @cd ${KSTAGEDIR}/${DISTDIR} ; \ awk -f ${SRCDIR}/release/scripts/mtree-to-plist.awk \ -v kernel=yes -v _kernconf=${INSTALLKERNEL} \ ${KSTAGEDIR}/kernel.meta ; \ cap_arg=`cd ${SRCDIR}/etc ; ${MAKE} -VCAP_MKDB_ENDIAN` ; \ pwd_arg=`cd ${SRCDIR}/etc ; ${MAKE} -VPWD_MKDB_ENDIAN` ; \ sed -e "s/%VERSION%/${PKG_VERSION}/" \ -e "s/%PKGNAME%/kernel-${INSTALLKERNEL:tl}${flavor}/" \ -e "s/%COMMENT%/FreeBSD ${INSTALLKERNEL} kernel ${flavor}/" \ -e "s/%DESC%/FreeBSD ${INSTALLKERNEL} kernel ${flavor}/" \ -e "s/%CAP_MKDB_ENDIAN%/$${cap_arg}/g" \ -e "s/%PWD_MKDB_ENDIAN%/$${pwd_arg}/g" \ ${SRCDIR}/release/packages/kernel.ucl \ > ${KSTAGEDIR}/${DISTDIR}/kernel.${INSTALLKERNEL}${flavor}.ucl ; \ awk -F\" ' \ /name/ { printf("===> Creating %s-", $$2); next } \ /version/ {print $$2; next } ' \ ${KSTAGEDIR}/${DISTDIR}/kernel.${INSTALLKERNEL}${flavor}.ucl ; \ pkg -o ABI_FILE=${WSTAGEDIR}/bin/sh -o ALLOW_BASE_SHLIBS=yes \ create -M ${KSTAGEDIR}/${DISTDIR}/kernel.${INSTALLKERNEL}${flavor}.ucl \ -p ${KSTAGEDIR}/${DISTDIR}/kernel.${INSTALLKERNEL}${flavor}.plist \ -r ${KSTAGEDIR}/${DISTDIR} \ -o ${REPODIR}/$$(pkg -o ABI_FILE=${WSTAGEDIR}/bin/sh config ABI)/${PKG_VERSION} .endfor .endif -.if ${BUILDKERNELS:[#]} > 1 && !defined(NO_INSTALLEXTRAKERNELS) +.if ${BUILDKERNELS:[#]} > 1 && ${NO_INSTALLEXTRAKERNELS} != "yes" .for _kernel in ${BUILDKERNELS:[2..-1]} .if exists(${KSTAGEDIR}/kernel.${_kernel}.meta) .for flavor in "" -debug @cd ${KSTAGEDIR}/kernel.${_kernel} ; \ awk -f ${SRCDIR}/release/scripts/mtree-to-plist.awk \ -v kernel=yes -v _kernconf=${_kernel} \ ${KSTAGEDIR}/kernel.${_kernel}.meta ; \ cap_arg=`cd ${SRCDIR}/etc ; ${MAKE} -VCAP_MKDB_ENDIAN` ; \ pwd_arg=`cd ${SRCDIR}/etc ; ${MAKE} -VPWD_MKDB_ENDIAN` ; \ sed -e "s/%VERSION%/${PKG_VERSION}/" \ -e "s/%PKGNAME%/kernel-${_kernel:tl}${flavor}/" \ -e "s/%COMMENT%/FreeBSD ${_kernel} kernel ${flavor}/" \ -e "s/%DESC%/FreeBSD ${_kernel} kernel ${flavor}/" \ -e "s/%CAP_MKDB_ENDIAN%/$${cap_arg}/g" \ -e "s/%PWD_MKDB_ENDIAN%/$${pwd_arg}/g" \ ${SRCDIR}/release/packages/kernel.ucl \ > ${KSTAGEDIR}/kernel.${_kernel}/kernel.${_kernel}${flavor}.ucl ; \ awk -F\" ' \ /name/ { printf("===> Creating %s-", $$2); next } \ /version/ {print $$2; next } ' \ ${KSTAGEDIR}/kernel.${_kernel}/kernel.${_kernel}${flavor}.ucl ; \ pkg -o ABI_FILE=${WSTAGEDIR}/bin/sh -o ALLOW_BASE_SHLIBS=yes \ create -M ${KSTAGEDIR}/kernel.${_kernel}/kernel.${_kernel}${flavor}.ucl \ -p ${KSTAGEDIR}/kernel.${_kernel}/kernel.${_kernel}${flavor}.plist \ -r ${KSTAGEDIR}/kernel.${_kernel} \ -o ${REPODIR}/$$(pkg -o ABI_FILE=${WSTAGEDIR}/bin/sh config ABI)/${PKG_VERSION} .endfor .endif .endfor .endif sign-packages: _pkgbootstrap .PHONY @[ -L "${REPODIR}/$$(pkg -o ABI_FILE=${WSTAGEDIR}/bin/sh config ABI)/latest" ] && \ unlink ${REPODIR}/$$(pkg -o ABI_FILE=${WSTAGEDIR}/bin/sh config ABI)/latest ; \ pkg -o ABI_FILE=${WSTAGEDIR}/bin/sh repo \ -o ${REPODIR}/$$(pkg -o ABI_FILE=${WSTAGEDIR}/bin/sh config ABI)/${PKG_VERSION} \ ${REPODIR}/$$(pkg -o ABI_FILE=${WSTAGEDIR}/bin/sh config ABI)/${PKG_VERSION} \ ${PKGSIGNKEY} ; \ ln -s ${REPODIR}/$$(pkg -o ABI_FILE=${WSTAGEDIR}/bin/sh config ABI)/${PKG_VERSION} \ ${REPODIR}/$$(pkg -o ABI_FILE=${WSTAGEDIR}/bin/sh config ABI)/latest # # # checkworld # # Run test suite on installed world. # checkworld: .PHONY @if [ ! -x ${LOCALBASE}/bin/kyua ]; then \ echo "You need kyua (devel/kyua) to run the test suite." | /usr/bin/fmt; \ exit 1; \ fi ${_+_}${LOCALBASE}/bin/kyua test -k ${TESTSBASE}/Kyuafile # # # doxygen # # Build the API documentation with doxygen # doxygen: .PHONY @if [ ! -x ${LOCALBASE}/bin/doxygen ]; then \ echo "You need doxygen (devel/doxygen) to generate the API documentation of the kernel." | /usr/bin/fmt; \ exit 1; \ fi ${_+_}cd ${.CURDIR}/tools/kerneldoc/subsys; ${MAKE} obj all # # update # # Update the source tree(s), by running svn/svnup to update to the # latest copy. # update: .PHONY .if defined(SVN_UPDATE) @echo "--------------------------------------------------------------" @echo ">>> Updating ${.CURDIR} using Subversion" @echo "--------------------------------------------------------------" @(cd ${.CURDIR}; ${SVN} update ${SVNFLAGS}) .endif # # ------------------------------------------------------------------------ # # From here onwards are utility targets used by the 'make world' and # related targets. If your 'world' breaks, you may like to try to fix # the problem and manually run the following targets to attempt to # complete the build. Beware, this is *not* guaranteed to work, you # need to have a pretty good grip on the current state of the system # to attempt to manually finish it. If in doubt, 'make world' again. # # # legacy: Build compatibility shims for the next three targets. This is a # minimal set of tools and shims necessary to compensate for older systems # which don't have the APIs required by the targets built in bootstrap-tools, # build-tools or cross-tools. # # ELF Tool Chain libraries are needed for ELF tools and dtrace tools. # r296685 fix cross-endian objcopy .if ${BOOTSTRAPPING} < 1100102 _elftoolchain_libs= lib/libelf lib/libdwarf .endif legacy: .PHONY .if ${BOOTSTRAPPING} < 800107 && ${BOOTSTRAPPING} != 0 @echo "ERROR: Source upgrades from versions prior to 8.0 are not supported."; \ false .endif .for _tool in tools/build ${_elftoolchain_libs} ${_+_}@${ECHODIR} "===> ${_tool} (obj,includes,all,install)"; \ cd ${.CURDIR}/${_tool}; \ ${MAKE} DIRPRFX=${_tool}/ obj; \ ${MAKE} DIRPRFX=${_tool}/ DESTDIR=${MAKEOBJDIRPREFIX}/legacy includes; \ ${MAKE} DIRPRFX=${_tool}/ all; \ ${MAKE} DIRPRFX=${_tool}/ DESTDIR=${MAKEOBJDIRPREFIX}/legacy install .endfor # # bootstrap-tools: Build tools needed for compatibility. These are binaries that # are built to build other binaries in the system. However, the focus of these # binaries is usually quite narrow. Bootstrap tools use the host's compiler and # libraries, augmented by -legacy. # _bt= _bootstrap-tools .if ${MK_GAMES} != "no" _strfile= usr.bin/fortune/strfile .endif .if ${MK_GCC} != "no" && ${MK_CXX} != "no" _gperf= gnu/usr.bin/gperf .endif .if ${MK_GROFF} != "no" _groff= gnu/usr.bin/groff \ usr.bin/soelim .endif .if ${MK_VT} != "no" _vtfontcvt= usr.bin/vtfontcvt .endif .if ${BOOTSTRAPPING} < 900002 _sed= usr.bin/sed .endif .if ${BOOTSTRAPPING} < 1000033 _libopenbsd= lib/libopenbsd _m4= usr.bin/m4 _lex= usr.bin/lex ${_bt}-usr.bin/m4: ${_bt}-lib/libopenbsd ${_bt}-usr.bin/lex: ${_bt}-usr.bin/m4 .endif .if ${BOOTSTRAPPING} < 1000026 _nmtree= lib/libnetbsd \ usr.sbin/nmtree ${_bt}-usr.sbin/nmtree: ${_bt}-lib/libnetbsd .endif .if ${BOOTSTRAPPING} < 1000027 _cat= bin/cat .endif # r264059 support for status= .if ${BOOTSTRAPPING} < 1100017 _dd= bin/dd .endif # r277259 crunchide: Correct 64-bit section header offset # r281674 crunchide: always include both 32- and 64-bit ELF support # r285986 crunchen: use STRIPBIN rather than STRIP .if ${BOOTSTRAPPING} < 1100078 _crunch= usr.sbin/crunch .endif .if ${BOOTSTRAPPING} >= 900040 && ${BOOTSTRAPPING} < 900041 _awk= usr.bin/awk .endif # r296926 -P keymap search path, MFC to stable/10 in r298297 .if ${BOOTSTRAPPING} < 1003501 || \ (${BOOTSTRAPPING} >= 1100000 && ${BOOTSTRAPPING} < 1100103) _kbdcontrol= usr.sbin/kbdcontrol .endif _yacc= lib/liby \ usr.bin/yacc ${_bt}-usr.bin/yacc: ${_bt}-lib/liby .if ${MK_BSNMP} != "no" _gensnmptree= usr.sbin/bsnmpd/gensnmptree .endif # We need to build tblgen when we're building clang either as # the bootstrap compiler, or as the part of the normal build. .if ${MK_CLANG_BOOTSTRAP} != "no" || ${MK_CLANG} != "no" _clang_tblgen= \ lib/clang/libllvmsupport \ lib/clang/libllvmtablegen \ usr.bin/clang/llvm-tblgen \ usr.bin/clang/clang-tblgen ${_bt}-usr.bin/clang/clang-tblgen: ${_bt}-lib/clang/libllvmtablegen ${_bt}-lib/clang/libllvmsupport ${_bt}-usr.bin/clang/llvm-tblgen: ${_bt}-lib/clang/libllvmtablegen ${_bt}-lib/clang/libllvmsupport .endif # Default to building the GPL DTC, but build the BSDL one if users explicitly # request it. _dtc= usr.bin/dtc .if ${MK_GPL_DTC} != "no" _dtc= gnu/usr.bin/dtc .endif .if ${MK_KERBEROS} != "no" _kerberos5_bootstrap_tools= \ kerberos5/tools/make-roken \ kerberos5/lib/libroken \ kerberos5/lib/libvers \ kerberos5/tools/asn1_compile \ kerberos5/tools/slc \ usr.bin/compile_et .ORDER: ${_kerberos5_bootstrap_tools:C/^/${_bt}-/g} .endif # r283777 makewhatis(1) replaced with mandoc version which builds a database. .if ${MK_MANDOCDB} != "no" && ${BOOTSTRAPPING} < 1100075 _libopenbsd?= lib/libopenbsd _makewhatis= lib/libsqlite3 \ usr.bin/mandoc ${_bt}-usr.bin/mandoc: ${_bt}-lib/libopenbsd ${_bt}-lib/libsqlite3 .endif bootstrap-tools: .PHONY # Please document (add comment) why something is in 'bootstrap-tools'. # Try to bound the building of the bootstrap-tool to just the # FreeBSD versions that need the tool built at this stage of the build. .for _tool in \ ${_clang_tblgen} \ ${_kerberos5_bootstrap_tools} \ ${_strfile} \ ${_gperf} \ ${_groff} \ ${_dtc} \ ${_awk} \ ${_cat} \ ${_dd} \ ${_kbdcontrol} \ usr.bin/lorder \ ${_libopenbsd} \ ${_makewhatis} \ usr.bin/rpcgen \ ${_sed} \ ${_yacc} \ ${_m4} \ ${_lex} \ usr.bin/xinstall \ ${_gensnmptree} \ usr.sbin/config \ ${_crunch} \ ${_nmtree} \ ${_vtfontcvt} \ usr.bin/localedef ${_bt}-${_tool}: .PHONY .MAKE ${_+_}@${ECHODIR} "===> ${_tool} (obj,all,install)"; \ cd ${.CURDIR}/${_tool}; \ ${MAKE} DIRPRFX=${_tool}/ obj; \ ${MAKE} DIRPRFX=${_tool}/ all; \ ${MAKE} DIRPRFX=${_tool}/ DESTDIR=${MAKEOBJDIRPREFIX}/legacy install bootstrap-tools: ${_bt}-${_tool} .endfor # # build-tools: Build special purpose build tools # .if !defined(NO_SHARE) _share= share/syscons/scrnmaps .endif .if ${MK_GCC} != "no" _gcc_tools= gnu/usr.bin/cc/cc_tools .endif .if ${MK_RESCUE} != "no" # rescue includes programs that have build-tools targets _rescue=rescue/rescue .endif .for _tool in \ bin/csh \ bin/sh \ ${LOCAL_TOOL_DIRS} \ lib/ncurses/ncurses \ lib/ncurses/ncursesw \ ${_rescue} \ ${_share} \ usr.bin/awk \ lib/libmagic \ usr.bin/mkesdb_static \ usr.bin/mkcsmapper_static \ usr.bin/vi/catalog build-tools_${_tool}: .PHONY ${_+_}@${ECHODIR} "===> ${_tool} (obj,build-tools)"; \ cd ${.CURDIR}/${_tool}; \ ${MAKE} DIRPRFX=${_tool}/ obj; \ ${MAKE} DIRPRFX=${_tool}/ build-tools build-tools: build-tools_${_tool} .endfor .for _tool in \ ${_gcc_tools} build-tools_${_tool}: .PHONY ${_+_}@${ECHODIR} "===> ${_tool} (obj,all)"; \ cd ${.CURDIR}/${_tool}; \ ${MAKE} DIRPRFX=${_tool}/ obj; \ ${MAKE} DIRPRFX=${_tool}/ all build-tools: build-tools_${_tool} .endfor # # kernel-tools: Build kernel-building tools # kernel-tools: .PHONY mkdir -p ${MAKEOBJDIRPREFIX}/usr mtree -deU -f ${.CURDIR}/etc/mtree/BSD.usr.dist \ -p ${MAKEOBJDIRPREFIX}/usr >/dev/null # # cross-tools: All the tools needed to build the rest of the system after # we get done with the earlier stages. It is the last set of tools needed # to begin building the target binaries. # .if ${TARGET_ARCH} != ${MACHINE_ARCH} .if ${TARGET_ARCH} == "amd64" || ${TARGET_ARCH} == "i386" _btxld= usr.sbin/btxld .endif .endif # Rebuild ctfconvert and ctfmerge to avoid difficult-to-diagnose failures # resulting from missing bug fixes or ELF Toolchain updates. .if ${MK_CDDL} != "no" _dtrace_tools= cddl/lib/libctf cddl/usr.bin/ctfconvert \ cddl/usr.bin/ctfmerge .endif # If we're given an XAS, don't build binutils. .if ${XAS:M/*} == "" .if ${MK_BINUTILS_BOOTSTRAP} != "no" _binutils= gnu/usr.bin/binutils .endif .if ${MK_ELFTOOLCHAIN_BOOTSTRAP} != "no" _elftctools= lib/libelftc \ lib/libpe \ usr.bin/elfcopy \ usr.bin/nm \ usr.bin/size \ usr.bin/strings # These are not required by the build, but can be useful for developers who # cross-build on a FreeBSD 10 host: _elftctools+= usr.bin/addr2line .endif .elif ${TARGET_ARCH} != ${MACHINE_ARCH} && ${MK_ELFTOOLCHAIN_BOOTSTRAP} != "no" # If cross-building with an external binutils we still need to build strip for # the target (for at least crunchide). _elftctools= lib/libelftc \ lib/libpe \ usr.bin/elfcopy .endif # If an full path to an external cross compiler is given, don't build # a cross compiler. .if ${XCC:N${CCACHE_BIN}:M/*} == "" && ${MK_CROSS_COMPILER} != "no" .if ${MK_CLANG_BOOTSTRAP} != "no" _clang= usr.bin/clang _clang_libs= lib/clang .endif .if ${MK_GCC_BOOTSTRAP} != "no" _cc= gnu/usr.bin/cc .endif .endif .if ${MK_USB} != "no" _usb_tools= sys/boot/usb/tools .endif cross-tools: .MAKE .PHONY .for _tool in \ ${_clang_libs} \ ${_clang} \ ${_binutils} \ ${_elftctools} \ ${_dtrace_tools} \ ${_cc} \ ${_btxld} \ ${_crunchide} \ ${_usb_tools} ${_+_}@${ECHODIR} "===> ${_tool} (obj,all,install)"; \ cd ${.CURDIR}/${_tool}; \ ${MAKE} DIRPRFX=${_tool}/ obj; \ ${MAKE} DIRPRFX=${_tool}/ all; \ ${MAKE} DIRPRFX=${_tool}/ DESTDIR=${MAKEOBJDIRPREFIX} install .endfor NXBDESTDIR= ${OBJTREE}/nxb-bin NXBENV= MAKEOBJDIRPREFIX=${OBJTREE}/nxb \ INSTALL="sh ${.CURDIR}/tools/install.sh" \ PATH=${PATH}:${OBJTREE}/gperf_for_gcc/usr/bin NXBMAKE= ${NXBENV} ${MAKE} \ LLVM_TBLGEN=${NXBDESTDIR}/usr/bin/llvm-tblgen \ CLANG_TBLGEN=${NXBDESTDIR}/usr/bin/clang-tblgen \ MACHINE=${TARGET} MACHINE_ARCH=${TARGET_ARCH} \ MK_GDB=no MK_TESTS=no \ SSP_CFLAGS= \ MK_HTML=no NO_LINT=yes MK_MAN=no \ -DNO_PIC MK_PROFILE=no -DNO_SHARED \ -DNO_CPU_CFLAGS MK_WARNS=no MK_CTF=no \ MK_CLANG_EXTRAS=no MK_CLANG_FULL=no \ MK_LLDB=no MK_DEBUG_FILES=no # native-xtools is the current target for qemu-user cross builds of ports # via poudriere and the imgact_binmisc kernel module. # For non-clang enabled targets that are still using the in tree gcc # we must build a gperf binary for one instance of its Makefiles. On # clang-enabled systems, the gperf binary is obsolete. native-xtools: .PHONY .if ${MK_GCC_BOOTSTRAP} != "no" mkdir -p ${OBJTREE}/gperf_for_gcc/usr/bin ${_+_}@${ECHODIR} "===> ${_gperf} (obj,all,install)"; \ cd ${.CURDIR}/${_gperf}; \ ${NXBMAKE} DIRPRFX=${_gperf}/ obj; \ ${NXBMAKE} DIRPRFX=${_gperf}/ all; \ ${NXBMAKE} DIRPRFX=${_gperf}/ DESTDIR=${OBJTREE}/gperf_for_gcc install .endif mkdir -p ${NXBDESTDIR}/bin ${NXBDESTDIR}/sbin ${NXBDESTDIR}/usr mtree -deU -f ${.CURDIR}/etc/mtree/BSD.usr.dist \ -p ${NXBDESTDIR}/usr >/dev/null mtree -deU -f ${.CURDIR}/etc/mtree/BSD.include.dist \ -p ${NXBDESTDIR}/usr/include >/dev/null .if ${MK_DEBUG_FILES} != "no" mtree -deU -f ${.CURDIR}/etc/mtree/BSD.debug.dist \ -p ${NXBDESTDIR}/usr/lib >/dev/null .endif .for _tool in \ bin/cat \ bin/chmod \ bin/cp \ bin/csh \ bin/echo \ bin/expr \ bin/hostname \ bin/ln \ bin/ls \ bin/mkdir \ bin/mv \ bin/ps \ bin/realpath \ bin/rm \ bin/rmdir \ bin/sh \ bin/sleep \ ${_clang_tblgen} \ usr.bin/ar \ ${_binutils} \ ${_elftctools} \ ${_cc} \ ${_gcc_tools} \ ${_clang_libs} \ ${_clang} \ sbin/md5 \ sbin/sysctl \ gnu/usr.bin/diff \ usr.bin/awk \ usr.bin/basename \ usr.bin/bmake \ usr.bin/bzip2 \ usr.bin/cmp \ usr.bin/dirname \ usr.bin/env \ usr.bin/fetch \ usr.bin/find \ usr.bin/grep \ usr.bin/gzip \ usr.bin/id \ usr.bin/lex \ usr.bin/lorder \ usr.bin/mktemp \ usr.bin/mt \ usr.bin/patch \ usr.bin/sed \ usr.bin/sort \ usr.bin/tar \ usr.bin/touch \ usr.bin/tr \ usr.bin/true \ usr.bin/uniq \ usr.bin/unzip \ usr.bin/xargs \ usr.bin/xinstall \ usr.bin/xz \ usr.bin/yacc \ usr.sbin/chown ${_+_}@${ECHODIR} "===> ${_tool} (obj,all,install)"; \ cd ${.CURDIR}/${_tool}; \ ${NXBMAKE} DIRPRFX=${_tool}/ obj; \ ${NXBMAKE} DIRPRFX=${_tool}/ all; \ ${NXBMAKE} DIRPRFX=${_tool}/ DESTDIR=${NXBDESTDIR} install .endfor # # hierarchy - ensure that all the needed directories are present # hierarchy hier: .MAKE .PHONY ${_+_}cd ${.CURDIR}/etc; ${HMAKE} distrib-dirs # # libraries - build all libraries, and install them under ${DESTDIR}. # # The list of libraries with dependents (${_prebuild_libs}) and their # interdependencies (__L) are built automatically by the # ${.CURDIR}/tools/make_libdeps.sh script. # libraries: .MAKE .PHONY ${_+_}cd ${.CURDIR}; \ ${MAKE} -f Makefile.inc1 _prereq_libs; \ ${MAKE} -f Makefile.inc1 _startup_libs; \ ${MAKE} -f Makefile.inc1 _prebuild_libs; \ ${MAKE} -f Makefile.inc1 _generic_libs # # static libgcc.a prerequisite for shared libc # _prereq_libs= gnu/lib/libssp/libssp_nonshared gnu/lib/libgcc lib/libcompiler_rt # These dependencies are not automatically generated: # # gnu/lib/csu, gnu/lib/libgcc, lib/csu and lib/libc must be built before # all shared libraries for ELF. # _startup_libs= gnu/lib/csu _startup_libs+= lib/csu _startup_libs+= gnu/lib/libgcc _startup_libs+= lib/libcompiler_rt _startup_libs+= lib/libc _startup_libs+= lib/libc_nonshared .if ${MK_LIBCPLUSPLUS} != "no" _startup_libs+= lib/libcxxrt .endif gnu/lib/libgcc__L: lib/libc__L gnu/lib/libgcc__L: lib/libc_nonshared__L .if ${MK_LIBCPLUSPLUS} != "no" lib/libcxxrt__L: gnu/lib/libgcc__L .endif _prebuild_libs= ${_kerberos5_lib_libasn1} \ ${_kerberos5_lib_libhdb} \ ${_kerberos5_lib_libheimbase} \ ${_kerberos5_lib_libheimntlm} \ ${_libsqlite3} \ ${_kerberos5_lib_libheimipcc} \ ${_kerberos5_lib_libhx509} ${_kerberos5_lib_libkrb5} \ ${_kerberos5_lib_libroken} \ ${_kerberos5_lib_libwind} \ lib/libbz2 ${_libcom_err} lib/libcrypt \ lib/libelf lib/libexpat \ lib/libfigpar \ ${_lib_libgssapi} \ lib/libkiconv lib/libkvm lib/liblzma lib/libmd lib/libnv \ ${_lib_casper} \ lib/ncurses/ncurses lib/ncurses/ncursesw \ lib/libopie lib/libpam/libpam ${_lib_libthr} \ ${_lib_libradius} lib/libsbuf lib/libtacplus \ lib/libgeom \ ${_cddl_lib_libumem} ${_cddl_lib_libnvpair} \ ${_cddl_lib_libuutil} \ ${_cddl_lib_libavl} \ ${_cddl_lib_libzfs_core} \ ${_cddl_lib_libctf} \ lib/libutil lib/libpjdlog ${_lib_libypclnt} lib/libz lib/msun \ ${_secure_lib_libcrypto} ${_lib_libldns} \ ${_secure_lib_libssh} ${_secure_lib_libssl} \ gnu/lib/libdialog .if ${MK_GNUCXX} != "no" _prebuild_libs+= gnu/lib/libstdc++ gnu/lib/libsupc++ gnu/lib/libstdc++__L: lib/msun__L gnu/lib/libsupc++__L: gnu/lib/libstdc++__L .endif .if ${MK_LIBCPLUSPLUS} != "no" _prebuild_libs+= lib/libc++ .endif lib/libgeom__L: lib/libexpat__L lib/libkvm__L: lib/libelf__L .if ${MK_LIBTHR} != "no" _lib_libthr= lib/libthr .endif .if ${MK_RADIUS_SUPPORT} != "no" _lib_libradius= lib/libradius .endif .if ${MK_OFED} != "no" _ofed_lib= contrib/ofed/usr.lib _prebuild_libs+= contrib/ofed/usr.lib/libosmcomp _prebuild_libs+= contrib/ofed/usr.lib/libopensm _prebuild_libs+= contrib/ofed/usr.lib/libibcommon _prebuild_libs+= contrib/ofed/usr.lib/libibverbs _prebuild_libs+= contrib/ofed/usr.lib/libibumad contrib/ofed/usr.lib/libopensm__L: lib/libthr__L contrib/ofed/usr.lib/libosmcomp__L: lib/libthr__L contrib/ofed/usr.lib/libibumad__L: contrib/ofed/usr.lib/libibcommon__L .endif .if ${MK_CASPER} != "no" _lib_casper= lib/libcasper .endif lib/libpjdlog__L: lib/libutil__L lib/libcasper__L: lib/libnv__L lib/liblzma__L: lib/libthr__L _generic_libs= ${_cddl_lib} gnu/lib ${_kerberos5_lib} lib ${_secure_lib} usr.bin/lex/lib ${_ofed_lib} .for _DIR in ${LOCAL_LIB_DIRS} .if exists(${.CURDIR}/${_DIR}/Makefile) && empty(_generic_libs:M${_DIR}) _generic_libs+= ${_DIR} .endif .endfor lib/libopie__L lib/libtacplus__L: lib/libmd__L .if ${MK_CDDL} != "no" _cddl_lib_libumem= cddl/lib/libumem _cddl_lib_libnvpair= cddl/lib/libnvpair _cddl_lib_libavl= cddl/lib/libavl _cddl_lib_libuutil= cddl/lib/libuutil _cddl_lib_libzfs_core= cddl/lib/libzfs_core _cddl_lib_libctf= cddl/lib/libctf _cddl_lib= cddl/lib cddl/lib/libzfs_core__L: cddl/lib/libnvpair__L cddl/lib/libzfs__L: lib/libgeom__L cddl/lib/libctf__L: lib/libz__L .endif # cddl/lib/libdtrace requires lib/libproc and lib/librtld_db; it's only built # on select architectures though (see cddl/lib/Makefile) .if ${MACHINE_CPUARCH} != "sparc64" _prebuild_libs+= lib/libproc lib/librtld_db .endif .if ${MK_CRYPT} != "no" .if ${MK_OPENSSL} != "no" _secure_lib_libcrypto= secure/lib/libcrypto _secure_lib_libssl= secure/lib/libssl lib/libradius__L secure/lib/libssl__L: secure/lib/libcrypto__L .if ${MK_LDNS} != "no" _lib_libldns= lib/libldns lib/libldns__L: secure/lib/libcrypto__L .endif .if ${MK_OPENSSH} != "no" _secure_lib_libssh= secure/lib/libssh secure/lib/libssh__L: lib/libz__L secure/lib/libcrypto__L lib/libcrypt__L .if ${MK_LDNS} != "no" secure/lib/libssh__L: lib/libldns__L .endif .if ${MK_KERBEROS_SUPPORT} != "no" secure/lib/libssh__L: lib/libgssapi__L kerberos5/lib/libkrb5__L \ kerberos5/lib/libhx509__L kerberos5/lib/libasn1__L lib/libcom_err__L \ lib/libmd__L kerberos5/lib/libroken__L .endif .endif .endif _secure_lib= secure/lib .endif .if ${MK_KERBEROS} != "no" kerberos5/lib/libasn1__L: lib/libcom_err__L kerberos5/lib/libroken__L kerberos5/lib/libhdb__L: kerberos5/lib/libasn1__L lib/libcom_err__L \ kerberos5/lib/libkrb5__L kerberos5/lib/libroken__L \ kerberos5/lib/libwind__L lib/libsqlite3__L kerberos5/lib/libheimntlm__L: secure/lib/libcrypto__L kerberos5/lib/libkrb5__L \ kerberos5/lib/libroken__L lib/libcom_err__L kerberos5/lib/libhx509__L: kerberos5/lib/libasn1__L lib/libcom_err__L \ secure/lib/libcrypto__L kerberos5/lib/libroken__L kerberos5/lib/libwind__L kerberos5/lib/libkrb5__L: kerberos5/lib/libasn1__L lib/libcom_err__L \ lib/libcrypt__L secure/lib/libcrypto__L kerberos5/lib/libhx509__L \ kerberos5/lib/libroken__L kerberos5/lib/libwind__L \ kerberos5/lib/libheimbase__L kerberos5/lib/libheimipcc__L kerberos5/lib/libroken__L: lib/libcrypt__L kerberos5/lib/libwind__L: kerberos5/lib/libroken__L lib/libcom_err__L kerberos5/lib/libheimbase__L: lib/libthr__L kerberos5/lib/libheimipcc__L: kerberos5/lib/libroken__L kerberos5/lib/libheimbase__L lib/libthr__L .endif lib/libsqlite3__L: lib/libthr__L .if ${MK_GSSAPI} != "no" _lib_libgssapi= lib/libgssapi .endif .if ${MK_KERBEROS} != "no" _kerberos5_lib= kerberos5/lib _kerberos5_lib_libasn1= kerberos5/lib/libasn1 _kerberos5_lib_libhdb= kerberos5/lib/libhdb _kerberos5_lib_libheimbase= kerberos5/lib/libheimbase _kerberos5_lib_libkrb5= kerberos5/lib/libkrb5 _kerberos5_lib_libhx509= kerberos5/lib/libhx509 _kerberos5_lib_libroken= kerberos5/lib/libroken _kerberos5_lib_libheimntlm= kerberos5/lib/libheimntlm _libsqlite3= lib/libsqlite3 _kerberos5_lib_libheimipcc= kerberos5/lib/libheimipcc _kerberos5_lib_libwind= kerberos5/lib/libwind _libcom_err= lib/libcom_err .endif .if ${MK_NIS} != "no" _lib_libypclnt= lib/libypclnt .endif .if ${MK_OPENSSL} == "no" lib/libradius__L: lib/libmd__L .endif lib/libproc__L: \ ${_cddl_lib_libctf:D${_cddl_lib_libctf}__L} lib/libelf__L lib/librtld_db__L lib/libutil__L .if ${MK_CXX} != "no" .if ${MK_LIBCPLUSPLUS} != "no" lib/libproc__L: lib/libcxxrt__L .else # This implies MK_GNUCXX != "no"; see lib/libproc lib/libproc__L: gnu/lib/libsupc++__L .endif .endif gnu/lib/libdialog__L: lib/msun__L lib/ncurses/ncursesw__L .for _lib in ${_prereq_libs} ${_lib}__PL: .PHONY .MAKE .if exists(${.CURDIR}/${_lib}) ${_+_}@${ECHODIR} "===> ${_lib} (obj,all,install)"; \ cd ${.CURDIR}/${_lib}; \ ${MAKE} MK_TESTS=no DIRPRFX=${_lib}/ obj; \ ${MAKE} MK_TESTS=no MK_PROFILE=no -DNO_PIC \ DIRPRFX=${_lib}/ all; \ ${MAKE} MK_TESTS=no MK_PROFILE=no -DNO_PIC \ DIRPRFX=${_lib}/ install .endif .endfor .for _lib in ${_startup_libs} ${_prebuild_libs} ${_generic_libs} ${_lib}__L: .PHONY .MAKE .if exists(${.CURDIR}/${_lib}) ${_+_}@${ECHODIR} "===> ${_lib} (obj,all,install)"; \ cd ${.CURDIR}/${_lib}; \ ${MAKE} MK_TESTS=no DIRPRFX=${_lib}/ obj; \ ${MAKE} MK_TESTS=no DIRPRFX=${_lib}/ all; \ ${MAKE} MK_TESTS=no DIRPRFX=${_lib}/ install .endif .endfor _prereq_libs: ${_prereq_libs:S/$/__PL/} _startup_libs: ${_startup_libs:S/$/__L/} _prebuild_libs: ${_prebuild_libs:S/$/__L/} _generic_libs: ${_generic_libs:S/$/__L/} # Enable SUBDIR_PARALLEL when not calling 'make all', unless called from # 'everything' with _PARALLEL_SUBDIR_OK set. This is because it is unlikely # that running 'make all' from the top-level, especially with a SUBDIR_OVERRIDE # or LOCAL_DIRS set, will have a reliable build if SUBDIRs are built in # parallel. This is safe for the world stage of buildworld though since it has # already built libraries in a proper order and installed includes into # WORLDTMP. Special handling is done for SUBDIR ordering for 'install*' to # avoid trashing a system if it crashes mid-install. .if !make(all) || defined(_PARALLEL_SUBDIR_OK) SUBDIR_PARALLEL= .endif .include .if make(check-old) || make(check-old-dirs) || \ make(check-old-files) || make(check-old-libs) || \ make(delete-old) || make(delete-old-dirs) || \ make(delete-old-files) || make(delete-old-libs) # # check for / delete old files section # .include "ObsoleteFiles.inc" OLD_LIBS_MESSAGE="Please be sure no application still uses those libraries, \ else you can not start such an application. Consult UPDATING for more \ information regarding how to cope with the removal/revision bump of a \ specific library." .if !defined(BATCH_DELETE_OLD_FILES) RM_I=-i .else RM_I=-v .endif delete-old-files: .PHONY @echo ">>> Removing old files (only deletes safe to delete libs)" # Ask for every old file if the user really wants to remove it. # It's annoying, but better safe than sorry. # NB: We cannot pass the list of OLD_FILES as a parameter because the # argument list will get too long. Using .for/.endfor make "loops" will make # the Makefile parser segfault. @exec 3<&0; \ cd ${.CURDIR}; \ ${MAKE} -f ${.CURDIR}/Makefile.inc1 ${.MAKEFLAGS} ${.TARGET} \ -V OLD_FILES -V "OLD_FILES:Musr/share/*.gz:R" | xargs -n1 | \ while read file; do \ if [ -f "${DESTDIR}/$${file}" -o -L "${DESTDIR}/$${file}" ]; then \ chflags noschg "${DESTDIR}/$${file}" 2>/dev/null || true; \ rm ${RM_I} "${DESTDIR}/$${file}" <&3; \ fi; \ for ext in debug symbols; do \ if ! [ -e "${DESTDIR}/$${file}" ] && [ -f \ "${DESTDIR}${DEBUGDIR}/$${file}.$${ext}" ]; then \ rm ${RM_I} "${DESTDIR}${DEBUGDIR}/$${file}.$${ext}" \ <&3; \ fi; \ done; \ done # Remove catpages without corresponding manpages. @exec 3<&0; \ find ${DESTDIR}/usr/share/man/cat* ! -type d | \ sed -ep -e's:${DESTDIR}/usr/share/man/cat:${DESTDIR}/usr/share/man/man:' | \ while read catpage; do \ read manpage; \ if [ ! -e "$${manpage}" ]; then \ rm ${RM_I} $${catpage} <&3; \ fi; \ done @echo ">>> Old files removed" check-old-files: .PHONY @echo ">>> Checking for old files" @cd ${.CURDIR}; \ ${MAKE} -f ${.CURDIR}/Makefile.inc1 ${.MAKEFLAGS} ${.TARGET} \ -V OLD_FILES -V "OLD_FILES:Musr/share/*.gz:R" | xargs -n1 | \ while read file; do \ if [ -f "${DESTDIR}/$${file}" -o -L "${DESTDIR}/$${file}" ]; then \ echo "${DESTDIR}/$${file}"; \ fi; \ for ext in debug symbols; do \ if [ -f "${DESTDIR}${DEBUGDIR}/$${file}.$${ext}" ]; then \ echo "${DESTDIR}${DEBUGDIR}/$${file}.$${ext}"; \ fi; \ done; \ done # Check for catpages without corresponding manpages. @find ${DESTDIR}/usr/share/man/cat* ! -type d | \ sed -ep -e's:${DESTDIR}/usr/share/man/cat:${DESTDIR}/usr/share/man/man:' | \ while read catpage; do \ read manpage; \ if [ ! -e "$${manpage}" ]; then \ echo $${catpage}; \ fi; \ done delete-old-libs: .PHONY @echo ">>> Removing old libraries" @echo "${OLD_LIBS_MESSAGE}" | fmt @exec 3<&0; \ cd ${.CURDIR}; \ ${MAKE} -f ${.CURDIR}/Makefile.inc1 ${.MAKEFLAGS} ${.TARGET} \ -V OLD_LIBS | xargs -n1 | \ while read file; do \ if [ -f "${DESTDIR}/$${file}" -o -L "${DESTDIR}/$${file}" ]; then \ chflags noschg "${DESTDIR}/$${file}" 2>/dev/null || true; \ rm ${RM_I} "${DESTDIR}/$${file}" <&3; \ fi; \ for ext in debug symbols; do \ if ! [ -e "${DESTDIR}/$${file}" ] && [ -f \ "${DESTDIR}${DEBUGDIR}/$${file}.$${ext}" ]; then \ rm ${RM_I} "${DESTDIR}${DEBUGDIR}/$${file}.$${ext}" \ <&3; \ fi; \ done; \ done @echo ">>> Old libraries removed" check-old-libs: .PHONY @echo ">>> Checking for old libraries" @cd ${.CURDIR}; \ ${MAKE} -f ${.CURDIR}/Makefile.inc1 ${.MAKEFLAGS} ${.TARGET} \ -V OLD_LIBS | xargs -n1 | \ while read file; do \ if [ -f "${DESTDIR}/$${file}" -o -L "${DESTDIR}/$${file}" ]; then \ echo "${DESTDIR}/$${file}"; \ fi; \ for ext in debug symbols; do \ if [ -f "${DESTDIR}${DEBUGDIR}/$${file}.$${ext}" ]; then \ echo "${DESTDIR}${DEBUGDIR}/$${file}.$${ext}"; \ fi; \ done; \ done delete-old-dirs: .PHONY @echo ">>> Removing old directories" @cd ${.CURDIR}; \ ${MAKE} -f ${.CURDIR}/Makefile.inc1 ${.MAKEFLAGS} ${.TARGET} \ -V OLD_DIRS | xargs -n1 | sort -r | \ while read dir; do \ if [ -d "${DESTDIR}/$${dir}" ]; then \ rmdir -v "${DESTDIR}/$${dir}" || true; \ elif [ -L "${DESTDIR}/$${dir}" ]; then \ echo "${DESTDIR}/$${dir} is a link, please remove everything manually."; \ fi; \ done @echo ">>> Old directories removed" check-old-dirs: .PHONY @echo ">>> Checking for old directories" @cd ${.CURDIR}; \ ${MAKE} -f ${.CURDIR}/Makefile.inc1 ${.MAKEFLAGS} ${.TARGET} \ -V OLD_DIRS | xargs -n1 | \ while read dir; do \ if [ -d "${DESTDIR}/$${dir}" ]; then \ echo "${DESTDIR}/$${dir}"; \ elif [ -L "${DESTDIR}/$${dir}" ]; then \ echo "${DESTDIR}/$${dir} is a link, please remove everything manually."; \ fi; \ done delete-old: delete-old-files delete-old-dirs .PHONY @echo "To remove old libraries run '${MAKE} delete-old-libs'." check-old: check-old-files check-old-libs check-old-dirs .PHONY @echo "To remove old files and directories run '${MAKE} delete-old'." @echo "To remove old libraries run '${MAKE} delete-old-libs'." .endif # # showconfig - show build configuration. # showconfig: .PHONY @(${MAKE} -n -f ${.CURDIR}/sys/conf/kern.opts.mk -V dummy -dg1; \ ${MAKE} -n -f ${.CURDIR}/share/mk/src.opts.mk -V dummy -dg1) 2>&1 | grep ^MK_ | sort -u .if !empty(KRNLOBJDIR) && !empty(KERNCONF) DTBOUTPUTPATH= ${KRNLOBJDIR}/${KERNCONF}/ .if !defined(FDT_DTS_FILE) || empty(FDT_DTS_FILE) .if exists(${KERNCONFDIR}/${KERNCONF}) FDT_DTS_FILE!= awk 'BEGIN {FS="="} /^makeoptions[[:space:]]+FDT_DTS_FILE/ {print $$2}' \ '${KERNCONFDIR}/${KERNCONF}' ; echo .endif .endif .endif .if !defined(DTBOUTPUTPATH) || !exists(${DTBOUTPUTPATH}) DTBOUTPUTPATH= ${.CURDIR} .endif # # Build 'standalone' Device Tree Blob # builddtb: .PHONY @PATH=${TMPPATH} MACHINE=${TARGET} \ ${.CURDIR}/sys/tools/fdt/make_dtb.sh ${.CURDIR}/sys \ "${FDT_DTS_FILE}" ${DTBOUTPUTPATH} ############### # cleanworld # In the following, the first 'rm' in a series will usually remove all # files and directories. If it does not, then there are probably some # files with file flags set, so this unsets them and tries the 'rm' a # second time. There are situations where this target will be cleaning # some directories via more than one method, but that duplication is # needed to correctly handle all the possible situations. Removing all # files without file flags set in the first 'rm' instance saves time, # because 'chflags' will need to operate on fewer files afterwards. # # It is expected that BW_CANONICALOBJDIR == the CANONICALOBJDIR as would be # created by bsd.obj.mk, except that we don't want to .include that file # in this makefile. # BW_CANONICALOBJDIR:=${OBJTREE}${.CURDIR} cleanworld: .PHONY .if exists(${BW_CANONICALOBJDIR}/) -rm -rf ${BW_CANONICALOBJDIR}/* -chflags -R 0 ${BW_CANONICALOBJDIR} rm -rf ${BW_CANONICALOBJDIR}/* .endif .if ${.CURDIR} == ${.OBJDIR} || ${.CURDIR}/obj == ${.OBJDIR} # To be safe in this case, fall back to a 'make cleandir' ${_+_}@cd ${.CURDIR}; ${MAKE} cleandir .endif .if defined(TARGET) && defined(TARGET_ARCH) .if ${TARGET} == ${MACHINE} && ${TARGET_ARCH} == ${MACHINE_ARCH} XDEV_CPUTYPE?=${CPUTYPE} .else XDEV_CPUTYPE?=${TARGET_CPUTYPE} .endif NOFUN=-DNO_FSCHG MK_HTML=no -DNO_LINT \ MK_MAN=no MK_NLS=no MK_PROFILE=no \ MK_KERBEROS=no MK_RESCUE=no MK_TESTS=no MK_WARNS=no \ TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \ CPUTYPE=${XDEV_CPUTYPE} XDDIR=${TARGET_ARCH}-freebsd XDTP?=/usr/${XDDIR} .if ${XDTP:N/*} .error XDTP variable should be an absolute path .endif CDBENV=MAKEOBJDIRPREFIX=${MAKEOBJDIRPREFIX}/${XDDIR} \ INSTALL="sh ${.CURDIR}/tools/install.sh" CDENV= ${CDBENV} \ TOOLS_PREFIX=${XDTP} CD2CFLAGS=-isystem ${XDDESTDIR}/usr/include -L${XDDESTDIR}/usr/lib \ --sysroot=${XDDESTDIR}/ -B${XDDESTDIR}/usr/libexec \ -B${XDDESTDIR}/usr/bin -B${XDDESTDIR}/usr/lib CD2ENV=${CDENV} CC="${CC} ${CD2CFLAGS}" CXX="${CXX} ${CD2CFLAGS}" \ CPP="${CPP} ${CD2CFLAGS}" \ MACHINE=${TARGET} MACHINE_ARCH=${TARGET_ARCH} CDTMP= ${MAKEOBJDIRPREFIX}/${XDDIR}/${.CURDIR}/tmp CDMAKE=${CDENV} PATH=${CDTMP}/usr/bin:${PATH} ${MAKE} ${NOFUN} CD2MAKE=${CD2ENV} PATH=${CDTMP}/usr/bin:${XDDESTDIR}/usr/bin:${PATH} ${MAKE} ${NOFUN} XDDESTDIR=${DESTDIR}/${XDTP} .if !defined(OSREL) OSREL!= uname -r | sed -e 's/[-(].*//' .endif .ORDER: xdev-build xdev-install xdev-links xdev: xdev-build xdev-install .PHONY .ORDER: _xb-worldtmp _xb-bootstrap-tools _xb-build-tools _xb-cross-tools xdev-build: _xb-worldtmp _xb-bootstrap-tools _xb-build-tools _xb-cross-tools .PHONY _xb-worldtmp: .PHONY mkdir -p ${CDTMP}/usr mtree -deU -f ${.CURDIR}/etc/mtree/BSD.usr.dist \ -p ${CDTMP}/usr >/dev/null _xb-bootstrap-tools: .PHONY .for _tool in \ ${_clang_tblgen} \ ${_gperf} ${_+_}@${ECHODIR} "===> ${_tool} (obj,all,install)"; \ cd ${.CURDIR}/${_tool}; \ ${CDMAKE} DIRPRFX=${_tool}/ obj; \ ${CDMAKE} DIRPRFX=${_tool}/ all; \ ${CDMAKE} DIRPRFX=${_tool}/ DESTDIR=${CDTMP} install .endfor _xb-build-tools: .PHONY ${_+_}@cd ${.CURDIR}; \ ${CDBENV} ${MAKE} -f Makefile.inc1 ${NOFUN} build-tools _xb-cross-tools: .PHONY .for _tool in \ ${_binutils} \ ${_elftctools} \ usr.bin/ar \ ${_clang_libs} \ ${_clang} \ ${_cc} ${_+_}@${ECHODIR} "===> xdev ${_tool} (obj,all)"; \ cd ${.CURDIR}/${_tool}; \ ${CDMAKE} DIRPRFX=${_tool}/ obj; \ ${CDMAKE} DIRPRFX=${_tool}/ all .endfor _xi-mtree: .PHONY ${_+_}@${ECHODIR} "mtree populating ${XDDESTDIR}" mkdir -p ${XDDESTDIR} mtree -deU -f ${.CURDIR}/etc/mtree/BSD.root.dist \ -p ${XDDESTDIR} >/dev/null mtree -deU -f ${.CURDIR}/etc/mtree/BSD.usr.dist \ -p ${XDDESTDIR}/usr >/dev/null mtree -deU -f ${.CURDIR}/etc/mtree/BSD.include.dist \ -p ${XDDESTDIR}/usr/include >/dev/null .if defined(LIBCOMPAT) mtree -deU -f ${.CURDIR}/etc/mtree/BSD.lib${libcompat}.dist \ -p ${XDDESTDIR}/usr >/dev/null .endif .if ${MK_TESTS} != "no" mkdir -p ${XDDESTDIR}${TESTSBASE} mtree -deU -f ${.CURDIR}/etc/mtree/BSD.tests.dist \ -p ${XDDESTDIR}${TESTSBASE} >/dev/null .endif .ORDER: xdev-build _xi-mtree _xi-cross-tools _xi-includes _xi-libraries xdev-install: xdev-build _xi-mtree _xi-cross-tools _xi-includes _xi-libraries .PHONY _xi-cross-tools: .PHONY @echo "_xi-cross-tools" .for _tool in \ ${_binutils} \ ${_elftctools} \ usr.bin/ar \ ${_clang_libs} \ ${_clang} \ ${_cc} ${_+_}@${ECHODIR} "===> xdev ${_tool} (install)"; \ cd ${.CURDIR}/${_tool}; \ ${CDMAKE} DIRPRFX=${_tool}/ install DESTDIR=${XDDESTDIR} .endfor _xi-includes: .PHONY ${_+_}cd ${.CURDIR}; ${CD2MAKE} -f Makefile.inc1 includes \ DESTDIR=${XDDESTDIR} _xi-libraries: .PHONY ${_+_}cd ${.CURDIR}; ${CD2MAKE} -f Makefile.inc1 libraries \ DESTDIR=${XDDESTDIR} xdev-links: .PHONY ${_+_}cd ${XDDESTDIR}/usr/bin; \ mkdir -p ../../../../usr/bin; \ for i in *; do \ ln -sf ../../${XDTP}/usr/bin/$$i \ ../../../../usr/bin/${XDDIR}-$$i; \ ln -sf ../../${XDTP}/usr/bin/$$i \ ../../../../usr/bin/${XDDIR}${OSREL}-$$i; \ done .else xdev xdev-build xdev-install xdev-links: .PHONY @echo "*** Error: Both TARGET and TARGET_ARCH must be defined for \"${.TARGET}\" target" .endif Index: user/ngie/release-pkg-fix-tests/etc/mtree/BSD.tests.dist =================================================================== --- user/ngie/release-pkg-fix-tests/etc/mtree/BSD.tests.dist (revision 299090) +++ user/ngie/release-pkg-fix-tests/etc/mtree/BSD.tests.dist (revision 299091) @@ -1,664 +1,666 @@ # $FreeBSD$ # # Please see the file src/etc/mtree/README before making changes to this file. # /set type=dir uname=root gname=wheel mode=0755 . bin cat .. chown .. date .. dd .. expr .. ls .. mv .. pax .. pkill .. sh builtins .. errors .. execution .. expansion .. parameters .. parser .. set-e .. .. sleep .. test .. .. cddl lib .. sbin .. usr.bin .. usr.sbin dtrace common aggs .. arithmetic .. arrays .. assocs .. begin .. bitfields .. buffering .. builtinvar .. cg .. clauses .. cpc .. decls .. drops .. dtraceUtil .. end .. enum .. error .. exit .. fbtprovider .. funcs .. grammar .. include .. inline .. io .. ip .. java_api .. json .. lexer .. llquantize .. mdb .. mib .. misc .. multiaggs .. offsetof .. operators .. pid .. plockstat .. pointers .. pragma .. predicates .. preprocessor .. print .. printa .. printf .. privs .. probes .. proc .. profile-n .. providers .. raise .. rates .. safety .. scalars .. sched .. scripting .. sdt .. sizeof .. speculation .. stability .. stack .. stackdepth .. stop .. strlen .. strtoll .. struct .. syscall .. sysevent .. tick-n .. trace .. tracemem .. translators .. typedef .. types .. uctf .. union .. usdt .. ustack .. vars .. version .. .. .. .. .. etc rc.d .. .. games .. gnu lib .. usr.bin diff .. .. .. lib atf libatf-c detail .. .. libatf-c++ detail .. .. test-programs .. .. libarchive .. libc c063 .. db .. gen execve .. posix_spawn .. .. hash data .. .. inet .. locale .. net getaddrinfo data .. .. .. nss .. regex data .. .. resolv .. rpc .. ssp .. setjmp .. stdio .. stdlib .. string .. sys .. time .. tls dso .. .. termios .. ttyio .. .. libcrypt .. libmp .. libnv .. libpam .. libproc .. librt .. libthr dlopen .. .. libutil .. libxo .. msun .. .. libexec atf atf-check .. atf-sh .. .. rtld-elf .. .. sbin dhclient .. devd .. growfs .. ifconfig .. mdconfig .. .. secure lib .. libexec .. usr.bin .. usr.sbin .. .. share examples tests atf .. plain .. .. .. .. sys acl .. aio .. fifo .. file .. geom class concat .. eli .. gate .. gpt .. mirror .. nop .. raid3 .. shsec .. stripe .. uzip etalon .. .. .. .. kern acct .. execve .. pipe .. .. kqueue .. mac bsdextended .. portacl .. .. mqueue .. netinet .. opencrypto .. pjdfstest chflags .. chmod .. chown .. ftruncate .. granular .. link .. mkdir .. mkfifo .. mknod .. open .. rename .. rmdir .. symlink .. truncate .. unlink .. .. posixshm .. + sys + .. vfs .. vm .. .. usr.bin apply .. basename .. bmake archives fmt_44bsd .. fmt_44bsd_mod .. fmt_oldbsd .. .. basic t0 .. t1 .. t2 .. t3 .. .. execution ellipsis .. empty .. joberr .. plus .. .. shell builtin .. meta .. path .. path_select .. replace .. select .. .. suffixes basic .. src_wild1 .. src_wild2 .. .. syntax directive-t0 .. enl .. funny-targets .. semi .. .. sysmk t0 2 1 .. .. mk .. .. t1 2 1 .. .. mk .. .. t2 2 1 .. .. mk .. .. .. variables modifier_M .. modifier_t .. opt_V .. t0 .. .. .. calendar .. cmp .. cpio .. col .. comm .. cut .. dirname .. file2c .. grep .. gzip .. ident .. join .. jot .. lastcomm .. limits .. m4 .. mkimg .. ncal .. opensm .. printf .. sdiff .. sed regress.multitest.out .. .. soelim .. tar .. timeout .. tr .. truncate .. units .. uudecode .. uuencode .. xargs .. xo .. yacc yacc .. .. .. usr.sbin etcupdate .. extattr .. fstyp .. makefs .. newsyslog .. nmtree .. pw .. rpcbind .. sa .. .. .. # vim: set expandtab ts=4 sw=4: Index: user/ngie/release-pkg-fix-tests/include/bitstring.h =================================================================== --- user/ngie/release-pkg-fix-tests/include/bitstring.h (revision 299090) +++ user/ngie/release-pkg-fix-tests/include/bitstring.h (revision 299091) @@ -1,35 +1,37 @@ /*- * Copyright (c) 2003 Poul-Henning Kamp * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _BITSTRING_H_ #define _BITSTRING_H_ +#include +#include #include #endif /* _BITSTRING_H_ */ Index: user/ngie/release-pkg-fix-tests/lib/libbluetooth/bluetooth.h =================================================================== --- user/ngie/release-pkg-fix-tests/lib/libbluetooth/bluetooth.h (revision 299090) +++ user/ngie/release-pkg-fix-tests/lib/libbluetooth/bluetooth.h (revision 299091) @@ -1,216 +1,218 @@ /* * bluetooth.h */ /*- * Copyright (c) 2001-2009 Maksim Yevmenkin * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $Id: bluetooth.h,v 1.5 2003/09/14 23:28:42 max Exp $ * $FreeBSD$ */ #ifndef _BLUETOOTH_H_ #define _BLUETOOTH_H_ #include -#include #include #include #include #include #include + #include #include +#include + #include #include #include #include #include __BEGIN_DECLS /* * Linux BlueZ compatibility */ #define bacmp(ba1, ba2) memcmp((ba1), (ba2), sizeof(bdaddr_t)) #define bacpy(dst, src) memcpy((dst), (src), sizeof(bdaddr_t)) #define ba2str(ba, str) bt_ntoa((ba), (str)) #define str2ba(str, ba) (bt_aton((str), (ba)) == 1? 0 : -1) #define htobs(d) htole16(d) #define htobl(d) htole32(d) #define btohs(d) le16toh(d) #define btohl(d) le32toh(d) /* * Interface to the outside world */ struct hostent * bt_gethostbyname (char const *name); struct hostent * bt_gethostbyaddr (char const *addr, int len, int type); struct hostent * bt_gethostent (void); void bt_sethostent (int stayopen); void bt_endhostent (void); struct protoent * bt_getprotobyname (char const *name); struct protoent * bt_getprotobynumber (int proto); struct protoent * bt_getprotoent (void); void bt_setprotoent (int stayopen); void bt_endprotoent (void); char const * bt_ntoa (bdaddr_t const *ba, char *str); int bt_aton (char const *str, bdaddr_t *ba); /* bt_devXXXX() functions (inspired by NetBSD) */ int bt_devaddr (char const *devname, bdaddr_t *addr); int bt_devname (char *devname, bdaddr_t const *addr); /* * Bluetooth HCI functions */ #define HCI_DEVMAX 32 /* arbitrary */ #define HCI_DEVNAME_SIZE NG_NODESIZ #define HCI_DEVFEATURES_SIZE NG_HCI_FEATURES_SIZE struct bt_devinfo { char devname[HCI_DEVNAME_SIZE]; uint32_t state; /* device/implementation specific */ bdaddr_t bdaddr; uint16_t _reserved0; uint8_t features[HCI_DEVFEATURES_SIZE]; /* buffer info */ uint16_t _reserved1; uint16_t cmd_free; uint16_t sco_size; uint16_t sco_pkts; uint16_t sco_free; uint16_t acl_size; uint16_t acl_pkts; uint16_t acl_free; /* stats */ uint32_t cmd_sent; uint32_t evnt_recv; uint32_t acl_recv; uint32_t acl_sent; uint32_t sco_recv; uint32_t sco_sent; uint32_t bytes_recv; uint32_t bytes_sent; /* misc/specific */ uint16_t link_policy_info; uint16_t packet_type_info; uint16_t role_switch_info; uint16_t debug; uint8_t _padding[20]; /* leave space for future additions */ }; struct bt_devreq { uint16_t opcode; uint8_t event; void *cparam; size_t clen; void *rparam; size_t rlen; }; struct bt_devfilter { bitstr_t bit_decl(packet_mask, 8); bitstr_t bit_decl(event_mask, 256); }; struct bt_devinquiry { bdaddr_t bdaddr; uint8_t pscan_rep_mode; uint8_t pscan_period_mode; uint8_t dev_class[3]; uint16_t clock_offset; int8_t rssi; uint8_t data[240]; }; typedef int (bt_devenum_cb_t)(int, struct bt_devinfo const *, void *); int bt_devopen (char const *devname); int bt_devclose(int s); int bt_devsend (int s, uint16_t opcode, void *param, size_t plen); ssize_t bt_devrecv (int s, void *buf, size_t size, time_t to); int bt_devreq (int s, struct bt_devreq *r, time_t to); int bt_devfilter(int s, struct bt_devfilter const *newp, struct bt_devfilter *oldp); void bt_devfilter_pkt_set(struct bt_devfilter *filter, uint8_t type); void bt_devfilter_pkt_clr(struct bt_devfilter *filter, uint8_t type); int bt_devfilter_pkt_tst(struct bt_devfilter const *filter, uint8_t type); void bt_devfilter_evt_set(struct bt_devfilter *filter, uint8_t event); void bt_devfilter_evt_clr(struct bt_devfilter *filter, uint8_t event); int bt_devfilter_evt_tst(struct bt_devfilter const *filter, uint8_t event); int bt_devinquiry(char const *devname, time_t length, int num_rsp, struct bt_devinquiry **ii); int bt_devinfo (struct bt_devinfo *di); int bt_devenum (bt_devenum_cb_t cb, void *arg); /* * bdaddr utility functions (from NetBSD) */ static __inline int bdaddr_same(const bdaddr_t *a, const bdaddr_t *b) { return (a->b[0] == b->b[0] && a->b[1] == b->b[1] && a->b[2] == b->b[2] && a->b[3] == b->b[3] && a->b[4] == b->b[4] && a->b[5] == b->b[5]); } static __inline int bdaddr_any(const bdaddr_t *a) { return (a->b[0] == 0 && a->b[1] == 0 && a->b[2] == 0 && a->b[3] == 0 && a->b[4] == 0 && a->b[5] == 0); } static __inline void bdaddr_copy(bdaddr_t *d, const bdaddr_t *s) { d->b[0] = s->b[0]; d->b[1] = s->b[1]; d->b[2] = s->b[2]; d->b[3] = s->b[3]; d->b[4] = s->b[4]; d->b[5] = s->b[5]; } __END_DECLS #endif /* ndef _BLUETOOTH_H_ */ Index: user/ngie/release-pkg-fix-tests/sbin/fsck_msdosfs/boot.c =================================================================== --- user/ngie/release-pkg-fix-tests/sbin/fsck_msdosfs/boot.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sbin/fsck_msdosfs/boot.c (revision 299091) @@ -1,275 +1,275 @@ /* * Copyright (C) 1995, 1997 Wolfgang Solfrank * Copyright (c) 1995 Martin Husemann * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include #ifndef lint __RCSID("$NetBSD: boot.c,v 1.11 2006/06/05 16:51:18 christos Exp "); static const char rcsid[] = "$FreeBSD$"; #endif /* not lint */ #include #include #include #include #include "ext.h" #include "fsutil.h" int readboot(int dosfs, struct bootblock *boot) { u_char block[DOSBOOTBLOCKSIZE]; u_char fsinfo[2 * DOSBOOTBLOCKSIZE]; u_char backup[DOSBOOTBLOCKSIZE]; int ret = FSOK; int i; if ((size_t)read(dosfs, block, sizeof block) != sizeof block) { perr("could not read boot block"); return FSFATAL; } if (block[510] != 0x55 || block[511] != 0xaa) { pfatal("Invalid signature in boot block: %02x%02x", block[511], block[510]); return FSFATAL; } memset(boot, 0, sizeof *boot); boot->ValidFat = -1; /* decode bios parameter block */ boot->bpbBytesPerSec = block[11] + (block[12] << 8); boot->bpbSecPerClust = block[13]; boot->bpbResSectors = block[14] + (block[15] << 8); boot->bpbFATs = block[16]; boot->bpbRootDirEnts = block[17] + (block[18] << 8); boot->bpbSectors = block[19] + (block[20] << 8); boot->bpbMedia = block[21]; boot->bpbFATsmall = block[22] + (block[23] << 8); boot->SecPerTrack = block[24] + (block[25] << 8); boot->bpbHeads = block[26] + (block[27] << 8); boot->bpbHiddenSecs = block[28] + (block[29] << 8) + (block[30] << 16) + (block[31] << 24); boot->bpbHugeSectors = block[32] + (block[33] << 8) + (block[34] << 16) + (block[35] << 24); boot->FATsecs = boot->bpbFATsmall; if (!boot->bpbRootDirEnts) boot->flags |= FAT32; if (boot->flags & FAT32) { boot->FATsecs = block[36] + (block[37] << 8) + (block[38] << 16) + (block[39] << 24); if (block[40] & 0x80) boot->ValidFat = block[40] & 0x0f; /* check version number: */ if (block[42] || block[43]) { /* Correct? XXX */ pfatal("Unknown file system version: %x.%x", block[43], block[42]); return FSFATAL; } boot->bpbRootClust = block[44] + (block[45] << 8) + (block[46] << 16) + (block[47] << 24); boot->bpbFSInfo = block[48] + (block[49] << 8); boot->bpbBackup = block[50] + (block[51] << 8); if (lseek(dosfs, boot->bpbFSInfo * boot->bpbBytesPerSec, SEEK_SET) != boot->bpbFSInfo * boot->bpbBytesPerSec || read(dosfs, fsinfo, sizeof fsinfo) != sizeof fsinfo) { perr("could not read fsinfo block"); return FSFATAL; } if (memcmp(fsinfo, "RRaA", 4) || memcmp(fsinfo + 0x1e4, "rrAa", 4) || fsinfo[0x1fc] || fsinfo[0x1fd] || fsinfo[0x1fe] != 0x55 || fsinfo[0x1ff] != 0xaa || fsinfo[0x3fc] || fsinfo[0x3fd] || fsinfo[0x3fe] != 0x55 || fsinfo[0x3ff] != 0xaa) { pwarn("Invalid signature in fsinfo block\n"); if (ask(0, "Fix")) { memcpy(fsinfo, "RRaA", 4); memcpy(fsinfo + 0x1e4, "rrAa", 4); fsinfo[0x1fc] = fsinfo[0x1fd] = 0; fsinfo[0x1fe] = 0x55; fsinfo[0x1ff] = 0xaa; fsinfo[0x3fc] = fsinfo[0x3fd] = 0; fsinfo[0x3fe] = 0x55; fsinfo[0x3ff] = 0xaa; if (lseek(dosfs, boot->bpbFSInfo * boot->bpbBytesPerSec, SEEK_SET) != boot->bpbFSInfo * boot->bpbBytesPerSec || write(dosfs, fsinfo, sizeof fsinfo) != sizeof fsinfo) { perr("Unable to write bpbFSInfo"); return FSFATAL; } ret = FSBOOTMOD; } else boot->bpbFSInfo = 0; } if (boot->bpbFSInfo) { boot->FSFree = fsinfo[0x1e8] + (fsinfo[0x1e9] << 8) + (fsinfo[0x1ea] << 16) + (fsinfo[0x1eb] << 24); boot->FSNext = fsinfo[0x1ec] + (fsinfo[0x1ed] << 8) + (fsinfo[0x1ee] << 16) + (fsinfo[0x1ef] << 24); } if (lseek(dosfs, boot->bpbBackup * boot->bpbBytesPerSec, SEEK_SET) != boot->bpbBackup * boot->bpbBytesPerSec || read(dosfs, backup, sizeof backup) != sizeof backup) { perr("could not read backup bootblock"); return FSFATAL; } backup[65] = block[65]; /* XXX */ if (memcmp(block + 11, backup + 11, 79)) { /* * XXX We require a reference that explains * that these bytes need to match, or should * drop the check. gdt@NetBSD has observed * filesystems that work fine under Windows XP * and NetBSD that do not match, so the * requirement is suspect. For now, just * print out useful information and continue. */ pfatal("backup (block %d) mismatch with primary bootblock:\n", boot->bpbBackup); for (i = 11; i < 11 + 90; i++) { if (block[i] != backup[i]) pfatal("\ti=%d\tprimary 0x%02x\tbackup 0x%02x\n", i, block[i], backup[i]); } } /* Check backup bpbFSInfo? XXX */ } boot->ClusterOffset = (boot->bpbRootDirEnts * 32 + boot->bpbBytesPerSec - 1) / boot->bpbBytesPerSec + boot->bpbResSectors + boot->bpbFATs * boot->FATsecs - CLUST_FIRST * boot->bpbSecPerClust; if (boot->bpbBytesPerSec % DOSBOOTBLOCKSIZE_REAL != 0) { pfatal("Invalid sector size: %u", boot->bpbBytesPerSec); return FSFATAL; } if (boot->bpbSecPerClust == 0) { pfatal("Invalid cluster size: %u", boot->bpbSecPerClust); return FSFATAL; } if (boot->bpbSectors) { boot->bpbHugeSectors = 0; boot->NumSectors = boot->bpbSectors; } else boot->NumSectors = boot->bpbHugeSectors; boot->NumClusters = (boot->NumSectors - boot->ClusterOffset) / boot->bpbSecPerClust; if (boot->flags&FAT32) boot->ClustMask = CLUST32_MASK; else if (boot->NumClusters < (CLUST_RSRVD&CLUST12_MASK)) boot->ClustMask = CLUST12_MASK; else if (boot->NumClusters < (CLUST_RSRVD&CLUST16_MASK)) boot->ClustMask = CLUST16_MASK; else { pfatal("Filesystem too big (%u clusters) for non-FAT32 partition", boot->NumClusters); return FSFATAL; } switch (boot->ClustMask) { case CLUST32_MASK: boot->NumFatEntries = (boot->FATsecs * boot->bpbBytesPerSec) / 4; break; case CLUST16_MASK: boot->NumFatEntries = (boot->FATsecs * boot->bpbBytesPerSec) / 2; break; default: boot->NumFatEntries = (boot->FATsecs * boot->bpbBytesPerSec * 2) / 3; break; } - if (boot->NumFatEntries < boot->NumClusters) { + if (boot->NumFatEntries < boot->NumClusters - CLUST_FIRST) { pfatal("FAT size too small, %u entries won't fit into %u sectors\n", boot->NumClusters, boot->FATsecs); return FSFATAL; } boot->ClusterSize = boot->bpbBytesPerSec * boot->bpbSecPerClust; boot->NumFiles = 1; boot->NumFree = 0; return ret; } int writefsinfo(int dosfs, struct bootblock *boot) { u_char fsinfo[2 * DOSBOOTBLOCKSIZE]; if (lseek(dosfs, boot->bpbFSInfo * boot->bpbBytesPerSec, SEEK_SET) != boot->bpbFSInfo * boot->bpbBytesPerSec || read(dosfs, fsinfo, sizeof fsinfo) != sizeof fsinfo) { perr("could not read fsinfo block"); return FSFATAL; } fsinfo[0x1e8] = (u_char)boot->FSFree; fsinfo[0x1e9] = (u_char)(boot->FSFree >> 8); fsinfo[0x1ea] = (u_char)(boot->FSFree >> 16); fsinfo[0x1eb] = (u_char)(boot->FSFree >> 24); fsinfo[0x1ec] = (u_char)boot->FSNext; fsinfo[0x1ed] = (u_char)(boot->FSNext >> 8); fsinfo[0x1ee] = (u_char)(boot->FSNext >> 16); fsinfo[0x1ef] = (u_char)(boot->FSNext >> 24); if (lseek(dosfs, boot->bpbFSInfo * boot->bpbBytesPerSec, SEEK_SET) != boot->bpbFSInfo * boot->bpbBytesPerSec || write(dosfs, fsinfo, sizeof fsinfo) != sizeof fsinfo) { perr("Unable to write bpbFSInfo"); return FSFATAL; } /* * Technically, we should return FSBOOTMOD here. * * However, since Win95 OSR2 (the first M$ OS that has * support for FAT32) doesn't maintain the FSINFO block * correctly, it has to be fixed pretty often. * * Therefor, we handle the FSINFO block only informally, * fixing it if necessary, but otherwise ignoring the * fact that it was incorrect. */ return 0; } Index: user/ngie/release-pkg-fix-tests/sbin/hastd/activemap.c =================================================================== --- user/ngie/release-pkg-fix-tests/sbin/hastd/activemap.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sbin/hastd/activemap.c (revision 299091) @@ -1,701 +1,701 @@ /*- * Copyright (c) 2009-2010 The FreeBSD Foundation * All rights reserved. * * This software was developed by Pawel Jakub Dawidek under sponsorship from * the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include /* powerof2() */ #include #include #include #include #include #include #include #include #include "activemap.h" #ifndef PJDLOG_ASSERT #include #define PJDLOG_ASSERT(...) assert(__VA_ARGS__) #endif #define ACTIVEMAP_MAGIC 0xac71e4 struct activemap { int am_magic; /* Magic value. */ off_t am_mediasize; /* Media size in bytes. */ uint32_t am_extentsize; /* Extent size in bytes, must be power of 2. */ uint8_t am_extentshift;/* 2 ^ extentbits == extentsize */ int am_nextents; /* Number of extents. */ size_t am_mapsize; /* Bitmap size in bytes. */ uint16_t *am_memtab; /* An array that holds number of pending writes per extent. */ bitstr_t *am_diskmap; /* On-disk bitmap of dirty extents. */ bitstr_t *am_memmap; /* In-memory bitmap of dirty extents. */ size_t am_diskmapsize; /* Map size rounded up to sector size. */ uint64_t am_ndirty; /* Number of dirty regions. */ bitstr_t *am_syncmap; /* Bitmap of extents to sync. */ off_t am_syncoff; /* Next synchronization offset. */ TAILQ_HEAD(skeepdirty, keepdirty) am_keepdirty; /* List of extents that we keep dirty to reduce bitmap updates. */ int am_nkeepdirty; /* Number of am_keepdirty elements. */ int am_nkeepdirty_limit; /* Maximum number of am_keepdirty elements. */ }; struct keepdirty { int kd_extent; TAILQ_ENTRY(keepdirty) kd_next; }; /* * Helper function taken from sys/systm.h to calculate extentshift. */ static uint32_t bitcount32(uint32_t x) { x = (x & 0x55555555) + ((x & 0xaaaaaaaa) >> 1); x = (x & 0x33333333) + ((x & 0xcccccccc) >> 2); x = (x + (x >> 4)) & 0x0f0f0f0f; x = (x + (x >> 8)); x = (x + (x >> 16)) & 0x000000ff; return (x); } static __inline int off2ext(const struct activemap *amp, off_t offset) { int extent; PJDLOG_ASSERT(offset >= 0 && offset < amp->am_mediasize); extent = (offset >> amp->am_extentshift); PJDLOG_ASSERT(extent >= 0 && extent < amp->am_nextents); return (extent); } static __inline off_t ext2off(const struct activemap *amp, int extent) { off_t offset; PJDLOG_ASSERT(extent >= 0 && extent < amp->am_nextents); offset = ((off_t)extent << amp->am_extentshift); PJDLOG_ASSERT(offset >= 0 && offset < amp->am_mediasize); return (offset); } /* * Function calculates number of requests needed to synchronize the given * extent. */ static __inline int ext2reqs(const struct activemap *amp, int ext) { off_t left; if (ext < amp->am_nextents - 1) return (((amp->am_extentsize - 1) / MAXPHYS) + 1); PJDLOG_ASSERT(ext == amp->am_nextents - 1); left = amp->am_mediasize % amp->am_extentsize; if (left == 0) left = amp->am_extentsize; return (((left - 1) / MAXPHYS) + 1); } /* * Initialize activemap structure and allocate memory for internal needs. * Function returns 0 on success and -1 if any of the allocations failed. */ int activemap_init(struct activemap **ampp, uint64_t mediasize, uint32_t extentsize, uint32_t sectorsize, uint32_t keepdirty) { struct activemap *amp; PJDLOG_ASSERT(ampp != NULL); PJDLOG_ASSERT(mediasize > 0); PJDLOG_ASSERT(extentsize > 0); PJDLOG_ASSERT(powerof2(extentsize)); PJDLOG_ASSERT(sectorsize > 0); PJDLOG_ASSERT(powerof2(sectorsize)); PJDLOG_ASSERT(keepdirty > 0); amp = malloc(sizeof(*amp)); if (amp == NULL) return (-1); amp->am_mediasize = mediasize; amp->am_nkeepdirty_limit = keepdirty; amp->am_extentsize = extentsize; amp->am_extentshift = bitcount32(extentsize - 1); amp->am_nextents = ((mediasize - 1) / extentsize) + 1; - amp->am_mapsize = sizeof(bitstr_t) * bitstr_size(amp->am_nextents); + amp->am_mapsize = bitstr_size(amp->am_nextents); amp->am_diskmapsize = roundup2(amp->am_mapsize, sectorsize); amp->am_ndirty = 0; amp->am_syncoff = -2; TAILQ_INIT(&->am_keepdirty); amp->am_nkeepdirty = 0; amp->am_memtab = calloc(amp->am_nextents, sizeof(amp->am_memtab[0])); amp->am_diskmap = calloc(1, amp->am_diskmapsize); amp->am_memmap = bit_alloc(amp->am_nextents); amp->am_syncmap = bit_alloc(amp->am_nextents); /* * Check to see if any of the allocations above failed. */ if (amp->am_memtab == NULL || amp->am_diskmap == NULL || amp->am_memmap == NULL || amp->am_syncmap == NULL) { if (amp->am_memtab != NULL) free(amp->am_memtab); if (amp->am_diskmap != NULL) free(amp->am_diskmap); if (amp->am_memmap != NULL) free(amp->am_memmap); if (amp->am_syncmap != NULL) free(amp->am_syncmap); amp->am_magic = 0; free(amp); errno = ENOMEM; return (-1); } amp->am_magic = ACTIVEMAP_MAGIC; *ampp = amp; return (0); } static struct keepdirty * keepdirty_find(struct activemap *amp, int extent) { struct keepdirty *kd; TAILQ_FOREACH(kd, &->am_keepdirty, kd_next) { if (kd->kd_extent == extent) break; } return (kd); } static bool keepdirty_add(struct activemap *amp, int extent) { struct keepdirty *kd; kd = keepdirty_find(amp, extent); if (kd != NULL) { /* * Only move element at the beginning. */ TAILQ_REMOVE(&->am_keepdirty, kd, kd_next); TAILQ_INSERT_HEAD(&->am_keepdirty, kd, kd_next); return (false); } /* * Add new element, but first remove the most unused one if * we have too many. */ if (amp->am_nkeepdirty >= amp->am_nkeepdirty_limit) { kd = TAILQ_LAST(&->am_keepdirty, skeepdirty); PJDLOG_ASSERT(kd != NULL); TAILQ_REMOVE(&->am_keepdirty, kd, kd_next); amp->am_nkeepdirty--; PJDLOG_ASSERT(amp->am_nkeepdirty > 0); } if (kd == NULL) kd = malloc(sizeof(*kd)); /* We can ignore allocation failure. */ if (kd != NULL) { kd->kd_extent = extent; amp->am_nkeepdirty++; TAILQ_INSERT_HEAD(&->am_keepdirty, kd, kd_next); } return (true); } static void keepdirty_fill(struct activemap *amp) { struct keepdirty *kd; TAILQ_FOREACH(kd, &->am_keepdirty, kd_next) bit_set(amp->am_diskmap, kd->kd_extent); } static void keepdirty_free(struct activemap *amp) { struct keepdirty *kd; while ((kd = TAILQ_FIRST(&->am_keepdirty)) != NULL) { TAILQ_REMOVE(&->am_keepdirty, kd, kd_next); amp->am_nkeepdirty--; free(kd); } PJDLOG_ASSERT(amp->am_nkeepdirty == 0); } /* * Function frees resources allocated by activemap_init() function. */ void activemap_free(struct activemap *amp) { PJDLOG_ASSERT(amp->am_magic == ACTIVEMAP_MAGIC); amp->am_magic = 0; keepdirty_free(amp); free(amp->am_memtab); free(amp->am_diskmap); free(amp->am_memmap); free(amp->am_syncmap); } /* * Function should be called before we handle write requests. It updates * internal structures and returns true if on-disk metadata should be updated. */ bool activemap_write_start(struct activemap *amp, off_t offset, off_t length) { bool modified; off_t end; int ext; PJDLOG_ASSERT(amp->am_magic == ACTIVEMAP_MAGIC); PJDLOG_ASSERT(length > 0); modified = false; end = offset + length - 1; for (ext = off2ext(amp, offset); ext <= off2ext(amp, end); ext++) { /* * If the number of pending writes is increased from 0, * we have to mark the extent as dirty also in on-disk bitmap. * By returning true we inform the caller that on-disk bitmap * was modified and has to be flushed to disk. */ if (amp->am_memtab[ext]++ == 0) { PJDLOG_ASSERT(!bit_test(amp->am_memmap, ext)); bit_set(amp->am_memmap, ext); amp->am_ndirty++; } if (keepdirty_add(amp, ext)) modified = true; } return (modified); } /* * Function should be called after receiving write confirmation. It updates * internal structures and returns true if on-disk metadata should be updated. */ bool activemap_write_complete(struct activemap *amp, off_t offset, off_t length) { bool modified; off_t end; int ext; PJDLOG_ASSERT(amp->am_magic == ACTIVEMAP_MAGIC); PJDLOG_ASSERT(length > 0); modified = false; end = offset + length - 1; for (ext = off2ext(amp, offset); ext <= off2ext(amp, end); ext++) { /* * If the number of pending writes goes down to 0, we have to * mark the extent as clean also in on-disk bitmap. * By returning true we inform the caller that on-disk bitmap * was modified and has to be flushed to disk. */ PJDLOG_ASSERT(amp->am_memtab[ext] > 0); PJDLOG_ASSERT(bit_test(amp->am_memmap, ext)); if (--amp->am_memtab[ext] == 0) { bit_clear(amp->am_memmap, ext); amp->am_ndirty--; if (keepdirty_find(amp, ext) == NULL) modified = true; } } return (modified); } /* * Function should be called after finishing synchronization of one extent. * It returns true if on-disk metadata should be updated. */ bool activemap_extent_complete(struct activemap *amp, int extent) { bool modified; int reqs; PJDLOG_ASSERT(amp->am_magic == ACTIVEMAP_MAGIC); PJDLOG_ASSERT(extent >= 0 && extent < amp->am_nextents); modified = false; reqs = ext2reqs(amp, extent); PJDLOG_ASSERT(amp->am_memtab[extent] >= reqs); amp->am_memtab[extent] -= reqs; PJDLOG_ASSERT(bit_test(amp->am_memmap, extent)); if (amp->am_memtab[extent] == 0) { bit_clear(amp->am_memmap, extent); amp->am_ndirty--; modified = true; } return (modified); } /* * Function returns number of dirty regions. */ uint64_t activemap_ndirty(const struct activemap *amp) { PJDLOG_ASSERT(amp->am_magic == ACTIVEMAP_MAGIC); return (amp->am_ndirty); } /* * Function compare on-disk bitmap and in-memory bitmap and returns true if * they differ and should be flushed to the disk. */ bool activemap_differ(const struct activemap *amp) { PJDLOG_ASSERT(amp->am_magic == ACTIVEMAP_MAGIC); return (memcmp(amp->am_diskmap, amp->am_memmap, amp->am_mapsize) != 0); } /* * Function returns number of bytes used by bitmap. */ size_t activemap_size(const struct activemap *amp) { PJDLOG_ASSERT(amp->am_magic == ACTIVEMAP_MAGIC); return (amp->am_mapsize); } /* * Function returns number of bytes needed for storing on-disk bitmap. * This is the same as activemap_size(), but rounded up to sector size. */ size_t activemap_ondisk_size(const struct activemap *amp) { PJDLOG_ASSERT(amp->am_magic == ACTIVEMAP_MAGIC); return (amp->am_diskmapsize); } /* * Function copies the given buffer read from disk to the internal bitmap. */ void activemap_copyin(struct activemap *amp, const unsigned char *buf, size_t size) { int ext; PJDLOG_ASSERT(amp->am_magic == ACTIVEMAP_MAGIC); PJDLOG_ASSERT(size >= amp->am_mapsize); memcpy(amp->am_diskmap, buf, amp->am_mapsize); memcpy(amp->am_memmap, buf, amp->am_mapsize); memcpy(amp->am_syncmap, buf, amp->am_mapsize); bit_ffs(amp->am_memmap, amp->am_nextents, &ext); if (ext == -1) { /* There are no dirty extents, so we can leave now. */ return; } /* * Set synchronization offset to the first dirty extent. */ activemap_sync_rewind(amp); /* * We have dirty extents and we want them to stay that way until * we synchronize, so we set number of pending writes to number * of requests needed to synchronize one extent. */ amp->am_ndirty = 0; for (; ext < amp->am_nextents; ext++) { if (bit_test(amp->am_memmap, ext)) { amp->am_memtab[ext] = ext2reqs(amp, ext); amp->am_ndirty++; } } } /* * Function merges the given bitmap with existing one. */ void activemap_merge(struct activemap *amp, const unsigned char *buf, size_t size) { bitstr_t *remmap = __DECONST(bitstr_t *, buf); int ext; PJDLOG_ASSERT(amp->am_magic == ACTIVEMAP_MAGIC); PJDLOG_ASSERT(size >= amp->am_mapsize); bit_ffs(remmap, amp->am_nextents, &ext); if (ext == -1) { /* There are no dirty extents, so we can leave now. */ return; } /* * We have dirty extents and we want them to stay that way until * we synchronize, so we set number of pending writes to number * of requests needed to synchronize one extent. */ for (; ext < amp->am_nextents; ext++) { /* Local extent already dirty. */ if (bit_test(amp->am_syncmap, ext)) continue; /* Remote extent isn't dirty. */ if (!bit_test(remmap, ext)) continue; bit_set(amp->am_syncmap, ext); bit_set(amp->am_memmap, ext); bit_set(amp->am_diskmap, ext); if (amp->am_memtab[ext] == 0) amp->am_ndirty++; amp->am_memtab[ext] = ext2reqs(amp, ext); } /* * Set synchronization offset to the first dirty extent. */ activemap_sync_rewind(amp); } /* * Function returns pointer to internal bitmap that should be written to disk. */ const unsigned char * activemap_bitmap(struct activemap *amp, size_t *sizep) { PJDLOG_ASSERT(amp->am_magic == ACTIVEMAP_MAGIC); if (sizep != NULL) *sizep = amp->am_diskmapsize; memcpy(amp->am_diskmap, amp->am_memmap, amp->am_mapsize); keepdirty_fill(amp); return ((const unsigned char *)amp->am_diskmap); } /* * Function calculates size needed to store bitmap on disk. */ size_t activemap_calc_ondisk_size(uint64_t mediasize, uint32_t extentsize, uint32_t sectorsize) { uint64_t nextents, mapsize; PJDLOG_ASSERT(mediasize > 0); PJDLOG_ASSERT(extentsize > 0); PJDLOG_ASSERT(powerof2(extentsize)); PJDLOG_ASSERT(sectorsize > 0); PJDLOG_ASSERT(powerof2(sectorsize)); nextents = ((mediasize - 1) / extentsize) + 1; - mapsize = sizeof(bitstr_t) * bitstr_size(nextents); + mapsize = bitstr_size(nextents); return (roundup2(mapsize, sectorsize)); } /* * Set synchronization offset to the first dirty extent. */ void activemap_sync_rewind(struct activemap *amp) { int ext; PJDLOG_ASSERT(amp->am_magic == ACTIVEMAP_MAGIC); bit_ffs(amp->am_syncmap, amp->am_nextents, &ext); if (ext == -1) { /* There are no extents to synchronize. */ amp->am_syncoff = -2; return; } /* * Mark that we want to start synchronization from the beginning. */ amp->am_syncoff = -1; } /* * Return next offset of where we should synchronize. */ off_t activemap_sync_offset(struct activemap *amp, off_t *lengthp, int *syncextp) { off_t syncoff, left; int ext; PJDLOG_ASSERT(amp->am_magic == ACTIVEMAP_MAGIC); PJDLOG_ASSERT(lengthp != NULL); PJDLOG_ASSERT(syncextp != NULL); *syncextp = -1; if (amp->am_syncoff == -2) return (-1); if (amp->am_syncoff >= 0 && (amp->am_syncoff + MAXPHYS >= amp->am_mediasize || off2ext(amp, amp->am_syncoff) != off2ext(amp, amp->am_syncoff + MAXPHYS))) { /* * We are about to change extent, so mark previous one as clean. */ ext = off2ext(amp, amp->am_syncoff); bit_clear(amp->am_syncmap, ext); *syncextp = ext; amp->am_syncoff = -1; } if (amp->am_syncoff == -1) { /* * Let's find first extent to synchronize. */ bit_ffs(amp->am_syncmap, amp->am_nextents, &ext); if (ext == -1) { amp->am_syncoff = -2; return (-1); } amp->am_syncoff = ext2off(amp, ext); } else { /* * We don't change extent, so just increase offset. */ amp->am_syncoff += MAXPHYS; if (amp->am_syncoff >= amp->am_mediasize) { amp->am_syncoff = -2; return (-1); } } syncoff = amp->am_syncoff; left = ext2off(amp, off2ext(amp, syncoff)) + amp->am_extentsize - syncoff; if (syncoff + left > amp->am_mediasize) left = amp->am_mediasize - syncoff; if (left > MAXPHYS) left = MAXPHYS; PJDLOG_ASSERT(left >= 0 && left <= MAXPHYS); PJDLOG_ASSERT(syncoff >= 0 && syncoff < amp->am_mediasize); PJDLOG_ASSERT(syncoff + left >= 0 && syncoff + left <= amp->am_mediasize); *lengthp = left; return (syncoff); } /* * Mark extent(s) containing the given region for synchronization. * Most likely one of the components is unavailable. */ bool activemap_need_sync(struct activemap *amp, off_t offset, off_t length) { bool modified; off_t end; int ext; PJDLOG_ASSERT(amp->am_magic == ACTIVEMAP_MAGIC); modified = false; end = offset + length - 1; for (ext = off2ext(amp, offset); ext <= off2ext(amp, end); ext++) { if (bit_test(amp->am_syncmap, ext)) { /* Already marked for synchronization. */ PJDLOG_ASSERT(bit_test(amp->am_memmap, ext)); continue; } bit_set(amp->am_syncmap, ext); if (!bit_test(amp->am_memmap, ext)) { bit_set(amp->am_memmap, ext); amp->am_ndirty++; } amp->am_memtab[ext] += ext2reqs(amp, ext); modified = true; } return (modified); } void activemap_dump(const struct activemap *amp) { int bit; printf("M: "); for (bit = 0; bit < amp->am_nextents; bit++) printf("%d", bit_test(amp->am_memmap, bit) ? 1 : 0); printf("\n"); printf("D: "); for (bit = 0; bit < amp->am_nextents; bit++) printf("%d", bit_test(amp->am_diskmap, bit) ? 1 : 0); printf("\n"); printf("S: "); for (bit = 0; bit < amp->am_nextents; bit++) printf("%d", bit_test(amp->am_syncmap, bit) ? 1 : 0); printf("\n"); } Index: user/ngie/release-pkg-fix-tests/share/man/man3/bitstring.3 =================================================================== --- user/ngie/release-pkg-fix-tests/share/man/man3/bitstring.3 (revision 299090) +++ user/ngie/release-pkg-fix-tests/share/man/man3/bitstring.3 (revision 299091) @@ -1,187 +1,263 @@ .\" Copyright (c) 1989, 1991, 1993 .\" The Regents of the University of California. All rights reserved. .\" .\" This code is derived from software contributed to Berkeley by .\" Paul Vixie. .\" Redistribution and use in source and binary forms, with or without .\" modification, are permitted provided that the following conditions .\" are met: .\" 1. Redistributions of source code must retain the above copyright .\" notice, this list of conditions and the following disclaimer. .\" 2. Redistributions in binary form must reproduce the above copyright .\" notice, this list of conditions and the following disclaimer in the .\" documentation and/or other materials provided with the distribution. .\" 3. Neither the name of the University nor the names of its contributors .\" may be used to endorse or promote products derived from this software .\" without specific prior written permission. .\" .\" THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND .\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE .\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE .\" ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE .\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL .\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS .\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) .\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT .\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF .\" SUCH DAMAGE. .\" +.\" Copyright (c) 2014 Spectra Logic Corporation +.\" All rights reserved. +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions, and the following disclaimer, +.\" without modification. +.\" 2. Redistributions in binary form must reproduce at minimum a disclaimer +.\" substantially similar to the "NO WARRANTY" disclaimer below +.\" ("Disclaimer") and any redistribution must be conditioned upon +.\" including a substantially similar Disclaimer requirement for further +.\" binary redistribution. +.\" +.\" NO WARRANTY +.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +.\" "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +.\" LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR +.\" A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +.\" HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +.\" STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING +.\" IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +.\" POSSIBILITY OF SUCH DAMAGES. +.\" .\" @(#)bitstring.3 8.1 (Berkeley) 7/19/93 .\" $FreeBSD$ .\" -.Dd October 17, 2015 +.Dd May 4, 2016 .Dt BITSTRING 3 .Os .Sh NAME .Nm bit_alloc , .Nm bit_clear , .Nm bit_decl , +.Nm bit_ffc , .Nm bit_ffs , +.Nm bit_ffc_at , +.Nm bit_ffs_at , .Nm bit_nclear , .Nm bit_nset , .Nm bit_set , -.Nm bitstr_size , -.Nm bit_test -.Nd bit-string manipulation macros +.Nm bit_test , +.Nm bitstr_size +.Nd bit-string manipulation functions and macros .Sh SYNOPSIS .In bitstring.h .Ft bitstr_t * .Fn bit_alloc "int nbits" .Ft void .Fn bit_decl "bitstr_t *name" "int nbits" .Ft void .Fn bit_clear "bitstr_t *name" "int bit" .Ft void .Fn bit_ffc "bitstr_t *name" "int nbits" "int *value" .Ft void .Fn bit_ffs "bitstr_t *name" "int nbits" "int *value" .Ft void +.Fn bit_ffc_at "bitstr_t *name" "int start" "int nbits" "int *value" +.Ft void +.Fn bit_ffs_at "bitstr_t *name" "int start" "int nbits" "int *value" +.Ft void .Fn bit_nclear "bitstr_t *name" "int start" "int stop" .Ft void .Fn bit_nset "bitstr_t *name" "int start" "int stop" .Ft void .Fn bit_set "bitstr_t *name" "int bit" .Ft int .Fn bitstr_size "int nbits" .Ft int .Fn bit_test "bitstr_t *name" "int bit" .Sh DESCRIPTION These macros operate on strings of bits. .Pp -The macro +The function .Fn bit_alloc returns a pointer of type .Dq Fa "bitstr_t *" to sufficient space to store .Fa nbits bits, or .Dv NULL if no space is available. +If successful, the returned bit string is initialized with all bits cleared. .Pp The macro .Fn bit_decl -allocates sufficient space to store +declares a bit string with sufficient space to store .Fa nbits -bits on the stack. +bits. +.Fn bit_decl +may be used to include statically sized bit strings in structure +definitions or to create bit strings on the stack. +Users of this macro are responsible for initialization of the bit string, +typically via a global initialization of the containing struct or use of the +.Fn bit_nset +or +.Fn bin_nclear +functions. .Pp The macro .Fn bitstr_size -returns the number of elements of type -.Fa bitstr_t -necessary to store +returns the number of bytes necessary to store .Fa nbits bits. This is useful for copying bit strings. .Pp -The macros +The functions .Fn bit_clear and .Fn bit_set clear or set the zero-based numbered bit .Fa bit , in the bit string .Ar name . .Pp The .Fn bit_nset and .Fn bit_nclear -macros +functions set or clear the zero-based numbered bits from .Fa start through .Fa stop in the bit string .Ar name . .Pp The .Fn bit_test -macro +function evaluates to non-zero if the zero-based numbered bit .Fa bit of bit string .Fa name is set, and zero otherwise. .Pp +The function +.Fn bit_ffc +stores in the location referenced by +.Fa value +the zero-based number of the first bit not set in the array of +.Fa nbits +bits referenced by +.Fa name . +If all bits are set, the location referenced by +.Fa value +is set to \-1. +.Pp The .Fn bit_ffs -macro +function stores in the location referenced by .Fa value the zero-based number of the first bit set in the array of .Fa nbits bits referenced by .Fa name . If no bits are set, the location referenced by .Fa value is set to \-1. .Pp -The macro -.Fn bit_ffc +The function +.Fn bit_ffc_at stores in the location referenced by .Fa value the zero-based number of the first bit not set in the array of .Fa nbits bits referenced by -.Fa name . -If all bits are set, the location referenced by +.Fa name , +at or after the zero-based bit index +.Fa start . +If all bits at or after +.Fa start +are set, the location referenced by .Fa value is set to \-1. .Pp -The arguments to these macros are evaluated only once and may safely +The +.Fn bit_ffs_at +function +stores in the location referenced by +.Fa value +the zero-based number of the first bit set in the array of +.Fa nbits +bits referenced by +.Fa name , +at or after the zero-based bit index +.Fa start . +If no bits are set after +.Fa start , +the location referenced by +.Fa value +is set to \-1. +.Pp +The arguments in bit string macros are evaluated only once and may safely have side effects. .Sh EXAMPLES .Bd -literal -offset indent #include #include \&... #define LPR_BUSY_BIT 0 #define LPR_FORMAT_BIT 1 #define LPR_DOWNLOAD_BIT 2 \&... #define LPR_AVAILABLE_BIT 9 #define LPR_MAX_BITS 10 make_lpr_available() { bitstr_t bit_decl(bitlist, LPR_MAX_BITS); ... bit_nclear(bitlist, 0, LPR_MAX_BITS - 1); ... if (!bit_test(bitlist, LPR_BUSY_BIT)) { bit_clear(bitlist, LPR_FORMAT_BIT); bit_clear(bitlist, LPR_DOWNLOAD_BIT); bit_set(bitlist, LPR_AVAILABLE_BIT); } } .Ed .Sh SEE ALSO .Xr malloc 3 , .Xr bitset 9 .Sh HISTORY The .Nm bitstring functions first appeared in .Bx 4.4 . Index: user/ngie/release-pkg-fix-tests/sys/amd64/amd64/pmap.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/amd64/amd64/pmap.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/amd64/amd64/pmap.c (revision 299091) @@ -1,7147 +1,7167 @@ /*- * Copyright (c) 1991 Regents of the University of California. * All rights reserved. * Copyright (c) 1994 John S. Dyson * All rights reserved. * Copyright (c) 1994 David Greenman * All rights reserved. * Copyright (c) 2003 Peter Wemm * All rights reserved. * Copyright (c) 2005-2010 Alan L. Cox * All rights reserved. * * This code is derived from software contributed to Berkeley by * the Systems Programming Group of the University of Utah Computer * Science Department and William Jolitz of UUNET Technologies Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by the University of * California, Berkeley and its contributors. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 */ /*- * Copyright (c) 2003 Networks Associates Technology, Inc. * All rights reserved. * * This software was developed for the FreeBSD Project by Jake Burkholder, * Safeport Network Services, and Network Associates Laboratories, the * Security Research Division of Network Associates, Inc. under * DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the DARPA * CHATS research program. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #define AMD64_NPT_AWARE #include __FBSDID("$FreeBSD$"); /* * Manages physical address maps. * * Since the information managed by this module is * also stored by the logical address mapping module, * this module may throw away valid virtual-to-physical * mappings at almost any time. However, invalidations * of virtual-to-physical mappings must be done as * requested. * * In order to cope with hardware architectures which * make virtual-to-physical map invalidates expensive, * this module may delay invalidate or reduced protection * operations until such time as they are actually * necessary. This module is given full information as * to which processors are currently using which maps, * and to when physical maps must be made correct. */ #include "opt_pmap.h" #include "opt_vm.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef SMP #include #endif static __inline boolean_t pmap_type_guest(pmap_t pmap) { return ((pmap->pm_type == PT_EPT) || (pmap->pm_type == PT_RVI)); } static __inline boolean_t pmap_emulate_ad_bits(pmap_t pmap) { return ((pmap->pm_flags & PMAP_EMULATE_AD_BITS) != 0); } static __inline pt_entry_t pmap_valid_bit(pmap_t pmap) { pt_entry_t mask; switch (pmap->pm_type) { case PT_X86: case PT_RVI: mask = X86_PG_V; break; case PT_EPT: if (pmap_emulate_ad_bits(pmap)) mask = EPT_PG_EMUL_V; else mask = EPT_PG_READ; break; default: panic("pmap_valid_bit: invalid pm_type %d", pmap->pm_type); } return (mask); } static __inline pt_entry_t pmap_rw_bit(pmap_t pmap) { pt_entry_t mask; switch (pmap->pm_type) { case PT_X86: case PT_RVI: mask = X86_PG_RW; break; case PT_EPT: if (pmap_emulate_ad_bits(pmap)) mask = EPT_PG_EMUL_RW; else mask = EPT_PG_WRITE; break; default: panic("pmap_rw_bit: invalid pm_type %d", pmap->pm_type); } return (mask); } static __inline pt_entry_t pmap_global_bit(pmap_t pmap) { pt_entry_t mask; switch (pmap->pm_type) { case PT_X86: mask = X86_PG_G; break; case PT_RVI: case PT_EPT: mask = 0; break; default: panic("pmap_global_bit: invalid pm_type %d", pmap->pm_type); } return (mask); } static __inline pt_entry_t pmap_accessed_bit(pmap_t pmap) { pt_entry_t mask; switch (pmap->pm_type) { case PT_X86: case PT_RVI: mask = X86_PG_A; break; case PT_EPT: if (pmap_emulate_ad_bits(pmap)) mask = EPT_PG_READ; else mask = EPT_PG_A; break; default: panic("pmap_accessed_bit: invalid pm_type %d", pmap->pm_type); } return (mask); } static __inline pt_entry_t pmap_modified_bit(pmap_t pmap) { pt_entry_t mask; switch (pmap->pm_type) { case PT_X86: case PT_RVI: mask = X86_PG_M; break; case PT_EPT: if (pmap_emulate_ad_bits(pmap)) mask = EPT_PG_WRITE; else mask = EPT_PG_M; break; default: panic("pmap_modified_bit: invalid pm_type %d", pmap->pm_type); } return (mask); } extern struct pcpu __pcpu[]; #if !defined(DIAGNOSTIC) #ifdef __GNUC_GNU_INLINE__ #define PMAP_INLINE __attribute__((__gnu_inline__)) inline #else #define PMAP_INLINE extern inline #endif #else #define PMAP_INLINE #endif #ifdef PV_STATS #define PV_STAT(x) do { x ; } while (0) #else #define PV_STAT(x) do { } while (0) #endif #define pa_index(pa) ((pa) >> PDRSHIFT) #define pa_to_pvh(pa) (&pv_table[pa_index(pa)]) #define NPV_LIST_LOCKS MAXCPU #define PHYS_TO_PV_LIST_LOCK(pa) \ (&pv_list_locks[pa_index(pa) % NPV_LIST_LOCKS]) #define CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa) do { \ struct rwlock **_lockp = (lockp); \ struct rwlock *_new_lock; \ \ _new_lock = PHYS_TO_PV_LIST_LOCK(pa); \ if (_new_lock != *_lockp) { \ if (*_lockp != NULL) \ rw_wunlock(*_lockp); \ *_lockp = _new_lock; \ rw_wlock(*_lockp); \ } \ } while (0) #define CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m) \ CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, VM_PAGE_TO_PHYS(m)) #define RELEASE_PV_LIST_LOCK(lockp) do { \ struct rwlock **_lockp = (lockp); \ \ if (*_lockp != NULL) { \ rw_wunlock(*_lockp); \ *_lockp = NULL; \ } \ } while (0) #define VM_PAGE_TO_PV_LIST_LOCK(m) \ PHYS_TO_PV_LIST_LOCK(VM_PAGE_TO_PHYS(m)) struct pmap kernel_pmap_store; vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */ vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */ int nkpt; SYSCTL_INT(_machdep, OID_AUTO, nkpt, CTLFLAG_RD, &nkpt, 0, "Number of kernel page table pages allocated on bootup"); static int ndmpdp; vm_paddr_t dmaplimit; vm_offset_t kernel_vm_end = VM_MIN_KERNEL_ADDRESS; pt_entry_t pg_nx; static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters"); static int pat_works = 1; SYSCTL_INT(_vm_pmap, OID_AUTO, pat_works, CTLFLAG_RD, &pat_works, 1, "Is page attribute table fully functional?"); static int pg_ps_enabled = 1; SYSCTL_INT(_vm_pmap, OID_AUTO, pg_ps_enabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &pg_ps_enabled, 0, "Are large page mappings enabled?"); #define PAT_INDEX_SIZE 8 static int pat_index[PAT_INDEX_SIZE]; /* cache mode to PAT index conversion */ static u_int64_t KPTphys; /* phys addr of kernel level 1 */ static u_int64_t KPDphys; /* phys addr of kernel level 2 */ u_int64_t KPDPphys; /* phys addr of kernel level 3 */ u_int64_t KPML4phys; /* phys addr of kernel level 4 */ static u_int64_t DMPDphys; /* phys addr of direct mapped level 2 */ static u_int64_t DMPDPphys; /* phys addr of direct mapped level 3 */ static int ndmpdpphys; /* number of DMPDPphys pages */ /* * pmap_mapdev support pre initialization (i.e. console) */ #define PMAP_PREINIT_MAPPING_COUNT 8 static struct pmap_preinit_mapping { vm_paddr_t pa; vm_offset_t va; vm_size_t sz; int mode; } pmap_preinit_mapping[PMAP_PREINIT_MAPPING_COUNT]; static int pmap_initialized; static struct rwlock_padalign pvh_global_lock; /* * Data for the pv entry allocation mechanism */ static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks); static struct mtx pv_chunks_mutex; static struct rwlock pv_list_locks[NPV_LIST_LOCKS]; static struct md_page *pv_table; /* * All those kernel PT submaps that BSD is so fond of */ pt_entry_t *CMAP1 = 0; caddr_t CADDR1 = 0; static vm_offset_t qframe = 0; static struct mtx qframe_mtx; static int pmap_flags = PMAP_PDE_SUPERPAGE; /* flags for x86 pmaps */ int pmap_pcid_enabled = 1; SYSCTL_INT(_vm_pmap, OID_AUTO, pcid_enabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &pmap_pcid_enabled, 0, "Is TLB Context ID enabled ?"); int invpcid_works = 0; SYSCTL_INT(_vm_pmap, OID_AUTO, invpcid_works, CTLFLAG_RD, &invpcid_works, 0, "Is the invpcid instruction available ?"); static int pmap_pcid_save_cnt_proc(SYSCTL_HANDLER_ARGS) { int i; uint64_t res; res = 0; CPU_FOREACH(i) { res += cpuid_to_pcpu[i]->pc_pm_save_cnt; } return (sysctl_handle_64(oidp, &res, 0, req)); } SYSCTL_PROC(_vm_pmap, OID_AUTO, pcid_save_cnt, CTLTYPE_U64 | CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, 0, pmap_pcid_save_cnt_proc, "QU", "Count of saved TLB context on switch"); /* * Crashdump maps. */ static caddr_t crashdumpmap; static void free_pv_chunk(struct pv_chunk *pc); static void free_pv_entry(pmap_t pmap, pv_entry_t pv); static pv_entry_t get_pv_entry(pmap_t pmap, struct rwlock **lockp); static int popcnt_pc_map_elem_pq(uint64_t elem); static vm_page_t reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp); static void reserve_pv_entries(pmap_t pmap, int needed, struct rwlock **lockp); static void pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, struct rwlock **lockp); static boolean_t pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, struct rwlock **lockp); static void pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, struct rwlock **lockp); static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va); static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va); static int pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode); static boolean_t pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va); static boolean_t pmap_demote_pde_locked(pmap_t pmap, pd_entry_t *pde, vm_offset_t va, struct rwlock **lockp); static boolean_t pmap_demote_pdpe(pmap_t pmap, pdp_entry_t *pdpe, vm_offset_t va); static boolean_t pmap_enter_pde(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, struct rwlock **lockp); static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp); static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte); static int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte); static void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode); static vm_page_t pmap_lookup_pt_page(pmap_t pmap, vm_offset_t va); static void pmap_pde_attr(pd_entry_t *pde, int cache_bits, int mask); static void pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va, struct rwlock **lockp); static boolean_t pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva, vm_prot_t prot); static void pmap_pte_attr(pt_entry_t *pte, int cache_bits, int mask); static int pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva, struct spglist *free, struct rwlock **lockp); static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva, pd_entry_t ptepde, struct spglist *free, struct rwlock **lockp); static void pmap_remove_pt_page(pmap_t pmap, vm_page_t mpte); static void pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, struct spglist *free); static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m, struct rwlock **lockp); static void pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t newpde); static void pmap_update_pde_invalidate(pmap_t, vm_offset_t va, pd_entry_t pde); static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp); static vm_page_t pmap_allocpde(pmap_t pmap, vm_offset_t va, struct rwlock **lockp); static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, struct rwlock **lockp); static void _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free); static int pmap_unuse_pt(pmap_t, vm_offset_t, pd_entry_t, struct spglist *); static vm_offset_t pmap_kmem_choose(vm_offset_t addr); /* * Move the kernel virtual free pointer to the next * 2MB. This is used to help improve performance * by using a large (2MB) page for much of the kernel * (.text, .data, .bss) */ static vm_offset_t pmap_kmem_choose(vm_offset_t addr) { vm_offset_t newaddr = addr; newaddr = roundup2(addr, NBPDR); return (newaddr); } /********************/ /* Inline functions */ /********************/ /* Return a non-clipped PD index for a given VA */ static __inline vm_pindex_t pmap_pde_pindex(vm_offset_t va) { return (va >> PDRSHIFT); } /* Return various clipped indexes for a given VA */ static __inline vm_pindex_t pmap_pte_index(vm_offset_t va) { return ((va >> PAGE_SHIFT) & ((1ul << NPTEPGSHIFT) - 1)); } static __inline vm_pindex_t pmap_pde_index(vm_offset_t va) { return ((va >> PDRSHIFT) & ((1ul << NPDEPGSHIFT) - 1)); } static __inline vm_pindex_t pmap_pdpe_index(vm_offset_t va) { return ((va >> PDPSHIFT) & ((1ul << NPDPEPGSHIFT) - 1)); } static __inline vm_pindex_t pmap_pml4e_index(vm_offset_t va) { return ((va >> PML4SHIFT) & ((1ul << NPML4EPGSHIFT) - 1)); } /* Return a pointer to the PML4 slot that corresponds to a VA */ static __inline pml4_entry_t * pmap_pml4e(pmap_t pmap, vm_offset_t va) { return (&pmap->pm_pml4[pmap_pml4e_index(va)]); } /* Return a pointer to the PDP slot that corresponds to a VA */ static __inline pdp_entry_t * pmap_pml4e_to_pdpe(pml4_entry_t *pml4e, vm_offset_t va) { pdp_entry_t *pdpe; pdpe = (pdp_entry_t *)PHYS_TO_DMAP(*pml4e & PG_FRAME); return (&pdpe[pmap_pdpe_index(va)]); } /* Return a pointer to the PDP slot that corresponds to a VA */ static __inline pdp_entry_t * pmap_pdpe(pmap_t pmap, vm_offset_t va) { pml4_entry_t *pml4e; pt_entry_t PG_V; PG_V = pmap_valid_bit(pmap); pml4e = pmap_pml4e(pmap, va); if ((*pml4e & PG_V) == 0) return (NULL); return (pmap_pml4e_to_pdpe(pml4e, va)); } /* Return a pointer to the PD slot that corresponds to a VA */ static __inline pd_entry_t * pmap_pdpe_to_pde(pdp_entry_t *pdpe, vm_offset_t va) { pd_entry_t *pde; pde = (pd_entry_t *)PHYS_TO_DMAP(*pdpe & PG_FRAME); return (&pde[pmap_pde_index(va)]); } /* Return a pointer to the PD slot that corresponds to a VA */ static __inline pd_entry_t * pmap_pde(pmap_t pmap, vm_offset_t va) { pdp_entry_t *pdpe; pt_entry_t PG_V; PG_V = pmap_valid_bit(pmap); pdpe = pmap_pdpe(pmap, va); if (pdpe == NULL || (*pdpe & PG_V) == 0) return (NULL); return (pmap_pdpe_to_pde(pdpe, va)); } /* Return a pointer to the PT slot that corresponds to a VA */ static __inline pt_entry_t * pmap_pde_to_pte(pd_entry_t *pde, vm_offset_t va) { pt_entry_t *pte; pte = (pt_entry_t *)PHYS_TO_DMAP(*pde & PG_FRAME); return (&pte[pmap_pte_index(va)]); } /* Return a pointer to the PT slot that corresponds to a VA */ static __inline pt_entry_t * pmap_pte(pmap_t pmap, vm_offset_t va) { pd_entry_t *pde; pt_entry_t PG_V; PG_V = pmap_valid_bit(pmap); pde = pmap_pde(pmap, va); if (pde == NULL || (*pde & PG_V) == 0) return (NULL); if ((*pde & PG_PS) != 0) /* compat with i386 pmap_pte() */ return ((pt_entry_t *)pde); return (pmap_pde_to_pte(pde, va)); } static __inline void pmap_resident_count_inc(pmap_t pmap, int count) { PMAP_LOCK_ASSERT(pmap, MA_OWNED); pmap->pm_stats.resident_count += count; } static __inline void pmap_resident_count_dec(pmap_t pmap, int count) { PMAP_LOCK_ASSERT(pmap, MA_OWNED); KASSERT(pmap->pm_stats.resident_count >= count, ("pmap %p resident count underflow %ld %d", pmap, pmap->pm_stats.resident_count, count)); pmap->pm_stats.resident_count -= count; } PMAP_INLINE pt_entry_t * vtopte(vm_offset_t va) { u_int64_t mask = ((1ul << (NPTEPGSHIFT + NPDEPGSHIFT + NPDPEPGSHIFT + NPML4EPGSHIFT)) - 1); KASSERT(va >= VM_MAXUSER_ADDRESS, ("vtopte on a uva/gpa 0x%0lx", va)); return (PTmap + ((va >> PAGE_SHIFT) & mask)); } static __inline pd_entry_t * vtopde(vm_offset_t va) { u_int64_t mask = ((1ul << (NPDEPGSHIFT + NPDPEPGSHIFT + NPML4EPGSHIFT)) - 1); KASSERT(va >= VM_MAXUSER_ADDRESS, ("vtopde on a uva/gpa 0x%0lx", va)); return (PDmap + ((va >> PDRSHIFT) & mask)); } static u_int64_t allocpages(vm_paddr_t *firstaddr, int n) { u_int64_t ret; ret = *firstaddr; bzero((void *)ret, n * PAGE_SIZE); *firstaddr += n * PAGE_SIZE; return (ret); } CTASSERT(powerof2(NDMPML4E)); /* number of kernel PDP slots */ #define NKPDPE(ptpgs) howmany(ptpgs, NPDEPG) static void nkpt_init(vm_paddr_t addr) { int pt_pages; #ifdef NKPT pt_pages = NKPT; #else pt_pages = howmany(addr, 1 << PDRSHIFT); pt_pages += NKPDPE(pt_pages); /* * Add some slop beyond the bare minimum required for bootstrapping * the kernel. * * This is quite important when allocating KVA for kernel modules. * The modules are required to be linked in the negative 2GB of * the address space. If we run out of KVA in this region then * pmap_growkernel() will need to allocate page table pages to map * the entire 512GB of KVA space which is an unnecessary tax on * physical memory. * * Secondly, device memory mapped as part of setting up the low- * level console(s) is taken from KVA, starting at virtual_avail. * This is because cninit() is called after pmap_bootstrap() but * before vm_init() and pmap_init(). 20MB for a frame buffer is * not uncommon. */ pt_pages += 32; /* 64MB additional slop. */ #endif nkpt = pt_pages; } static void create_pagetables(vm_paddr_t *firstaddr) { int i, j, ndm1g, nkpdpe; pt_entry_t *pt_p; pd_entry_t *pd_p; pdp_entry_t *pdp_p; pml4_entry_t *p4_p; /* Allocate page table pages for the direct map */ ndmpdp = howmany(ptoa(Maxmem), NBPDP); if (ndmpdp < 4) /* Minimum 4GB of dirmap */ ndmpdp = 4; ndmpdpphys = howmany(ndmpdp, NPDPEPG); if (ndmpdpphys > NDMPML4E) { /* * Each NDMPML4E allows 512 GB, so limit to that, * and then readjust ndmpdp and ndmpdpphys. */ printf("NDMPML4E limits system to %d GB\n", NDMPML4E * 512); Maxmem = atop(NDMPML4E * NBPML4); ndmpdpphys = NDMPML4E; ndmpdp = NDMPML4E * NPDEPG; } DMPDPphys = allocpages(firstaddr, ndmpdpphys); ndm1g = 0; if ((amd_feature & AMDID_PAGE1GB) != 0) ndm1g = ptoa(Maxmem) >> PDPSHIFT; if (ndm1g < ndmpdp) DMPDphys = allocpages(firstaddr, ndmpdp - ndm1g); dmaplimit = (vm_paddr_t)ndmpdp << PDPSHIFT; /* Allocate pages */ KPML4phys = allocpages(firstaddr, 1); KPDPphys = allocpages(firstaddr, NKPML4E); /* * Allocate the initial number of kernel page table pages required to * bootstrap. We defer this until after all memory-size dependent * allocations are done (e.g. direct map), so that we don't have to * build in too much slop in our estimate. * * Note that when NKPML4E > 1, we have an empty page underneath * all but the KPML4I'th one, so we need NKPML4E-1 extra (zeroed) * pages. (pmap_enter requires a PD page to exist for each KPML4E.) */ nkpt_init(*firstaddr); nkpdpe = NKPDPE(nkpt); KPTphys = allocpages(firstaddr, nkpt); KPDphys = allocpages(firstaddr, nkpdpe); /* Fill in the underlying page table pages */ /* Nominally read-only (but really R/W) from zero to physfree */ /* XXX not fully used, underneath 2M pages */ pt_p = (pt_entry_t *)KPTphys; for (i = 0; ptoa(i) < *firstaddr; i++) pt_p[i] = ptoa(i) | X86_PG_RW | X86_PG_V | X86_PG_G; /* Now map the page tables at their location within PTmap */ pd_p = (pd_entry_t *)KPDphys; for (i = 0; i < nkpt; i++) pd_p[i] = (KPTphys + ptoa(i)) | X86_PG_RW | X86_PG_V; /* Map from zero to end of allocations under 2M pages */ /* This replaces some of the KPTphys entries above */ for (i = 0; (i << PDRSHIFT) < *firstaddr; i++) pd_p[i] = (i << PDRSHIFT) | X86_PG_RW | X86_PG_V | PG_PS | X86_PG_G; /* And connect up the PD to the PDP (leaving room for L4 pages) */ pdp_p = (pdp_entry_t *)(KPDPphys + ptoa(KPML4I - KPML4BASE)); for (i = 0; i < nkpdpe; i++) pdp_p[i + KPDPI] = (KPDphys + ptoa(i)) | X86_PG_RW | X86_PG_V | PG_U; /* * Now, set up the direct map region using 2MB and/or 1GB pages. If * the end of physical memory is not aligned to a 1GB page boundary, * then the residual physical memory is mapped with 2MB pages. Later, * if pmap_mapdev{_attr}() uses the direct map for non-write-back * memory, pmap_change_attr() will demote any 2MB or 1GB page mappings * that are partially used. */ pd_p = (pd_entry_t *)DMPDphys; for (i = NPDEPG * ndm1g, j = 0; i < NPDEPG * ndmpdp; i++, j++) { pd_p[j] = (vm_paddr_t)i << PDRSHIFT; /* Preset PG_M and PG_A because demotion expects it. */ pd_p[j] |= X86_PG_RW | X86_PG_V | PG_PS | X86_PG_G | X86_PG_M | X86_PG_A; } pdp_p = (pdp_entry_t *)DMPDPphys; for (i = 0; i < ndm1g; i++) { pdp_p[i] = (vm_paddr_t)i << PDPSHIFT; /* Preset PG_M and PG_A because demotion expects it. */ pdp_p[i] |= X86_PG_RW | X86_PG_V | PG_PS | X86_PG_G | X86_PG_M | X86_PG_A; } for (j = 0; i < ndmpdp; i++, j++) { pdp_p[i] = DMPDphys + ptoa(j); pdp_p[i] |= X86_PG_RW | X86_PG_V | PG_U; } /* And recursively map PML4 to itself in order to get PTmap */ p4_p = (pml4_entry_t *)KPML4phys; p4_p[PML4PML4I] = KPML4phys; p4_p[PML4PML4I] |= X86_PG_RW | X86_PG_V | PG_U; /* Connect the Direct Map slot(s) up to the PML4. */ for (i = 0; i < ndmpdpphys; i++) { p4_p[DMPML4I + i] = DMPDPphys + ptoa(i); p4_p[DMPML4I + i] |= X86_PG_RW | X86_PG_V | PG_U; } /* Connect the KVA slots up to the PML4 */ for (i = 0; i < NKPML4E; i++) { p4_p[KPML4BASE + i] = KPDPphys + ptoa(i); p4_p[KPML4BASE + i] |= X86_PG_RW | X86_PG_V | PG_U; } } /* * Bootstrap the system enough to run with virtual memory. * * On amd64 this is called after mapping has already been enabled * and just syncs the pmap module with what has already been done. * [We can't call it easily with mapping off since the kernel is not * mapped with PA == VA, hence we would have to relocate every address * from the linked base (virtual) address "KERNBASE" to the actual * (physical) address starting relative to 0] */ void pmap_bootstrap(vm_paddr_t *firstaddr) { vm_offset_t va; pt_entry_t *pte; int i; /* * Create an initial set of page tables to run the kernel in. */ create_pagetables(firstaddr); /* * Add a physical memory segment (vm_phys_seg) corresponding to the * preallocated kernel page table pages so that vm_page structures * representing these pages will be created. The vm_page structures * are required for promotion of the corresponding kernel virtual * addresses to superpage mappings. */ vm_phys_add_seg(KPTphys, KPTphys + ptoa(nkpt)); virtual_avail = (vm_offset_t) KERNBASE + *firstaddr; virtual_avail = pmap_kmem_choose(virtual_avail); virtual_end = VM_MAX_KERNEL_ADDRESS; /* XXX do %cr0 as well */ load_cr4(rcr4() | CR4_PGE); load_cr3(KPML4phys); if (cpu_stdext_feature & CPUID_STDEXT_SMEP) load_cr4(rcr4() | CR4_SMEP); /* * Initialize the kernel pmap (which is statically allocated). */ PMAP_LOCK_INIT(kernel_pmap); kernel_pmap->pm_pml4 = (pdp_entry_t *)PHYS_TO_DMAP(KPML4phys); kernel_pmap->pm_cr3 = KPML4phys; CPU_FILL(&kernel_pmap->pm_active); /* don't allow deactivation */ TAILQ_INIT(&kernel_pmap->pm_pvchunk); kernel_pmap->pm_flags = pmap_flags; /* * Initialize the global pv list lock. */ rw_init(&pvh_global_lock, "pmap pv global"); /* * Reserve some special page table entries/VA space for temporary * mapping of pages. */ #define SYSMAP(c, p, v, n) \ v = (c)va; va += ((n)*PAGE_SIZE); p = pte; pte += (n); va = virtual_avail; pte = vtopte(va); /* * Crashdump maps. The first page is reused as CMAP1 for the * memory test. */ SYSMAP(caddr_t, CMAP1, crashdumpmap, MAXDUMPPGS) CADDR1 = crashdumpmap; virtual_avail = va; /* Initialize the PAT MSR. */ pmap_init_pat(); /* Initialize TLB Context Id. */ TUNABLE_INT_FETCH("vm.pmap.pcid_enabled", &pmap_pcid_enabled); if ((cpu_feature2 & CPUID2_PCID) != 0 && pmap_pcid_enabled) { /* Check for INVPCID support */ invpcid_works = (cpu_stdext_feature & CPUID_STDEXT_INVPCID) != 0; for (i = 0; i < MAXCPU; i++) { kernel_pmap->pm_pcids[i].pm_pcid = PMAP_PCID_KERN; kernel_pmap->pm_pcids[i].pm_gen = 1; } __pcpu[0].pc_pcid_next = PMAP_PCID_KERN + 1; __pcpu[0].pc_pcid_gen = 1; /* * pcpu area for APs is zeroed during AP startup. * pc_pcid_next and pc_pcid_gen are initialized by AP * during pcpu setup. */ load_cr4(rcr4() | CR4_PCIDE); } else { pmap_pcid_enabled = 0; } } /* * Setup the PAT MSR. */ void pmap_init_pat(void) { int pat_table[PAT_INDEX_SIZE]; uint64_t pat_msr; u_long cr0, cr4; int i; /* Bail if this CPU doesn't implement PAT. */ if ((cpu_feature & CPUID_PAT) == 0) panic("no PAT??"); /* Set default PAT index table. */ for (i = 0; i < PAT_INDEX_SIZE; i++) pat_table[i] = -1; pat_table[PAT_WRITE_BACK] = 0; pat_table[PAT_WRITE_THROUGH] = 1; pat_table[PAT_UNCACHEABLE] = 3; pat_table[PAT_WRITE_COMBINING] = 3; pat_table[PAT_WRITE_PROTECTED] = 3; pat_table[PAT_UNCACHED] = 3; /* Initialize default PAT entries. */ pat_msr = PAT_VALUE(0, PAT_WRITE_BACK) | PAT_VALUE(1, PAT_WRITE_THROUGH) | PAT_VALUE(2, PAT_UNCACHED) | PAT_VALUE(3, PAT_UNCACHEABLE) | PAT_VALUE(4, PAT_WRITE_BACK) | PAT_VALUE(5, PAT_WRITE_THROUGH) | PAT_VALUE(6, PAT_UNCACHED) | PAT_VALUE(7, PAT_UNCACHEABLE); if (pat_works) { /* * Leave the indices 0-3 at the default of WB, WT, UC-, and UC. * Program 5 and 6 as WP and WC. * Leave 4 and 7 as WB and UC. */ pat_msr &= ~(PAT_MASK(5) | PAT_MASK(6)); pat_msr |= PAT_VALUE(5, PAT_WRITE_PROTECTED) | PAT_VALUE(6, PAT_WRITE_COMBINING); pat_table[PAT_UNCACHED] = 2; pat_table[PAT_WRITE_PROTECTED] = 5; pat_table[PAT_WRITE_COMBINING] = 6; } else { /* * Just replace PAT Index 2 with WC instead of UC-. */ pat_msr &= ~PAT_MASK(2); pat_msr |= PAT_VALUE(2, PAT_WRITE_COMBINING); pat_table[PAT_WRITE_COMBINING] = 2; } /* Disable PGE. */ cr4 = rcr4(); load_cr4(cr4 & ~CR4_PGE); /* Disable caches (CD = 1, NW = 0). */ cr0 = rcr0(); load_cr0((cr0 & ~CR0_NW) | CR0_CD); /* Flushes caches and TLBs. */ wbinvd(); invltlb(); /* Update PAT and index table. */ wrmsr(MSR_PAT, pat_msr); for (i = 0; i < PAT_INDEX_SIZE; i++) pat_index[i] = pat_table[i]; /* Flush caches and TLBs again. */ wbinvd(); invltlb(); /* Restore caches and PGE. */ load_cr0(cr0); load_cr4(cr4); } /* * Initialize a vm_page's machine-dependent fields. */ void pmap_page_init(vm_page_t m) { TAILQ_INIT(&m->md.pv_list); m->md.pat_mode = PAT_WRITE_BACK; } /* * Initialize the pmap module. * Called by vm_init, to initialize any structures that the pmap * system needs to map virtual memory. */ void pmap_init(void) { struct pmap_preinit_mapping *ppim; vm_page_t mpte; vm_size_t s; int error, i, pv_npg; /* * Initialize the vm page array entries for the kernel pmap's * page table pages. */ for (i = 0; i < nkpt; i++) { mpte = PHYS_TO_VM_PAGE(KPTphys + (i << PAGE_SHIFT)); KASSERT(mpte >= vm_page_array && mpte < &vm_page_array[vm_page_array_size], ("pmap_init: page table page is out of range")); mpte->pindex = pmap_pde_pindex(KERNBASE) + i; mpte->phys_addr = KPTphys + (i << PAGE_SHIFT); } /* * If the kernel is running on a virtual machine, then it must assume * that MCA is enabled by the hypervisor. Moreover, the kernel must * be prepared for the hypervisor changing the vendor and family that * are reported by CPUID. Consequently, the workaround for AMD Family * 10h Erratum 383 is enabled if the processor's feature set does not * include at least one feature that is only supported by older Intel * or newer AMD processors. */ if (vm_guest == VM_GUEST_VM && (cpu_feature & CPUID_SS) == 0 && (cpu_feature2 & (CPUID2_SSSE3 | CPUID2_SSE41 | CPUID2_AESNI | CPUID2_AVX | CPUID2_XSAVE)) == 0 && (amd_feature2 & (AMDID2_XOP | AMDID2_FMA4)) == 0) workaround_erratum383 = 1; /* * Are large page mappings enabled? */ TUNABLE_INT_FETCH("vm.pmap.pg_ps_enabled", &pg_ps_enabled); if (pg_ps_enabled) { KASSERT(MAXPAGESIZES > 1 && pagesizes[1] == 0, ("pmap_init: can't assign to pagesizes[1]")); pagesizes[1] = NBPDR; } /* * Initialize the pv chunk list mutex. */ mtx_init(&pv_chunks_mutex, "pmap pv chunk list", NULL, MTX_DEF); /* * Initialize the pool of pv list locks. */ for (i = 0; i < NPV_LIST_LOCKS; i++) rw_init(&pv_list_locks[i], "pmap pv list"); /* * Calculate the size of the pv head table for superpages. */ pv_npg = howmany(vm_phys_segs[vm_phys_nsegs - 1].end, NBPDR); /* * Allocate memory for the pv head table for superpages. */ s = (vm_size_t)(pv_npg * sizeof(struct md_page)); s = round_page(s); pv_table = (struct md_page *)kmem_malloc(kernel_arena, s, M_WAITOK | M_ZERO); for (i = 0; i < pv_npg; i++) TAILQ_INIT(&pv_table[i].pv_list); pmap_initialized = 1; for (i = 0; i < PMAP_PREINIT_MAPPING_COUNT; i++) { ppim = pmap_preinit_mapping + i; if (ppim->va == 0) continue; /* Make the direct map consistent */ if (ppim->pa < dmaplimit && ppim->pa + ppim->sz < dmaplimit) { (void)pmap_change_attr(PHYS_TO_DMAP(ppim->pa), ppim->sz, ppim->mode); } if (!bootverbose) continue; printf("PPIM %u: PA=%#lx, VA=%#lx, size=%#lx, mode=%#x\n", i, ppim->pa, ppim->va, ppim->sz, ppim->mode); } mtx_init(&qframe_mtx, "qfrmlk", NULL, MTX_SPIN); error = vmem_alloc(kernel_arena, PAGE_SIZE, M_BESTFIT | M_WAITOK, (vmem_addr_t *)&qframe); if (error != 0) panic("qframe allocation failed"); } static SYSCTL_NODE(_vm_pmap, OID_AUTO, pde, CTLFLAG_RD, 0, "2MB page mapping counters"); static u_long pmap_pde_demotions; SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, demotions, CTLFLAG_RD, &pmap_pde_demotions, 0, "2MB page demotions"); static u_long pmap_pde_mappings; SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, mappings, CTLFLAG_RD, &pmap_pde_mappings, 0, "2MB page mappings"); static u_long pmap_pde_p_failures; SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, p_failures, CTLFLAG_RD, &pmap_pde_p_failures, 0, "2MB page promotion failures"); static u_long pmap_pde_promotions; SYSCTL_ULONG(_vm_pmap_pde, OID_AUTO, promotions, CTLFLAG_RD, &pmap_pde_promotions, 0, "2MB page promotions"); static SYSCTL_NODE(_vm_pmap, OID_AUTO, pdpe, CTLFLAG_RD, 0, "1GB page mapping counters"); static u_long pmap_pdpe_demotions; SYSCTL_ULONG(_vm_pmap_pdpe, OID_AUTO, demotions, CTLFLAG_RD, &pmap_pdpe_demotions, 0, "1GB page demotions"); /*************************************************** * Low level helper routines..... ***************************************************/ static pt_entry_t pmap_swap_pat(pmap_t pmap, pt_entry_t entry) { int x86_pat_bits = X86_PG_PTE_PAT | X86_PG_PDE_PAT; switch (pmap->pm_type) { case PT_X86: case PT_RVI: /* Verify that both PAT bits are not set at the same time */ KASSERT((entry & x86_pat_bits) != x86_pat_bits, ("Invalid PAT bits in entry %#lx", entry)); /* Swap the PAT bits if one of them is set */ if ((entry & x86_pat_bits) != 0) entry ^= x86_pat_bits; break; case PT_EPT: /* * Nothing to do - the memory attributes are represented * the same way for regular pages and superpages. */ break; default: panic("pmap_switch_pat_bits: bad pm_type %d", pmap->pm_type); } return (entry); } /* * Determine the appropriate bits to set in a PTE or PDE for a specified * caching mode. */ static int pmap_cache_bits(pmap_t pmap, int mode, boolean_t is_pde) { int cache_bits, pat_flag, pat_idx; if (mode < 0 || mode >= PAT_INDEX_SIZE || pat_index[mode] < 0) panic("Unknown caching mode %d\n", mode); switch (pmap->pm_type) { case PT_X86: case PT_RVI: /* The PAT bit is different for PTE's and PDE's. */ pat_flag = is_pde ? X86_PG_PDE_PAT : X86_PG_PTE_PAT; /* Map the caching mode to a PAT index. */ pat_idx = pat_index[mode]; /* Map the 3-bit index value into the PAT, PCD, and PWT bits. */ cache_bits = 0; if (pat_idx & 0x4) cache_bits |= pat_flag; if (pat_idx & 0x2) cache_bits |= PG_NC_PCD; if (pat_idx & 0x1) cache_bits |= PG_NC_PWT; break; case PT_EPT: cache_bits = EPT_PG_IGNORE_PAT | EPT_PG_MEMORY_TYPE(mode); break; default: panic("unsupported pmap type %d", pmap->pm_type); } return (cache_bits); } static int pmap_cache_mask(pmap_t pmap, boolean_t is_pde) { int mask; switch (pmap->pm_type) { case PT_X86: case PT_RVI: mask = is_pde ? X86_PG_PDE_CACHE : X86_PG_PTE_CACHE; break; case PT_EPT: mask = EPT_PG_IGNORE_PAT | EPT_PG_MEMORY_TYPE(0x7); break; default: panic("pmap_cache_mask: invalid pm_type %d", pmap->pm_type); } return (mask); } static __inline boolean_t pmap_ps_enabled(pmap_t pmap) { return (pg_ps_enabled && (pmap->pm_flags & PMAP_PDE_SUPERPAGE) != 0); } static void pmap_update_pde_store(pmap_t pmap, pd_entry_t *pde, pd_entry_t newpde) { switch (pmap->pm_type) { case PT_X86: break; case PT_RVI: case PT_EPT: /* * XXX * This is a little bogus since the generation number is * supposed to be bumped up when a region of the address * space is invalidated in the page tables. * * In this case the old PDE entry is valid but yet we want * to make sure that any mappings using the old entry are * invalidated in the TLB. * * The reason this works as expected is because we rendezvous * "all" host cpus and force any vcpu context to exit as a * side-effect. */ atomic_add_acq_long(&pmap->pm_eptgen, 1); break; default: panic("pmap_update_pde_store: bad pm_type %d", pmap->pm_type); } pde_store(pde, newpde); } /* * After changing the page size for the specified virtual address in the page * table, flush the corresponding entries from the processor's TLB. Only the * calling processor's TLB is affected. * * The calling thread must be pinned to a processor. */ static void pmap_update_pde_invalidate(pmap_t pmap, vm_offset_t va, pd_entry_t newpde) { pt_entry_t PG_G; if (pmap_type_guest(pmap)) return; KASSERT(pmap->pm_type == PT_X86, ("pmap_update_pde_invalidate: invalid type %d", pmap->pm_type)); PG_G = pmap_global_bit(pmap); if ((newpde & PG_PS) == 0) /* Demotion: flush a specific 2MB page mapping. */ invlpg(va); else if ((newpde & PG_G) == 0) /* * Promotion: flush every 4KB page mapping from the TLB * because there are too many to flush individually. */ invltlb(); else { /* * Promotion: flush every 4KB page mapping from the TLB, * including any global (PG_G) mappings. */ invltlb_glob(); } } #ifdef SMP /* * For SMP, these functions have to use the IPI mechanism for coherence. * * N.B.: Before calling any of the following TLB invalidation functions, * the calling processor must ensure that all stores updating a non- * kernel page table are globally performed. Otherwise, another * processor could cache an old, pre-update entry without being * invalidated. This can happen one of two ways: (1) The pmap becomes * active on another processor after its pm_active field is checked by * one of the following functions but before a store updating the page * table is globally performed. (2) The pmap becomes active on another * processor before its pm_active field is checked but due to * speculative loads one of the following functions stills reads the * pmap as inactive on the other processor. * * The kernel page table is exempt because its pm_active field is * immutable. The kernel page table is always active on every * processor. */ /* * Interrupt the cpus that are executing in the guest context. * This will force the vcpu to exit and the cached EPT mappings * will be invalidated by the host before the next vmresume. */ static __inline void pmap_invalidate_ept(pmap_t pmap) { int ipinum; sched_pin(); KASSERT(!CPU_ISSET(curcpu, &pmap->pm_active), ("pmap_invalidate_ept: absurd pm_active")); /* * The TLB mappings associated with a vcpu context are not * flushed each time a different vcpu is chosen to execute. * * This is in contrast with a process's vtop mappings that * are flushed from the TLB on each context switch. * * Therefore we need to do more than just a TLB shootdown on * the active cpus in 'pmap->pm_active'. To do this we keep * track of the number of invalidations performed on this pmap. * * Each vcpu keeps a cache of this counter and compares it * just before a vmresume. If the counter is out-of-date an * invept will be done to flush stale mappings from the TLB. */ atomic_add_acq_long(&pmap->pm_eptgen, 1); /* * Force the vcpu to exit and trap back into the hypervisor. */ ipinum = pmap->pm_flags & PMAP_NESTED_IPIMASK; ipi_selected(pmap->pm_active, ipinum); sched_unpin(); } void pmap_invalidate_page(pmap_t pmap, vm_offset_t va) { cpuset_t *mask; u_int cpuid, i; if (pmap_type_guest(pmap)) { pmap_invalidate_ept(pmap); return; } KASSERT(pmap->pm_type == PT_X86, ("pmap_invalidate_page: invalid type %d", pmap->pm_type)); sched_pin(); if (pmap == kernel_pmap) { invlpg(va); mask = &all_cpus; } else { cpuid = PCPU_GET(cpuid); if (pmap == PCPU_GET(curpmap)) invlpg(va); else if (pmap_pcid_enabled) pmap->pm_pcids[cpuid].pm_gen = 0; if (pmap_pcid_enabled) { CPU_FOREACH(i) { if (cpuid != i) pmap->pm_pcids[i].pm_gen = 0; } } mask = &pmap->pm_active; } smp_masked_invlpg(*mask, va); sched_unpin(); } /* 4k PTEs -- Chosen to exceed the total size of Broadwell L2 TLB */ #define PMAP_INVLPG_THRESHOLD (4 * 1024 * PAGE_SIZE) void pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) { cpuset_t *mask; vm_offset_t addr; u_int cpuid, i; if (eva - sva >= PMAP_INVLPG_THRESHOLD) { pmap_invalidate_all(pmap); return; } if (pmap_type_guest(pmap)) { pmap_invalidate_ept(pmap); return; } KASSERT(pmap->pm_type == PT_X86, ("pmap_invalidate_range: invalid type %d", pmap->pm_type)); sched_pin(); cpuid = PCPU_GET(cpuid); if (pmap == kernel_pmap) { for (addr = sva; addr < eva; addr += PAGE_SIZE) invlpg(addr); mask = &all_cpus; } else { if (pmap == PCPU_GET(curpmap)) { for (addr = sva; addr < eva; addr += PAGE_SIZE) invlpg(addr); } else if (pmap_pcid_enabled) { pmap->pm_pcids[cpuid].pm_gen = 0; } if (pmap_pcid_enabled) { CPU_FOREACH(i) { if (cpuid != i) pmap->pm_pcids[i].pm_gen = 0; } } mask = &pmap->pm_active; } smp_masked_invlpg_range(*mask, sva, eva); sched_unpin(); } void pmap_invalidate_all(pmap_t pmap) { cpuset_t *mask; struct invpcid_descr d; u_int cpuid, i; if (pmap_type_guest(pmap)) { pmap_invalidate_ept(pmap); return; } KASSERT(pmap->pm_type == PT_X86, ("pmap_invalidate_all: invalid type %d", pmap->pm_type)); sched_pin(); if (pmap == kernel_pmap) { if (pmap_pcid_enabled && invpcid_works) { bzero(&d, sizeof(d)); invpcid(&d, INVPCID_CTXGLOB); } else { invltlb_glob(); } mask = &all_cpus; } else { cpuid = PCPU_GET(cpuid); if (pmap == PCPU_GET(curpmap)) { if (pmap_pcid_enabled) { if (invpcid_works) { d.pcid = pmap->pm_pcids[cpuid].pm_pcid; d.pad = 0; d.addr = 0; invpcid(&d, INVPCID_CTX); } else { load_cr3(pmap->pm_cr3 | pmap->pm_pcids [PCPU_GET(cpuid)].pm_pcid); } } else { invltlb(); } } else if (pmap_pcid_enabled) { pmap->pm_pcids[cpuid].pm_gen = 0; } if (pmap_pcid_enabled) { CPU_FOREACH(i) { if (cpuid != i) pmap->pm_pcids[i].pm_gen = 0; } } mask = &pmap->pm_active; } smp_masked_invltlb(*mask, pmap); sched_unpin(); } void pmap_invalidate_cache(void) { sched_pin(); wbinvd(); smp_cache_flush(); sched_unpin(); } struct pde_action { cpuset_t invalidate; /* processors that invalidate their TLB */ pmap_t pmap; vm_offset_t va; pd_entry_t *pde; pd_entry_t newpde; u_int store; /* processor that updates the PDE */ }; static void pmap_update_pde_action(void *arg) { struct pde_action *act = arg; if (act->store == PCPU_GET(cpuid)) pmap_update_pde_store(act->pmap, act->pde, act->newpde); } static void pmap_update_pde_teardown(void *arg) { struct pde_action *act = arg; if (CPU_ISSET(PCPU_GET(cpuid), &act->invalidate)) pmap_update_pde_invalidate(act->pmap, act->va, act->newpde); } /* * Change the page size for the specified virtual address in a way that * prevents any possibility of the TLB ever having two entries that map the * same virtual address using different page sizes. This is the recommended * workaround for Erratum 383 on AMD Family 10h processors. It prevents a * machine check exception for a TLB state that is improperly diagnosed as a * hardware error. */ static void pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t newpde) { struct pde_action act; cpuset_t active, other_cpus; u_int cpuid; sched_pin(); cpuid = PCPU_GET(cpuid); other_cpus = all_cpus; CPU_CLR(cpuid, &other_cpus); if (pmap == kernel_pmap || pmap_type_guest(pmap)) active = all_cpus; else { active = pmap->pm_active; } if (CPU_OVERLAP(&active, &other_cpus)) { act.store = cpuid; act.invalidate = active; act.va = va; act.pmap = pmap; act.pde = pde; act.newpde = newpde; CPU_SET(cpuid, &active); smp_rendezvous_cpus(active, smp_no_rendevous_barrier, pmap_update_pde_action, pmap_update_pde_teardown, &act); } else { pmap_update_pde_store(pmap, pde, newpde); if (CPU_ISSET(cpuid, &active)) pmap_update_pde_invalidate(pmap, va, newpde); } sched_unpin(); } #else /* !SMP */ /* * Normal, non-SMP, invalidation functions. */ void pmap_invalidate_page(pmap_t pmap, vm_offset_t va) { if (pmap->pm_type == PT_RVI || pmap->pm_type == PT_EPT) { pmap->pm_eptgen++; return; } KASSERT(pmap->pm_type == PT_X86, ("pmap_invalidate_range: unknown type %d", pmap->pm_type)); if (pmap == kernel_pmap || pmap == PCPU_GET(curpmap)) invlpg(va); else if (pmap_pcid_enabled) pmap->pm_pcids[0].pm_gen = 0; } void pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) { vm_offset_t addr; if (pmap->pm_type == PT_RVI || pmap->pm_type == PT_EPT) { pmap->pm_eptgen++; return; } KASSERT(pmap->pm_type == PT_X86, ("pmap_invalidate_range: unknown type %d", pmap->pm_type)); if (pmap == kernel_pmap || pmap == PCPU_GET(curpmap)) { for (addr = sva; addr < eva; addr += PAGE_SIZE) invlpg(addr); } else if (pmap_pcid_enabled) { pmap->pm_pcids[0].pm_gen = 0; } } void pmap_invalidate_all(pmap_t pmap) { struct invpcid_descr d; if (pmap->pm_type == PT_RVI || pmap->pm_type == PT_EPT) { pmap->pm_eptgen++; return; } KASSERT(pmap->pm_type == PT_X86, ("pmap_invalidate_all: unknown type %d", pmap->pm_type)); if (pmap == kernel_pmap) { if (pmap_pcid_enabled && invpcid_works) { bzero(&d, sizeof(d)); invpcid(&d, INVPCID_CTXGLOB); } else { invltlb_glob(); } } else if (pmap == PCPU_GET(curpmap)) { if (pmap_pcid_enabled) { if (invpcid_works) { d.pcid = pmap->pm_pcids[0].pm_pcid; d.pad = 0; d.addr = 0; invpcid(&d, INVPCID_CTX); } else { load_cr3(pmap->pm_cr3 | pmap->pm_pcids[0]. pm_pcid); } } else { invltlb(); } } else if (pmap_pcid_enabled) { pmap->pm_pcids[0].pm_gen = 0; } } PMAP_INLINE void pmap_invalidate_cache(void) { wbinvd(); } static void pmap_update_pde(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, pd_entry_t newpde) { pmap_update_pde_store(pmap, pde, newpde); if (pmap == kernel_pmap || pmap == PCPU_GET(curpmap)) pmap_update_pde_invalidate(pmap, va, newpde); else pmap->pm_pcids[0].pm_gen = 0; } #endif /* !SMP */ #define PMAP_CLFLUSH_THRESHOLD (2 * 1024 * 1024) void pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva, boolean_t force) { if (force) { sva &= ~(vm_offset_t)cpu_clflush_line_size; } else { KASSERT((sva & PAGE_MASK) == 0, ("pmap_invalidate_cache_range: sva not page-aligned")); KASSERT((eva & PAGE_MASK) == 0, ("pmap_invalidate_cache_range: eva not page-aligned")); } if ((cpu_feature & CPUID_SS) != 0 && !force) ; /* If "Self Snoop" is supported and allowed, do nothing. */ else if ((cpu_stdext_feature & CPUID_STDEXT_CLFLUSHOPT) != 0 && eva - sva < PMAP_CLFLUSH_THRESHOLD) { /* * XXX: Some CPUs fault, hang, or trash the local APIC * registers if we use CLFLUSH on the local APIC * range. The local APIC is always uncached, so we * don't need to flush for that range anyway. */ if (pmap_kextract(sva) == lapic_paddr) return; /* * Otherwise, do per-cache line flush. Use the mfence * instruction to insure that previous stores are * included in the write-back. The processor * propagates flush to other processors in the cache * coherence domain. */ mfence(); for (; sva < eva; sva += cpu_clflush_line_size) clflushopt(sva); mfence(); } else if ((cpu_feature & CPUID_CLFSH) != 0 && eva - sva < PMAP_CLFLUSH_THRESHOLD) { if (pmap_kextract(sva) == lapic_paddr) return; /* * Writes are ordered by CLFLUSH on Intel CPUs. */ if (cpu_vendor_id != CPU_VENDOR_INTEL) mfence(); for (; sva < eva; sva += cpu_clflush_line_size) clflush(sva); if (cpu_vendor_id != CPU_VENDOR_INTEL) mfence(); } else { /* * No targeted cache flush methods are supported by CPU, * or the supplied range is bigger than 2MB. * Globally invalidate cache. */ pmap_invalidate_cache(); } } /* * Remove the specified set of pages from the data and instruction caches. * * In contrast to pmap_invalidate_cache_range(), this function does not * rely on the CPU's self-snoop feature, because it is intended for use * when moving pages into a different cache domain. */ void pmap_invalidate_cache_pages(vm_page_t *pages, int count) { vm_offset_t daddr, eva; int i; bool useclflushopt; useclflushopt = (cpu_stdext_feature & CPUID_STDEXT_CLFLUSHOPT) != 0; if (count >= PMAP_CLFLUSH_THRESHOLD / PAGE_SIZE || ((cpu_feature & CPUID_CLFSH) == 0 && !useclflushopt)) pmap_invalidate_cache(); else { if (useclflushopt || cpu_vendor_id != CPU_VENDOR_INTEL) mfence(); for (i = 0; i < count; i++) { daddr = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pages[i])); eva = daddr + PAGE_SIZE; for (; daddr < eva; daddr += cpu_clflush_line_size) { if (useclflushopt) clflushopt(daddr); else clflush(daddr); } } if (useclflushopt || cpu_vendor_id != CPU_VENDOR_INTEL) mfence(); } } /* * Routine: pmap_extract * Function: * Extract the physical page address associated * with the given map/virtual_address pair. */ vm_paddr_t pmap_extract(pmap_t pmap, vm_offset_t va) { pdp_entry_t *pdpe; pd_entry_t *pde; pt_entry_t *pte, PG_V; vm_paddr_t pa; pa = 0; PG_V = pmap_valid_bit(pmap); PMAP_LOCK(pmap); pdpe = pmap_pdpe(pmap, va); if (pdpe != NULL && (*pdpe & PG_V) != 0) { if ((*pdpe & PG_PS) != 0) pa = (*pdpe & PG_PS_FRAME) | (va & PDPMASK); else { pde = pmap_pdpe_to_pde(pdpe, va); if ((*pde & PG_V) != 0) { if ((*pde & PG_PS) != 0) { pa = (*pde & PG_PS_FRAME) | (va & PDRMASK); } else { pte = pmap_pde_to_pte(pde, va); pa = (*pte & PG_FRAME) | (va & PAGE_MASK); } } } } PMAP_UNLOCK(pmap); return (pa); } /* * Routine: pmap_extract_and_hold * Function: * Atomically extract and hold the physical page * with the given pmap and virtual address pair * if that mapping permits the given protection. */ vm_page_t pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot) { pd_entry_t pde, *pdep; pt_entry_t pte, PG_RW, PG_V; vm_paddr_t pa; vm_page_t m; pa = 0; m = NULL; PG_RW = pmap_rw_bit(pmap); PG_V = pmap_valid_bit(pmap); PMAP_LOCK(pmap); retry: pdep = pmap_pde(pmap, va); if (pdep != NULL && (pde = *pdep)) { if (pde & PG_PS) { if ((pde & PG_RW) || (prot & VM_PROT_WRITE) == 0) { if (vm_page_pa_tryrelock(pmap, (pde & PG_PS_FRAME) | (va & PDRMASK), &pa)) goto retry; m = PHYS_TO_VM_PAGE((pde & PG_PS_FRAME) | (va & PDRMASK)); vm_page_hold(m); } } else { pte = *pmap_pde_to_pte(pdep, va); if ((pte & PG_V) && ((pte & PG_RW) || (prot & VM_PROT_WRITE) == 0)) { if (vm_page_pa_tryrelock(pmap, pte & PG_FRAME, &pa)) goto retry; m = PHYS_TO_VM_PAGE(pte & PG_FRAME); vm_page_hold(m); } } } PA_UNLOCK_COND(pa); PMAP_UNLOCK(pmap); return (m); } vm_paddr_t pmap_kextract(vm_offset_t va) { pd_entry_t pde; vm_paddr_t pa; if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) { pa = DMAP_TO_PHYS(va); } else { pde = *vtopde(va); if (pde & PG_PS) { pa = (pde & PG_PS_FRAME) | (va & PDRMASK); } else { /* * Beware of a concurrent promotion that changes the * PDE at this point! For example, vtopte() must not * be used to access the PTE because it would use the * new PDE. It is, however, safe to use the old PDE * because the page table page is preserved by the * promotion. */ pa = *pmap_pde_to_pte(&pde, va); pa = (pa & PG_FRAME) | (va & PAGE_MASK); } } return (pa); } /*************************************************** * Low level mapping routines..... ***************************************************/ /* * Add a wired page to the kva. * Note: not SMP coherent. */ PMAP_INLINE void pmap_kenter(vm_offset_t va, vm_paddr_t pa) { pt_entry_t *pte; pte = vtopte(va); pte_store(pte, pa | X86_PG_RW | X86_PG_V | X86_PG_G); } static __inline void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode) { pt_entry_t *pte; int cache_bits; pte = vtopte(va); cache_bits = pmap_cache_bits(kernel_pmap, mode, 0); pte_store(pte, pa | X86_PG_RW | X86_PG_V | X86_PG_G | cache_bits); } /* * Remove a page from the kernel pagetables. * Note: not SMP coherent. */ PMAP_INLINE void pmap_kremove(vm_offset_t va) { pt_entry_t *pte; pte = vtopte(va); pte_clear(pte); } /* * Used to map a range of physical addresses into kernel * virtual address space. * * The value passed in '*virt' is a suggested virtual address for * the mapping. Architectures which can support a direct-mapped * physical to virtual region can return the appropriate address * within that region, leaving '*virt' unchanged. Other * architectures should map the pages starting at '*virt' and * update '*virt' with the first usable address after the mapped * region. */ vm_offset_t pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot) { return PHYS_TO_DMAP(start); } /* * Add a list of wired pages to the kva * this routine is only used for temporary * kernel mappings that do not need to have * page modification or references recorded. * Note that old mappings are simply written * over. The page *must* be wired. * Note: SMP coherent. Uses a ranged shootdown IPI. */ void pmap_qenter(vm_offset_t sva, vm_page_t *ma, int count) { pt_entry_t *endpte, oldpte, pa, *pte; vm_page_t m; int cache_bits; oldpte = 0; pte = vtopte(sva); endpte = pte + count; while (pte < endpte) { m = *ma++; cache_bits = pmap_cache_bits(kernel_pmap, m->md.pat_mode, 0); pa = VM_PAGE_TO_PHYS(m) | cache_bits; if ((*pte & (PG_FRAME | X86_PG_PTE_CACHE)) != pa) { oldpte |= *pte; pte_store(pte, pa | X86_PG_G | X86_PG_RW | X86_PG_V); } pte++; } if (__predict_false((oldpte & X86_PG_V) != 0)) pmap_invalidate_range(kernel_pmap, sva, sva + count * PAGE_SIZE); } /* * This routine tears out page mappings from the * kernel -- it is meant only for temporary mappings. * Note: SMP coherent. Uses a ranged shootdown IPI. */ void pmap_qremove(vm_offset_t sva, int count) { vm_offset_t va; va = sva; while (count-- > 0) { KASSERT(va >= VM_MIN_KERNEL_ADDRESS, ("usermode va %lx", va)); pmap_kremove(va); va += PAGE_SIZE; } pmap_invalidate_range(kernel_pmap, sva, va); } /*************************************************** * Page table page management routines..... ***************************************************/ static __inline void pmap_free_zero_pages(struct spglist *free) { vm_page_t m; while ((m = SLIST_FIRST(free)) != NULL) { SLIST_REMOVE_HEAD(free, plinks.s.ss); /* Preserve the page's PG_ZERO setting. */ vm_page_free_toq(m); } } /* * Schedule the specified unused page table page to be freed. Specifically, * add the page to the specified list of pages that will be released to the * physical memory manager after the TLB has been updated. */ static __inline void pmap_add_delayed_free_list(vm_page_t m, struct spglist *free, boolean_t set_PG_ZERO) { if (set_PG_ZERO) m->flags |= PG_ZERO; else m->flags &= ~PG_ZERO; SLIST_INSERT_HEAD(free, m, plinks.s.ss); } /* * Inserts the specified page table page into the specified pmap's collection * of idle page table pages. Each of a pmap's page table pages is responsible * for mapping a distinct range of virtual addresses. The pmap's collection is * ordered by this virtual address range. */ static __inline int pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte) { PMAP_LOCK_ASSERT(pmap, MA_OWNED); return (vm_radix_insert(&pmap->pm_root, mpte)); } /* * Looks for a page table page mapping the specified virtual address in the * specified pmap's collection of idle page table pages. Returns NULL if there * is no page table page corresponding to the specified virtual address. */ static __inline vm_page_t pmap_lookup_pt_page(pmap_t pmap, vm_offset_t va) { PMAP_LOCK_ASSERT(pmap, MA_OWNED); return (vm_radix_lookup(&pmap->pm_root, pmap_pde_pindex(va))); } /* * Removes the specified page table page from the specified pmap's collection * of idle page table pages. The specified page table page must be a member of * the pmap's collection. */ static __inline void pmap_remove_pt_page(pmap_t pmap, vm_page_t mpte) { PMAP_LOCK_ASSERT(pmap, MA_OWNED); vm_radix_remove(&pmap->pm_root, mpte->pindex); } /* * Decrements a page table page's wire count, which is used to record the * number of valid page table entries within the page. If the wire count * drops to zero, then the page table page is unmapped. Returns TRUE if the * page table page was unmapped and FALSE otherwise. */ static inline boolean_t pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) { --m->wire_count; if (m->wire_count == 0) { _pmap_unwire_ptp(pmap, va, m, free); return (TRUE); } else return (FALSE); } static void _pmap_unwire_ptp(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) { PMAP_LOCK_ASSERT(pmap, MA_OWNED); /* * unmap the page table page */ if (m->pindex >= (NUPDE + NUPDPE)) { /* PDP page */ pml4_entry_t *pml4; pml4 = pmap_pml4e(pmap, va); *pml4 = 0; } else if (m->pindex >= NUPDE) { /* PD page */ pdp_entry_t *pdp; pdp = pmap_pdpe(pmap, va); *pdp = 0; } else { /* PTE page */ pd_entry_t *pd; pd = pmap_pde(pmap, va); *pd = 0; } pmap_resident_count_dec(pmap, 1); if (m->pindex < NUPDE) { /* We just released a PT, unhold the matching PD */ vm_page_t pdpg; pdpg = PHYS_TO_VM_PAGE(*pmap_pdpe(pmap, va) & PG_FRAME); pmap_unwire_ptp(pmap, va, pdpg, free); } if (m->pindex >= NUPDE && m->pindex < (NUPDE + NUPDPE)) { /* We just released a PD, unhold the matching PDP */ vm_page_t pdppg; pdppg = PHYS_TO_VM_PAGE(*pmap_pml4e(pmap, va) & PG_FRAME); pmap_unwire_ptp(pmap, va, pdppg, free); } /* * This is a release store so that the ordinary store unmapping * the page table page is globally performed before TLB shoot- * down is begun. */ atomic_subtract_rel_int(&vm_cnt.v_wire_count, 1); /* * Put page on a list so that it is released after * *ALL* TLB shootdown is done */ pmap_add_delayed_free_list(m, free, TRUE); } /* * After removing a page table entry, this routine is used to * conditionally free the page, and manage the hold/wire counts. */ static int pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pd_entry_t ptepde, struct spglist *free) { vm_page_t mpte; if (va >= VM_MAXUSER_ADDRESS) return (0); KASSERT(ptepde != 0, ("pmap_unuse_pt: ptepde != 0")); mpte = PHYS_TO_VM_PAGE(ptepde & PG_FRAME); return (pmap_unwire_ptp(pmap, va, mpte, free)); } void pmap_pinit0(pmap_t pmap) { int i; PMAP_LOCK_INIT(pmap); pmap->pm_pml4 = (pml4_entry_t *)PHYS_TO_DMAP(KPML4phys); pmap->pm_cr3 = KPML4phys; pmap->pm_root.rt_root = 0; CPU_ZERO(&pmap->pm_active); TAILQ_INIT(&pmap->pm_pvchunk); bzero(&pmap->pm_stats, sizeof pmap->pm_stats); pmap->pm_flags = pmap_flags; CPU_FOREACH(i) { pmap->pm_pcids[i].pm_pcid = PMAP_PCID_NONE; pmap->pm_pcids[i].pm_gen = 0; } PCPU_SET(curpmap, kernel_pmap); pmap_activate(curthread); CPU_FILL(&kernel_pmap->pm_active); } /* * Initialize a preallocated and zeroed pmap structure, * such as one in a vmspace structure. */ int pmap_pinit_type(pmap_t pmap, enum pmap_type pm_type, int flags) { vm_page_t pml4pg; vm_paddr_t pml4phys; int i; /* * allocate the page directory page */ while ((pml4pg = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) VM_WAIT; pml4phys = VM_PAGE_TO_PHYS(pml4pg); pmap->pm_pml4 = (pml4_entry_t *)PHYS_TO_DMAP(pml4phys); CPU_FOREACH(i) { pmap->pm_pcids[i].pm_pcid = PMAP_PCID_NONE; pmap->pm_pcids[i].pm_gen = 0; } pmap->pm_cr3 = ~0; /* initialize to an invalid value */ if ((pml4pg->flags & PG_ZERO) == 0) pagezero(pmap->pm_pml4); /* * Do not install the host kernel mappings in the nested page * tables. These mappings are meaningless in the guest physical * address space. */ if ((pmap->pm_type = pm_type) == PT_X86) { pmap->pm_cr3 = pml4phys; /* Wire in kernel global address entries. */ for (i = 0; i < NKPML4E; i++) { pmap->pm_pml4[KPML4BASE + i] = (KPDPphys + ptoa(i)) | X86_PG_RW | X86_PG_V | PG_U; } for (i = 0; i < ndmpdpphys; i++) { pmap->pm_pml4[DMPML4I + i] = (DMPDPphys + ptoa(i)) | X86_PG_RW | X86_PG_V | PG_U; } /* install self-referential address mapping entry(s) */ pmap->pm_pml4[PML4PML4I] = VM_PAGE_TO_PHYS(pml4pg) | X86_PG_V | X86_PG_RW | X86_PG_A | X86_PG_M; } pmap->pm_root.rt_root = 0; CPU_ZERO(&pmap->pm_active); TAILQ_INIT(&pmap->pm_pvchunk); bzero(&pmap->pm_stats, sizeof pmap->pm_stats); pmap->pm_flags = flags; pmap->pm_eptgen = 0; return (1); } int pmap_pinit(pmap_t pmap) { return (pmap_pinit_type(pmap, PT_X86, pmap_flags)); } /* * This routine is called if the desired page table page does not exist. * * If page table page allocation fails, this routine may sleep before * returning NULL. It sleeps only if a lock pointer was given. * * Note: If a page allocation fails at page table level two or three, * one or two pages may be held during the wait, only to be released * afterwards. This conservative approach is easily argued to avoid * race conditions. */ static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp) { vm_page_t m, pdppg, pdpg; pt_entry_t PG_A, PG_M, PG_RW, PG_V; PMAP_LOCK_ASSERT(pmap, MA_OWNED); PG_A = pmap_accessed_bit(pmap); PG_M = pmap_modified_bit(pmap); PG_V = pmap_valid_bit(pmap); PG_RW = pmap_rw_bit(pmap); /* * Allocate a page table page. */ if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) { if (lockp != NULL) { RELEASE_PV_LIST_LOCK(lockp); PMAP_UNLOCK(pmap); rw_runlock(&pvh_global_lock); VM_WAIT; rw_rlock(&pvh_global_lock); PMAP_LOCK(pmap); } /* * Indicate the need to retry. While waiting, the page table * page may have been allocated. */ return (NULL); } if ((m->flags & PG_ZERO) == 0) pmap_zero_page(m); /* * Map the pagetable page into the process address space, if * it isn't already there. */ if (ptepindex >= (NUPDE + NUPDPE)) { pml4_entry_t *pml4; vm_pindex_t pml4index; /* Wire up a new PDPE page */ pml4index = ptepindex - (NUPDE + NUPDPE); pml4 = &pmap->pm_pml4[pml4index]; *pml4 = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M; } else if (ptepindex >= NUPDE) { vm_pindex_t pml4index; vm_pindex_t pdpindex; pml4_entry_t *pml4; pdp_entry_t *pdp; /* Wire up a new PDE page */ pdpindex = ptepindex - NUPDE; pml4index = pdpindex >> NPML4EPGSHIFT; pml4 = &pmap->pm_pml4[pml4index]; if ((*pml4 & PG_V) == 0) { /* Have to allocate a new pdp, recurse */ if (_pmap_allocpte(pmap, NUPDE + NUPDPE + pml4index, lockp) == NULL) { --m->wire_count; atomic_subtract_int(&vm_cnt.v_wire_count, 1); vm_page_free_zero(m); return (NULL); } } else { /* Add reference to pdp page */ pdppg = PHYS_TO_VM_PAGE(*pml4 & PG_FRAME); pdppg->wire_count++; } pdp = (pdp_entry_t *)PHYS_TO_DMAP(*pml4 & PG_FRAME); /* Now find the pdp page */ pdp = &pdp[pdpindex & ((1ul << NPDPEPGSHIFT) - 1)]; *pdp = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M; } else { vm_pindex_t pml4index; vm_pindex_t pdpindex; pml4_entry_t *pml4; pdp_entry_t *pdp; pd_entry_t *pd; /* Wire up a new PTE page */ pdpindex = ptepindex >> NPDPEPGSHIFT; pml4index = pdpindex >> NPML4EPGSHIFT; /* First, find the pdp and check that its valid. */ pml4 = &pmap->pm_pml4[pml4index]; if ((*pml4 & PG_V) == 0) { /* Have to allocate a new pd, recurse */ if (_pmap_allocpte(pmap, NUPDE + pdpindex, lockp) == NULL) { --m->wire_count; atomic_subtract_int(&vm_cnt.v_wire_count, 1); vm_page_free_zero(m); return (NULL); } pdp = (pdp_entry_t *)PHYS_TO_DMAP(*pml4 & PG_FRAME); pdp = &pdp[pdpindex & ((1ul << NPDPEPGSHIFT) - 1)]; } else { pdp = (pdp_entry_t *)PHYS_TO_DMAP(*pml4 & PG_FRAME); pdp = &pdp[pdpindex & ((1ul << NPDPEPGSHIFT) - 1)]; if ((*pdp & PG_V) == 0) { /* Have to allocate a new pd, recurse */ if (_pmap_allocpte(pmap, NUPDE + pdpindex, lockp) == NULL) { --m->wire_count; atomic_subtract_int(&vm_cnt.v_wire_count, 1); vm_page_free_zero(m); return (NULL); } } else { /* Add reference to the pd page */ pdpg = PHYS_TO_VM_PAGE(*pdp & PG_FRAME); pdpg->wire_count++; } } pd = (pd_entry_t *)PHYS_TO_DMAP(*pdp & PG_FRAME); /* Now we know where the page directory page is */ pd = &pd[ptepindex & ((1ul << NPDEPGSHIFT) - 1)]; *pd = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V | PG_A | PG_M; } pmap_resident_count_inc(pmap, 1); return (m); } static vm_page_t pmap_allocpde(pmap_t pmap, vm_offset_t va, struct rwlock **lockp) { vm_pindex_t pdpindex, ptepindex; pdp_entry_t *pdpe, PG_V; vm_page_t pdpg; PG_V = pmap_valid_bit(pmap); retry: pdpe = pmap_pdpe(pmap, va); if (pdpe != NULL && (*pdpe & PG_V) != 0) { /* Add a reference to the pd page. */ pdpg = PHYS_TO_VM_PAGE(*pdpe & PG_FRAME); pdpg->wire_count++; } else { /* Allocate a pd page. */ ptepindex = pmap_pde_pindex(va); pdpindex = ptepindex >> NPDPEPGSHIFT; pdpg = _pmap_allocpte(pmap, NUPDE + pdpindex, lockp); if (pdpg == NULL && lockp != NULL) goto retry; } return (pdpg); } static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, struct rwlock **lockp) { vm_pindex_t ptepindex; pd_entry_t *pd, PG_V; vm_page_t m; PG_V = pmap_valid_bit(pmap); /* * Calculate pagetable page index */ ptepindex = pmap_pde_pindex(va); retry: /* * Get the page directory entry */ pd = pmap_pde(pmap, va); /* * This supports switching from a 2MB page to a * normal 4K page. */ if (pd != NULL && (*pd & (PG_PS | PG_V)) == (PG_PS | PG_V)) { if (!pmap_demote_pde_locked(pmap, pd, va, lockp)) { /* * Invalidation of the 2MB page mapping may have caused * the deallocation of the underlying PD page. */ pd = NULL; } } /* * If the page table page is mapped, we just increment the * hold count, and activate it. */ if (pd != NULL && (*pd & PG_V) != 0) { m = PHYS_TO_VM_PAGE(*pd & PG_FRAME); m->wire_count++; } else { /* * Here if the pte page isn't mapped, or if it has been * deallocated. */ m = _pmap_allocpte(pmap, ptepindex, lockp); if (m == NULL && lockp != NULL) goto retry; } return (m); } /*************************************************** * Pmap allocation/deallocation routines. ***************************************************/ /* * Release any resources held by the given physical map. * Called when a pmap initialized by pmap_pinit is being released. * Should only be called if the map contains no valid mappings. */ void pmap_release(pmap_t pmap) { vm_page_t m; int i; KASSERT(pmap->pm_stats.resident_count == 0, ("pmap_release: pmap resident count %ld != 0", pmap->pm_stats.resident_count)); KASSERT(vm_radix_is_empty(&pmap->pm_root), ("pmap_release: pmap has reserved page table page(s)")); KASSERT(CPU_EMPTY(&pmap->pm_active), ("releasing active pmap %p", pmap)); m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pmap->pm_pml4)); for (i = 0; i < NKPML4E; i++) /* KVA */ pmap->pm_pml4[KPML4BASE + i] = 0; for (i = 0; i < ndmpdpphys; i++)/* Direct Map */ pmap->pm_pml4[DMPML4I + i] = 0; pmap->pm_pml4[PML4PML4I] = 0; /* Recursive Mapping */ m->wire_count--; atomic_subtract_int(&vm_cnt.v_wire_count, 1); vm_page_free_zero(m); } static int kvm_size(SYSCTL_HANDLER_ARGS) { unsigned long ksize = VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS; return sysctl_handle_long(oidp, &ksize, 0, req); } SYSCTL_PROC(_vm, OID_AUTO, kvm_size, CTLTYPE_LONG|CTLFLAG_RD, 0, 0, kvm_size, "LU", "Size of KVM"); static int kvm_free(SYSCTL_HANDLER_ARGS) { unsigned long kfree = VM_MAX_KERNEL_ADDRESS - kernel_vm_end; return sysctl_handle_long(oidp, &kfree, 0, req); } SYSCTL_PROC(_vm, OID_AUTO, kvm_free, CTLTYPE_LONG|CTLFLAG_RD, 0, 0, kvm_free, "LU", "Amount of KVM free"); /* * grow the number of kernel page table entries, if needed */ void pmap_growkernel(vm_offset_t addr) { vm_paddr_t paddr; vm_page_t nkpg; pd_entry_t *pde, newpdir; pdp_entry_t *pdpe; mtx_assert(&kernel_map->system_mtx, MA_OWNED); /* * Return if "addr" is within the range of kernel page table pages * that were preallocated during pmap bootstrap. Moreover, leave * "kernel_vm_end" and the kernel page table as they were. * * The correctness of this action is based on the following * argument: vm_map_insert() allocates contiguous ranges of the * kernel virtual address space. It calls this function if a range * ends after "kernel_vm_end". If the kernel is mapped between * "kernel_vm_end" and "addr", then the range cannot begin at * "kernel_vm_end". In fact, its beginning address cannot be less * than the kernel. Thus, there is no immediate need to allocate * any new kernel page table pages between "kernel_vm_end" and * "KERNBASE". */ if (KERNBASE < addr && addr <= KERNBASE + nkpt * NBPDR) return; addr = roundup2(addr, NBPDR); if (addr - 1 >= kernel_map->max_offset) addr = kernel_map->max_offset; while (kernel_vm_end < addr) { pdpe = pmap_pdpe(kernel_pmap, kernel_vm_end); if ((*pdpe & X86_PG_V) == 0) { /* We need a new PDP entry */ nkpg = vm_page_alloc(NULL, kernel_vm_end >> PDPSHIFT, VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO); if (nkpg == NULL) panic("pmap_growkernel: no memory to grow kernel"); if ((nkpg->flags & PG_ZERO) == 0) pmap_zero_page(nkpg); paddr = VM_PAGE_TO_PHYS(nkpg); *pdpe = (pdp_entry_t)(paddr | X86_PG_V | X86_PG_RW | X86_PG_A | X86_PG_M); continue; /* try again */ } pde = pmap_pdpe_to_pde(pdpe, kernel_vm_end); if ((*pde & X86_PG_V) != 0) { kernel_vm_end = (kernel_vm_end + NBPDR) & ~PDRMASK; if (kernel_vm_end - 1 >= kernel_map->max_offset) { kernel_vm_end = kernel_map->max_offset; break; } continue; } nkpg = vm_page_alloc(NULL, pmap_pde_pindex(kernel_vm_end), VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO); if (nkpg == NULL) panic("pmap_growkernel: no memory to grow kernel"); if ((nkpg->flags & PG_ZERO) == 0) pmap_zero_page(nkpg); paddr = VM_PAGE_TO_PHYS(nkpg); newpdir = paddr | X86_PG_V | X86_PG_RW | X86_PG_A | X86_PG_M; pde_store(pde, newpdir); kernel_vm_end = (kernel_vm_end + NBPDR) & ~PDRMASK; if (kernel_vm_end - 1 >= kernel_map->max_offset) { kernel_vm_end = kernel_map->max_offset; break; } } } /*************************************************** * page management routines. ***************************************************/ CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE); CTASSERT(_NPCM == 3); CTASSERT(_NPCPV == 168); static __inline struct pv_chunk * pv_to_chunk(pv_entry_t pv) { return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK)); } #define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap) #define PC_FREE0 0xfffffffffffffffful #define PC_FREE1 0xfffffffffffffffful #define PC_FREE2 0x000000fffffffffful static const uint64_t pc_freemask[_NPCM] = { PC_FREE0, PC_FREE1, PC_FREE2 }; #ifdef PV_STATS static int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail; SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_count, CTLFLAG_RD, &pc_chunk_count, 0, "Current number of pv entry chunks"); SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_allocs, CTLFLAG_RD, &pc_chunk_allocs, 0, "Current number of pv entry chunks allocated"); SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_frees, CTLFLAG_RD, &pc_chunk_frees, 0, "Current number of pv entry chunks frees"); SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_tryfail, CTLFLAG_RD, &pc_chunk_tryfail, 0, "Number of times tried to get a chunk page but failed."); static long pv_entry_frees, pv_entry_allocs, pv_entry_count; static int pv_entry_spare; SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_frees, CTLFLAG_RD, &pv_entry_frees, 0, "Current number of pv entry frees"); SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_allocs, CTLFLAG_RD, &pv_entry_allocs, 0, "Current number of pv entry allocs"); SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_count, CTLFLAG_RD, &pv_entry_count, 0, "Current number of pv entries"); SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_spare, CTLFLAG_RD, &pv_entry_spare, 0, "Current number of spare pv entries"); #endif /* * We are in a serious low memory condition. Resort to * drastic measures to free some pages so we can allocate * another pv entry chunk. * * Returns NULL if PV entries were reclaimed from the specified pmap. * * We do not, however, unmap 2mpages because subsequent accesses will * allocate per-page pv entries until repromotion occurs, thereby * exacerbating the shortage of free pv entries. */ static vm_page_t reclaim_pv_chunk(pmap_t locked_pmap, struct rwlock **lockp) { struct pch new_tail; struct pv_chunk *pc; struct md_page *pvh; pd_entry_t *pde; pmap_t pmap; pt_entry_t *pte, tpte; pt_entry_t PG_G, PG_A, PG_M, PG_RW; pv_entry_t pv; vm_offset_t va; vm_page_t m, m_pc; struct spglist free; uint64_t inuse; int bit, field, freed; rw_assert(&pvh_global_lock, RA_LOCKED); PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED); KASSERT(lockp != NULL, ("reclaim_pv_chunk: lockp is NULL")); pmap = NULL; m_pc = NULL; PG_G = PG_A = PG_M = PG_RW = 0; SLIST_INIT(&free); TAILQ_INIT(&new_tail); mtx_lock(&pv_chunks_mutex); while ((pc = TAILQ_FIRST(&pv_chunks)) != NULL && SLIST_EMPTY(&free)) { TAILQ_REMOVE(&pv_chunks, pc, pc_lru); mtx_unlock(&pv_chunks_mutex); if (pmap != pc->pc_pmap) { if (pmap != NULL) { pmap_invalidate_all(pmap); if (pmap != locked_pmap) PMAP_UNLOCK(pmap); } pmap = pc->pc_pmap; /* Avoid deadlock and lock recursion. */ if (pmap > locked_pmap) { RELEASE_PV_LIST_LOCK(lockp); PMAP_LOCK(pmap); } else if (pmap != locked_pmap && !PMAP_TRYLOCK(pmap)) { pmap = NULL; TAILQ_INSERT_TAIL(&new_tail, pc, pc_lru); mtx_lock(&pv_chunks_mutex); continue; } PG_G = pmap_global_bit(pmap); PG_A = pmap_accessed_bit(pmap); PG_M = pmap_modified_bit(pmap); PG_RW = pmap_rw_bit(pmap); } /* * Destroy every non-wired, 4 KB page mapping in the chunk. */ freed = 0; for (field = 0; field < _NPCM; field++) { for (inuse = ~pc->pc_map[field] & pc_freemask[field]; inuse != 0; inuse &= ~(1UL << bit)) { bit = bsfq(inuse); pv = &pc->pc_pventry[field * 64 + bit]; va = pv->pv_va; pde = pmap_pde(pmap, va); if ((*pde & PG_PS) != 0) continue; pte = pmap_pde_to_pte(pde, va); if ((*pte & PG_W) != 0) continue; tpte = pte_load_clear(pte); if ((tpte & PG_G) != 0) pmap_invalidate_page(pmap, va); m = PHYS_TO_VM_PAGE(tpte & PG_FRAME); if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) vm_page_dirty(m); if ((tpte & PG_A) != 0) vm_page_aflag_set(m, PGA_REFERENCED); CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); m->md.pv_gen++; if (TAILQ_EMPTY(&m->md.pv_list) && (m->flags & PG_FICTITIOUS) == 0) { pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); if (TAILQ_EMPTY(&pvh->pv_list)) { vm_page_aflag_clear(m, PGA_WRITEABLE); } } pc->pc_map[field] |= 1UL << bit; pmap_unuse_pt(pmap, va, *pde, &free); freed++; } } if (freed == 0) { TAILQ_INSERT_TAIL(&new_tail, pc, pc_lru); mtx_lock(&pv_chunks_mutex); continue; } /* Every freed mapping is for a 4 KB page. */ pmap_resident_count_dec(pmap, freed); PV_STAT(atomic_add_long(&pv_entry_frees, freed)); PV_STAT(atomic_add_int(&pv_entry_spare, freed)); PV_STAT(atomic_subtract_long(&pv_entry_count, freed)); TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); if (pc->pc_map[0] == PC_FREE0 && pc->pc_map[1] == PC_FREE1 && pc->pc_map[2] == PC_FREE2) { PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV)); PV_STAT(atomic_subtract_int(&pc_chunk_count, 1)); PV_STAT(atomic_add_int(&pc_chunk_frees, 1)); /* Entire chunk is free; return it. */ m_pc = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc)); dump_drop_page(m_pc->phys_addr); mtx_lock(&pv_chunks_mutex); break; } TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); TAILQ_INSERT_TAIL(&new_tail, pc, pc_lru); mtx_lock(&pv_chunks_mutex); /* One freed pv entry in locked_pmap is sufficient. */ if (pmap == locked_pmap) break; } TAILQ_CONCAT(&pv_chunks, &new_tail, pc_lru); mtx_unlock(&pv_chunks_mutex); if (pmap != NULL) { pmap_invalidate_all(pmap); if (pmap != locked_pmap) PMAP_UNLOCK(pmap); } if (m_pc == NULL && !SLIST_EMPTY(&free)) { m_pc = SLIST_FIRST(&free); SLIST_REMOVE_HEAD(&free, plinks.s.ss); /* Recycle a freed page table page. */ m_pc->wire_count = 1; atomic_add_int(&vm_cnt.v_wire_count, 1); } pmap_free_zero_pages(&free); return (m_pc); } /* * free the pv_entry back to the free list */ static void free_pv_entry(pmap_t pmap, pv_entry_t pv) { struct pv_chunk *pc; int idx, field, bit; rw_assert(&pvh_global_lock, RA_LOCKED); PMAP_LOCK_ASSERT(pmap, MA_OWNED); PV_STAT(atomic_add_long(&pv_entry_frees, 1)); PV_STAT(atomic_add_int(&pv_entry_spare, 1)); PV_STAT(atomic_subtract_long(&pv_entry_count, 1)); pc = pv_to_chunk(pv); idx = pv - &pc->pc_pventry[0]; field = idx / 64; bit = idx % 64; pc->pc_map[field] |= 1ul << bit; if (pc->pc_map[0] != PC_FREE0 || pc->pc_map[1] != PC_FREE1 || pc->pc_map[2] != PC_FREE2) { /* 98% of the time, pc is already at the head of the list. */ if (__predict_false(pc != TAILQ_FIRST(&pmap->pm_pvchunk))) { TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); } return; } TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); free_pv_chunk(pc); } static void free_pv_chunk(struct pv_chunk *pc) { vm_page_t m; mtx_lock(&pv_chunks_mutex); TAILQ_REMOVE(&pv_chunks, pc, pc_lru); mtx_unlock(&pv_chunks_mutex); PV_STAT(atomic_subtract_int(&pv_entry_spare, _NPCPV)); PV_STAT(atomic_subtract_int(&pc_chunk_count, 1)); PV_STAT(atomic_add_int(&pc_chunk_frees, 1)); /* entire chunk is free, return it */ m = PHYS_TO_VM_PAGE(DMAP_TO_PHYS((vm_offset_t)pc)); dump_drop_page(m->phys_addr); vm_page_unwire(m, PQ_NONE); vm_page_free(m); } /* * Returns a new PV entry, allocating a new PV chunk from the system when * needed. If this PV chunk allocation fails and a PV list lock pointer was * given, a PV chunk is reclaimed from an arbitrary pmap. Otherwise, NULL is * returned. * * The given PV list lock may be released. */ static pv_entry_t get_pv_entry(pmap_t pmap, struct rwlock **lockp) { int bit, field; pv_entry_t pv; struct pv_chunk *pc; vm_page_t m; rw_assert(&pvh_global_lock, RA_LOCKED); PMAP_LOCK_ASSERT(pmap, MA_OWNED); PV_STAT(atomic_add_long(&pv_entry_allocs, 1)); retry: pc = TAILQ_FIRST(&pmap->pm_pvchunk); if (pc != NULL) { for (field = 0; field < _NPCM; field++) { if (pc->pc_map[field]) { bit = bsfq(pc->pc_map[field]); break; } } if (field < _NPCM) { pv = &pc->pc_pventry[field * 64 + bit]; pc->pc_map[field] &= ~(1ul << bit); /* If this was the last item, move it to tail */ if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0 && pc->pc_map[2] == 0) { TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); } PV_STAT(atomic_add_long(&pv_entry_count, 1)); PV_STAT(atomic_subtract_int(&pv_entry_spare, 1)); return (pv); } } /* No free items, allocate another chunk */ m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED); if (m == NULL) { if (lockp == NULL) { PV_STAT(pc_chunk_tryfail++); return (NULL); } m = reclaim_pv_chunk(pmap, lockp); if (m == NULL) goto retry; } PV_STAT(atomic_add_int(&pc_chunk_count, 1)); PV_STAT(atomic_add_int(&pc_chunk_allocs, 1)); dump_add_page(m->phys_addr); pc = (void *)PHYS_TO_DMAP(m->phys_addr); pc->pc_pmap = pmap; pc->pc_map[0] = PC_FREE0 & ~1ul; /* preallocated bit 0 */ pc->pc_map[1] = PC_FREE1; pc->pc_map[2] = PC_FREE2; mtx_lock(&pv_chunks_mutex); TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru); mtx_unlock(&pv_chunks_mutex); pv = &pc->pc_pventry[0]; TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); PV_STAT(atomic_add_long(&pv_entry_count, 1)); PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV - 1)); return (pv); } /* * Returns the number of one bits within the given PV chunk map element. * * The erratas for Intel processors state that "POPCNT Instruction May * Take Longer to Execute Than Expected". It is believed that the * issue is the spurious dependency on the destination register. * Provide a hint to the register rename logic that the destination * value is overwritten, by clearing it, as suggested in the * optimization manual. It should be cheap for unaffected processors * as well. * * Reference numbers for erratas are * 4th Gen Core: HSD146 * 5th Gen Core: BDM85 */ static int popcnt_pc_map_elem_pq(uint64_t elem) { u_long result; __asm __volatile("xorl %k0,%k0;popcntq %1,%0" : "=&r" (result) : "rm" (elem)); return (result); } /* * Ensure that the number of spare PV entries in the specified pmap meets or * exceeds the given count, "needed". * * The given PV list lock may be released. */ static void reserve_pv_entries(pmap_t pmap, int needed, struct rwlock **lockp) { struct pch new_tail; struct pv_chunk *pc; int avail, free; vm_page_t m; rw_assert(&pvh_global_lock, RA_LOCKED); PMAP_LOCK_ASSERT(pmap, MA_OWNED); KASSERT(lockp != NULL, ("reserve_pv_entries: lockp is NULL")); /* * Newly allocated PV chunks must be stored in a private list until * the required number of PV chunks have been allocated. Otherwise, * reclaim_pv_chunk() could recycle one of these chunks. In * contrast, these chunks must be added to the pmap upon allocation. */ TAILQ_INIT(&new_tail); retry: avail = 0; TAILQ_FOREACH(pc, &pmap->pm_pvchunk, pc_list) { #ifndef __POPCNT__ if ((cpu_feature2 & CPUID2_POPCNT) == 0) { free = bitcount64(pc->pc_map[0]); free += bitcount64(pc->pc_map[1]); free += bitcount64(pc->pc_map[2]); } else #endif { free = popcnt_pc_map_elem_pq(pc->pc_map[0]); free += popcnt_pc_map_elem_pq(pc->pc_map[1]); free += popcnt_pc_map_elem_pq(pc->pc_map[2]); } if (free == 0) break; avail += free; if (avail >= needed) break; } for (; avail < needed; avail += _NPCPV) { m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED); if (m == NULL) { m = reclaim_pv_chunk(pmap, lockp); if (m == NULL) goto retry; } PV_STAT(atomic_add_int(&pc_chunk_count, 1)); PV_STAT(atomic_add_int(&pc_chunk_allocs, 1)); dump_add_page(m->phys_addr); pc = (void *)PHYS_TO_DMAP(m->phys_addr); pc->pc_pmap = pmap; pc->pc_map[0] = PC_FREE0; pc->pc_map[1] = PC_FREE1; pc->pc_map[2] = PC_FREE2; TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); TAILQ_INSERT_TAIL(&new_tail, pc, pc_lru); PV_STAT(atomic_add_int(&pv_entry_spare, _NPCPV)); } if (!TAILQ_EMPTY(&new_tail)) { mtx_lock(&pv_chunks_mutex); TAILQ_CONCAT(&pv_chunks, &new_tail, pc_lru); mtx_unlock(&pv_chunks_mutex); } } /* * First find and then remove the pv entry for the specified pmap and virtual * address from the specified pv list. Returns the pv entry if found and NULL * otherwise. This operation can be performed on pv lists for either 4KB or * 2MB page mappings. */ static __inline pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va) { pv_entry_t pv; rw_assert(&pvh_global_lock, RA_LOCKED); TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { if (pmap == PV_PMAP(pv) && va == pv->pv_va) { TAILQ_REMOVE(&pvh->pv_list, pv, pv_next); pvh->pv_gen++; break; } } return (pv); } /* * After demotion from a 2MB page mapping to 512 4KB page mappings, * destroy the pv entry for the 2MB page mapping and reinstantiate the pv * entries for each of the 4KB page mappings. */ static void pmap_pv_demote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, struct rwlock **lockp) { struct md_page *pvh; struct pv_chunk *pc; pv_entry_t pv; vm_offset_t va_last; vm_page_t m; int bit, field; rw_assert(&pvh_global_lock, RA_LOCKED); PMAP_LOCK_ASSERT(pmap, MA_OWNED); KASSERT((pa & PDRMASK) == 0, ("pmap_pv_demote_pde: pa is not 2mpage aligned")); CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa); /* * Transfer the 2mpage's pv entry for this mapping to the first * page's pv list. Once this transfer begins, the pv list lock * must not be released until the last pv entry is reinstantiated. */ pvh = pa_to_pvh(pa); va = trunc_2mpage(va); pv = pmap_pvh_remove(pvh, pmap, va); KASSERT(pv != NULL, ("pmap_pv_demote_pde: pv not found")); m = PHYS_TO_VM_PAGE(pa); TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); m->md.pv_gen++; /* Instantiate the remaining NPTEPG - 1 pv entries. */ PV_STAT(atomic_add_long(&pv_entry_allocs, NPTEPG - 1)); va_last = va + NBPDR - PAGE_SIZE; for (;;) { pc = TAILQ_FIRST(&pmap->pm_pvchunk); KASSERT(pc->pc_map[0] != 0 || pc->pc_map[1] != 0 || pc->pc_map[2] != 0, ("pmap_pv_demote_pde: missing spare")); for (field = 0; field < _NPCM; field++) { while (pc->pc_map[field]) { bit = bsfq(pc->pc_map[field]); pc->pc_map[field] &= ~(1ul << bit); pv = &pc->pc_pventry[field * 64 + bit]; va += PAGE_SIZE; pv->pv_va = va; m++; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_pv_demote_pde: page %p is not managed", m)); TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); m->md.pv_gen++; if (va == va_last) goto out; } } TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); } out: if (pc->pc_map[0] == 0 && pc->pc_map[1] == 0 && pc->pc_map[2] == 0) { TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); } PV_STAT(atomic_add_long(&pv_entry_count, NPTEPG - 1)); PV_STAT(atomic_subtract_int(&pv_entry_spare, NPTEPG - 1)); } /* * After promotion from 512 4KB page mappings to a single 2MB page mapping, * replace the many pv entries for the 4KB page mappings by a single pv entry * for the 2MB page mapping. */ static void pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, struct rwlock **lockp) { struct md_page *pvh; pv_entry_t pv; vm_offset_t va_last; vm_page_t m; rw_assert(&pvh_global_lock, RA_LOCKED); KASSERT((pa & PDRMASK) == 0, ("pmap_pv_promote_pde: pa is not 2mpage aligned")); CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa); /* * Transfer the first page's pv entry for this mapping to the 2mpage's * pv list. Aside from avoiding the cost of a call to get_pv_entry(), * a transfer avoids the possibility that get_pv_entry() calls * reclaim_pv_chunk() and that reclaim_pv_chunk() removes one of the * mappings that is being promoted. */ m = PHYS_TO_VM_PAGE(pa); va = trunc_2mpage(va); pv = pmap_pvh_remove(&m->md, pmap, va); KASSERT(pv != NULL, ("pmap_pv_promote_pde: pv not found")); pvh = pa_to_pvh(pa); TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next); pvh->pv_gen++; /* Free the remaining NPTEPG - 1 pv entries. */ va_last = va + NBPDR - PAGE_SIZE; do { m++; va += PAGE_SIZE; pmap_pvh_free(&m->md, pmap, va); } while (va < va_last); } /* * First find and then destroy the pv entry for the specified pmap and virtual * address. This operation can be performed on pv lists for either 4KB or 2MB * page mappings. */ static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va) { pv_entry_t pv; pv = pmap_pvh_remove(pvh, pmap, va); KASSERT(pv != NULL, ("pmap_pvh_free: pv not found")); free_pv_entry(pmap, pv); } /* * Conditionally create the PV entry for a 4KB page mapping if the required * memory can be allocated without resorting to reclamation. */ static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m, struct rwlock **lockp) { pv_entry_t pv; rw_assert(&pvh_global_lock, RA_LOCKED); PMAP_LOCK_ASSERT(pmap, MA_OWNED); /* Pass NULL instead of the lock pointer to disable reclamation. */ if ((pv = get_pv_entry(pmap, NULL)) != NULL) { pv->pv_va = va; CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); m->md.pv_gen++; return (TRUE); } else return (FALSE); } /* * Conditionally create the PV entry for a 2MB page mapping if the required * memory can be allocated without resorting to reclamation. */ static boolean_t pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa, struct rwlock **lockp) { struct md_page *pvh; pv_entry_t pv; rw_assert(&pvh_global_lock, RA_LOCKED); PMAP_LOCK_ASSERT(pmap, MA_OWNED); /* Pass NULL instead of the lock pointer to disable reclamation. */ if ((pv = get_pv_entry(pmap, NULL)) != NULL) { pv->pv_va = va; CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa); pvh = pa_to_pvh(pa); TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next); pvh->pv_gen++; return (TRUE); } else return (FALSE); } /* * Fills a page table page with mappings to consecutive physical pages. */ static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte) { pt_entry_t *pte; for (pte = firstpte; pte < firstpte + NPTEPG; pte++) { *pte = newpte; newpte += PAGE_SIZE; } } /* * Tries to demote a 2MB page mapping. If demotion fails, the 2MB page * mapping is invalidated. */ static boolean_t pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va) { struct rwlock *lock; boolean_t rv; lock = NULL; rv = pmap_demote_pde_locked(pmap, pde, va, &lock); if (lock != NULL) rw_wunlock(lock); return (rv); } static boolean_t pmap_demote_pde_locked(pmap_t pmap, pd_entry_t *pde, vm_offset_t va, struct rwlock **lockp) { pd_entry_t newpde, oldpde; pt_entry_t *firstpte, newpte; pt_entry_t PG_A, PG_G, PG_M, PG_RW, PG_V; vm_paddr_t mptepa; vm_page_t mpte; struct spglist free; int PG_PTE_CACHE; PG_G = pmap_global_bit(pmap); PG_A = pmap_accessed_bit(pmap); PG_M = pmap_modified_bit(pmap); PG_RW = pmap_rw_bit(pmap); PG_V = pmap_valid_bit(pmap); PG_PTE_CACHE = pmap_cache_mask(pmap, 0); PMAP_LOCK_ASSERT(pmap, MA_OWNED); oldpde = *pde; KASSERT((oldpde & (PG_PS | PG_V)) == (PG_PS | PG_V), ("pmap_demote_pde: oldpde is missing PG_PS and/or PG_V")); if ((oldpde & PG_A) != 0 && (mpte = pmap_lookup_pt_page(pmap, va)) != NULL) pmap_remove_pt_page(pmap, mpte); else { KASSERT((oldpde & PG_W) == 0, ("pmap_demote_pde: page table page for a wired mapping" " is missing")); /* * Invalidate the 2MB page mapping and return "failure" if the * mapping was never accessed or the allocation of the new * page table page fails. If the 2MB page mapping belongs to * the direct map region of the kernel's address space, then * the page allocation request specifies the highest possible * priority (VM_ALLOC_INTERRUPT). Otherwise, the priority is * normal. Page table pages are preallocated for every other * part of the kernel address space, so the direct map region * is the only part of the kernel address space that must be * handled here. */ if ((oldpde & PG_A) == 0 || (mpte = vm_page_alloc(NULL, pmap_pde_pindex(va), (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS ? VM_ALLOC_INTERRUPT : VM_ALLOC_NORMAL) | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED)) == NULL) { SLIST_INIT(&free); pmap_remove_pde(pmap, pde, trunc_2mpage(va), &free, lockp); pmap_invalidate_page(pmap, trunc_2mpage(va)); pmap_free_zero_pages(&free); CTR2(KTR_PMAP, "pmap_demote_pde: failure for va %#lx" " in pmap %p", va, pmap); return (FALSE); } if (va < VM_MAXUSER_ADDRESS) pmap_resident_count_inc(pmap, 1); } mptepa = VM_PAGE_TO_PHYS(mpte); firstpte = (pt_entry_t *)PHYS_TO_DMAP(mptepa); newpde = mptepa | PG_M | PG_A | (oldpde & PG_U) | PG_RW | PG_V; KASSERT((oldpde & PG_A) != 0, ("pmap_demote_pde: oldpde is missing PG_A")); KASSERT((oldpde & (PG_M | PG_RW)) != PG_RW, ("pmap_demote_pde: oldpde is missing PG_M")); newpte = oldpde & ~PG_PS; newpte = pmap_swap_pat(pmap, newpte); /* * If the page table page is new, initialize it. */ if (mpte->wire_count == 1) { mpte->wire_count = NPTEPG; pmap_fill_ptp(firstpte, newpte); } KASSERT((*firstpte & PG_FRAME) == (newpte & PG_FRAME), ("pmap_demote_pde: firstpte and newpte map different physical" " addresses")); /* * If the mapping has changed attributes, update the page table * entries. */ if ((*firstpte & PG_PTE_PROMOTE) != (newpte & PG_PTE_PROMOTE)) pmap_fill_ptp(firstpte, newpte); /* * The spare PV entries must be reserved prior to demoting the * mapping, that is, prior to changing the PDE. Otherwise, the state * of the PDE and the PV lists will be inconsistent, which can result * in reclaim_pv_chunk() attempting to remove a PV entry from the * wrong PV list and pmap_pv_demote_pde() failing to find the expected * PV entry for the 2MB page mapping that is being demoted. */ if ((oldpde & PG_MANAGED) != 0) reserve_pv_entries(pmap, NPTEPG - 1, lockp); /* * Demote the mapping. This pmap is locked. The old PDE has * PG_A set. If the old PDE has PG_RW set, it also has PG_M * set. Thus, there is no danger of a race with another * processor changing the setting of PG_A and/or PG_M between * the read above and the store below. */ if (workaround_erratum383) pmap_update_pde(pmap, va, pde, newpde); else pde_store(pde, newpde); /* * Invalidate a stale recursive mapping of the page table page. */ if (va >= VM_MAXUSER_ADDRESS) pmap_invalidate_page(pmap, (vm_offset_t)vtopte(va)); /* * Demote the PV entry. */ if ((oldpde & PG_MANAGED) != 0) pmap_pv_demote_pde(pmap, va, oldpde & PG_PS_FRAME, lockp); atomic_add_long(&pmap_pde_demotions, 1); CTR2(KTR_PMAP, "pmap_demote_pde: success for va %#lx" " in pmap %p", va, pmap); return (TRUE); } /* * pmap_remove_kernel_pde: Remove a kernel superpage mapping. */ static void pmap_remove_kernel_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va) { pd_entry_t newpde; vm_paddr_t mptepa; vm_page_t mpte; KASSERT(pmap == kernel_pmap, ("pmap %p is not kernel_pmap", pmap)); PMAP_LOCK_ASSERT(pmap, MA_OWNED); mpte = pmap_lookup_pt_page(pmap, va); if (mpte == NULL) panic("pmap_remove_kernel_pde: Missing pt page."); pmap_remove_pt_page(pmap, mpte); mptepa = VM_PAGE_TO_PHYS(mpte); newpde = mptepa | X86_PG_M | X86_PG_A | X86_PG_RW | X86_PG_V; /* * Initialize the page table page. */ pagezero((void *)PHYS_TO_DMAP(mptepa)); /* * Demote the mapping. */ if (workaround_erratum383) pmap_update_pde(pmap, va, pde, newpde); else pde_store(pde, newpde); /* * Invalidate a stale recursive mapping of the page table page. */ pmap_invalidate_page(pmap, (vm_offset_t)vtopte(va)); } /* * pmap_remove_pde: do the things to unmap a superpage in a process */ static int pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva, struct spglist *free, struct rwlock **lockp) { struct md_page *pvh; pd_entry_t oldpde; vm_offset_t eva, va; vm_page_t m, mpte; pt_entry_t PG_G, PG_A, PG_M, PG_RW; PG_G = pmap_global_bit(pmap); PG_A = pmap_accessed_bit(pmap); PG_M = pmap_modified_bit(pmap); PG_RW = pmap_rw_bit(pmap); PMAP_LOCK_ASSERT(pmap, MA_OWNED); KASSERT((sva & PDRMASK) == 0, ("pmap_remove_pde: sva is not 2mpage aligned")); oldpde = pte_load_clear(pdq); if (oldpde & PG_W) pmap->pm_stats.wired_count -= NBPDR / PAGE_SIZE; /* * Machines that don't support invlpg, also don't support * PG_G. */ if (oldpde & PG_G) pmap_invalidate_page(kernel_pmap, sva); pmap_resident_count_dec(pmap, NBPDR / PAGE_SIZE); if (oldpde & PG_MANAGED) { CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, oldpde & PG_PS_FRAME); pvh = pa_to_pvh(oldpde & PG_PS_FRAME); pmap_pvh_free(pvh, pmap, sva); eva = sva + NBPDR; for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME); va < eva; va += PAGE_SIZE, m++) { if ((oldpde & (PG_M | PG_RW)) == (PG_M | PG_RW)) vm_page_dirty(m); if (oldpde & PG_A) vm_page_aflag_set(m, PGA_REFERENCED); if (TAILQ_EMPTY(&m->md.pv_list) && TAILQ_EMPTY(&pvh->pv_list)) vm_page_aflag_clear(m, PGA_WRITEABLE); } } if (pmap == kernel_pmap) { pmap_remove_kernel_pde(pmap, pdq, sva); } else { mpte = pmap_lookup_pt_page(pmap, sva); if (mpte != NULL) { pmap_remove_pt_page(pmap, mpte); pmap_resident_count_dec(pmap, 1); KASSERT(mpte->wire_count == NPTEPG, ("pmap_remove_pde: pte page wire count error")); mpte->wire_count = 0; pmap_add_delayed_free_list(mpte, free, FALSE); atomic_subtract_int(&vm_cnt.v_wire_count, 1); } } return (pmap_unuse_pt(pmap, sva, *pmap_pdpe(pmap, sva), free)); } /* * pmap_remove_pte: do the things to unmap a page in a process */ static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va, pd_entry_t ptepde, struct spglist *free, struct rwlock **lockp) { struct md_page *pvh; pt_entry_t oldpte, PG_A, PG_M, PG_RW; vm_page_t m; PG_A = pmap_accessed_bit(pmap); PG_M = pmap_modified_bit(pmap); PG_RW = pmap_rw_bit(pmap); PMAP_LOCK_ASSERT(pmap, MA_OWNED); oldpte = pte_load_clear(ptq); if (oldpte & PG_W) pmap->pm_stats.wired_count -= 1; pmap_resident_count_dec(pmap, 1); if (oldpte & PG_MANAGED) { m = PHYS_TO_VM_PAGE(oldpte & PG_FRAME); if ((oldpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) vm_page_dirty(m); if (oldpte & PG_A) vm_page_aflag_set(m, PGA_REFERENCED); CHANGE_PV_LIST_LOCK_TO_VM_PAGE(lockp, m); pmap_pvh_free(&m->md, pmap, va); if (TAILQ_EMPTY(&m->md.pv_list) && (m->flags & PG_FICTITIOUS) == 0) { pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); if (TAILQ_EMPTY(&pvh->pv_list)) vm_page_aflag_clear(m, PGA_WRITEABLE); } } return (pmap_unuse_pt(pmap, va, ptepde, free)); } /* * Remove a single page from a process address space */ static void pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, struct spglist *free) { struct rwlock *lock; pt_entry_t *pte, PG_V; PG_V = pmap_valid_bit(pmap); PMAP_LOCK_ASSERT(pmap, MA_OWNED); if ((*pde & PG_V) == 0) return; pte = pmap_pde_to_pte(pde, va); if ((*pte & PG_V) == 0) return; lock = NULL; pmap_remove_pte(pmap, pte, va, *pde, free, &lock); if (lock != NULL) rw_wunlock(lock); pmap_invalidate_page(pmap, va); } /* * Remove the given range of addresses from the specified map. * * It is assumed that the start and end are properly * rounded to the page size. */ void pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) { struct rwlock *lock; vm_offset_t va, va_next; pml4_entry_t *pml4e; pdp_entry_t *pdpe; pd_entry_t ptpaddr, *pde; pt_entry_t *pte, PG_G, PG_V; struct spglist free; int anyvalid; PG_G = pmap_global_bit(pmap); PG_V = pmap_valid_bit(pmap); /* * Perform an unsynchronized read. This is, however, safe. */ if (pmap->pm_stats.resident_count == 0) return; anyvalid = 0; SLIST_INIT(&free); rw_rlock(&pvh_global_lock); PMAP_LOCK(pmap); /* * special handling of removing one page. a very * common operation and easy to short circuit some * code. */ if (sva + PAGE_SIZE == eva) { pde = pmap_pde(pmap, sva); if (pde && (*pde & PG_PS) == 0) { pmap_remove_page(pmap, sva, pde, &free); goto out; } } lock = NULL; for (; sva < eva; sva = va_next) { if (pmap->pm_stats.resident_count == 0) break; pml4e = pmap_pml4e(pmap, sva); if ((*pml4e & PG_V) == 0) { va_next = (sva + NBPML4) & ~PML4MASK; if (va_next < sva) va_next = eva; continue; } pdpe = pmap_pml4e_to_pdpe(pml4e, sva); if ((*pdpe & PG_V) == 0) { va_next = (sva + NBPDP) & ~PDPMASK; if (va_next < sva) va_next = eva; continue; } /* * Calculate index for next page table. */ va_next = (sva + NBPDR) & ~PDRMASK; if (va_next < sva) va_next = eva; pde = pmap_pdpe_to_pde(pdpe, sva); ptpaddr = *pde; /* * Weed out invalid mappings. */ if (ptpaddr == 0) continue; /* * Check for large page. */ if ((ptpaddr & PG_PS) != 0) { /* * Are we removing the entire large page? If not, * demote the mapping and fall through. */ if (sva + NBPDR == va_next && eva >= va_next) { /* * The TLB entry for a PG_G mapping is * invalidated by pmap_remove_pde(). */ if ((ptpaddr & PG_G) == 0) anyvalid = 1; pmap_remove_pde(pmap, pde, sva, &free, &lock); continue; } else if (!pmap_demote_pde_locked(pmap, pde, sva, &lock)) { /* The large page mapping was destroyed. */ continue; } else ptpaddr = *pde; } /* * Limit our scan to either the end of the va represented * by the current page table page, or to the end of the * range being removed. */ if (va_next > eva) va_next = eva; va = va_next; for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++, sva += PAGE_SIZE) { if (*pte == 0) { if (va != va_next) { pmap_invalidate_range(pmap, va, sva); va = va_next; } continue; } if ((*pte & PG_G) == 0) anyvalid = 1; else if (va == va_next) va = sva; if (pmap_remove_pte(pmap, pte, sva, ptpaddr, &free, &lock)) { sva += PAGE_SIZE; break; } } if (va != va_next) pmap_invalidate_range(pmap, va, sva); } if (lock != NULL) rw_wunlock(lock); out: if (anyvalid) pmap_invalidate_all(pmap); rw_runlock(&pvh_global_lock); PMAP_UNLOCK(pmap); pmap_free_zero_pages(&free); } /* * Routine: pmap_remove_all * Function: * Removes this physical page from * all physical maps in which it resides. * Reflects back modify bits to the pager. * * Notes: * Original versions of this routine were very * inefficient because they iteratively called * pmap_remove (slow...) */ void pmap_remove_all(vm_page_t m) { struct md_page *pvh; pv_entry_t pv; pmap_t pmap; pt_entry_t *pte, tpte, PG_A, PG_M, PG_RW; pd_entry_t *pde; vm_offset_t va; struct spglist free; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_remove_all: page %p is not managed", m)); SLIST_INIT(&free); rw_wlock(&pvh_global_lock); if ((m->flags & PG_FICTITIOUS) != 0) goto small_mappings; pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); while ((pv = TAILQ_FIRST(&pvh->pv_list)) != NULL) { pmap = PV_PMAP(pv); PMAP_LOCK(pmap); va = pv->pv_va; pde = pmap_pde(pmap, va); (void)pmap_demote_pde(pmap, pde, va); PMAP_UNLOCK(pmap); } small_mappings: while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { pmap = PV_PMAP(pv); PMAP_LOCK(pmap); PG_A = pmap_accessed_bit(pmap); PG_M = pmap_modified_bit(pmap); PG_RW = pmap_rw_bit(pmap); pmap_resident_count_dec(pmap, 1); pde = pmap_pde(pmap, pv->pv_va); KASSERT((*pde & PG_PS) == 0, ("pmap_remove_all: found" " a 2mpage in page %p's pv list", m)); pte = pmap_pde_to_pte(pde, pv->pv_va); tpte = pte_load_clear(pte); if (tpte & PG_W) pmap->pm_stats.wired_count--; if (tpte & PG_A) vm_page_aflag_set(m, PGA_REFERENCED); /* * Update the vm_page_t clean and reference bits. */ if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) vm_page_dirty(m); pmap_unuse_pt(pmap, pv->pv_va, *pde, &free); pmap_invalidate_page(pmap, pv->pv_va); TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); m->md.pv_gen++; free_pv_entry(pmap, pv); PMAP_UNLOCK(pmap); } vm_page_aflag_clear(m, PGA_WRITEABLE); rw_wunlock(&pvh_global_lock); pmap_free_zero_pages(&free); } /* * pmap_protect_pde: do the things to protect a 2mpage in a process */ static boolean_t pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva, vm_prot_t prot) { pd_entry_t newpde, oldpde; vm_offset_t eva, va; vm_page_t m; boolean_t anychanged; pt_entry_t PG_G, PG_M, PG_RW; PG_G = pmap_global_bit(pmap); PG_M = pmap_modified_bit(pmap); PG_RW = pmap_rw_bit(pmap); PMAP_LOCK_ASSERT(pmap, MA_OWNED); KASSERT((sva & PDRMASK) == 0, ("pmap_protect_pde: sva is not 2mpage aligned")); anychanged = FALSE; retry: oldpde = newpde = *pde; if (oldpde & PG_MANAGED) { eva = sva + NBPDR; for (va = sva, m = PHYS_TO_VM_PAGE(oldpde & PG_PS_FRAME); va < eva; va += PAGE_SIZE, m++) if ((oldpde & (PG_M | PG_RW)) == (PG_M | PG_RW)) vm_page_dirty(m); } if ((prot & VM_PROT_WRITE) == 0) newpde &= ~(PG_RW | PG_M); if ((prot & VM_PROT_EXECUTE) == 0) newpde |= pg_nx; if (newpde != oldpde) { if (!atomic_cmpset_long(pde, oldpde, newpde)) goto retry; if (oldpde & PG_G) pmap_invalidate_page(pmap, sva); else anychanged = TRUE; } return (anychanged); } /* * Set the physical protection on the * specified range of this map as requested. */ void pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) { vm_offset_t va_next; pml4_entry_t *pml4e; pdp_entry_t *pdpe; pd_entry_t ptpaddr, *pde; pt_entry_t *pte, PG_G, PG_M, PG_RW, PG_V; boolean_t anychanged, pv_lists_locked; KASSERT((prot & ~VM_PROT_ALL) == 0, ("invalid prot %x", prot)); if (prot == VM_PROT_NONE) { pmap_remove(pmap, sva, eva); return; } if ((prot & (VM_PROT_WRITE|VM_PROT_EXECUTE)) == (VM_PROT_WRITE|VM_PROT_EXECUTE)) return; PG_G = pmap_global_bit(pmap); PG_M = pmap_modified_bit(pmap); PG_V = pmap_valid_bit(pmap); PG_RW = pmap_rw_bit(pmap); pv_lists_locked = FALSE; resume: anychanged = FALSE; PMAP_LOCK(pmap); for (; sva < eva; sva = va_next) { pml4e = pmap_pml4e(pmap, sva); if ((*pml4e & PG_V) == 0) { va_next = (sva + NBPML4) & ~PML4MASK; if (va_next < sva) va_next = eva; continue; } pdpe = pmap_pml4e_to_pdpe(pml4e, sva); if ((*pdpe & PG_V) == 0) { va_next = (sva + NBPDP) & ~PDPMASK; if (va_next < sva) va_next = eva; continue; } va_next = (sva + NBPDR) & ~PDRMASK; if (va_next < sva) va_next = eva; pde = pmap_pdpe_to_pde(pdpe, sva); ptpaddr = *pde; /* * Weed out invalid mappings. */ if (ptpaddr == 0) continue; /* * Check for large page. */ if ((ptpaddr & PG_PS) != 0) { /* * Are we protecting the entire large page? If not, * demote the mapping and fall through. */ if (sva + NBPDR == va_next && eva >= va_next) { /* * The TLB entry for a PG_G mapping is * invalidated by pmap_protect_pde(). */ if (pmap_protect_pde(pmap, pde, sva, prot)) anychanged = TRUE; continue; } else { if (!pv_lists_locked) { pv_lists_locked = TRUE; if (!rw_try_rlock(&pvh_global_lock)) { if (anychanged) pmap_invalidate_all( pmap); PMAP_UNLOCK(pmap); rw_rlock(&pvh_global_lock); goto resume; } } if (!pmap_demote_pde(pmap, pde, sva)) { /* * The large page mapping was * destroyed. */ continue; } } } if (va_next > eva) va_next = eva; for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++, sva += PAGE_SIZE) { pt_entry_t obits, pbits; vm_page_t m; retry: obits = pbits = *pte; if ((pbits & PG_V) == 0) continue; if ((prot & VM_PROT_WRITE) == 0) { if ((pbits & (PG_MANAGED | PG_M | PG_RW)) == (PG_MANAGED | PG_M | PG_RW)) { m = PHYS_TO_VM_PAGE(pbits & PG_FRAME); vm_page_dirty(m); } pbits &= ~(PG_RW | PG_M); } if ((prot & VM_PROT_EXECUTE) == 0) pbits |= pg_nx; if (pbits != obits) { if (!atomic_cmpset_long(pte, obits, pbits)) goto retry; if (obits & PG_G) pmap_invalidate_page(pmap, sva); else anychanged = TRUE; } } } if (anychanged) pmap_invalidate_all(pmap); if (pv_lists_locked) rw_runlock(&pvh_global_lock); PMAP_UNLOCK(pmap); } /* * Tries to promote the 512, contiguous 4KB page mappings that are within a * single page table page (PTP) to a single 2MB page mapping. For promotion * to occur, two conditions must be met: (1) the 4KB page mappings must map * aligned, contiguous physical memory and (2) the 4KB page mappings must have * identical characteristics. */ static void pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va, struct rwlock **lockp) { pd_entry_t newpde; pt_entry_t *firstpte, oldpte, pa, *pte; pt_entry_t PG_G, PG_A, PG_M, PG_RW, PG_V; vm_page_t mpte; int PG_PTE_CACHE; PG_A = pmap_accessed_bit(pmap); PG_G = pmap_global_bit(pmap); PG_M = pmap_modified_bit(pmap); PG_V = pmap_valid_bit(pmap); PG_RW = pmap_rw_bit(pmap); PG_PTE_CACHE = pmap_cache_mask(pmap, 0); PMAP_LOCK_ASSERT(pmap, MA_OWNED); /* * Examine the first PTE in the specified PTP. Abort if this PTE is * either invalid, unused, or does not map the first 4KB physical page * within a 2MB page. */ firstpte = (pt_entry_t *)PHYS_TO_DMAP(*pde & PG_FRAME); setpde: newpde = *firstpte; if ((newpde & ((PG_FRAME & PDRMASK) | PG_A | PG_V)) != (PG_A | PG_V)) { atomic_add_long(&pmap_pde_p_failures, 1); CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx" " in pmap %p", va, pmap); return; } if ((newpde & (PG_M | PG_RW)) == PG_RW) { /* * When PG_M is already clear, PG_RW can be cleared without * a TLB invalidation. */ if (!atomic_cmpset_long(firstpte, newpde, newpde & ~PG_RW)) goto setpde; newpde &= ~PG_RW; } /* * Examine each of the other PTEs in the specified PTP. Abort if this * PTE maps an unexpected 4KB physical page or does not have identical * characteristics to the first PTE. */ pa = (newpde & (PG_PS_FRAME | PG_A | PG_V)) + NBPDR - PAGE_SIZE; for (pte = firstpte + NPTEPG - 1; pte > firstpte; pte--) { setpte: oldpte = *pte; if ((oldpte & (PG_FRAME | PG_A | PG_V)) != pa) { atomic_add_long(&pmap_pde_p_failures, 1); CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx" " in pmap %p", va, pmap); return; } if ((oldpte & (PG_M | PG_RW)) == PG_RW) { /* * When PG_M is already clear, PG_RW can be cleared * without a TLB invalidation. */ if (!atomic_cmpset_long(pte, oldpte, oldpte & ~PG_RW)) goto setpte; oldpte &= ~PG_RW; CTR2(KTR_PMAP, "pmap_promote_pde: protect for va %#lx" " in pmap %p", (oldpte & PG_FRAME & PDRMASK) | (va & ~PDRMASK), pmap); } if ((oldpte & PG_PTE_PROMOTE) != (newpde & PG_PTE_PROMOTE)) { atomic_add_long(&pmap_pde_p_failures, 1); CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx" " in pmap %p", va, pmap); return; } pa -= PAGE_SIZE; } /* * Save the page table page in its current state until the PDE * mapping the superpage is demoted by pmap_demote_pde() or * destroyed by pmap_remove_pde(). */ mpte = PHYS_TO_VM_PAGE(*pde & PG_FRAME); KASSERT(mpte >= vm_page_array && mpte < &vm_page_array[vm_page_array_size], ("pmap_promote_pde: page table page is out of range")); KASSERT(mpte->pindex == pmap_pde_pindex(va), ("pmap_promote_pde: page table page's pindex is wrong")); if (pmap_insert_pt_page(pmap, mpte)) { atomic_add_long(&pmap_pde_p_failures, 1); CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx in pmap %p", va, pmap); return; } /* * Promote the pv entries. */ if ((newpde & PG_MANAGED) != 0) pmap_pv_promote_pde(pmap, va, newpde & PG_PS_FRAME, lockp); /* * Propagate the PAT index to its proper position. */ newpde = pmap_swap_pat(pmap, newpde); /* * Map the superpage. */ if (workaround_erratum383) pmap_update_pde(pmap, va, pde, PG_PS | newpde); else pde_store(pde, PG_PS | newpde); atomic_add_long(&pmap_pde_promotions, 1); CTR2(KTR_PMAP, "pmap_promote_pde: success for va %#lx" " in pmap %p", va, pmap); } /* * Insert the given physical page (p) at * the specified virtual address (v) in the * target physical map with the protection requested. * * If specified, the page will be wired down, meaning * that the related pte can not be reclaimed. * * NB: This is the only routine which MAY NOT lazy-evaluate * or lose information. That is, this routine must actually * insert this page into the given map NOW. */ int pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, u_int flags, int8_t psind __unused) { struct rwlock *lock; pd_entry_t *pde; pt_entry_t *pte, PG_G, PG_A, PG_M, PG_RW, PG_V; pt_entry_t newpte, origpte; pv_entry_t pv; vm_paddr_t opa, pa; vm_page_t mpte, om; boolean_t nosleep; PG_A = pmap_accessed_bit(pmap); PG_G = pmap_global_bit(pmap); PG_M = pmap_modified_bit(pmap); PG_V = pmap_valid_bit(pmap); PG_RW = pmap_rw_bit(pmap); va = trunc_page(va); KASSERT(va <= VM_MAX_KERNEL_ADDRESS, ("pmap_enter: toobig")); KASSERT(va < UPT_MIN_ADDRESS || va >= UPT_MAX_ADDRESS, ("pmap_enter: invalid to pmap_enter page table pages (va: 0x%lx)", va)); KASSERT((m->oflags & VPO_UNMANAGED) != 0 || va < kmi.clean_sva || va >= kmi.clean_eva, ("pmap_enter: managed mapping within the clean submap")); if ((m->oflags & VPO_UNMANAGED) == 0 && !vm_page_xbusied(m)) VM_OBJECT_ASSERT_LOCKED(m->object); pa = VM_PAGE_TO_PHYS(m); newpte = (pt_entry_t)(pa | PG_A | PG_V); if ((flags & VM_PROT_WRITE) != 0) newpte |= PG_M; if ((prot & VM_PROT_WRITE) != 0) newpte |= PG_RW; KASSERT((newpte & (PG_M | PG_RW)) != PG_M, ("pmap_enter: flags includes VM_PROT_WRITE but prot doesn't")); if ((prot & VM_PROT_EXECUTE) == 0) newpte |= pg_nx; if ((flags & PMAP_ENTER_WIRED) != 0) newpte |= PG_W; if (va < VM_MAXUSER_ADDRESS) newpte |= PG_U; if (pmap == kernel_pmap) newpte |= PG_G; newpte |= pmap_cache_bits(pmap, m->md.pat_mode, 0); /* * Set modified bit gratuitously for writeable mappings if * the page is unmanaged. We do not want to take a fault * to do the dirty bit accounting for these mappings. */ if ((m->oflags & VPO_UNMANAGED) != 0) { if ((newpte & PG_RW) != 0) newpte |= PG_M; } mpte = NULL; lock = NULL; rw_rlock(&pvh_global_lock); PMAP_LOCK(pmap); /* * In the case that a page table page is not * resident, we are creating it here. */ retry: pde = pmap_pde(pmap, va); if (pde != NULL && (*pde & PG_V) != 0 && ((*pde & PG_PS) == 0 || pmap_demote_pde_locked(pmap, pde, va, &lock))) { pte = pmap_pde_to_pte(pde, va); if (va < VM_MAXUSER_ADDRESS && mpte == NULL) { mpte = PHYS_TO_VM_PAGE(*pde & PG_FRAME); mpte->wire_count++; } } else if (va < VM_MAXUSER_ADDRESS) { /* * Here if the pte page isn't mapped, or if it has been * deallocated. */ nosleep = (flags & PMAP_ENTER_NOSLEEP) != 0; mpte = _pmap_allocpte(pmap, pmap_pde_pindex(va), nosleep ? NULL : &lock); if (mpte == NULL && nosleep) { if (lock != NULL) rw_wunlock(lock); rw_runlock(&pvh_global_lock); PMAP_UNLOCK(pmap); return (KERN_RESOURCE_SHORTAGE); } goto retry; } else panic("pmap_enter: invalid page directory va=%#lx", va); origpte = *pte; /* * Is the specified virtual address already mapped? */ if ((origpte & PG_V) != 0) { /* * Wiring change, just update stats. We don't worry about * wiring PT pages as they remain resident as long as there * are valid mappings in them. Hence, if a user page is wired, * the PT page will be also. */ if ((newpte & PG_W) != 0 && (origpte & PG_W) == 0) pmap->pm_stats.wired_count++; else if ((newpte & PG_W) == 0 && (origpte & PG_W) != 0) pmap->pm_stats.wired_count--; /* * Remove the extra PT page reference. */ if (mpte != NULL) { mpte->wire_count--; KASSERT(mpte->wire_count > 0, ("pmap_enter: missing reference to page table page," " va: 0x%lx", va)); } /* * Has the physical page changed? */ opa = origpte & PG_FRAME; if (opa == pa) { /* * No, might be a protection or wiring change. */ if ((origpte & PG_MANAGED) != 0) { newpte |= PG_MANAGED; if ((newpte & PG_RW) != 0) vm_page_aflag_set(m, PGA_WRITEABLE); } if (((origpte ^ newpte) & ~(PG_M | PG_A)) == 0) goto unchanged; goto validate; } } else { /* * Increment the counters. */ if ((newpte & PG_W) != 0) pmap->pm_stats.wired_count++; pmap_resident_count_inc(pmap, 1); } /* * Enter on the PV list if part of our managed memory. */ if ((m->oflags & VPO_UNMANAGED) == 0) { newpte |= PG_MANAGED; pv = get_pv_entry(pmap, &lock); pv->pv_va = va; CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, pa); TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); m->md.pv_gen++; if ((newpte & PG_RW) != 0) vm_page_aflag_set(m, PGA_WRITEABLE); } /* * Update the PTE. */ if ((origpte & PG_V) != 0) { validate: origpte = pte_load_store(pte, newpte); opa = origpte & PG_FRAME; if (opa != pa) { if ((origpte & PG_MANAGED) != 0) { om = PHYS_TO_VM_PAGE(opa); if ((origpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) vm_page_dirty(om); if ((origpte & PG_A) != 0) vm_page_aflag_set(om, PGA_REFERENCED); CHANGE_PV_LIST_LOCK_TO_PHYS(&lock, opa); pmap_pvh_free(&om->md, pmap, va); if ((om->aflags & PGA_WRITEABLE) != 0 && TAILQ_EMPTY(&om->md.pv_list) && ((om->flags & PG_FICTITIOUS) != 0 || TAILQ_EMPTY(&pa_to_pvh(opa)->pv_list))) vm_page_aflag_clear(om, PGA_WRITEABLE); } } else if ((newpte & PG_M) == 0 && (origpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) { if ((origpte & PG_MANAGED) != 0) vm_page_dirty(m); /* * Although the PTE may still have PG_RW set, TLB * invalidation may nonetheless be required because * the PTE no longer has PG_M set. */ } else if ((origpte & PG_NX) != 0 || (newpte & PG_NX) == 0) { /* * This PTE change does not require TLB invalidation. */ goto unchanged; } if ((origpte & PG_A) != 0) pmap_invalidate_page(pmap, va); } else pte_store(pte, newpte); unchanged: /* * If both the page table page and the reservation are fully * populated, then attempt promotion. */ if ((mpte == NULL || mpte->wire_count == NPTEPG) && pmap_ps_enabled(pmap) && (m->flags & PG_FICTITIOUS) == 0 && vm_reserv_level_iffullpop(m) == 0) pmap_promote_pde(pmap, pde, va, &lock); if (lock != NULL) rw_wunlock(lock); rw_runlock(&pvh_global_lock); PMAP_UNLOCK(pmap); return (KERN_SUCCESS); } /* * Tries to create a 2MB page mapping. Returns TRUE if successful and FALSE * otherwise. Fails if (1) a page table page cannot be allocated without * blocking, (2) a mapping already exists at the specified virtual address, or * (3) a pv entry cannot be allocated without reclaiming another pv entry. */ static boolean_t pmap_enter_pde(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, struct rwlock **lockp) { pd_entry_t *pde, newpde; pt_entry_t PG_V; vm_page_t mpde; struct spglist free; PG_V = pmap_valid_bit(pmap); rw_assert(&pvh_global_lock, RA_LOCKED); PMAP_LOCK_ASSERT(pmap, MA_OWNED); if ((mpde = pmap_allocpde(pmap, va, NULL)) == NULL) { CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx" " in pmap %p", va, pmap); return (FALSE); } pde = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mpde)); pde = &pde[pmap_pde_index(va)]; if ((*pde & PG_V) != 0) { KASSERT(mpde->wire_count > 1, ("pmap_enter_pde: mpde's wire count is too low")); mpde->wire_count--; CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx" " in pmap %p", va, pmap); return (FALSE); } newpde = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(pmap, m->md.pat_mode, 1) | PG_PS | PG_V; if ((m->oflags & VPO_UNMANAGED) == 0) { newpde |= PG_MANAGED; /* * Abort this mapping if its PV entry could not be created. */ if (!pmap_pv_insert_pde(pmap, va, VM_PAGE_TO_PHYS(m), lockp)) { SLIST_INIT(&free); if (pmap_unwire_ptp(pmap, va, mpde, &free)) { + /* + * Although "va" is not mapped, paging- + * structure caches could nonetheless have + * entries that refer to the freed page table + * pages. Invalidate those entries. + */ pmap_invalidate_page(pmap, va); pmap_free_zero_pages(&free); } CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx" " in pmap %p", va, pmap); return (FALSE); } } if ((prot & VM_PROT_EXECUTE) == 0) newpde |= pg_nx; if (va < VM_MAXUSER_ADDRESS) newpde |= PG_U; /* * Increment counters. */ pmap_resident_count_inc(pmap, NBPDR / PAGE_SIZE); /* * Map the superpage. */ pde_store(pde, newpde); atomic_add_long(&pmap_pde_mappings, 1); CTR2(KTR_PMAP, "pmap_enter_pde: success for va %#lx" " in pmap %p", va, pmap); return (TRUE); } /* * Maps a sequence of resident pages belonging to the same object. * The sequence begins with the given page m_start. This page is * mapped at the given virtual address start. Each subsequent page is * mapped at a virtual address that is offset from start by the same * amount as the page is offset from m_start within the object. The * last page in the sequence is the page with the largest offset from * m_start that can be mapped at a virtual address less than the given * virtual address end. Not every virtual page between start and end * is mapped; only those for which a resident page exists with the * corresponding offset from m_start are mapped. */ void pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, vm_page_t m_start, vm_prot_t prot) { struct rwlock *lock; vm_offset_t va; vm_page_t m, mpte; vm_pindex_t diff, psize; VM_OBJECT_ASSERT_LOCKED(m_start->object); psize = atop(end - start); mpte = NULL; m = m_start; lock = NULL; rw_rlock(&pvh_global_lock); PMAP_LOCK(pmap); while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { va = start + ptoa(diff); if ((va & PDRMASK) == 0 && va + NBPDR <= end && m->psind == 1 && pmap_ps_enabled(pmap) && pmap_enter_pde(pmap, va, m, prot, &lock)) m = &m[NBPDR / PAGE_SIZE - 1]; else mpte = pmap_enter_quick_locked(pmap, va, m, prot, mpte, &lock); m = TAILQ_NEXT(m, listq); } if (lock != NULL) rw_wunlock(lock); rw_runlock(&pvh_global_lock); PMAP_UNLOCK(pmap); } /* * this code makes some *MAJOR* assumptions: * 1. Current pmap & pmap exists. * 2. Not wired. * 3. Read access. * 4. No page table pages. * but is *MUCH* faster than pmap_enter... */ void pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot) { struct rwlock *lock; lock = NULL; rw_rlock(&pvh_global_lock); PMAP_LOCK(pmap); (void)pmap_enter_quick_locked(pmap, va, m, prot, NULL, &lock); if (lock != NULL) rw_wunlock(lock); rw_runlock(&pvh_global_lock); PMAP_UNLOCK(pmap); } static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, vm_page_t mpte, struct rwlock **lockp) { struct spglist free; pt_entry_t *pte, PG_V; vm_paddr_t pa; KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva || (m->oflags & VPO_UNMANAGED) != 0, ("pmap_enter_quick_locked: managed mapping within the clean submap")); PG_V = pmap_valid_bit(pmap); rw_assert(&pvh_global_lock, RA_LOCKED); PMAP_LOCK_ASSERT(pmap, MA_OWNED); /* * In the case that a page table page is not * resident, we are creating it here. */ if (va < VM_MAXUSER_ADDRESS) { vm_pindex_t ptepindex; pd_entry_t *ptepa; /* * Calculate pagetable page index */ ptepindex = pmap_pde_pindex(va); if (mpte && (mpte->pindex == ptepindex)) { mpte->wire_count++; } else { /* * Get the page directory entry */ ptepa = pmap_pde(pmap, va); /* * If the page table page is mapped, we just increment * the hold count, and activate it. Otherwise, we * attempt to allocate a page table page. If this * attempt fails, we don't retry. Instead, we give up. */ if (ptepa && (*ptepa & PG_V) != 0) { if (*ptepa & PG_PS) return (NULL); mpte = PHYS_TO_VM_PAGE(*ptepa & PG_FRAME); mpte->wire_count++; } else { /* * Pass NULL instead of the PV list lock * pointer, because we don't intend to sleep. */ mpte = _pmap_allocpte(pmap, ptepindex, NULL); if (mpte == NULL) return (mpte); } } pte = (pt_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mpte)); pte = &pte[pmap_pte_index(va)]; } else { mpte = NULL; pte = vtopte(va); } if (*pte) { if (mpte != NULL) { mpte->wire_count--; mpte = NULL; } return (mpte); } /* * Enter on the PV list if part of our managed memory. */ if ((m->oflags & VPO_UNMANAGED) == 0 && !pmap_try_insert_pv_entry(pmap, va, m, lockp)) { if (mpte != NULL) { SLIST_INIT(&free); if (pmap_unwire_ptp(pmap, va, mpte, &free)) { + /* + * Although "va" is not mapped, paging- + * structure caches could nonetheless have + * entries that refer to the freed page table + * pages. Invalidate those entries. + */ pmap_invalidate_page(pmap, va); pmap_free_zero_pages(&free); } mpte = NULL; } return (mpte); } /* * Increment counters */ pmap_resident_count_inc(pmap, 1); pa = VM_PAGE_TO_PHYS(m) | pmap_cache_bits(pmap, m->md.pat_mode, 0); if ((prot & VM_PROT_EXECUTE) == 0) pa |= pg_nx; /* * Now validate mapping with RO protection */ if ((m->oflags & VPO_UNMANAGED) != 0) pte_store(pte, pa | PG_V | PG_U); else pte_store(pte, pa | PG_V | PG_U | PG_MANAGED); return (mpte); } /* * Make a temporary mapping for a physical address. This is only intended * to be used for panic dumps. */ void * pmap_kenter_temporary(vm_paddr_t pa, int i) { vm_offset_t va; va = (vm_offset_t)crashdumpmap + (i * PAGE_SIZE); pmap_kenter(va, pa); invlpg(va); return ((void *)crashdumpmap); } /* * This code maps large physical mmap regions into the * processor address space. Note that some shortcuts * are taken, but the code works. */ void pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object, vm_pindex_t pindex, vm_size_t size) { pd_entry_t *pde; pt_entry_t PG_A, PG_M, PG_RW, PG_V; vm_paddr_t pa, ptepa; vm_page_t p, pdpg; int pat_mode; PG_A = pmap_accessed_bit(pmap); PG_M = pmap_modified_bit(pmap); PG_V = pmap_valid_bit(pmap); PG_RW = pmap_rw_bit(pmap); VM_OBJECT_ASSERT_WLOCKED(object); KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, ("pmap_object_init_pt: non-device object")); if ((addr & (NBPDR - 1)) == 0 && (size & (NBPDR - 1)) == 0) { if (!pmap_ps_enabled(pmap)) return; if (!vm_object_populate(object, pindex, pindex + atop(size))) return; p = vm_page_lookup(object, pindex); KASSERT(p->valid == VM_PAGE_BITS_ALL, ("pmap_object_init_pt: invalid page %p", p)); pat_mode = p->md.pat_mode; /* * Abort the mapping if the first page is not physically * aligned to a 2MB page boundary. */ ptepa = VM_PAGE_TO_PHYS(p); if (ptepa & (NBPDR - 1)) return; /* * Skip the first page. Abort the mapping if the rest of * the pages are not physically contiguous or have differing * memory attributes. */ p = TAILQ_NEXT(p, listq); for (pa = ptepa + PAGE_SIZE; pa < ptepa + size; pa += PAGE_SIZE) { KASSERT(p->valid == VM_PAGE_BITS_ALL, ("pmap_object_init_pt: invalid page %p", p)); if (pa != VM_PAGE_TO_PHYS(p) || pat_mode != p->md.pat_mode) return; p = TAILQ_NEXT(p, listq); } /* * Map using 2MB pages. Since "ptepa" is 2M aligned and * "size" is a multiple of 2M, adding the PAT setting to "pa" * will not affect the termination of this loop. */ PMAP_LOCK(pmap); for (pa = ptepa | pmap_cache_bits(pmap, pat_mode, 1); pa < ptepa + size; pa += NBPDR) { pdpg = pmap_allocpde(pmap, addr, NULL); if (pdpg == NULL) { /* * The creation of mappings below is only an * optimization. If a page directory page * cannot be allocated without blocking, * continue on to the next mapping rather than * blocking. */ addr += NBPDR; continue; } pde = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(pdpg)); pde = &pde[pmap_pde_index(addr)]; if ((*pde & PG_V) == 0) { pde_store(pde, pa | PG_PS | PG_M | PG_A | PG_U | PG_RW | PG_V); pmap_resident_count_inc(pmap, NBPDR / PAGE_SIZE); atomic_add_long(&pmap_pde_mappings, 1); } else { /* Continue on if the PDE is already valid. */ pdpg->wire_count--; KASSERT(pdpg->wire_count > 0, ("pmap_object_init_pt: missing reference " "to page directory page, va: 0x%lx", addr)); } addr += NBPDR; } PMAP_UNLOCK(pmap); } } /* * Clear the wired attribute from the mappings for the specified range of * addresses in the given pmap. Every valid mapping within that range * must have the wired attribute set. In contrast, invalid mappings * cannot have the wired attribute set, so they are ignored. * * The wired attribute of the page table entry is not a hardware feature, * so there is no need to invalidate any TLB entries. */ void pmap_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) { vm_offset_t va_next; pml4_entry_t *pml4e; pdp_entry_t *pdpe; pd_entry_t *pde; pt_entry_t *pte, PG_V; boolean_t pv_lists_locked; PG_V = pmap_valid_bit(pmap); pv_lists_locked = FALSE; resume: PMAP_LOCK(pmap); for (; sva < eva; sva = va_next) { pml4e = pmap_pml4e(pmap, sva); if ((*pml4e & PG_V) == 0) { va_next = (sva + NBPML4) & ~PML4MASK; if (va_next < sva) va_next = eva; continue; } pdpe = pmap_pml4e_to_pdpe(pml4e, sva); if ((*pdpe & PG_V) == 0) { va_next = (sva + NBPDP) & ~PDPMASK; if (va_next < sva) va_next = eva; continue; } va_next = (sva + NBPDR) & ~PDRMASK; if (va_next < sva) va_next = eva; pde = pmap_pdpe_to_pde(pdpe, sva); if ((*pde & PG_V) == 0) continue; if ((*pde & PG_PS) != 0) { if ((*pde & PG_W) == 0) panic("pmap_unwire: pde %#jx is missing PG_W", (uintmax_t)*pde); /* * Are we unwiring the entire large page? If not, * demote the mapping and fall through. */ if (sva + NBPDR == va_next && eva >= va_next) { atomic_clear_long(pde, PG_W); pmap->pm_stats.wired_count -= NBPDR / PAGE_SIZE; continue; } else { if (!pv_lists_locked) { pv_lists_locked = TRUE; if (!rw_try_rlock(&pvh_global_lock)) { PMAP_UNLOCK(pmap); rw_rlock(&pvh_global_lock); /* Repeat sva. */ goto resume; } } if (!pmap_demote_pde(pmap, pde, sva)) panic("pmap_unwire: demotion failed"); } } if (va_next > eva) va_next = eva; for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++, sva += PAGE_SIZE) { if ((*pte & PG_V) == 0) continue; if ((*pte & PG_W) == 0) panic("pmap_unwire: pte %#jx is missing PG_W", (uintmax_t)*pte); /* * PG_W must be cleared atomically. Although the pmap * lock synchronizes access to PG_W, another processor * could be setting PG_M and/or PG_A concurrently. */ atomic_clear_long(pte, PG_W); pmap->pm_stats.wired_count--; } } if (pv_lists_locked) rw_runlock(&pvh_global_lock); PMAP_UNLOCK(pmap); } /* * Copy the range specified by src_addr/len * from the source map to the range dst_addr/len * in the destination map. * * This routine is only advisory and need not do anything. */ void pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len, vm_offset_t src_addr) { struct rwlock *lock; struct spglist free; vm_offset_t addr; vm_offset_t end_addr = src_addr + len; vm_offset_t va_next; pt_entry_t PG_A, PG_M, PG_V; if (dst_addr != src_addr) return; if (dst_pmap->pm_type != src_pmap->pm_type) return; /* * EPT page table entries that require emulation of A/D bits are * sensitive to clearing the PG_A bit (aka EPT_PG_READ). Although * we clear PG_M (aka EPT_PG_WRITE) concomitantly, the PG_U bit * (aka EPT_PG_EXECUTE) could still be set. Since some EPT * implementations flag an EPT misconfiguration for exec-only * mappings we skip this function entirely for emulated pmaps. */ if (pmap_emulate_ad_bits(dst_pmap)) return; lock = NULL; rw_rlock(&pvh_global_lock); if (dst_pmap < src_pmap) { PMAP_LOCK(dst_pmap); PMAP_LOCK(src_pmap); } else { PMAP_LOCK(src_pmap); PMAP_LOCK(dst_pmap); } PG_A = pmap_accessed_bit(dst_pmap); PG_M = pmap_modified_bit(dst_pmap); PG_V = pmap_valid_bit(dst_pmap); for (addr = src_addr; addr < end_addr; addr = va_next) { pt_entry_t *src_pte, *dst_pte; vm_page_t dstmpde, dstmpte, srcmpte; pml4_entry_t *pml4e; pdp_entry_t *pdpe; pd_entry_t srcptepaddr, *pde; KASSERT(addr < UPT_MIN_ADDRESS, ("pmap_copy: invalid to pmap_copy page tables")); pml4e = pmap_pml4e(src_pmap, addr); if ((*pml4e & PG_V) == 0) { va_next = (addr + NBPML4) & ~PML4MASK; if (va_next < addr) va_next = end_addr; continue; } pdpe = pmap_pml4e_to_pdpe(pml4e, addr); if ((*pdpe & PG_V) == 0) { va_next = (addr + NBPDP) & ~PDPMASK; if (va_next < addr) va_next = end_addr; continue; } va_next = (addr + NBPDR) & ~PDRMASK; if (va_next < addr) va_next = end_addr; pde = pmap_pdpe_to_pde(pdpe, addr); srcptepaddr = *pde; if (srcptepaddr == 0) continue; if (srcptepaddr & PG_PS) { if ((addr & PDRMASK) != 0 || addr + NBPDR > end_addr) continue; dstmpde = pmap_allocpde(dst_pmap, addr, NULL); if (dstmpde == NULL) break; pde = (pd_entry_t *) PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dstmpde)); pde = &pde[pmap_pde_index(addr)]; if (*pde == 0 && ((srcptepaddr & PG_MANAGED) == 0 || pmap_pv_insert_pde(dst_pmap, addr, srcptepaddr & PG_PS_FRAME, &lock))) { *pde = srcptepaddr & ~PG_W; pmap_resident_count_inc(dst_pmap, NBPDR / PAGE_SIZE); atomic_add_long(&pmap_pde_mappings, 1); } else dstmpde->wire_count--; continue; } srcptepaddr &= PG_FRAME; srcmpte = PHYS_TO_VM_PAGE(srcptepaddr); KASSERT(srcmpte->wire_count > 0, ("pmap_copy: source page table page is unused")); if (va_next > end_addr) va_next = end_addr; src_pte = (pt_entry_t *)PHYS_TO_DMAP(srcptepaddr); src_pte = &src_pte[pmap_pte_index(addr)]; dstmpte = NULL; while (addr < va_next) { pt_entry_t ptetemp; ptetemp = *src_pte; /* * we only virtual copy managed pages */ if ((ptetemp & PG_MANAGED) != 0) { if (dstmpte != NULL && dstmpte->pindex == pmap_pde_pindex(addr)) dstmpte->wire_count++; else if ((dstmpte = pmap_allocpte(dst_pmap, addr, NULL)) == NULL) goto out; dst_pte = (pt_entry_t *) PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dstmpte)); dst_pte = &dst_pte[pmap_pte_index(addr)]; if (*dst_pte == 0 && pmap_try_insert_pv_entry(dst_pmap, addr, PHYS_TO_VM_PAGE(ptetemp & PG_FRAME), &lock)) { /* * Clear the wired, modified, and * accessed (referenced) bits * during the copy. */ *dst_pte = ptetemp & ~(PG_W | PG_M | PG_A); pmap_resident_count_inc(dst_pmap, 1); } else { SLIST_INIT(&free); if (pmap_unwire_ptp(dst_pmap, addr, dstmpte, &free)) { + /* + * Although "addr" is not + * mapped, paging-structure + * caches could nonetheless + * have entries that refer to + * the freed page table pages. + * Invalidate those entries. + */ pmap_invalidate_page(dst_pmap, addr); pmap_free_zero_pages(&free); } goto out; } if (dstmpte->wire_count >= srcmpte->wire_count) break; } addr += PAGE_SIZE; src_pte++; } } out: if (lock != NULL) rw_wunlock(lock); rw_runlock(&pvh_global_lock); PMAP_UNLOCK(src_pmap); PMAP_UNLOCK(dst_pmap); } /* * pmap_zero_page zeros the specified hardware page by mapping * the page into KVM and using bzero to clear its contents. */ void pmap_zero_page(vm_page_t m) { vm_offset_t va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); pagezero((void *)va); } /* * pmap_zero_page_area zeros the specified hardware page by mapping * the page into KVM and using bzero to clear its contents. * * off and size may not cover an area beyond a single hardware page. */ void pmap_zero_page_area(vm_page_t m, int off, int size) { vm_offset_t va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); if (off == 0 && size == PAGE_SIZE) pagezero((void *)va); else bzero((char *)va + off, size); } /* * pmap_zero_page_idle zeros the specified hardware page by mapping * the page into KVM and using bzero to clear its contents. This * is intended to be called from the vm_pagezero process only and * outside of Giant. */ void pmap_zero_page_idle(vm_page_t m) { vm_offset_t va = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)); pagezero((void *)va); } /* * pmap_copy_page copies the specified (machine independent) * page by mapping the page into virtual memory and using * bcopy to copy the page, one machine dependent page at a * time. */ void pmap_copy_page(vm_page_t msrc, vm_page_t mdst) { vm_offset_t src = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(msrc)); vm_offset_t dst = PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mdst)); pagecopy((void *)src, (void *)dst); } int unmapped_buf_allowed = 1; void pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[], vm_offset_t b_offset, int xfersize) { void *a_cp, *b_cp; vm_page_t pages[2]; vm_offset_t vaddr[2], a_pg_offset, b_pg_offset; int cnt; boolean_t mapped; while (xfersize > 0) { a_pg_offset = a_offset & PAGE_MASK; pages[0] = ma[a_offset >> PAGE_SHIFT]; b_pg_offset = b_offset & PAGE_MASK; pages[1] = mb[b_offset >> PAGE_SHIFT]; cnt = min(xfersize, PAGE_SIZE - a_pg_offset); cnt = min(cnt, PAGE_SIZE - b_pg_offset); mapped = pmap_map_io_transient(pages, vaddr, 2, FALSE); a_cp = (char *)vaddr[0] + a_pg_offset; b_cp = (char *)vaddr[1] + b_pg_offset; bcopy(a_cp, b_cp, cnt); if (__predict_false(mapped)) pmap_unmap_io_transient(pages, vaddr, 2, FALSE); a_offset += cnt; b_offset += cnt; xfersize -= cnt; } } /* * Returns true if the pmap's pv is one of the first * 16 pvs linked to from this page. This count may * be changed upwards or downwards in the future; it * is only necessary that true be returned for a small * subset of pmaps for proper page aging. */ boolean_t pmap_page_exists_quick(pmap_t pmap, vm_page_t m) { struct md_page *pvh; struct rwlock *lock; pv_entry_t pv; int loops = 0; boolean_t rv; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_page_exists_quick: page %p is not managed", m)); rv = FALSE; rw_rlock(&pvh_global_lock); lock = VM_PAGE_TO_PV_LIST_LOCK(m); rw_rlock(lock); TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { if (PV_PMAP(pv) == pmap) { rv = TRUE; break; } loops++; if (loops >= 16) break; } if (!rv && loops < 16 && (m->flags & PG_FICTITIOUS) == 0) { pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { if (PV_PMAP(pv) == pmap) { rv = TRUE; break; } loops++; if (loops >= 16) break; } } rw_runlock(lock); rw_runlock(&pvh_global_lock); return (rv); } /* * pmap_page_wired_mappings: * * Return the number of managed mappings to the given physical page * that are wired. */ int pmap_page_wired_mappings(vm_page_t m) { struct rwlock *lock; struct md_page *pvh; pmap_t pmap; pt_entry_t *pte; pv_entry_t pv; int count, md_gen, pvh_gen; if ((m->oflags & VPO_UNMANAGED) != 0) return (0); rw_rlock(&pvh_global_lock); lock = VM_PAGE_TO_PV_LIST_LOCK(m); rw_rlock(lock); restart: count = 0; TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { pmap = PV_PMAP(pv); if (!PMAP_TRYLOCK(pmap)) { md_gen = m->md.pv_gen; rw_runlock(lock); PMAP_LOCK(pmap); rw_rlock(lock); if (md_gen != m->md.pv_gen) { PMAP_UNLOCK(pmap); goto restart; } } pte = pmap_pte(pmap, pv->pv_va); if ((*pte & PG_W) != 0) count++; PMAP_UNLOCK(pmap); } if ((m->flags & PG_FICTITIOUS) == 0) { pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { pmap = PV_PMAP(pv); if (!PMAP_TRYLOCK(pmap)) { md_gen = m->md.pv_gen; pvh_gen = pvh->pv_gen; rw_runlock(lock); PMAP_LOCK(pmap); rw_rlock(lock); if (md_gen != m->md.pv_gen || pvh_gen != pvh->pv_gen) { PMAP_UNLOCK(pmap); goto restart; } } pte = pmap_pde(pmap, pv->pv_va); if ((*pte & PG_W) != 0) count++; PMAP_UNLOCK(pmap); } } rw_runlock(lock); rw_runlock(&pvh_global_lock); return (count); } /* * Returns TRUE if the given page is mapped individually or as part of * a 2mpage. Otherwise, returns FALSE. */ boolean_t pmap_page_is_mapped(vm_page_t m) { struct rwlock *lock; boolean_t rv; if ((m->oflags & VPO_UNMANAGED) != 0) return (FALSE); rw_rlock(&pvh_global_lock); lock = VM_PAGE_TO_PV_LIST_LOCK(m); rw_rlock(lock); rv = !TAILQ_EMPTY(&m->md.pv_list) || ((m->flags & PG_FICTITIOUS) == 0 && !TAILQ_EMPTY(&pa_to_pvh(VM_PAGE_TO_PHYS(m))->pv_list)); rw_runlock(lock); rw_runlock(&pvh_global_lock); return (rv); } /* * Destroy all managed, non-wired mappings in the given user-space * pmap. This pmap cannot be active on any processor besides the * caller. - * + * * This function cannot be applied to the kernel pmap. Moreover, it * is not intended for general use. It is only to be used during * process termination. Consequently, it can be implemented in ways * that make it faster than pmap_remove(). First, it can more quickly * destroy mappings by iterating over the pmap's collection of PV * entries, rather than searching the page table. Second, it doesn't * have to test and clear the page table entries atomically, because * no processor is currently accessing the user address space. In * particular, a page table entry's dirty bit won't change state once * this function starts. */ void pmap_remove_pages(pmap_t pmap) { pd_entry_t ptepde; pt_entry_t *pte, tpte; pt_entry_t PG_M, PG_RW, PG_V; struct spglist free; vm_page_t m, mpte, mt; pv_entry_t pv; struct md_page *pvh; struct pv_chunk *pc, *npc; struct rwlock *lock; int64_t bit; uint64_t inuse, bitmask; int allfree, field, freed, idx; boolean_t superpage; vm_paddr_t pa; /* * Assert that the given pmap is only active on the current * CPU. Unfortunately, we cannot block another CPU from * activating the pmap while this function is executing. */ KASSERT(pmap == PCPU_GET(curpmap), ("non-current pmap %p", pmap)); #ifdef INVARIANTS { cpuset_t other_cpus; other_cpus = all_cpus; critical_enter(); CPU_CLR(PCPU_GET(cpuid), &other_cpus); CPU_AND(&other_cpus, &pmap->pm_active); critical_exit(); KASSERT(CPU_EMPTY(&other_cpus), ("pmap active %p", pmap)); } #endif lock = NULL; PG_M = pmap_modified_bit(pmap); PG_V = pmap_valid_bit(pmap); PG_RW = pmap_rw_bit(pmap); SLIST_INIT(&free); rw_rlock(&pvh_global_lock); PMAP_LOCK(pmap); TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) { allfree = 1; freed = 0; for (field = 0; field < _NPCM; field++) { inuse = ~pc->pc_map[field] & pc_freemask[field]; while (inuse != 0) { bit = bsfq(inuse); bitmask = 1UL << bit; idx = field * 64 + bit; pv = &pc->pc_pventry[idx]; inuse &= ~bitmask; pte = pmap_pdpe(pmap, pv->pv_va); ptepde = *pte; pte = pmap_pdpe_to_pde(pte, pv->pv_va); tpte = *pte; if ((tpte & (PG_PS | PG_V)) == PG_V) { superpage = FALSE; ptepde = tpte; pte = (pt_entry_t *)PHYS_TO_DMAP(tpte & PG_FRAME); pte = &pte[pmap_pte_index(pv->pv_va)]; tpte = *pte; } else { /* * Keep track whether 'tpte' is a * superpage explicitly instead of * relying on PG_PS being set. * * This is because PG_PS is numerically * identical to PG_PTE_PAT and thus a * regular page could be mistaken for * a superpage. */ superpage = TRUE; } if ((tpte & PG_V) == 0) { panic("bad pte va %lx pte %lx", pv->pv_va, tpte); } /* * We cannot remove wired pages from a process' mapping at this time */ if (tpte & PG_W) { allfree = 0; continue; } if (superpage) pa = tpte & PG_PS_FRAME; else pa = tpte & PG_FRAME; m = PHYS_TO_VM_PAGE(pa); KASSERT(m->phys_addr == pa, ("vm_page_t %p phys_addr mismatch %016jx %016jx", m, (uintmax_t)m->phys_addr, (uintmax_t)tpte)); KASSERT((m->flags & PG_FICTITIOUS) != 0 || m < &vm_page_array[vm_page_array_size], ("pmap_remove_pages: bad tpte %#jx", (uintmax_t)tpte)); pte_clear(pte); /* * Update the vm_page_t clean/reference bits. */ if ((tpte & (PG_M | PG_RW)) == (PG_M | PG_RW)) { if (superpage) { for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++) vm_page_dirty(mt); } else vm_page_dirty(m); } CHANGE_PV_LIST_LOCK_TO_VM_PAGE(&lock, m); /* Mark free */ pc->pc_map[field] |= bitmask; if (superpage) { pmap_resident_count_dec(pmap, NBPDR / PAGE_SIZE); pvh = pa_to_pvh(tpte & PG_PS_FRAME); TAILQ_REMOVE(&pvh->pv_list, pv, pv_next); pvh->pv_gen++; if (TAILQ_EMPTY(&pvh->pv_list)) { for (mt = m; mt < &m[NBPDR / PAGE_SIZE]; mt++) if ((mt->aflags & PGA_WRITEABLE) != 0 && TAILQ_EMPTY(&mt->md.pv_list)) vm_page_aflag_clear(mt, PGA_WRITEABLE); } mpte = pmap_lookup_pt_page(pmap, pv->pv_va); if (mpte != NULL) { pmap_remove_pt_page(pmap, mpte); pmap_resident_count_dec(pmap, 1); KASSERT(mpte->wire_count == NPTEPG, ("pmap_remove_pages: pte page wire count error")); mpte->wire_count = 0; pmap_add_delayed_free_list(mpte, &free, FALSE); atomic_subtract_int(&vm_cnt.v_wire_count, 1); } } else { pmap_resident_count_dec(pmap, 1); TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); m->md.pv_gen++; if ((m->aflags & PGA_WRITEABLE) != 0 && TAILQ_EMPTY(&m->md.pv_list) && (m->flags & PG_FICTITIOUS) == 0) { pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); if (TAILQ_EMPTY(&pvh->pv_list)) vm_page_aflag_clear(m, PGA_WRITEABLE); } } pmap_unuse_pt(pmap, pv->pv_va, ptepde, &free); freed++; } } PV_STAT(atomic_add_long(&pv_entry_frees, freed)); PV_STAT(atomic_add_int(&pv_entry_spare, freed)); PV_STAT(atomic_subtract_long(&pv_entry_count, freed)); if (allfree) { TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); free_pv_chunk(pc); } } if (lock != NULL) rw_wunlock(lock); pmap_invalidate_all(pmap); rw_runlock(&pvh_global_lock); PMAP_UNLOCK(pmap); pmap_free_zero_pages(&free); } static boolean_t pmap_page_test_mappings(vm_page_t m, boolean_t accessed, boolean_t modified) { struct rwlock *lock; pv_entry_t pv; struct md_page *pvh; pt_entry_t *pte, mask; pt_entry_t PG_A, PG_M, PG_RW, PG_V; pmap_t pmap; int md_gen, pvh_gen; boolean_t rv; rv = FALSE; rw_rlock(&pvh_global_lock); lock = VM_PAGE_TO_PV_LIST_LOCK(m); rw_rlock(lock); restart: TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { pmap = PV_PMAP(pv); if (!PMAP_TRYLOCK(pmap)) { md_gen = m->md.pv_gen; rw_runlock(lock); PMAP_LOCK(pmap); rw_rlock(lock); if (md_gen != m->md.pv_gen) { PMAP_UNLOCK(pmap); goto restart; } } pte = pmap_pte(pmap, pv->pv_va); mask = 0; if (modified) { PG_M = pmap_modified_bit(pmap); PG_RW = pmap_rw_bit(pmap); mask |= PG_RW | PG_M; } if (accessed) { PG_A = pmap_accessed_bit(pmap); PG_V = pmap_valid_bit(pmap); mask |= PG_V | PG_A; } rv = (*pte & mask) == mask; PMAP_UNLOCK(pmap); if (rv) goto out; } if ((m->flags & PG_FICTITIOUS) == 0) { pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { pmap = PV_PMAP(pv); if (!PMAP_TRYLOCK(pmap)) { md_gen = m->md.pv_gen; pvh_gen = pvh->pv_gen; rw_runlock(lock); PMAP_LOCK(pmap); rw_rlock(lock); if (md_gen != m->md.pv_gen || pvh_gen != pvh->pv_gen) { PMAP_UNLOCK(pmap); goto restart; } } pte = pmap_pde(pmap, pv->pv_va); mask = 0; if (modified) { PG_M = pmap_modified_bit(pmap); PG_RW = pmap_rw_bit(pmap); mask |= PG_RW | PG_M; } if (accessed) { PG_A = pmap_accessed_bit(pmap); PG_V = pmap_valid_bit(pmap); mask |= PG_V | PG_A; } rv = (*pte & mask) == mask; PMAP_UNLOCK(pmap); if (rv) goto out; } } out: rw_runlock(lock); rw_runlock(&pvh_global_lock); return (rv); } /* * pmap_is_modified: * * Return whether or not the specified physical page was modified * in any physical maps. */ boolean_t pmap_is_modified(vm_page_t m) { KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_is_modified: page %p is not managed", m)); /* * If the page is not exclusive busied, then PGA_WRITEABLE cannot be * concurrently set while the object is locked. Thus, if PGA_WRITEABLE * is clear, no PTEs can have PG_M set. */ VM_OBJECT_ASSERT_WLOCKED(m->object); if (!vm_page_xbusied(m) && (m->aflags & PGA_WRITEABLE) == 0) return (FALSE); return (pmap_page_test_mappings(m, FALSE, TRUE)); } /* * pmap_is_prefaultable: * * Return whether or not the specified virtual address is eligible * for prefault. */ boolean_t pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr) { pd_entry_t *pde; pt_entry_t *pte, PG_V; boolean_t rv; PG_V = pmap_valid_bit(pmap); rv = FALSE; PMAP_LOCK(pmap); pde = pmap_pde(pmap, addr); if (pde != NULL && (*pde & (PG_PS | PG_V)) == PG_V) { pte = pmap_pde_to_pte(pde, addr); rv = (*pte & PG_V) == 0; } PMAP_UNLOCK(pmap); return (rv); } /* * pmap_is_referenced: * * Return whether or not the specified physical page was referenced * in any physical maps. */ boolean_t pmap_is_referenced(vm_page_t m) { KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_is_referenced: page %p is not managed", m)); return (pmap_page_test_mappings(m, TRUE, FALSE)); } /* * Clear the write and modified bits in each of the given page's mappings. */ void pmap_remove_write(vm_page_t m) { struct md_page *pvh; pmap_t pmap; struct rwlock *lock; pv_entry_t next_pv, pv; pd_entry_t *pde; pt_entry_t oldpte, *pte, PG_M, PG_RW; vm_offset_t va; int pvh_gen, md_gen; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_remove_write: page %p is not managed", m)); /* * If the page is not exclusive busied, then PGA_WRITEABLE cannot be * set by another thread while the object is locked. Thus, * if PGA_WRITEABLE is clear, no page table entries need updating. */ VM_OBJECT_ASSERT_WLOCKED(m->object); if (!vm_page_xbusied(m) && (m->aflags & PGA_WRITEABLE) == 0) return; rw_rlock(&pvh_global_lock); lock = VM_PAGE_TO_PV_LIST_LOCK(m); pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); retry_pv_loop: rw_wlock(lock); if ((m->flags & PG_FICTITIOUS) != 0) goto small_mappings; TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_next, next_pv) { pmap = PV_PMAP(pv); if (!PMAP_TRYLOCK(pmap)) { pvh_gen = pvh->pv_gen; rw_wunlock(lock); PMAP_LOCK(pmap); rw_wlock(lock); if (pvh_gen != pvh->pv_gen) { PMAP_UNLOCK(pmap); rw_wunlock(lock); goto retry_pv_loop; } } PG_RW = pmap_rw_bit(pmap); va = pv->pv_va; pde = pmap_pde(pmap, va); if ((*pde & PG_RW) != 0) (void)pmap_demote_pde_locked(pmap, pde, va, &lock); KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m), ("inconsistent pv lock %p %p for page %p", lock, VM_PAGE_TO_PV_LIST_LOCK(m), m)); PMAP_UNLOCK(pmap); } small_mappings: TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { pmap = PV_PMAP(pv); if (!PMAP_TRYLOCK(pmap)) { pvh_gen = pvh->pv_gen; md_gen = m->md.pv_gen; rw_wunlock(lock); PMAP_LOCK(pmap); rw_wlock(lock); if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) { PMAP_UNLOCK(pmap); rw_wunlock(lock); goto retry_pv_loop; } } PG_M = pmap_modified_bit(pmap); PG_RW = pmap_rw_bit(pmap); pde = pmap_pde(pmap, pv->pv_va); KASSERT((*pde & PG_PS) == 0, ("pmap_remove_write: found a 2mpage in page %p's pv list", m)); pte = pmap_pde_to_pte(pde, pv->pv_va); retry: oldpte = *pte; if (oldpte & PG_RW) { if (!atomic_cmpset_long(pte, oldpte, oldpte & ~(PG_RW | PG_M))) goto retry; if ((oldpte & PG_M) != 0) vm_page_dirty(m); pmap_invalidate_page(pmap, pv->pv_va); } PMAP_UNLOCK(pmap); } rw_wunlock(lock); vm_page_aflag_clear(m, PGA_WRITEABLE); rw_runlock(&pvh_global_lock); } static __inline boolean_t safe_to_clear_referenced(pmap_t pmap, pt_entry_t pte) { if (!pmap_emulate_ad_bits(pmap)) return (TRUE); KASSERT(pmap->pm_type == PT_EPT, ("invalid pm_type %d", pmap->pm_type)); /* * XWR = 010 or 110 will cause an unconditional EPT misconfiguration * so we don't let the referenced (aka EPT_PG_READ) bit to be cleared * if the EPT_PG_WRITE bit is set. */ if ((pte & EPT_PG_WRITE) != 0) return (FALSE); /* * XWR = 100 is allowed only if the PMAP_SUPPORTS_EXEC_ONLY is set. */ if ((pte & EPT_PG_EXECUTE) == 0 || ((pmap->pm_flags & PMAP_SUPPORTS_EXEC_ONLY) != 0)) return (TRUE); else return (FALSE); } #define PMAP_TS_REFERENCED_MAX 5 /* * pmap_ts_referenced: * * Return a count of reference bits for a page, clearing those bits. * It is not necessary for every reference bit to be cleared, but it * is necessary that 0 only be returned when there are truly no * reference bits set. * * XXX: The exact number of bits to check and clear is a matter that * should be tested and standardized at some point in the future for * optimal aging of shared pages. */ int pmap_ts_referenced(vm_page_t m) { struct md_page *pvh; pv_entry_t pv, pvf; pmap_t pmap; struct rwlock *lock; pd_entry_t oldpde, *pde; pt_entry_t *pte, PG_A; vm_offset_t va; vm_paddr_t pa; int cleared, md_gen, not_cleared, pvh_gen; struct spglist free; boolean_t demoted; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_ts_referenced: page %p is not managed", m)); SLIST_INIT(&free); cleared = 0; pa = VM_PAGE_TO_PHYS(m); lock = PHYS_TO_PV_LIST_LOCK(pa); pvh = pa_to_pvh(pa); rw_rlock(&pvh_global_lock); rw_wlock(lock); retry: not_cleared = 0; if ((m->flags & PG_FICTITIOUS) != 0 || (pvf = TAILQ_FIRST(&pvh->pv_list)) == NULL) goto small_mappings; pv = pvf; do { if (pvf == NULL) pvf = pv; pmap = PV_PMAP(pv); if (!PMAP_TRYLOCK(pmap)) { pvh_gen = pvh->pv_gen; rw_wunlock(lock); PMAP_LOCK(pmap); rw_wlock(lock); if (pvh_gen != pvh->pv_gen) { PMAP_UNLOCK(pmap); goto retry; } } PG_A = pmap_accessed_bit(pmap); va = pv->pv_va; pde = pmap_pde(pmap, pv->pv_va); oldpde = *pde; if ((*pde & PG_A) != 0) { /* * Since this reference bit is shared by 512 4KB * pages, it should not be cleared every time it is * tested. Apply a simple "hash" function on the * physical page number, the virtual superpage number, * and the pmap address to select one 4KB page out of * the 512 on which testing the reference bit will * result in clearing that reference bit. This * function is designed to avoid the selection of the * same 4KB page for every 2MB page mapping. * * On demotion, a mapping that hasn't been referenced * is simply destroyed. To avoid the possibility of a * subsequent page fault on a demoted wired mapping, * always leave its reference bit set. Moreover, * since the superpage is wired, the current state of * its reference bit won't affect page replacement. */ if ((((pa >> PAGE_SHIFT) ^ (pv->pv_va >> PDRSHIFT) ^ (uintptr_t)pmap) & (NPTEPG - 1)) == 0 && (*pde & PG_W) == 0) { if (safe_to_clear_referenced(pmap, oldpde)) { atomic_clear_long(pde, PG_A); pmap_invalidate_page(pmap, pv->pv_va); demoted = FALSE; } else if (pmap_demote_pde_locked(pmap, pde, pv->pv_va, &lock)) { /* * Remove the mapping to a single page * so that a subsequent access may * repromote. Since the underlying * page table page is fully populated, * this removal never frees a page * table page. */ demoted = TRUE; va += VM_PAGE_TO_PHYS(m) - (oldpde & PG_PS_FRAME); pte = pmap_pde_to_pte(pde, va); pmap_remove_pte(pmap, pte, va, *pde, NULL, &lock); pmap_invalidate_page(pmap, va); } else demoted = TRUE; if (demoted) { /* * The superpage mapping was removed * entirely and therefore 'pv' is no * longer valid. */ if (pvf == pv) pvf = NULL; pv = NULL; } cleared++; KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m), ("inconsistent pv lock %p %p for page %p", lock, VM_PAGE_TO_PV_LIST_LOCK(m), m)); } else not_cleared++; } PMAP_UNLOCK(pmap); /* Rotate the PV list if it has more than one entry. */ if (pv != NULL && TAILQ_NEXT(pv, pv_next) != NULL) { TAILQ_REMOVE(&pvh->pv_list, pv, pv_next); TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next); pvh->pv_gen++; } if (cleared + not_cleared >= PMAP_TS_REFERENCED_MAX) goto out; } while ((pv = TAILQ_FIRST(&pvh->pv_list)) != pvf); small_mappings: if ((pvf = TAILQ_FIRST(&m->md.pv_list)) == NULL) goto out; pv = pvf; do { if (pvf == NULL) pvf = pv; pmap = PV_PMAP(pv); if (!PMAP_TRYLOCK(pmap)) { pvh_gen = pvh->pv_gen; md_gen = m->md.pv_gen; rw_wunlock(lock); PMAP_LOCK(pmap); rw_wlock(lock); if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) { PMAP_UNLOCK(pmap); goto retry; } } PG_A = pmap_accessed_bit(pmap); pde = pmap_pde(pmap, pv->pv_va); KASSERT((*pde & PG_PS) == 0, ("pmap_ts_referenced: found a 2mpage in page %p's pv list", m)); pte = pmap_pde_to_pte(pde, pv->pv_va); if ((*pte & PG_A) != 0) { if (safe_to_clear_referenced(pmap, *pte)) { atomic_clear_long(pte, PG_A); pmap_invalidate_page(pmap, pv->pv_va); cleared++; } else if ((*pte & PG_W) == 0) { /* * Wired pages cannot be paged out so * doing accessed bit emulation for * them is wasted effort. We do the * hard work for unwired pages only. */ pmap_remove_pte(pmap, pte, pv->pv_va, *pde, &free, &lock); pmap_invalidate_page(pmap, pv->pv_va); cleared++; if (pvf == pv) pvf = NULL; pv = NULL; KASSERT(lock == VM_PAGE_TO_PV_LIST_LOCK(m), ("inconsistent pv lock %p %p for page %p", lock, VM_PAGE_TO_PV_LIST_LOCK(m), m)); } else not_cleared++; } PMAP_UNLOCK(pmap); /* Rotate the PV list if it has more than one entry. */ if (pv != NULL && TAILQ_NEXT(pv, pv_next) != NULL) { TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); m->md.pv_gen++; } } while ((pv = TAILQ_FIRST(&m->md.pv_list)) != pvf && cleared + not_cleared < PMAP_TS_REFERENCED_MAX); out: rw_wunlock(lock); rw_runlock(&pvh_global_lock); pmap_free_zero_pages(&free); return (cleared + not_cleared); } /* * Apply the given advice to the specified range of addresses within the * given pmap. Depending on the advice, clear the referenced and/or * modified flags in each mapping and set the mapped page's dirty field. */ void pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice) { struct rwlock *lock; pml4_entry_t *pml4e; pdp_entry_t *pdpe; pd_entry_t oldpde, *pde; pt_entry_t *pte, PG_A, PG_G, PG_M, PG_RW, PG_V; vm_offset_t va_next; vm_page_t m; boolean_t anychanged, pv_lists_locked; if (advice != MADV_DONTNEED && advice != MADV_FREE) return; /* * A/D bit emulation requires an alternate code path when clearing * the modified and accessed bits below. Since this function is * advisory in nature we skip it entirely for pmaps that require * A/D bit emulation. */ if (pmap_emulate_ad_bits(pmap)) return; PG_A = pmap_accessed_bit(pmap); PG_G = pmap_global_bit(pmap); PG_M = pmap_modified_bit(pmap); PG_V = pmap_valid_bit(pmap); PG_RW = pmap_rw_bit(pmap); pv_lists_locked = FALSE; resume: anychanged = FALSE; PMAP_LOCK(pmap); for (; sva < eva; sva = va_next) { pml4e = pmap_pml4e(pmap, sva); if ((*pml4e & PG_V) == 0) { va_next = (sva + NBPML4) & ~PML4MASK; if (va_next < sva) va_next = eva; continue; } pdpe = pmap_pml4e_to_pdpe(pml4e, sva); if ((*pdpe & PG_V) == 0) { va_next = (sva + NBPDP) & ~PDPMASK; if (va_next < sva) va_next = eva; continue; } va_next = (sva + NBPDR) & ~PDRMASK; if (va_next < sva) va_next = eva; pde = pmap_pdpe_to_pde(pdpe, sva); oldpde = *pde; if ((oldpde & PG_V) == 0) continue; else if ((oldpde & PG_PS) != 0) { if ((oldpde & PG_MANAGED) == 0) continue; if (!pv_lists_locked) { pv_lists_locked = TRUE; if (!rw_try_rlock(&pvh_global_lock)) { if (anychanged) pmap_invalidate_all(pmap); PMAP_UNLOCK(pmap); rw_rlock(&pvh_global_lock); goto resume; } } lock = NULL; if (!pmap_demote_pde_locked(pmap, pde, sva, &lock)) { if (lock != NULL) rw_wunlock(lock); /* * The large page mapping was destroyed. */ continue; } /* * Unless the page mappings are wired, remove the * mapping to a single page so that a subsequent * access may repromote. Since the underlying page * table page is fully populated, this removal never * frees a page table page. */ if ((oldpde & PG_W) == 0) { pte = pmap_pde_to_pte(pde, sva); KASSERT((*pte & PG_V) != 0, ("pmap_advise: invalid PTE")); pmap_remove_pte(pmap, pte, sva, *pde, NULL, &lock); anychanged = TRUE; } if (lock != NULL) rw_wunlock(lock); } if (va_next > eva) va_next = eva; for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++, sva += PAGE_SIZE) { if ((*pte & (PG_MANAGED | PG_V)) != (PG_MANAGED | PG_V)) continue; else if ((*pte & (PG_M | PG_RW)) == (PG_M | PG_RW)) { if (advice == MADV_DONTNEED) { /* * Future calls to pmap_is_modified() * can be avoided by making the page * dirty now. */ m = PHYS_TO_VM_PAGE(*pte & PG_FRAME); vm_page_dirty(m); } atomic_clear_long(pte, PG_M | PG_A); } else if ((*pte & PG_A) != 0) atomic_clear_long(pte, PG_A); else continue; if ((*pte & PG_G) != 0) pmap_invalidate_page(pmap, sva); else anychanged = TRUE; } } if (anychanged) pmap_invalidate_all(pmap); if (pv_lists_locked) rw_runlock(&pvh_global_lock); PMAP_UNLOCK(pmap); } /* * Clear the modify bits on the specified physical page. */ void pmap_clear_modify(vm_page_t m) { struct md_page *pvh; pmap_t pmap; pv_entry_t next_pv, pv; pd_entry_t oldpde, *pde; pt_entry_t oldpte, *pte, PG_M, PG_RW, PG_V; struct rwlock *lock; vm_offset_t va; int md_gen, pvh_gen; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_clear_modify: page %p is not managed", m)); VM_OBJECT_ASSERT_WLOCKED(m->object); KASSERT(!vm_page_xbusied(m), ("pmap_clear_modify: page %p is exclusive busied", m)); /* * If the page is not PGA_WRITEABLE, then no PTEs can have PG_M set. * If the object containing the page is locked and the page is not * exclusive busied, then PGA_WRITEABLE cannot be concurrently set. */ if ((m->aflags & PGA_WRITEABLE) == 0) return; pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); rw_rlock(&pvh_global_lock); lock = VM_PAGE_TO_PV_LIST_LOCK(m); rw_wlock(lock); restart: if ((m->flags & PG_FICTITIOUS) != 0) goto small_mappings; TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_next, next_pv) { pmap = PV_PMAP(pv); if (!PMAP_TRYLOCK(pmap)) { pvh_gen = pvh->pv_gen; rw_wunlock(lock); PMAP_LOCK(pmap); rw_wlock(lock); if (pvh_gen != pvh->pv_gen) { PMAP_UNLOCK(pmap); goto restart; } } PG_M = pmap_modified_bit(pmap); PG_V = pmap_valid_bit(pmap); PG_RW = pmap_rw_bit(pmap); va = pv->pv_va; pde = pmap_pde(pmap, va); oldpde = *pde; if ((oldpde & PG_RW) != 0) { if (pmap_demote_pde_locked(pmap, pde, va, &lock)) { if ((oldpde & PG_W) == 0) { /* * Write protect the mapping to a * single page so that a subsequent * write access may repromote. */ va += VM_PAGE_TO_PHYS(m) - (oldpde & PG_PS_FRAME); pte = pmap_pde_to_pte(pde, va); oldpte = *pte; if ((oldpte & PG_V) != 0) { while (!atomic_cmpset_long(pte, oldpte, oldpte & ~(PG_M | PG_RW))) oldpte = *pte; vm_page_dirty(m); pmap_invalidate_page(pmap, va); } } } } PMAP_UNLOCK(pmap); } small_mappings: TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { pmap = PV_PMAP(pv); if (!PMAP_TRYLOCK(pmap)) { md_gen = m->md.pv_gen; pvh_gen = pvh->pv_gen; rw_wunlock(lock); PMAP_LOCK(pmap); rw_wlock(lock); if (pvh_gen != pvh->pv_gen || md_gen != m->md.pv_gen) { PMAP_UNLOCK(pmap); goto restart; } } PG_M = pmap_modified_bit(pmap); PG_RW = pmap_rw_bit(pmap); pde = pmap_pde(pmap, pv->pv_va); KASSERT((*pde & PG_PS) == 0, ("pmap_clear_modify: found" " a 2mpage in page %p's pv list", m)); pte = pmap_pde_to_pte(pde, pv->pv_va); if ((*pte & (PG_M | PG_RW)) == (PG_M | PG_RW)) { atomic_clear_long(pte, PG_M); pmap_invalidate_page(pmap, pv->pv_va); } PMAP_UNLOCK(pmap); } rw_wunlock(lock); rw_runlock(&pvh_global_lock); } /* * Miscellaneous support routines follow */ /* Adjust the cache mode for a 4KB page mapped via a PTE. */ static __inline void pmap_pte_attr(pt_entry_t *pte, int cache_bits, int mask) { u_int opte, npte; /* * The cache mode bits are all in the low 32-bits of the * PTE, so we can just spin on updating the low 32-bits. */ do { opte = *(u_int *)pte; npte = opte & ~mask; npte |= cache_bits; } while (npte != opte && !atomic_cmpset_int((u_int *)pte, opte, npte)); } /* Adjust the cache mode for a 2MB page mapped via a PDE. */ static __inline void pmap_pde_attr(pd_entry_t *pde, int cache_bits, int mask) { u_int opde, npde; /* * The cache mode bits are all in the low 32-bits of the * PDE, so we can just spin on updating the low 32-bits. */ do { opde = *(u_int *)pde; npde = opde & ~mask; npde |= cache_bits; } while (npde != opde && !atomic_cmpset_int((u_int *)pde, opde, npde)); } /* * Map a set of physical memory pages into the kernel virtual * address space. Return a pointer to where it is mapped. This * routine is intended to be used for mapping device memory, * NOT real memory. */ void * pmap_mapdev_attr(vm_paddr_t pa, vm_size_t size, int mode) { struct pmap_preinit_mapping *ppim; vm_offset_t va, offset; vm_size_t tmpsize; int i; offset = pa & PAGE_MASK; size = round_page(offset + size); pa = trunc_page(pa); if (!pmap_initialized) { va = 0; for (i = 0; i < PMAP_PREINIT_MAPPING_COUNT; i++) { ppim = pmap_preinit_mapping + i; if (ppim->va == 0) { ppim->pa = pa; ppim->sz = size; ppim->mode = mode; ppim->va = virtual_avail; virtual_avail += size; va = ppim->va; break; } } if (va == 0) panic("%s: too many preinit mappings", __func__); } else { /* * If we have a preinit mapping, re-use it. */ for (i = 0; i < PMAP_PREINIT_MAPPING_COUNT; i++) { ppim = pmap_preinit_mapping + i; if (ppim->pa == pa && ppim->sz == size && ppim->mode == mode) return ((void *)(ppim->va + offset)); } /* * If the specified range of physical addresses fits within * the direct map window, use the direct map. */ if (pa < dmaplimit && pa + size < dmaplimit) { va = PHYS_TO_DMAP(pa); if (!pmap_change_attr(va, size, mode)) return ((void *)(va + offset)); } va = kva_alloc(size); if (va == 0) panic("%s: Couldn't allocate KVA", __func__); } for (tmpsize = 0; tmpsize < size; tmpsize += PAGE_SIZE) pmap_kenter_attr(va + tmpsize, pa + tmpsize, mode); pmap_invalidate_range(kernel_pmap, va, va + tmpsize); pmap_invalidate_cache_range(va, va + tmpsize, FALSE); return ((void *)(va + offset)); } void * pmap_mapdev(vm_paddr_t pa, vm_size_t size) { return (pmap_mapdev_attr(pa, size, PAT_UNCACHEABLE)); } void * pmap_mapbios(vm_paddr_t pa, vm_size_t size) { return (pmap_mapdev_attr(pa, size, PAT_WRITE_BACK)); } void pmap_unmapdev(vm_offset_t va, vm_size_t size) { struct pmap_preinit_mapping *ppim; vm_offset_t offset; int i; /* If we gave a direct map region in pmap_mapdev, do nothing */ if (va >= DMAP_MIN_ADDRESS && va < DMAP_MAX_ADDRESS) return; offset = va & PAGE_MASK; size = round_page(offset + size); va = trunc_page(va); for (i = 0; i < PMAP_PREINIT_MAPPING_COUNT; i++) { ppim = pmap_preinit_mapping + i; if (ppim->va == va && ppim->sz == size) { if (pmap_initialized) return; ppim->pa = 0; ppim->va = 0; ppim->sz = 0; ppim->mode = 0; if (va + size == virtual_avail) virtual_avail = va; return; } } if (pmap_initialized) kva_free(va, size); } /* * Tries to demote a 1GB page mapping. */ static boolean_t pmap_demote_pdpe(pmap_t pmap, pdp_entry_t *pdpe, vm_offset_t va) { pdp_entry_t newpdpe, oldpdpe; pd_entry_t *firstpde, newpde, *pde; pt_entry_t PG_A, PG_M, PG_RW, PG_V; vm_paddr_t mpdepa; vm_page_t mpde; PG_A = pmap_accessed_bit(pmap); PG_M = pmap_modified_bit(pmap); PG_V = pmap_valid_bit(pmap); PG_RW = pmap_rw_bit(pmap); PMAP_LOCK_ASSERT(pmap, MA_OWNED); oldpdpe = *pdpe; KASSERT((oldpdpe & (PG_PS | PG_V)) == (PG_PS | PG_V), ("pmap_demote_pdpe: oldpdpe is missing PG_PS and/or PG_V")); if ((mpde = vm_page_alloc(NULL, va >> PDPSHIFT, VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED)) == NULL) { CTR2(KTR_PMAP, "pmap_demote_pdpe: failure for va %#lx" " in pmap %p", va, pmap); return (FALSE); } mpdepa = VM_PAGE_TO_PHYS(mpde); firstpde = (pd_entry_t *)PHYS_TO_DMAP(mpdepa); newpdpe = mpdepa | PG_M | PG_A | (oldpdpe & PG_U) | PG_RW | PG_V; KASSERT((oldpdpe & PG_A) != 0, ("pmap_demote_pdpe: oldpdpe is missing PG_A")); KASSERT((oldpdpe & (PG_M | PG_RW)) != PG_RW, ("pmap_demote_pdpe: oldpdpe is missing PG_M")); newpde = oldpdpe; /* * Initialize the page directory page. */ for (pde = firstpde; pde < firstpde + NPDEPG; pde++) { *pde = newpde; newpde += NBPDR; } /* * Demote the mapping. */ *pdpe = newpdpe; /* * Invalidate a stale recursive mapping of the page directory page. */ pmap_invalidate_page(pmap, (vm_offset_t)vtopde(va)); pmap_pdpe_demotions++; CTR2(KTR_PMAP, "pmap_demote_pdpe: success for va %#lx" " in pmap %p", va, pmap); return (TRUE); } /* * Sets the memory attribute for the specified page. */ void pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma) { m->md.pat_mode = ma; /* * If "m" is a normal page, update its direct mapping. This update * can be relied upon to perform any cache operations that are * required for data coherence. */ if ((m->flags & PG_FICTITIOUS) == 0 && pmap_change_attr(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)), PAGE_SIZE, m->md.pat_mode)) panic("memory attribute change on the direct map failed"); } /* * Changes the specified virtual address range's memory type to that given by * the parameter "mode". The specified virtual address range must be * completely contained within either the direct map or the kernel map. If * the virtual address range is contained within the kernel map, then the * memory type for each of the corresponding ranges of the direct map is also * changed. (The corresponding ranges of the direct map are those ranges that * map the same physical pages as the specified virtual address range.) These * changes to the direct map are necessary because Intel describes the * behavior of their processors as "undefined" if two or more mappings to the * same physical page have different memory types. * * Returns zero if the change completed successfully, and either EINVAL or * ENOMEM if the change failed. Specifically, EINVAL is returned if some part * of the virtual address range was not mapped, and ENOMEM is returned if * there was insufficient memory available to complete the change. In the * latter case, the memory type may have been changed on some part of the * virtual address range or the direct map. */ int pmap_change_attr(vm_offset_t va, vm_size_t size, int mode) { int error; PMAP_LOCK(kernel_pmap); error = pmap_change_attr_locked(va, size, mode); PMAP_UNLOCK(kernel_pmap); return (error); } static int pmap_change_attr_locked(vm_offset_t va, vm_size_t size, int mode) { vm_offset_t base, offset, tmpva; vm_paddr_t pa_start, pa_end; pdp_entry_t *pdpe; pd_entry_t *pde; pt_entry_t *pte; int cache_bits_pte, cache_bits_pde, error; boolean_t changed; PMAP_LOCK_ASSERT(kernel_pmap, MA_OWNED); base = trunc_page(va); offset = va & PAGE_MASK; size = round_page(offset + size); /* * Only supported on kernel virtual addresses, including the direct * map but excluding the recursive map. */ if (base < DMAP_MIN_ADDRESS) return (EINVAL); cache_bits_pde = pmap_cache_bits(kernel_pmap, mode, 1); cache_bits_pte = pmap_cache_bits(kernel_pmap, mode, 0); changed = FALSE; /* * Pages that aren't mapped aren't supported. Also break down 2MB pages * into 4KB pages if required. */ for (tmpva = base; tmpva < base + size; ) { pdpe = pmap_pdpe(kernel_pmap, tmpva); if (pdpe == NULL || *pdpe == 0) return (EINVAL); if (*pdpe & PG_PS) { /* * If the current 1GB page already has the required * memory type, then we need not demote this page. Just * increment tmpva to the next 1GB page frame. */ if ((*pdpe & X86_PG_PDE_CACHE) == cache_bits_pde) { tmpva = trunc_1gpage(tmpva) + NBPDP; continue; } /* * If the current offset aligns with a 1GB page frame * and there is at least 1GB left within the range, then * we need not break down this page into 2MB pages. */ if ((tmpva & PDPMASK) == 0 && tmpva + PDPMASK < base + size) { tmpva += NBPDP; continue; } if (!pmap_demote_pdpe(kernel_pmap, pdpe, tmpva)) return (ENOMEM); } pde = pmap_pdpe_to_pde(pdpe, tmpva); if (*pde == 0) return (EINVAL); if (*pde & PG_PS) { /* * If the current 2MB page already has the required * memory type, then we need not demote this page. Just * increment tmpva to the next 2MB page frame. */ if ((*pde & X86_PG_PDE_CACHE) == cache_bits_pde) { tmpva = trunc_2mpage(tmpva) + NBPDR; continue; } /* * If the current offset aligns with a 2MB page frame * and there is at least 2MB left within the range, then * we need not break down this page into 4KB pages. */ if ((tmpva & PDRMASK) == 0 && tmpva + PDRMASK < base + size) { tmpva += NBPDR; continue; } if (!pmap_demote_pde(kernel_pmap, pde, tmpva)) return (ENOMEM); } pte = pmap_pde_to_pte(pde, tmpva); if (*pte == 0) return (EINVAL); tmpva += PAGE_SIZE; } error = 0; /* * Ok, all the pages exist, so run through them updating their * cache mode if required. */ pa_start = pa_end = 0; for (tmpva = base; tmpva < base + size; ) { pdpe = pmap_pdpe(kernel_pmap, tmpva); if (*pdpe & PG_PS) { if ((*pdpe & X86_PG_PDE_CACHE) != cache_bits_pde) { pmap_pde_attr(pdpe, cache_bits_pde, X86_PG_PDE_CACHE); changed = TRUE; } if (tmpva >= VM_MIN_KERNEL_ADDRESS && (*pdpe & PG_PS_FRAME) < dmaplimit) { if (pa_start == pa_end) { /* Start physical address run. */ pa_start = *pdpe & PG_PS_FRAME; pa_end = pa_start + NBPDP; } else if (pa_end == (*pdpe & PG_PS_FRAME)) pa_end += NBPDP; else { /* Run ended, update direct map. */ error = pmap_change_attr_locked( PHYS_TO_DMAP(pa_start), pa_end - pa_start, mode); if (error != 0) break; /* Start physical address run. */ pa_start = *pdpe & PG_PS_FRAME; pa_end = pa_start + NBPDP; } } tmpva = trunc_1gpage(tmpva) + NBPDP; continue; } pde = pmap_pdpe_to_pde(pdpe, tmpva); if (*pde & PG_PS) { if ((*pde & X86_PG_PDE_CACHE) != cache_bits_pde) { pmap_pde_attr(pde, cache_bits_pde, X86_PG_PDE_CACHE); changed = TRUE; } if (tmpva >= VM_MIN_KERNEL_ADDRESS && (*pde & PG_PS_FRAME) < dmaplimit) { if (pa_start == pa_end) { /* Start physical address run. */ pa_start = *pde & PG_PS_FRAME; pa_end = pa_start + NBPDR; } else if (pa_end == (*pde & PG_PS_FRAME)) pa_end += NBPDR; else { /* Run ended, update direct map. */ error = pmap_change_attr_locked( PHYS_TO_DMAP(pa_start), pa_end - pa_start, mode); if (error != 0) break; /* Start physical address run. */ pa_start = *pde & PG_PS_FRAME; pa_end = pa_start + NBPDR; } } tmpva = trunc_2mpage(tmpva) + NBPDR; } else { pte = pmap_pde_to_pte(pde, tmpva); if ((*pte & X86_PG_PTE_CACHE) != cache_bits_pte) { pmap_pte_attr(pte, cache_bits_pte, X86_PG_PTE_CACHE); changed = TRUE; } if (tmpva >= VM_MIN_KERNEL_ADDRESS && (*pte & PG_PS_FRAME) < dmaplimit) { if (pa_start == pa_end) { /* Start physical address run. */ pa_start = *pte & PG_FRAME; pa_end = pa_start + PAGE_SIZE; } else if (pa_end == (*pte & PG_FRAME)) pa_end += PAGE_SIZE; else { /* Run ended, update direct map. */ error = pmap_change_attr_locked( PHYS_TO_DMAP(pa_start), pa_end - pa_start, mode); if (error != 0) break; /* Start physical address run. */ pa_start = *pte & PG_FRAME; pa_end = pa_start + PAGE_SIZE; } } tmpva += PAGE_SIZE; } } if (error == 0 && pa_start != pa_end) error = pmap_change_attr_locked(PHYS_TO_DMAP(pa_start), pa_end - pa_start, mode); /* * Flush CPU caches if required to make sure any data isn't cached that * shouldn't be, etc. */ if (changed) { pmap_invalidate_range(kernel_pmap, base, tmpva); pmap_invalidate_cache_range(base, tmpva, FALSE); } return (error); } /* * Demotes any mapping within the direct map region that covers more than the * specified range of physical addresses. This range's size must be a power * of two and its starting address must be a multiple of its size. Since the * demotion does not change any attributes of the mapping, a TLB invalidation * is not mandatory. The caller may, however, request a TLB invalidation. */ void pmap_demote_DMAP(vm_paddr_t base, vm_size_t len, boolean_t invalidate) { pdp_entry_t *pdpe; pd_entry_t *pde; vm_offset_t va; boolean_t changed; if (len == 0) return; KASSERT(powerof2(len), ("pmap_demote_DMAP: len is not a power of 2")); KASSERT((base & (len - 1)) == 0, ("pmap_demote_DMAP: base is not a multiple of len")); if (len < NBPDP && base < dmaplimit) { va = PHYS_TO_DMAP(base); changed = FALSE; PMAP_LOCK(kernel_pmap); pdpe = pmap_pdpe(kernel_pmap, va); if ((*pdpe & X86_PG_V) == 0) panic("pmap_demote_DMAP: invalid PDPE"); if ((*pdpe & PG_PS) != 0) { if (!pmap_demote_pdpe(kernel_pmap, pdpe, va)) panic("pmap_demote_DMAP: PDPE failed"); changed = TRUE; } if (len < NBPDR) { pde = pmap_pdpe_to_pde(pdpe, va); if ((*pde & X86_PG_V) == 0) panic("pmap_demote_DMAP: invalid PDE"); if ((*pde & PG_PS) != 0) { if (!pmap_demote_pde(kernel_pmap, pde, va)) panic("pmap_demote_DMAP: PDE failed"); changed = TRUE; } } if (changed && invalidate) pmap_invalidate_page(kernel_pmap, va); PMAP_UNLOCK(kernel_pmap); } } /* * perform the pmap work for mincore */ int pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa) { pd_entry_t *pdep; pt_entry_t pte, PG_A, PG_M, PG_RW, PG_V; vm_paddr_t pa; int val; PG_A = pmap_accessed_bit(pmap); PG_M = pmap_modified_bit(pmap); PG_V = pmap_valid_bit(pmap); PG_RW = pmap_rw_bit(pmap); PMAP_LOCK(pmap); retry: pdep = pmap_pde(pmap, addr); if (pdep != NULL && (*pdep & PG_V)) { if (*pdep & PG_PS) { pte = *pdep; /* Compute the physical address of the 4KB page. */ pa = ((*pdep & PG_PS_FRAME) | (addr & PDRMASK)) & PG_FRAME; val = MINCORE_SUPER; } else { pte = *pmap_pde_to_pte(pdep, addr); pa = pte & PG_FRAME; val = 0; } } else { pte = 0; pa = 0; val = 0; } if ((pte & PG_V) != 0) { val |= MINCORE_INCORE; if ((pte & (PG_M | PG_RW)) == (PG_M | PG_RW)) val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER; if ((pte & PG_A) != 0) val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER; } if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) != (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) && (pte & (PG_MANAGED | PG_V)) == (PG_MANAGED | PG_V)) { /* Ensure that "PHYS_TO_VM_PAGE(pa)->object" doesn't change. */ if (vm_page_pa_tryrelock(pmap, pa, locked_pa)) goto retry; } else PA_UNLOCK_COND(*locked_pa); PMAP_UNLOCK(pmap); return (val); } static uint64_t pmap_pcid_alloc(pmap_t pmap, u_int cpuid) { uint32_t gen, new_gen, pcid_next; CRITICAL_ASSERT(curthread); gen = PCPU_GET(pcid_gen); if (pmap->pm_pcids[cpuid].pm_pcid == PMAP_PCID_KERN || pmap->pm_pcids[cpuid].pm_gen == gen) return (CR3_PCID_SAVE); pcid_next = PCPU_GET(pcid_next); KASSERT(pcid_next <= PMAP_PCID_OVERMAX, ("cpu %d pcid_next %#x", cpuid, pcid_next)); if (pcid_next == PMAP_PCID_OVERMAX) { new_gen = gen + 1; if (new_gen == 0) new_gen = 1; PCPU_SET(pcid_gen, new_gen); pcid_next = PMAP_PCID_KERN + 1; } else { new_gen = gen; } pmap->pm_pcids[cpuid].pm_pcid = pcid_next; pmap->pm_pcids[cpuid].pm_gen = new_gen; PCPU_SET(pcid_next, pcid_next + 1); return (0); } void pmap_activate_sw(struct thread *td) { pmap_t oldpmap, pmap; uint64_t cached, cr3; u_int cpuid; oldpmap = PCPU_GET(curpmap); pmap = vmspace_pmap(td->td_proc->p_vmspace); if (oldpmap == pmap) return; cpuid = PCPU_GET(cpuid); #ifdef SMP CPU_SET_ATOMIC(cpuid, &pmap->pm_active); #else CPU_SET(cpuid, &pmap->pm_active); #endif cr3 = rcr3(); if (pmap_pcid_enabled) { cached = pmap_pcid_alloc(pmap, cpuid); KASSERT(pmap->pm_pcids[cpuid].pm_pcid >= 0 && pmap->pm_pcids[cpuid].pm_pcid < PMAP_PCID_OVERMAX, ("pmap %p cpu %d pcid %#x", pmap, cpuid, pmap->pm_pcids[cpuid].pm_pcid)); KASSERT(pmap->pm_pcids[cpuid].pm_pcid != PMAP_PCID_KERN || pmap == kernel_pmap, ("non-kernel pmap thread %p pmap %p cpu %d pcid %#x", td, pmap, cpuid, pmap->pm_pcids[cpuid].pm_pcid)); if (!cached || (cr3 & ~CR3_PCID_MASK) != pmap->pm_cr3) { load_cr3(pmap->pm_cr3 | pmap->pm_pcids[cpuid].pm_pcid | cached); if (cached) PCPU_INC(pm_save_cnt); } } else if (cr3 != pmap->pm_cr3) { load_cr3(pmap->pm_cr3); } PCPU_SET(curpmap, pmap); #ifdef SMP CPU_CLR_ATOMIC(cpuid, &oldpmap->pm_active); #else CPU_CLR(cpuid, &oldpmap->pm_active); #endif } void pmap_activate(struct thread *td) { critical_enter(); pmap_activate_sw(td); critical_exit(); } void pmap_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz) { } /* * Increase the starting virtual address of the given mapping if a * different alignment might result in more superpage mappings. */ void pmap_align_superpage(vm_object_t object, vm_ooffset_t offset, vm_offset_t *addr, vm_size_t size) { vm_offset_t superpage_offset; if (size < NBPDR) return; if (object != NULL && (object->flags & OBJ_COLORED) != 0) offset += ptoa(object->pg_color); superpage_offset = offset & PDRMASK; if (size - ((NBPDR - superpage_offset) & PDRMASK) < NBPDR || (*addr & PDRMASK) == superpage_offset) return; if ((*addr & PDRMASK) < superpage_offset) *addr = (*addr & ~PDRMASK) + superpage_offset; else *addr = ((*addr + PDRMASK) & ~PDRMASK) + superpage_offset; } #ifdef INVARIANTS static unsigned long num_dirty_emulations; SYSCTL_ULONG(_vm_pmap, OID_AUTO, num_dirty_emulations, CTLFLAG_RW, &num_dirty_emulations, 0, NULL); static unsigned long num_accessed_emulations; SYSCTL_ULONG(_vm_pmap, OID_AUTO, num_accessed_emulations, CTLFLAG_RW, &num_accessed_emulations, 0, NULL); static unsigned long num_superpage_accessed_emulations; SYSCTL_ULONG(_vm_pmap, OID_AUTO, num_superpage_accessed_emulations, CTLFLAG_RW, &num_superpage_accessed_emulations, 0, NULL); static unsigned long ad_emulation_superpage_promotions; SYSCTL_ULONG(_vm_pmap, OID_AUTO, ad_emulation_superpage_promotions, CTLFLAG_RW, &ad_emulation_superpage_promotions, 0, NULL); #endif /* INVARIANTS */ int pmap_emulate_accessed_dirty(pmap_t pmap, vm_offset_t va, int ftype) { int rv; struct rwlock *lock; vm_page_t m, mpte; pd_entry_t *pde; pt_entry_t *pte, PG_A, PG_M, PG_RW, PG_V; boolean_t pv_lists_locked; KASSERT(ftype == VM_PROT_READ || ftype == VM_PROT_WRITE, ("pmap_emulate_accessed_dirty: invalid fault type %d", ftype)); if (!pmap_emulate_ad_bits(pmap)) return (-1); PG_A = pmap_accessed_bit(pmap); PG_M = pmap_modified_bit(pmap); PG_V = pmap_valid_bit(pmap); PG_RW = pmap_rw_bit(pmap); rv = -1; lock = NULL; pv_lists_locked = FALSE; retry: PMAP_LOCK(pmap); pde = pmap_pde(pmap, va); if (pde == NULL || (*pde & PG_V) == 0) goto done; if ((*pde & PG_PS) != 0) { if (ftype == VM_PROT_READ) { #ifdef INVARIANTS atomic_add_long(&num_superpage_accessed_emulations, 1); #endif *pde |= PG_A; rv = 0; } goto done; } pte = pmap_pde_to_pte(pde, va); if ((*pte & PG_V) == 0) goto done; if (ftype == VM_PROT_WRITE) { if ((*pte & PG_RW) == 0) goto done; /* * Set the modified and accessed bits simultaneously. * * Intel EPT PTEs that do software emulation of A/D bits map * PG_A and PG_M to EPT_PG_READ and EPT_PG_WRITE respectively. * An EPT misconfiguration is triggered if the PTE is writable * but not readable (WR=10). This is avoided by setting PG_A * and PG_M simultaneously. */ *pte |= PG_M | PG_A; } else { *pte |= PG_A; } /* try to promote the mapping */ if (va < VM_MAXUSER_ADDRESS) mpte = PHYS_TO_VM_PAGE(*pde & PG_FRAME); else mpte = NULL; m = PHYS_TO_VM_PAGE(*pte & PG_FRAME); if ((mpte == NULL || mpte->wire_count == NPTEPG) && pmap_ps_enabled(pmap) && (m->flags & PG_FICTITIOUS) == 0 && vm_reserv_level_iffullpop(m) == 0) { if (!pv_lists_locked) { pv_lists_locked = TRUE; if (!rw_try_rlock(&pvh_global_lock)) { PMAP_UNLOCK(pmap); rw_rlock(&pvh_global_lock); goto retry; } } pmap_promote_pde(pmap, pde, va, &lock); #ifdef INVARIANTS atomic_add_long(&ad_emulation_superpage_promotions, 1); #endif } #ifdef INVARIANTS if (ftype == VM_PROT_WRITE) atomic_add_long(&num_dirty_emulations, 1); else atomic_add_long(&num_accessed_emulations, 1); #endif rv = 0; /* success */ done: if (lock != NULL) rw_wunlock(lock); if (pv_lists_locked) rw_runlock(&pvh_global_lock); PMAP_UNLOCK(pmap); return (rv); } void pmap_get_mapping(pmap_t pmap, vm_offset_t va, uint64_t *ptr, int *num) { pml4_entry_t *pml4; pdp_entry_t *pdp; pd_entry_t *pde; pt_entry_t *pte, PG_V; int idx; idx = 0; PG_V = pmap_valid_bit(pmap); PMAP_LOCK(pmap); pml4 = pmap_pml4e(pmap, va); ptr[idx++] = *pml4; if ((*pml4 & PG_V) == 0) goto done; pdp = pmap_pml4e_to_pdpe(pml4, va); ptr[idx++] = *pdp; if ((*pdp & PG_V) == 0 || (*pdp & PG_PS) != 0) goto done; pde = pmap_pdpe_to_pde(pdp, va); ptr[idx++] = *pde; if ((*pde & PG_V) == 0 || (*pde & PG_PS) != 0) goto done; pte = pmap_pde_to_pte(pde, va); ptr[idx++] = *pte; done: PMAP_UNLOCK(pmap); *num = idx; } /** * Get the kernel virtual address of a set of physical pages. If there are * physical addresses not covered by the DMAP perform a transient mapping * that will be removed when calling pmap_unmap_io_transient. * * \param page The pages the caller wishes to obtain the virtual * address on the kernel memory map. * \param vaddr On return contains the kernel virtual memory address * of the pages passed in the page parameter. * \param count Number of pages passed in. * \param can_fault TRUE if the thread using the mapped pages can take * page faults, FALSE otherwise. * * \returns TRUE if the caller must call pmap_unmap_io_transient when * finished or FALSE otherwise. * */ boolean_t pmap_map_io_transient(vm_page_t page[], vm_offset_t vaddr[], int count, boolean_t can_fault) { vm_paddr_t paddr; boolean_t needs_mapping; pt_entry_t *pte; int cache_bits, error, i; /* * Allocate any KVA space that we need, this is done in a separate * loop to prevent calling vmem_alloc while pinned. */ needs_mapping = FALSE; for (i = 0; i < count; i++) { paddr = VM_PAGE_TO_PHYS(page[i]); if (__predict_false(paddr >= dmaplimit)) { error = vmem_alloc(kernel_arena, PAGE_SIZE, M_BESTFIT | M_WAITOK, &vaddr[i]); KASSERT(error == 0, ("vmem_alloc failed: %d", error)); needs_mapping = TRUE; } else { vaddr[i] = PHYS_TO_DMAP(paddr); } } /* Exit early if everything is covered by the DMAP */ if (!needs_mapping) return (FALSE); /* * NB: The sequence of updating a page table followed by accesses * to the corresponding pages used in the !DMAP case is subject to * the situation described in the "AMD64 Architecture Programmer's * Manual Volume 2: System Programming" rev. 3.23, "7.3.1 Special * Coherency Considerations". Therefore, issuing the INVLPG right * after modifying the PTE bits is crucial. */ if (!can_fault) sched_pin(); for (i = 0; i < count; i++) { paddr = VM_PAGE_TO_PHYS(page[i]); if (paddr >= dmaplimit) { if (can_fault) { /* * Slow path, since we can get page faults * while mappings are active don't pin the * thread to the CPU and instead add a global * mapping visible to all CPUs. */ pmap_qenter(vaddr[i], &page[i], 1); } else { pte = vtopte(vaddr[i]); cache_bits = pmap_cache_bits(kernel_pmap, page[i]->md.pat_mode, 0); pte_store(pte, paddr | X86_PG_RW | X86_PG_V | cache_bits); invlpg(vaddr[i]); } } } return (needs_mapping); } void pmap_unmap_io_transient(vm_page_t page[], vm_offset_t vaddr[], int count, boolean_t can_fault) { vm_paddr_t paddr; int i; if (!can_fault) sched_unpin(); for (i = 0; i < count; i++) { paddr = VM_PAGE_TO_PHYS(page[i]); if (paddr >= dmaplimit) { if (can_fault) pmap_qremove(vaddr[i], 1); vmem_free(kernel_arena, vaddr[i], PAGE_SIZE); } } } vm_offset_t pmap_quick_enter_page(vm_page_t m) { vm_paddr_t paddr; paddr = VM_PAGE_TO_PHYS(m); if (paddr < dmaplimit) return (PHYS_TO_DMAP(paddr)); mtx_lock_spin(&qframe_mtx); KASSERT(*vtopte(qframe) == 0, ("qframe busy")); pte_store(vtopte(qframe), paddr | X86_PG_RW | X86_PG_V | X86_PG_A | X86_PG_M | pmap_cache_bits(kernel_pmap, m->md.pat_mode, 0)); return (qframe); } void pmap_quick_remove_page(vm_offset_t addr) { if (addr != qframe) return; pte_store(vtopte(qframe), 0); invlpg(qframe); mtx_unlock_spin(&qframe_mtx); } #include "opt_ddb.h" #ifdef DDB #include DB_SHOW_COMMAND(pte, pmap_print_pte) { pmap_t pmap; pml4_entry_t *pml4; pdp_entry_t *pdp; pd_entry_t *pde; pt_entry_t *pte, PG_V; vm_offset_t va; if (have_addr) { va = (vm_offset_t)addr; pmap = PCPU_GET(curpmap); /* XXX */ } else { db_printf("show pte addr\n"); return; } PG_V = pmap_valid_bit(pmap); pml4 = pmap_pml4e(pmap, va); db_printf("VA %#016lx pml4e %#016lx", va, *pml4); if ((*pml4 & PG_V) == 0) { db_printf("\n"); return; } pdp = pmap_pml4e_to_pdpe(pml4, va); db_printf(" pdpe %#016lx", *pdp); if ((*pdp & PG_V) == 0 || (*pdp & PG_PS) != 0) { db_printf("\n"); return; } pde = pmap_pdpe_to_pde(pdp, va); db_printf(" pde %#016lx", *pde); if ((*pde & PG_V) == 0 || (*pde & PG_PS) != 0) { db_printf("\n"); return; } pte = pmap_pde_to_pte(pde, va); db_printf(" pte %#016lx\n", *pte); } DB_SHOW_COMMAND(phys2dmap, pmap_phys2dmap) { vm_paddr_t a; if (have_addr) { a = (vm_paddr_t)addr; db_printf("0x%jx\n", (uintmax_t)PHYS_TO_DMAP(a)); } else { db_printf("show phys2dmap addr\n"); } } #endif Index: user/ngie/release-pkg-fix-tests/sys/arm/allwinner/files.allwinner =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/allwinner/files.allwinner (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/allwinner/files.allwinner (revision 299091) @@ -1,47 +1,48 @@ # $FreeBSD$ kern/kern_clocksource.c standard arm/allwinner/a10_ahci.c optional ahci arm/allwinner/a10_codec.c optional sound arm/allwinner/a10_common.c standard arm/allwinner/a10_dmac.c standard arm/allwinner/a10_ehci.c optional ehci arm/allwinner/aw_usbphy.c optional ehci arm/allwinner/a10_gpio.c optional gpio arm/allwinner/a10_mmc.c optional mmc arm/allwinner/a10_sramc.c standard arm/allwinner/aw_nmi.c optional intrng arm/allwinner/aw_if_dwc.c optional dwc arm/allwinner/aw_rtc.c standard arm/allwinner/aw_wdog.c standard arm/allwinner/a20/a20_cpu_cfg.c standard arm/allwinner/allwinner_machdep.c standard arm/allwinner/aw_mp.c optional smp arm/allwinner/axp209.c optional axp209 +arm/allwinner/if_awg.c optional awg arm/allwinner/if_emac.c optional emac arm/allwinner/sunxi_dma_if.m standard dev/iicbus/twsi/a10_twsi.c optional twsi #arm/allwinner/console.c standard arm/allwinner/a10_fb.c optional vt arm/allwinner/a10_hdmi.c optional hdmi arm/allwinner/a10_hdmiaudio.c optional hdmi sound arm/arm/hdmi_if.m optional hdmi arm/allwinner/aw_reset.c standard arm/allwinner/aw_ccu.c standard arm/allwinner/clk/aw_ahbclk.c standard arm/allwinner/clk/aw_apbclk.c standard arm/allwinner/clk/aw_axiclk.c standard arm/allwinner/clk/aw_codecclk.c standard arm/allwinner/clk/aw_cpuclk.c standard arm/allwinner/clk/aw_debeclk.c standard arm/allwinner/clk/aw_gate.c standard arm/allwinner/clk/aw_gmacclk.c standard arm/allwinner/clk/aw_hdmiclk.c standard arm/allwinner/clk/aw_lcdclk.c standard arm/allwinner/clk/aw_modclk.c standard arm/allwinner/clk/aw_mmcclk.c standard arm/allwinner/clk/aw_oscclk.c standard arm/allwinner/clk/aw_pll.c standard arm/allwinner/clk/aw_usbclk.c standard Index: user/ngie/release-pkg-fix-tests/sys/arm/allwinner/if_awg.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/allwinner/if_awg.c (nonexistent) +++ user/ngie/release-pkg-fix-tests/sys/arm/allwinner/if_awg.c (revision 299091) @@ -0,0 +1,1418 @@ +/*- + * Copyright (c) 2016 Jared McNeill + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +/* + * Allwinner Gigabit Ethernet MAC (EMAC) controller + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include +#include +#include + +#include +#include +#include + +#include "miibus_if.h" + +#define RD4(sc, reg) bus_read_4((sc)->res[0], (reg)) +#define WR4(sc, reg, val) bus_write_4((sc)->res[0], (reg), (val)) + +#define AWG_LOCK(sc) mtx_lock(&(sc)->mtx) +#define AWG_UNLOCK(sc) mtx_unlock(&(sc)->mtx); +#define AWG_ASSERT_LOCKED(sc) mtx_assert(&(sc)->mtx, MA_OWNED) +#define AWG_ASSERT_UNLOCKED(sc) mtx_assert(&(sc)->mtx, MA_NOTOWNED) + +#define DESC_ALIGN 4 +#define TX_DESC_COUNT 256 +#define TX_DESC_SIZE (sizeof(struct emac_desc) * TX_DESC_COUNT) +#define RX_DESC_COUNT 256 +#define RX_DESC_SIZE (sizeof(struct emac_desc) * RX_DESC_COUNT) + +#define DESC_OFF(n) ((n) * sizeof(struct emac_desc)) +#define TX_NEXT(n) (((n) + 1) & (TX_DESC_COUNT - 1)) +#define TX_SKIP(n, o) (((n) + (o)) & (TX_DESC_COUNT - 1)) +#define RX_NEXT(n) (((n) + 1) & (RX_DESC_COUNT - 1)) + +#define TX_MAX_SEGS 10 + +#define SOFT_RST_RETRY 1000 +#define MII_BUSY_RETRY 1000 +#define MDIO_FREQ 2500000 + +#define BURST_LEN_DEFAULT 8 +#define RX_TX_PRI_DEFAULT 0 +#define PAUSE_TIME_DEFAULT 0x400 +#define TX_INTERVAL_DEFAULT 64 + +/* Burst length of RX and TX DMA transfers */ +static int awg_burst_len = BURST_LEN_DEFAULT; +TUNABLE_INT("hw.awg.burst_len", &awg_burst_len); + +/* RX / TX DMA priority. If 1, RX DMA has priority over TX DMA. */ +static int awg_rx_tx_pri = RX_TX_PRI_DEFAULT; +TUNABLE_INT("hw.awg.rx_tx_pri", &awg_rx_tx_pri); + +/* Pause time field in the transmitted control frame */ +static int awg_pause_time = PAUSE_TIME_DEFAULT; +TUNABLE_INT("hw.awg.pause_time", &awg_pause_time); + +/* Request a TX interrupt every descriptors */ +static int awg_tx_interval = TX_INTERVAL_DEFAULT; +TUNABLE_INT("hw.awg.tx_interval", &awg_tx_interval); + +static struct ofw_compat_data compat_data[] = { + { "allwinner,sun8i-a83t-emac", 1 }, + { NULL, 0 } +}; + +struct awg_bufmap { + bus_dmamap_t map; + struct mbuf *mbuf; +}; + +struct awg_txring { + bus_dma_tag_t desc_tag; + bus_dmamap_t desc_map; + struct emac_desc *desc_ring; + bus_addr_t desc_ring_paddr; + bus_dma_tag_t buf_tag; + struct awg_bufmap buf_map[TX_DESC_COUNT]; + u_int cur, next, queued; +}; + +struct awg_rxring { + bus_dma_tag_t desc_tag; + bus_dmamap_t desc_map; + struct emac_desc *desc_ring; + bus_addr_t desc_ring_paddr; + bus_dma_tag_t buf_tag; + struct awg_bufmap buf_map[RX_DESC_COUNT]; + u_int cur; +}; + +struct awg_softc { + struct resource *res[2]; + struct mtx mtx; + if_t ifp; + device_t miibus; + struct callout stat_ch; + struct task link_task; + void *ih; + u_int mdc_div_ratio_m; + int link; + int if_flags; + + struct awg_txring tx; + struct awg_rxring rx; +}; + +static struct resource_spec awg_spec[] = { + { SYS_RES_MEMORY, 0, RF_ACTIVE }, + { SYS_RES_IRQ, 0, RF_ACTIVE }, + { -1, 0 } +}; + +static int +awg_miibus_readreg(device_t dev, int phy, int reg) +{ + struct awg_softc *sc; + int retry, val; + + sc = device_get_softc(dev); + val = 0; + + WR4(sc, EMAC_MII_CMD, + (sc->mdc_div_ratio_m << MDC_DIV_RATIO_M_SHIFT) | + (phy << PHY_ADDR_SHIFT) | + (reg << PHY_REG_ADDR_SHIFT) | + MII_BUSY); + for (retry = MII_BUSY_RETRY; retry > 0; retry--) { + if ((RD4(sc, EMAC_MII_CMD) & MII_BUSY) == 0) { + val = RD4(sc, EMAC_MII_DATA); + break; + } + DELAY(10); + } + + if (retry == 0) + device_printf(dev, "phy read timeout, phy=%d reg=%d\n", + phy, reg); + + return (val); +} + +static int +awg_miibus_writereg(device_t dev, int phy, int reg, int val) +{ + struct awg_softc *sc; + int retry; + + sc = device_get_softc(dev); + + WR4(sc, EMAC_MII_DATA, val); + WR4(sc, EMAC_MII_CMD, + (sc->mdc_div_ratio_m << MDC_DIV_RATIO_M_SHIFT) | + (phy << PHY_ADDR_SHIFT) | + (reg << PHY_REG_ADDR_SHIFT) | + MII_WR | MII_BUSY); + for (retry = MII_BUSY_RETRY; retry > 0; retry--) { + if ((RD4(sc, EMAC_MII_CMD) & MII_BUSY) == 0) + break; + DELAY(10); + } + + if (retry == 0) + device_printf(dev, "phy write timeout, phy=%d reg=%d\n", + phy, reg); + + return (0); +} + +static void +awg_update_link_locked(struct awg_softc *sc) +{ + struct mii_data *mii; + uint32_t val; + + AWG_ASSERT_LOCKED(sc); + + if ((if_getdrvflags(sc->ifp) & IFF_DRV_RUNNING) == 0) + return; + mii = device_get_softc(sc->miibus); + + if ((mii->mii_media_status & (IFM_ACTIVE | IFM_AVALID)) == + (IFM_ACTIVE | IFM_AVALID)) { + switch (IFM_SUBTYPE(mii->mii_media_active)) { + case IFM_1000_T: + case IFM_1000_SX: + case IFM_100_TX: + case IFM_10_T: + sc->link = 1; + break; + default: + sc->link = 0; + break; + } + } else + sc->link = 0; + + if (sc->link == 0) + return; + + val = RD4(sc, EMAC_BASIC_CTL_0); + val &= ~(BASIC_CTL_SPEED | BASIC_CTL_DUPLEX); + + if (IFM_SUBTYPE(mii->mii_media_active) == IFM_1000_T || + IFM_SUBTYPE(mii->mii_media_active) == IFM_1000_SX) + val |= BASIC_CTL_SPEED_1000 << BASIC_CTL_SPEED_SHIFT; + else if (IFM_SUBTYPE(mii->mii_media_active) == IFM_100_TX) + val |= BASIC_CTL_SPEED_100 << BASIC_CTL_SPEED_SHIFT; + else + val |= BASIC_CTL_SPEED_10 << BASIC_CTL_SPEED_SHIFT; + + if ((IFM_OPTIONS(mii->mii_media_active) & IFM_FDX) != 0) + val |= BASIC_CTL_DUPLEX; + + WR4(sc, EMAC_BASIC_CTL_0, val); + + val = RD4(sc, EMAC_RX_CTL_0); + val &= ~RX_FLOW_CTL_EN; + if ((IFM_OPTIONS(mii->mii_media_active) & IFM_ETH_RXPAUSE) != 0) + val |= RX_FLOW_CTL_EN; + WR4(sc, EMAC_RX_CTL_0, val); + + val = RD4(sc, EMAC_TX_FLOW_CTL); + val &= ~(PAUSE_TIME|TX_FLOW_CTL_EN); + if ((IFM_OPTIONS(mii->mii_media_active) & IFM_ETH_TXPAUSE) != 0) + val |= TX_FLOW_CTL_EN; + if ((IFM_OPTIONS(mii->mii_media_active) & IFM_FDX) != 0) + val |= awg_pause_time << PAUSE_TIME_SHIFT; + WR4(sc, EMAC_TX_FLOW_CTL, val); +} + +static void +awg_link_task(void *arg, int pending) +{ + struct awg_softc *sc; + + sc = arg; + + AWG_LOCK(sc); + awg_update_link_locked(sc); + AWG_UNLOCK(sc); +} + +static void +awg_miibus_statchg(device_t dev) +{ + struct awg_softc *sc; + + sc = device_get_softc(dev); + + taskqueue_enqueue(taskqueue_swi, &sc->link_task); +} + +static void +awg_media_status(if_t ifp, struct ifmediareq *ifmr) +{ + struct awg_softc *sc; + struct mii_data *mii; + + sc = if_getsoftc(ifp); + mii = device_get_softc(sc->miibus); + + AWG_LOCK(sc); + mii_pollstat(mii); + ifmr->ifm_active = mii->mii_media_active; + ifmr->ifm_status = mii->mii_media_status; + AWG_UNLOCK(sc); +} + +static int +awg_media_change(if_t ifp) +{ + struct awg_softc *sc; + struct mii_data *mii; + int error; + + sc = if_getsoftc(ifp); + mii = device_get_softc(sc->miibus); + + AWG_LOCK(sc); + error = mii_mediachg(mii); + AWG_UNLOCK(sc); + + return (error); +} + +static void +awg_setup_txdesc(struct awg_softc *sc, int index, int flags, bus_addr_t paddr, + u_int len) +{ + uint32_t status, size; + + if (paddr == 0 || len == 0) { + status = 0; + size = 0; + --sc->tx.queued; + } else { + status = TX_DESC_CTL; + size = flags | len; + if ((index & (awg_tx_interval - 1)) == 0) + size |= htole32(TX_INT_CTL); + ++sc->tx.queued; + } + + sc->tx.desc_ring[index].addr = htole32((uint32_t)paddr); + sc->tx.desc_ring[index].size = htole32(size); + sc->tx.desc_ring[index].status = htole32(status); +} + +static int +awg_setup_txbuf(struct awg_softc *sc, int index, struct mbuf **mp) +{ + bus_dma_segment_t segs[TX_MAX_SEGS]; + int error, nsegs, cur, i, flags; + u_int csum_flags; + struct mbuf *m; + + m = *mp; + error = bus_dmamap_load_mbuf_sg(sc->tx.buf_tag, + sc->tx.buf_map[index].map, m, segs, &nsegs, BUS_DMA_NOWAIT); + if (error == EFBIG) { + m = m_collapse(m, M_NOWAIT, TX_MAX_SEGS); + if (m == NULL) + return (0); + *mp = m; + error = bus_dmamap_load_mbuf_sg(sc->tx.buf_tag, + sc->tx.buf_map[index].map, m, segs, &nsegs, BUS_DMA_NOWAIT); + } + if (error != 0) + return (0); + + bus_dmamap_sync(sc->tx.buf_tag, sc->tx.buf_map[index].map, + BUS_DMASYNC_PREWRITE); + + flags = TX_FIR_DESC; + if ((m->m_pkthdr.csum_flags & CSUM_IP) != 0) { + if ((m->m_pkthdr.csum_flags & (CSUM_TCP|CSUM_UDP)) != 0) + csum_flags = TX_CHECKSUM_CTL_FULL; + else + csum_flags = TX_CHECKSUM_CTL_IP; + flags |= (csum_flags << TX_CHECKSUM_CTL_SHIFT); + } + + for (cur = index, i = 0; i < nsegs; i++) { + sc->tx.buf_map[cur].mbuf = (i == 0 ? m : NULL); + if (i == nsegs - 1) + flags |= TX_LAST_DESC; + awg_setup_txdesc(sc, cur, flags, segs[i].ds_addr, + segs[i].ds_len); + flags &= ~TX_FIR_DESC; + cur = TX_NEXT(cur); + } + + return (nsegs); +} + +static void +awg_setup_rxdesc(struct awg_softc *sc, int index, bus_addr_t paddr) +{ + uint32_t status, size; + + status = RX_DESC_CTL; + size = MCLBYTES - 1; + + sc->rx.desc_ring[index].addr = htole32((uint32_t)paddr); + sc->rx.desc_ring[index].size = htole32(size); + sc->rx.desc_ring[index].next = + htole32(sc->rx.desc_ring_paddr + DESC_OFF(RX_NEXT(index))); + sc->rx.desc_ring[index].status = htole32(status); +} + +static int +awg_setup_rxbuf(struct awg_softc *sc, int index, struct mbuf *m) +{ + bus_dma_segment_t seg; + int error, nsegs; + + m_adj(m, ETHER_ALIGN); + + error = bus_dmamap_load_mbuf_sg(sc->rx.buf_tag, + sc->rx.buf_map[index].map, m, &seg, &nsegs, 0); + if (error != 0) + return (error); + + bus_dmamap_sync(sc->rx.buf_tag, sc->rx.buf_map[index].map, + BUS_DMASYNC_PREREAD); + + sc->rx.buf_map[index].mbuf = m; + awg_setup_rxdesc(sc, index, seg.ds_addr); + + return (0); +} + +static struct mbuf * +awg_alloc_mbufcl(struct awg_softc *sc) +{ + struct mbuf *m; + + m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); + if (m != NULL) + m->m_pkthdr.len = m->m_len = m->m_ext.ext_size; + + return (m); +} + +static void +awg_start_locked(struct awg_softc *sc) +{ + struct mbuf *m; + uint32_t val; + if_t ifp; + int cnt, nsegs; + + AWG_ASSERT_LOCKED(sc); + + if (!sc->link) + return; + + ifp = sc->ifp; + + if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING|IFF_DRV_OACTIVE)) != + IFF_DRV_RUNNING) + return; + + for (cnt = 0; ; cnt++) { + if (sc->tx.queued >= TX_DESC_COUNT - TX_MAX_SEGS) { + if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, 0); + break; + } + + m = if_dequeue(ifp); + if (m == NULL) + break; + + nsegs = awg_setup_txbuf(sc, sc->tx.cur, &m); + if (nsegs == 0) { + if_sendq_prepend(ifp, m); + break; + } + if_bpfmtap(ifp, m); + sc->tx.cur = TX_SKIP(sc->tx.cur, nsegs); + } + + if (cnt != 0) { + bus_dmamap_sync(sc->tx.desc_tag, sc->tx.desc_map, + BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); + + /* Start and run TX DMA */ + val = RD4(sc, EMAC_TX_CTL_1); + WR4(sc, EMAC_TX_CTL_1, val | TX_DMA_START); + } +} + +static void +awg_start(if_t ifp) +{ + struct awg_softc *sc; + + sc = if_getsoftc(ifp); + + AWG_LOCK(sc); + awg_start_locked(sc); + AWG_UNLOCK(sc); +} + +static void +awg_tick(void *softc) +{ + struct awg_softc *sc; + struct mii_data *mii; + if_t ifp; + int link; + + sc = softc; + ifp = sc->ifp; + mii = device_get_softc(sc->miibus); + + AWG_ASSERT_LOCKED(sc); + + if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) + return; + + link = sc->link; + mii_tick(mii); + if (sc->link && !link) + awg_start_locked(sc); + + callout_reset(&sc->stat_ch, hz, awg_tick, sc); +} + +/* Bit Reversal - http://aggregate.org/MAGIC/#Bit%20Reversal */ +static uint32_t +bitrev32(uint32_t x) +{ + x = (((x & 0xaaaaaaaa) >> 1) | ((x & 0x55555555) << 1)); + x = (((x & 0xcccccccc) >> 2) | ((x & 0x33333333) << 2)); + x = (((x & 0xf0f0f0f0) >> 4) | ((x & 0x0f0f0f0f) << 4)); + x = (((x & 0xff00ff00) >> 8) | ((x & 0x00ff00ff) << 8)); + + return (x >> 16) | (x << 16); +} + +static void +awg_setup_rxfilter(struct awg_softc *sc) +{ + uint32_t val, crc, hashreg, hashbit, hash[2], machi, maclo; + int mc_count, mcnt, i; + uint8_t *eaddr, *mta; + if_t ifp; + + AWG_ASSERT_LOCKED(sc); + + ifp = sc->ifp; + val = 0; + hash[0] = hash[1] = 0; + + mc_count = if_multiaddr_count(ifp, -1); + + if (if_getflags(ifp) & IFF_PROMISC) + val |= DIS_ADDR_FILTER; + else if (if_getflags(ifp) & IFF_ALLMULTI) { + val |= RX_ALL_MULTICAST; + hash[0] = hash[1] = ~0; + } else if (mc_count > 0) { + val |= HASH_MULTICAST; + + mta = malloc(sizeof(unsigned char) * ETHER_ADDR_LEN * mc_count, + M_DEVBUF, M_NOWAIT); + if (mta == NULL) { + if_printf(ifp, + "failed to allocate temporary multicast list\n"); + return; + } + + if_multiaddr_array(ifp, mta, &mcnt, mc_count); + for (i = 0; i < mcnt; i++) { + crc = ether_crc32_le(mta + (i * ETHER_ADDR_LEN), + ETHER_ADDR_LEN) & 0x7f; + crc = bitrev32(~crc) >> 26; + hashreg = (crc >> 5); + hashbit = (crc & 0x1f); + hash[hashreg] |= (1 << hashbit); + } + + free(mta, M_DEVBUF); + } + + /* Write our unicast address */ + eaddr = IF_LLADDR(ifp); + machi = (eaddr[5] << 8) | eaddr[4]; + maclo = (eaddr[3] << 24) | (eaddr[2] << 16) | (eaddr[1] << 8) | + (eaddr[0] << 0); + WR4(sc, EMAC_ADDR_HIGH(0), machi); + WR4(sc, EMAC_ADDR_LOW(0), maclo); + + /* Multicast hash filters */ + WR4(sc, EMAC_RX_HASH_0, hash[1]); + WR4(sc, EMAC_RX_HASH_1, hash[0]); + + /* RX frame filter config */ + WR4(sc, EMAC_RX_FRM_FLT, val); +} + +static void +awg_init_locked(struct awg_softc *sc) +{ + struct mii_data *mii; + uint32_t val; + if_t ifp; + + mii = device_get_softc(sc->miibus); + ifp = sc->ifp; + + AWG_ASSERT_LOCKED(sc); + + if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) + return; + + awg_setup_rxfilter(sc); + + /* Configure DMA burst length and priorities */ + val = awg_burst_len << BASIC_CTL_BURST_LEN_SHIFT; + if (awg_rx_tx_pri) + val |= BASIC_CTL_RX_TX_PRI; + WR4(sc, EMAC_BASIC_CTL_1, val); + + /* Enable interrupts */ + WR4(sc, EMAC_INT_EN, RX_INT_EN | TX_INT_EN | TX_BUF_UA_INT_EN); + + /* Enable transmit DMA */ + val = RD4(sc, EMAC_TX_CTL_1); + WR4(sc, EMAC_TX_CTL_1, val | TX_DMA_EN | TX_MD); + + /* Enable receive DMA */ + val = RD4(sc, EMAC_RX_CTL_1); + WR4(sc, EMAC_RX_CTL_1, val | RX_DMA_EN | RX_MD); + + /* Enable transmitter */ + val = RD4(sc, EMAC_TX_CTL_0); + WR4(sc, EMAC_TX_CTL_0, val | TX_EN); + + /* Enable receiver */ + val = RD4(sc, EMAC_RX_CTL_0); + WR4(sc, EMAC_RX_CTL_0, val | RX_EN | CHECK_CRC); + + if_setdrvflagbits(ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE); + + mii_mediachg(mii); + callout_reset(&sc->stat_ch, hz, awg_tick, sc); +} + +static void +awg_init(void *softc) +{ + struct awg_softc *sc; + + sc = softc; + + AWG_LOCK(sc); + awg_init_locked(sc); + AWG_UNLOCK(sc); +} + +static void +awg_stop(struct awg_softc *sc) +{ + if_t ifp; + uint32_t val; + + AWG_ASSERT_LOCKED(sc); + + ifp = sc->ifp; + + callout_stop(&sc->stat_ch); + + /* Stop transmit DMA and flush data in the TX FIFO */ + val = RD4(sc, EMAC_TX_CTL_1); + val &= ~TX_DMA_EN; + val |= FLUSH_TX_FIFO; + WR4(sc, EMAC_TX_CTL_1, val); + + /* Disable transmitter */ + val = RD4(sc, EMAC_TX_CTL_0); + WR4(sc, EMAC_TX_CTL_0, val & ~TX_EN); + + /* Disable receiver */ + val = RD4(sc, EMAC_RX_CTL_0); + WR4(sc, EMAC_RX_CTL_0, val & ~RX_EN); + + /* Disable interrupts */ + WR4(sc, EMAC_INT_EN, 0); + + /* Disable transmit DMA */ + val = RD4(sc, EMAC_TX_CTL_1); + WR4(sc, EMAC_TX_CTL_1, val & ~TX_DMA_EN); + + /* Disable receive DMA */ + val = RD4(sc, EMAC_RX_CTL_1); + WR4(sc, EMAC_RX_CTL_1, val & ~RX_DMA_EN); + + sc->link = 0; + + if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING | IFF_DRV_OACTIVE); +} + +static void +awg_rxintr(struct awg_softc *sc) +{ + if_t ifp; + struct mbuf *m, *m0; + int error, index, len; + uint32_t status; + + ifp = sc->ifp; + + bus_dmamap_sync(sc->rx.desc_tag, sc->rx.desc_map, + BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); + + for (index = sc->rx.cur; ; index = RX_NEXT(index)) { + status = le32toh(sc->rx.desc_ring[index].status); + if ((status & RX_DESC_CTL) != 0) + break; + + bus_dmamap_sync(sc->rx.buf_tag, sc->rx.buf_map[index].map, + BUS_DMASYNC_POSTREAD); + bus_dmamap_unload(sc->rx.buf_tag, sc->rx.buf_map[index].map); + + len = (status & RX_FRM_LEN) >> RX_FRM_LEN_SHIFT; + if (len != 0) { + m = sc->rx.buf_map[index].mbuf; + m->m_pkthdr.rcvif = ifp; + m->m_pkthdr.len = len; + m->m_len = len; + if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); + + if ((if_getcapenable(ifp) & IFCAP_RXCSUM) != 0 && + (status & RX_FRM_TYPE) != 0) { + m->m_pkthdr.csum_flags = CSUM_IP_CHECKED; + if ((status & RX_HEADER_ERR) == 0) + m->m_pkthdr.csum_flags |= CSUM_IP_VALID; + if ((status & RX_PAYLOAD_ERR) == 0) { + m->m_pkthdr.csum_flags |= + CSUM_DATA_VALID | CSUM_PSEUDO_HDR; + m->m_pkthdr.csum_data = 0xffff; + } + } + + AWG_UNLOCK(sc); + if_input(ifp, m); + AWG_LOCK(sc); + } + + if ((m0 = awg_alloc_mbufcl(sc)) != NULL) { + error = awg_setup_rxbuf(sc, index, m0); + if (error != 0) { + /* XXX hole in RX ring */ + } + } else + if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1); + } + + if (index != sc->rx.cur) { + bus_dmamap_sync(sc->rx.desc_tag, sc->rx.desc_map, + BUS_DMASYNC_PREWRITE); + } + + sc->rx.cur = index; +} + +static void +awg_txintr(struct awg_softc *sc) +{ + struct awg_bufmap *bmap; + struct emac_desc *desc; + uint32_t status; + if_t ifp; + int i; + + AWG_ASSERT_LOCKED(sc); + + bus_dmamap_sync(sc->tx.desc_tag, sc->tx.desc_map, + BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); + + ifp = sc->ifp; + for (i = sc->tx.next; sc->tx.queued > 0; i = TX_NEXT(i)) { + desc = &sc->tx.desc_ring[i]; + status = le32toh(desc->status); + if ((status & TX_DESC_CTL) != 0) + break; + bmap = &sc->tx.buf_map[i]; + if (bmap->mbuf != NULL) { + bus_dmamap_sync(sc->tx.buf_tag, bmap->map, + BUS_DMASYNC_POSTWRITE); + bus_dmamap_unload(sc->tx.buf_tag, bmap->map); + m_freem(bmap->mbuf); + bmap->mbuf = NULL; + } + awg_setup_txdesc(sc, i, 0, 0, 0); + if_setdrvflagbits(ifp, 0, IFF_DRV_OACTIVE); + if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); + } + + sc->tx.next = i; + + bus_dmamap_sync(sc->tx.desc_tag, sc->tx.desc_map, + BUS_DMASYNC_PREWRITE); +} + +static void +awg_intr(void *arg) +{ + struct awg_softc *sc; + uint32_t val; + + sc = arg; + + AWG_LOCK(sc); + val = RD4(sc, EMAC_INT_STA); + WR4(sc, EMAC_INT_STA, val); + + if (val & RX_INT) + awg_rxintr(sc); + + if (val & (TX_INT|TX_BUF_UA_INT)) { + awg_txintr(sc); + if (!if_sendq_empty(sc->ifp)) + awg_start_locked(sc); + } + + AWG_UNLOCK(sc); +} + +static int +awg_ioctl(if_t ifp, u_long cmd, caddr_t data) +{ + struct awg_softc *sc; + struct mii_data *mii; + struct ifreq *ifr; + int flags, mask, error; + + sc = if_getsoftc(ifp); + mii = device_get_softc(sc->miibus); + ifr = (struct ifreq *)data; + error = 0; + + switch (cmd) { + case SIOCSIFFLAGS: + AWG_LOCK(sc); + if (if_getflags(ifp) & IFF_UP) { + if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { + flags = if_getflags(ifp) ^ sc->if_flags; + if ((flags & (IFF_PROMISC|IFF_ALLMULTI)) != 0) + awg_setup_rxfilter(sc); + } else + awg_init_locked(sc); + } else { + if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) + awg_stop(sc); + } + sc->if_flags = if_getflags(ifp); + AWG_UNLOCK(sc); + break; + case SIOCADDMULTI: + case SIOCDELMULTI: + if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) { + AWG_LOCK(sc); + awg_setup_rxfilter(sc); + AWG_UNLOCK(sc); + } + break; + case SIOCSIFMEDIA: + case SIOCGIFMEDIA: + error = ifmedia_ioctl(ifp, ifr, &mii->mii_media, cmd); + break; + case SIOCSIFCAP: + mask = ifr->ifr_reqcap ^ if_getcapenable(ifp); + if (mask & IFCAP_VLAN_MTU) + if_togglecapenable(ifp, IFCAP_VLAN_MTU); + if (mask & IFCAP_RXCSUM) + if_togglecapenable(ifp, IFCAP_RXCSUM); + if (mask & IFCAP_TXCSUM) + if_togglecapenable(ifp, IFCAP_TXCSUM); + if ((if_getcapenable(ifp) & (IFCAP_RXCSUM|IFCAP_TXCSUM)) != 0) + if_sethwassistbits(ifp, CSUM_IP, 0); + else + if_sethwassistbits(ifp, 0, CSUM_IP); + break; + default: + error = ether_ioctl(ifp, cmd, data); + break; + } + + return (error); +} + +static int +awg_setup_extres(device_t dev) +{ + struct awg_softc *sc; + hwreset_t rst_ahb; + clk_t clk_ahb, clk_tx, clk_tx_parent; + regulator_t reg; + const char *tx_parent_name; + char *phy_type; + phandle_t node; + uint64_t freq; + int error, div; + + sc = device_get_softc(dev); + node = ofw_bus_get_node(dev); + rst_ahb = NULL; + clk_ahb = NULL; + clk_tx = NULL; + clk_tx_parent = NULL; + reg = NULL; + phy_type = NULL; + + /* Get AHB clock and reset resources */ + error = hwreset_get_by_ofw_name(dev, "ahb", &rst_ahb); + if (error != 0) { + device_printf(dev, "cannot get ahb reset\n"); + goto fail; + } + error = clk_get_by_ofw_name(dev, "ahb", &clk_ahb); + if (error != 0) { + device_printf(dev, "cannot get ahb clock\n"); + goto fail; + } + + /* Configure PHY for MII or RGMII mode */ + if (OF_getprop_alloc(node, "phy-mode", 1, (void **)&phy_type)) { + if (bootverbose) + device_printf(dev, "PHY type: %s\n", phy_type); + + if (strcmp(phy_type, "rgmii") == 0) + tx_parent_name = "emac_int_tx"; + else + tx_parent_name = "mii_phy_tx"; + free(phy_type, M_OFWPROP); + + /* Get the TX clock */ + error = clk_get_by_ofw_name(dev, "tx", &clk_tx); + if (error != 0) { + device_printf(dev, "cannot get tx clock\n"); + goto fail; + } + + /* Find the desired parent clock based on phy-mode property */ + error = clk_get_by_name(dev, tx_parent_name, &clk_tx_parent); + if (error != 0) { + device_printf(dev, "cannot get clock '%s'\n", + tx_parent_name); + goto fail; + } + + /* Set TX clock parent */ + error = clk_set_parent_by_clk(clk_tx, clk_tx_parent); + if (error != 0) { + device_printf(dev, "cannot set tx clock parent\n"); + goto fail; + } + + /* Enable TX clock */ + error = clk_enable(clk_tx); + if (error != 0) { + device_printf(dev, "cannot enable tx clock\n"); + goto fail; + } + } + + /* Enable AHB clock */ + error = clk_enable(clk_ahb); + if (error != 0) { + device_printf(dev, "cannot enable ahb clock\n"); + goto fail; + } + + /* De-assert reset */ + error = hwreset_deassert(rst_ahb); + if (error != 0) { + device_printf(dev, "cannot de-assert ahb reset\n"); + goto fail; + } + + /* Enable PHY regulator if applicable */ + if (regulator_get_by_ofw_property(dev, "phy-supply", ®) == 0) { + error = regulator_enable(reg); + if (error != 0) { + device_printf(dev, "cannot enable PHY regulator\n"); + goto fail; + } + } + + /* Determine MDC clock divide ratio based on AHB clock */ + error = clk_get_freq(clk_ahb, &freq); + if (error != 0) { + device_printf(dev, "cannot get AHB clock frequency\n"); + goto fail; + } + div = freq / MDIO_FREQ; + if (div <= 16) + sc->mdc_div_ratio_m = MDC_DIV_RATIO_M_16; + else if (div <= 32) + sc->mdc_div_ratio_m = MDC_DIV_RATIO_M_32; + else if (div <= 64) + sc->mdc_div_ratio_m = MDC_DIV_RATIO_M_64; + else if (div <= 128) + sc->mdc_div_ratio_m = MDC_DIV_RATIO_M_128; + else { + device_printf(dev, "cannot determine MDC clock divide ratio\n"); + error = ENXIO; + goto fail; + } + + if (bootverbose) + device_printf(dev, "AHB frequency %llu Hz, MDC div: 0x%x\n", + freq, sc->mdc_div_ratio_m); + + return (0); + +fail: + free(phy_type, M_OFWPROP); + + if (reg != NULL) + regulator_release(reg); + if (clk_tx_parent != NULL) + clk_release(clk_tx_parent); + if (clk_tx != NULL) + clk_release(clk_tx); + if (clk_ahb != NULL) + clk_release(clk_ahb); + if (rst_ahb != NULL) + hwreset_release(rst_ahb); + return (error); +} + +static void +awg_get_eaddr(device_t dev, uint8_t *eaddr) +{ + struct awg_softc *sc; + uint32_t maclo, machi, rnd; + + sc = device_get_softc(dev); + + machi = RD4(sc, EMAC_ADDR_HIGH(0)) & 0xffff; + maclo = RD4(sc, EMAC_ADDR_LOW(0)); + + if (maclo == 0xffffffff && machi == 0xffff) { + /* MAC address in hardware is invalid, create one */ + rnd = arc4random(); + maclo = 0x00f2 | (rnd & 0xffff0000); + machi = rnd & 0xffff; + } + + eaddr[0] = maclo & 0xff; + eaddr[1] = (maclo >> 8) & 0xff; + eaddr[2] = (maclo >> 16) & 0xff; + eaddr[3] = (maclo >> 24) & 0xff; + eaddr[4] = machi & 0xff; + eaddr[5] = (machi >> 8) & 0xff; +} + +#ifdef AWG_DEBUG +static void +awg_dump_regs(device_t dev) +{ + static const struct { + const char *name; + u_int reg; + } regs[] = { + { "BASIC_CTL_0", EMAC_BASIC_CTL_0 }, + { "BASIC_CTL_1", EMAC_BASIC_CTL_1 }, + { "INT_STA", EMAC_INT_STA }, + { "INT_EN", EMAC_INT_EN }, + { "TX_CTL_0", EMAC_TX_CTL_0 }, + { "TX_CTL_1", EMAC_TX_CTL_1 }, + { "TX_FLOW_CTL", EMAC_TX_FLOW_CTL }, + { "TX_DMA_LIST", EMAC_TX_DMA_LIST }, + { "RX_CTL_0", EMAC_RX_CTL_0 }, + { "RX_CTL_1", EMAC_RX_CTL_1 }, + { "RX_DMA_LIST", EMAC_RX_DMA_LIST }, + { "RX_FRM_FLT", EMAC_RX_FRM_FLT }, + { "RX_HASH_0", EMAC_RX_HASH_0 }, + { "RX_HASH_1", EMAC_RX_HASH_1 }, + { "MII_CMD", EMAC_MII_CMD }, + { "ADDR_HIGH0", EMAC_ADDR_HIGH(0) }, + { "ADDR_LOW0", EMAC_ADDR_LOW(0) }, + { "TX_DMA_STA", EMAC_TX_DMA_STA }, + { "TX_DMA_CUR_DESC", EMAC_TX_DMA_CUR_DESC }, + { "TX_DMA_CUR_BUF", EMAC_TX_DMA_CUR_BUF }, + { "RX_DMA_STA", EMAC_RX_DMA_STA }, + { "RX_DMA_CUR_DESC", EMAC_RX_DMA_CUR_DESC }, + { "RX_DMA_CUR_BUF", EMAC_RX_DMA_CUR_BUF }, + { "RGMII_STA", EMAC_RGMII_STA }, + }; + struct awg_softc *sc; + unsigned int n; + + sc = device_get_softc(dev); + + for (n = 0; n < nitems(regs); n++) + device_printf(dev, " %-20s %08x\n", regs[n].name, + RD4(sc, regs[n].reg)); +} +#endif + +static int +awg_reset(device_t dev) +{ + struct awg_softc *sc; + int retry; + + sc = device_get_softc(dev); + + /* Soft reset all registers and logic */ + WR4(sc, EMAC_BASIC_CTL_1, BASIC_CTL_SOFT_RST); + + /* Wait for soft reset bit to self-clear */ + for (retry = SOFT_RST_RETRY; retry > 0; retry--) { + if ((RD4(sc, EMAC_BASIC_CTL_1) & BASIC_CTL_SOFT_RST) == 0) + break; + DELAY(10); + } + if (retry == 0) { + device_printf(dev, "soft reset timed out\n"); +#ifdef AWG_DEBUG + awg_dump_regs(dev); +#endif + return (ETIMEDOUT); + } + + return (0); +} + +static void +awg_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) +{ + if (error != 0) + return; + *(bus_addr_t *)arg = segs[0].ds_addr; +} + +static int +awg_setup_dma(device_t dev) +{ + struct awg_softc *sc; + struct mbuf *m; + int error, i; + + sc = device_get_softc(dev); + + /* Setup TX ring */ + error = bus_dma_tag_create( + bus_get_dma_tag(dev), /* Parent tag */ + DESC_ALIGN, 0, /* alignment, boundary */ + BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + TX_DESC_SIZE, 1, /* maxsize, nsegs */ + TX_DESC_SIZE, /* maxsegsize */ + 0, /* flags */ + NULL, NULL, /* lockfunc, lockarg */ + &sc->tx.desc_tag); + if (error != 0) { + device_printf(dev, "cannot create TX descriptor ring tag\n"); + return (error); + } + + error = bus_dmamem_alloc(sc->tx.desc_tag, (void **)&sc->tx.desc_ring, + BUS_DMA_COHERENT | BUS_DMA_WAITOK | BUS_DMA_ZERO, &sc->tx.desc_map); + if (error != 0) { + device_printf(dev, "cannot allocate TX descriptor ring\n"); + return (error); + } + + error = bus_dmamap_load(sc->tx.desc_tag, sc->tx.desc_map, + sc->tx.desc_ring, TX_DESC_SIZE, awg_dmamap_cb, + &sc->tx.desc_ring_paddr, 0); + if (error != 0) { + device_printf(dev, "cannot load TX descriptor ring\n"); + return (error); + } + + for (i = 0; i < TX_DESC_COUNT; i++) + sc->tx.desc_ring[i].next = + htole32(sc->tx.desc_ring_paddr + DESC_OFF(TX_NEXT(i))); + + error = bus_dma_tag_create( + bus_get_dma_tag(dev), /* Parent tag */ + 1, 0, /* alignment, boundary */ + BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + MCLBYTES, TX_MAX_SEGS, /* maxsize, nsegs */ + MCLBYTES, /* maxsegsize */ + 0, /* flags */ + NULL, NULL, /* lockfunc, lockarg */ + &sc->tx.buf_tag); + if (error != 0) { + device_printf(dev, "cannot create TX buffer tag\n"); + return (error); + } + + sc->tx.queued = TX_DESC_COUNT; + for (i = 0; i < TX_DESC_COUNT; i++) { + error = bus_dmamap_create(sc->tx.buf_tag, 0, + &sc->tx.buf_map[i].map); + if (error != 0) { + device_printf(dev, "cannot create TX buffer map\n"); + return (error); + } + awg_setup_txdesc(sc, i, 0, 0, 0); + } + + /* Setup RX ring */ + error = bus_dma_tag_create( + bus_get_dma_tag(dev), /* Parent tag */ + DESC_ALIGN, 0, /* alignment, boundary */ + BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + RX_DESC_SIZE, 1, /* maxsize, nsegs */ + RX_DESC_SIZE, /* maxsegsize */ + 0, /* flags */ + NULL, NULL, /* lockfunc, lockarg */ + &sc->rx.desc_tag); + if (error != 0) { + device_printf(dev, "cannot create RX descriptor ring tag\n"); + return (error); + } + + error = bus_dmamem_alloc(sc->rx.desc_tag, (void **)&sc->rx.desc_ring, + BUS_DMA_COHERENT | BUS_DMA_WAITOK | BUS_DMA_ZERO, &sc->rx.desc_map); + if (error != 0) { + device_printf(dev, "cannot allocate RX descriptor ring\n"); + return (error); + } + + error = bus_dmamap_load(sc->rx.desc_tag, sc->rx.desc_map, + sc->rx.desc_ring, RX_DESC_SIZE, awg_dmamap_cb, + &sc->rx.desc_ring_paddr, 0); + if (error != 0) { + device_printf(dev, "cannot load RX descriptor ring\n"); + return (error); + } + + error = bus_dma_tag_create( + bus_get_dma_tag(dev), /* Parent tag */ + 1, 0, /* alignment, boundary */ + BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ + BUS_SPACE_MAXADDR, /* highaddr */ + NULL, NULL, /* filter, filterarg */ + MCLBYTES, 1, /* maxsize, nsegs */ + MCLBYTES, /* maxsegsize */ + 0, /* flags */ + NULL, NULL, /* lockfunc, lockarg */ + &sc->rx.buf_tag); + if (error != 0) { + device_printf(dev, "cannot create RX buffer tag\n"); + return (error); + } + + for (i = 0; i < RX_DESC_COUNT; i++) { + error = bus_dmamap_create(sc->rx.buf_tag, 0, + &sc->rx.buf_map[i].map); + if (error != 0) { + device_printf(dev, "cannot create RX buffer map\n"); + return (error); + } + if ((m = awg_alloc_mbufcl(sc)) == NULL) { + device_printf(dev, "cannot allocate RX mbuf\n"); + return (ENOMEM); + } + error = awg_setup_rxbuf(sc, i, m); + if (error != 0) { + device_printf(dev, "cannot create RX buffer\n"); + return (error); + } + } + bus_dmamap_sync(sc->rx.desc_tag, sc->rx.desc_map, + BUS_DMASYNC_PREWRITE); + + /* Write transmit and receive descriptor base address registers */ + WR4(sc, EMAC_TX_DMA_LIST, sc->tx.desc_ring_paddr); + WR4(sc, EMAC_RX_DMA_LIST, sc->rx.desc_ring_paddr); + + return (0); +} + +static int +awg_probe(device_t dev) +{ + if (!ofw_bus_status_okay(dev)) + return (ENXIO); + + if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0) + return (ENXIO); + + device_set_desc(dev, "Allwinner Gigabit Ethernet"); + return (BUS_PROBE_DEFAULT); +} + +static int +awg_attach(device_t dev) +{ + uint8_t eaddr[ETHER_ADDR_LEN]; + struct awg_softc *sc; + phandle_t node; + int error; + + sc = device_get_softc(dev); + node = ofw_bus_get_node(dev); + + if (bus_alloc_resources(dev, awg_spec, sc->res) != 0) { + device_printf(dev, "cannot allocate resources for device\n"); + return (ENXIO); + } + + mtx_init(&sc->mtx, device_get_nameunit(dev), MTX_NETWORK_LOCK, MTX_DEF); + callout_init_mtx(&sc->stat_ch, &sc->mtx, 0); + TASK_INIT(&sc->link_task, 0, awg_link_task, sc); + + /* Setup clocks and regulators */ + error = awg_setup_extres(dev); + if (error != 0) + return (error); + + /* Read MAC address before resetting the chip */ + awg_get_eaddr(dev, eaddr); + + /* Soft reset EMAC core */ + error = awg_reset(dev); + if (error != 0) + return (error); + + /* Setup DMA descriptors */ + error = awg_setup_dma(dev); + if (error != 0) + return (error); + + /* Install interrupt handler */ + error = bus_setup_intr(dev, sc->res[1], INTR_TYPE_NET | INTR_MPSAFE, + NULL, awg_intr, sc, &sc->ih); + if (error != 0) { + device_printf(dev, "cannot setup interrupt handler\n"); + return (error); + } + + /* Setup ethernet interface */ + sc->ifp = if_alloc(IFT_ETHER); + if_setsoftc(sc->ifp, sc); + if_initname(sc->ifp, device_get_name(dev), device_get_unit(dev)); + if_setflags(sc->ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST); + if_setstartfn(sc->ifp, awg_start); + if_setioctlfn(sc->ifp, awg_ioctl); + if_setinitfn(sc->ifp, awg_init); + if_setsendqlen(sc->ifp, TX_DESC_COUNT - 1); + if_setsendqready(sc->ifp); + if_sethwassist(sc->ifp, CSUM_IP | CSUM_UDP | CSUM_TCP); + if_setcapabilities(sc->ifp, IFCAP_VLAN_MTU | IFCAP_HWCSUM); + if_setcapenable(sc->ifp, if_getcapabilities(sc->ifp)); + + /* Attach MII driver */ + error = mii_attach(dev, &sc->miibus, sc->ifp, awg_media_change, + awg_media_status, BMSR_DEFCAPMASK, MII_PHY_ANY, MII_OFFSET_ANY, + MIIF_DOPAUSE); + if (error != 0) { + device_printf(dev, "cannot attach PHY\n"); + return (error); + } + + /* Attach ethernet interface */ + ether_ifattach(sc->ifp, eaddr); + + return (0); +} + +static device_method_t awg_methods[] = { + /* Device interface */ + DEVMETHOD(device_probe, awg_probe), + DEVMETHOD(device_attach, awg_attach), + + /* MII interface */ + DEVMETHOD(miibus_readreg, awg_miibus_readreg), + DEVMETHOD(miibus_writereg, awg_miibus_writereg), + DEVMETHOD(miibus_statchg, awg_miibus_statchg), + + DEVMETHOD_END +}; + +static driver_t awg_driver = { + "awg", + awg_methods, + sizeof(struct awg_softc), +}; + +static devclass_t awg_devclass; + +DRIVER_MODULE(awg, simplebus, awg_driver, awg_devclass, 0, 0); +DRIVER_MODULE(miibus, awg, miibus_driver, miibus_devclass, 0, 0); + +MODULE_DEPEND(awg, ether, 1, 1, 1); +MODULE_DEPEND(awg, miibus, 1, 1, 1); Property changes on: user/ngie/release-pkg-fix-tests/sys/arm/allwinner/if_awg.c ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: user/ngie/release-pkg-fix-tests/sys/arm/allwinner/if_awgreg.h =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/allwinner/if_awgreg.h (nonexistent) +++ user/ngie/release-pkg-fix-tests/sys/arm/allwinner/if_awgreg.h (revision 299091) @@ -0,0 +1,181 @@ +/*- + * Copyright (c) 2016 Jared McNeill + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +/* + * Allwinner Gigabit Ethernet + */ + +#ifndef __IF_AWGREG_H__ +#define __IF_AWGREG_H__ + +#define EMAC_BASIC_CTL_0 0x00 +#define BASIC_CTL_SPEED (0x3 << 2) +#define BASIC_CTL_SPEED_SHIFT 2 +#define BASIC_CTL_SPEED_1000 0 +#define BASIC_CTL_SPEED_10 2 +#define BASIC_CTL_SPEED_100 3 +#define BASIC_CTL_LOOPBACK (1 << 1) +#define BASIC_CTL_DUPLEX (1 << 0) +#define EMAC_BASIC_CTL_1 0x04 +#define BASIC_CTL_BURST_LEN (0x3f << 24) +#define BASIC_CTL_BURST_LEN_SHIFT 24 +#define BASIC_CTL_RX_TX_PRI (1 << 1) +#define BASIC_CTL_SOFT_RST (1 << 0) +#define EMAC_INT_STA 0x08 +#define RX_BUF_UA_INT (1 << 10) +#define RX_INT (1 << 8) +#define TX_UNDERFLOW_INT (1 << 4) +#define TX_BUF_UA_INT (1 << 2) +#define TX_DMA_STOPPED_INT (1 << 1) +#define TX_INT (1 << 0) +#define EMAC_INT_EN 0x0c +#define RX_BUF_UA_INT_EN (1 << 10) +#define RX_INT_EN (1 << 8) +#define TX_UNDERFLOW_INT_EN (1 << 4) +#define TX_BUF_UA_INT_EN (1 << 2) +#define TX_DMA_STOPPED_INT_EN (1 << 1) +#define TX_INT_EN (1 << 0) +#define EMAC_TX_CTL_0 0x10 +#define TX_EN (1 << 31) +#define EMAC_TX_CTL_1 0x14 +#define TX_DMA_START (1 << 31) +#define TX_DMA_EN (1 << 30) +#define TX_MD (1 << 1) +#define FLUSH_TX_FIFO (1 << 0) +#define EMAC_TX_FLOW_CTL 0x1c +#define PAUSE_TIME (0xffff << 4) +#define PAUSE_TIME_SHIFT 4 +#define TX_FLOW_CTL_EN (1 << 0) +#define EMAC_TX_DMA_LIST 0x20 +#define EMAC_RX_CTL_0 0x24 +#define RX_EN (1 << 31) +#define JUMBO_FRM_EN (1 << 29) +#define STRIP_FCS (1 << 28) +#define CHECK_CRC (1 << 27) +#define RX_FLOW_CTL_EN (1 << 16) +#define EMAC_RX_CTL_1 0x28 +#define RX_DMA_START (1 << 31) +#define RX_DMA_EN (1 << 30) +#define RX_MD (1 << 1) +#define EMAC_RX_DMA_LIST 0x34 +#define EMAC_RX_FRM_FLT 0x38 +#define DIS_ADDR_FILTER (1 << 31) +#define DIS_BROADCAST (1 << 17) +#define RX_ALL_MULTICAST (1 << 16) +#define CTL_FRM_FILTER (0x3 << 12) +#define CTL_FRM_FILTER_SHIFT 12 +#define HASH_MULTICAST (1 << 9) +#define HASH_UNICAST (1 << 8) +#define SA_FILTER_EN (1 << 6) +#define SA_INV_FILTER (1 << 5) +#define DA_INV_FILTER (1 << 4) +#define FLT_MD (1 << 1) +#define RX_ALL (1 << 0) +#define EMAC_RX_HASH_0 0x40 +#define EMAC_RX_HASH_1 0x44 +#define EMAC_MII_CMD 0x48 +#define MDC_DIV_RATIO_M (0x7 << 20) +#define MDC_DIV_RATIO_M_16 0 +#define MDC_DIV_RATIO_M_32 1 +#define MDC_DIV_RATIO_M_64 2 +#define MDC_DIV_RATIO_M_128 3 +#define MDC_DIV_RATIO_M_SHIFT 20 +#define PHY_ADDR (0x1f << 12) +#define PHY_ADDR_SHIFT 12 +#define PHY_REG_ADDR (0x1f << 4) +#define PHY_REG_ADDR_SHIFT 4 +#define MII_WR (1 << 1) +#define MII_BUSY (1 << 0) +#define EMAC_MII_DATA 0x4c +#define EMAC_ADDR_HIGH(n) (0x50 + (n) * 8) +#define EMAC_ADDR_LOW(n) (0x54 + (n) * 8) +#define EMAC_TX_DMA_STA 0x80 +#define EMAC_TX_DMA_CUR_DESC 0x84 +#define EMAC_TX_DMA_CUR_BUF 0x88 +#define EMAC_RX_DMA_STA 0xc0 +#define EMAC_RX_DMA_CUR_DESC 0xc4 +#define EMAC_RX_DMA_CUR_BUF 0xc8 +#define EMAC_RGMII_STA 0xd0 + +struct emac_desc { + uint32_t status; +/* Transmit */ +#define TX_DESC_CTL (1 << 31) +#define TX_HEADER_ERR (1 << 16) +#define TX_LENGTH_ERR (1 << 14) +#define TX_PAYLOAD_ERR (1 << 12) +#define TX_CRS_ERR (1 << 10) +#define TX_COL_ERR_0 (1 << 9) +#define TX_COL_ERR_1 (1 << 8) +#define TX_COL_CNT (0xf << 3) +#define TX_COL_CNT_SHIFT 3 +#define TX_DEFER_ERR (1 << 2) +#define TX_UNDERFLOW_ERR (1 << 1) +#define TX_DEFER (1 << 0) +/* Receive */ +#define RX_DESC_CTL (1 << 31) +#define RX_DAF_FAIL (1 << 30) +#define RX_FRM_LEN (0x3fff << 16) +#define RX_FRM_LEN_SHIFT 16 +#define RX_NO_ENOUGH_BUF_ERR (1 << 14) +#define RX_SAF_FAIL (1 << 13) +#define RX_OVERFLOW_ERR (1 << 11) +#define RX_FIR_DESC (1 << 9) +#define RX_LAST_DESC (1 << 8) +#define RX_HEADER_ERR (1 << 7) +#define RX_COL_ERR (1 << 6) +#define RX_FRM_TYPE (1 << 5) +#define RX_LENGTH_ERR (1 << 4) +#define RX_PHY_ERR (1 << 3) +#define RX_CRC_ERR (1 << 1) +#define RX_PAYLOAD_ERR (1 << 0) + + uint32_t size; +/* Transmit */ +#define TX_INT_CTL (1 << 31) +#define TX_LAST_DESC (1 << 30) +#define TX_FIR_DESC (1 << 29) +#define TX_CHECKSUM_CTL (0x3 << 27) +#define TX_CHECKSUM_CTL_IP 1 +#define TX_CHECKSUM_CTL_NO_PSE 2 +#define TX_CHECKSUM_CTL_FULL 3 +#define TX_CHECKSUM_CTL_SHIFT 27 +#define TX_CRC_CTL (1 << 26) +#define TX_BUF_SIZE (0xfff << 0) +#define TX_BUF_SIZE_SHIFT 0 +/* Receive */ +#define RX_INT_CTL (1 << 31) +#define RX_BUF_SIZE (0xfff << 0) +#define RX_BUF_SIZE_SHIFT 0 + + uint32_t addr; + + uint32_t next; +} __packed; + +#endif /* !__IF_AWGREG_H__ */ Property changes on: user/ngie/release-pkg-fix-tests/sys/arm/allwinner/if_awgreg.h ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: user/ngie/release-pkg-fix-tests/sys/arm/amlogic/aml8726/aml8726_mmc.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/amlogic/aml8726/aml8726_mmc.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/amlogic/aml8726/aml8726_mmc.c (revision 299091) @@ -1,1102 +1,1102 @@ /*- * Copyright 2013-2015 John Wehle * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Amlogic aml8726 MMC host controller driver. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "gpio_if.h" #include "mmcbr_if.h" struct aml8726_mmc_gpio { device_t dev; uint32_t pin; uint32_t pol; }; struct aml8726_mmc_softc { device_t dev; struct resource *res[2]; struct mtx mtx; struct callout ch; uint32_t port; unsigned int ref_freq; struct aml8726_mmc_gpio pwr_en; int voltages[2]; struct aml8726_mmc_gpio vselect; bus_dma_tag_t dmatag; bus_dmamap_t dmamap; void *ih_cookie; struct mmc_host host; int bus_busy; struct mmc_command *cmd; uint32_t stop_timeout; }; static struct resource_spec aml8726_mmc_spec[] = { { SYS_RES_MEMORY, 0, RF_ACTIVE }, { SYS_RES_IRQ, 0, RF_ACTIVE }, { -1, 0 } }; #define AML_MMC_LOCK(sc) mtx_lock(&(sc)->mtx) #define AML_MMC_UNLOCK(sc) mtx_unlock(&(sc)->mtx) #define AML_MMC_LOCK_ASSERT(sc) mtx_assert(&(sc)->mtx, MA_OWNED) #define AML_MMC_LOCK_INIT(sc) \ mtx_init(&(sc)->mtx, device_get_nameunit((sc)->dev), \ "mmc", MTX_DEF) #define AML_MMC_LOCK_DESTROY(sc) mtx_destroy(&(sc)->mtx); #define CSR_WRITE_4(sc, reg, val) bus_write_4((sc)->res[0], reg, (val)) #define CSR_READ_4(sc, reg) bus_read_4((sc)->res[0], reg) #define CSR_BARRIER(sc, reg) bus_barrier((sc)->res[0], reg, 4, \ (BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE)) #define PWR_ON_FLAG(pol) ((pol) == 0 ? GPIO_PIN_LOW : \ GPIO_PIN_HIGH) #define PWR_OFF_FLAG(pol) ((pol) == 0 ? GPIO_PIN_HIGH : \ GPIO_PIN_LOW) #define MSECS_TO_TICKS(ms) (((ms)*hz)/1000 + 1) static void aml8726_mmc_timeout(void *arg); static unsigned int aml8726_mmc_clk(phandle_t node) { pcell_t prop; ssize_t len; phandle_t clk_node; len = OF_getencprop(node, "clocks", &prop, sizeof(prop)); if ((len / sizeof(prop)) != 1 || prop == 0 || (clk_node = OF_node_from_xref(prop)) == 0) return (0); len = OF_getencprop(clk_node, "clock-frequency", &prop, sizeof(prop)); if ((len / sizeof(prop)) != 1 || prop == 0) return (0); return ((unsigned int)prop); } static uint32_t aml8726_mmc_freq(struct aml8726_mmc_softc *sc, uint32_t divisor) { return (sc->ref_freq / ((divisor + 1) * 2)); } static uint32_t aml8726_mmc_div(struct aml8726_mmc_softc *sc, uint32_t desired_freq) { uint32_t divisor; divisor = sc->ref_freq / (desired_freq * 2); if (divisor == 0) divisor = 1; divisor -= 1; if (aml8726_mmc_freq(sc, divisor) > desired_freq) divisor += 1; if (divisor > (AML_MMC_CONFIG_CMD_CLK_DIV_MASK >> AML_MMC_CONFIG_CMD_CLK_DIV_SHIFT)) { divisor = AML_MMC_CONFIG_CMD_CLK_DIV_MASK >> AML_MMC_CONFIG_CMD_CLK_DIV_SHIFT; } return (divisor); } static void aml8726_mmc_mapmem(void *arg, bus_dma_segment_t *segs, int nseg, int error) { bus_addr_t *busaddrp; /* * There should only be one bus space address since * bus_dma_tag_create was called with nsegments = 1. */ busaddrp = (bus_addr_t *)arg; *busaddrp = segs->ds_addr; } static int aml8726_mmc_power_off(struct aml8726_mmc_softc *sc) { if (sc->pwr_en.dev == NULL) return (0); return (GPIO_PIN_SET(sc->pwr_en.dev, sc->pwr_en.pin, PWR_OFF_FLAG(sc->pwr_en.pol))); } static int aml8726_mmc_power_on(struct aml8726_mmc_softc *sc) { if (sc->pwr_en.dev == NULL) return (0); return (GPIO_PIN_SET(sc->pwr_en.dev, sc->pwr_en.pin, PWR_ON_FLAG(sc->pwr_en.pol))); } static void aml8726_mmc_soft_reset(struct aml8726_mmc_softc *sc, boolean_t enable_irq) { uint32_t icr; icr = AML_MMC_IRQ_CONFIG_SOFT_RESET; if (enable_irq == true) icr |= AML_MMC_IRQ_CONFIG_CMD_DONE_EN; CSR_WRITE_4(sc, AML_MMC_IRQ_CONFIG_REG, icr); CSR_BARRIER(sc, AML_MMC_IRQ_CONFIG_REG); } static int aml8726_mmc_start_command(struct aml8726_mmc_softc *sc, struct mmc_command *cmd) { struct mmc_ios *ios = &sc->host.ios; bus_addr_t baddr; uint32_t block_size; uint32_t bus_width; uint32_t cmdr; uint32_t extr; uint32_t mcfgr; uint32_t nbits_per_pkg; uint32_t timeout; int error; struct mmc_data *data; if (cmd->opcode > 0x3f) return (MMC_ERR_INVALID); /* * Ensure the hardware state machine is in a known state. */ aml8726_mmc_soft_reset(sc, true); /* * Start and transmission bits are per section 4.7.2 of the: * * SD Specifications Part 1 - * Physicaly Layer Simplified Specification + * Physical Layer Simplified Specification * Version 4.10 */ cmdr = AML_MMC_CMD_START_BIT | AML_MMC_CMD_TRANS_BIT_HOST | cmd->opcode; baddr = 0; extr = 0; mcfgr = sc->port; timeout = AML_MMC_CMD_TIMEOUT; /* * If this is a linked command, then use the previous timeout. */ if (cmd == cmd->mrq->stop && sc->stop_timeout) timeout = sc->stop_timeout; sc->stop_timeout = 0; if ((cmd->flags & MMC_RSP_136) != 0) { cmdr |= AML_MMC_CMD_RESP_CRC7_FROM_8; cmdr |= (133 << AML_MMC_CMD_RESP_BITS_SHIFT); } else if ((cmd->flags & MMC_RSP_PRESENT) != 0) cmdr |= (45 << AML_MMC_CMD_RESP_BITS_SHIFT); if ((cmd->flags & MMC_RSP_CRC) == 0) cmdr |= AML_MMC_CMD_RESP_NO_CRC7; if ((cmd->flags & MMC_RSP_BUSY) != 0) cmdr |= AML_MMC_CMD_CHECK_DAT0_BUSY; data = cmd->data; if (data && data->len && (data->flags & (MMC_DATA_READ | MMC_DATA_WRITE)) != 0) { block_size = data->len; if ((data->flags & MMC_DATA_MULTI) != 0) { block_size = MMC_SECTOR_SIZE; if ((data->len % block_size) != 0) return (MMC_ERR_INVALID); } cmdr |= (((data->len / block_size) - 1) << AML_MMC_CMD_REP_PKG_CNT_SHIFT); mcfgr |= (data->flags & MMC_DATA_STREAM) ? AML_MMC_MULT_CONFIG_STREAM_EN : 0; /* * The number of bits per package equals the number * of data bits + the number of CRC bits. There are * 16 bits of CRC calculate per bus line. * * A completed package appears to be detected by when * a counter decremented by the width underflows, thus * a value of zero always transfers 1 (or 4 bits depending * on the mode) which is why bus_width is subtracted. */ bus_width = (ios->bus_width == bus_width_4) ? 4 : 1; nbits_per_pkg = block_size * 8 + 16 * bus_width - bus_width; if (nbits_per_pkg > 0x3fff) return (MMC_ERR_INVALID); extr |= (nbits_per_pkg << AML_MMC_EXTENSION_PKT_SIZE_SHIFT); error = bus_dmamap_load(sc->dmatag, sc->dmamap, data->data, data->len, aml8726_mmc_mapmem, &baddr, BUS_DMA_NOWAIT); if (error) return (MMC_ERR_NO_MEMORY); if ((data->flags & MMC_DATA_READ) != 0) { cmdr |= AML_MMC_CMD_RESP_HAS_DATA; bus_dmamap_sync(sc->dmatag, sc->dmamap, BUS_DMASYNC_PREREAD); timeout = AML_MMC_READ_TIMEOUT * (data->len / block_size); } else { cmdr |= AML_MMC_CMD_CMD_HAS_DATA; bus_dmamap_sync(sc->dmatag, sc->dmamap, BUS_DMASYNC_PREWRITE); timeout = AML_MMC_WRITE_TIMEOUT * (data->len / block_size); } /* * Stop terminates a multiblock read / write and thus * can take as long to execute as an actual read / write. */ if (cmd->mrq->stop != NULL) sc->stop_timeout = timeout; } sc->cmd = cmd; cmd->error = MMC_ERR_NONE; if (timeout > AML_MMC_MAX_TIMEOUT) timeout = AML_MMC_MAX_TIMEOUT; callout_reset(&sc->ch, MSECS_TO_TICKS(timeout), aml8726_mmc_timeout, sc); CSR_WRITE_4(sc, AML_MMC_CMD_ARGUMENT_REG, cmd->arg); CSR_WRITE_4(sc, AML_MMC_MULT_CONFIG_REG, mcfgr); CSR_WRITE_4(sc, AML_MMC_EXTENSION_REG, extr); CSR_WRITE_4(sc, AML_MMC_DMA_ADDR_REG, (uint32_t)baddr); CSR_WRITE_4(sc, AML_MMC_CMD_SEND_REG, cmdr); CSR_BARRIER(sc, AML_MMC_CMD_SEND_REG); return (MMC_ERR_NONE); } static void aml8726_mmc_finish_command(struct aml8726_mmc_softc *sc, int mmc_error) { int mmc_stop_error; struct mmc_command *cmd; struct mmc_command *stop_cmd; struct mmc_data *data; AML_MMC_LOCK_ASSERT(sc); /* Clear all interrupts since the request is no longer in flight. */ CSR_WRITE_4(sc, AML_MMC_IRQ_STATUS_REG, AML_MMC_IRQ_STATUS_CLEAR_IRQ); CSR_BARRIER(sc, AML_MMC_IRQ_STATUS_REG); /* In some cases (e.g. finish called via timeout) this is a NOP. */ callout_stop(&sc->ch); cmd = sc->cmd; sc->cmd = NULL; cmd->error = mmc_error; data = cmd->data; if (data && data->len && (data->flags & (MMC_DATA_READ | MMC_DATA_WRITE)) != 0) { if ((data->flags & MMC_DATA_READ) != 0) bus_dmamap_sync(sc->dmatag, sc->dmamap, BUS_DMASYNC_POSTREAD); else bus_dmamap_sync(sc->dmatag, sc->dmamap, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(sc->dmatag, sc->dmamap); } /* * If there's a linked stop command, then start the stop command. * In order to establish a known state attempt the stop command * even if the original request encountered an error. */ stop_cmd = (cmd->mrq->stop != cmd) ? cmd->mrq->stop : NULL; if (stop_cmd != NULL) { mmc_stop_error = aml8726_mmc_start_command(sc, stop_cmd); if (mmc_stop_error == MMC_ERR_NONE) { AML_MMC_UNLOCK(sc); return; } stop_cmd->error = mmc_stop_error; } AML_MMC_UNLOCK(sc); /* Execute the callback after dropping the lock. */ if (cmd->mrq) cmd->mrq->done(cmd->mrq); } static void aml8726_mmc_timeout(void *arg) { struct aml8726_mmc_softc *sc = (struct aml8726_mmc_softc *)arg; /* * The command failed to complete in time so forcefully * terminate it. */ aml8726_mmc_soft_reset(sc, false); /* * Ensure the command has terminated before continuing on * to things such as bus_dmamap_sync / bus_dmamap_unload. */ while ((CSR_READ_4(sc, AML_MMC_IRQ_STATUS_REG) & AML_MMC_IRQ_STATUS_CMD_BUSY) != 0) cpu_spinwait(); aml8726_mmc_finish_command(sc, MMC_ERR_TIMEOUT); } static void aml8726_mmc_intr(void *arg) { struct aml8726_mmc_softc *sc = (struct aml8726_mmc_softc *)arg; uint32_t cmdr; uint32_t isr; uint32_t mcfgr; uint32_t previous_byte; uint32_t resp; int mmc_error; unsigned int i; AML_MMC_LOCK(sc); isr = CSR_READ_4(sc, AML_MMC_IRQ_STATUS_REG); cmdr = CSR_READ_4(sc, AML_MMC_CMD_SEND_REG); if (sc->cmd == NULL) goto spurious; mmc_error = MMC_ERR_NONE; if ((isr & AML_MMC_IRQ_STATUS_CMD_DONE_IRQ) != 0) { /* Check for CRC errors if the command has completed. */ if ((cmdr & AML_MMC_CMD_RESP_NO_CRC7) == 0 && (isr & AML_MMC_IRQ_STATUS_RESP_CRC7_OK) == 0) mmc_error = MMC_ERR_BADCRC; if ((cmdr & AML_MMC_CMD_RESP_HAS_DATA) != 0 && (isr & AML_MMC_IRQ_STATUS_RD_CRC16_OK) == 0) mmc_error = MMC_ERR_BADCRC; if ((cmdr & AML_MMC_CMD_CMD_HAS_DATA) != 0 && (isr & AML_MMC_IRQ_STATUS_WR_CRC16_OK) == 0) mmc_error = MMC_ERR_BADCRC; } else { spurious: /* * Clear spurious interrupts while leaving intacted any * interrupts that may have occurred after we read the * interrupt status register. */ CSR_WRITE_4(sc, AML_MMC_IRQ_STATUS_REG, (AML_MMC_IRQ_STATUS_CLEAR_IRQ & isr)); CSR_BARRIER(sc, AML_MMC_IRQ_STATUS_REG); AML_MMC_UNLOCK(sc); return; } if ((cmdr & AML_MMC_CMD_RESP_BITS_MASK) != 0) { mcfgr = sc->port; mcfgr |= AML_MMC_MULT_CONFIG_RESP_READOUT_EN; CSR_WRITE_4(sc, AML_MMC_MULT_CONFIG_REG, mcfgr); if ((cmdr & AML_MMC_CMD_RESP_CRC7_FROM_8) != 0) { /* * Controller supplies 135:8 instead of * 127:0 so discard the leading 8 bits * and provide a trailing 8 zero bits * where the CRC belongs. */ previous_byte = 0; for (i = 0; i < 4; i++) { resp = CSR_READ_4(sc, AML_MMC_CMD_ARGUMENT_REG); sc->cmd->resp[3 - i] = (resp << 8) | previous_byte; previous_byte = (resp >> 24) & 0xff; } } else sc->cmd->resp[0] = CSR_READ_4(sc, AML_MMC_CMD_ARGUMENT_REG); } if ((isr & AML_MMC_IRQ_STATUS_CMD_BUSY) != 0 && /* * A multiblock operation may keep the hardware * busy until stop transmission is executed. */ (isr & AML_MMC_IRQ_STATUS_CMD_DONE_IRQ) == 0) { if (mmc_error == MMC_ERR_NONE) mmc_error = MMC_ERR_FAILED; /* * Issue a soft reset to terminate the command. * * Ensure the command has terminated before continuing on * to things such as bus_dmamap_sync / bus_dmamap_unload. */ aml8726_mmc_soft_reset(sc, false); while ((CSR_READ_4(sc, AML_MMC_IRQ_STATUS_REG) & AML_MMC_IRQ_STATUS_CMD_BUSY) != 0) cpu_spinwait(); } aml8726_mmc_finish_command(sc, mmc_error); } static int aml8726_mmc_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, "amlogic,aml8726-mmc")) return (ENXIO); device_set_desc(dev, "Amlogic aml8726 MMC"); return (BUS_PROBE_DEFAULT); } static int aml8726_mmc_attach(device_t dev) { struct aml8726_mmc_softc *sc = device_get_softc(dev); char *function_name; char *voltages; char *voltage; int error; int nvoltages; pcell_t prop[3]; phandle_t node; ssize_t len; device_t child; sc->dev = dev; node = ofw_bus_get_node(dev); sc->ref_freq = aml8726_mmc_clk(node); if (sc->ref_freq == 0) { device_printf(dev, "missing clocks attribute in FDT\n"); return (ENXIO); } /* * The pins must be specified as part of the device in order * to know which port to used. */ len = OF_getencprop(node, "pinctrl-0", prop, sizeof(prop)); if ((len / sizeof(prop[0])) != 1 || prop[0] == 0) { device_printf(dev, "missing pinctrl-0 attribute in FDT\n"); return (ENXIO); } len = OF_getprop_alloc(OF_node_from_xref(prop[0]), "amlogic,function", sizeof(char), (void **)&function_name); if (len < 0) { device_printf(dev, "missing amlogic,function attribute in FDT\n"); return (ENXIO); } if (strncmp("sdio-a", function_name, len) == 0) sc->port = AML_MMC_MULT_CONFIG_PORT_A; else if (strncmp("sdio-b", function_name, len) == 0) sc->port = AML_MMC_MULT_CONFIG_PORT_B; else if (strncmp("sdio-c", function_name, len) == 0) sc->port = AML_MMC_MULT_CONFIG_PORT_C; else { device_printf(dev, "unknown function attribute %.*s in FDT\n", len, function_name); free(function_name, M_OFWPROP); return (ENXIO); } free(function_name, M_OFWPROP); sc->pwr_en.dev = NULL; len = OF_getencprop(node, "mmc-pwr-en", prop, sizeof(prop)); if (len > 0) { if ((len / sizeof(prop[0])) == 3) { sc->pwr_en.dev = OF_device_from_xref(prop[0]); sc->pwr_en.pin = prop[1]; sc->pwr_en.pol = prop[2]; } if (sc->pwr_en.dev == NULL) { device_printf(dev, "unable to process mmc-pwr-en attribute in FDT\n"); return (ENXIO); } /* Turn off power and then configure the output driver. */ if (aml8726_mmc_power_off(sc) != 0 || GPIO_PIN_SETFLAGS(sc->pwr_en.dev, sc->pwr_en.pin, GPIO_PIN_OUTPUT) != 0) { device_printf(dev, "could not use gpio to control power\n"); return (ENXIO); } } len = OF_getprop_alloc(node, "mmc-voltages", sizeof(char), (void **)&voltages); if (len < 0) { device_printf(dev, "missing mmc-voltages attribute in FDT\n"); return (ENXIO); } sc->voltages[0] = 0; sc->voltages[1] = 0; voltage = voltages; nvoltages = 0; while (len && nvoltages < 2) { if (strncmp("1.8", voltage, len) == 0) sc->voltages[nvoltages] = MMC_OCR_LOW_VOLTAGE; else if (strncmp("3.3", voltage, len) == 0) sc->voltages[nvoltages] = MMC_OCR_320_330 | MMC_OCR_330_340; else { device_printf(dev, "unknown voltage attribute %.*s in FDT\n", len, voltage); free(voltages, M_OFWPROP); return (ENXIO); } nvoltages++; /* queue up next string */ while (*voltage && len) { voltage++; len--; } if (len) { voltage++; len--; } } free(voltages, M_OFWPROP); sc->vselect.dev = NULL; len = OF_getencprop(node, "mmc-vselect", prop, sizeof(prop)); if (len > 0) { if ((len / sizeof(prop[0])) == 2) { sc->vselect.dev = OF_device_from_xref(prop[0]); sc->vselect.pin = prop[1]; sc->vselect.pol = 1; } if (sc->vselect.dev == NULL) { device_printf(dev, "unable to process mmc-vselect attribute in FDT\n"); return (ENXIO); } /* * With the power off select voltage 0 and then * configure the output driver. */ if (GPIO_PIN_SET(sc->vselect.dev, sc->vselect.pin, 0) != 0 || GPIO_PIN_SETFLAGS(sc->vselect.dev, sc->vselect.pin, GPIO_PIN_OUTPUT) != 0) { device_printf(dev, "could not use gpio to set voltage\n"); return (ENXIO); } } if (nvoltages == 0) { device_printf(dev, "no voltages in FDT\n"); return (ENXIO); } else if (nvoltages == 1 && sc->vselect.dev != NULL) { device_printf(dev, "only one voltage in FDT\n"); return (ENXIO); } else if (nvoltages == 2 && sc->vselect.dev == NULL) { device_printf(dev, "too many voltages in FDT\n"); return (ENXIO); } if (bus_alloc_resources(dev, aml8726_mmc_spec, sc->res)) { device_printf(dev, "could not allocate resources for device\n"); return (ENXIO); } AML_MMC_LOCK_INIT(sc); error = bus_dma_tag_create(bus_get_dma_tag(dev), AML_MMC_ALIGN_DMA, 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, AML_MMC_MAX_DMA, 1, AML_MMC_MAX_DMA, 0, NULL, NULL, &sc->dmatag); if (error) goto fail; error = bus_dmamap_create(sc->dmatag, 0, &sc->dmamap); if (error) goto fail; error = bus_setup_intr(dev, sc->res[1], INTR_TYPE_MISC | INTR_MPSAFE, NULL, aml8726_mmc_intr, sc, &sc->ih_cookie); if (error) { device_printf(dev, "could not setup interrupt handler\n"); goto fail; } callout_init_mtx(&sc->ch, &sc->mtx, CALLOUT_RETURNUNLOCKED); sc->host.f_min = aml8726_mmc_freq(sc, aml8726_mmc_div(sc, 200000)); sc->host.f_max = aml8726_mmc_freq(sc, aml8726_mmc_div(sc, 50000000)); sc->host.host_ocr = sc->voltages[0] | sc->voltages[1]; sc->host.caps = MMC_CAP_4_BIT_DATA | MMC_CAP_HSPEED; child = device_add_child(dev, "mmc", -1); if (!child) { device_printf(dev, "could not add mmc\n"); error = ENXIO; goto fail; } error = device_probe_and_attach(child); if (error) { device_printf(dev, "could not attach mmc\n"); goto fail; } return (0); fail: if (sc->ih_cookie) bus_teardown_intr(dev, sc->res[1], sc->ih_cookie); if (sc->dmamap) bus_dmamap_destroy(sc->dmatag, sc->dmamap); if (sc->dmatag) bus_dma_tag_destroy(sc->dmatag); AML_MMC_LOCK_DESTROY(sc); aml8726_mmc_power_off(sc); bus_release_resources(dev, aml8726_mmc_spec, sc->res); return (error); } static int aml8726_mmc_detach(device_t dev) { struct aml8726_mmc_softc *sc = device_get_softc(dev); AML_MMC_LOCK(sc); if (sc->cmd != NULL) { AML_MMC_UNLOCK(sc); return (EBUSY); } /* * Turn off the power, reset the hardware state machine, * disable the interrupts, and clear the interrupts. */ (void)aml8726_mmc_power_off(sc); aml8726_mmc_soft_reset(sc, false); CSR_WRITE_4(sc, AML_MMC_IRQ_STATUS_REG, AML_MMC_IRQ_STATUS_CLEAR_IRQ); /* This should be a NOP since no command was in flight. */ callout_stop(&sc->ch); AML_MMC_UNLOCK(sc); bus_generic_detach(dev); bus_teardown_intr(dev, sc->res[1], sc->ih_cookie); bus_dmamap_destroy(sc->dmatag, sc->dmamap); bus_dma_tag_destroy(sc->dmatag); AML_MMC_LOCK_DESTROY(sc); bus_release_resources(dev, aml8726_mmc_spec, sc->res); return (0); } static int aml8726_mmc_shutdown(device_t dev) { struct aml8726_mmc_softc *sc = device_get_softc(dev); /* * Turn off the power, reset the hardware state machine, * disable the interrupts, and clear the interrupts. */ (void)aml8726_mmc_power_off(sc); aml8726_mmc_soft_reset(sc, false); CSR_WRITE_4(sc, AML_MMC_IRQ_STATUS_REG, AML_MMC_IRQ_STATUS_CLEAR_IRQ); return (0); } static int aml8726_mmc_update_ios(device_t bus, device_t child) { struct aml8726_mmc_softc *sc = device_get_softc(bus); struct mmc_ios *ios = &sc->host.ios; int error; int i; uint32_t cfgr; cfgr = (2 << AML_MMC_CONFIG_WR_CRC_STAT_SHIFT) | (2 << AML_MMC_CONFIG_WR_DELAY_SHIFT) | AML_MMC_CONFIG_DMA_ENDIAN_SBW | (39 << AML_MMC_CONFIG_CMD_ARG_BITS_SHIFT); switch (ios->bus_width) { case bus_width_4: cfgr |= AML_MMC_CONFIG_BUS_WIDTH_4; break; case bus_width_1: cfgr |= AML_MMC_CONFIG_BUS_WIDTH_1; break; default: return (EINVAL); } cfgr |= aml8726_mmc_div(sc, ios->clock) << AML_MMC_CONFIG_CMD_CLK_DIV_SHIFT; CSR_WRITE_4(sc, AML_MMC_CONFIG_REG, cfgr); error = 0; switch (ios->power_mode) { case power_up: /* * Configure and power on the regulator so that the * voltage stabilizes prior to powering on the card. */ if (sc->vselect.dev != NULL) { for (i = 0; i < 2; i++) if ((sc->voltages[i] & (1 << ios->vdd)) != 0) break; if (i >= 2) return (EINVAL); error = GPIO_PIN_SET(sc->vselect.dev, sc->vselect.pin, i); } break; case power_on: error = aml8726_mmc_power_on(sc); break; case power_off: error = aml8726_mmc_power_off(sc); break; default: return (EINVAL); } return (error); } static int aml8726_mmc_request(device_t bus, device_t child, struct mmc_request *req) { struct aml8726_mmc_softc *sc = device_get_softc(bus); int mmc_error; AML_MMC_LOCK(sc); if (sc->cmd != NULL) { AML_MMC_UNLOCK(sc); return (EBUSY); } mmc_error = aml8726_mmc_start_command(sc, req->cmd); AML_MMC_UNLOCK(sc); /* Execute the callback after dropping the lock. */ if (mmc_error != MMC_ERR_NONE) { req->cmd->error = mmc_error; req->done(req); } return (0); } static int aml8726_mmc_read_ivar(device_t bus, device_t child, int which, uintptr_t *result) { struct aml8726_mmc_softc *sc = device_get_softc(bus); switch (which) { case MMCBR_IVAR_BUS_MODE: *(int *)result = sc->host.ios.bus_mode; break; case MMCBR_IVAR_BUS_WIDTH: *(int *)result = sc->host.ios.bus_width; break; case MMCBR_IVAR_CHIP_SELECT: *(int *)result = sc->host.ios.chip_select; break; case MMCBR_IVAR_CLOCK: *(int *)result = sc->host.ios.clock; break; case MMCBR_IVAR_F_MIN: *(int *)result = sc->host.f_min; break; case MMCBR_IVAR_F_MAX: *(int *)result = sc->host.f_max; break; case MMCBR_IVAR_HOST_OCR: *(int *)result = sc->host.host_ocr; break; case MMCBR_IVAR_MODE: *(int *)result = sc->host.mode; break; case MMCBR_IVAR_OCR: *(int *)result = sc->host.ocr; break; case MMCBR_IVAR_POWER_MODE: *(int *)result = sc->host.ios.power_mode; break; case MMCBR_IVAR_VDD: *(int *)result = sc->host.ios.vdd; break; case MMCBR_IVAR_CAPS: *(int *)result = sc->host.caps; break; case MMCBR_IVAR_MAX_DATA: *(int *)result = AML_MMC_MAX_DMA / MMC_SECTOR_SIZE; break; default: return (EINVAL); } return (0); } static int aml8726_mmc_write_ivar(device_t bus, device_t child, int which, uintptr_t value) { struct aml8726_mmc_softc *sc = device_get_softc(bus); switch (which) { case MMCBR_IVAR_BUS_MODE: sc->host.ios.bus_mode = value; break; case MMCBR_IVAR_BUS_WIDTH: sc->host.ios.bus_width = value; break; case MMCBR_IVAR_CHIP_SELECT: sc->host.ios.chip_select = value; break; case MMCBR_IVAR_CLOCK: sc->host.ios.clock = value; break; case MMCBR_IVAR_MODE: sc->host.mode = value; break; case MMCBR_IVAR_OCR: sc->host.ocr = value; break; case MMCBR_IVAR_POWER_MODE: sc->host.ios.power_mode = value; break; case MMCBR_IVAR_VDD: sc->host.ios.vdd = value; break; /* These are read-only */ case MMCBR_IVAR_CAPS: case MMCBR_IVAR_HOST_OCR: case MMCBR_IVAR_F_MIN: case MMCBR_IVAR_F_MAX: case MMCBR_IVAR_MAX_DATA: default: return (EINVAL); } return (0); } static int aml8726_mmc_get_ro(device_t bus, device_t child) { return (0); } static int aml8726_mmc_acquire_host(device_t bus, device_t child) { struct aml8726_mmc_softc *sc = device_get_softc(bus); AML_MMC_LOCK(sc); while (sc->bus_busy) mtx_sleep(sc, &sc->mtx, PZERO, "mmc", hz / 5); sc->bus_busy++; AML_MMC_UNLOCK(sc); return (0); } static int aml8726_mmc_release_host(device_t bus, device_t child) { struct aml8726_mmc_softc *sc = device_get_softc(bus); AML_MMC_LOCK(sc); sc->bus_busy--; wakeup(sc); AML_MMC_UNLOCK(sc); return (0); } static device_method_t aml8726_mmc_methods[] = { /* Device interface */ DEVMETHOD(device_probe, aml8726_mmc_probe), DEVMETHOD(device_attach, aml8726_mmc_attach), DEVMETHOD(device_detach, aml8726_mmc_detach), DEVMETHOD(device_shutdown, aml8726_mmc_shutdown), /* Bus interface */ DEVMETHOD(bus_read_ivar, aml8726_mmc_read_ivar), DEVMETHOD(bus_write_ivar, aml8726_mmc_write_ivar), /* MMC bridge interface */ DEVMETHOD(mmcbr_update_ios, aml8726_mmc_update_ios), DEVMETHOD(mmcbr_request, aml8726_mmc_request), DEVMETHOD(mmcbr_get_ro, aml8726_mmc_get_ro), DEVMETHOD(mmcbr_acquire_host, aml8726_mmc_acquire_host), DEVMETHOD(mmcbr_release_host, aml8726_mmc_release_host), DEVMETHOD_END }; static driver_t aml8726_mmc_driver = { "aml8726_mmc", aml8726_mmc_methods, sizeof(struct aml8726_mmc_softc), }; static devclass_t aml8726_mmc_devclass; DRIVER_MODULE(aml8726_mmc, simplebus, aml8726_mmc_driver, aml8726_mmc_devclass, 0, 0); MODULE_DEPEND(aml8726_mmc, aml8726_gpio, 1, 1, 1); DRIVER_MODULE(mmc, aml8726_mmc, mmc_driver, mmc_devclass, NULL, NULL); MODULE_DEPEND(aml8726_mmc, mmc, 1, 1, 1); Index: user/ngie/release-pkg-fix-tests/sys/arm/amlogic/aml8726/aml8726_mmc.h =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/amlogic/aml8726/aml8726_mmc.h (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/amlogic/aml8726/aml8726_mmc.h (revision 299091) @@ -1,123 +1,123 @@ /*- * Copyright 2013-2015 John Wehle * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _ARM_AMLOGIC_AML8726_MMC_H #define _ARM_AMLOGIC_AML8726_MMC_H #define AML_MMC_ALIGN_DMA 4 #define AML_MMC_MAX_DMA 4096 /* * Timeouts are in milliseconds * * Read and write are per section 4.6.2 of the: * * SD Specifications Part 1 - * Physicaly Layer Simplified Specification + * Physical Layer Simplified Specification * Version 4.10 */ #define AML_MMC_CMD_TIMEOUT 50 #define AML_MMC_READ_TIMEOUT 100 #define AML_MMC_WRITE_TIMEOUT 500 #define AML_MMC_MAX_TIMEOUT 5000 #define AML_MMC_CMD_ARGUMENT_REG 0 #define AML_MMC_CMD_SEND_REG 4 #define AML_MMC_CMD_REP_PKG_CNT_MASK (0xffU << 24) #define AML_MMC_CMD_REP_PKG_CNT_SHIFT 24 #define AML_MMC_CMD_CMD_HAS_DATA (1 << 20) #define AML_MMC_CMD_CHECK_DAT0_BUSY (1 << 19) #define AML_MMC_CMD_RESP_CRC7_FROM_8 (1 << 18) #define AML_MMC_CMD_RESP_HAS_DATA (1 << 17) #define AML_MMC_CMD_RESP_NO_CRC7 (1 << 16) #define AML_MMC_CMD_RESP_BITS_MASK (0xff << 8) #define AML_MMC_CMD_RESP_BITS_SHIFT 8 #define AML_MMC_CMD_START_BIT (0 << 7) #define AML_MMC_CMD_TRANS_BIT_HOST (1 << 6) #define AML_MMC_CMD_INDEX_MASK 0x3f #define AML_MMC_CMD_INDEX_SHIFT 0 #define AML_MMC_CONFIG_REG 8 #define AML_MMC_CONFIG_WR_CRC_STAT_MASK (7U << 29) #define AML_MMC_CONFIG_WR_CRC_STAT_SHIFT 29 #define AML_MMC_CONFIG_WR_DELAY_MASK (0x3f << 23) #define AML_MMC_CONFIG_WR_DELAY_SHIFT 23 #define AML_MMC_CONFIG_DMA_ENDIAN_MASK (3 << 21) #define AML_MMC_CONFIG_DMA_ENDIAN_NC (0 << 21) #define AML_MMC_CONFIG_DMA_ENDIAN_SB (1 << 21) #define AML_MMC_CONFIG_DMA_ENDIAN_SW (2 << 21) #define AML_MMC_CONFIG_DMA_ENDIAN_SBW (3 << 21) #define AML_MMC_CONFIG_BUS_WIDTH_MASK (1 << 20) #define AML_MMC_CONFIG_BUS_WIDTH_1 (0 << 20) #define AML_MMC_CONFIG_BUS_WIDTH_4 (1 << 20) #define AML_MMC_CONFIG_DATA_NEG_EDGE (1 << 19) #define AML_MMC_CONFIG_DONT_DELAY_DATA (1 << 18) #define AML_MMC_CONFIG_CMD_ARG_BITS_MASK (0x3f << 12) #define AML_MMC_CONFIG_CMD_ARG_BITS_SHIFT 12 #define AML_MMC_CONFIG_CMD_POS_EDGE (1 << 11) #define AML_MMC_CONFIG_CMD_NO_CRC (1 << 10) #define AML_MMC_CONFIG_CMD_CLK_DIV_MASK 0x3ff #define AML_MMC_CONFIG_CMD_CLK_DIV_SHIFT 0 #define AML_MMC_IRQ_STATUS_REG 12 #define AML_MMC_IRQ_STATUS_TIMER_CNT_MASK (0x1fffU << 19) #define AML_MMC_IRQ_STATUS_TIMER_CNT_SHIFT 19 #define AML_MMC_IRQ_STATUS_TIMER_EN (1 << 18) #define AML_MMC_IRQ_STATUS_TIMEOUT_IRQ (1 << 16) #define AML_MMC_IRQ_STATUS_CMD_DONE_IRQ (1 << 9) #define AML_MMC_IRQ_STATUS_WR_CRC16_OK (1 << 7) #define AML_MMC_IRQ_STATUS_RD_CRC16_OK (1 << 6) #define AML_MMC_IRQ_STATUS_RESP_CRC7_OK (1 << 5) #define AML_MMC_IRQ_STATUS_CMD_BUSY (1 << 4) #define AML_MMC_IRQ_STATUS_CLEAR_IRQ 0x10700 #define AML_MMC_IRQ_CONFIG_REG 16 #define AML_MMC_IRQ_CONFIG_SOFT_RESET (1 << 15) #define AML_MMC_IRQ_CONFIG_CMD_DONE_EN (1 << 4) #define AML_MMC_MULT_CONFIG_REG 20 #define AML_MMC_MULT_CONFIG_RESP_INDEX_MASK (0xf << 12) #define AML_MMC_MULT_CONFIG_RESP_INDEX_SHIFT 12 #define AML_MMC_MULT_CONFIG_RESP_READOUT_EN (1 << 8) #define AML_MMC_MULT_CONFIG_STREAM_8_MODE (1 << 5) #define AML_MMC_MULT_CONFIG_STREAM_EN (1 << 4) #define AML_MMC_MULT_CONFIG_PORT_MASK 3 #define AML_MMC_MULT_CONFIG_PORT_A 0 #define AML_MMC_MULT_CONFIG_PORT_B 1 #define AML_MMC_MULT_CONFIG_PORT_C 2 #define AML_MMC_DMA_ADDR_REG 24 #define AML_MMC_EXTENSION_REG 28 #define AML_MMC_EXTENSION_NO_CRC16 (1 << 30) #define AML_MMC_EXTENSION_PKT_SIZE_MASK (0x3fff << 16) #define AML_MMC_EXTENSION_PKT_SIZE_SHIFT 16 #endif /* _ARM_AMLOGIC_AML8726_MMC_H */ Index: user/ngie/release-pkg-fix-tests/sys/arm/amlogic/aml8726/aml8726_sdxc-m8.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/amlogic/aml8726/aml8726_sdxc-m8.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/amlogic/aml8726/aml8726_sdxc-m8.c (revision 299091) @@ -1,1381 +1,1381 @@ /*- * Copyright 2015 John Wehle * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Amlogic aml8726-m8 (and later) SDXC host controller driver. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "gpio_if.h" #include "mmcbr_if.h" /* * The table is sorted from highest to lowest and * last entry in the table is mark by freq == 0. */ struct { uint32_t voltage; uint32_t freq; uint32_t rx_phase; } aml8726_sdxc_clk_phases[] = { { MMC_OCR_LOW_VOLTAGE | MMC_OCR_320_330 | MMC_OCR_330_340, 100000000, 1 }, { MMC_OCR_320_330 | MMC_OCR_330_340, 45000000, 15 }, { MMC_OCR_LOW_VOLTAGE, 45000000, 11 }, { MMC_OCR_LOW_VOLTAGE | MMC_OCR_320_330 | MMC_OCR_330_340, 24999999, 15 }, { MMC_OCR_LOW_VOLTAGE | MMC_OCR_320_330 | MMC_OCR_330_340, 5000000, 23 }, { MMC_OCR_LOW_VOLTAGE | MMC_OCR_320_330 | MMC_OCR_330_340, 1000000, 55 }, { MMC_OCR_LOW_VOLTAGE | MMC_OCR_320_330 | MMC_OCR_330_340, 0, 1061 }, }; struct aml8726_sdxc_gpio { device_t dev; uint32_t pin; uint32_t pol; }; struct aml8726_sdxc_softc { device_t dev; boolean_t auto_fill_flush; struct resource *res[2]; struct mtx mtx; struct callout ch; unsigned int ref_freq; struct aml8726_sdxc_gpio pwr_en; int voltages[2]; struct aml8726_sdxc_gpio vselect; struct aml8726_sdxc_gpio card_rst; bus_dma_tag_t dmatag; bus_dmamap_t dmamap; void *ih_cookie; struct mmc_host host; int bus_busy; struct { uint32_t time; uint32_t error; } busy; struct mmc_command *cmd; }; static struct resource_spec aml8726_sdxc_spec[] = { { SYS_RES_MEMORY, 0, RF_ACTIVE }, { SYS_RES_IRQ, 0, RF_ACTIVE }, { -1, 0 } }; #define AML_SDXC_LOCK(sc) mtx_lock(&(sc)->mtx) #define AML_SDXC_UNLOCK(sc) mtx_unlock(&(sc)->mtx) #define AML_SDXC_LOCK_ASSERT(sc) mtx_assert(&(sc)->mtx, MA_OWNED) #define AML_SDXC_LOCK_INIT(sc) \ mtx_init(&(sc)->mtx, device_get_nameunit((sc)->dev), \ "sdxc", MTX_DEF) #define AML_SDXC_LOCK_DESTROY(sc) mtx_destroy(&(sc)->mtx); #define CSR_WRITE_4(sc, reg, val) bus_write_4((sc)->res[0], reg, (val)) #define CSR_READ_4(sc, reg) bus_read_4((sc)->res[0], reg) #define CSR_BARRIER(sc, reg) bus_barrier((sc)->res[0], reg, 4, \ (BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE)) #define PIN_ON_FLAG(pol) ((pol) == 0 ? \ GPIO_PIN_LOW : GPIO_PIN_HIGH) #define PIN_OFF_FLAG(pol) ((pol) == 0 ? \ GPIO_PIN_HIGH : GPIO_PIN_LOW) #define msecs_to_ticks(ms) (((ms)*hz)/1000 + 1) static void aml8726_sdxc_timeout(void *arg); static void aml8726_sdxc_mapmem(void *arg, bus_dma_segment_t *segs, int nseg, int error) { bus_addr_t *busaddrp; /* * There should only be one bus space address since * bus_dma_tag_create was called with nsegments = 1. */ busaddrp = (bus_addr_t *)arg; *busaddrp = segs->ds_addr; } static int aml8726_sdxc_power_off(struct aml8726_sdxc_softc *sc) { if (sc->pwr_en.dev == NULL) return (0); return (GPIO_PIN_SET(sc->pwr_en.dev, sc->pwr_en.pin, PIN_OFF_FLAG(sc->pwr_en.pol))); } static int aml8726_sdxc_power_on(struct aml8726_sdxc_softc *sc) { if (sc->pwr_en.dev == NULL) return (0); return (GPIO_PIN_SET(sc->pwr_en.dev, sc->pwr_en.pin, PIN_ON_FLAG(sc->pwr_en.pol))); } static void aml8726_sdxc_soft_reset(struct aml8726_sdxc_softc *sc) { CSR_WRITE_4(sc, AML_SDXC_SOFT_RESET_REG, AML_SDXC_SOFT_RESET); CSR_BARRIER(sc, AML_SDXC_SOFT_RESET_REG); DELAY(5); } static void aml8726_sdxc_engage_dma(struct aml8726_sdxc_softc *sc) { int i; uint32_t pdmar; uint32_t sr; struct mmc_data *data; data = sc->cmd->data; if (data == NULL || data->len == 0) return; /* * Engaging the DMA hardware is recommended before writing * to AML_SDXC_SEND_REG so that the FIFOs are ready to go. * * Presumably AML_SDXC_CNTRL_REG and AML_SDXC_DMA_ADDR_REG * must be set up prior to this happening. */ pdmar = CSR_READ_4(sc, AML_SDXC_PDMA_REG); pdmar &= ~AML_SDXC_PDMA_RX_FLUSH_MODE_SW; pdmar |= AML_SDXC_PDMA_DMA_EN; if (sc->auto_fill_flush == true) { CSR_WRITE_4(sc, AML_SDXC_PDMA_REG, pdmar); CSR_BARRIER(sc, AML_SDXC_PDMA_REG); return; } if ((data->flags & MMC_DATA_READ) != 0) { pdmar |= AML_SDXC_PDMA_RX_FLUSH_MODE_SW; CSR_WRITE_4(sc, AML_SDXC_PDMA_REG, pdmar); CSR_BARRIER(sc, AML_SDXC_PDMA_REG); } else { pdmar |= AML_SDXC_PDMA_TX_FILL; CSR_WRITE_4(sc, AML_SDXC_PDMA_REG, pdmar); CSR_BARRIER(sc, AML_SDXC_PDMA_REG); /* * Wait up to 100us for data to show up. */ for (i = 0; i < 100; i++) { sr = CSR_READ_4(sc, AML_SDXC_STATUS_REG); if ((sr & AML_SDXC_STATUS_TX_CNT_MASK) != 0) break; DELAY(1); } if (i >= 100) device_printf(sc->dev, "TX FIFO fill timeout\n"); } } static void aml8726_sdxc_disengage_dma(struct aml8726_sdxc_softc *sc) { int i; uint32_t pdmar; uint32_t sr; struct mmc_data *data; data = sc->cmd->data; if (data == NULL || data->len == 0) return; pdmar = CSR_READ_4(sc, AML_SDXC_PDMA_REG); if (sc->auto_fill_flush == true) { pdmar &= ~AML_SDXC_PDMA_DMA_EN; CSR_WRITE_4(sc, AML_SDXC_PDMA_REG, pdmar); CSR_BARRIER(sc, AML_SDXC_PDMA_REG); return; } if ((data->flags & MMC_DATA_READ) != 0) { pdmar |= AML_SDXC_PDMA_RX_FLUSH_NOW; CSR_WRITE_4(sc, AML_SDXC_PDMA_REG, pdmar); CSR_BARRIER(sc, AML_SDXC_PDMA_REG); /* * Wait up to 100us for data to drain. */ for (i = 0; i < 100; i++) { sr = CSR_READ_4(sc, AML_SDXC_STATUS_REG); if ((sr & AML_SDXC_STATUS_RX_CNT_MASK) == 0) break; DELAY(1); } if (i >= 100) device_printf(sc->dev, "RX FIFO drain timeout\n"); } pdmar &= ~(AML_SDXC_PDMA_DMA_EN | AML_SDXC_PDMA_RX_FLUSH_MODE_SW); CSR_WRITE_4(sc, AML_SDXC_PDMA_REG, pdmar); CSR_BARRIER(sc, AML_SDXC_PDMA_REG); } static int aml8726_sdxc_start_command(struct aml8726_sdxc_softc *sc, struct mmc_command *cmd) { bus_addr_t baddr; uint32_t block_size; uint32_t ctlr; uint32_t ier; uint32_t sndr; uint32_t timeout; int error; struct mmc_data *data; AML_SDXC_LOCK_ASSERT(sc); if (cmd->opcode > 0x3f) return (MMC_ERR_INVALID); /* * Ensure the hardware state machine is in a known state. */ aml8726_sdxc_soft_reset(sc); sndr = cmd->opcode; if ((cmd->flags & MMC_RSP_136) != 0) { sndr |= AML_SDXC_SEND_CMD_HAS_RESP; sndr |= AML_SDXC_SEND_RESP_136; /* * According to the SD spec the 136 bit response is * used for getting the CID or CSD in which case the * CRC7 is embedded in the contents rather than being * calculated over the entire response (the controller * always checks the CRC7 over the entire response). */ sndr |= AML_SDXC_SEND_RESP_NO_CRC7_CHECK; } else if ((cmd->flags & MMC_RSP_PRESENT) != 0) sndr |= AML_SDXC_SEND_CMD_HAS_RESP; if ((cmd->flags & MMC_RSP_CRC) == 0) sndr |= AML_SDXC_SEND_RESP_NO_CRC7_CHECK; if (cmd->opcode == MMC_STOP_TRANSMISSION) sndr |= AML_SDXC_SEND_DATA_STOP; data = cmd->data; baddr = 0; ctlr = CSR_READ_4(sc, AML_SDXC_CNTRL_REG); ier = AML_SDXC_IRQ_ENABLE_STANDARD; timeout = AML_SDXC_CMD_TIMEOUT; ctlr &= ~AML_SDXC_CNTRL_PKG_LEN_MASK; if (data && data->len && (data->flags & (MMC_DATA_READ | MMC_DATA_WRITE)) != 0) { block_size = data->len; if ((data->flags & MMC_DATA_MULTI) != 0) { block_size = MMC_SECTOR_SIZE; if ((data->len % block_size) != 0) return (MMC_ERR_INVALID); } if (block_size > 512) return (MMC_ERR_INVALID); sndr |= AML_SDXC_SEND_CMD_HAS_DATA; sndr |= ((data->flags & MMC_DATA_WRITE) != 0) ? AML_SDXC_SEND_DATA_WRITE : 0; sndr |= (((data->len / block_size) - 1) << AML_SDXC_SEND_REP_PKG_CNT_SHIFT); ctlr |= ((block_size < 512) ? block_size : 0) << AML_SDXC_CNTRL_PKG_LEN_SHIFT; ier &= ~AML_SDXC_IRQ_ENABLE_RESP_OK; ier |= (sc->auto_fill_flush == true || (data->flags & MMC_DATA_WRITE) != 0) ? AML_SDXC_IRQ_ENABLE_DMA_DONE : AML_SDXC_IRQ_ENABLE_TRANSFER_DONE_OK; error = bus_dmamap_load(sc->dmatag, sc->dmamap, data->data, data->len, aml8726_sdxc_mapmem, &baddr, BUS_DMA_NOWAIT); if (error) return (MMC_ERR_NO_MEMORY); if ((data->flags & MMC_DATA_READ) != 0) { bus_dmamap_sync(sc->dmatag, sc->dmamap, BUS_DMASYNC_PREREAD); timeout = AML_SDXC_READ_TIMEOUT * (data->len / block_size); } else { bus_dmamap_sync(sc->dmatag, sc->dmamap, BUS_DMASYNC_PREWRITE); timeout = AML_SDXC_WRITE_TIMEOUT * (data->len / block_size); } } sc->cmd = cmd; cmd->error = MMC_ERR_NONE; sc->busy.time = 0; sc->busy.error = MMC_ERR_NONE; if (timeout > AML_SDXC_MAX_TIMEOUT) timeout = AML_SDXC_MAX_TIMEOUT; callout_reset(&sc->ch, msecs_to_ticks(timeout), aml8726_sdxc_timeout, sc); CSR_WRITE_4(sc, AML_SDXC_IRQ_ENABLE_REG, ier); CSR_WRITE_4(sc, AML_SDXC_CNTRL_REG, ctlr); CSR_WRITE_4(sc, AML_SDXC_DMA_ADDR_REG, (uint32_t)baddr); CSR_WRITE_4(sc, AML_SDXC_CMD_ARGUMENT_REG, cmd->arg); aml8726_sdxc_engage_dma(sc); CSR_WRITE_4(sc, AML_SDXC_SEND_REG, sndr); CSR_BARRIER(sc, AML_SDXC_SEND_REG); return (MMC_ERR_NONE); } static void aml8726_sdxc_finish_command(struct aml8726_sdxc_softc *sc, int mmc_error) { int mmc_stop_error; struct mmc_command *cmd; struct mmc_command *stop_cmd; struct mmc_data *data; AML_SDXC_LOCK_ASSERT(sc); /* Clear all interrupts since the request is no longer in flight. */ CSR_WRITE_4(sc, AML_SDXC_IRQ_STATUS_REG, AML_SDXC_IRQ_STATUS_CLEAR); CSR_BARRIER(sc, AML_SDXC_IRQ_STATUS_REG); /* In some cases (e.g. finish called via timeout) this is a NOP. */ callout_stop(&sc->ch); cmd = sc->cmd; sc->cmd = NULL; cmd->error = mmc_error; data = cmd->data; if (data && data->len && (data->flags & (MMC_DATA_READ | MMC_DATA_WRITE)) != 0) { if ((data->flags & MMC_DATA_READ) != 0) bus_dmamap_sync(sc->dmatag, sc->dmamap, BUS_DMASYNC_POSTREAD); else bus_dmamap_sync(sc->dmatag, sc->dmamap, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(sc->dmatag, sc->dmamap); } /* * If there's a linked stop command, then start the stop command. * In order to establish a known state attempt the stop command * even if the original request encountered an error. */ stop_cmd = (cmd->mrq->stop != cmd) ? cmd->mrq->stop : NULL; if (stop_cmd != NULL) { /* - * If the original command executed successfuly, then + * If the original command executed successfully, then * the hardware will also have automatically executed * a stop command so don't bother with the one supplied * with the original request. */ if (mmc_error == MMC_ERR_NONE) { stop_cmd->error = MMC_ERR_NONE; stop_cmd->resp[0] = cmd->resp[0]; stop_cmd->resp[1] = cmd->resp[1]; stop_cmd->resp[2] = cmd->resp[2]; stop_cmd->resp[3] = cmd->resp[3]; } else { mmc_stop_error = aml8726_sdxc_start_command(sc, stop_cmd); if (mmc_stop_error == MMC_ERR_NONE) { AML_SDXC_UNLOCK(sc); return; } stop_cmd->error = mmc_stop_error; } } AML_SDXC_UNLOCK(sc); /* Execute the callback after dropping the lock. */ if (cmd->mrq != NULL) cmd->mrq->done(cmd->mrq); } static void aml8726_sdxc_timeout(void *arg) { struct aml8726_sdxc_softc *sc = (struct aml8726_sdxc_softc *)arg; /* * The command failed to complete in time so forcefully * terminate it. */ aml8726_sdxc_soft_reset(sc); /* * Ensure the command has terminated before continuing on * to things such as bus_dmamap_sync / bus_dmamap_unload. */ while ((CSR_READ_4(sc, AML_SDXC_STATUS_REG) & AML_SDXC_STATUS_BUSY) != 0) cpu_spinwait(); aml8726_sdxc_finish_command(sc, MMC_ERR_TIMEOUT); } static void aml8726_sdxc_busy_check(void *arg) { struct aml8726_sdxc_softc *sc = (struct aml8726_sdxc_softc *)arg; uint32_t sr; sc->busy.time += AML_SDXC_BUSY_POLL_INTVL; sr = CSR_READ_4(sc, AML_SDXC_STATUS_REG); if ((sr & AML_SDXC_STATUS_DAT0) == 0) { if (sc->busy.time < AML_SDXC_BUSY_TIMEOUT) { callout_reset(&sc->ch, msecs_to_ticks(AML_SDXC_BUSY_POLL_INTVL), aml8726_sdxc_busy_check, sc); AML_SDXC_UNLOCK(sc); return; } if (sc->busy.error == MMC_ERR_NONE) sc->busy.error = MMC_ERR_TIMEOUT; } aml8726_sdxc_finish_command(sc, sc->busy.error); } static void aml8726_sdxc_intr(void *arg) { struct aml8726_sdxc_softc *sc = (struct aml8726_sdxc_softc *)arg; uint32_t isr; uint32_t pdmar; uint32_t sndr; uint32_t sr; int i; int mmc_error; int start; int stop; AML_SDXC_LOCK(sc); isr = CSR_READ_4(sc, AML_SDXC_IRQ_STATUS_REG); sndr = CSR_READ_4(sc, AML_SDXC_SEND_REG); sr = CSR_READ_4(sc, AML_SDXC_STATUS_REG); if (sc->cmd == NULL) goto spurious; mmc_error = MMC_ERR_NONE; if ((isr & (AML_SDXC_IRQ_STATUS_TX_FIFO_EMPTY | AML_SDXC_IRQ_STATUS_RX_FIFO_FULL)) != 0) mmc_error = MMC_ERR_FIFO; else if ((isr & (AML_SDXC_IRQ_ENABLE_A_PKG_CRC_ERR | AML_SDXC_IRQ_ENABLE_RESP_CRC_ERR)) != 0) mmc_error = MMC_ERR_BADCRC; else if ((isr & (AML_SDXC_IRQ_ENABLE_A_PKG_TIMEOUT_ERR | AML_SDXC_IRQ_ENABLE_RESP_TIMEOUT_ERR)) != 0) mmc_error = MMC_ERR_TIMEOUT; else if ((isr & (AML_SDXC_IRQ_STATUS_RESP_OK | AML_SDXC_IRQ_STATUS_DMA_DONE | AML_SDXC_IRQ_STATUS_TRANSFER_DONE_OK)) != 0) { ; } else { spurious: /* * Clear spurious interrupts while leaving intacted any * interrupts that may have occurred after we read the * interrupt status register. */ CSR_WRITE_4(sc, AML_SDXC_IRQ_STATUS_REG, (AML_SDXC_IRQ_STATUS_CLEAR & isr)); CSR_BARRIER(sc, AML_SDXC_IRQ_STATUS_REG); AML_SDXC_UNLOCK(sc); return; } aml8726_sdxc_disengage_dma(sc); if ((sndr & AML_SDXC_SEND_CMD_HAS_RESP) != 0) { start = 0; stop = 1; if ((sndr & AML_SDXC_SEND_RESP_136) != 0) { start = 1; stop = start + 4; } for (i = start; i < stop; i++) { pdmar = CSR_READ_4(sc, AML_SDXC_PDMA_REG); pdmar &= ~(AML_SDXC_PDMA_DMA_EN | AML_SDXC_PDMA_RESP_INDEX_MASK); pdmar |= i << AML_SDXC_PDMA_RESP_INDEX_SHIFT; CSR_WRITE_4(sc, AML_SDXC_PDMA_REG, pdmar); sc->cmd->resp[(stop - 1) - i] = CSR_READ_4(sc, AML_SDXC_CMD_ARGUMENT_REG); } } if ((sr & AML_SDXC_STATUS_BUSY) != 0 && /* * A multiblock operation may keep the hardware * busy until stop transmission is executed. */ (isr & (AML_SDXC_IRQ_STATUS_DMA_DONE | AML_SDXC_IRQ_STATUS_TRANSFER_DONE_OK)) == 0) { if (mmc_error == MMC_ERR_NONE) mmc_error = MMC_ERR_FAILED; /* * Issue a soft reset to terminate the command. * * Ensure the command has terminated before continuing on * to things such as bus_dmamap_sync / bus_dmamap_unload. */ aml8726_sdxc_soft_reset(sc); while ((CSR_READ_4(sc, AML_SDXC_STATUS_REG) & AML_SDXC_STATUS_BUSY) != 0) cpu_spinwait(); } /* * The stop command can be generated either manually or * automatically by the hardware if MISC_MANUAL_STOP_MODE * has not been set. In either case check for busy. */ if (((sc->cmd->flags & MMC_RSP_BUSY) != 0 || (sndr & AML_SDXC_SEND_INDEX_MASK) == MMC_STOP_TRANSMISSION) && (sr & AML_SDXC_STATUS_DAT0) == 0) { sc->busy.error = mmc_error; callout_reset(&sc->ch, msecs_to_ticks(AML_SDXC_BUSY_POLL_INTVL), aml8726_sdxc_busy_check, sc); CSR_WRITE_4(sc, AML_SDXC_IRQ_STATUS_REG, (AML_SDXC_IRQ_STATUS_CLEAR & isr)); CSR_BARRIER(sc, AML_SDXC_IRQ_STATUS_REG); AML_SDXC_UNLOCK(sc); return; } aml8726_sdxc_finish_command(sc, mmc_error); } static int aml8726_sdxc_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, "amlogic,aml8726-sdxc-m8")) return (ENXIO); device_set_desc(dev, "Amlogic aml8726-m8 SDXC"); return (BUS_PROBE_DEFAULT); } static int aml8726_sdxc_attach(device_t dev) { struct aml8726_sdxc_softc *sc = device_get_softc(dev); char *voltages; char *voltage; int error; int nvoltages; pcell_t prop[3]; phandle_t node; ssize_t len; device_t child; uint32_t ectlr; uint32_t miscr; uint32_t pdmar; sc->dev = dev; sc->auto_fill_flush = false; pdmar = AML_SDXC_PDMA_DMA_URGENT | (49 << AML_SDXC_PDMA_TX_THOLD_SHIFT) | (7 << AML_SDXC_PDMA_RX_THOLD_SHIFT) | (15 << AML_SDXC_PDMA_RD_BURST_SHIFT) | (7 << AML_SDXC_PDMA_WR_BURST_SHIFT); miscr = (2 << AML_SDXC_MISC_WCRC_OK_PAT_SHIFT) | (5 << AML_SDXC_MISC_WCRC_ERR_PAT_SHIFT); ectlr = (12 << AML_SDXC_ENH_CNTRL_SDIO_IRQ_PERIOD_SHIFT); /* * Certain bitfields are dependent on the hardware revision. */ switch (aml8726_soc_hw_rev) { case AML_SOC_HW_REV_M8: switch (aml8726_soc_metal_rev) { case AML_SOC_M8_METAL_REV_M2_A: sc->auto_fill_flush = true; miscr |= (6 << AML_SDXC_MISC_TXSTART_THOLD_SHIFT); ectlr |= (64 << AML_SDXC_ENH_CNTRL_RX_FULL_THOLD_SHIFT) | AML_SDXC_ENH_CNTRL_WR_RESP_MODE_SKIP_M8M2; break; default: miscr |= (7 << AML_SDXC_MISC_TXSTART_THOLD_SHIFT); ectlr |= (63 << AML_SDXC_ENH_CNTRL_RX_FULL_THOLD_SHIFT) | AML_SDXC_ENH_CNTRL_DMA_NO_WR_RESP_CHECK_M8 | (255 << AML_SDXC_ENH_CNTRL_RX_TIMEOUT_SHIFT_M8); break; } break; case AML_SOC_HW_REV_M8B: miscr |= (7 << AML_SDXC_MISC_TXSTART_THOLD_SHIFT); ectlr |= (63 << AML_SDXC_ENH_CNTRL_RX_FULL_THOLD_SHIFT) | AML_SDXC_ENH_CNTRL_DMA_NO_WR_RESP_CHECK_M8 | (255 << AML_SDXC_ENH_CNTRL_RX_TIMEOUT_SHIFT_M8); break; default: device_printf(dev, "unsupported SoC\n"); return (ENXIO); /* NOTREACHED */ } node = ofw_bus_get_node(dev); len = OF_getencprop(node, "clock-frequency", prop, sizeof(prop)); if ((len / sizeof(prop[0])) != 1 || prop[0] == 0) { device_printf(dev, "missing clock-frequency attribute in FDT\n"); return (ENXIO); } sc->ref_freq = prop[0]; sc->pwr_en.dev = NULL; len = OF_getencprop(node, "mmc-pwr-en", prop, sizeof(prop)); if (len > 0) { if ((len / sizeof(prop[0])) == 3) { sc->pwr_en.dev = OF_device_from_xref(prop[0]); sc->pwr_en.pin = prop[1]; sc->pwr_en.pol = prop[2]; } if (sc->pwr_en.dev == NULL) { device_printf(dev, "unable to process mmc-pwr-en attribute in FDT\n"); return (ENXIO); } /* Turn off power and then configure the output driver. */ if (aml8726_sdxc_power_off(sc) != 0 || GPIO_PIN_SETFLAGS(sc->pwr_en.dev, sc->pwr_en.pin, GPIO_PIN_OUTPUT) != 0) { device_printf(dev, "could not use gpio to control power\n"); return (ENXIO); } } len = OF_getprop_alloc(node, "mmc-voltages", sizeof(char), (void **)&voltages); if (len < 0) { device_printf(dev, "missing mmc-voltages attribute in FDT\n"); return (ENXIO); } sc->voltages[0] = 0; sc->voltages[1] = 0; voltage = voltages; nvoltages = 0; while (len && nvoltages < 2) { if (strncmp("1.8", voltage, len) == 0) sc->voltages[nvoltages] = MMC_OCR_LOW_VOLTAGE; else if (strncmp("3.3", voltage, len) == 0) sc->voltages[nvoltages] = MMC_OCR_320_330 | MMC_OCR_330_340; else { device_printf(dev, "unknown voltage attribute %.*s in FDT\n", len, voltage); free(voltages, M_OFWPROP); return (ENXIO); } nvoltages++; /* queue up next string */ while (*voltage && len) { voltage++; len--; } if (len) { voltage++; len--; } } free(voltages, M_OFWPROP); sc->vselect.dev = NULL; len = OF_getencprop(node, "mmc-vselect", prop, sizeof(prop)); if (len > 0) { if ((len / sizeof(prop[0])) == 2) { sc->vselect.dev = OF_device_from_xref(prop[0]); sc->vselect.pin = prop[1]; sc->vselect.pol = 1; } if (sc->vselect.dev == NULL) { device_printf(dev, "unable to process mmc-vselect attribute in FDT\n"); return (ENXIO); } /* * With the power off select voltage 0 and then * configure the output driver. */ if (GPIO_PIN_SET(sc->vselect.dev, sc->vselect.pin, 0) != 0 || GPIO_PIN_SETFLAGS(sc->vselect.dev, sc->vselect.pin, GPIO_PIN_OUTPUT) != 0) { device_printf(dev, "could not use gpio to set voltage\n"); return (ENXIO); } } if (nvoltages == 0) { device_printf(dev, "no voltages in FDT\n"); return (ENXIO); } else if (nvoltages == 1 && sc->vselect.dev != NULL) { device_printf(dev, "only one voltage in FDT\n"); return (ENXIO); } else if (nvoltages == 2 && sc->vselect.dev == NULL) { device_printf(dev, "too many voltages in FDT\n"); return (ENXIO); } sc->card_rst.dev = NULL; len = OF_getencprop(node, "mmc-rst", prop, sizeof(prop)); if (len > 0) { if ((len / sizeof(prop[0])) == 3) { sc->card_rst.dev = OF_device_from_xref(prop[0]); sc->card_rst.pin = prop[1]; sc->card_rst.pol = prop[2]; } if (sc->card_rst.dev == NULL) { device_printf(dev, "unable to process mmc-rst attribute in FDT\n"); return (ENXIO); } } if (bus_alloc_resources(dev, aml8726_sdxc_spec, sc->res)) { device_printf(dev, "could not allocate resources for device\n"); return (ENXIO); } AML_SDXC_LOCK_INIT(sc); error = bus_dma_tag_create(bus_get_dma_tag(dev), AML_SDXC_ALIGN_DMA, 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, AML_SDXC_MAX_DMA, 1, AML_SDXC_MAX_DMA, 0, NULL, NULL, &sc->dmatag); if (error) goto fail; error = bus_dmamap_create(sc->dmatag, 0, &sc->dmamap); if (error) goto fail; error = bus_setup_intr(dev, sc->res[1], INTR_TYPE_MISC | INTR_MPSAFE, NULL, aml8726_sdxc_intr, sc, &sc->ih_cookie); if (error) { device_printf(dev, "could not setup interrupt handler\n"); goto fail; } callout_init_mtx(&sc->ch, &sc->mtx, CALLOUT_RETURNUNLOCKED); sc->host.f_min = 200000; sc->host.f_max = 100000000; sc->host.host_ocr = sc->voltages[0] | sc->voltages[1]; sc->host.caps = MMC_CAP_8_BIT_DATA | MMC_CAP_4_BIT_DATA | MMC_CAP_HSPEED; aml8726_sdxc_soft_reset(sc); CSR_WRITE_4(sc, AML_SDXC_PDMA_REG, pdmar); CSR_WRITE_4(sc, AML_SDXC_MISC_REG, miscr); CSR_WRITE_4(sc, AML_SDXC_ENH_CNTRL_REG, ectlr); child = device_add_child(dev, "mmc", -1); if (!child) { device_printf(dev, "could not add mmc\n"); error = ENXIO; goto fail; } error = device_probe_and_attach(child); if (error) { device_printf(dev, "could not attach mmc\n"); goto fail; } return (0); fail: if (sc->ih_cookie) bus_teardown_intr(dev, sc->res[1], sc->ih_cookie); if (sc->dmamap) bus_dmamap_destroy(sc->dmatag, sc->dmamap); if (sc->dmatag) bus_dma_tag_destroy(sc->dmatag); AML_SDXC_LOCK_DESTROY(sc); (void)aml8726_sdxc_power_off(sc); bus_release_resources(dev, aml8726_sdxc_spec, sc->res); return (error); } static int aml8726_sdxc_detach(device_t dev) { struct aml8726_sdxc_softc *sc = device_get_softc(dev); AML_SDXC_LOCK(sc); if (sc->cmd != NULL) { AML_SDXC_UNLOCK(sc); return (EBUSY); } /* * Turn off the power, reset the hardware state machine, * and disable the interrupts. */ aml8726_sdxc_power_off(sc); aml8726_sdxc_soft_reset(sc); CSR_WRITE_4(sc, AML_SDXC_IRQ_ENABLE_REG, 0); AML_SDXC_UNLOCK(sc); bus_generic_detach(dev); bus_teardown_intr(dev, sc->res[1], sc->ih_cookie); bus_dmamap_destroy(sc->dmatag, sc->dmamap); bus_dma_tag_destroy(sc->dmatag); AML_SDXC_LOCK_DESTROY(sc); bus_release_resources(dev, aml8726_sdxc_spec, sc->res); return (0); } static int aml8726_sdxc_shutdown(device_t dev) { struct aml8726_sdxc_softc *sc = device_get_softc(dev); /* * Turn off the power, reset the hardware state machine, * and disable the interrupts. */ aml8726_sdxc_power_off(sc); aml8726_sdxc_soft_reset(sc); CSR_WRITE_4(sc, AML_SDXC_IRQ_ENABLE_REG, 0); return (0); } static int aml8726_sdxc_update_ios(device_t bus, device_t child) { struct aml8726_sdxc_softc *sc = device_get_softc(bus); struct mmc_ios *ios = &sc->host.ios; unsigned int divisor; int error; int i; uint32_t cctlr; uint32_t clk2r; uint32_t ctlr; uint32_t freq; ctlr = (7 << AML_SDXC_CNTRL_TX_ENDIAN_SHIFT) | (7 << AML_SDXC_CNTRL_RX_ENDIAN_SHIFT) | (0xf << AML_SDXC_CNTRL_RX_PERIOD_SHIFT) | (0x7f << AML_SDXC_CNTRL_RX_TIMEOUT_SHIFT); switch (ios->bus_width) { case bus_width_8: ctlr |= AML_SDXC_CNTRL_BUS_WIDTH_8; break; case bus_width_4: ctlr |= AML_SDXC_CNTRL_BUS_WIDTH_4; break; case bus_width_1: ctlr |= AML_SDXC_CNTRL_BUS_WIDTH_1; break; default: return (EINVAL); } CSR_WRITE_4(sc, AML_SDXC_CNTRL_REG, ctlr); /* * Disable clocks and then clock module prior to setting desired values. */ cctlr = CSR_READ_4(sc, AML_SDXC_CLK_CNTRL_REG); cctlr &= ~(AML_SDXC_CLK_CNTRL_TX_CLK_EN | AML_SDXC_CLK_CNTRL_RX_CLK_EN | AML_SDXC_CLK_CNTRL_SD_CLK_EN); CSR_WRITE_4(sc, AML_SDXC_CLK_CNTRL_REG, cctlr); CSR_BARRIER(sc, AML_SDXC_CLK_CNTRL_REG); cctlr &= ~AML_SDXC_CLK_CNTRL_CLK_MODULE_EN; CSR_WRITE_4(sc, AML_SDXC_CLK_CNTRL_REG, cctlr); CSR_BARRIER(sc, AML_SDXC_CLK_CNTRL_REG); /* * aml8726-m8 * * Clock select 1 fclk_div2 (1.275 GHz) */ cctlr &= ~AML_SDXC_CLK_CNTRL_CLK_SEL_MASK; cctlr |= (1 << AML_SDXC_CLK_CNTRL_CLK_SEL_SHIFT); divisor = sc->ref_freq / ios->clock - 1; if (divisor == 0 || divisor == -1) divisor = 1; if ((sc->ref_freq / (divisor + 1)) > ios->clock) divisor += 1; if (divisor > (AML_SDXC_CLK_CNTRL_CLK_DIV_MASK >> AML_SDXC_CLK_CNTRL_CLK_DIV_SHIFT)) divisor = AML_SDXC_CLK_CNTRL_CLK_DIV_MASK >> AML_SDXC_CLK_CNTRL_CLK_DIV_SHIFT; cctlr &= ~AML_SDXC_CLK_CNTRL_CLK_DIV_MASK; cctlr |= divisor << AML_SDXC_CLK_CNTRL_CLK_DIV_SHIFT; cctlr &= ~AML_SDXC_CLK_CNTRL_MEM_PWR_MASK; cctlr |= AML_SDXC_CLK_CNTRL_MEM_PWR_ON; CSR_WRITE_4(sc, AML_SDXC_CLK_CNTRL_REG, cctlr); CSR_BARRIER(sc, AML_SDXC_CLK_CNTRL_REG); /* * Enable clock module and then clocks after setting desired values. */ cctlr |= AML_SDXC_CLK_CNTRL_CLK_MODULE_EN; CSR_WRITE_4(sc, AML_SDXC_CLK_CNTRL_REG, cctlr); CSR_BARRIER(sc, AML_SDXC_CLK_CNTRL_REG); cctlr |= AML_SDXC_CLK_CNTRL_TX_CLK_EN | AML_SDXC_CLK_CNTRL_RX_CLK_EN | AML_SDXC_CLK_CNTRL_SD_CLK_EN; CSR_WRITE_4(sc, AML_SDXC_CLK_CNTRL_REG, cctlr); CSR_BARRIER(sc, AML_SDXC_CLK_CNTRL_REG); freq = sc->ref_freq / divisor; for (i = 0; aml8726_sdxc_clk_phases[i].voltage; i++) { if ((aml8726_sdxc_clk_phases[i].voltage & (1 << ios->vdd)) != 0 && freq > aml8726_sdxc_clk_phases[i].freq) break; if (aml8726_sdxc_clk_phases[i].freq == 0) break; } clk2r = (1 << AML_SDXC_CLK2_SD_PHASE_SHIFT) | (aml8726_sdxc_clk_phases[i].rx_phase << AML_SDXC_CLK2_RX_PHASE_SHIFT); CSR_WRITE_4(sc, AML_SDXC_CLK2_REG, clk2r); CSR_BARRIER(sc, AML_SDXC_CLK2_REG); error = 0; switch (ios->power_mode) { case power_up: /* * Configure and power on the regulator so that the * voltage stabilizes prior to powering on the card. */ if (sc->vselect.dev != NULL) { for (i = 0; i < 2; i++) if ((sc->voltages[i] & (1 << ios->vdd)) != 0) break; if (i >= 2) return (EINVAL); error = GPIO_PIN_SET(sc->vselect.dev, sc->vselect.pin, i); } break; case power_on: error = aml8726_sdxc_power_on(sc); if (error) break; if (sc->card_rst.dev != NULL) { if (GPIO_PIN_SET(sc->card_rst.dev, sc->card_rst.pin, PIN_ON_FLAG(sc->card_rst.pol)) != 0 || GPIO_PIN_SETFLAGS(sc->card_rst.dev, sc->card_rst.pin, GPIO_PIN_OUTPUT) != 0) error = ENXIO; DELAY(5); if (GPIO_PIN_SET(sc->card_rst.dev, sc->card_rst.pin, PIN_OFF_FLAG(sc->card_rst.pol)) != 0) error = ENXIO; DELAY(5); if (error) { device_printf(sc->dev, "could not use gpio to reset card\n"); break; } } break; case power_off: error = aml8726_sdxc_power_off(sc); break; default: return (EINVAL); } return (error); } static int aml8726_sdxc_request(device_t bus, device_t child, struct mmc_request *req) { struct aml8726_sdxc_softc *sc = device_get_softc(bus); int mmc_error; AML_SDXC_LOCK(sc); if (sc->cmd != NULL) { AML_SDXC_UNLOCK(sc); return (EBUSY); } mmc_error = aml8726_sdxc_start_command(sc, req->cmd); AML_SDXC_UNLOCK(sc); /* Execute the callback after dropping the lock. */ if (mmc_error != MMC_ERR_NONE) { req->cmd->error = mmc_error; req->done(req); } return (0); } static int aml8726_sdxc_read_ivar(device_t bus, device_t child, int which, uintptr_t *result) { struct aml8726_sdxc_softc *sc = device_get_softc(bus); switch (which) { case MMCBR_IVAR_BUS_MODE: *(int *)result = sc->host.ios.bus_mode; break; case MMCBR_IVAR_BUS_WIDTH: *(int *)result = sc->host.ios.bus_width; break; case MMCBR_IVAR_CHIP_SELECT: *(int *)result = sc->host.ios.chip_select; break; case MMCBR_IVAR_CLOCK: *(int *)result = sc->host.ios.clock; break; case MMCBR_IVAR_F_MIN: *(int *)result = sc->host.f_min; break; case MMCBR_IVAR_F_MAX: *(int *)result = sc->host.f_max; break; case MMCBR_IVAR_HOST_OCR: *(int *)result = sc->host.host_ocr; break; case MMCBR_IVAR_MODE: *(int *)result = sc->host.mode; break; case MMCBR_IVAR_OCR: *(int *)result = sc->host.ocr; break; case MMCBR_IVAR_POWER_MODE: *(int *)result = sc->host.ios.power_mode; break; case MMCBR_IVAR_VDD: *(int *)result = sc->host.ios.vdd; break; case MMCBR_IVAR_CAPS: *(int *)result = sc->host.caps; break; case MMCBR_IVAR_MAX_DATA: *(int *)result = AML_SDXC_MAX_DMA / MMC_SECTOR_SIZE; break; default: return (EINVAL); } return (0); } static int aml8726_sdxc_write_ivar(device_t bus, device_t child, int which, uintptr_t value) { struct aml8726_sdxc_softc *sc = device_get_softc(bus); switch (which) { case MMCBR_IVAR_BUS_MODE: sc->host.ios.bus_mode = value; break; case MMCBR_IVAR_BUS_WIDTH: sc->host.ios.bus_width = value; break; case MMCBR_IVAR_CHIP_SELECT: sc->host.ios.chip_select = value; break; case MMCBR_IVAR_CLOCK: sc->host.ios.clock = value; break; case MMCBR_IVAR_MODE: sc->host.mode = value; break; case MMCBR_IVAR_OCR: sc->host.ocr = value; break; case MMCBR_IVAR_POWER_MODE: sc->host.ios.power_mode = value; break; case MMCBR_IVAR_VDD: sc->host.ios.vdd = value; break; /* These are read-only */ case MMCBR_IVAR_CAPS: case MMCBR_IVAR_HOST_OCR: case MMCBR_IVAR_F_MIN: case MMCBR_IVAR_F_MAX: case MMCBR_IVAR_MAX_DATA: default: return (EINVAL); } return (0); } static int aml8726_sdxc_get_ro(device_t bus, device_t child) { return (0); } static int aml8726_sdxc_acquire_host(device_t bus, device_t child) { struct aml8726_sdxc_softc *sc = device_get_softc(bus); AML_SDXC_LOCK(sc); while (sc->bus_busy) mtx_sleep(sc, &sc->mtx, PZERO, "sdxc", hz / 5); sc->bus_busy++; AML_SDXC_UNLOCK(sc); return (0); } static int aml8726_sdxc_release_host(device_t bus, device_t child) { struct aml8726_sdxc_softc *sc = device_get_softc(bus); AML_SDXC_LOCK(sc); sc->bus_busy--; wakeup(sc); AML_SDXC_UNLOCK(sc); return (0); } static device_method_t aml8726_sdxc_methods[] = { /* Device interface */ DEVMETHOD(device_probe, aml8726_sdxc_probe), DEVMETHOD(device_attach, aml8726_sdxc_attach), DEVMETHOD(device_detach, aml8726_sdxc_detach), DEVMETHOD(device_shutdown, aml8726_sdxc_shutdown), /* Bus interface */ DEVMETHOD(bus_read_ivar, aml8726_sdxc_read_ivar), DEVMETHOD(bus_write_ivar, aml8726_sdxc_write_ivar), /* MMC bridge interface */ DEVMETHOD(mmcbr_update_ios, aml8726_sdxc_update_ios), DEVMETHOD(mmcbr_request, aml8726_sdxc_request), DEVMETHOD(mmcbr_get_ro, aml8726_sdxc_get_ro), DEVMETHOD(mmcbr_acquire_host, aml8726_sdxc_acquire_host), DEVMETHOD(mmcbr_release_host, aml8726_sdxc_release_host), DEVMETHOD_END }; static driver_t aml8726_sdxc_driver = { "aml8726_sdxc", aml8726_sdxc_methods, sizeof(struct aml8726_sdxc_softc), }; static devclass_t aml8726_sdxc_devclass; DRIVER_MODULE(aml8726_sdxc, simplebus, aml8726_sdxc_driver, aml8726_sdxc_devclass, 0, 0); MODULE_DEPEND(aml8726_sdxc, aml8726_gpio, 1, 1, 1); DRIVER_MODULE(mmc, aml8726_sdxc, mmc_driver, mmc_devclass, NULL, NULL); MODULE_DEPEND(aml8726_sdxc, mmc, 1, 1, 1); Index: user/ngie/release-pkg-fix-tests/sys/arm/amlogic/aml8726/aml8726_sdxc-m8.h =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/amlogic/aml8726/aml8726_sdxc-m8.h (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/amlogic/aml8726/aml8726_sdxc-m8.h (revision 299091) @@ -1,223 +1,223 @@ /*- * Copyright 2015 John Wehle * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _ARM_AMLOGIC_AML8726_SDXC_M8_H #define _ARM_AMLOGIC_AML8726_SDXC_M8_H #define AML_SDXC_ALIGN_DMA 4 #define AML_SDXC_MAX_DMA 4096 /* * Timeouts are in milliseconds * * Read and write are per section 4.6.2 of the: * * SD Specifications Part 1 - * Physicaly Layer Simplified Specification + * Physical Layer Simplified Specification * Version 4.10 */ #define AML_SDXC_CMD_TIMEOUT 50 #define AML_SDXC_READ_TIMEOUT 100 #define AML_SDXC_WRITE_TIMEOUT 500 #define AML_SDXC_MAX_TIMEOUT 5000 #define AML_SDXC_BUSY_POLL_INTVL 1 #define AML_SDXC_BUSY_TIMEOUT 1000 /* * There's some disagreements between the S805 documentation * and the Amlogic Linux platform code regarding the exact * layout of various registers ... when in doubt we follow * the platform code. */ #define AML_SDXC_CMD_ARGUMENT_REG 0 #define AML_SDXC_SEND_REG 4 #define AML_SDXC_SEND_REP_PKG_CNT_MASK (0xffffU << 16) #define AML_SDXC_SEND_REP_PKG_CNT_SHIFT 16 #define AML_SDXC_SEND_DATA_STOP (1 << 11) #define AML_SDXC_SEND_DATA_WRITE (1 << 10) #define AML_SDXC_SEND_RESP_NO_CRC7_CHECK (1 << 9) #define AML_SDXC_SEND_RESP_136 (1 << 8) #define AML_SDXC_SEND_CMD_HAS_DATA (1 << 7) #define AML_SDXC_SEND_CMD_HAS_RESP (1 << 6) #define AML_SDXC_SEND_INDEX_MASK 0x3f #define AML_SDXC_SEND_INDEX_SHIFT 0 #define AML_SDXC_CNTRL_REG 8 #define AML_SDXC_CNTRL_TX_ENDIAN_MASK (7 << 29) #define AML_SDXC_CNTRL_TX_ENDIAN_SHIFT 29 #define AML_SDXC_CNTRL_RX_ENDIAN_MASK (7 << 24) #define AML_SDXC_CNTRL_RX_ENDIAN_SHIFT 24 #define AML_SDXC_CNTRL_RX_PERIOD_SHIFT 20 #define AML_SDXC_CNTRL_RX_TIMEOUT_SHIFT 13 #define AML_SDXC_CNTRL_PKG_LEN_MASK (0x1ff << 4) #define AML_SDXC_CNTRL_PKG_LEN_SHIFT 4 #define AML_SDXC_CNTRL_BUS_WIDTH_MASK (3 << 0) #define AML_SDXC_CNTRL_BUS_WIDTH_1 (0 << 0) #define AML_SDXC_CNTRL_BUS_WIDTH_4 (1 << 0) #define AML_SDXC_CNTRL_BUS_WIDTH_8 (2 << 0) #define AML_SDXC_STATUS_REG 12 #define AML_SDXC_STATUS_TX_CNT_MASK (0x7f << 13) #define AML_SDXC_STATUS_TX_CNT_SHIFT 13 #define AML_SDXC_STATUS_RX_CNT_MASK (0x7f << 6) #define AML_SDXC_STATUS_RX_CNT_SHIFT 6 #define AML_SDXC_STATUS_CMD (1 << 5) #define AML_SDXC_STATUS_DAT3 (1 << 4) #define AML_SDXC_STATUS_DAT2 (1 << 3) #define AML_SDXC_STATUS_DAT1 (1 << 2) #define AML_SDXC_STATUS_DAT0 (1 << 1) #define AML_SDXC_STATUS_BUSY (1 << 0) #define AML_SDXC_CLK_CNTRL_REG 16 #define AML_SDXC_CLK_CNTRL_MEM_PWR_MASK (3 << 25) #define AML_SDXC_CLK_CNTRL_MEM_PWR_OFF (3 << 25) #define AML_SDXC_CLK_CNTRL_MEM_PWR_ON (0 << 25) #define AML_SDXC_CLK_CNTRL_CLK_SEL_MASK (3 << 16) #define AML_SDXC_CLK_CNTRL_CLK_SEL_SHIFT 16 #define AML_SDXC_CLK_CNTRL_CLK_MODULE_EN (1 << 15) #define AML_SDXC_CLK_CNTRL_SD_CLK_EN (1 << 14) #define AML_SDXC_CLK_CNTRL_RX_CLK_EN (1 << 13) #define AML_SDXC_CLK_CNTRL_TX_CLK_EN (1 << 12) #define AML_SDXC_CLK_CNTRL_CLK_DIV_MASK 0x0fff #define AML_SDXC_CLK_CNTRL_CLK_DIV_SHIFT 0 #define AML_SDXC_DMA_ADDR_REG 20 #define AML_SDXC_PDMA_REG 24 #define AML_SDXC_PDMA_TX_FILL (1U << 31) #define AML_SDXC_PDMA_RX_FLUSH_NOW (1 << 30) #define AML_SDXC_PDMA_RX_FLUSH_MODE_SW (1 << 29) #define AML_SDXC_PDMA_TX_THOLD_MASK (0x3f << 22) #define AML_SDXC_PDMA_TX_THOLD_SHIFT 22 #define AML_SDXC_PDMA_RX_THOLD_MASK (0x3f << 15) #define AML_SDXC_PDMA_RX_THOLD_SHIFT 15 #define AML_SDXC_PDMA_RD_BURST_MASK (0x1f << 10) #define AML_SDXC_PDMA_RD_BURST_SHIFT 10 #define AML_SDXC_PDMA_WR_BURST_MASK (0x1f << 5) #define AML_SDXC_PDMA_WR_BURST_SHIFT 5 #define AML_SDXC_PDMA_DMA_URGENT (1 << 4) #define AML_SDXC_PDMA_RESP_INDEX_MASK (7 << 1) #define AML_SDXC_PDMA_RESP_INDEX_SHIFT 1 #define AML_SDXC_PDMA_DMA_EN (1 << 0) #define AML_SDXC_MISC_REG 28 #define AML_SDXC_MISC_TXSTART_THOLD_MASK (7U << 29) #define AML_SDXC_MISC_TXSTART_THOLD_SHIFT 29 #define AML_SDXC_MISC_MANUAL_STOP_MODE (1 << 28) #define AML_SDXC_MISC_WCRC_OK_PAT_MASK (7 << 7) #define AML_SDXC_MISC_WCRC_OK_PAT_SHIFT 7 #define AML_SDXC_MISC_WCRC_ERR_PAT_MASK (7 << 4) #define AML_SDXC_MISC_WCRC_ERR_PAT_SHIFT 4 #define AML_SDXC_DATA_REG 32 #define AML_SDXC_IRQ_ENABLE_REG 36 #define AML_SDXC_IRQ_ENABLE_TX_FIFO_EMPTY (1 << 13) #define AML_SDXC_IRQ_ENABLE_RX_FIFO_FULL (1 << 12) #define AML_SDXC_IRQ_ENABLE_DMA_DONE (1 << 11) #define AML_SDXC_IRQ_ENABLE_TRANSFER_DONE_OK (1 << 7) #define AML_SDXC_IRQ_ENABLE_A_PKG_CRC_ERR (1 << 6) #define AML_SDXC_IRQ_ENABLE_A_PKG_TIMEOUT_ERR (1 << 5) #define AML_SDXC_IRQ_ENABLE_A_PKG_DONE_OK (1 << 4) #define AML_SDXC_IRQ_ENABLE_RESP_CRC_ERR (1 << 2) #define AML_SDXC_IRQ_ENABLE_RESP_TIMEOUT_ERR (1 << 1) #define AML_SDXC_IRQ_ENABLE_RESP_OK (1 << 0) #define AML_SDXC_IRQ_ENABLE_STANDARD \ (AML_SDXC_IRQ_ENABLE_TX_FIFO_EMPTY | \ AML_SDXC_IRQ_ENABLE_RX_FIFO_FULL | \ AML_SDXC_IRQ_ENABLE_A_PKG_CRC_ERR | \ AML_SDXC_IRQ_ENABLE_A_PKG_TIMEOUT_ERR | \ AML_SDXC_IRQ_ENABLE_RESP_CRC_ERR | \ AML_SDXC_IRQ_ENABLE_RESP_TIMEOUT_ERR | \ AML_SDXC_IRQ_ENABLE_RESP_OK) #define AML_SDXC_IRQ_STATUS_REG 40 #define AML_SDXC_IRQ_STATUS_TX_FIFO_EMPTY (1 << 13) #define AML_SDXC_IRQ_STATUS_RX_FIFO_FULL (1 << 12) #define AML_SDXC_IRQ_STATUS_DMA_DONE (1 << 11) #define AML_SDXC_IRQ_STATUS_TRANSFER_DONE_OK (1 << 7) #define AML_SDXC_IRQ_STATUS_A_PKG_CRC_ERR (1 << 6) #define AML_SDXC_IRQ_STATUS_A_PKG_TIMEOUT_ERR (1 << 5) #define AML_SDXC_IRQ_STATUS_A_PKG_DONE_OK (1 << 4) #define AML_SDXC_IRQ_STATUS_RESP_CRC_ERR (1 << 2) #define AML_SDXC_IRQ_STATUS_RESP_TIMEOUT_ERR (1 << 1) #define AML_SDXC_IRQ_STATUS_RESP_OK (1 << 0) #define AML_SDXC_IRQ_STATUS_CLEAR \ (AML_SDXC_IRQ_STATUS_TX_FIFO_EMPTY | \ AML_SDXC_IRQ_STATUS_RX_FIFO_FULL | \ AML_SDXC_IRQ_STATUS_DMA_DONE | \ AML_SDXC_IRQ_STATUS_TRANSFER_DONE_OK | \ AML_SDXC_IRQ_STATUS_A_PKG_CRC_ERR | \ AML_SDXC_IRQ_STATUS_A_PKG_TIMEOUT_ERR | \ AML_SDXC_IRQ_STATUS_RESP_CRC_ERR | \ AML_SDXC_IRQ_STATUS_RESP_TIMEOUT_ERR | \ AML_SDXC_IRQ_STATUS_RESP_OK) #define AML_SDXC_SOFT_RESET_REG 44 #define AML_SDXC_SOFT_RESET_DMA (1 << 5) #define AML_SDXC_SOFT_RESET_TX_PHY (1 << 4) #define AML_SDXC_SOFT_RESET_RX_PHY (1 << 3) #define AML_SDXC_SOFT_RESET_TX_FIFO (1 << 2) #define AML_SDXC_SOFT_RESET_RX_FIFO (1 << 1) #define AML_SDXC_SOFT_RESET_MAIN (1 << 0) #define AML_SDXC_SOFT_RESET \ (AML_SDXC_SOFT_RESET_DMA | \ AML_SDXC_SOFT_RESET_TX_FIFO | \ AML_SDXC_SOFT_RESET_RX_FIFO | \ AML_SDXC_SOFT_RESET_MAIN) #define AML_SDXC_ENH_CNTRL_REG 52 #define AML_SDXC_ENH_CNTRL_TX_EMPTY_THOLD_MASK (0x7f << 25) #define AML_SDXC_ENH_CNTRL_TX_EMPTY_THOLD_SHIFT 25 #define AML_SDXC_ENH_CNTRL_RX_FULL_THOLD_MASK (0x7f << 18) #define AML_SDXC_ENH_CNTRL_RX_FULL_THOLD_SHIFT 18 #define AML_SDXC_ENH_CNTRL_SDIO_IRQ_PERIOD_MASK (0xff << 8) #define AML_SDXC_ENH_CNTRL_SDIO_IRQ_PERIOD_SHIFT 8 #define AML_SDXC_ENH_CNTRL_DMA_NO_WR_RESP_CHECK_M8 (1 << 17) #define AML_SDXC_ENH_CNTRL_DMA_NO_RD_RESP_CHECK_M8 (1 << 16) #define AML_SDXC_ENH_CNTRL_RX_TIMEOUT_MASK_M8 (0xff << 0) #define AML_SDXC_ENH_CNTRL_RX_TIMEOUT_SHIFT_M8 0 #define AML_SDXC_ENH_CNTRL_NO_DMA_CHECK_M8M2 (1 << 2) #define AML_SDXC_ENH_CNTRL_NO_WR_RESP_CHECK_M8M2 (1 << 1) #define AML_SDXC_ENH_CNTRL_WR_RESP_MODE_SKIP_M8M2 (1 << 0) #define AML_SDXC_CLK2_REG 56 #define AML_SDXC_CLK2_SD_PHASE_MASK (0x3ff << 12) #define AML_SDXC_CLK2_SD_PHASE_SHIFT 12 #define AML_SDXC_CLK2_RX_PHASE_MASK (0x3ff << 0) #define AML_SDXC_CLK2_RX_PHASE_SHIFT 0 #endif /* _ARM_AMLOGIC_AML8726_SDXC_M8_H */ Index: user/ngie/release-pkg-fix-tests/sys/arm/arm/cpufunc_asm_arm11.S =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/arm/cpufunc_asm_arm11.S (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/arm/cpufunc_asm_arm11.S (revision 299091) @@ -1,102 +1,102 @@ /* $NetBSD: cpufunc_asm_arm11.S,v 1.2 2005/12/11 12:16:41 christos Exp $ */ /* * Copyright (c) 2002, 2005 ARM Limited * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the company may not be used to endorse or promote * products derived from this software without specific prior written * permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * ARM11 assembly functions for CPU / MMU / TLB specific operations * - * XXX We make no attempt at present to take advantage of the v6 memroy + * XXX We make no attempt at present to take advantage of the v6 memory * architecture or physically tagged cache. */ #include __FBSDID("$FreeBSD$"); /* * TLB functions */ ENTRY(arm11_tlb_flushID_SE) mcr p15, 0, r0, c8, c6, 1 /* flush D tlb single entry */ mcr p15, 0, r0, c8, c5, 1 /* flush I tlb single entry */ mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ RET END(arm11_tlb_flushID_SE) /* * Context switch. * * These is the CPU-specific parts of the context switcher cpu_switch() * These functions actually perform the TTB reload. * * NOTE: Special calling convention * r1, r4-r13 must be preserved */ ENTRY(arm11_context_switch) /* * We can assume that the caches will only contain kernel addresses * at this point. So no need to flush them again. */ mcr p15, 0, r0, c7, c10, 4 /* drain the write buffer */ mcr p15, 0, r0, c2, c0, 0 /* set the new TTB */ mcr p15, 0, r0, c8, c7, 0 /* and flush the I+D tlbs */ /* Paranoia -- make sure the pipeline is empty. */ nop nop nop RET END(arm11_context_switch) /* * TLB functions */ ENTRY(arm11_tlb_flushID) mcr p15, 0, r0, c8, c7, 0 /* flush I+D tlb */ mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ mov pc, lr END(arm11_tlb_flushID) ENTRY(arm11_tlb_flushD) mcr p15, 0, r0, c8, c6, 0 /* flush D tlb */ mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ mov pc, lr END(arm11_tlb_flushD) ENTRY(arm11_tlb_flushD_SE) mcr p15, 0, r0, c8, c6, 1 /* flush D tlb single entry */ mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ mov pc, lr END(arm11_tlb_flushD_SE) /* * Other functions */ ENTRY(arm11_drain_writebuf) mcr p15, 0, r0, c7, c10, 4 /* drain write buffer */ mov pc, lr END(arm11_drain_writebuf) Index: user/ngie/release-pkg-fix-tests/sys/arm/arm/exception.S =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/arm/exception.S (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/arm/exception.S (revision 299091) @@ -1,504 +1,504 @@ /* $NetBSD: exception.S,v 1.13 2003/10/31 16:30:15 scw Exp $ */ /*- * Copyright (c) 1994-1997 Mark Brinicombe. * Copyright (c) 1994 Brini. * All rights reserved. * * This code is derived from software written for Brini by Mark Brinicombe * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Brini. * 4. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * RiscBSD kernel project * * exception.S * * Low level handlers for exception vectors * * Created : 24/09/94 * * Based on kate/display/abort.s * */ #include "assym.s" #include #include #include #include #include __FBSDID("$FreeBSD$"); #ifdef KDTRACE_HOOKS .bss .align 4 .global _C_LABEL(dtrace_invop_jump_addr) _C_LABEL(dtrace_invop_jump_addr): .word 0 .word 0 #endif .text .align 2 /* * ASM macros for pushing and pulling trapframes from the stack * * These macros are used to handle the irqframe and trapframe structures * defined above. */ /* * PUSHFRAME - macro to push a trap frame on the stack in the current mode * Since the current mode is used, the SVC lr field is not defined. * * NOTE: r13 and r14 are stored separately as a work around for the * SA110 rev 2 STM^ bug */ #if __ARM_ARCH < 6 #define PUSHFRAME \ sub sp, sp, #4; /* Align the stack */ \ str lr, [sp, #-4]!; /* Push the return address */ \ sub sp, sp, #(4*17); /* Adjust the stack pointer */ \ stmia sp, {r0-r12}; /* Push the user mode registers */ \ add r0, sp, #(4*13); /* Adjust the stack pointer */ \ stmia r0, {r13-r14}^; /* Push the user mode registers */ \ mov r0, r0; /* NOP for previous instruction */ \ mrs r0, spsr; /* Put the SPSR on the stack */ \ str r0, [sp, #-4]!; \ ldr r0, =ARM_RAS_START; \ mov r1, #0; \ str r1, [r0]; \ mov r1, #0xffffffff; \ str r1, [r0, #4]; #else #define PUSHFRAME \ sub sp, sp, #4; /* Align the stack */ \ str lr, [sp, #-4]!; /* Push the return address */ \ sub sp, sp, #(4*17); /* Adjust the stack pointer */ \ stmia sp, {r0-r12}; /* Push the user mode registers */ \ add r0, sp, #(4*13); /* Adjust the stack pointer */ \ stmia r0, {r13-r14}^; /* Push the user mode registers */ \ mov r0, r0; /* NOP for previous instruction */ \ mrs r0, spsr; /* Put the SPSR on the stack */ \ str r0, [sp, #-4]!; #endif /* * PULLFRAME - macro to pull a trap frame from the stack in the current mode * Since the current mode is used, the SVC lr field is ignored. */ #if __ARM_ARCH < 6 #define PULLFRAME \ ldr r0, [sp], #4; /* Get the SPSR from stack */ \ msr spsr_fsxc, r0; \ ldmia sp, {r0-r14}^; /* Restore registers (usr mode) */ \ mov r0, r0; /* NOP for previous instruction */ \ add sp, sp, #(4*17); /* Adjust the stack pointer */ \ ldr lr, [sp], #4; /* Pull the return address */ \ add sp, sp, #4 /* Align the stack */ #else #define PULLFRAME \ ldr r0, [sp], #4 ; /* Get the SPSR from stack */ \ msr spsr_fsxc, r0; \ clrex; \ ldmia sp, {r0-r14}^; /* Restore registers (usr mode) */ \ mov r0, r0; /* NOP for previous instruction */ \ add sp, sp, #(4*17); /* Adjust the stack pointer */ \ ldr lr, [sp], #4; /* Pull the return address */ \ add sp, sp, #4 /* Align the stack */ #endif /* * PUSHFRAMEINSVC - macro to push a trap frame on the stack in SVC32 mode * This should only be used if the processor is not currently in SVC32 * mode. The processor mode is switched to SVC mode and the trap frame is * stored. The SVC lr field is used to store the previous value of * lr in SVC mode. * * NOTE: r13 and r14 are stored separately as a work around for the * SA110 rev 2 STM^ bug */ #if __ARM_ARCH < 6 #define PUSHFRAMEINSVC \ stmdb sp, {r0-r3}; /* Save 4 registers */ \ mov r0, lr; /* Save xxx32 r14 */ \ mov r1, sp; /* Save xxx32 sp */ \ mrs r3, spsr; /* Save xxx32 spsr */ \ mrs r2, cpsr; /* Get the CPSR */ \ bic r2, r2, #(PSR_MODE); /* Fix for SVC mode */ \ orr r2, r2, #(PSR_SVC32_MODE); \ msr cpsr_c, r2; /* Punch into SVC mode */ \ mov r2, sp; /* Save SVC sp */ \ bic sp, sp, #7; /* Align sp to an 8-byte addrress */ \ sub sp, sp, #(4 * 17); /* Pad trapframe to keep alignment */ \ /* and for dtrace to emulate push/pop */ \ str r0, [sp, #-4]!; /* Push return address */ \ str lr, [sp, #-4]!; /* Push SVC lr */ \ str r2, [sp, #-4]!; /* Push SVC sp */ \ msr spsr_fsxc, r3; /* Restore correct spsr */ \ ldmdb r1, {r0-r3}; /* Restore 4 regs from xxx mode */ \ sub sp, sp, #(4*15); /* Adjust the stack pointer */ \ stmia sp, {r0-r12}; /* Push the user mode registers */ \ add r0, sp, #(4*13); /* Adjust the stack pointer */ \ stmia r0, {r13-r14}^; /* Push the user mode registers */ \ mov r0, r0; /* NOP for previous instruction */ \ ldr r5, =ARM_RAS_START; /* Check if there's any RAS */ \ ldr r4, [r5, #4]; /* reset it to point at the */ \ cmp r4, #0xffffffff; /* end of memory if necessary; */ \ movne r1, #0xffffffff; /* leave value in r4 for later */ \ - strne r1, [r5, #4]; /* comparision against PC. */ \ + strne r1, [r5, #4]; /* comparison against PC. */ \ ldr r3, [r5]; /* Retrieve global RAS_START */ \ cmp r3, #0; /* and reset it if non-zero. */ \ movne r1, #0; /* If non-zero RAS_START and */ \ strne r1, [r5]; /* PC was lower than RAS_END, */ \ ldrne r1, [r0, #16]; /* adjust the saved PC so that */ \ cmpne r4, r1; /* execution later resumes at */ \ strhi r3, [r0, #16]; /* the RAS_START location. */ \ mrs r0, spsr; \ str r0, [sp, #-4]! #else #define PUSHFRAMEINSVC \ stmdb sp, {r0-r3}; /* Save 4 registers */ \ mov r0, lr; /* Save xxx32 r14 */ \ mov r1, sp; /* Save xxx32 sp */ \ mrs r3, spsr; /* Save xxx32 spsr */ \ mrs r2, cpsr; /* Get the CPSR */ \ bic r2, r2, #(PSR_MODE); /* Fix for SVC mode */ \ orr r2, r2, #(PSR_SVC32_MODE); \ msr cpsr_c, r2; /* Punch into SVC mode */ \ mov r2, sp; /* Save SVC sp */ \ bic sp, sp, #7; /* Align sp to an 8-byte addrress */ \ sub sp, sp, #(4 * 17); /* Pad trapframe to keep alignment */ \ /* and for dtrace to emulate push/pop */ \ str r0, [sp, #-4]!; /* Push return address */ \ str lr, [sp, #-4]!; /* Push SVC lr */ \ str r2, [sp, #-4]!; /* Push SVC sp */ \ msr spsr_fsxc, r3; /* Restore correct spsr */ \ ldmdb r1, {r0-r3}; /* Restore 4 regs from xxx mode */ \ sub sp, sp, #(4*15); /* Adjust the stack pointer */ \ stmia sp, {r0-r12}; /* Push the user mode registers */ \ add r0, sp, #(4*13); /* Adjust the stack pointer */ \ stmia r0, {r13-r14}^; /* Push the user mode registers */ \ mov r0, r0; /* NOP for previous instruction */ \ mrs r0, spsr; /* Put the SPSR on the stack */ \ str r0, [sp, #-4]! #endif /* * PULLFRAMEFROMSVCANDEXIT - macro to pull a trap frame from the stack * in SVC32 mode and restore the saved processor mode and PC. * This should be used when the SVC lr register needs to be restored on * exit. */ #if __ARM_ARCH < 6 #define PULLFRAMEFROMSVCANDEXIT \ ldr r0, [sp], #4; /* Get the SPSR from stack */ \ msr spsr_fsxc, r0; /* restore SPSR */ \ ldmia sp, {r0-r14}^; /* Restore registers (usr mode) */ \ mov r0, r0; /* NOP for previous instruction */ \ add sp, sp, #(4*15); /* Adjust the stack pointer */ \ ldmia sp, {sp, lr, pc}^ /* Restore lr and exit */ #else #define PULLFRAMEFROMSVCANDEXIT \ ldr r0, [sp], #4; /* Get the SPSR from stack */ \ msr spsr_fsxc, r0; /* restore SPSR */ \ clrex; \ ldmia sp, {r0-r14}^; /* Restore registers (usr mode) */ \ mov r0, r0; /* NOP for previous instruction */ \ add sp, sp, #(4*15); /* Adjust the stack pointer */ \ ldmia sp, {sp, lr, pc}^ /* Restore lr and exit */ #endif /* * Unwind hints so we can unwind past functions that use * PULLFRAMEFROMSVCANDEXIT. They are run in reverse order. * As the last thing we do is restore the stack pointer * we can ignore the padding at the end of struct trapframe. */ #define UNWINDSVCFRAME \ .save {r13-r15}; /* Restore sp, lr, pc */ \ .pad #(2*4); /* Skip user sp and lr */ \ .save {r0-r12}; /* Restore r0-r12 */ \ .pad #(4) /* Skip spsr */ #define DO_AST \ ldr r0, [sp]; /* Get the SPSR from stack */ \ mrs r4, cpsr; /* save CPSR */ \ orr r1, r4, #(PSR_I|PSR_F); \ msr cpsr_c, r1; /* Disable interrupts */ \ and r0, r0, #(PSR_MODE); /* Returning to USR mode? */ \ teq r0, #(PSR_USR32_MODE); \ bne 2f; /* Nope, get out now */ \ bic r4, r4, #(PSR_I|PSR_F); \ 1: GET_CURTHREAD_PTR(r5); \ ldr r1, [r5, #(TD_FLAGS)]; \ and r1, r1, #(TDF_ASTPENDING|TDF_NEEDRESCHED); \ teq r1, #0; \ beq 2f; /* Nope. Just bail */ \ msr cpsr_c, r4; /* Restore interrupts */ \ mov r0, sp; \ bl _C_LABEL(ast); /* ast(frame) */ \ orr r0, r4, #(PSR_I|PSR_F); \ msr cpsr_c, r0; \ b 1b; \ 2: /* * Entry point for a Software Interrupt (SWI). * * The hardware switches to svc32 mode on a swi, so we're already on the * right stack; just build a trapframe and call the handler. */ ASENTRY_NP(swi_entry) PUSHFRAME /* Build the trapframe on the */ mov r0, sp /* scv32 stack, pass it to the */ bl _C_LABEL(swi_handler) /* swi handler. */ /* * The fork_trampoline() code in swtch.S aranges for the MI fork_exit() * to return to swi_exit here, to return to userland. The net effect is * that a newly created thread appears to return from a SWI just like * the parent thread that created it. */ ASEENTRY_NP(swi_exit) DO_AST /* Handle pending signals. */ PULLFRAME /* Deallocate trapframe. */ movs pc, lr /* Return to userland. */ STOP_UNWINDING /* Don't unwind into user mode. */ EEND(swi_exit) END(swi_entry) /* * Standard exception exit handler. * * This is used to return from all exceptions except SWI. It uses DO_AST and * PULLFRAMEFROMSVCANDEXIT and can only be called if the exception entry code * used PUSHFRAMEINSVC. * * If the return is to user mode, this uses DO_AST to deliver any pending * signals and/or handle TDF_NEEDRESCHED first. */ ASENTRY_NP(exception_exit) DO_AST /* Handle pending signals. */ PULLFRAMEFROMSVCANDEXIT /* Return. */ UNWINDSVCFRAME /* Special unwinding for exceptions. */ END(exception_exit) /* * Entry point for a Prefetch Abort exception. * * The hardware switches to the abort mode stack; we switch to svc32 before * calling the handler, then return directly to the original mode/stack * on exit (without transitioning back through the abort mode stack). */ ASENTRY_NP(prefetch_abort_entry) #ifdef __XSCALE__ nop /* Make absolutely sure any pending */ nop /* imprecise aborts have occurred. */ #endif sub lr, lr, #4 /* Adjust the lr. Transition to scv32 */ PUSHFRAMEINSVC /* mode stack, build trapframe there. */ adr lr, exception_exit /* Return from handler via standard */ mov r0, sp /* exception exit routine. Pass the */ mov r1, #1 /* Type flag */ b _C_LABEL(abort_handler) END(prefetch_abort_entry) /* * Entry point for a Data Abort exception. * * The hardware switches to the abort mode stack; we switch to svc32 before * calling the handler, then return directly to the original mode/stack * on exit (without transitioning back through the abort mode stack). */ ASENTRY_NP(data_abort_entry) #ifdef __XSCALE__ nop /* Make absolutely sure any pending */ nop /* imprecise aborts have occurred. */ #endif sub lr, lr, #8 /* Adjust the lr. Transition to scv32 */ PUSHFRAMEINSVC /* mode stack, build trapframe there. */ adr lr, exception_exit /* Exception exit routine */ mov r0, sp /* Trapframe to the handler */ mov r1, #0 /* Type flag */ b _C_LABEL(abort_handler) END(data_abort_entry) /* * Entry point for an Undefined Instruction exception. * * The hardware switches to the undefined mode stack; we switch to svc32 before * calling the handler, then return directly to the original mode/stack * on exit (without transitioning back through the undefined mode stack). */ ASENTRY_NP(undefined_entry) PUSHFRAMEINSVC /* mode stack, build trapframe there. */ mov r4, r0 /* R0 contains SPSR */ adr lr, exception_exit /* Return from handler via standard */ mov r0, sp /* exception exit routine. pass frame */ ldr r2, [sp, #(TF_PC)] /* load pc */ #if __ARM_ARCH >= 7 tst r4, #(PSR_T) /* test if PSR_T */ subne r2, r2, #(THUMB_INSN_SIZE) subeq r2, r2, #(INSN_SIZE) #else sub r2, r2, #(INSN_SIZE) /* fix pc */ #endif str r2, [sp, #TF_PC] /* store pc */ #ifdef KDTRACE_HOOKS /* Check if dtrace is enabled */ ldr r1, =_C_LABEL(dtrace_invop_jump_addr) ldr r3, [r1] cmp r3, #0 beq undefinedinstruction and r4, r4, #(PSR_MODE) /* Mask out unneeded bits */ cmp r4, #(PSR_USR32_MODE) /* Check if we came from usermode */ beq undefinedinstruction ldr r4, [r2] /* load instrution */ ldr r1, =FBT_BREAKPOINT /* load fbt inv op */ cmp r1, r4 bne undefinedinstruction bx r3 /* call invop_jump_addr */ #endif b undefinedinstruction /* call stadnard handler */ END(undefined_entry) /* * Entry point for a normal IRQ. * * The hardware switches to the IRQ mode stack; we switch to svc32 before * calling the handler, then return directly to the original mode/stack * on exit (without transitioning back through the IRQ mode stack). */ ASENTRY_NP(irq_entry) sub lr, lr, #4 /* Adjust the lr. Transition to scv32 */ PUSHFRAMEINSVC /* mode stack, build trapframe there. */ adr lr, exception_exit /* Return from handler via standard */ mov r0, sp /* exception exit routine. Pass the */ b _C_LABEL(intr_irq_handler)/* trapframe to the handler. */ END(irq_entry) /* * Entry point for an FIQ interrupt. * * We don't currently support FIQ handlers very much. Something can * install itself in the FIQ vector using code (that may or may not work * these days) in fiq.c. If nobody does that and an FIQ happens, this * default handler just disables FIQs and otherwise ignores it. */ ASENTRY_NP(fiq_entry) mrs r8, cpsr /* FIQ handling isn't supported, */ bic r8, #(PSR_F) /* just disable FIQ and return. */ msr cpsr_c, r8 /* The r8 we trash here is the */ subs pc, lr, #4 /* banked FIQ-mode r8. */ END(fiq_entry) /* * Entry point for an Address Exception exception. * This is an arm26 exception that should never happen. */ ASENTRY_NP(addr_exception_entry) mov r3, lr mrs r2, spsr mrs r1, cpsr adr r0, Laddr_exception_msg b _C_LABEL(panic) Laddr_exception_msg: .asciz "Address Exception CPSR=0x%08x SPSR=0x%08x LR=0x%08x\n" .balign 4 END(addr_exception_entry) /* * Entry point for the system Reset vector. * This should never happen, so panic. */ ASENTRY_NP(reset_entry) mov r1, lr adr r0, Lreset_panicmsg b _C_LABEL(panic) /* NOTREACHED */ Lreset_panicmsg: .asciz "Reset vector called, LR = 0x%08x" .balign 4 END(reset_entry) /* * page0 and page0_data -- An image of the ARM vectors which is copied to * the ARM vectors page (high or low) as part of CPU initialization. The * code that does the copy assumes that page0_data holds one 32-bit word * of data for each of the predefined ARM vectors. It also assumes that * page0_data follows the vectors in page0, but other stuff can appear * between the two. We currently leave room between the two for some fiq * handler code to be copied in. */ .global _C_LABEL(page0), _C_LABEL(page0_data) _C_LABEL(page0): ldr pc, .Lreset_entry ldr pc, .Lundefined_entry ldr pc, .Lswi_entry ldr pc, .Lprefetch_abort_entry ldr pc, .Ldata_abort_entry ldr pc, .Laddr_exception_entry ldr pc, .Lirq_entry .fiqv: ldr pc, .Lfiq_entry .space 256 /* room for some fiq handler code */ _C_LABEL(page0_data): .Lreset_entry: .word reset_entry .Lundefined_entry: .word undefined_entry .Lswi_entry: .word swi_entry .Lprefetch_abort_entry: .word prefetch_abort_entry .Ldata_abort_entry: .word data_abort_entry .Laddr_exception_entry: .word addr_exception_entry .Lirq_entry: .word irq_entry .Lfiq_entry: .word fiq_entry /* * These items are used by the code in fiq.c to install what it calls the * "null" handler. It's actually our default vector entry that just jumps * to the default handler which just disables FIQs and returns. */ .global _C_LABEL(fiq_nullhandler_code), _C_LABEL(fiq_nullhandler_size) _C_LABEL(fiq_nullhandler_code): .word .fiqv _C_LABEL(fiq_nullhandler_size): .word 4 Index: user/ngie/release-pkg-fix-tests/sys/arm/arm/generic_timer.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/arm/generic_timer.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/arm/generic_timer.c (revision 299091) @@ -1,536 +1,540 @@ /*- * Copyright (c) 2011 The FreeBSD Foundation * Copyright (c) 2013 Ruslan Bukin * All rights reserved. * * Based on mpcore_timer.c developed by Ben Gray * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /** * Cortex-A7, Cortex-A15, ARMv8 and later Generic Timer */ #include "opt_acpi.h" #include "opt_platform.h" #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef MULTIDELAY #include /* For arm_set_delay */ #endif #ifdef FDT #include #include #include #include #endif #ifdef DEV_ACPI #include #include #endif #define GT_CTRL_ENABLE (1 << 0) #define GT_CTRL_INT_MASK (1 << 1) #define GT_CTRL_INT_STAT (1 << 2) #define GT_REG_CTRL 0 #define GT_REG_TVAL 1 #define GT_CNTKCTL_PL0PTEN (1 << 9) /* PL0 Physical timer reg access */ #define GT_CNTKCTL_PL0VTEN (1 << 8) /* PL0 Virtual timer reg access */ #define GT_CNTKCTL_EVNTI (0xf << 4) /* Virtual counter event bits */ #define GT_CNTKCTL_EVNTDIR (1 << 3) /* Virtual counter event transition */ #define GT_CNTKCTL_EVNTEN (1 << 2) /* Enables virtual counter events */ #define GT_CNTKCTL_PL0VCTEN (1 << 1) /* PL0 CNTVCT and CNTFRQ access */ #define GT_CNTKCTL_PL0PCTEN (1 << 0) /* PL0 CNTPCT and CNTFRQ access */ struct arm_tmr_softc { struct resource *res[4]; void *ihl[4]; uint32_t clkfreq; struct eventtimer et; bool physical; }; static struct arm_tmr_softc *arm_tmr_sc = NULL; static struct resource_spec timer_spec[] = { { SYS_RES_IRQ, 0, RF_ACTIVE }, /* Secure */ { SYS_RES_IRQ, 1, RF_ACTIVE }, /* Non-secure */ - { SYS_RES_IRQ, 2, RF_ACTIVE }, /* Virt */ + { SYS_RES_IRQ, 2, RF_ACTIVE | RF_OPTIONAL }, /* Virt */ { SYS_RES_IRQ, 3, RF_ACTIVE | RF_OPTIONAL }, /* Hyp */ { -1, 0 } }; static timecounter_get_t arm_tmr_get_timecount; static struct timecounter arm_tmr_timecount = { .tc_name = "ARM MPCore Timecounter", .tc_get_timecount = arm_tmr_get_timecount, .tc_poll_pps = NULL, .tc_counter_mask = ~0u, .tc_frequency = 0, .tc_quality = 1000, }; #ifdef __arm__ #define get_el0(x) cp15_## x ##_get() #define get_el1(x) cp15_## x ##_get() #define set_el0(x, val) cp15_## x ##_set(val) #define set_el1(x, val) cp15_## x ##_set(val) #else /* __aarch64__ */ #define get_el0(x) READ_SPECIALREG(x ##_el0) #define get_el1(x) READ_SPECIALREG(x ##_el1) #define set_el0(x, val) WRITE_SPECIALREG(x ##_el0, val) #define set_el1(x, val) WRITE_SPECIALREG(x ##_el1, val) #endif static uint32_t arm_tmr_fill_vdso_timehands(struct vdso_timehands *vdso_th, struct timecounter *tc); static void arm_tmr_do_delay(int usec, void *); static int get_freq(void) { return (get_el0(cntfrq)); } static long get_cntxct(bool physical) { uint64_t val; isb(); if (physical) val = get_el0(cntpct); else val = get_el0(cntvct); return (val); } static int set_ctrl(uint32_t val, bool physical) { if (physical) set_el0(cntp_ctl, val); else set_el0(cntv_ctl, val); isb(); return (0); } static int set_tval(uint32_t val, bool physical) { if (physical) set_el0(cntp_tval, val); else set_el0(cntv_tval, val); isb(); return (0); } static int get_ctrl(bool physical) { uint32_t val; if (physical) val = get_el0(cntp_ctl); else val = get_el0(cntv_ctl); return (val); } static void setup_user_access(void *arg __unused) { uint32_t cntkctl; cntkctl = get_el1(cntkctl); cntkctl &= ~(GT_CNTKCTL_PL0PTEN | GT_CNTKCTL_PL0VTEN | GT_CNTKCTL_EVNTEN); if (arm_tmr_sc->physical) { cntkctl |= GT_CNTKCTL_PL0PCTEN; cntkctl &= ~GT_CNTKCTL_PL0VCTEN; } else { cntkctl |= GT_CNTKCTL_PL0VCTEN; cntkctl &= ~GT_CNTKCTL_PL0PCTEN; } set_el1(cntkctl, cntkctl); isb(); } static void tmr_setup_user_access(void *arg __unused) { smp_rendezvous(NULL, setup_user_access, NULL, NULL); } SYSINIT(tmr_ua, SI_SUB_SMP, SI_ORDER_SECOND, tmr_setup_user_access, NULL); static unsigned arm_tmr_get_timecount(struct timecounter *tc) { return (get_cntxct(arm_tmr_sc->physical)); } static int arm_tmr_start(struct eventtimer *et, sbintime_t first, sbintime_t period __unused) { struct arm_tmr_softc *sc; int counts, ctrl; sc = (struct arm_tmr_softc *)et->et_priv; if (first != 0) { counts = ((uint32_t)et->et_frequency * first) >> 32; ctrl = get_ctrl(sc->physical); ctrl &= ~GT_CTRL_INT_MASK; ctrl |= GT_CTRL_ENABLE; set_tval(counts, sc->physical); set_ctrl(ctrl, sc->physical); return (0); } return (EINVAL); } static int arm_tmr_stop(struct eventtimer *et) { struct arm_tmr_softc *sc; int ctrl; sc = (struct arm_tmr_softc *)et->et_priv; ctrl = get_ctrl(sc->physical); ctrl &= GT_CTRL_ENABLE; set_ctrl(ctrl, sc->physical); return (0); } static int arm_tmr_intr(void *arg) { struct arm_tmr_softc *sc; int ctrl; sc = (struct arm_tmr_softc *)arg; ctrl = get_ctrl(sc->physical); if (ctrl & GT_CTRL_INT_STAT) { ctrl |= GT_CTRL_INT_MASK; set_ctrl(ctrl, sc->physical); } if (sc->et.et_active) sc->et.et_event_cb(&sc->et, sc->et.et_arg); return (FILTER_HANDLED); } #ifdef FDT static int arm_tmr_fdt_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (ofw_bus_is_compatible(dev, "arm,armv7-timer")) { device_set_desc(dev, "ARMv7 Generic Timer"); return (BUS_PROBE_DEFAULT); } else if (ofw_bus_is_compatible(dev, "arm,armv8-timer")) { device_set_desc(dev, "ARMv8 Generic Timer"); return (BUS_PROBE_DEFAULT); } return (ENXIO); } #endif #ifdef DEV_ACPI static void arm_tmr_acpi_identify(driver_t *driver, device_t parent) { ACPI_TABLE_GTDT *gtdt; vm_paddr_t physaddr; device_t dev; physaddr = acpi_find_table(ACPI_SIG_GTDT); if (physaddr == 0) return; gtdt = acpi_map_table(physaddr, ACPI_SIG_GTDT); if (gtdt == NULL) { device_printf(parent, "gic: Unable to map the GTDT\n"); return; } dev = BUS_ADD_CHILD(parent, BUS_PASS_TIMER + BUS_PASS_ORDER_MIDDLE, "generic_timer", -1); if (dev == NULL) { device_printf(parent, "add gic child failed\n"); goto out; } BUS_SET_RESOURCE(parent, dev, SYS_RES_IRQ, 0, gtdt->SecureEl1Interrupt, 1); BUS_SET_RESOURCE(parent, dev, SYS_RES_IRQ, 1, gtdt->NonSecureEl1Interrupt, 1); BUS_SET_RESOURCE(parent, dev, SYS_RES_IRQ, 2, gtdt->VirtualTimerInterrupt, 1); out: acpi_unmap_table(gtdt); } static int arm_tmr_acpi_probe(device_t dev) { device_set_desc(dev, "ARM Generic Timer"); return (BUS_PROBE_NOWILDCARD); } #endif static int arm_tmr_attach(device_t dev) { struct arm_tmr_softc *sc; #ifdef FDT phandle_t node; pcell_t clock; #endif int error; int i; sc = device_get_softc(dev); if (arm_tmr_sc) return (ENXIO); #ifdef FDT /* Get the base clock frequency */ node = ofw_bus_get_node(dev); if (node > 0) { error = OF_getencprop(node, "clock-frequency", &clock, sizeof(clock)); if (error > 0) sc->clkfreq = clock; } #endif if (sc->clkfreq == 0) { /* Try to get clock frequency from timer */ sc->clkfreq = get_freq(); } if (sc->clkfreq == 0) { device_printf(dev, "No clock frequency specified\n"); return (ENXIO); } if (bus_alloc_resources(dev, timer_spec, sc->res)) { device_printf(dev, "could not allocate resources\n"); return (ENXIO); } #ifdef __arm__ sc->physical = true; #else /* __aarch64__ */ - sc->physical = false; + /* If we do not have a virtual timer use the physical. */ + sc->physical = (sc->res[2] == NULL) ? true : false; #endif arm_tmr_sc = sc; /* Setup secure, non-secure and virtual IRQs handler */ for (i = 0; i < 3; i++) { + /* If we do not have the interrupt, skip it. */ + if (sc->res[i] == NULL) + continue; error = bus_setup_intr(dev, sc->res[i], INTR_TYPE_CLK, arm_tmr_intr, NULL, sc, &sc->ihl[i]); if (error) { device_printf(dev, "Unable to alloc int resource.\n"); return (ENXIO); } } arm_cpu_fill_vdso_timehands = arm_tmr_fill_vdso_timehands; arm_tmr_timecount.tc_frequency = sc->clkfreq; tc_init(&arm_tmr_timecount); sc->et.et_name = "ARM MPCore Eventtimer"; sc->et.et_flags = ET_FLAGS_ONESHOT | ET_FLAGS_PERCPU; sc->et.et_quality = 1000; sc->et.et_frequency = sc->clkfreq; - sc->et.et_min_period = (0x00000002LLU << 32) / sc->et.et_frequency; + sc->et.et_min_period = (0x00000010LLU << 32) / sc->et.et_frequency; sc->et.et_max_period = (0xfffffffeLLU << 32) / sc->et.et_frequency; sc->et.et_start = arm_tmr_start; sc->et.et_stop = arm_tmr_stop; sc->et.et_priv = sc; et_register(&sc->et); #ifdef MULTIDELAY arm_set_delay(arm_tmr_do_delay, sc); #endif return (0); } #ifdef FDT static device_method_t arm_tmr_fdt_methods[] = { DEVMETHOD(device_probe, arm_tmr_fdt_probe), DEVMETHOD(device_attach, arm_tmr_attach), { 0, 0 } }; static driver_t arm_tmr_fdt_driver = { "generic_timer", arm_tmr_fdt_methods, sizeof(struct arm_tmr_softc), }; static devclass_t arm_tmr_fdt_devclass; EARLY_DRIVER_MODULE(timer, simplebus, arm_tmr_fdt_driver, arm_tmr_fdt_devclass, 0, 0, BUS_PASS_TIMER + BUS_PASS_ORDER_MIDDLE); EARLY_DRIVER_MODULE(timer, ofwbus, arm_tmr_fdt_driver, arm_tmr_fdt_devclass, 0, 0, BUS_PASS_TIMER + BUS_PASS_ORDER_MIDDLE); #endif #ifdef DEV_ACPI static device_method_t arm_tmr_acpi_methods[] = { DEVMETHOD(device_identify, arm_tmr_acpi_identify), DEVMETHOD(device_probe, arm_tmr_acpi_probe), DEVMETHOD(device_attach, arm_tmr_attach), { 0, 0 } }; static driver_t arm_tmr_acpi_driver = { "generic_timer", arm_tmr_acpi_methods, sizeof(struct arm_tmr_softc), }; static devclass_t arm_tmr_acpi_devclass; EARLY_DRIVER_MODULE(timer, acpi, arm_tmr_acpi_driver, arm_tmr_acpi_devclass, 0, 0, BUS_PASS_TIMER + BUS_PASS_ORDER_MIDDLE); #endif static void arm_tmr_do_delay(int usec, void *arg) { struct arm_tmr_softc *sc = arg; int32_t counts, counts_per_usec; uint32_t first, last; /* Get the number of times to count */ counts_per_usec = ((arm_tmr_timecount.tc_frequency / 1000000) + 1); /* * Clamp the timeout at a maximum value (about 32 seconds with * a 66MHz clock). *Nobody* should be delay()ing for anywhere * near that length of time and if they are, they should be hung * out to dry. */ if (usec >= (0x80000000U / counts_per_usec)) counts = (0x80000000U / counts_per_usec) - 1; else counts = usec * counts_per_usec; first = get_cntxct(sc->physical); while (counts > 0) { last = get_cntxct(sc->physical); counts -= (int32_t)(last - first); first = last; } } #ifndef MULTIDELAY void DELAY(int usec) { int32_t counts; /* * Check the timers are setup, if not just * use a for loop for the meantime */ if (arm_tmr_sc == NULL) { for (; usec > 0; usec--) for (counts = 200; counts > 0; counts--) /* * Prevent the compiler from optimizing * out the loop */ cpufunc_nullop(); } else arm_tmr_do_delay(usec, arm_tmr_sc); } #endif static uint32_t arm_tmr_fill_vdso_timehands(struct vdso_timehands *vdso_th, struct timecounter *tc) { vdso_th->th_physical = arm_tmr_sc->physical; bzero(vdso_th->th_res, sizeof(vdso_th->th_res)); return (tc == &arm_tmr_timecount); } Index: user/ngie/release-pkg-fix-tests/sys/arm/arm/gic.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/arm/gic.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/arm/gic.c (revision 299091) @@ -1,1570 +1,1570 @@ /*- * Copyright (c) 2011 The FreeBSD Foundation * All rights reserved. * * Developed by Damjan Marion * * Based on OMAP4 GIC code by Ben Gray * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include "opt_platform.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef INTRNG #include #endif #include #include #include #include #include #include #include #ifdef INTRNG #include "pic_if.h" #endif #define GIC_DEBUG_SPURIOUS /* We are using GICv2 register naming */ /* Distributor Registers */ #define GICD_CTLR 0x000 /* v1 ICDDCR */ #define GICD_TYPER 0x004 /* v1 ICDICTR */ #define GICD_IIDR 0x008 /* v1 ICDIIDR */ #define GICD_IGROUPR(n) (0x0080 + ((n) * 4)) /* v1 ICDISER */ #define GICD_ISENABLER(n) (0x0100 + ((n) * 4)) /* v1 ICDISER */ #define GICD_ICENABLER(n) (0x0180 + ((n) * 4)) /* v1 ICDICER */ #define GICD_ISPENDR(n) (0x0200 + ((n) * 4)) /* v1 ICDISPR */ #define GICD_ICPENDR(n) (0x0280 + ((n) * 4)) /* v1 ICDICPR */ #define GICD_ICACTIVER(n) (0x0380 + ((n) * 4)) /* v1 ICDABR */ #define GICD_IPRIORITYR(n) (0x0400 + ((n) * 4)) /* v1 ICDIPR */ #define GICD_ITARGETSR(n) (0x0800 + ((n) * 4)) /* v1 ICDIPTR */ #define GICD_ICFGR(n) (0x0C00 + ((n) * 4)) /* v1 ICDICFR */ #define GICD_SGIR(n) (0x0F00 + ((n) * 4)) /* v1 ICDSGIR */ #define GICD_SGI_TARGET_SHIFT 16 /* CPU Registers */ #define GICC_CTLR 0x0000 /* v1 ICCICR */ #define GICC_PMR 0x0004 /* v1 ICCPMR */ #define GICC_BPR 0x0008 /* v1 ICCBPR */ #define GICC_IAR 0x000C /* v1 ICCIAR */ #define GICC_EOIR 0x0010 /* v1 ICCEOIR */ #define GICC_RPR 0x0014 /* v1 ICCRPR */ #define GICC_HPPIR 0x0018 /* v1 ICCHPIR */ #define GICC_ABPR 0x001C /* v1 ICCABPR */ #define GICC_IIDR 0x00FC /* v1 ICCIIDR*/ #define GIC_FIRST_SGI 0 /* Irqs 0-15 are SGIs/IPIs. */ #define GIC_LAST_SGI 15 #define GIC_FIRST_PPI 16 /* Irqs 16-31 are private (per */ #define GIC_LAST_PPI 31 /* core) peripheral interrupts. */ #define GIC_FIRST_SPI 32 /* Irqs 32+ are shared peripherals. */ /* First bit is a polarity bit (0 - low, 1 - high) */ #define GICD_ICFGR_POL_LOW (0 << 0) #define GICD_ICFGR_POL_HIGH (1 << 0) #define GICD_ICFGR_POL_MASK 0x1 /* Second bit is a trigger bit (0 - level, 1 - edge) */ #define GICD_ICFGR_TRIG_LVL (0 << 1) #define GICD_ICFGR_TRIG_EDGE (1 << 1) #define GICD_ICFGR_TRIG_MASK 0x2 #ifndef GIC_DEFAULT_ICFGR_INIT #define GIC_DEFAULT_ICFGR_INIT 0x00000000 #endif #ifdef INTRNG struct gic_irqsrc { struct intr_irqsrc gi_isrc; uint32_t gi_irq; enum intr_polarity gi_pol; enum intr_trigger gi_trig; #define GI_FLAG_EARLY_EOI (1 << 0) u_int gi_flags; }; static u_int gic_irq_cpu; static int arm_gic_intr(void *); static int arm_gic_bind_intr(device_t dev, struct intr_irqsrc *isrc); #ifdef SMP static u_int sgi_to_ipi[GIC_LAST_SGI - GIC_FIRST_SGI + 1]; static u_int sgi_first_unused = GIC_FIRST_SGI; #endif #endif #ifdef INTRNG struct arm_gic_range { uint64_t bus; uint64_t host; uint64_t size; }; struct arm_gic_devinfo { struct ofw_bus_devinfo obdinfo; struct resource_list rl; }; #endif struct arm_gic_softc { device_t gic_dev; #ifdef INTRNG void * gic_intrhand; struct gic_irqsrc * gic_irqs; #endif struct resource * gic_res[3]; bus_space_tag_t gic_c_bst; bus_space_tag_t gic_d_bst; bus_space_handle_t gic_c_bsh; bus_space_handle_t gic_d_bsh; uint8_t ver; struct mtx mutex; uint32_t nirqs; #ifdef GIC_DEBUG_SPURIOUS uint32_t last_irq[MAXCPU]; #endif #ifdef INTRNG /* FDT child data */ pcell_t addr_cells; pcell_t size_cells; int nranges; struct arm_gic_range * ranges; #endif }; #ifdef INTRNG #define GIC_INTR_ISRC(sc, irq) (&sc->gic_irqs[irq].gi_isrc) #endif static struct resource_spec arm_gic_spec[] = { { SYS_RES_MEMORY, 0, RF_ACTIVE }, /* Distributor registers */ { SYS_RES_MEMORY, 1, RF_ACTIVE }, /* CPU Interrupt Intf. registers */ #ifdef INTRNG { SYS_RES_IRQ, 0, RF_ACTIVE | RF_OPTIONAL }, /* Parent interrupt */ #endif { -1, 0 } }; static u_int arm_gic_map[MAXCPU]; static struct arm_gic_softc *gic_sc = NULL; #define gic_c_read_4(_sc, _reg) \ bus_space_read_4((_sc)->gic_c_bst, (_sc)->gic_c_bsh, (_reg)) #define gic_c_write_4(_sc, _reg, _val) \ bus_space_write_4((_sc)->gic_c_bst, (_sc)->gic_c_bsh, (_reg), (_val)) #define gic_d_read_4(_sc, _reg) \ bus_space_read_4((_sc)->gic_d_bst, (_sc)->gic_d_bsh, (_reg)) #define gic_d_write_1(_sc, _reg, _val) \ bus_space_write_1((_sc)->gic_d_bst, (_sc)->gic_d_bsh, (_reg), (_val)) #define gic_d_write_4(_sc, _reg, _val) \ bus_space_write_4((_sc)->gic_d_bst, (_sc)->gic_d_bsh, (_reg), (_val)) #ifndef INTRNG static int gic_config_irq(int irq, enum intr_trigger trig, enum intr_polarity pol); static void gic_post_filter(void *); #endif static struct ofw_compat_data compat_data[] = { {"arm,gic", true}, /* Non-standard, used in FreeBSD dts. */ {"arm,gic-400", true}, {"arm,cortex-a15-gic", true}, {"arm,cortex-a9-gic", true}, {"arm,cortex-a7-gic", true}, {"arm,arm11mp-gic", true}, {"brcm,brahma-b15-gic", true}, {"qcom,msm-qgic2", true}, {NULL, false} }; static int arm_gic_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_search_compatible(dev, compat_data)->ocd_data) return (ENXIO); device_set_desc(dev, "ARM Generic Interrupt Controller"); return (BUS_PROBE_DEFAULT); } #ifdef INTRNG static inline void gic_irq_unmask(struct arm_gic_softc *sc, u_int irq) { gic_d_write_4(sc, GICD_ISENABLER(irq >> 5), (1UL << (irq & 0x1F))); } static inline void gic_irq_mask(struct arm_gic_softc *sc, u_int irq) { gic_d_write_4(sc, GICD_ICENABLER(irq >> 5), (1UL << (irq & 0x1F))); } #endif static uint8_t gic_cpu_mask(struct arm_gic_softc *sc) { uint32_t mask; int i; /* Read the current cpuid mask by reading ITARGETSR{0..7} */ for (i = 0; i < 8; i++) { mask = gic_d_read_4(sc, GICD_ITARGETSR(i)); if (mask != 0) break; } /* No mask found, assume we are on CPU interface 0 */ if (mask == 0) return (1); /* Collect the mask in the lower byte */ mask |= mask >> 16; mask |= mask >> 8; return (mask); } #ifdef SMP #ifdef INTRNG static void arm_gic_init_secondary(device_t dev) { struct arm_gic_softc *sc = device_get_softc(dev); u_int irq, cpu; /* Set the mask so we can find this CPU to send it IPIs */ cpu = PCPU_GET(cpuid); arm_gic_map[cpu] = gic_cpu_mask(sc); for (irq = 0; irq < sc->nirqs; irq += 4) gic_d_write_4(sc, GICD_IPRIORITYR(irq >> 2), 0); /* Set all the interrupts to be in Group 0 (secure) */ for (irq = 0; irq < sc->nirqs; irq += 32) { gic_d_write_4(sc, GICD_IGROUPR(irq >> 5), 0); } /* Enable CPU interface */ gic_c_write_4(sc, GICC_CTLR, 1); /* Set priority mask register. */ gic_c_write_4(sc, GICC_PMR, 0xff); /* Enable interrupt distribution */ gic_d_write_4(sc, GICD_CTLR, 0x01); /* Unmask attached SGI interrupts. */ for (irq = GIC_FIRST_SGI; irq <= GIC_LAST_SGI; irq++) if (intr_isrc_init_on_cpu(GIC_INTR_ISRC(sc, irq), cpu)) gic_irq_unmask(sc, irq); /* Unmask attached PPI interrupts. */ for (irq = GIC_FIRST_PPI; irq <= GIC_LAST_PPI; irq++) if (intr_isrc_init_on_cpu(GIC_INTR_ISRC(sc, irq), cpu)) gic_irq_unmask(sc, irq); } #else static void arm_gic_init_secondary(device_t dev) { struct arm_gic_softc *sc = device_get_softc(dev); int i; /* Set the mask so we can find this CPU to send it IPIs */ arm_gic_map[PCPU_GET(cpuid)] = gic_cpu_mask(sc); for (i = 0; i < sc->nirqs; i += 4) gic_d_write_4(sc, GICD_IPRIORITYR(i >> 2), 0); /* Set all the interrupts to be in Group 0 (secure) */ for (i = 0; i < sc->nirqs; i += 32) { gic_d_write_4(sc, GICD_IGROUPR(i >> 5), 0); } /* Enable CPU interface */ gic_c_write_4(sc, GICC_CTLR, 1); /* Set priority mask register. */ gic_c_write_4(sc, GICC_PMR, 0xff); /* Enable interrupt distribution */ gic_d_write_4(sc, GICD_CTLR, 0x01); /* * Activate the timer interrupts: virtual, secure, and non-secure. */ gic_d_write_4(sc, GICD_ISENABLER(27 >> 5), (1UL << (27 & 0x1F))); gic_d_write_4(sc, GICD_ISENABLER(29 >> 5), (1UL << (29 & 0x1F))); gic_d_write_4(sc, GICD_ISENABLER(30 >> 5), (1UL << (30 & 0x1F))); } #endif /* INTRNG */ #endif /* SMP */ #ifndef INTRNG int gic_decode_fdt(phandle_t iparent, pcell_t *intr, int *interrupt, int *trig, int *pol) { static u_int num_intr_cells; static phandle_t self; struct ofw_compat_data *ocd; if (self == 0) { for (ocd = compat_data; ocd->ocd_str != NULL; ocd++) { if (fdt_is_compatible(iparent, ocd->ocd_str)) { self = iparent; break; } } } if (self != iparent) return (ENXIO); if (num_intr_cells == 0) { if (OF_searchencprop(OF_node_from_xref(iparent), "#interrupt-cells", &num_intr_cells, sizeof(num_intr_cells)) == -1) { num_intr_cells = 1; } } if (num_intr_cells == 1) { *interrupt = fdt32_to_cpu(intr[0]); *trig = INTR_TRIGGER_CONFORM; *pol = INTR_POLARITY_CONFORM; } else { if (fdt32_to_cpu(intr[0]) == 0) *interrupt = fdt32_to_cpu(intr[1]) + GIC_FIRST_SPI; else *interrupt = fdt32_to_cpu(intr[1]) + GIC_FIRST_PPI; /* * In intr[2], bits[3:0] are trigger type and level flags. * 1 = low-to-high edge triggered * 2 = high-to-low edge triggered * 4 = active high level-sensitive * 8 = active low level-sensitive * The hardware only supports active-high-level or rising-edge * for SPIs */ if (*interrupt >= GIC_FIRST_SPI && fdt32_to_cpu(intr[2]) & 0x0a) { printf("unsupported trigger/polarity configuration " "0x%02x\n", fdt32_to_cpu(intr[2]) & 0x0f); } *pol = INTR_POLARITY_CONFORM; if (fdt32_to_cpu(intr[2]) & 0x03) *trig = INTR_TRIGGER_EDGE; else *trig = INTR_TRIGGER_LEVEL; } return (0); } #endif #ifdef INTRNG static inline intptr_t gic_xref(device_t dev) { #ifdef FDT return (OF_xref_from_node(ofw_bus_get_node(dev))); #else return (0); #endif } static int arm_gic_register_isrcs(struct arm_gic_softc *sc, uint32_t num) { int error; uint32_t irq; struct gic_irqsrc *irqs; struct intr_irqsrc *isrc; const char *name; irqs = malloc(num * sizeof(struct gic_irqsrc), M_DEVBUF, M_WAITOK | M_ZERO); name = device_get_nameunit(sc->gic_dev); for (irq = 0; irq < num; irq++) { irqs[irq].gi_irq = irq; irqs[irq].gi_pol = INTR_POLARITY_CONFORM; irqs[irq].gi_trig = INTR_TRIGGER_CONFORM; isrc = &irqs[irq].gi_isrc; if (irq <= GIC_LAST_SGI) { error = intr_isrc_register(isrc, sc->gic_dev, INTR_ISRCF_IPI, "%s,i%u", name, irq - GIC_FIRST_SGI); } else if (irq <= GIC_LAST_PPI) { error = intr_isrc_register(isrc, sc->gic_dev, INTR_ISRCF_PPI, "%s,p%u", name, irq - GIC_FIRST_PPI); } else { error = intr_isrc_register(isrc, sc->gic_dev, 0, "%s,s%u", name, irq - GIC_FIRST_SPI); } if (error != 0) { /* XXX call intr_isrc_deregister() */ free(irqs, M_DEVBUF); return (error); } } sc->gic_irqs = irqs; sc->nirqs = num; return (0); } static int arm_gic_fill_ranges(phandle_t node, struct arm_gic_softc *sc) { pcell_t host_cells; cell_t *base_ranges; ssize_t nbase_ranges; int i, j, k; host_cells = 1; OF_getencprop(OF_parent(node), "#address-cells", &host_cells, sizeof(host_cells)); sc->addr_cells = 2; OF_getencprop(node, "#address-cells", &sc->addr_cells, sizeof(sc->addr_cells)); sc->size_cells = 2; OF_getencprop(node, "#size-cells", &sc->size_cells, sizeof(sc->size_cells)); nbase_ranges = OF_getproplen(node, "ranges"); if (nbase_ranges < 0) return (-1); sc->nranges = nbase_ranges / sizeof(cell_t) / (sc->addr_cells + host_cells + sc->size_cells); if (sc->nranges == 0) return (0); sc->ranges = malloc(sc->nranges * sizeof(sc->ranges[0]), M_DEVBUF, M_WAITOK); base_ranges = malloc(nbase_ranges, M_DEVBUF, M_WAITOK); OF_getencprop(node, "ranges", base_ranges, nbase_ranges); for (i = 0, j = 0; i < sc->nranges; i++) { sc->ranges[i].bus = 0; for (k = 0; k < sc->addr_cells; k++) { sc->ranges[i].bus <<= 32; sc->ranges[i].bus |= base_ranges[j++]; } sc->ranges[i].host = 0; for (k = 0; k < host_cells; k++) { sc->ranges[i].host <<= 32; sc->ranges[i].host |= base_ranges[j++]; } sc->ranges[i].size = 0; for (k = 0; k < sc->size_cells; k++) { sc->ranges[i].size <<= 32; sc->ranges[i].size |= base_ranges[j++]; } } free(base_ranges, M_DEVBUF); return (sc->nranges); } static bool arm_gic_add_children(device_t dev) { struct arm_gic_softc *sc; struct arm_gic_devinfo *dinfo; phandle_t child, node; device_t cdev; sc = device_get_softc(dev); node = ofw_bus_get_node(dev); /* If we have no children don't probe for them */ child = OF_child(node); if (child == 0) return (false); if (arm_gic_fill_ranges(node, sc) < 0) { device_printf(dev, "Have a child, but no ranges\n"); return (false); } for (; child != 0; child = OF_peer(child)) { dinfo = malloc(sizeof(*dinfo), M_DEVBUF, M_WAITOK | M_ZERO); if (ofw_bus_gen_setup_devinfo(&dinfo->obdinfo, child) != 0) { free(dinfo, M_DEVBUF); continue; } resource_list_init(&dinfo->rl); ofw_bus_reg_to_rl(dev, child, sc->addr_cells, sc->size_cells, &dinfo->rl); cdev = device_add_child(dev, NULL, -1); if (cdev == NULL) { device_printf(dev, "<%s>: device_add_child failed\n", dinfo->obdinfo.obd_name); resource_list_free(&dinfo->rl); ofw_bus_gen_destroy_devinfo(&dinfo->obdinfo); free(dinfo, M_DEVBUF); continue; } device_set_ivars(cdev, dinfo); } return (true); } #endif static int arm_gic_attach(device_t dev) { struct arm_gic_softc *sc; int i; uint32_t icciidr, mask, nirqs; #ifdef INTRNG phandle_t pxref; intptr_t xref = gic_xref(dev); #endif if (gic_sc) return (ENXIO); sc = device_get_softc(dev); if (bus_alloc_resources(dev, arm_gic_spec, sc->gic_res)) { device_printf(dev, "could not allocate resources\n"); return (ENXIO); } sc->gic_dev = dev; gic_sc = sc; /* Initialize mutex */ mtx_init(&sc->mutex, "GIC lock", "", MTX_SPIN); /* Distributor Interface */ sc->gic_d_bst = rman_get_bustag(sc->gic_res[0]); sc->gic_d_bsh = rman_get_bushandle(sc->gic_res[0]); /* CPU Interface */ sc->gic_c_bst = rman_get_bustag(sc->gic_res[1]); sc->gic_c_bsh = rman_get_bushandle(sc->gic_res[1]); /* Disable interrupt forwarding to the CPU interface */ gic_d_write_4(sc, GICD_CTLR, 0x00); /* Get the number of interrupts */ nirqs = gic_d_read_4(sc, GICD_TYPER); nirqs = 32 * ((nirqs & 0x1f) + 1); #ifdef INTRNG if (arm_gic_register_isrcs(sc, nirqs)) { device_printf(dev, "could not register irqs\n"); goto cleanup; } #else sc->nirqs = nirqs; /* Set up function pointers */ arm_post_filter = gic_post_filter; arm_config_irq = gic_config_irq; #endif icciidr = gic_c_read_4(sc, GICC_IIDR); device_printf(dev,"pn 0x%x, arch 0x%x, rev 0x%x, implementer 0x%x irqs %u\n", icciidr>>20, (icciidr>>16) & 0xF, (icciidr>>12) & 0xf, (icciidr & 0xfff), sc->nirqs); /* Set all global interrupts to be level triggered, active low. */ for (i = 32; i < sc->nirqs; i += 16) { gic_d_write_4(sc, GICD_ICFGR(i >> 4), GIC_DEFAULT_ICFGR_INIT); } /* Disable all interrupts. */ for (i = 32; i < sc->nirqs; i += 32) { gic_d_write_4(sc, GICD_ICENABLER(i >> 5), 0xFFFFFFFF); } /* Find the current cpu mask */ mask = gic_cpu_mask(sc); /* Set the mask so we can find this CPU to send it IPIs */ arm_gic_map[PCPU_GET(cpuid)] = mask; /* Set all four targets to this cpu */ mask |= mask << 8; mask |= mask << 16; for (i = 0; i < sc->nirqs; i += 4) { gic_d_write_4(sc, GICD_IPRIORITYR(i >> 2), 0); if (i > 32) { gic_d_write_4(sc, GICD_ITARGETSR(i >> 2), mask); } } /* Set all the interrupts to be in Group 0 (secure) */ for (i = 0; i < sc->nirqs; i += 32) { gic_d_write_4(sc, GICD_IGROUPR(i >> 5), 0); } /* Enable CPU interface */ gic_c_write_4(sc, GICC_CTLR, 1); /* Set priority mask register. */ gic_c_write_4(sc, GICC_PMR, 0xff); /* Enable interrupt distribution */ gic_d_write_4(sc, GICD_CTLR, 0x01); #ifndef INTRNG return (0); #else /* * Now, when everything is initialized, it's right time to * register interrupt controller to interrupt framefork. */ if (intr_pic_register(dev, xref) != 0) { device_printf(dev, "could not register PIC\n"); goto cleanup; } /* * Controller is root if: * - doesn't have interrupt parent * - his interrupt parent is this controller */ pxref = ofw_bus_find_iparent(ofw_bus_get_node(dev)); if (pxref == 0 || xref == pxref) { if (intr_pic_claim_root(dev, xref, arm_gic_intr, sc, GIC_LAST_SGI - GIC_FIRST_SGI + 1) != 0) { device_printf(dev, "could not set PIC as a root\n"); intr_pic_deregister(dev, xref); goto cleanup; } } else { if (sc->gic_res[2] == NULL) { device_printf(dev, "not root PIC must have defined interrupt\n"); intr_pic_deregister(dev, xref); goto cleanup; } if (bus_setup_intr(dev, sc->gic_res[2], INTR_TYPE_CLK, arm_gic_intr, NULL, sc, &sc->gic_intrhand)) { device_printf(dev, "could not setup irq handler\n"); intr_pic_deregister(dev, xref); goto cleanup; } } OF_device_register_xref(xref, dev); /* If we have children probe and attach them */ if (arm_gic_add_children(dev)) { bus_generic_probe(dev); return (bus_generic_attach(dev)); } return (0); cleanup: /* * XXX - not implemented arm_gic_detach() should be called ! */ if (sc->gic_irqs != NULL) free(sc->gic_irqs, M_DEVBUF); bus_release_resources(dev, arm_gic_spec, sc->gic_res); return(ENXIO); #endif } #ifdef INTRNG static struct resource * arm_gic_alloc_resource(device_t bus, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { struct arm_gic_softc *sc; struct arm_gic_devinfo *di; struct resource_list_entry *rle; int j; KASSERT(type == SYS_RES_MEMORY, ("Invalid resoure type %x", type)); sc = device_get_softc(bus); /* * Request for the default allocation with a given rid: use resource * list stored in the local device info. */ if (RMAN_IS_DEFAULT_RANGE(start, end)) { if ((di = device_get_ivars(child)) == NULL) return (NULL); if (type == SYS_RES_IOPORT) type = SYS_RES_MEMORY; rle = resource_list_find(&di->rl, type, *rid); if (rle == NULL) { if (bootverbose) device_printf(bus, "no default resources for " "rid = %d, type = %d\n", *rid, type); return (NULL); } start = rle->start; end = rle->end; count = rle->count; } /* Remap through ranges property */ for (j = 0; j < sc->nranges; j++) { if (start >= sc->ranges[j].bus && end < sc->ranges[j].bus + sc->ranges[j].size) { start -= sc->ranges[j].bus; start += sc->ranges[j].host; end -= sc->ranges[j].bus; end += sc->ranges[j].host; break; } } if (j == sc->nranges && sc->nranges != 0) { if (bootverbose) device_printf(bus, "Could not map resource " "%#jx-%#jx\n", (uintmax_t)start, (uintmax_t)end); return (NULL); } return (bus_generic_alloc_resource(bus, child, type, rid, start, end, count, flags)); } static const struct ofw_bus_devinfo * arm_gic_ofw_get_devinfo(device_t bus __unused, device_t child) { struct arm_gic_devinfo *di; di = device_get_ivars(child); return (&di->obdinfo); } static int arm_gic_intr(void *arg) { struct arm_gic_softc *sc = arg; struct gic_irqsrc *gi; uint32_t irq_active_reg, irq; struct trapframe *tf; irq_active_reg = gic_c_read_4(sc, GICC_IAR); irq = irq_active_reg & 0x3FF; /* * 1. We do EOI here because recent read value from active interrupt * register must be used for it. Another approach is to save this * value into associated interrupt source. * 2. EOI must be done on same CPU where interrupt has fired. Thus * we must ensure that interrupted thread does not migrate to * another CPU. * 3. EOI cannot be delayed by any preemption which could happen on * critical_exit() used in MI intr code, when interrupt thread is * scheduled. See next point. * 4. IPI_RENDEZVOUS assumes that no preemption is permitted during * an action and any use of critical_exit() could break this * assumption. See comments within smp_rendezvous_action(). * 5. We always return FILTER_HANDLED as this is an interrupt * controller dispatch function. Otherwise, in cascaded interrupt * case, the whole interrupt subtree would be masked. */ if (irq >= sc->nirqs) { #ifdef GIC_DEBUG_SPURIOUS device_printf(sc->gic_dev, "Spurious interrupt detected: last irq: %d on CPU%d\n", sc->last_irq[PCPU_GET(cpuid)], PCPU_GET(cpuid)); #endif return (FILTER_HANDLED); } tf = curthread->td_intr_frame; dispatch_irq: gi = sc->gic_irqs + irq; /* * Note that GIC_FIRST_SGI is zero and is not used in 'if' statement * as compiler complains that comparing u_int >= 0 is always true. */ if (irq <= GIC_LAST_SGI) { #ifdef SMP /* Call EOI for all IPI before dispatch. */ gic_c_write_4(sc, GICC_EOIR, irq_active_reg); intr_ipi_dispatch(sgi_to_ipi[gi->gi_irq], tf); goto next_irq; #else device_printf(sc->gic_dev, "SGI %u on UP system detected\n", irq - GIC_FIRST_SGI); gic_c_write_4(sc, GICC_EOIR, irq_active_reg); goto next_irq; #endif } #ifdef GIC_DEBUG_SPURIOUS sc->last_irq[PCPU_GET(cpuid)] = irq; #endif if ((gi->gi_flags & GI_FLAG_EARLY_EOI) == GI_FLAG_EARLY_EOI) gic_c_write_4(sc, GICC_EOIR, irq_active_reg); if (intr_isrc_dispatch(&gi->gi_isrc, tf) != 0) { gic_irq_mask(sc, irq); if ((gi->gi_flags & GI_FLAG_EARLY_EOI) != GI_FLAG_EARLY_EOI) gic_c_write_4(sc, GICC_EOIR, irq_active_reg); device_printf(sc->gic_dev, "Stray irq %u disabled\n", irq); } next_irq: arm_irq_memory_barrier(irq); irq_active_reg = gic_c_read_4(sc, GICC_IAR); irq = irq_active_reg & 0x3FF; if (irq < sc->nirqs) goto dispatch_irq; return (FILTER_HANDLED); } static void gic_config(struct arm_gic_softc *sc, u_int irq, enum intr_trigger trig, enum intr_polarity pol) { uint32_t reg; uint32_t mask; if (irq < GIC_FIRST_SPI) return; mtx_lock_spin(&sc->mutex); reg = gic_d_read_4(sc, GICD_ICFGR(irq >> 4)); mask = (reg >> 2*(irq % 16)) & 0x3; if (pol == INTR_POLARITY_LOW) { mask &= ~GICD_ICFGR_POL_MASK; mask |= GICD_ICFGR_POL_LOW; } else if (pol == INTR_POLARITY_HIGH) { mask &= ~GICD_ICFGR_POL_MASK; mask |= GICD_ICFGR_POL_HIGH; } if (trig == INTR_TRIGGER_LEVEL) { mask &= ~GICD_ICFGR_TRIG_MASK; mask |= GICD_ICFGR_TRIG_LVL; } else if (trig == INTR_TRIGGER_EDGE) { mask &= ~GICD_ICFGR_TRIG_MASK; mask |= GICD_ICFGR_TRIG_EDGE; } /* Set mask */ reg = reg & ~(0x3 << 2*(irq % 16)); reg = reg | (mask << 2*(irq % 16)); gic_d_write_4(sc, GICD_ICFGR(irq >> 4), reg); mtx_unlock_spin(&sc->mutex); } static int gic_bind(struct arm_gic_softc *sc, u_int irq, cpuset_t *cpus) { uint32_t cpu, end, mask; end = min(mp_ncpus, 8); for (cpu = end; cpu < MAXCPU; cpu++) if (CPU_ISSET(cpu, cpus)) return (EINVAL); for (mask = 0, cpu = 0; cpu < end; cpu++) if (CPU_ISSET(cpu, cpus)) mask |= 1 << cpu; gic_d_write_1(sc, GICD_ITARGETSR(0) + irq, mask); return (0); } #ifdef FDT static int gic_map_fdt(device_t dev, u_int ncells, pcell_t *cells, u_int *irqp, enum intr_polarity *polp, enum intr_trigger *trigp) { if (ncells == 1) { *irqp = cells[0]; *polp = INTR_POLARITY_CONFORM; *trigp = INTR_TRIGGER_CONFORM; return (0); } if (ncells == 3) { u_int irq, tripol; /* * The 1st cell is the interrupt type: * 0 = SPI * 1 = PPI * The 2nd cell contains the interrupt number: * [0 - 987] for SPI * [0 - 15] for PPI * The 3rd cell is the flags, encoded as follows: * bits[3:0] trigger type and level flags * 1 = low-to-high edge triggered * 2 = high-to-low edge triggered * 4 = active high level-sensitive * 8 = active low level-sensitive * bits[15:8] PPI interrupt cpu mask * Each bit corresponds to each of the 8 possible cpus * attached to the GIC. A bit set to '1' indicated * the interrupt is wired to that CPU. */ switch (cells[0]) { case 0: irq = GIC_FIRST_SPI + cells[1]; /* SPI irq is checked later. */ break; case 1: irq = GIC_FIRST_PPI + cells[1]; if (irq > GIC_LAST_PPI) { device_printf(dev, "unsupported PPI interrupt " "number %u\n", cells[1]); return (EINVAL); } break; default: device_printf(dev, "unsupported interrupt type " "configuration %u\n", cells[0]); return (EINVAL); } tripol = cells[2] & 0xff; if (tripol & 0xf0 || (tripol & 0x0a && cells[0] == 0)) device_printf(dev, "unsupported trigger/polarity " "configuration 0x%02x\n", tripol); *irqp = irq; *polp = INTR_POLARITY_CONFORM; *trigp = tripol & 0x03 ? INTR_TRIGGER_EDGE : INTR_TRIGGER_LEVEL; return (0); } return (EINVAL); } #endif static int gic_map_intr(device_t dev, struct intr_map_data *data, u_int *irqp, enum intr_polarity *polp, enum intr_trigger *trigp) { u_int irq; enum intr_polarity pol; enum intr_trigger trig; struct arm_gic_softc *sc; sc = device_get_softc(dev); switch (data->type) { #ifdef FDT case INTR_MAP_DATA_FDT: if (gic_map_fdt(dev, data->fdt.ncells, data->fdt.cells, &irq, &pol, &trig) != 0) return (EINVAL); break; #endif default: return (EINVAL); } if (irq >= sc->nirqs) return (EINVAL); if (pol != INTR_POLARITY_CONFORM && pol != INTR_POLARITY_LOW && pol != INTR_POLARITY_HIGH) return (EINVAL); if (trig != INTR_TRIGGER_CONFORM && trig != INTR_TRIGGER_EDGE && trig != INTR_TRIGGER_LEVEL) return (EINVAL); *irqp = irq; if (polp != NULL) *polp = pol; if (trigp != NULL) *trigp = trig; return (0); } static int arm_gic_map_intr(device_t dev, struct intr_map_data *data, struct intr_irqsrc **isrcp) { int error; u_int irq; struct arm_gic_softc *sc; error = gic_map_intr(dev, data, &irq, NULL, NULL); if (error == 0) { sc = device_get_softc(dev); *isrcp = GIC_INTR_ISRC(sc, irq); } return (error); } static int arm_gic_setup_intr(device_t dev, struct intr_irqsrc *isrc, struct resource *res, struct intr_map_data *data) { struct arm_gic_softc *sc = device_get_softc(dev); struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc; u_int irq; enum intr_trigger trig; enum intr_polarity pol; if (data == NULL) return (ENOTSUP); /* Get config for resource. */ if (gic_map_intr(dev, data, &irq, &pol, &trig)) return (EINVAL); if (gi->gi_irq != irq) return (EINVAL); /* Compare config if this is not first setup. */ if (isrc->isrc_handlers != 0) { if ((pol != INTR_POLARITY_CONFORM && pol != gi->gi_pol) || (trig != INTR_TRIGGER_CONFORM && trig != gi->gi_trig)) return (EINVAL); else return (0); } if (pol == INTR_POLARITY_CONFORM) pol = INTR_POLARITY_LOW; /* just pick some */ if (trig == INTR_TRIGGER_CONFORM) trig = INTR_TRIGGER_EDGE; /* just pick some */ gi->gi_pol = pol; gi->gi_trig = trig; /* Edge triggered interrupts need an early EOI sent */ if (gi->gi_pol == INTR_TRIGGER_EDGE) gi->gi_flags |= GI_FLAG_EARLY_EOI; /* * XXX - In case that per CPU interrupt is going to be enabled in time * when SMP is already started, we need some IPI call which * enables it on others CPUs. Further, it's more complicated as * pic_enable_source() and pic_disable_source() should act on * per CPU basis only. Thus, it should be solved here somehow. */ if (isrc->isrc_flags & INTR_ISRCF_PPI) CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu); gic_config(sc, gi->gi_irq, trig, pol); arm_gic_bind_intr(dev, isrc); return (0); } static int arm_gic_teardown_intr(device_t dev, struct intr_irqsrc *isrc, struct resource *res, struct intr_map_data *data) { struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc; if (isrc->isrc_handlers == 0) { gi->gi_pol = INTR_POLARITY_CONFORM; gi->gi_trig = INTR_TRIGGER_CONFORM; } return (0); } static void arm_gic_enable_intr(device_t dev, struct intr_irqsrc *isrc) { struct arm_gic_softc *sc = device_get_softc(dev); struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc; arm_irq_memory_barrier(gi->gi_irq); gic_irq_unmask(sc, gi->gi_irq); } static void arm_gic_disable_intr(device_t dev, struct intr_irqsrc *isrc) { struct arm_gic_softc *sc = device_get_softc(dev); struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc; gic_irq_mask(sc, gi->gi_irq); } static void arm_gic_pre_ithread(device_t dev, struct intr_irqsrc *isrc) { struct arm_gic_softc *sc = device_get_softc(dev); struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc; arm_gic_disable_intr(dev, isrc); gic_c_write_4(sc, GICC_EOIR, gi->gi_irq); } static void arm_gic_post_ithread(device_t dev, struct intr_irqsrc *isrc) { arm_irq_memory_barrier(0); arm_gic_enable_intr(dev, isrc); } static void arm_gic_post_filter(device_t dev, struct intr_irqsrc *isrc) { struct arm_gic_softc *sc = device_get_softc(dev); struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc; /* EOI for edge-triggered done earlier. */ if ((gi->gi_flags & GI_FLAG_EARLY_EOI) == GI_FLAG_EARLY_EOI) return; arm_irq_memory_barrier(0); gic_c_write_4(sc, GICC_EOIR, gi->gi_irq); } static int arm_gic_bind_intr(device_t dev, struct intr_irqsrc *isrc) { struct arm_gic_softc *sc = device_get_softc(dev); struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc; if (gi->gi_irq < GIC_FIRST_SPI) return (EINVAL); if (CPU_EMPTY(&isrc->isrc_cpu)) { gic_irq_cpu = intr_irq_next_cpu(gic_irq_cpu, &all_cpus); CPU_SETOF(gic_irq_cpu, &isrc->isrc_cpu); } return (gic_bind(sc, gi->gi_irq, &isrc->isrc_cpu)); } #ifdef SMP static void arm_gic_ipi_send(device_t dev, struct intr_irqsrc *isrc, cpuset_t cpus, u_int ipi) { struct arm_gic_softc *sc = device_get_softc(dev); struct gic_irqsrc *gi = (struct gic_irqsrc *)isrc; uint32_t val = 0, i; for (i = 0; i < MAXCPU; i++) if (CPU_ISSET(i, &cpus)) val |= arm_gic_map[i] << GICD_SGI_TARGET_SHIFT; gic_d_write_4(sc, GICD_SGIR(0), val | gi->gi_irq); } static int arm_gic_ipi_setup(device_t dev, u_int ipi, struct intr_irqsrc **isrcp) { struct intr_irqsrc *isrc; struct arm_gic_softc *sc = device_get_softc(dev); if (sgi_first_unused > GIC_LAST_SGI) return (ENOSPC); isrc = GIC_INTR_ISRC(sc, sgi_first_unused); sgi_to_ipi[sgi_first_unused++] = ipi; CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu); *isrcp = isrc; return (0); } #endif #else static int arm_gic_next_irq(struct arm_gic_softc *sc, int last_irq) { uint32_t active_irq; active_irq = gic_c_read_4(sc, GICC_IAR); /* - * Immediatly EOIR the SGIs, because doing so requires the other + * Immediately EOIR the SGIs, because doing so requires the other * bits (ie CPU number), not just the IRQ number, and we do not * have this information later. */ if ((active_irq & 0x3ff) <= GIC_LAST_SGI) gic_c_write_4(sc, GICC_EOIR, active_irq); active_irq &= 0x3FF; if (active_irq == 0x3FF) { if (last_irq == -1) device_printf(sc->gic_dev, "Spurious interrupt detected\n"); return -1; } return active_irq; } static int arm_gic_config(device_t dev, int irq, enum intr_trigger trig, enum intr_polarity pol) { struct arm_gic_softc *sc = device_get_softc(dev); uint32_t reg; uint32_t mask; /* Function is public-accessible, so validate input arguments */ if ((irq < 0) || (irq >= sc->nirqs)) goto invalid_args; if ((trig != INTR_TRIGGER_EDGE) && (trig != INTR_TRIGGER_LEVEL) && (trig != INTR_TRIGGER_CONFORM)) goto invalid_args; if ((pol != INTR_POLARITY_HIGH) && (pol != INTR_POLARITY_LOW) && (pol != INTR_POLARITY_CONFORM)) goto invalid_args; mtx_lock_spin(&sc->mutex); reg = gic_d_read_4(sc, GICD_ICFGR(irq >> 4)); mask = (reg >> 2*(irq % 16)) & 0x3; if (pol == INTR_POLARITY_LOW) { mask &= ~GICD_ICFGR_POL_MASK; mask |= GICD_ICFGR_POL_LOW; } else if (pol == INTR_POLARITY_HIGH) { mask &= ~GICD_ICFGR_POL_MASK; mask |= GICD_ICFGR_POL_HIGH; } if (trig == INTR_TRIGGER_LEVEL) { mask &= ~GICD_ICFGR_TRIG_MASK; mask |= GICD_ICFGR_TRIG_LVL; } else if (trig == INTR_TRIGGER_EDGE) { mask &= ~GICD_ICFGR_TRIG_MASK; mask |= GICD_ICFGR_TRIG_EDGE; } /* Set mask */ reg = reg & ~(0x3 << 2*(irq % 16)); reg = reg | (mask << 2*(irq % 16)); gic_d_write_4(sc, GICD_ICFGR(irq >> 4), reg); mtx_unlock_spin(&sc->mutex); return (0); invalid_args: device_printf(dev, "gic_config_irg, invalid parameters\n"); return (EINVAL); } static void arm_gic_mask(device_t dev, int irq) { struct arm_gic_softc *sc = device_get_softc(dev); gic_d_write_4(sc, GICD_ICENABLER(irq >> 5), (1UL << (irq & 0x1F))); gic_c_write_4(sc, GICC_EOIR, irq); /* XXX - not allowed */ } static void arm_gic_unmask(device_t dev, int irq) { struct arm_gic_softc *sc = device_get_softc(dev); if (irq > GIC_LAST_SGI) arm_irq_memory_barrier(irq); gic_d_write_4(sc, GICD_ISENABLER(irq >> 5), (1UL << (irq & 0x1F))); } #ifdef SMP static void arm_gic_ipi_send(device_t dev, cpuset_t cpus, u_int ipi) { struct arm_gic_softc *sc = device_get_softc(dev); uint32_t val = 0, i; for (i = 0; i < MAXCPU; i++) if (CPU_ISSET(i, &cpus)) val |= arm_gic_map[i] << GICD_SGI_TARGET_SHIFT; gic_d_write_4(sc, GICD_SGIR(0), val | ipi); } static int arm_gic_ipi_read(device_t dev, int i) { if (i != -1) { /* * The intr code will automagically give the frame pointer * if the interrupt argument is 0. */ if ((unsigned int)i > 16) return (0); return (i); } return (0x3ff); } static void arm_gic_ipi_clear(device_t dev, int ipi) { /* no-op */ } #endif static void gic_post_filter(void *arg) { struct arm_gic_softc *sc = gic_sc; uintptr_t irq = (uintptr_t) arg; if (irq > GIC_LAST_SGI) arm_irq_memory_barrier(irq); gic_c_write_4(sc, GICC_EOIR, irq); } static int gic_config_irq(int irq, enum intr_trigger trig, enum intr_polarity pol) { return (arm_gic_config(gic_sc->gic_dev, irq, trig, pol)); } void arm_mask_irq(uintptr_t nb) { arm_gic_mask(gic_sc->gic_dev, nb); } void arm_unmask_irq(uintptr_t nb) { arm_gic_unmask(gic_sc->gic_dev, nb); } int arm_get_next_irq(int last_irq) { return (arm_gic_next_irq(gic_sc, last_irq)); } #ifdef SMP void intr_pic_init_secondary(void) { arm_gic_init_secondary(gic_sc->gic_dev); } void pic_ipi_send(cpuset_t cpus, u_int ipi) { arm_gic_ipi_send(gic_sc->gic_dev, cpus, ipi); } int pic_ipi_read(int i) { return (arm_gic_ipi_read(gic_sc->gic_dev, i)); } void pic_ipi_clear(int ipi) { arm_gic_ipi_clear(gic_sc->gic_dev, ipi); } #endif #endif /* INTRNG */ static device_method_t arm_gic_methods[] = { /* Device interface */ DEVMETHOD(device_probe, arm_gic_probe), DEVMETHOD(device_attach, arm_gic_attach), #ifdef INTRNG /* Bus interface */ DEVMETHOD(bus_add_child, bus_generic_add_child), DEVMETHOD(bus_alloc_resource, arm_gic_alloc_resource), DEVMETHOD(bus_release_resource, bus_generic_release_resource), DEVMETHOD(bus_activate_resource,bus_generic_activate_resource), /* ofw_bus interface */ DEVMETHOD(ofw_bus_get_devinfo, arm_gic_ofw_get_devinfo), DEVMETHOD(ofw_bus_get_compat, ofw_bus_gen_get_compat), DEVMETHOD(ofw_bus_get_model, ofw_bus_gen_get_model), DEVMETHOD(ofw_bus_get_name, ofw_bus_gen_get_name), DEVMETHOD(ofw_bus_get_node, ofw_bus_gen_get_node), DEVMETHOD(ofw_bus_get_type, ofw_bus_gen_get_type), /* Interrupt controller interface */ DEVMETHOD(pic_disable_intr, arm_gic_disable_intr), DEVMETHOD(pic_enable_intr, arm_gic_enable_intr), DEVMETHOD(pic_map_intr, arm_gic_map_intr), DEVMETHOD(pic_setup_intr, arm_gic_setup_intr), DEVMETHOD(pic_teardown_intr, arm_gic_teardown_intr), DEVMETHOD(pic_post_filter, arm_gic_post_filter), DEVMETHOD(pic_post_ithread, arm_gic_post_ithread), DEVMETHOD(pic_pre_ithread, arm_gic_pre_ithread), #ifdef SMP DEVMETHOD(pic_bind_intr, arm_gic_bind_intr), DEVMETHOD(pic_init_secondary, arm_gic_init_secondary), DEVMETHOD(pic_ipi_send, arm_gic_ipi_send), DEVMETHOD(pic_ipi_setup, arm_gic_ipi_setup), #endif #endif { 0, 0 } }; static driver_t arm_gic_driver = { "gic", arm_gic_methods, sizeof(struct arm_gic_softc), }; static devclass_t arm_gic_devclass; EARLY_DRIVER_MODULE(gic, simplebus, arm_gic_driver, arm_gic_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); EARLY_DRIVER_MODULE(gic, ofwbus, arm_gic_driver, arm_gic_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); #ifdef INTRNG /* * GICv2m support -- the GICv2 MSI/MSI-X controller. */ #define GICV2M_MSI_TYPER 0x008 #define MSI_TYPER_SPI_BASE(x) (((x) >> 16) & 0x3ff) #define MSI_TYPER_SPI_COUNT(x) (((x) >> 0) & 0x3ff) #define GICv2M_MSI_SETSPI_NS 0x040 #define GICV2M_MSI_IIDR 0xFCC struct arm_gicv2m_softc { struct resource *sc_mem; struct mtx sc_mutex; u_int sc_spi_start; u_int sc_spi_count; u_int sc_spi_offset; }; static struct ofw_compat_data gicv2m_compat_data[] = { {"arm,gic-v2m-frame", true}, {NULL, false} }; static int arm_gicv2m_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_search_compatible(dev, gicv2m_compat_data)->ocd_data) return (ENXIO); device_set_desc(dev, "ARM Generic Interrupt Controller MSI/MSIX"); return (BUS_PROBE_DEFAULT); } static int arm_gicv2m_attach(device_t dev) { struct arm_gicv2m_softc *sc; uint32_t typer; int rid; sc = device_get_softc(dev); rid = 0; sc->sc_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->sc_mem == NULL) { device_printf(dev, "Unable to allocate resources\n"); return (ENXIO); } typer = bus_read_4(sc->sc_mem, GICV2M_MSI_TYPER); sc->sc_spi_start = MSI_TYPER_SPI_BASE(typer); sc->sc_spi_count = MSI_TYPER_SPI_COUNT(typer); mtx_init(&sc->sc_mutex, "GICv2m lock", "", MTX_DEF); if (bootverbose) device_printf(dev, "using spi %u to %u\n", sc->sc_spi_start, sc->sc_spi_start + sc->sc_spi_count - 1); return (0); } static device_method_t arm_gicv2m_methods[] = { /* Device interface */ DEVMETHOD(device_probe, arm_gicv2m_probe), DEVMETHOD(device_attach, arm_gicv2m_attach), /* End */ DEVMETHOD_END }; DEFINE_CLASS_0(gicv2m, arm_gicv2m_driver, arm_gicv2m_methods, sizeof(struct arm_gicv2m_softc)); static devclass_t arm_gicv2m_devclass; EARLY_DRIVER_MODULE(gicv2m, gic, arm_gicv2m_driver, arm_gicv2m_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); #endif Index: user/ngie/release-pkg-fix-tests/sys/arm/arm/machdep.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/arm/machdep.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/arm/machdep.c (revision 299091) @@ -1,2015 +1,2015 @@ /* $NetBSD: arm32_machdep.c,v 1.44 2004/03/24 15:34:47 atatat Exp $ */ /*- * Copyright (c) 2004 Olivier Houchard * Copyright (c) 1994-1998 Mark Brinicombe. * Copyright (c) 1994 Brini. * All rights reserved. * * This code is derived from software written for Brini by Mark Brinicombe * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Mark Brinicombe * for the NetBSD Project. * 4. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * Machine dependant functions for kernel setup + * Machine dependent functions for kernel setup * * Created : 17/09/94 * Updated : 18/04/01 updated for new wscons */ #include "opt_compat.h" #include "opt_ddb.h" #include "opt_kstack_pages.h" #include "opt_platform.h" #include "opt_sched.h" #include "opt_timer.h" #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef FDT #include #include #include #endif #ifdef DDB #include #if __ARM_ARCH >= 6 DB_SHOW_COMMAND(cp15, db_show_cp15) { u_int reg; reg = cp15_midr_get(); db_printf("Cpu ID: 0x%08x\n", reg); reg = cp15_ctr_get(); db_printf("Current Cache Lvl ID: 0x%08x\n",reg); reg = cp15_sctlr_get(); db_printf("Ctrl: 0x%08x\n",reg); reg = cp15_actlr_get(); db_printf("Aux Ctrl: 0x%08x\n",reg); reg = cp15_id_pfr0_get(); db_printf("Processor Feat 0: 0x%08x\n", reg); reg = cp15_id_pfr1_get(); db_printf("Processor Feat 1: 0x%08x\n", reg); reg = cp15_id_dfr0_get(); db_printf("Debug Feat 0: 0x%08x\n", reg); reg = cp15_id_afr0_get(); db_printf("Auxiliary Feat 0: 0x%08x\n", reg); reg = cp15_id_mmfr0_get(); db_printf("Memory Model Feat 0: 0x%08x\n", reg); reg = cp15_id_mmfr1_get(); db_printf("Memory Model Feat 1: 0x%08x\n", reg); reg = cp15_id_mmfr2_get(); db_printf("Memory Model Feat 2: 0x%08x\n", reg); reg = cp15_id_mmfr3_get(); db_printf("Memory Model Feat 3: 0x%08x\n", reg); reg = cp15_ttbr_get(); db_printf("TTB0: 0x%08x\n", reg); } DB_SHOW_COMMAND(vtop, db_show_vtop) { u_int reg; if (have_addr) { cp15_ats1cpr_set(addr); reg = cp15_par_get(); db_printf("Physical address reg: 0x%08x\n",reg); } else db_printf("show vtop \n"); } #endif /* __ARM_ARCH >= 6 */ #endif /* DDB */ #ifdef DEBUG #define debugf(fmt, args...) printf(fmt, ##args) #else #define debugf(fmt, args...) #endif #if defined(COMPAT_FREEBSD4) || defined(COMPAT_FREEBSD5) || \ defined(COMPAT_FREEBSD6) || defined(COMPAT_FREEBSD7) || \ defined(COMPAT_FREEBSD9) #error FreeBSD/arm doesn't provide compatibility with releases prior to 10 #endif struct pcpu __pcpu[MAXCPU]; struct pcpu *pcpup = &__pcpu[0]; static struct trapframe proc0_tf; uint32_t cpu_reset_address = 0; int cold = 1; vm_offset_t vector_page; int (*_arm_memcpy)(void *, void *, int, int) = NULL; int (*_arm_bzero)(void *, int, int) = NULL; int _min_memcpy_size = 0; int _min_bzero_size = 0; extern int *end; #ifdef FDT static char *loader_envp; vm_paddr_t pmap_pa; #if __ARM_ARCH >= 6 vm_offset_t systempage; vm_offset_t irqstack; vm_offset_t undstack; vm_offset_t abtstack; #else /* * This is the number of L2 page tables required for covering max * (hypothetical) memsize of 4GB and all kernel mappings (vectors, msgbuf, * stacks etc.), uprounded to be divisible by 4. */ #define KERNEL_PT_MAX 78 static struct pv_addr kernel_pt_table[KERNEL_PT_MAX]; struct pv_addr systempage; static struct pv_addr msgbufpv; struct pv_addr irqstack; struct pv_addr undstack; struct pv_addr abtstack; static struct pv_addr kernelstack; #endif #endif #if defined(LINUX_BOOT_ABI) #define LBABI_MAX_BANKS 10 #define CMDLINE_GUARD "FreeBSD:" uint32_t board_id; struct arm_lbabi_tag *atag_list; char linux_command_line[LBABI_MAX_COMMAND_LINE + 1]; char atags[LBABI_MAX_COMMAND_LINE * 2]; uint32_t memstart[LBABI_MAX_BANKS]; uint32_t memsize[LBABI_MAX_BANKS]; uint32_t membanks; #endif #ifdef MULTIDELAY static delay_func *delay_impl; static void *delay_arg; #endif static uint32_t board_revision; /* hex representation of uint64_t */ static char board_serial[32]; SYSCTL_NODE(_hw, OID_AUTO, board, CTLFLAG_RD, 0, "Board attributes"); SYSCTL_UINT(_hw_board, OID_AUTO, revision, CTLFLAG_RD, &board_revision, 0, "Board revision"); SYSCTL_STRING(_hw_board, OID_AUTO, serial, CTLFLAG_RD, board_serial, 0, "Board serial"); int vfp_exists; SYSCTL_INT(_hw, HW_FLOATINGPT, floatingpoint, CTLFLAG_RD, &vfp_exists, 0, "Floating point support enabled"); void board_set_serial(uint64_t serial) { snprintf(board_serial, sizeof(board_serial)-1, "%016jx", serial); } void board_set_revision(uint32_t revision) { board_revision = revision; } void sendsig(catcher, ksi, mask) sig_t catcher; ksiginfo_t *ksi; sigset_t *mask; { struct thread *td; struct proc *p; struct trapframe *tf; struct sigframe *fp, frame; struct sigacts *psp; struct sysentvec *sysent; int onstack; int sig; int code; td = curthread; p = td->td_proc; PROC_LOCK_ASSERT(p, MA_OWNED); sig = ksi->ksi_signo; code = ksi->ksi_code; psp = p->p_sigacts; mtx_assert(&psp->ps_mtx, MA_OWNED); tf = td->td_frame; onstack = sigonstack(tf->tf_usr_sp); CTR4(KTR_SIG, "sendsig: td=%p (%s) catcher=%p sig=%d", td, p->p_comm, catcher, sig); /* Allocate and validate space for the signal handler context. */ if ((td->td_pflags & TDP_ALTSTACK) != 0 && !(onstack) && SIGISMEMBER(psp->ps_sigonstack, sig)) { fp = (struct sigframe *)((uintptr_t)td->td_sigstk.ss_sp + td->td_sigstk.ss_size); #if defined(COMPAT_43) td->td_sigstk.ss_flags |= SS_ONSTACK; #endif } else fp = (struct sigframe *)td->td_frame->tf_usr_sp; /* make room on the stack */ fp--; /* make the stack aligned */ fp = (struct sigframe *)STACKALIGN(fp); /* Populate the siginfo frame. */ get_mcontext(td, &frame.sf_uc.uc_mcontext, 0); frame.sf_si = ksi->ksi_info; frame.sf_uc.uc_sigmask = *mask; frame.sf_uc.uc_stack.ss_flags = (td->td_pflags & TDP_ALTSTACK ) ? ((onstack) ? SS_ONSTACK : 0) : SS_DISABLE; frame.sf_uc.uc_stack = td->td_sigstk; mtx_unlock(&psp->ps_mtx); PROC_UNLOCK(td->td_proc); /* Copy the sigframe out to the user's stack. */ if (copyout(&frame, fp, sizeof(*fp)) != 0) { /* Process has trashed its stack. Kill it. */ CTR2(KTR_SIG, "sendsig: sigexit td=%p fp=%p", td, fp); PROC_LOCK(p); sigexit(td, SIGILL); } /* * Build context to run handler in. We invoke the handler * directly, only returning via the trampoline. Note the * trampoline version numbers are coordinated with machine- * dependent code in libc. */ tf->tf_r0 = sig; tf->tf_r1 = (register_t)&fp->sf_si; tf->tf_r2 = (register_t)&fp->sf_uc; /* the trampoline uses r5 as the uc address */ tf->tf_r5 = (register_t)&fp->sf_uc; tf->tf_pc = (register_t)catcher; tf->tf_usr_sp = (register_t)fp; sysent = p->p_sysent; if (sysent->sv_sigcode_base != 0) tf->tf_usr_lr = (register_t)sysent->sv_sigcode_base; else tf->tf_usr_lr = (register_t)(sysent->sv_psstrings - *(sysent->sv_szsigcode)); /* Set the mode to enter in the signal handler */ #if __ARM_ARCH >= 7 if ((register_t)catcher & 1) tf->tf_spsr |= PSR_T; else tf->tf_spsr &= ~PSR_T; #endif CTR3(KTR_SIG, "sendsig: return td=%p pc=%#x sp=%#x", td, tf->tf_usr_lr, tf->tf_usr_sp); PROC_LOCK(p); mtx_lock(&psp->ps_mtx); } struct kva_md_info kmi; /* * arm32_vector_init: * * Initialize the vector page, and select whether or not to * relocate the vectors. * * NOTE: We expect the vector page to be mapped at its expected * destination. */ extern unsigned int page0[], page0_data[]; void arm_vector_init(vm_offset_t va, int which) { unsigned int *vectors = (int *) va; unsigned int *vectors_data = vectors + (page0_data - page0); int vec; /* * Loop through the vectors we're taking over, and copy the * vector's insn and data word. */ for (vec = 0; vec < ARM_NVEC; vec++) { if ((which & (1 << vec)) == 0) { /* Don't want to take over this vector. */ continue; } vectors[vec] = page0[vec]; vectors_data[vec] = page0_data[vec]; } /* Now sync the vectors. */ icache_sync(va, (ARM_NVEC * 2) * sizeof(u_int)); vector_page = va; if (va == ARM_VECTORS_HIGH) { /* * Assume the MD caller knows what it's doing here, and * really does want the vector page relocated. * * Note: This has to be done here (and not just in * cpu_setup()) because the vector page needs to be * accessible *before* cpu_startup() is called. * Think ddb(9) ... * * NOTE: If the CPU control register is not readable, * this will totally fail! We'll just assume that * any system that has high vector support has a * readable CPU control register, for now. If we * ever encounter one that does not, we'll have to * rethink this. */ cpu_control(CPU_CONTROL_VECRELOC, CPU_CONTROL_VECRELOC); } } static void cpu_startup(void *dummy) { struct pcb *pcb = thread0.td_pcb; const unsigned int mbyte = 1024 * 1024; #if __ARM_ARCH < 6 && !defined(ARM_CACHE_LOCK_ENABLE) vm_page_t m; #endif identify_arm_cpu(); vm_ksubmap_init(&kmi); /* * Display the RAM layout. */ printf("real memory = %ju (%ju MB)\n", (uintmax_t)arm32_ptob(realmem), (uintmax_t)arm32_ptob(realmem) / mbyte); printf("avail memory = %ju (%ju MB)\n", (uintmax_t)arm32_ptob(vm_cnt.v_free_count), (uintmax_t)arm32_ptob(vm_cnt.v_free_count) / mbyte); if (bootverbose) { arm_physmem_print_tables(); devmap_print_table(); } bufinit(); vm_pager_bufferinit(); pcb->pcb_regs.sf_sp = (u_int)thread0.td_kstack + USPACE_SVC_STACK_TOP; pmap_set_pcb_pagedir(kernel_pmap, pcb); #if __ARM_ARCH < 6 vector_page_setprot(VM_PROT_READ); pmap_postinit(); #ifdef ARM_CACHE_LOCK_ENABLE pmap_kenter_user(ARM_TP_ADDRESS, ARM_TP_ADDRESS); arm_lock_cache_line(ARM_TP_ADDRESS); #else m = vm_page_alloc(NULL, 0, VM_ALLOC_NOOBJ | VM_ALLOC_ZERO); pmap_kenter_user(ARM_TP_ADDRESS, VM_PAGE_TO_PHYS(m)); #endif *(uint32_t *)ARM_RAS_START = 0; *(uint32_t *)ARM_RAS_END = 0xffffffff; #endif } SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL); /* * Flush the D-cache for non-DMA I/O so that the I-cache can * be made coherent later. */ void cpu_flush_dcache(void *ptr, size_t len) { dcache_wb_poc((vm_offset_t)ptr, (vm_paddr_t)vtophys(ptr), len); } /* Get current clock frequency for the given cpu id. */ int cpu_est_clockrate(int cpu_id, uint64_t *rate) { return (ENXIO); } void cpu_idle(int busy) { CTR2(KTR_SPARE2, "cpu_idle(%d) at %d", busy, curcpu); spinlock_enter(); #ifndef NO_EVENTTIMERS if (!busy) cpu_idleclock(); #endif if (!sched_runnable()) cpu_sleep(0); #ifndef NO_EVENTTIMERS if (!busy) cpu_activeclock(); #endif spinlock_exit(); CTR2(KTR_SPARE2, "cpu_idle(%d) at %d done", busy, curcpu); } int cpu_idle_wakeup(int cpu) { return (0); } /* * Most ARM platforms don't need to do anything special to init their clocks * (they get intialized during normal device attachment), and by not defining a * cpu_initclocks() function they get this generic one. Any platform that needs * to do something special can just provide their own implementation, which will * override this one due to the weak linkage. */ void arm_generic_initclocks(void) { #ifndef NO_EVENTTIMERS #ifdef SMP if (PCPU_GET(cpuid) == 0) cpu_initclocks_bsp(); else cpu_initclocks_ap(); #else cpu_initclocks_bsp(); #endif #endif } __weak_reference(arm_generic_initclocks, cpu_initclocks); #ifdef MULTIDELAY void arm_set_delay(delay_func *impl, void *arg) { KASSERT(impl != NULL, ("No DELAY implementation")); delay_impl = impl; delay_arg = arg; } void DELAY(int usec) { delay_impl(usec, delay_arg); } #endif int fill_regs(struct thread *td, struct reg *regs) { struct trapframe *tf = td->td_frame; bcopy(&tf->tf_r0, regs->r, sizeof(regs->r)); regs->r_sp = tf->tf_usr_sp; regs->r_lr = tf->tf_usr_lr; regs->r_pc = tf->tf_pc; regs->r_cpsr = tf->tf_spsr; return (0); } int fill_fpregs(struct thread *td, struct fpreg *regs) { bzero(regs, sizeof(*regs)); return (0); } int set_regs(struct thread *td, struct reg *regs) { struct trapframe *tf = td->td_frame; bcopy(regs->r, &tf->tf_r0, sizeof(regs->r)); tf->tf_usr_sp = regs->r_sp; tf->tf_usr_lr = regs->r_lr; tf->tf_pc = regs->r_pc; tf->tf_spsr &= ~PSR_FLAGS; tf->tf_spsr |= regs->r_cpsr & PSR_FLAGS; return (0); } int set_fpregs(struct thread *td, struct fpreg *regs) { return (0); } int fill_dbregs(struct thread *td, struct dbreg *regs) { return (0); } int set_dbregs(struct thread *td, struct dbreg *regs) { return (0); } static int ptrace_read_int(struct thread *td, vm_offset_t addr, uint32_t *v) { if (proc_readmem(td, td->td_proc, addr, v, sizeof(*v)) != sizeof(*v)) return (ENOMEM); return (0); } static int ptrace_write_int(struct thread *td, vm_offset_t addr, uint32_t v) { if (proc_writemem(td, td->td_proc, addr, &v, sizeof(v)) != sizeof(v)) return (ENOMEM); return (0); } static u_int ptrace_get_usr_reg(void *cookie, int reg) { int ret; struct thread *td = cookie; KASSERT(((reg >= 0) && (reg <= ARM_REG_NUM_PC)), ("reg is outside range")); switch(reg) { case ARM_REG_NUM_PC: ret = td->td_frame->tf_pc; break; case ARM_REG_NUM_LR: ret = td->td_frame->tf_usr_lr; break; case ARM_REG_NUM_SP: ret = td->td_frame->tf_usr_sp; break; default: ret = *((register_t*)&td->td_frame->tf_r0 + reg); break; } return (ret); } static u_int ptrace_get_usr_int(void* cookie, vm_offset_t offset, u_int* val) { struct thread *td = cookie; u_int error; error = ptrace_read_int(td, offset, val); return (error); } /** * This function parses current instruction opcode and decodes * any possible jump (change in PC) which might occur after * the instruction is executed. * * @param td Thread structure of analysed task * @param cur_instr Currently executed instruction * @param alt_next_address Pointer to the variable where * the destination address of the * jump instruction shall be stored. * * @return <0> when jump is possible * otherwise */ static int ptrace_get_alternative_next(struct thread *td, uint32_t cur_instr, uint32_t *alt_next_address) { int error; if (inst_branch(cur_instr) || inst_call(cur_instr) || inst_return(cur_instr)) { error = arm_predict_branch(td, cur_instr, td->td_frame->tf_pc, alt_next_address, ptrace_get_usr_reg, ptrace_get_usr_int); return (error); } return (EINVAL); } int ptrace_single_step(struct thread *td) { struct proc *p; int error, error_alt; uint32_t cur_instr, alt_next = 0; /* TODO: This needs to be updated for Thumb-2 */ if ((td->td_frame->tf_spsr & PSR_T) != 0) return (EINVAL); KASSERT(td->td_md.md_ptrace_instr == 0, ("Didn't clear single step")); KASSERT(td->td_md.md_ptrace_instr_alt == 0, ("Didn't clear alternative single step")); p = td->td_proc; PROC_UNLOCK(p); error = ptrace_read_int(td, td->td_frame->tf_pc, &cur_instr); if (error) goto out; error = ptrace_read_int(td, td->td_frame->tf_pc + INSN_SIZE, &td->td_md.md_ptrace_instr); if (error == 0) { error = ptrace_write_int(td, td->td_frame->tf_pc + INSN_SIZE, PTRACE_BREAKPOINT); if (error) { td->td_md.md_ptrace_instr = 0; } else { td->td_md.md_ptrace_addr = td->td_frame->tf_pc + INSN_SIZE; } } error_alt = ptrace_get_alternative_next(td, cur_instr, &alt_next); if (error_alt == 0) { error_alt = ptrace_read_int(td, alt_next, &td->td_md.md_ptrace_instr_alt); if (error_alt) { td->td_md.md_ptrace_instr_alt = 0; } else { error_alt = ptrace_write_int(td, alt_next, PTRACE_BREAKPOINT); if (error_alt) td->td_md.md_ptrace_instr_alt = 0; else td->td_md.md_ptrace_addr_alt = alt_next; } } out: PROC_LOCK(p); return ((error != 0) && (error_alt != 0)); } int ptrace_clear_single_step(struct thread *td) { struct proc *p; /* TODO: This needs to be updated for Thumb-2 */ if ((td->td_frame->tf_spsr & PSR_T) != 0) return (EINVAL); if (td->td_md.md_ptrace_instr != 0) { p = td->td_proc; PROC_UNLOCK(p); ptrace_write_int(td, td->td_md.md_ptrace_addr, td->td_md.md_ptrace_instr); PROC_LOCK(p); td->td_md.md_ptrace_instr = 0; } if (td->td_md.md_ptrace_instr_alt != 0) { p = td->td_proc; PROC_UNLOCK(p); ptrace_write_int(td, td->td_md.md_ptrace_addr_alt, td->td_md.md_ptrace_instr_alt); PROC_LOCK(p); td->td_md.md_ptrace_instr_alt = 0; } return (0); } int ptrace_set_pc(struct thread *td, unsigned long addr) { td->td_frame->tf_pc = addr; return (0); } void cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size) { } void spinlock_enter(void) { struct thread *td; register_t cspr; td = curthread; if (td->td_md.md_spinlock_count == 0) { cspr = disable_interrupts(PSR_I | PSR_F); td->td_md.md_spinlock_count = 1; td->td_md.md_saved_cspr = cspr; } else td->td_md.md_spinlock_count++; critical_enter(); } void spinlock_exit(void) { struct thread *td; register_t cspr; td = curthread; critical_exit(); cspr = td->td_md.md_saved_cspr; td->td_md.md_spinlock_count--; if (td->td_md.md_spinlock_count == 0) restore_interrupts(cspr); } /* * Clear registers on exec */ void exec_setregs(struct thread *td, struct image_params *imgp, u_long stack) { struct trapframe *tf = td->td_frame; memset(tf, 0, sizeof(*tf)); tf->tf_usr_sp = stack; tf->tf_usr_lr = imgp->entry_addr; tf->tf_svc_lr = 0x77777777; tf->tf_pc = imgp->entry_addr; tf->tf_spsr = PSR_USR32_MODE; } /* * Get machine context. */ int get_mcontext(struct thread *td, mcontext_t *mcp, int clear_ret) { struct trapframe *tf = td->td_frame; __greg_t *gr = mcp->__gregs; if (clear_ret & GET_MC_CLEAR_RET) { gr[_REG_R0] = 0; gr[_REG_CPSR] = tf->tf_spsr & ~PSR_C; } else { gr[_REG_R0] = tf->tf_r0; gr[_REG_CPSR] = tf->tf_spsr; } gr[_REG_R1] = tf->tf_r1; gr[_REG_R2] = tf->tf_r2; gr[_REG_R3] = tf->tf_r3; gr[_REG_R4] = tf->tf_r4; gr[_REG_R5] = tf->tf_r5; gr[_REG_R6] = tf->tf_r6; gr[_REG_R7] = tf->tf_r7; gr[_REG_R8] = tf->tf_r8; gr[_REG_R9] = tf->tf_r9; gr[_REG_R10] = tf->tf_r10; gr[_REG_R11] = tf->tf_r11; gr[_REG_R12] = tf->tf_r12; gr[_REG_SP] = tf->tf_usr_sp; gr[_REG_LR] = tf->tf_usr_lr; gr[_REG_PC] = tf->tf_pc; return (0); } /* * Set machine context. * * However, we don't set any but the user modifiable flags, and we won't * touch the cs selector. */ int set_mcontext(struct thread *td, mcontext_t *mcp) { struct trapframe *tf = td->td_frame; const __greg_t *gr = mcp->__gregs; tf->tf_r0 = gr[_REG_R0]; tf->tf_r1 = gr[_REG_R1]; tf->tf_r2 = gr[_REG_R2]; tf->tf_r3 = gr[_REG_R3]; tf->tf_r4 = gr[_REG_R4]; tf->tf_r5 = gr[_REG_R5]; tf->tf_r6 = gr[_REG_R6]; tf->tf_r7 = gr[_REG_R7]; tf->tf_r8 = gr[_REG_R8]; tf->tf_r9 = gr[_REG_R9]; tf->tf_r10 = gr[_REG_R10]; tf->tf_r11 = gr[_REG_R11]; tf->tf_r12 = gr[_REG_R12]; tf->tf_usr_sp = gr[_REG_SP]; tf->tf_usr_lr = gr[_REG_LR]; tf->tf_pc = gr[_REG_PC]; tf->tf_spsr = gr[_REG_CPSR]; return (0); } /* * MPSAFE */ int sys_sigreturn(td, uap) struct thread *td; struct sigreturn_args /* { const struct __ucontext *sigcntxp; } */ *uap; { ucontext_t uc; int spsr; if (uap == NULL) return (EFAULT); if (copyin(uap->sigcntxp, &uc, sizeof(uc))) return (EFAULT); /* * Make sure the processor mode has not been tampered with and * interrupts have not been disabled. */ spsr = uc.uc_mcontext.__gregs[_REG_CPSR]; if ((spsr & PSR_MODE) != PSR_USR32_MODE || (spsr & (PSR_I | PSR_F)) != 0) return (EINVAL); /* Restore register context. */ set_mcontext(td, &uc.uc_mcontext); /* Restore signal mask. */ kern_sigprocmask(td, SIG_SETMASK, &uc.uc_sigmask, NULL, 0); return (EJUSTRETURN); } /* * Construct a PCB from a trapframe. This is called from kdb_trap() where * we want to start a backtrace from the function that caused us to enter * the debugger. We have the context in the trapframe, but base the trace * on the PCB. The PCB doesn't have to be perfect, as long as it contains * enough for a backtrace. */ void makectx(struct trapframe *tf, struct pcb *pcb) { pcb->pcb_regs.sf_r4 = tf->tf_r4; pcb->pcb_regs.sf_r5 = tf->tf_r5; pcb->pcb_regs.sf_r6 = tf->tf_r6; pcb->pcb_regs.sf_r7 = tf->tf_r7; pcb->pcb_regs.sf_r8 = tf->tf_r8; pcb->pcb_regs.sf_r9 = tf->tf_r9; pcb->pcb_regs.sf_r10 = tf->tf_r10; pcb->pcb_regs.sf_r11 = tf->tf_r11; pcb->pcb_regs.sf_r12 = tf->tf_r12; pcb->pcb_regs.sf_pc = tf->tf_pc; pcb->pcb_regs.sf_lr = tf->tf_usr_lr; pcb->pcb_regs.sf_sp = tf->tf_usr_sp; } /* * Fake up a boot descriptor table */ vm_offset_t fake_preload_metadata(struct arm_boot_params *abp __unused, void *dtb_ptr, size_t dtb_size) { #ifdef DDB vm_offset_t zstart = 0, zend = 0; #endif vm_offset_t lastaddr; int i = 0; static uint32_t fake_preload[35]; fake_preload[i++] = MODINFO_NAME; fake_preload[i++] = strlen("kernel") + 1; strcpy((char*)&fake_preload[i++], "kernel"); i += 1; fake_preload[i++] = MODINFO_TYPE; fake_preload[i++] = strlen("elf kernel") + 1; strcpy((char*)&fake_preload[i++], "elf kernel"); i += 2; fake_preload[i++] = MODINFO_ADDR; fake_preload[i++] = sizeof(vm_offset_t); fake_preload[i++] = KERNVIRTADDR; fake_preload[i++] = MODINFO_SIZE; fake_preload[i++] = sizeof(uint32_t); fake_preload[i++] = (uint32_t)&end - KERNVIRTADDR; #ifdef DDB if (*(uint32_t *)KERNVIRTADDR == MAGIC_TRAMP_NUMBER) { fake_preload[i++] = MODINFO_METADATA|MODINFOMD_SSYM; fake_preload[i++] = sizeof(vm_offset_t); fake_preload[i++] = *(uint32_t *)(KERNVIRTADDR + 4); fake_preload[i++] = MODINFO_METADATA|MODINFOMD_ESYM; fake_preload[i++] = sizeof(vm_offset_t); fake_preload[i++] = *(uint32_t *)(KERNVIRTADDR + 8); lastaddr = *(uint32_t *)(KERNVIRTADDR + 8); zend = lastaddr; zstart = *(uint32_t *)(KERNVIRTADDR + 4); db_fetch_ksymtab(zstart, zend); } else #endif lastaddr = (vm_offset_t)&end; if (dtb_ptr != NULL) { /* Copy DTB to KVA space and insert it into module chain. */ lastaddr = roundup(lastaddr, sizeof(int)); fake_preload[i++] = MODINFO_METADATA | MODINFOMD_DTBP; fake_preload[i++] = sizeof(uint32_t); fake_preload[i++] = (uint32_t)lastaddr; memmove((void *)lastaddr, dtb_ptr, dtb_size); lastaddr += dtb_size; lastaddr = roundup(lastaddr, sizeof(int)); } fake_preload[i++] = 0; fake_preload[i] = 0; preload_metadata = (void *)fake_preload; init_static_kenv(NULL, 0); return (lastaddr); } void pcpu0_init(void) { #if __ARM_ARCH >= 6 set_curthread(&thread0); #endif pcpu_init(pcpup, 0, sizeof(struct pcpu)); PCPU_SET(curthread, &thread0); } #if defined(LINUX_BOOT_ABI) /* Convert the U-Boot command line into FreeBSD kenv and boot options. */ static void cmdline_set_env(char *cmdline, const char *guard) { char *cmdline_next, *env; size_t size, guard_len; int i; size = strlen(cmdline); /* Skip leading spaces. */ for (; isspace(*cmdline) && (size > 0); cmdline++) size--; /* Test and remove guard. */ if (guard != NULL && guard[0] != '\0') { guard_len = strlen(guard); if (strncasecmp(cmdline, guard, guard_len) != 0) return; cmdline += guard_len; size -= guard_len; } /* Skip leading spaces. */ for (; isspace(*cmdline) && (size > 0); cmdline++) size--; /* Replace ',' with '\0'. */ /* TODO: implement escaping for ',' character. */ cmdline_next = cmdline; while(strsep(&cmdline_next, ",") != NULL) ; init_static_kenv(cmdline, 0); /* Parse boothowto. */ for (i = 0; howto_names[i].ev != NULL; i++) { env = kern_getenv(howto_names[i].ev); if (env != NULL) { if (strtoul(env, NULL, 10) != 0) boothowto |= howto_names[i].mask; freeenv(env); } } } vm_offset_t linux_parse_boot_param(struct arm_boot_params *abp) { struct arm_lbabi_tag *walker; uint32_t revision; uint64_t serial; int size; vm_offset_t lastaddr; #ifdef FDT struct fdt_header *dtb_ptr; uint32_t dtb_size; #endif /* * Linux boot ABI: r0 = 0, r1 is the board type (!= 0) and r2 * is atags or dtb pointer. If all of these aren't satisfied, * then punt. Unfortunately, it looks like DT enabled kernels * doesn't uses board type and U-Boot delivers 0 in r1 for them. */ if (abp->abp_r0 != 0 || abp->abp_r2 == 0) return (0); #ifdef FDT /* Test if r2 point to valid DTB. */ dtb_ptr = (struct fdt_header *)abp->abp_r2; if (fdt_check_header(dtb_ptr) == 0) { dtb_size = fdt_totalsize(dtb_ptr); return (fake_preload_metadata(abp, dtb_ptr, dtb_size)); } #endif board_id = abp->abp_r1; walker = (struct arm_lbabi_tag *)abp->abp_r2; if (ATAG_TAG(walker) != ATAG_CORE) return 0; atag_list = walker; while (ATAG_TAG(walker) != ATAG_NONE) { switch (ATAG_TAG(walker)) { case ATAG_CORE: break; case ATAG_MEM: arm_physmem_hardware_region(walker->u.tag_mem.start, walker->u.tag_mem.size); break; case ATAG_INITRD2: break; case ATAG_SERIAL: serial = walker->u.tag_sn.high; serial <<= 32; serial |= walker->u.tag_sn.low; board_set_serial(serial); break; case ATAG_REVISION: revision = walker->u.tag_rev.rev; board_set_revision(revision); break; case ATAG_CMDLINE: size = ATAG_SIZE(walker) - sizeof(struct arm_lbabi_header); size = min(size, LBABI_MAX_COMMAND_LINE); strncpy(linux_command_line, walker->u.tag_cmd.command, size); linux_command_line[size] = '\0'; break; default: break; } walker = ATAG_NEXT(walker); } /* Save a copy for later */ bcopy(atag_list, atags, (char *)walker - (char *)atag_list + ATAG_SIZE(walker)); lastaddr = fake_preload_metadata(abp, NULL, 0); cmdline_set_env(linux_command_line, CMDLINE_GUARD); return lastaddr; } #endif #if defined(FREEBSD_BOOT_LOADER) vm_offset_t freebsd_parse_boot_param(struct arm_boot_params *abp) { vm_offset_t lastaddr = 0; void *mdp; void *kmdp; #ifdef DDB vm_offset_t ksym_start; vm_offset_t ksym_end; #endif /* * Mask metadata pointer: it is supposed to be on page boundary. If * the first argument (mdp) doesn't point to a valid address the * bootloader must have passed us something else than the metadata * ptr, so we give up. Also give up if we cannot find metadta section * the loader creates that we get all this data out of. */ if ((mdp = (void *)(abp->abp_r0 & ~PAGE_MASK)) == NULL) return 0; preload_metadata = mdp; kmdp = preload_search_by_type("elf kernel"); if (kmdp == NULL) return 0; boothowto = MD_FETCH(kmdp, MODINFOMD_HOWTO, int); loader_envp = MD_FETCH(kmdp, MODINFOMD_ENVP, char *); init_static_kenv(loader_envp, 0); lastaddr = MD_FETCH(kmdp, MODINFOMD_KERNEND, vm_offset_t); #ifdef DDB ksym_start = MD_FETCH(kmdp, MODINFOMD_SSYM, uintptr_t); ksym_end = MD_FETCH(kmdp, MODINFOMD_ESYM, uintptr_t); db_fetch_ksymtab(ksym_start, ksym_end); #endif return lastaddr; } #endif vm_offset_t default_parse_boot_param(struct arm_boot_params *abp) { vm_offset_t lastaddr; #if defined(LINUX_BOOT_ABI) if ((lastaddr = linux_parse_boot_param(abp)) != 0) return lastaddr; #endif #if defined(FREEBSD_BOOT_LOADER) if ((lastaddr = freebsd_parse_boot_param(abp)) != 0) return lastaddr; #endif /* Fall back to hardcoded metadata. */ lastaddr = fake_preload_metadata(abp, NULL, 0); return lastaddr; } /* * Stub version of the boot parameter parsing routine. We are * called early in initarm, before even VM has been initialized. * This routine needs to preserve any data that the boot loader * has passed in before the kernel starts to grow past the end * of the BSS, traditionally the place boot-loaders put this data. * * Since this is called so early, things that depend on the vm system * being setup (including access to some SoC's serial ports), about * all that can be done in this routine is to copy the arguments. * * This is the default boot parameter parsing routine. Individual * kernels/boards can override this weak function with one of their * own. We just fake metadata... */ __weak_reference(default_parse_boot_param, parse_boot_param); /* * Initialize proc0 */ void init_proc0(vm_offset_t kstack) { proc_linkup0(&proc0, &thread0); thread0.td_kstack = kstack; thread0.td_pcb = (struct pcb *) (thread0.td_kstack + kstack_pages * PAGE_SIZE) - 1; thread0.td_pcb->pcb_flags = 0; thread0.td_pcb->pcb_vfpcpu = -1; thread0.td_pcb->pcb_vfpstate.fpscr = VFPSCR_DN; thread0.td_frame = &proc0_tf; pcpup->pc_curpcb = thread0.td_pcb; } int arm_predict_branch(void *cookie, u_int insn, register_t pc, register_t *new_pc, u_int (*fetch_reg)(void*, int), u_int (*read_int)(void*, vm_offset_t, u_int*)) { u_int addr, nregs, offset = 0; int error = 0; switch ((insn >> 24) & 0xf) { case 0x2: /* add pc, reg1, #value */ case 0x0: /* add pc, reg1, reg2, lsl #offset */ addr = fetch_reg(cookie, (insn >> 16) & 0xf); if (((insn >> 16) & 0xf) == 15) addr += 8; if (insn & 0x0200000) { offset = (insn >> 7) & 0x1e; offset = (insn & 0xff) << (32 - offset) | (insn & 0xff) >> offset; } else { offset = fetch_reg(cookie, insn & 0x0f); if ((insn & 0x0000ff0) != 0x00000000) { if (insn & 0x10) nregs = fetch_reg(cookie, (insn >> 8) & 0xf); else nregs = (insn >> 7) & 0x1f; switch ((insn >> 5) & 3) { case 0: /* lsl */ offset = offset << nregs; break; case 1: /* lsr */ offset = offset >> nregs; break; default: break; /* XXX */ } } *new_pc = addr + offset; return (0); } case 0xa: /* b ... */ case 0xb: /* bl ... */ addr = ((insn << 2) & 0x03ffffff); if (addr & 0x02000000) addr |= 0xfc000000; *new_pc = (pc + 8 + addr); return (0); case 0x7: /* ldr pc, [pc, reg, lsl #2] */ addr = fetch_reg(cookie, insn & 0xf); addr = pc + 8 + (addr << 2); error = read_int(cookie, addr, &addr); *new_pc = addr; return (error); case 0x1: /* mov pc, reg */ *new_pc = fetch_reg(cookie, insn & 0xf); return (0); case 0x4: case 0x5: /* ldr pc, [reg] */ addr = fetch_reg(cookie, (insn >> 16) & 0xf); /* ldr pc, [reg, #offset] */ if (insn & (1 << 24)) offset = insn & 0xfff; if (insn & 0x00800000) addr += offset; else addr -= offset; error = read_int(cookie, addr, &addr); *new_pc = addr; return (error); case 0x8: /* ldmxx reg, {..., pc} */ case 0x9: addr = fetch_reg(cookie, (insn >> 16) & 0xf); nregs = (insn & 0x5555) + ((insn >> 1) & 0x5555); nregs = (nregs & 0x3333) + ((nregs >> 2) & 0x3333); nregs = (nregs + (nregs >> 4)) & 0x0f0f; nregs = (nregs + (nregs >> 8)) & 0x001f; switch ((insn >> 23) & 0x3) { case 0x0: /* ldmda */ addr = addr - 0; break; case 0x1: /* ldmia */ addr = addr + 0 + ((nregs - 1) << 2); break; case 0x2: /* ldmdb */ addr = addr - 4; break; case 0x3: /* ldmib */ addr = addr + 4 + ((nregs - 1) << 2); break; } error = read_int(cookie, addr, &addr); *new_pc = addr; return (error); default: return (EINVAL); } } #if __ARM_ARCH >= 6 void set_stackptrs(int cpu) { set_stackptr(PSR_IRQ32_MODE, irqstack + ((IRQ_STACK_SIZE * PAGE_SIZE) * (cpu + 1))); set_stackptr(PSR_ABT32_MODE, abtstack + ((ABT_STACK_SIZE * PAGE_SIZE) * (cpu + 1))); set_stackptr(PSR_UND32_MODE, undstack + ((UND_STACK_SIZE * PAGE_SIZE) * (cpu + 1))); } #else void set_stackptrs(int cpu) { set_stackptr(PSR_IRQ32_MODE, irqstack.pv_va + ((IRQ_STACK_SIZE * PAGE_SIZE) * (cpu + 1))); set_stackptr(PSR_ABT32_MODE, abtstack.pv_va + ((ABT_STACK_SIZE * PAGE_SIZE) * (cpu + 1))); set_stackptr(PSR_UND32_MODE, undstack.pv_va + ((UND_STACK_SIZE * PAGE_SIZE) * (cpu + 1))); } #endif #ifdef EFI #define efi_next_descriptor(ptr, size) \ ((struct efi_md *)(((uint8_t *) ptr) + size)) static void add_efi_map_entries(struct efi_map_header *efihdr, struct mem_region *mr, int *mrcnt) { struct efi_md *map, *p; const char *type; size_t efisz, memory_size; int ndesc, i, j; static const char *types[] = { "Reserved", "LoaderCode", "LoaderData", "BootServicesCode", "BootServicesData", "RuntimeServicesCode", "RuntimeServicesData", "ConventionalMemory", "UnusableMemory", "ACPIReclaimMemory", "ACPIMemoryNVS", "MemoryMappedIO", "MemoryMappedIOPortSpace", "PalCode" }; *mrcnt = 0; /* * Memory map data provided by UEFI via the GetMemoryMap * Boot Services API. */ efisz = roundup2(sizeof(struct efi_map_header), 0x10); map = (struct efi_md *)((uint8_t *)efihdr + efisz); if (efihdr->descriptor_size == 0) return; ndesc = efihdr->memory_size / efihdr->descriptor_size; if (boothowto & RB_VERBOSE) printf("%23s %12s %12s %8s %4s\n", "Type", "Physical", "Virtual", "#Pages", "Attr"); memory_size = 0; for (i = 0, j = 0, p = map; i < ndesc; i++, p = efi_next_descriptor(p, efihdr->descriptor_size)) { if (boothowto & RB_VERBOSE) { if (p->md_type <= EFI_MD_TYPE_PALCODE) type = types[p->md_type]; else type = ""; printf("%23s %012llx %12p %08llx ", type, p->md_phys, p->md_virt, p->md_pages); if (p->md_attr & EFI_MD_ATTR_UC) printf("UC "); if (p->md_attr & EFI_MD_ATTR_WC) printf("WC "); if (p->md_attr & EFI_MD_ATTR_WT) printf("WT "); if (p->md_attr & EFI_MD_ATTR_WB) printf("WB "); if (p->md_attr & EFI_MD_ATTR_UCE) printf("UCE "); if (p->md_attr & EFI_MD_ATTR_WP) printf("WP "); if (p->md_attr & EFI_MD_ATTR_RP) printf("RP "); if (p->md_attr & EFI_MD_ATTR_XP) printf("XP "); if (p->md_attr & EFI_MD_ATTR_RT) printf("RUNTIME"); printf("\n"); } switch (p->md_type) { case EFI_MD_TYPE_CODE: case EFI_MD_TYPE_DATA: case EFI_MD_TYPE_BS_CODE: case EFI_MD_TYPE_BS_DATA: case EFI_MD_TYPE_FREE: /* * We're allowed to use any entry with these types. */ break; default: continue; } j++; if (j >= FDT_MEM_REGIONS) break; mr[j].mr_start = p->md_phys; mr[j].mr_size = p->md_pages * PAGE_SIZE; memory_size += mr[j].mr_size; } *mrcnt = j; } #endif /* EFI */ #ifdef FDT static char * kenv_next(char *cp) { if (cp != NULL) { while (*cp != 0) cp++; cp++; if (*cp == 0) cp = NULL; } return (cp); } static void print_kenv(void) { char *cp; debugf("loader passed (static) kenv:\n"); if (loader_envp == NULL) { debugf(" no env, null ptr\n"); return; } debugf(" loader_envp = 0x%08x\n", (uint32_t)loader_envp); for (cp = loader_envp; cp != NULL; cp = kenv_next(cp)) debugf(" %x %s\n", (uint32_t)cp, cp); } #if __ARM_ARCH < 6 void * initarm(struct arm_boot_params *abp) { struct mem_region mem_regions[FDT_MEM_REGIONS]; struct pv_addr kernel_l1pt; struct pv_addr dpcpu; vm_offset_t dtbp, freemempos, l2_start, lastaddr; uint64_t memsize; uint32_t l2size; char *env; void *kmdp; u_int l1pagetable; int i, j, err_devmap, mem_regions_sz; lastaddr = parse_boot_param(abp); arm_physmem_kernaddr = abp->abp_physaddr; memsize = 0; cpuinfo_init(); set_cpufuncs(); /* * Find the dtb passed in by the boot loader. */ kmdp = preload_search_by_type("elf kernel"); if (kmdp != NULL) dtbp = MD_FETCH(kmdp, MODINFOMD_DTBP, vm_offset_t); else dtbp = (vm_offset_t)NULL; #if defined(FDT_DTB_STATIC) /* * In case the device tree blob was not retrieved (from metadata) try * to use the statically embedded one. */ if (dtbp == (vm_offset_t)NULL) dtbp = (vm_offset_t)&fdt_static_dtb; #endif if (OF_install(OFW_FDT, 0) == FALSE) panic("Cannot install FDT"); if (OF_init((void *)dtbp) != 0) panic("OF_init failed with the found device tree"); /* Grab physical memory regions information from device tree. */ if (fdt_get_mem_regions(mem_regions, &mem_regions_sz, &memsize) != 0) panic("Cannot get physical memory regions"); arm_physmem_hardware_regions(mem_regions, mem_regions_sz); /* Grab reserved memory regions information from device tree. */ if (fdt_get_reserved_regions(mem_regions, &mem_regions_sz) == 0) arm_physmem_exclude_regions(mem_regions, mem_regions_sz, EXFLAG_NODUMP | EXFLAG_NOALLOC); /* Platform-specific initialisation */ platform_probe_and_attach(); pcpu0_init(); /* Do basic tuning, hz etc */ init_param1(); /* Calculate number of L2 tables needed for mapping vm_page_array */ l2size = (memsize / PAGE_SIZE) * sizeof(struct vm_page); l2size = (l2size >> L1_S_SHIFT) + 1; /* * Add one table for end of kernel map, one for stacks, msgbuf and * L1 and L2 tables map and one for vectors map. */ l2size += 3; /* Make it divisible by 4 */ l2size = (l2size + 3) & ~3; freemempos = (lastaddr + PAGE_MASK) & ~PAGE_MASK; /* Define a macro to simplify memory allocation */ #define valloc_pages(var, np) \ alloc_pages((var).pv_va, (np)); \ (var).pv_pa = (var).pv_va + (abp->abp_physaddr - KERNVIRTADDR); #define alloc_pages(var, np) \ (var) = freemempos; \ freemempos += (np * PAGE_SIZE); \ memset((char *)(var), 0, ((np) * PAGE_SIZE)); while (((freemempos - L1_TABLE_SIZE) & (L1_TABLE_SIZE - 1)) != 0) freemempos += PAGE_SIZE; valloc_pages(kernel_l1pt, L1_TABLE_SIZE / PAGE_SIZE); for (i = 0, j = 0; i < l2size; ++i) { if (!(i % (PAGE_SIZE / L2_TABLE_SIZE_REAL))) { valloc_pages(kernel_pt_table[i], L2_TABLE_SIZE / PAGE_SIZE); j = i; } else { kernel_pt_table[i].pv_va = kernel_pt_table[j].pv_va + L2_TABLE_SIZE_REAL * (i - j); kernel_pt_table[i].pv_pa = kernel_pt_table[i].pv_va - KERNVIRTADDR + abp->abp_physaddr; } } /* * Allocate a page for the system page mapped to 0x00000000 * or 0xffff0000. This page will just contain the system vectors * and can be shared by all processes. */ valloc_pages(systempage, 1); /* Allocate dynamic per-cpu area. */ valloc_pages(dpcpu, DPCPU_SIZE / PAGE_SIZE); dpcpu_init((void *)dpcpu.pv_va, 0); /* Allocate stacks for all modes */ valloc_pages(irqstack, IRQ_STACK_SIZE * MAXCPU); valloc_pages(abtstack, ABT_STACK_SIZE * MAXCPU); valloc_pages(undstack, UND_STACK_SIZE * MAXCPU); valloc_pages(kernelstack, kstack_pages * MAXCPU); valloc_pages(msgbufpv, round_page(msgbufsize) / PAGE_SIZE); /* * Now we start construction of the L1 page table * We start by mapping the L2 page tables into the L1. * This means that we can replace L1 mappings later on if necessary */ l1pagetable = kernel_l1pt.pv_va; /* * Try to map as much as possible of kernel text and data using * 1MB section mapping and for the rest of initial kernel address * space use L2 coarse tables. * * Link L2 tables for mapping remainder of kernel (modulo 1MB) * and kernel structures */ l2_start = lastaddr & ~(L1_S_OFFSET); for (i = 0 ; i < l2size - 1; i++) pmap_link_l2pt(l1pagetable, l2_start + i * L1_S_SIZE, &kernel_pt_table[i]); pmap_curmaxkvaddr = l2_start + (l2size - 1) * L1_S_SIZE; /* Map kernel code and data */ pmap_map_chunk(l1pagetable, KERNVIRTADDR, abp->abp_physaddr, (((uint32_t)(lastaddr) - KERNVIRTADDR) + PAGE_MASK) & ~PAGE_MASK, VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); /* Map L1 directory and allocated L2 page tables */ pmap_map_chunk(l1pagetable, kernel_l1pt.pv_va, kernel_l1pt.pv_pa, L1_TABLE_SIZE, VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE); pmap_map_chunk(l1pagetable, kernel_pt_table[0].pv_va, kernel_pt_table[0].pv_pa, L2_TABLE_SIZE_REAL * l2size, VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE); /* Map allocated DPCPU, stacks and msgbuf */ pmap_map_chunk(l1pagetable, dpcpu.pv_va, dpcpu.pv_pa, freemempos - dpcpu.pv_va, VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); /* Link and map the vector page */ pmap_link_l2pt(l1pagetable, ARM_VECTORS_HIGH, &kernel_pt_table[l2size - 1]); pmap_map_entry(l1pagetable, ARM_VECTORS_HIGH, systempage.pv_pa, VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE, PTE_CACHE); /* Establish static device mappings. */ err_devmap = platform_devmap_init(); devmap_bootstrap(l1pagetable, NULL); vm_max_kernel_address = platform_lastaddr(); cpu_domains((DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL * 2)) | DOMAIN_CLIENT); pmap_pa = kernel_l1pt.pv_pa; cpu_setttb(kernel_l1pt.pv_pa); cpu_tlb_flushID(); cpu_domains(DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL * 2)); /* * Now that proper page tables are installed, call cpu_setup() to enable * instruction and data caches and other chip-specific features. */ cpu_setup(); /* * Only after the SOC registers block is mapped we can perform device * tree fixups, as they may attempt to read parameters from hardware. */ OF_interpret("perform-fixup", 0); platform_gpio_init(); cninit(); debugf("initarm: console initialized\n"); debugf(" arg1 kmdp = 0x%08x\n", (uint32_t)kmdp); debugf(" boothowto = 0x%08x\n", boothowto); debugf(" dtbp = 0x%08x\n", (uint32_t)dtbp); print_kenv(); env = kern_getenv("kernelname"); if (env != NULL) { strlcpy(kernelname, env, sizeof(kernelname)); freeenv(env); } if (err_devmap != 0) printf("WARNING: could not fully configure devmap, error=%d\n", err_devmap); platform_late_init(); /* * Pages were allocated during the secondary bootstrap for the * stacks for different CPU modes. * We must now set the r13 registers in the different CPU modes to * point to these stacks. * Since the ARM stacks use STMFD etc. we must set r13 to the top end * of the stack memory. */ cpu_control(CPU_CONTROL_MMU_ENABLE, CPU_CONTROL_MMU_ENABLE); set_stackptrs(0); /* * We must now clean the cache again.... * Cleaning may be done by reading new data to displace any * dirty data in the cache. This will have happened in cpu_setttb() * but since we are boot strapping the addresses used for the read * may have just been remapped and thus the cache could be out * of sync. A re-clean after the switch will cure this. * After booting there are no gross relocations of the kernel thus * this problem will not occur after initarm(). */ cpu_idcache_wbinv_all(); undefined_init(); init_proc0(kernelstack.pv_va); arm_vector_init(ARM_VECTORS_HIGH, ARM_VEC_ALL); pmap_bootstrap(freemempos, &kernel_l1pt); msgbufp = (void *)msgbufpv.pv_va; msgbufinit(msgbufp, msgbufsize); mutex_init(); /* * Exclude the kernel (and all the things we allocated which immediately * follow the kernel) from the VM allocation pool but not from crash * dumps. virtual_avail is a global variable which tracks the kva we've * "allocated" while setting up pmaps. * * Prepare the list of physical memory available to the vm subsystem. */ arm_physmem_exclude_region(abp->abp_physaddr, (virtual_avail - KERNVIRTADDR), EXFLAG_NOALLOC); arm_physmem_init_kernel_globals(); init_param2(physmem); dbg_monitor_init(); kdb_init(); return ((void *)(kernelstack.pv_va + USPACE_SVC_STACK_TOP - sizeof(struct pcb))); } #else /* __ARM_ARCH < 6 */ void * initarm(struct arm_boot_params *abp) { struct mem_region mem_regions[FDT_MEM_REGIONS]; vm_paddr_t lastaddr; vm_offset_t dtbp, kernelstack, dpcpu; char *env; void *kmdp; int err_devmap, mem_regions_sz; #ifdef EFI struct efi_map_header *efihdr; #endif /* get last allocated physical address */ arm_physmem_kernaddr = abp->abp_physaddr; lastaddr = parse_boot_param(abp) - KERNVIRTADDR + arm_physmem_kernaddr; set_cpufuncs(); cpuinfo_init(); /* * Find the dtb passed in by the boot loader. */ kmdp = preload_search_by_type("elf kernel"); dtbp = MD_FETCH(kmdp, MODINFOMD_DTBP, vm_offset_t); #if defined(FDT_DTB_STATIC) /* * In case the device tree blob was not retrieved (from metadata) try * to use the statically embedded one. */ if (dtbp == (vm_offset_t)NULL) dtbp = (vm_offset_t)&fdt_static_dtb; #endif if (OF_install(OFW_FDT, 0) == FALSE) panic("Cannot install FDT"); if (OF_init((void *)dtbp) != 0) panic("OF_init failed with the found device tree"); #if defined(LINUX_BOOT_ABI) if (loader_envp == NULL && fdt_get_chosen_bootargs(linux_command_line, LBABI_MAX_COMMAND_LINE) == 0) cmdline_set_env(linux_command_line, CMDLINE_GUARD); #endif #ifdef EFI efihdr = (struct efi_map_header *)preload_search_info(kmdp, MODINFO_METADATA | MODINFOMD_EFI_MAP); if (efihdr != NULL) { add_efi_map_entries(efihdr, mem_regions, &mem_regions_sz); } else #endif { /* Grab physical memory regions information from device tree. */ if (fdt_get_mem_regions(mem_regions, &mem_regions_sz,NULL) != 0) panic("Cannot get physical memory regions"); } arm_physmem_hardware_regions(mem_regions, mem_regions_sz); /* Grab reserved memory regions information from device tree. */ if (fdt_get_reserved_regions(mem_regions, &mem_regions_sz) == 0) arm_physmem_exclude_regions(mem_regions, mem_regions_sz, EXFLAG_NODUMP | EXFLAG_NOALLOC); /* * Set TEX remapping registers. * Setup kernel page tables and switch to kernel L1 page table. */ pmap_set_tex(); pmap_bootstrap_prepare(lastaddr); /* * Now that proper page tables are installed, call cpu_setup() to enable * instruction and data caches and other chip-specific features. */ cpu_setup(); /* Platform-specific initialisation */ platform_probe_and_attach(); pcpu0_init(); /* Do basic tuning, hz etc */ init_param1(); /* * Allocate a page for the system page mapped to 0xffff0000 * This page will just contain the system vectors and can be * shared by all processes. */ systempage = pmap_preboot_get_pages(1); /* Map the vector page. */ pmap_preboot_map_pages(systempage, ARM_VECTORS_HIGH, 1); if (virtual_end >= ARM_VECTORS_HIGH) virtual_end = ARM_VECTORS_HIGH - 1; /* Allocate dynamic per-cpu area. */ dpcpu = pmap_preboot_get_vpages(DPCPU_SIZE / PAGE_SIZE); dpcpu_init((void *)dpcpu, 0); /* Allocate stacks for all modes */ irqstack = pmap_preboot_get_vpages(IRQ_STACK_SIZE * MAXCPU); abtstack = pmap_preboot_get_vpages(ABT_STACK_SIZE * MAXCPU); undstack = pmap_preboot_get_vpages(UND_STACK_SIZE * MAXCPU ); kernelstack = pmap_preboot_get_vpages(kstack_pages * MAXCPU); /* Allocate message buffer. */ msgbufp = (void *)pmap_preboot_get_vpages( round_page(msgbufsize) / PAGE_SIZE); /* * Pages were allocated during the secondary bootstrap for the * stacks for different CPU modes. * We must now set the r13 registers in the different CPU modes to * point to these stacks. * Since the ARM stacks use STMFD etc. we must set r13 to the top end * of the stack memory. */ set_stackptrs(0); mutex_init(); /* Establish static device mappings. */ err_devmap = platform_devmap_init(); devmap_bootstrap(0, NULL); vm_max_kernel_address = platform_lastaddr(); /* * Only after the SOC registers block is mapped we can perform device * tree fixups, as they may attempt to read parameters from hardware. */ OF_interpret("perform-fixup", 0); platform_gpio_init(); cninit(); debugf("initarm: console initialized\n"); debugf(" arg1 kmdp = 0x%08x\n", (uint32_t)kmdp); debugf(" boothowto = 0x%08x\n", boothowto); debugf(" dtbp = 0x%08x\n", (uint32_t)dtbp); debugf(" lastaddr1: 0x%08x\n", lastaddr); print_kenv(); env = kern_getenv("kernelname"); if (env != NULL) strlcpy(kernelname, env, sizeof(kernelname)); if (err_devmap != 0) printf("WARNING: could not fully configure devmap, error=%d\n", err_devmap); platform_late_init(); /* * We must now clean the cache again.... * Cleaning may be done by reading new data to displace any * dirty data in the cache. This will have happened in cpu_setttb() * but since we are boot strapping the addresses used for the read * may have just been remapped and thus the cache could be out * of sync. A re-clean after the switch will cure this. * After booting there are no gross relocations of the kernel thus * this problem will not occur after initarm(). */ /* Set stack for exception handlers */ undefined_init(); init_proc0(kernelstack); arm_vector_init(ARM_VECTORS_HIGH, ARM_VEC_ALL); enable_interrupts(PSR_A); pmap_bootstrap(0); /* Exclude the kernel (and all the things we allocated which immediately * follow the kernel) from the VM allocation pool but not from crash * dumps. virtual_avail is a global variable which tracks the kva we've * "allocated" while setting up pmaps. * * Prepare the list of physical memory available to the vm subsystem. */ arm_physmem_exclude_region(abp->abp_physaddr, pmap_preboot_get_pages(0) - abp->abp_physaddr, EXFLAG_NOALLOC); arm_physmem_init_kernel_globals(); init_param2(physmem); /* Init message buffer. */ msgbufinit(msgbufp, msgbufsize); dbg_monitor_init(); kdb_init(); return ((void *)STACKALIGN(thread0.td_pcb)); } #endif /* __ARM_ARCH < 6 */ #endif /* FDT */ uint32_t (*arm_cpu_fill_vdso_timehands)(struct vdso_timehands *, struct timecounter *); uint32_t cpu_fill_vdso_timehands(struct vdso_timehands *vdso_th, struct timecounter *tc) { return (arm_cpu_fill_vdso_timehands != NULL ? arm_cpu_fill_vdso_timehands(vdso_th, tc) : 0); } Index: user/ngie/release-pkg-fix-tests/sys/arm/arm/mpcore_timer.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/arm/mpcore_timer.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/arm/mpcore_timer.c (revision 299091) @@ -1,546 +1,546 @@ /*- * Copyright (c) 2011 The FreeBSD Foundation * All rights reserved. * * Developed by Ben Gray * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /** * The ARM Cortex-A9 core can support a global timer plus a private and * watchdog timer per core. This driver reserves memory and interrupt * resources for accessing both timer register sets, these resources are * stored globally and used to setup the timecount and eventtimer. * * The timecount timer uses the global 64-bit counter, whereas the * per-CPU eventtimer uses the private 32-bit counters. * * * REF: ARM Cortex-A9 MPCore, Technical Reference Manual (rev. r2p2) */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* Private (per-CPU) timer register map */ #define PRV_TIMER_LOAD 0x0000 #define PRV_TIMER_COUNT 0x0004 #define PRV_TIMER_CTRL 0x0008 #define PRV_TIMER_INTR 0x000C #define PRV_TIMER_CTR_PRESCALER_SHIFT 8 #define PRV_TIMER_CTRL_IRQ_ENABLE (1UL << 2) #define PRV_TIMER_CTRL_AUTO_RELOAD (1UL << 1) #define PRV_TIMER_CTRL_TIMER_ENABLE (1UL << 0) #define PRV_TIMER_INTR_EVENT (1UL << 0) /* Global timer register map */ #define GBL_TIMER_COUNT_LOW 0x0000 #define GBL_TIMER_COUNT_HIGH 0x0004 #define GBL_TIMER_CTRL 0x0008 #define GBL_TIMER_INTR 0x000C #define GBL_TIMER_CTR_PRESCALER_SHIFT 8 #define GBL_TIMER_CTRL_AUTO_INC (1UL << 3) #define GBL_TIMER_CTRL_IRQ_ENABLE (1UL << 2) #define GBL_TIMER_CTRL_COMP_ENABLE (1UL << 1) #define GBL_TIMER_CTRL_TIMER_ENABLE (1UL << 0) #define GBL_TIMER_INTR_EVENT (1UL << 0) struct arm_tmr_softc { device_t dev; int irqrid; int memrid; struct resource * gbl_mem; struct resource * prv_mem; struct resource * prv_irq; uint64_t clkfreq; struct eventtimer et; }; static struct eventtimer *arm_tmr_et; static struct timecounter *arm_tmr_tc; static uint64_t arm_tmr_freq; static boolean_t arm_tmr_freq_varies; #define tmr_prv_read_4(sc, reg) bus_read_4((sc)->prv_mem, reg) #define tmr_prv_write_4(sc, reg, val) bus_write_4((sc)->prv_mem, reg, val) #define tmr_gbl_read_4(sc, reg) bus_read_4((sc)->gbl_mem, reg) #define tmr_gbl_write_4(sc, reg, val) bus_write_4((sc)->gbl_mem, reg, val) static timecounter_get_t arm_tmr_get_timecount; static struct timecounter arm_tmr_timecount = { .tc_name = "MPCore", .tc_get_timecount = arm_tmr_get_timecount, .tc_poll_pps = NULL, .tc_counter_mask = ~0u, .tc_frequency = 0, .tc_quality = 800, }; #define TMR_GBL 0x01 #define TMR_PRV 0x02 #define TMR_BOTH (TMR_GBL | TMR_PRV) #define TMR_NONE 0 static struct ofw_compat_data compat_data[] = { {"arm,mpcore-timers", TMR_BOTH}, /* Non-standard, FreeBSD. */ {"arm,cortex-a9-global-timer", TMR_GBL}, {"arm,cortex-a5-global-timer", TMR_GBL}, {"arm,cortex-a9-twd-timer", TMR_PRV}, {"arm,cortex-a5-twd-timer", TMR_PRV}, {"arm,arm11mp-twd-timer", TMR_PRV}, {NULL, TMR_NONE} }; /** * arm_tmr_get_timecount - reads the timecount (global) timer * @tc: pointer to arm_tmr_timecount struct * * We only read the lower 32-bits, the timecount stuff only uses 32-bits * so (for now?) ignore the upper 32-bits. * * RETURNS * The lower 32-bits of the counter. */ static unsigned arm_tmr_get_timecount(struct timecounter *tc) { struct arm_tmr_softc *sc; sc = tc->tc_priv; return (tmr_gbl_read_4(sc, GBL_TIMER_COUNT_LOW)); } /** * arm_tmr_start - starts the eventtimer (private) timer * @et: pointer to eventtimer struct * @first: the number of seconds and fractional sections to trigger in * @period: the period (in seconds and fractional sections) to set * * If the eventtimer is required to be in oneshot mode, period will be * NULL and first will point to the time to trigger. If in periodic mode * period will contain the time period and first may optionally contain * the time for the first period. * * RETURNS * Always returns 0 */ static int arm_tmr_start(struct eventtimer *et, sbintime_t first, sbintime_t period) { struct arm_tmr_softc *sc; uint32_t load, count; uint32_t ctrl; sc = et->et_priv; tmr_prv_write_4(sc, PRV_TIMER_CTRL, 0); tmr_prv_write_4(sc, PRV_TIMER_INTR, PRV_TIMER_INTR_EVENT); ctrl = PRV_TIMER_CTRL_IRQ_ENABLE | PRV_TIMER_CTRL_TIMER_ENABLE; if (period != 0) { load = ((uint32_t)et->et_frequency * period) >> 32; ctrl |= PRV_TIMER_CTRL_AUTO_RELOAD; } else load = 0; if (first != 0) count = (uint32_t)((et->et_frequency * first) >> 32); else count = load; tmr_prv_write_4(sc, PRV_TIMER_LOAD, load); tmr_prv_write_4(sc, PRV_TIMER_COUNT, count); tmr_prv_write_4(sc, PRV_TIMER_CTRL, ctrl); return (0); } /** * arm_tmr_stop - stops the eventtimer (private) timer * @et: pointer to eventtimer struct * * Simply stops the private timer by clearing all bits in the ctrl register. * * RETURNS * Always returns 0 */ static int arm_tmr_stop(struct eventtimer *et) { struct arm_tmr_softc *sc; sc = et->et_priv; tmr_prv_write_4(sc, PRV_TIMER_CTRL, 0); tmr_prv_write_4(sc, PRV_TIMER_INTR, PRV_TIMER_INTR_EVENT); return (0); } /** * arm_tmr_intr - ISR for the eventtimer (private) timer * @arg: pointer to arm_tmr_softc struct * * Clears the event register and then calls the eventtimer callback. * * RETURNS * Always returns FILTER_HANDLED */ static int arm_tmr_intr(void *arg) { struct arm_tmr_softc *sc; sc = arg; tmr_prv_write_4(sc, PRV_TIMER_INTR, PRV_TIMER_INTR_EVENT); if (sc->et.et_active) sc->et.et_event_cb(&sc->et, sc->et.et_arg); return (FILTER_HANDLED); } /** * arm_tmr_probe - timer probe routine * @dev: new device * * The probe function returns success when probed with the fdt compatible * string set to "arm,mpcore-timers". * * RETURNS * BUS_PROBE_DEFAULT if the fdt device is compatible, otherwise ENXIO. */ static int arm_tmr_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == TMR_NONE) return (ENXIO); device_set_desc(dev, "ARM MPCore Timers"); return (BUS_PROBE_DEFAULT); } static int attach_tc(struct arm_tmr_softc *sc) { int rid; if (arm_tmr_tc != NULL) return (EBUSY); rid = sc->memrid; sc->gbl_mem = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->gbl_mem == NULL) { device_printf(sc->dev, "could not allocate gbl mem resources\n"); return (ENXIO); } tmr_gbl_write_4(sc, GBL_TIMER_CTRL, 0x00000000); arm_tmr_timecount.tc_frequency = sc->clkfreq; arm_tmr_timecount.tc_priv = sc; tc_init(&arm_tmr_timecount); arm_tmr_tc = &arm_tmr_timecount; tmr_gbl_write_4(sc, GBL_TIMER_CTRL, GBL_TIMER_CTRL_TIMER_ENABLE); return (0); } static int attach_et(struct arm_tmr_softc *sc) { void *ihl; int irid, mrid; if (arm_tmr_et != NULL) return (EBUSY); mrid = sc->memrid; sc->prv_mem = bus_alloc_resource_any(sc->dev, SYS_RES_MEMORY, &mrid, RF_ACTIVE); if (sc->prv_mem == NULL) { device_printf(sc->dev, "could not allocate prv mem resources\n"); return (ENXIO); } tmr_prv_write_4(sc, PRV_TIMER_CTRL, 0x00000000); irid = sc->irqrid; sc->prv_irq = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &irid, RF_ACTIVE); if (sc->prv_irq == NULL) { bus_release_resource(sc->dev, SYS_RES_MEMORY, mrid, sc->prv_mem); device_printf(sc->dev, "could not allocate prv irq resources\n"); return (ENXIO); } if (bus_setup_intr(sc->dev, sc->prv_irq, INTR_TYPE_CLK, arm_tmr_intr, NULL, sc, &ihl) != 0) { bus_release_resource(sc->dev, SYS_RES_MEMORY, mrid, sc->prv_mem); bus_release_resource(sc->dev, SYS_RES_IRQ, irid, sc->prv_irq); device_printf(sc->dev, "unable to setup the et irq handler.\n"); return (ENXIO); } /* * Setup and register the eventtimer. Most event timers set their min * and max period values to some value calculated from the clock * frequency. We might not know yet what our runtime clock frequency * will be, so we just use some safe values. A max of 2 seconds ensures * that even if our base clock frequency is 2GHz (meaning a 4GHz CPU), * we won't overflow our 32-bit timer count register. A min of 20 * nanoseconds is pretty much completely arbitrary. */ sc->et.et_name = "MPCore"; sc->et.et_flags = ET_FLAGS_PERIODIC | ET_FLAGS_ONESHOT | ET_FLAGS_PERCPU; sc->et.et_quality = 1000; sc->et.et_frequency = sc->clkfreq; sc->et.et_min_period = 20 * SBT_1NS; sc->et.et_max_period = 2 * SBT_1S; sc->et.et_start = arm_tmr_start; sc->et.et_stop = arm_tmr_stop; sc->et.et_priv = sc; et_register(&sc->et); arm_tmr_et = &sc->et; return (0); } /** * arm_tmr_attach - attaches the timer to the simplebus * @dev: new device * * Reserves memory and interrupt resources, stores the softc structure * globally and registers both the timecount and eventtimer objects. * * RETURNS - * Zero on sucess or ENXIO if an error occuried. + * Zero on success or ENXIO if an error occuried. */ static int arm_tmr_attach(device_t dev) { struct arm_tmr_softc *sc; phandle_t node; pcell_t clock; int et_err, tc_err, tmrtype; sc = device_get_softc(dev); sc->dev = dev; if (arm_tmr_freq_varies) { sc->clkfreq = arm_tmr_freq; } else { if (arm_tmr_freq != 0) { sc->clkfreq = arm_tmr_freq; } else { /* Get the base clock frequency */ node = ofw_bus_get_node(dev); if ((OF_getencprop(node, "clock-frequency", &clock, sizeof(clock))) <= 0) { device_printf(dev, "missing clock-frequency " "attribute in FDT\n"); return (ENXIO); } sc->clkfreq = clock; } } tmrtype = ofw_bus_search_compatible(dev, compat_data)->ocd_data; tc_err = ENXIO; et_err = ENXIO; /* * If we're handling the global timer and it is fixed-frequency, set it * up to use as a timecounter. If it's variable frequency it won't work * as a timecounter. We also can't use it for DELAY(), so hopefully the * platform provides its own implementation. If it doesn't, ours will * get used, but since the frequency isn't set, it will only use the * bogus loop counter. */ if (tmrtype & TMR_GBL) { if (!arm_tmr_freq_varies) tc_err = attach_tc(sc); else if (bootverbose) device_printf(sc->dev, "not using variable-frequency device as timecounter"); sc->memrid++; sc->irqrid++; } /* If we are handling the private timer, set it up as an eventtimer. */ if (tmrtype & TMR_PRV) { et_err = attach_et(sc); } /* * If we didn't successfully set up a timecounter or eventtimer then we * didn't actually attach at all, return error. */ if (tc_err != 0 && et_err != 0) { return (ENXIO); } return (0); } static device_method_t arm_tmr_methods[] = { DEVMETHOD(device_probe, arm_tmr_probe), DEVMETHOD(device_attach, arm_tmr_attach), { 0, 0 } }; static driver_t arm_tmr_driver = { "mp_tmr", arm_tmr_methods, sizeof(struct arm_tmr_softc), }; static devclass_t arm_tmr_devclass; EARLY_DRIVER_MODULE(mp_tmr, simplebus, arm_tmr_driver, arm_tmr_devclass, 0, 0, BUS_PASS_TIMER + BUS_PASS_ORDER_MIDDLE); EARLY_DRIVER_MODULE(mp_tmr, ofwbus, arm_tmr_driver, arm_tmr_devclass, 0, 0, BUS_PASS_TIMER + BUS_PASS_ORDER_MIDDLE); /* * Handle a change in clock frequency. The mpcore timer runs at half the CPU * frequency. When the CPU frequency changes due to power-saving or thermal - * managment, the platform-specific code that causes the frequency change calls + * management, the platform-specific code that causes the frequency change calls * this routine to inform the clock driver, and we in turn inform the event * timer system, which actually updates the value in et->frequency for us and * reschedules the current event(s) in a way that's atomic with respect to * start/stop/intr code that may be running on various CPUs at the time of the * call. * * This routine can also be called by a platform's early init code. If the * value passed is ARM_TMR_FREQUENCY_VARIES, that will cause the attach() code * to register as an eventtimer, but not a timecounter. If the value passed in * is any other non-zero value it is used as the fixed frequency for the timer. */ void arm_tmr_change_frequency(uint64_t newfreq) { if (newfreq == ARM_TMR_FREQUENCY_VARIES) { arm_tmr_freq_varies = true; return; } arm_tmr_freq = newfreq; if (arm_tmr_et != NULL) et_change_frequency(arm_tmr_et, newfreq); } /** * DELAY - Delay for at least usec microseconds. * @usec: number of microseconds to delay by * * This function is called all over the kernel and is suppose to provide a * consistent delay. This function may also be called before the console * is setup so no printf's can be called here. * * RETURNS: * nothing */ static void __used /* Must emit function code for the weak ref below. */ arm_tmr_DELAY(int usec) { struct arm_tmr_softc *sc; int32_t counts_per_usec; int32_t counts; uint32_t first, last; /* Check the timers are setup, if not just use a for loop for the meantime */ if (arm_tmr_tc == NULL || arm_tmr_timecount.tc_frequency == 0) { for (; usec > 0; usec--) for (counts = 200; counts > 0; counts--) cpufunc_nullop(); /* Prevent gcc from optimizing * out the loop */ return; } sc = arm_tmr_tc->tc_priv; /* Get the number of times to count */ counts_per_usec = ((arm_tmr_timecount.tc_frequency / 1000000) + 1); /* * Clamp the timeout at a maximum value (about 32 seconds with * a 66MHz clock). *Nobody* should be delay()ing for anywhere * near that length of time and if they are, they should be hung * out to dry. */ if (usec >= (0x80000000U / counts_per_usec)) counts = (0x80000000U / counts_per_usec) - 1; else counts = usec * counts_per_usec; first = tmr_gbl_read_4(sc, GBL_TIMER_COUNT_LOW); while (counts > 0) { last = tmr_gbl_read_4(sc, GBL_TIMER_COUNT_LOW); counts -= (int32_t)(last - first); first = last; } } /* * Supply a DELAY() implementation via weak linkage. A platform may want to use * the mpcore per-cpu eventtimers but provide its own DELAY() routine, * especially when the core frequency can change on the fly. */ __weak_reference(arm_tmr_DELAY, DELAY); Index: user/ngie/release-pkg-fix-tests/sys/arm/arm/pmap-v4.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/arm/pmap-v4.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/arm/pmap-v4.c (revision 299091) @@ -1,4874 +1,4874 @@ /* From: $NetBSD: pmap.c,v 1.148 2004/04/03 04:35:48 bsh Exp $ */ /*- * Copyright 2004 Olivier Houchard. * Copyright 2003 Wasabi Systems, Inc. * All rights reserved. * * Written by Steve C. Woodford for Wasabi Systems, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed for the NetBSD Project by * Wasabi Systems, Inc. * 4. The name of Wasabi Systems, Inc. may not be used to endorse * or promote products derived from this software without specific prior * written permission. * * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /*- * Copyright (c) 2002-2003 Wasabi Systems, Inc. * Copyright (c) 2001 Richard Earnshaw * Copyright (c) 2001-2002 Christopher Gilbert * All rights reserved. * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /*- * Copyright (c) 1999 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation * by Charles M. Hannum. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /*- * Copyright (c) 1994-1998 Mark Brinicombe. * Copyright (c) 1994 Brini. * All rights reserved. * * This code is derived from software written for Brini by Mark Brinicombe * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Mark Brinicombe. * 4. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * * RiscBSD kernel project * * pmap.c * - * Machine dependant vm stuff + * Machine dependent vm stuff * * Created : 20/09/94 */ /* * Special compilation symbols * PMAP_DEBUG - Build in pmap_debug_level code * * Note that pmap_mapdev() and pmap_unmapdev() are implemented in arm/devmap.c */ /* Include header files */ #include "opt_vm.h" #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef PMAP_DEBUG #define PDEBUG(_lev_,_stat_) \ if (pmap_debug_level >= (_lev_)) \ ((_stat_)) #define dprintf printf int pmap_debug_level = 0; #define PMAP_INLINE #else /* PMAP_DEBUG */ #define PDEBUG(_lev_,_stat_) /* Nothing */ #define dprintf(x, arg...) #define PMAP_INLINE __inline #endif /* PMAP_DEBUG */ extern struct pv_addr systempage; extern int last_fault_code; /* * Internal function prototypes */ static void pmap_free_pv_entry (pv_entry_t); static pv_entry_t pmap_get_pv_entry(void); static int pmap_enter_locked(pmap_t, vm_offset_t, vm_page_t, vm_prot_t, u_int); static vm_paddr_t pmap_extract_locked(pmap_t pmap, vm_offset_t va); static void pmap_fix_cache(struct vm_page *, pmap_t, vm_offset_t); static void pmap_alloc_l1(pmap_t); static void pmap_free_l1(pmap_t); static int pmap_clearbit(struct vm_page *, u_int); static struct l2_bucket *pmap_get_l2_bucket(pmap_t, vm_offset_t); static struct l2_bucket *pmap_alloc_l2_bucket(pmap_t, vm_offset_t); static void pmap_free_l2_bucket(pmap_t, struct l2_bucket *, u_int); static vm_offset_t kernel_pt_lookup(vm_paddr_t); static MALLOC_DEFINE(M_VMPMAP, "pmap", "PMAP L1"); vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */ vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */ vm_offset_t pmap_curmaxkvaddr; vm_paddr_t kernel_l1pa; vm_offset_t kernel_vm_end = 0; vm_offset_t vm_max_kernel_address; struct pmap kernel_pmap_store; static pt_entry_t *csrc_pte, *cdst_pte; static vm_offset_t csrcp, cdstp, qmap_addr; static struct mtx cmtx, qmap_mtx; static void pmap_init_l1(struct l1_ttable *, pd_entry_t *); /* * These routines are called when the CPU type is identified to set up * the PTE prototypes, cache modes, etc. * * The variables are always here, just in case LKMs need to reference * them (though, they shouldn't). */ pt_entry_t pte_l1_s_cache_mode; pt_entry_t pte_l1_s_cache_mode_pt; pt_entry_t pte_l1_s_cache_mask; pt_entry_t pte_l2_l_cache_mode; pt_entry_t pte_l2_l_cache_mode_pt; pt_entry_t pte_l2_l_cache_mask; pt_entry_t pte_l2_s_cache_mode; pt_entry_t pte_l2_s_cache_mode_pt; pt_entry_t pte_l2_s_cache_mask; pt_entry_t pte_l2_s_prot_u; pt_entry_t pte_l2_s_prot_w; pt_entry_t pte_l2_s_prot_mask; pt_entry_t pte_l1_s_proto; pt_entry_t pte_l1_c_proto; pt_entry_t pte_l2_s_proto; void (*pmap_copy_page_func)(vm_paddr_t, vm_paddr_t); void (*pmap_copy_page_offs_func)(vm_paddr_t a_phys, vm_offset_t a_offs, vm_paddr_t b_phys, vm_offset_t b_offs, int cnt); void (*pmap_zero_page_func)(vm_paddr_t, int, int); struct msgbuf *msgbufp = NULL; /* * Crashdump maps. */ static caddr_t crashdumpmap; extern void bcopy_page(vm_offset_t, vm_offset_t); extern void bzero_page(vm_offset_t); extern vm_offset_t alloc_firstaddr; char *_tmppt; /* * Metadata for L1 translation tables. */ struct l1_ttable { /* Entry on the L1 Table list */ SLIST_ENTRY(l1_ttable) l1_link; /* Entry on the L1 Least Recently Used list */ TAILQ_ENTRY(l1_ttable) l1_lru; /* Track how many domains are allocated from this L1 */ volatile u_int l1_domain_use_count; /* * A free-list of domain numbers for this L1. * We avoid using ffs() and a bitmap to track domains since ffs() * is slow on ARM. */ u_int8_t l1_domain_first; u_int8_t l1_domain_free[PMAP_DOMAINS]; /* Physical address of this L1 page table */ vm_paddr_t l1_physaddr; /* KVA of this L1 page table */ pd_entry_t *l1_kva; }; /* * Convert a virtual address into its L1 table index. That is, the * index used to locate the L2 descriptor table pointer in an L1 table. * This is basically used to index l1->l1_kva[]. * * Each L2 descriptor table represents 1MB of VA space. */ #define L1_IDX(va) (((vm_offset_t)(va)) >> L1_S_SHIFT) /* * L1 Page Tables are tracked using a Least Recently Used list. * - New L1s are allocated from the HEAD. * - Freed L1s are added to the TAIl. * - Recently accessed L1s (where an 'access' is some change to one of * the userland pmaps which owns this L1) are moved to the TAIL. */ static TAILQ_HEAD(, l1_ttable) l1_lru_list; /* * A list of all L1 tables */ static SLIST_HEAD(, l1_ttable) l1_list; static struct mtx l1_lru_lock; /* * The l2_dtable tracks L2_BUCKET_SIZE worth of L1 slots. * * This is normally 16MB worth L2 page descriptors for any given pmap. * Reference counts are maintained for L2 descriptors so they can be * freed when empty. */ struct l2_dtable { /* The number of L2 page descriptors allocated to this l2_dtable */ u_int l2_occupancy; /* List of L2 page descriptors */ struct l2_bucket { pt_entry_t *l2b_kva; /* KVA of L2 Descriptor Table */ vm_paddr_t l2b_phys; /* Physical address of same */ u_short l2b_l1idx; /* This L2 table's L1 index */ u_short l2b_occupancy; /* How many active descriptors */ } l2_bucket[L2_BUCKET_SIZE]; }; /* pmap_kenter_internal flags */ #define KENTER_CACHE 0x1 #define KENTER_USER 0x2 /* * Given an L1 table index, calculate the corresponding l2_dtable index * and bucket index within the l2_dtable. */ #define L2_IDX(l1idx) (((l1idx) >> L2_BUCKET_LOG2) & \ (L2_SIZE - 1)) #define L2_BUCKET(l1idx) ((l1idx) & (L2_BUCKET_SIZE - 1)) /* * Given a virtual address, this macro returns the * virtual address required to drop into the next L2 bucket. */ #define L2_NEXT_BUCKET(va) (((va) & L1_S_FRAME) + L1_S_SIZE) /* * We try to map the page tables write-through, if possible. However, not * all CPUs have a write-through cache mode, so on those we have to sync * the cache when we frob page tables. * * We try to evaluate this at compile time, if possible. However, it's * not always possible to do that, hence this run-time var. */ int pmap_needs_pte_sync; /* * Macro to determine if a mapping might be resident in the * instruction cache and/or TLB */ #define PV_BEEN_EXECD(f) (((f) & (PVF_REF | PVF_EXEC)) == (PVF_REF | PVF_EXEC)) /* * Macro to determine if a mapping might be resident in the * data cache and/or TLB */ #define PV_BEEN_REFD(f) (((f) & PVF_REF) != 0) #ifndef PMAP_SHPGPERPROC #define PMAP_SHPGPERPROC 200 #endif #define pmap_is_current(pm) ((pm) == kernel_pmap || \ curproc->p_vmspace->vm_map.pmap == (pm)) static uma_zone_t pvzone = NULL; uma_zone_t l2zone; static uma_zone_t l2table_zone; static vm_offset_t pmap_kernel_l2dtable_kva; static vm_offset_t pmap_kernel_l2ptp_kva; static vm_paddr_t pmap_kernel_l2ptp_phys; static int pv_entry_count=0, pv_entry_max=0, pv_entry_high_water=0; static struct rwlock pvh_global_lock; void pmap_copy_page_offs_generic(vm_paddr_t a_phys, vm_offset_t a_offs, vm_paddr_t b_phys, vm_offset_t b_offs, int cnt); #if ARM_MMU_XSCALE == 1 void pmap_copy_page_offs_xscale(vm_paddr_t a_phys, vm_offset_t a_offs, vm_paddr_t b_phys, vm_offset_t b_offs, int cnt); #endif /* * This list exists for the benefit of pmap_map_chunk(). It keeps track * of the kernel L2 tables during bootstrap, so that pmap_map_chunk() can * find them as necessary. * * Note that the data on this list MUST remain valid after initarm() returns, * as pmap_bootstrap() uses it to contruct L2 table metadata. */ SLIST_HEAD(, pv_addr) kernel_pt_list = SLIST_HEAD_INITIALIZER(kernel_pt_list); static void pmap_init_l1(struct l1_ttable *l1, pd_entry_t *l1pt) { int i; l1->l1_kva = l1pt; l1->l1_domain_use_count = 0; l1->l1_domain_first = 0; for (i = 0; i < PMAP_DOMAINS; i++) l1->l1_domain_free[i] = i + 1; /* * Copy the kernel's L1 entries to each new L1. */ if (l1pt != kernel_pmap->pm_l1->l1_kva) memcpy(l1pt, kernel_pmap->pm_l1->l1_kva, L1_TABLE_SIZE); if ((l1->l1_physaddr = pmap_extract(kernel_pmap, (vm_offset_t)l1pt)) == 0) panic("pmap_init_l1: can't get PA of L1 at %p", l1pt); SLIST_INSERT_HEAD(&l1_list, l1, l1_link); TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); } static vm_offset_t kernel_pt_lookup(vm_paddr_t pa) { struct pv_addr *pv; SLIST_FOREACH(pv, &kernel_pt_list, pv_list) { if (pv->pv_pa == pa) return (pv->pv_va); } return (0); } #if ARM_MMU_GENERIC != 0 void pmap_pte_init_generic(void) { pte_l1_s_cache_mode = L1_S_B|L1_S_C; pte_l1_s_cache_mask = L1_S_CACHE_MASK_generic; pte_l2_l_cache_mode = L2_B|L2_C; pte_l2_l_cache_mask = L2_L_CACHE_MASK_generic; pte_l2_s_cache_mode = L2_B|L2_C; pte_l2_s_cache_mask = L2_S_CACHE_MASK_generic; /* * If we have a write-through cache, set B and C. If * we have a write-back cache, then we assume setting * only C will make those pages write-through. */ if (cpufuncs.cf_dcache_wb_range == (void *) cpufunc_nullop) { pte_l1_s_cache_mode_pt = L1_S_B|L1_S_C; pte_l2_l_cache_mode_pt = L2_B|L2_C; pte_l2_s_cache_mode_pt = L2_B|L2_C; } else { pte_l1_s_cache_mode_pt = L1_S_C; pte_l2_l_cache_mode_pt = L2_C; pte_l2_s_cache_mode_pt = L2_C; } pte_l2_s_prot_u = L2_S_PROT_U_generic; pte_l2_s_prot_w = L2_S_PROT_W_generic; pte_l2_s_prot_mask = L2_S_PROT_MASK_generic; pte_l1_s_proto = L1_S_PROTO_generic; pte_l1_c_proto = L1_C_PROTO_generic; pte_l2_s_proto = L2_S_PROTO_generic; pmap_copy_page_func = pmap_copy_page_generic; pmap_copy_page_offs_func = pmap_copy_page_offs_generic; pmap_zero_page_func = pmap_zero_page_generic; } #endif /* ARM_MMU_GENERIC != 0 */ #if ARM_MMU_XSCALE == 1 #if (ARM_NMMUS > 1) || defined (CPU_XSCALE_CORE3) static u_int xscale_use_minidata; #endif void pmap_pte_init_xscale(void) { uint32_t auxctl; int write_through = 0; pte_l1_s_cache_mode = L1_S_B|L1_S_C|L1_S_XSCALE_P; pte_l1_s_cache_mask = L1_S_CACHE_MASK_xscale; pte_l2_l_cache_mode = L2_B|L2_C; pte_l2_l_cache_mask = L2_L_CACHE_MASK_xscale; pte_l2_s_cache_mode = L2_B|L2_C; pte_l2_s_cache_mask = L2_S_CACHE_MASK_xscale; pte_l1_s_cache_mode_pt = L1_S_C; pte_l2_l_cache_mode_pt = L2_C; pte_l2_s_cache_mode_pt = L2_C; #ifdef XSCALE_CACHE_READ_WRITE_ALLOCATE /* * The XScale core has an enhanced mode where writes that * miss the cache cause a cache line to be allocated. This * is significantly faster than the traditional, write-through * behavior of this case. */ pte_l1_s_cache_mode |= L1_S_XSCALE_TEX(TEX_XSCALE_X); pte_l2_l_cache_mode |= L2_XSCALE_L_TEX(TEX_XSCALE_X); pte_l2_s_cache_mode |= L2_XSCALE_T_TEX(TEX_XSCALE_X); #endif /* XSCALE_CACHE_READ_WRITE_ALLOCATE */ #ifdef XSCALE_CACHE_WRITE_THROUGH /* * Some versions of the XScale core have various bugs in * their cache units, the work-around for which is to run * the cache in write-through mode. Unfortunately, this * has a major (negative) impact on performance. So, we * go ahead and run fast-and-loose, in the hopes that we * don't line up the planets in a way that will trip the * bugs. * * However, we give you the option to be slow-but-correct. */ write_through = 1; #elif defined(XSCALE_CACHE_WRITE_BACK) /* force write back cache mode */ write_through = 0; #elif defined(CPU_XSCALE_PXA2X0) /* * Intel PXA2[15]0 processors are known to have a bug in * write-back cache on revision 4 and earlier (stepping * A[01] and B[012]). Fixed for C0 and later. */ { uint32_t id, type; id = cpu_ident(); type = id & ~(CPU_ID_XSCALE_COREREV_MASK|CPU_ID_REVISION_MASK); if (type == CPU_ID_PXA250 || type == CPU_ID_PXA210) { if ((id & CPU_ID_REVISION_MASK) < 5) { /* write through for stepping A0-1 and B0-2 */ write_through = 1; } } } #endif /* XSCALE_CACHE_WRITE_THROUGH */ if (write_through) { pte_l1_s_cache_mode = L1_S_C; pte_l2_l_cache_mode = L2_C; pte_l2_s_cache_mode = L2_C; } #if (ARM_NMMUS > 1) xscale_use_minidata = 1; #endif pte_l2_s_prot_u = L2_S_PROT_U_xscale; pte_l2_s_prot_w = L2_S_PROT_W_xscale; pte_l2_s_prot_mask = L2_S_PROT_MASK_xscale; pte_l1_s_proto = L1_S_PROTO_xscale; pte_l1_c_proto = L1_C_PROTO_xscale; pte_l2_s_proto = L2_S_PROTO_xscale; #ifdef CPU_XSCALE_CORE3 pmap_copy_page_func = pmap_copy_page_generic; pmap_copy_page_offs_func = pmap_copy_page_offs_generic; pmap_zero_page_func = pmap_zero_page_generic; xscale_use_minidata = 0; /* Make sure it is L2-cachable */ pte_l1_s_cache_mode |= L1_S_XSCALE_TEX(TEX_XSCALE_T); pte_l1_s_cache_mode_pt = pte_l1_s_cache_mode &~ L1_S_XSCALE_P; pte_l2_l_cache_mode |= L2_XSCALE_L_TEX(TEX_XSCALE_T) ; pte_l2_l_cache_mode_pt = pte_l1_s_cache_mode; pte_l2_s_cache_mode |= L2_XSCALE_T_TEX(TEX_XSCALE_T); pte_l2_s_cache_mode_pt = pte_l2_s_cache_mode; #else pmap_copy_page_func = pmap_copy_page_xscale; pmap_copy_page_offs_func = pmap_copy_page_offs_xscale; pmap_zero_page_func = pmap_zero_page_xscale; #endif /* * Disable ECC protection of page table access, for now. */ __asm __volatile("mrc p15, 0, %0, c1, c0, 1" : "=r" (auxctl)); auxctl &= ~XSCALE_AUXCTL_P; __asm __volatile("mcr p15, 0, %0, c1, c0, 1" : : "r" (auxctl)); } /* * xscale_setup_minidata: * * Set up the mini-data cache clean area. We require the * caller to allocate the right amount of physically and * virtually contiguous space. */ extern vm_offset_t xscale_minidata_clean_addr; extern vm_size_t xscale_minidata_clean_size; /* already initialized */ void xscale_setup_minidata(vm_offset_t l1pt, vm_offset_t va, vm_paddr_t pa) { pd_entry_t *pde = (pd_entry_t *) l1pt; pt_entry_t *pte; vm_size_t size; uint32_t auxctl; xscale_minidata_clean_addr = va; /* Round it to page size. */ size = (xscale_minidata_clean_size + L2_S_OFFSET) & L2_S_FRAME; for (; size != 0; va += L2_S_SIZE, pa += L2_S_SIZE, size -= L2_S_SIZE) { pte = (pt_entry_t *) kernel_pt_lookup( pde[L1_IDX(va)] & L1_C_ADDR_MASK); if (pte == NULL) panic("xscale_setup_minidata: can't find L2 table for " "VA 0x%08x", (u_int32_t) va); pte[l2pte_index(va)] = L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); } /* * Configure the mini-data cache for write-back with * read/write-allocate. * * NOTE: In order to reconfigure the mini-data cache, we must * make sure it contains no valid data! In order to do that, * we must issue a global data cache invalidate command! * * WE ASSUME WE ARE RUNNING UN-CACHED WHEN THIS ROUTINE IS CALLED! * THIS IS VERY IMPORTANT! */ /* Invalidate data and mini-data. */ __asm __volatile("mcr p15, 0, %0, c7, c6, 0" : : "r" (0)); __asm __volatile("mrc p15, 0, %0, c1, c0, 1" : "=r" (auxctl)); auxctl = (auxctl & ~XSCALE_AUXCTL_MD_MASK) | XSCALE_AUXCTL_MD_WB_RWA; __asm __volatile("mcr p15, 0, %0, c1, c0, 1" : : "r" (auxctl)); } #endif /* * Allocate an L1 translation table for the specified pmap. * This is called at pmap creation time. */ static void pmap_alloc_l1(pmap_t pm) { struct l1_ttable *l1; u_int8_t domain; /* * Remove the L1 at the head of the LRU list */ mtx_lock(&l1_lru_lock); l1 = TAILQ_FIRST(&l1_lru_list); TAILQ_REMOVE(&l1_lru_list, l1, l1_lru); /* * Pick the first available domain number, and update * the link to the next number. */ domain = l1->l1_domain_first; l1->l1_domain_first = l1->l1_domain_free[domain]; /* * If there are still free domain numbers in this L1, * put it back on the TAIL of the LRU list. */ if (++l1->l1_domain_use_count < PMAP_DOMAINS) TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); mtx_unlock(&l1_lru_lock); /* * Fix up the relevant bits in the pmap structure */ pm->pm_l1 = l1; pm->pm_domain = domain + 1; } /* * Free an L1 translation table. * This is called at pmap destruction time. */ static void pmap_free_l1(pmap_t pm) { struct l1_ttable *l1 = pm->pm_l1; mtx_lock(&l1_lru_lock); /* * If this L1 is currently on the LRU list, remove it. */ if (l1->l1_domain_use_count < PMAP_DOMAINS) TAILQ_REMOVE(&l1_lru_list, l1, l1_lru); /* * Free up the domain number which was allocated to the pmap */ l1->l1_domain_free[pm->pm_domain - 1] = l1->l1_domain_first; l1->l1_domain_first = pm->pm_domain - 1; l1->l1_domain_use_count--; /* * The L1 now must have at least 1 free domain, so add * it back to the LRU list. If the use count is zero, * put it at the head of the list, otherwise it goes * to the tail. */ if (l1->l1_domain_use_count == 0) { TAILQ_INSERT_HEAD(&l1_lru_list, l1, l1_lru); } else TAILQ_INSERT_TAIL(&l1_lru_list, l1, l1_lru); mtx_unlock(&l1_lru_lock); } /* * Returns a pointer to the L2 bucket associated with the specified pmap * and VA, or NULL if no L2 bucket exists for the address. */ static PMAP_INLINE struct l2_bucket * pmap_get_l2_bucket(pmap_t pm, vm_offset_t va) { struct l2_dtable *l2; struct l2_bucket *l2b; u_short l1idx; l1idx = L1_IDX(va); if ((l2 = pm->pm_l2[L2_IDX(l1idx)]) == NULL || (l2b = &l2->l2_bucket[L2_BUCKET(l1idx)])->l2b_kva == NULL) return (NULL); return (l2b); } /* * Returns a pointer to the L2 bucket associated with the specified pmap * and VA. * * If no L2 bucket exists, perform the necessary allocations to put an L2 * bucket/page table in place. * * Note that if a new L2 bucket/page was allocated, the caller *must* * increment the bucket occupancy counter appropriately *before* * releasing the pmap's lock to ensure no other thread or cpu deallocates * the bucket/page in the meantime. */ static struct l2_bucket * pmap_alloc_l2_bucket(pmap_t pm, vm_offset_t va) { struct l2_dtable *l2; struct l2_bucket *l2b; u_short l1idx; l1idx = L1_IDX(va); PMAP_ASSERT_LOCKED(pm); rw_assert(&pvh_global_lock, RA_WLOCKED); if ((l2 = pm->pm_l2[L2_IDX(l1idx)]) == NULL) { /* * No mapping at this address, as there is * no entry in the L1 table. * Need to allocate a new l2_dtable. */ PMAP_UNLOCK(pm); rw_wunlock(&pvh_global_lock); if ((l2 = uma_zalloc(l2table_zone, M_NOWAIT)) == NULL) { rw_wlock(&pvh_global_lock); PMAP_LOCK(pm); return (NULL); } rw_wlock(&pvh_global_lock); PMAP_LOCK(pm); if (pm->pm_l2[L2_IDX(l1idx)] != NULL) { /* * Someone already allocated the l2_dtable while * we were doing the same. */ uma_zfree(l2table_zone, l2); l2 = pm->pm_l2[L2_IDX(l1idx)]; } else { bzero(l2, sizeof(*l2)); /* * Link it into the parent pmap */ pm->pm_l2[L2_IDX(l1idx)] = l2; } } l2b = &l2->l2_bucket[L2_BUCKET(l1idx)]; /* * Fetch pointer to the L2 page table associated with the address. */ if (l2b->l2b_kva == NULL) { pt_entry_t *ptep; /* * No L2 page table has been allocated. Chances are, this * is because we just allocated the l2_dtable, above. */ l2->l2_occupancy++; PMAP_UNLOCK(pm); rw_wunlock(&pvh_global_lock); ptep = uma_zalloc(l2zone, M_NOWAIT); rw_wlock(&pvh_global_lock); PMAP_LOCK(pm); if (l2b->l2b_kva != NULL) { /* We lost the race. */ l2->l2_occupancy--; uma_zfree(l2zone, ptep); return (l2b); } l2b->l2b_phys = vtophys(ptep); if (ptep == NULL) { /* * Oops, no more L2 page tables available at this * time. We may need to deallocate the l2_dtable * if we allocated a new one above. */ l2->l2_occupancy--; if (l2->l2_occupancy == 0) { pm->pm_l2[L2_IDX(l1idx)] = NULL; uma_zfree(l2table_zone, l2); } return (NULL); } l2b->l2b_kva = ptep; l2b->l2b_l1idx = l1idx; } return (l2b); } static PMAP_INLINE void #ifndef PMAP_INCLUDE_PTE_SYNC pmap_free_l2_ptp(pt_entry_t *l2) #else pmap_free_l2_ptp(boolean_t need_sync, pt_entry_t *l2) #endif { #ifdef PMAP_INCLUDE_PTE_SYNC /* * Note: With a write-back cache, we may need to sync this * L2 table before re-using it. * This is because it may have belonged to a non-current * pmap, in which case the cache syncs would have been * skipped when the pages were being unmapped. If the * L2 table were then to be immediately re-allocated to * the *current* pmap, it may well contain stale mappings * which have not yet been cleared by a cache write-back * and so would still be visible to the mmu. */ if (need_sync) PTE_SYNC_RANGE(l2, L2_TABLE_SIZE_REAL / sizeof(pt_entry_t)); #endif uma_zfree(l2zone, l2); } /* * One or more mappings in the specified L2 descriptor table have just been * invalidated. * * Garbage collect the metadata and descriptor table itself if necessary. * * The pmap lock must be acquired when this is called (not necessary * for the kernel pmap). */ static void pmap_free_l2_bucket(pmap_t pm, struct l2_bucket *l2b, u_int count) { struct l2_dtable *l2; pd_entry_t *pl1pd, l1pd; pt_entry_t *ptep; u_short l1idx; /* * Update the bucket's reference count according to how many * PTEs the caller has just invalidated. */ l2b->l2b_occupancy -= count; /* * Note: * * Level 2 page tables allocated to the kernel pmap are never freed * as that would require checking all Level 1 page tables and * removing any references to the Level 2 page table. See also the * comment elsewhere about never freeing bootstrap L2 descriptors. * * We make do with just invalidating the mapping in the L2 table. * * This isn't really a big deal in practice and, in fact, leads * to a performance win over time as we don't need to continually * alloc/free. */ if (l2b->l2b_occupancy > 0 || pm == kernel_pmap) return; /* * There are no more valid mappings in this level 2 page table. * Go ahead and NULL-out the pointer in the bucket, then * free the page table. */ l1idx = l2b->l2b_l1idx; ptep = l2b->l2b_kva; l2b->l2b_kva = NULL; pl1pd = &pm->pm_l1->l1_kva[l1idx]; /* * If the L1 slot matches the pmap's domain * number, then invalidate it. */ l1pd = *pl1pd & (L1_TYPE_MASK | L1_C_DOM_MASK); if (l1pd == (L1_C_DOM(pm->pm_domain) | L1_TYPE_C)) { *pl1pd = 0; PTE_SYNC(pl1pd); } /* * Release the L2 descriptor table back to the pool cache. */ #ifndef PMAP_INCLUDE_PTE_SYNC pmap_free_l2_ptp(ptep); #else pmap_free_l2_ptp(!pmap_is_current(pm), ptep); #endif /* * Update the reference count in the associated l2_dtable */ l2 = pm->pm_l2[L2_IDX(l1idx)]; if (--l2->l2_occupancy > 0) return; /* * There are no more valid mappings in any of the Level 1 * slots managed by this l2_dtable. Go ahead and NULL-out * the pointer in the parent pmap and free the l2_dtable. */ pm->pm_l2[L2_IDX(l1idx)] = NULL; uma_zfree(l2table_zone, l2); } /* * Pool cache constructors for L2 descriptor tables, metadata and pmap * structures. */ static int pmap_l2ptp_ctor(void *mem, int size, void *arg, int flags) { #ifndef PMAP_INCLUDE_PTE_SYNC struct l2_bucket *l2b; pt_entry_t *ptep, pte; vm_offset_t va = (vm_offset_t)mem & ~PAGE_MASK; /* * The mappings for these page tables were initially made using * pmap_kenter() by the pool subsystem. Therefore, the cache- * mode will not be right for page table mappings. To avoid * polluting the pmap_kenter() code with a special case for * page tables, we simply fix up the cache-mode here if it's not * correct. */ l2b = pmap_get_l2_bucket(kernel_pmap, va); ptep = &l2b->l2b_kva[l2pte_index(va)]; pte = *ptep; if ((pte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) { /* * Page tables must have the cache-mode set to * Write-Thru. */ *ptep = (pte & ~L2_S_CACHE_MASK) | pte_l2_s_cache_mode_pt; PTE_SYNC(ptep); cpu_tlb_flushD_SE(va); cpu_cpwait(); } #endif memset(mem, 0, L2_TABLE_SIZE_REAL); PTE_SYNC_RANGE(mem, L2_TABLE_SIZE_REAL / sizeof(pt_entry_t)); return (0); } /* * A bunch of routines to conditionally flush the caches/TLB depending * on whether the specified pmap actually needs to be flushed at any * given time. */ static PMAP_INLINE void pmap_tlb_flushID_SE(pmap_t pm, vm_offset_t va) { if (pmap_is_current(pm)) cpu_tlb_flushID_SE(va); } static PMAP_INLINE void pmap_tlb_flushD_SE(pmap_t pm, vm_offset_t va) { if (pmap_is_current(pm)) cpu_tlb_flushD_SE(va); } static PMAP_INLINE void pmap_tlb_flushID(pmap_t pm) { if (pmap_is_current(pm)) cpu_tlb_flushID(); } static PMAP_INLINE void pmap_tlb_flushD(pmap_t pm) { if (pmap_is_current(pm)) cpu_tlb_flushD(); } static int pmap_has_valid_mapping(pmap_t pm, vm_offset_t va) { pd_entry_t *pde; pt_entry_t *ptep; if (pmap_get_pde_pte(pm, va, &pde, &ptep) && ptep && ((*ptep & L2_TYPE_MASK) != L2_TYPE_INV)) return (1); return (0); } static PMAP_INLINE void pmap_idcache_wbinv_range(pmap_t pm, vm_offset_t va, vm_size_t len) { vm_size_t rest; CTR4(KTR_PMAP, "pmap_dcache_wbinv_range: pmap %p is_kernel %d va 0x%08x" " len 0x%x ", pm, pm == kernel_pmap, va, len); if (pmap_is_current(pm) || pm == kernel_pmap) { rest = MIN(PAGE_SIZE - (va & PAGE_MASK), len); while (len > 0) { if (pmap_has_valid_mapping(pm, va)) { cpu_idcache_wbinv_range(va, rest); cpu_l2cache_wbinv_range(va, rest); } len -= rest; va += rest; rest = MIN(PAGE_SIZE, len); } } } static PMAP_INLINE void pmap_dcache_wb_range(pmap_t pm, vm_offset_t va, vm_size_t len, boolean_t do_inv, boolean_t rd_only) { vm_size_t rest; CTR4(KTR_PMAP, "pmap_dcache_wb_range: pmap %p is_kernel %d va 0x%08x " "len 0x%x ", pm, pm == kernel_pmap, va, len); CTR2(KTR_PMAP, " do_inv %d rd_only %d", do_inv, rd_only); if (pmap_is_current(pm)) { rest = MIN(PAGE_SIZE - (va & PAGE_MASK), len); while (len > 0) { if (pmap_has_valid_mapping(pm, va)) { if (do_inv && rd_only) { cpu_dcache_inv_range(va, rest); cpu_l2cache_inv_range(va, rest); } else if (do_inv) { cpu_dcache_wbinv_range(va, rest); cpu_l2cache_wbinv_range(va, rest); } else if (!rd_only) { cpu_dcache_wb_range(va, rest); cpu_l2cache_wb_range(va, rest); } } len -= rest; va += rest; rest = MIN(PAGE_SIZE, len); } } } static PMAP_INLINE void pmap_idcache_wbinv_all(pmap_t pm) { if (pmap_is_current(pm)) { cpu_idcache_wbinv_all(); cpu_l2cache_wbinv_all(); } } #ifdef notyet static PMAP_INLINE void pmap_dcache_wbinv_all(pmap_t pm) { if (pmap_is_current(pm)) { cpu_dcache_wbinv_all(); cpu_l2cache_wbinv_all(); } } #endif /* * PTE_SYNC_CURRENT: * * Make sure the pte is written out to RAM. * We need to do this for one of two cases: * - We're dealing with the kernel pmap * - There is no pmap active in the cache/tlb. * - The specified pmap is 'active' in the cache/tlb. */ #ifdef PMAP_INCLUDE_PTE_SYNC #define PTE_SYNC_CURRENT(pm, ptep) \ do { \ if (PMAP_NEEDS_PTE_SYNC && \ pmap_is_current(pm)) \ PTE_SYNC(ptep); \ } while (/*CONSTCOND*/0) #else #define PTE_SYNC_CURRENT(pm, ptep) /* nothing */ #endif /* * cacheable == -1 means we must make the entry uncacheable, 1 means * cacheable; */ static __inline void pmap_set_cache_entry(pv_entry_t pv, pmap_t pm, vm_offset_t va, int cacheable) { struct l2_bucket *l2b; pt_entry_t *ptep, pte; l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va); ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; if (cacheable == 1) { pte = (*ptep & ~L2_S_CACHE_MASK) | pte_l2_s_cache_mode; if (l2pte_valid(pte)) { if (PV_BEEN_EXECD(pv->pv_flags)) { pmap_tlb_flushID_SE(pv->pv_pmap, pv->pv_va); } else if (PV_BEEN_REFD(pv->pv_flags)) { pmap_tlb_flushD_SE(pv->pv_pmap, pv->pv_va); } } } else { pte = *ptep &~ L2_S_CACHE_MASK; if ((va != pv->pv_va || pm != pv->pv_pmap) && l2pte_valid(pte)) { if (PV_BEEN_EXECD(pv->pv_flags)) { pmap_idcache_wbinv_range(pv->pv_pmap, pv->pv_va, PAGE_SIZE); pmap_tlb_flushID_SE(pv->pv_pmap, pv->pv_va); } else if (PV_BEEN_REFD(pv->pv_flags)) { pmap_dcache_wb_range(pv->pv_pmap, pv->pv_va, PAGE_SIZE, TRUE, (pv->pv_flags & PVF_WRITE) == 0); pmap_tlb_flushD_SE(pv->pv_pmap, pv->pv_va); } } } *ptep = pte; PTE_SYNC_CURRENT(pv->pv_pmap, ptep); } static void pmap_fix_cache(struct vm_page *pg, pmap_t pm, vm_offset_t va) { int pmwc = 0; int writable = 0, kwritable = 0, uwritable = 0; int entries = 0, kentries = 0, uentries = 0; struct pv_entry *pv; rw_assert(&pvh_global_lock, RA_WLOCKED); /* the cache gets written back/invalidated on context switch. * therefore, if a user page shares an entry in the same page or * with the kernel map and at least one is writable, then the * cache entry must be set write-through. */ TAILQ_FOREACH(pv, &pg->md.pv_list, pv_list) { /* generate a count of the pv_entry uses */ if (pv->pv_flags & PVF_WRITE) { if (pv->pv_pmap == kernel_pmap) kwritable++; else if (pv->pv_pmap == pm) uwritable++; writable++; } if (pv->pv_pmap == kernel_pmap) kentries++; else { if (pv->pv_pmap == pm) uentries++; entries++; } } /* * check if the user duplicate mapping has * been removed. */ if ((pm != kernel_pmap) && (((uentries > 1) && uwritable) || (uwritable > 1))) pmwc = 1; TAILQ_FOREACH(pv, &pg->md.pv_list, pv_list) { /* check for user uncachable conditions - order is important */ if (pm != kernel_pmap && (pv->pv_pmap == pm || pv->pv_pmap == kernel_pmap)) { if ((uentries > 1 && uwritable) || uwritable > 1) { /* user duplicate mapping */ if (pv->pv_pmap != kernel_pmap) pv->pv_flags |= PVF_MWC; if (!(pv->pv_flags & PVF_NC)) { pv->pv_flags |= PVF_NC; pmap_set_cache_entry(pv, pm, va, -1); } continue; } else /* no longer a duplicate user */ pv->pv_flags &= ~PVF_MWC; } /* * check for kernel uncachable conditions * kernel writable or kernel readable with writable user entry */ if ((kwritable && (entries || kentries > 1)) || (kwritable > 1) || ((kwritable != writable) && kentries && (pv->pv_pmap == kernel_pmap || (pv->pv_flags & PVF_WRITE) || (pv->pv_flags & PVF_MWC)))) { if (!(pv->pv_flags & PVF_NC)) { pv->pv_flags |= PVF_NC; pmap_set_cache_entry(pv, pm, va, -1); } continue; } /* kernel and user are cachable */ if ((pm == kernel_pmap) && !(pv->pv_flags & PVF_MWC) && (pv->pv_flags & PVF_NC)) { pv->pv_flags &= ~PVF_NC; if (pg->md.pv_memattr != VM_MEMATTR_UNCACHEABLE) pmap_set_cache_entry(pv, pm, va, 1); continue; } /* user is no longer sharable and writable */ if (pm != kernel_pmap && (pv->pv_pmap == pm || pv->pv_pmap == kernel_pmap) && !pmwc && (pv->pv_flags & PVF_NC)) { pv->pv_flags &= ~(PVF_NC | PVF_MWC); if (pg->md.pv_memattr != VM_MEMATTR_UNCACHEABLE) pmap_set_cache_entry(pv, pm, va, 1); } } if ((kwritable == 0) && (writable == 0)) { pg->md.pvh_attrs &= ~PVF_MOD; vm_page_aflag_clear(pg, PGA_WRITEABLE); return; } } /* * Modify pte bits for all ptes corresponding to the given physical address. * We use `maskbits' rather than `clearbits' because we're always passing * constants and the latter would require an extra inversion at run-time. */ static int pmap_clearbit(struct vm_page *pg, u_int maskbits) { struct l2_bucket *l2b; struct pv_entry *pv; pt_entry_t *ptep, npte, opte; pmap_t pm; vm_offset_t va; u_int oflags; int count = 0; rw_wlock(&pvh_global_lock); if (maskbits & PVF_WRITE) maskbits |= PVF_MOD; /* * Clear saved attributes (modify, reference) */ pg->md.pvh_attrs &= ~(maskbits & (PVF_MOD | PVF_REF)); if (TAILQ_EMPTY(&pg->md.pv_list)) { rw_wunlock(&pvh_global_lock); return (0); } /* * Loop over all current mappings setting/clearing as appropos */ TAILQ_FOREACH(pv, &pg->md.pv_list, pv_list) { va = pv->pv_va; pm = pv->pv_pmap; oflags = pv->pv_flags; if (!(oflags & maskbits)) { if ((maskbits & PVF_WRITE) && (pv->pv_flags & PVF_NC)) { if (pg->md.pv_memattr != VM_MEMATTR_UNCACHEABLE) { PMAP_LOCK(pm); l2b = pmap_get_l2_bucket(pm, va); ptep = &l2b->l2b_kva[l2pte_index(va)]; *ptep |= pte_l2_s_cache_mode; PTE_SYNC(ptep); PMAP_UNLOCK(pm); } pv->pv_flags &= ~(PVF_NC | PVF_MWC); } continue; } pv->pv_flags &= ~maskbits; PMAP_LOCK(pm); l2b = pmap_get_l2_bucket(pm, va); ptep = &l2b->l2b_kva[l2pte_index(va)]; npte = opte = *ptep; if (maskbits & (PVF_WRITE|PVF_MOD)) { if ((pv->pv_flags & PVF_NC)) { /* * Entry is not cacheable: * * Don't turn caching on again if this is a * modified emulation. This would be - * inconsitent with the settings created by + * inconsistent with the settings created by * pmap_fix_cache(). Otherwise, it's safe - * to re-enable cacheing. + * to re-enable caching. * * There's no need to call pmap_fix_cache() * here: all pages are losing their write * permission. */ if (maskbits & PVF_WRITE) { if (pg->md.pv_memattr != VM_MEMATTR_UNCACHEABLE) npte |= pte_l2_s_cache_mode; pv->pv_flags &= ~(PVF_NC | PVF_MWC); } } else if (opte & L2_S_PROT_W) { vm_page_dirty(pg); /* * Entry is writable/cacheable: check if pmap * is current if it is flush it, otherwise it * won't be in the cache */ if (PV_BEEN_EXECD(oflags)) pmap_idcache_wbinv_range(pm, pv->pv_va, PAGE_SIZE); else if (PV_BEEN_REFD(oflags)) pmap_dcache_wb_range(pm, pv->pv_va, PAGE_SIZE, (maskbits & PVF_REF) ? TRUE : FALSE, FALSE); } /* make the pte read only */ npte &= ~L2_S_PROT_W; } if (maskbits & PVF_REF) { if ((pv->pv_flags & PVF_NC) == 0 && (maskbits & (PVF_WRITE|PVF_MOD)) == 0) { /* * Check npte here; we may have already * done the wbinv above, and the validity * of the PTE is the same for opte and * npte. */ if (npte & L2_S_PROT_W) { if (PV_BEEN_EXECD(oflags)) pmap_idcache_wbinv_range(pm, pv->pv_va, PAGE_SIZE); else if (PV_BEEN_REFD(oflags)) pmap_dcache_wb_range(pm, pv->pv_va, PAGE_SIZE, TRUE, FALSE); } else if ((npte & L2_TYPE_MASK) != L2_TYPE_INV) { /* XXXJRT need idcache_inv_range */ if (PV_BEEN_EXECD(oflags)) pmap_idcache_wbinv_range(pm, pv->pv_va, PAGE_SIZE); else if (PV_BEEN_REFD(oflags)) pmap_dcache_wb_range(pm, pv->pv_va, PAGE_SIZE, TRUE, TRUE); } } /* * Make the PTE invalid so that we will take a * page fault the next time the mapping is * referenced. */ npte &= ~L2_TYPE_MASK; npte |= L2_TYPE_INV; } if (npte != opte) { count++; *ptep = npte; PTE_SYNC(ptep); /* Flush the TLB entry if a current pmap. */ if (PV_BEEN_EXECD(oflags)) pmap_tlb_flushID_SE(pm, pv->pv_va); else if (PV_BEEN_REFD(oflags)) pmap_tlb_flushD_SE(pm, pv->pv_va); } PMAP_UNLOCK(pm); } if (maskbits & PVF_WRITE) vm_page_aflag_clear(pg, PGA_WRITEABLE); rw_wunlock(&pvh_global_lock); return (count); } /* * main pv_entry manipulation functions: * pmap_enter_pv: enter a mapping onto a vm_page list * pmap_remove_pv: remove a mappiing from a vm_page list * * NOTE: pmap_enter_pv expects to lock the pvh itself * pmap_remove_pv expects the caller to lock the pvh before calling */ /* * pmap_enter_pv: enter a mapping onto a vm_page's PV list * * => caller should hold the proper lock on pvh_global_lock * => caller should have pmap locked * => we will (someday) gain the lock on the vm_page's PV list * => caller should adjust ptp's wire_count before calling * => caller should not adjust pmap's wire_count */ static void pmap_enter_pv(struct vm_page *pg, struct pv_entry *pve, pmap_t pm, vm_offset_t va, u_int flags) { rw_assert(&pvh_global_lock, RA_WLOCKED); PMAP_ASSERT_LOCKED(pm); if (pg->md.pv_kva != 0) { pve->pv_pmap = kernel_pmap; pve->pv_va = pg->md.pv_kva; pve->pv_flags = PVF_WRITE | PVF_UNMAN; if (pm != kernel_pmap) PMAP_LOCK(kernel_pmap); TAILQ_INSERT_HEAD(&pg->md.pv_list, pve, pv_list); TAILQ_INSERT_HEAD(&kernel_pmap->pm_pvlist, pve, pv_plist); if (pm != kernel_pmap) PMAP_UNLOCK(kernel_pmap); pg->md.pv_kva = 0; if ((pve = pmap_get_pv_entry()) == NULL) panic("pmap_kenter_pv: no pv entries"); } pve->pv_pmap = pm; pve->pv_va = va; pve->pv_flags = flags; TAILQ_INSERT_HEAD(&pg->md.pv_list, pve, pv_list); TAILQ_INSERT_HEAD(&pm->pm_pvlist, pve, pv_plist); pg->md.pvh_attrs |= flags & (PVF_REF | PVF_MOD); if (pve->pv_flags & PVF_WIRED) ++pm->pm_stats.wired_count; vm_page_aflag_set(pg, PGA_REFERENCED); } /* * * pmap_find_pv: Find a pv entry * * => caller should hold lock on vm_page */ static PMAP_INLINE struct pv_entry * pmap_find_pv(struct vm_page *pg, pmap_t pm, vm_offset_t va) { struct pv_entry *pv; rw_assert(&pvh_global_lock, RA_WLOCKED); TAILQ_FOREACH(pv, &pg->md.pv_list, pv_list) if (pm == pv->pv_pmap && va == pv->pv_va) break; return (pv); } /* * vector_page_setprot: * * Manipulate the protection of the vector page. */ void vector_page_setprot(int prot) { struct l2_bucket *l2b; pt_entry_t *ptep; l2b = pmap_get_l2_bucket(kernel_pmap, vector_page); ptep = &l2b->l2b_kva[l2pte_index(vector_page)]; *ptep = (*ptep & ~L1_S_PROT_MASK) | L2_S_PROT(PTE_KERNEL, prot); PTE_SYNC(ptep); cpu_tlb_flushD_SE(vector_page); cpu_cpwait(); } /* * pmap_remove_pv: try to remove a mapping from a pv_list * * => caller should hold proper lock on pmap_main_lock * => pmap should be locked * => caller should hold lock on vm_page [so that attrs can be adjusted] * => caller should adjust ptp's wire_count and free PTP if needed * => caller should NOT adjust pmap's wire_count * => we return the removed pve */ static void pmap_nuke_pv(struct vm_page *pg, pmap_t pm, struct pv_entry *pve) { struct pv_entry *pv; rw_assert(&pvh_global_lock, RA_WLOCKED); PMAP_ASSERT_LOCKED(pm); TAILQ_REMOVE(&pg->md.pv_list, pve, pv_list); TAILQ_REMOVE(&pm->pm_pvlist, pve, pv_plist); if (pve->pv_flags & PVF_WIRED) --pm->pm_stats.wired_count; if (pg->md.pvh_attrs & PVF_MOD) vm_page_dirty(pg); if (TAILQ_FIRST(&pg->md.pv_list) == NULL) pg->md.pvh_attrs &= ~PVF_REF; else vm_page_aflag_set(pg, PGA_REFERENCED); if ((pve->pv_flags & PVF_NC) && ((pm == kernel_pmap) || (pve->pv_flags & PVF_WRITE) || !(pve->pv_flags & PVF_MWC))) pmap_fix_cache(pg, pm, 0); else if (pve->pv_flags & PVF_WRITE) { TAILQ_FOREACH(pve, &pg->md.pv_list, pv_list) if (pve->pv_flags & PVF_WRITE) break; if (!pve) { pg->md.pvh_attrs &= ~PVF_MOD; vm_page_aflag_clear(pg, PGA_WRITEABLE); } } pv = TAILQ_FIRST(&pg->md.pv_list); if (pv != NULL && (pv->pv_flags & PVF_UNMAN) && TAILQ_NEXT(pv, pv_list) == NULL) { pm = kernel_pmap; pg->md.pv_kva = pv->pv_va; /* a recursive pmap_nuke_pv */ TAILQ_REMOVE(&pg->md.pv_list, pv, pv_list); TAILQ_REMOVE(&pm->pm_pvlist, pv, pv_plist); if (pv->pv_flags & PVF_WIRED) --pm->pm_stats.wired_count; pg->md.pvh_attrs &= ~PVF_REF; pg->md.pvh_attrs &= ~PVF_MOD; vm_page_aflag_clear(pg, PGA_WRITEABLE); pmap_free_pv_entry(pv); } } static struct pv_entry * pmap_remove_pv(struct vm_page *pg, pmap_t pm, vm_offset_t va) { struct pv_entry *pve; rw_assert(&pvh_global_lock, RA_WLOCKED); pve = TAILQ_FIRST(&pg->md.pv_list); while (pve) { if (pve->pv_pmap == pm && pve->pv_va == va) { /* match? */ pmap_nuke_pv(pg, pm, pve); break; } pve = TAILQ_NEXT(pve, pv_list); } if (pve == NULL && pg->md.pv_kva == va) pg->md.pv_kva = 0; return(pve); /* return removed pve */ } /* * * pmap_modify_pv: Update pv flags * * => caller should hold lock on vm_page [so that attrs can be adjusted] * => caller should NOT adjust pmap's wire_count * => we return the old flags * * Modify a physical-virtual mapping in the pv table */ static u_int pmap_modify_pv(struct vm_page *pg, pmap_t pm, vm_offset_t va, u_int clr_mask, u_int set_mask) { struct pv_entry *npv; u_int flags, oflags; PMAP_ASSERT_LOCKED(pm); rw_assert(&pvh_global_lock, RA_WLOCKED); if ((npv = pmap_find_pv(pg, pm, va)) == NULL) return (0); /* * There is at least one VA mapping this page. */ if (clr_mask & (PVF_REF | PVF_MOD)) pg->md.pvh_attrs |= set_mask & (PVF_REF | PVF_MOD); oflags = npv->pv_flags; npv->pv_flags = flags = (oflags & ~clr_mask) | set_mask; if ((flags ^ oflags) & PVF_WIRED) { if (flags & PVF_WIRED) ++pm->pm_stats.wired_count; else --pm->pm_stats.wired_count; } if ((flags ^ oflags) & PVF_WRITE) pmap_fix_cache(pg, pm, 0); return (oflags); } /* Function to set the debug level of the pmap code */ #ifdef PMAP_DEBUG void pmap_debug(int level) { pmap_debug_level = level; dprintf("pmap_debug: level=%d\n", pmap_debug_level); } #endif /* PMAP_DEBUG */ void pmap_pinit0(struct pmap *pmap) { PDEBUG(1, printf("pmap_pinit0: pmap = %08x\n", (u_int32_t) pmap)); bcopy(kernel_pmap, pmap, sizeof(*pmap)); bzero(&pmap->pm_mtx, sizeof(pmap->pm_mtx)); PMAP_LOCK_INIT(pmap); } /* * Initialize a vm_page's machine-dependent fields. */ void pmap_page_init(vm_page_t m) { TAILQ_INIT(&m->md.pv_list); m->md.pv_memattr = VM_MEMATTR_DEFAULT; } /* * Initialize the pmap module. * Called by vm_init, to initialize any structures that the pmap * system needs to map virtual memory. */ void pmap_init(void) { int shpgperproc = PMAP_SHPGPERPROC; l2zone = uma_zcreate("L2 Table", L2_TABLE_SIZE_REAL, pmap_l2ptp_ctor, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE); l2table_zone = uma_zcreate("L2 Table", sizeof(struct l2_dtable), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE); /* * Initialize the PV entry allocator. */ pvzone = uma_zcreate("PV ENTRY", sizeof (struct pv_entry), NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE); TUNABLE_INT_FETCH("vm.pmap.shpgperproc", &shpgperproc); pv_entry_max = shpgperproc * maxproc + vm_cnt.v_page_count; uma_zone_reserve_kva(pvzone, pv_entry_max); pv_entry_high_water = 9 * (pv_entry_max / 10); /* * Now it is safe to enable pv_table recording. */ PDEBUG(1, printf("pmap_init: done!\n")); } int pmap_fault_fixup(pmap_t pm, vm_offset_t va, vm_prot_t ftype, int user) { struct l2_dtable *l2; struct l2_bucket *l2b; pd_entry_t *pl1pd, l1pd; pt_entry_t *ptep, pte; vm_paddr_t pa; u_int l1idx; int rv = 0; l1idx = L1_IDX(va); rw_wlock(&pvh_global_lock); PMAP_LOCK(pm); /* * If there is no l2_dtable for this address, then the process * has no business accessing it. * * Note: This will catch userland processes trying to access * kernel addresses. */ l2 = pm->pm_l2[L2_IDX(l1idx)]; if (l2 == NULL) goto out; /* * Likewise if there is no L2 descriptor table */ l2b = &l2->l2_bucket[L2_BUCKET(l1idx)]; if (l2b->l2b_kva == NULL) goto out; /* * Check the PTE itself. */ ptep = &l2b->l2b_kva[l2pte_index(va)]; pte = *ptep; if (pte == 0) goto out; /* * Catch a userland access to the vector page mapped at 0x0 */ if (user && (pte & L2_S_PROT_U) == 0) goto out; if (va == vector_page) goto out; pa = l2pte_pa(pte); if ((ftype & VM_PROT_WRITE) && (pte & L2_S_PROT_W) == 0) { /* * This looks like a good candidate for "page modified" * emulation... */ struct pv_entry *pv; struct vm_page *pg; /* Extract the physical address of the page */ if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) { goto out; } /* Get the current flags for this page. */ pv = pmap_find_pv(pg, pm, va); if (pv == NULL) { goto out; } /* * Do the flags say this page is writable? If not then it * is a genuine write fault. If yes then the write fault is * our fault as we did not reflect the write access in the * PTE. Now we know a write has occurred we can correct this * and also set the modified bit */ if ((pv->pv_flags & PVF_WRITE) == 0) { goto out; } pg->md.pvh_attrs |= PVF_REF | PVF_MOD; vm_page_dirty(pg); pv->pv_flags |= PVF_REF | PVF_MOD; /* * Re-enable write permissions for the page. No need to call * pmap_fix_cache(), since this is just a * modified-emulation fault, and the PVF_WRITE bit isn't * changing. We've already set the cacheable bits based on * the assumption that we can write to this page. */ *ptep = (pte & ~L2_TYPE_MASK) | L2_S_PROTO | L2_S_PROT_W; PTE_SYNC(ptep); rv = 1; } else if ((pte & L2_TYPE_MASK) == L2_TYPE_INV) { /* * This looks like a good candidate for "page referenced" * emulation. */ struct pv_entry *pv; struct vm_page *pg; /* Extract the physical address of the page */ if ((pg = PHYS_TO_VM_PAGE(pa)) == NULL) goto out; /* Get the current flags for this page. */ pv = pmap_find_pv(pg, pm, va); if (pv == NULL) goto out; pg->md.pvh_attrs |= PVF_REF; pv->pv_flags |= PVF_REF; *ptep = (pte & ~L2_TYPE_MASK) | L2_S_PROTO; PTE_SYNC(ptep); rv = 1; } /* * We know there is a valid mapping here, so simply * fix up the L1 if necessary. */ pl1pd = &pm->pm_l1->l1_kva[l1idx]; l1pd = l2b->l2b_phys | L1_C_DOM(pm->pm_domain) | L1_C_PROTO; if (*pl1pd != l1pd) { *pl1pd = l1pd; PTE_SYNC(pl1pd); rv = 1; } #ifdef DEBUG /* * If 'rv == 0' at this point, it generally indicates that there is a * stale TLB entry for the faulting address. This happens when two or * more processes are sharing an L1. Since we don't flush the TLB on * a context switch between such processes, we can take domain faults * for mappings which exist at the same VA in both processes. EVEN IF * WE'VE RECENTLY FIXED UP THE CORRESPONDING L1 in pmap_enter(), for * example. * * This is extremely likely to happen if pmap_enter() updated the L1 * entry for a recently entered mapping. In this case, the TLB is * flushed for the new mapping, but there may still be TLB entries for * other mappings belonging to other processes in the 1MB range * covered by the L1 entry. * * Since 'rv == 0', we know that the L1 already contains the correct * value, so the fault must be due to a stale TLB entry. * * Since we always need to flush the TLB anyway in the case where we * fixed up the L1, or frobbed the L2 PTE, we effectively deal with * stale TLB entries dynamically. * * However, the above condition can ONLY happen if the current L1 is * being shared. If it happens when the L1 is unshared, it indicates * that other parts of the pmap are not doing their job WRT managing * the TLB. */ if (rv == 0 && pm->pm_l1->l1_domain_use_count == 1) { printf("fixup: pm %p, va 0x%lx, ftype %d - nothing to do!\n", pm, (u_long)va, ftype); printf("fixup: l2 %p, l2b %p, ptep %p, pl1pd %p\n", l2, l2b, ptep, pl1pd); printf("fixup: pte 0x%x, l1pd 0x%x, last code 0x%x\n", pte, l1pd, last_fault_code); #ifdef DDB Debugger(); #endif } #endif cpu_tlb_flushID_SE(va); cpu_cpwait(); rv = 1; out: rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(pm); return (rv); } void pmap_postinit(void) { struct l2_bucket *l2b; struct l1_ttable *l1; pd_entry_t *pl1pt; pt_entry_t *ptep, pte; vm_offset_t va, eva; u_int loop, needed; needed = (maxproc / PMAP_DOMAINS) + ((maxproc % PMAP_DOMAINS) ? 1 : 0); needed -= 1; l1 = malloc(sizeof(*l1) * needed, M_VMPMAP, M_WAITOK); for (loop = 0; loop < needed; loop++, l1++) { /* Allocate a L1 page table */ va = (vm_offset_t)contigmalloc(L1_TABLE_SIZE, M_VMPMAP, 0, 0x0, 0xffffffff, L1_TABLE_SIZE, 0); if (va == 0) panic("Cannot allocate L1 KVM"); eva = va + L1_TABLE_SIZE; pl1pt = (pd_entry_t *)va; while (va < eva) { l2b = pmap_get_l2_bucket(kernel_pmap, va); ptep = &l2b->l2b_kva[l2pte_index(va)]; pte = *ptep; pte = (pte & ~L2_S_CACHE_MASK) | pte_l2_s_cache_mode_pt; *ptep = pte; PTE_SYNC(ptep); cpu_tlb_flushD_SE(va); va += PAGE_SIZE; } pmap_init_l1(l1, pl1pt); } #ifdef DEBUG printf("pmap_postinit: Allocated %d static L1 descriptor tables\n", needed); #endif } /* * This is used to stuff certain critical values into the PCB where they * can be accessed quickly from cpu_switch() et al. */ void pmap_set_pcb_pagedir(pmap_t pm, struct pcb *pcb) { struct l2_bucket *l2b; pcb->pcb_pagedir = pm->pm_l1->l1_physaddr; pcb->pcb_dacr = (DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL * 2)) | (DOMAIN_CLIENT << (pm->pm_domain * 2)); if (vector_page < KERNBASE) { pcb->pcb_pl1vec = &pm->pm_l1->l1_kva[L1_IDX(vector_page)]; l2b = pmap_get_l2_bucket(pm, vector_page); pcb->pcb_l1vec = l2b->l2b_phys | L1_C_PROTO | L1_C_DOM(pm->pm_domain) | L1_C_DOM(PMAP_DOMAIN_KERNEL); } else pcb->pcb_pl1vec = NULL; } void pmap_activate(struct thread *td) { pmap_t pm; struct pcb *pcb; pm = vmspace_pmap(td->td_proc->p_vmspace); pcb = td->td_pcb; critical_enter(); pmap_set_pcb_pagedir(pm, pcb); if (td == curthread) { u_int cur_dacr, cur_ttb; __asm __volatile("mrc p15, 0, %0, c2, c0, 0" : "=r"(cur_ttb)); __asm __volatile("mrc p15, 0, %0, c3, c0, 0" : "=r"(cur_dacr)); cur_ttb &= ~(L1_TABLE_SIZE - 1); if (cur_ttb == (u_int)pcb->pcb_pagedir && cur_dacr == pcb->pcb_dacr) { /* * No need to switch address spaces. */ critical_exit(); return; } /* * We MUST, I repeat, MUST fix up the L1 entry corresponding * to 'vector_page' in the incoming L1 table before switching * to it otherwise subsequent interrupts/exceptions (including * domain faults!) will jump into hyperspace. */ if (pcb->pcb_pl1vec) { *pcb->pcb_pl1vec = pcb->pcb_l1vec; /* * Don't need to PTE_SYNC() at this point since * cpu_setttb() is about to flush both the cache * and the TLB. */ } cpu_domains(pcb->pcb_dacr); cpu_setttb(pcb->pcb_pagedir); } critical_exit(); } static int pmap_set_pt_cache_mode(pd_entry_t *kl1, vm_offset_t va) { pd_entry_t *pdep, pde; pt_entry_t *ptep, pte; vm_offset_t pa; int rv = 0; /* * Make sure the descriptor itself has the correct cache mode */ pdep = &kl1[L1_IDX(va)]; pde = *pdep; if (l1pte_section_p(pde)) { if ((pde & L1_S_CACHE_MASK) != pte_l1_s_cache_mode_pt) { *pdep = (pde & ~L1_S_CACHE_MASK) | pte_l1_s_cache_mode_pt; PTE_SYNC(pdep); cpu_dcache_wbinv_range((vm_offset_t)pdep, sizeof(*pdep)); cpu_l2cache_wbinv_range((vm_offset_t)pdep, sizeof(*pdep)); rv = 1; } } else { pa = (vm_paddr_t)(pde & L1_C_ADDR_MASK); ptep = (pt_entry_t *)kernel_pt_lookup(pa); if (ptep == NULL) panic("pmap_bootstrap: No L2 for L2 @ va %p\n", ptep); ptep = &ptep[l2pte_index(va)]; pte = *ptep; if ((pte & L2_S_CACHE_MASK) != pte_l2_s_cache_mode_pt) { *ptep = (pte & ~L2_S_CACHE_MASK) | pte_l2_s_cache_mode_pt; PTE_SYNC(ptep); cpu_dcache_wbinv_range((vm_offset_t)ptep, sizeof(*ptep)); cpu_l2cache_wbinv_range((vm_offset_t)ptep, sizeof(*ptep)); rv = 1; } } return (rv); } static void pmap_alloc_specials(vm_offset_t *availp, int pages, vm_offset_t *vap, pt_entry_t **ptep) { vm_offset_t va = *availp; struct l2_bucket *l2b; if (ptep) { l2b = pmap_get_l2_bucket(kernel_pmap, va); if (l2b == NULL) panic("pmap_alloc_specials: no l2b for 0x%x", va); *ptep = &l2b->l2b_kva[l2pte_index(va)]; } *vap = va; *availp = va + (PAGE_SIZE * pages); } /* * Bootstrap the system enough to run with virtual memory. * * On the arm this is called after mapping has already been enabled * and just syncs the pmap module with what has already been done. * [We can't call it easily with mapping off since the kernel is not * mapped with PA == VA, hence we would have to relocate every address * from the linked base (virtual) address "KERNBASE" to the actual * (physical) address starting relative to 0] */ #define PMAP_STATIC_L2_SIZE 16 void pmap_bootstrap(vm_offset_t firstaddr, struct pv_addr *l1pt) { static struct l1_ttable static_l1; static struct l2_dtable static_l2[PMAP_STATIC_L2_SIZE]; struct l1_ttable *l1 = &static_l1; struct l2_dtable *l2; struct l2_bucket *l2b; pd_entry_t pde; pd_entry_t *kernel_l1pt = (pd_entry_t *)l1pt->pv_va; pt_entry_t *ptep; pt_entry_t *qmap_pte; vm_paddr_t pa; vm_offset_t va; vm_size_t size; int l1idx, l2idx, l2next = 0; PDEBUG(1, printf("firstaddr = %08x, lastaddr = %08x\n", firstaddr, vm_max_kernel_address)); virtual_avail = firstaddr; kernel_pmap->pm_l1 = l1; kernel_l1pa = l1pt->pv_pa; /* * Scan the L1 translation table created by initarm() and create * the required metadata for all valid mappings found in it. */ for (l1idx = 0; l1idx < (L1_TABLE_SIZE / sizeof(pd_entry_t)); l1idx++) { pde = kernel_l1pt[l1idx]; /* * We're only interested in Coarse mappings. * pmap_extract() can deal with section mappings without * recourse to checking L2 metadata. */ if ((pde & L1_TYPE_MASK) != L1_TYPE_C) continue; /* * Lookup the KVA of this L2 descriptor table */ pa = (vm_paddr_t)(pde & L1_C_ADDR_MASK); ptep = (pt_entry_t *)kernel_pt_lookup(pa); if (ptep == NULL) { panic("pmap_bootstrap: No L2 for va 0x%x, pa 0x%lx", (u_int)l1idx << L1_S_SHIFT, (long unsigned int)pa); } /* * Fetch the associated L2 metadata structure. * Allocate a new one if necessary. */ if ((l2 = kernel_pmap->pm_l2[L2_IDX(l1idx)]) == NULL) { if (l2next == PMAP_STATIC_L2_SIZE) panic("pmap_bootstrap: out of static L2s"); kernel_pmap->pm_l2[L2_IDX(l1idx)] = l2 = &static_l2[l2next++]; } /* * One more L1 slot tracked... */ l2->l2_occupancy++; /* * Fill in the details of the L2 descriptor in the * appropriate bucket. */ l2b = &l2->l2_bucket[L2_BUCKET(l1idx)]; l2b->l2b_kva = ptep; l2b->l2b_phys = pa; l2b->l2b_l1idx = l1idx; /* * Establish an initial occupancy count for this descriptor */ for (l2idx = 0; l2idx < (L2_TABLE_SIZE_REAL / sizeof(pt_entry_t)); l2idx++) { if ((ptep[l2idx] & L2_TYPE_MASK) != L2_TYPE_INV) { l2b->l2b_occupancy++; } } /* * Make sure the descriptor itself has the correct cache mode. * If not, fix it, but whine about the problem. Port-meisters * should consider this a clue to fix up their initarm() * function. :) */ if (pmap_set_pt_cache_mode(kernel_l1pt, (vm_offset_t)ptep)) { printf("pmap_bootstrap: WARNING! wrong cache mode for " "L2 pte @ %p\n", ptep); } } /* * Ensure the primary (kernel) L1 has the correct cache mode for * a page table. Bitch if it is not correctly set. */ for (va = (vm_offset_t)kernel_l1pt; va < ((vm_offset_t)kernel_l1pt + L1_TABLE_SIZE); va += PAGE_SIZE) { if (pmap_set_pt_cache_mode(kernel_l1pt, va)) printf("pmap_bootstrap: WARNING! wrong cache mode for " "primary L1 @ 0x%x\n", va); } cpu_dcache_wbinv_all(); cpu_l2cache_wbinv_all(); cpu_tlb_flushID(); cpu_cpwait(); PMAP_LOCK_INIT(kernel_pmap); CPU_FILL(&kernel_pmap->pm_active); kernel_pmap->pm_domain = PMAP_DOMAIN_KERNEL; TAILQ_INIT(&kernel_pmap->pm_pvlist); /* * Initialize the global pv list lock. */ rw_init_flags(&pvh_global_lock, "pmap pv global", RW_RECURSE); /* * Reserve some special page table entries/VA space for temporary * mapping of pages. */ pmap_alloc_specials(&virtual_avail, 1, &csrcp, &csrc_pte); pmap_set_pt_cache_mode(kernel_l1pt, (vm_offset_t)csrc_pte); pmap_alloc_specials(&virtual_avail, 1, &cdstp, &cdst_pte); pmap_set_pt_cache_mode(kernel_l1pt, (vm_offset_t)cdst_pte); pmap_alloc_specials(&virtual_avail, 1, &qmap_addr, &qmap_pte); pmap_set_pt_cache_mode(kernel_l1pt, (vm_offset_t)qmap_pte); size = ((vm_max_kernel_address - pmap_curmaxkvaddr) + L1_S_OFFSET) / L1_S_SIZE; pmap_alloc_specials(&virtual_avail, round_page(size * L2_TABLE_SIZE_REAL) / PAGE_SIZE, &pmap_kernel_l2ptp_kva, NULL); size = howmany(size, L2_BUCKET_SIZE); pmap_alloc_specials(&virtual_avail, round_page(size * sizeof(struct l2_dtable)) / PAGE_SIZE, &pmap_kernel_l2dtable_kva, NULL); pmap_alloc_specials(&virtual_avail, 1, (vm_offset_t*)&_tmppt, NULL); pmap_alloc_specials(&virtual_avail, MAXDUMPPGS, (vm_offset_t *)&crashdumpmap, NULL); SLIST_INIT(&l1_list); TAILQ_INIT(&l1_lru_list); mtx_init(&l1_lru_lock, "l1 list lock", NULL, MTX_DEF); pmap_init_l1(l1, kernel_l1pt); cpu_dcache_wbinv_all(); cpu_l2cache_wbinv_all(); virtual_avail = round_page(virtual_avail); virtual_end = vm_max_kernel_address; kernel_vm_end = pmap_curmaxkvaddr; mtx_init(&cmtx, "TMP mappings mtx", NULL, MTX_DEF); mtx_init(&qmap_mtx, "quick mapping mtx", NULL, MTX_DEF); pmap_set_pcb_pagedir(kernel_pmap, thread0.td_pcb); } /*************************************************** * Pmap allocation/deallocation routines. ***************************************************/ /* * Release any resources held by the given physical map. * Called when a pmap initialized by pmap_pinit is being released. * Should only be called if the map contains no valid mappings. */ void pmap_release(pmap_t pmap) { struct pcb *pcb; pmap_idcache_wbinv_all(pmap); cpu_l2cache_wbinv_all(); pmap_tlb_flushID(pmap); cpu_cpwait(); if (vector_page < KERNBASE) { struct pcb *curpcb = PCPU_GET(curpcb); pcb = thread0.td_pcb; if (pmap_is_current(pmap)) { /* * Frob the L1 entry corresponding to the vector * page so that it contains the kernel pmap's domain * number. This will ensure pmap_remove() does not * pull the current vector page out from under us. */ critical_enter(); *pcb->pcb_pl1vec = pcb->pcb_l1vec; cpu_domains(pcb->pcb_dacr); cpu_setttb(pcb->pcb_pagedir); critical_exit(); } pmap_remove(pmap, vector_page, vector_page + PAGE_SIZE); /* * Make sure cpu_switch(), et al, DTRT. This is safe to do * since this process has no remaining mappings of its own. */ curpcb->pcb_pl1vec = pcb->pcb_pl1vec; curpcb->pcb_l1vec = pcb->pcb_l1vec; curpcb->pcb_dacr = pcb->pcb_dacr; curpcb->pcb_pagedir = pcb->pcb_pagedir; } pmap_free_l1(pmap); dprintf("pmap_release()\n"); } /* * Helper function for pmap_grow_l2_bucket() */ static __inline int pmap_grow_map(vm_offset_t va, pt_entry_t cache_mode, vm_paddr_t *pap) { struct l2_bucket *l2b; pt_entry_t *ptep; vm_paddr_t pa; struct vm_page *pg; pg = vm_page_alloc(NULL, 0, VM_ALLOC_NOOBJ | VM_ALLOC_WIRED); if (pg == NULL) return (1); pa = VM_PAGE_TO_PHYS(pg); if (pap) *pap = pa; l2b = pmap_get_l2_bucket(kernel_pmap, va); ptep = &l2b->l2b_kva[l2pte_index(va)]; *ptep = L2_S_PROTO | pa | cache_mode | L2_S_PROT(PTE_KERNEL, VM_PROT_READ | VM_PROT_WRITE); PTE_SYNC(ptep); return (0); } /* * This is the same as pmap_alloc_l2_bucket(), except that it is only * used by pmap_growkernel(). */ static __inline struct l2_bucket * pmap_grow_l2_bucket(pmap_t pm, vm_offset_t va) { struct l2_dtable *l2; struct l2_bucket *l2b; struct l1_ttable *l1; pd_entry_t *pl1pd; u_short l1idx; vm_offset_t nva; l1idx = L1_IDX(va); if ((l2 = pm->pm_l2[L2_IDX(l1idx)]) == NULL) { /* * No mapping at this address, as there is * no entry in the L1 table. * Need to allocate a new l2_dtable. */ nva = pmap_kernel_l2dtable_kva; if ((nva & PAGE_MASK) == 0) { /* * Need to allocate a backing page */ if (pmap_grow_map(nva, pte_l2_s_cache_mode, NULL)) return (NULL); } l2 = (struct l2_dtable *)nva; nva += sizeof(struct l2_dtable); if ((nva & PAGE_MASK) < (pmap_kernel_l2dtable_kva & PAGE_MASK)) { /* * The new l2_dtable straddles a page boundary. * Map in another page to cover it. */ if (pmap_grow_map(nva, pte_l2_s_cache_mode, NULL)) return (NULL); } pmap_kernel_l2dtable_kva = nva; /* * Link it into the parent pmap */ pm->pm_l2[L2_IDX(l1idx)] = l2; memset(l2, 0, sizeof(*l2)); } l2b = &l2->l2_bucket[L2_BUCKET(l1idx)]; /* * Fetch pointer to the L2 page table associated with the address. */ if (l2b->l2b_kva == NULL) { pt_entry_t *ptep; /* * No L2 page table has been allocated. Chances are, this * is because we just allocated the l2_dtable, above. */ nva = pmap_kernel_l2ptp_kva; ptep = (pt_entry_t *)nva; if ((nva & PAGE_MASK) == 0) { /* * Need to allocate a backing page */ if (pmap_grow_map(nva, pte_l2_s_cache_mode_pt, &pmap_kernel_l2ptp_phys)) return (NULL); PTE_SYNC_RANGE(ptep, PAGE_SIZE / sizeof(pt_entry_t)); } memset(ptep, 0, L2_TABLE_SIZE_REAL); l2->l2_occupancy++; l2b->l2b_kva = ptep; l2b->l2b_l1idx = l1idx; l2b->l2b_phys = pmap_kernel_l2ptp_phys; pmap_kernel_l2ptp_kva += L2_TABLE_SIZE_REAL; pmap_kernel_l2ptp_phys += L2_TABLE_SIZE_REAL; } /* Distribute new L1 entry to all other L1s */ SLIST_FOREACH(l1, &l1_list, l1_link) { pl1pd = &l1->l1_kva[L1_IDX(va)]; *pl1pd = l2b->l2b_phys | L1_C_DOM(PMAP_DOMAIN_KERNEL) | L1_C_PROTO; PTE_SYNC(pl1pd); } return (l2b); } /* * grow the number of kernel page table entries, if needed */ void pmap_growkernel(vm_offset_t addr) { pmap_t kpm = kernel_pmap; if (addr <= pmap_curmaxkvaddr) return; /* we are OK */ /* * whoops! we need to add kernel PTPs */ /* Map 1MB at a time */ for (; pmap_curmaxkvaddr < addr; pmap_curmaxkvaddr += L1_S_SIZE) pmap_grow_l2_bucket(kpm, pmap_curmaxkvaddr); /* * flush out the cache, expensive but growkernel will happen so * rarely */ cpu_dcache_wbinv_all(); cpu_l2cache_wbinv_all(); cpu_tlb_flushD(); cpu_cpwait(); kernel_vm_end = pmap_curmaxkvaddr; } /* * Remove all pages from specified address space * this aids process exit speeds. Also, this code * is special cased for current process only, but * can have the more generic (and slightly slower) * mode enabled. This is much faster than pmap_remove * in the case of running down an entire address space. */ void pmap_remove_pages(pmap_t pmap) { struct pv_entry *pv, *npv; struct l2_bucket *l2b = NULL; vm_page_t m; pt_entry_t *pt; rw_wlock(&pvh_global_lock); PMAP_LOCK(pmap); cpu_idcache_wbinv_all(); cpu_l2cache_wbinv_all(); for (pv = TAILQ_FIRST(&pmap->pm_pvlist); pv; pv = npv) { if (pv->pv_flags & PVF_WIRED || pv->pv_flags & PVF_UNMAN) { /* Cannot remove wired or unmanaged pages now. */ npv = TAILQ_NEXT(pv, pv_plist); continue; } pmap->pm_stats.resident_count--; l2b = pmap_get_l2_bucket(pmap, pv->pv_va); KASSERT(l2b != NULL, ("No L2 bucket in pmap_remove_pages")); pt = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; m = PHYS_TO_VM_PAGE(*pt & L2_S_FRAME); KASSERT((vm_offset_t)m >= KERNBASE, ("Trying to access non-existent page va %x pte %x", pv->pv_va, *pt)); *pt = 0; PTE_SYNC(pt); npv = TAILQ_NEXT(pv, pv_plist); pmap_nuke_pv(m, pmap, pv); if (TAILQ_EMPTY(&m->md.pv_list)) vm_page_aflag_clear(m, PGA_WRITEABLE); pmap_free_pv_entry(pv); pmap_free_l2_bucket(pmap, l2b, 1); } rw_wunlock(&pvh_global_lock); cpu_tlb_flushID(); cpu_cpwait(); PMAP_UNLOCK(pmap); } /*************************************************** * Low level mapping routines..... ***************************************************/ #ifdef ARM_HAVE_SUPERSECTIONS /* Map a super section into the KVA. */ void pmap_kenter_supersection(vm_offset_t va, uint64_t pa, int flags) { pd_entry_t pd = L1_S_PROTO | L1_S_SUPERSEC | (pa & L1_SUP_FRAME) | (((pa >> 32) & 0xf) << 20) | L1_S_PROT(PTE_KERNEL, VM_PROT_READ|VM_PROT_WRITE) | L1_S_DOM(PMAP_DOMAIN_KERNEL); struct l1_ttable *l1; vm_offset_t va0, va_end; KASSERT(((va | pa) & L1_SUP_OFFSET) == 0, ("Not a valid super section mapping")); if (flags & SECTION_CACHE) pd |= pte_l1_s_cache_mode; else if (flags & SECTION_PT) pd |= pte_l1_s_cache_mode_pt; va0 = va & L1_SUP_FRAME; va_end = va + L1_SUP_SIZE; SLIST_FOREACH(l1, &l1_list, l1_link) { va = va0; for (; va < va_end; va += L1_S_SIZE) { l1->l1_kva[L1_IDX(va)] = pd; PTE_SYNC(&l1->l1_kva[L1_IDX(va)]); } } } #endif /* Map a section into the KVA. */ void pmap_kenter_section(vm_offset_t va, vm_offset_t pa, int flags) { pd_entry_t pd = L1_S_PROTO | pa | L1_S_PROT(PTE_KERNEL, VM_PROT_READ|VM_PROT_WRITE) | L1_S_DOM(PMAP_DOMAIN_KERNEL); struct l1_ttable *l1; KASSERT(((va | pa) & L1_S_OFFSET) == 0, ("Not a valid section mapping")); if (flags & SECTION_CACHE) pd |= pte_l1_s_cache_mode; else if (flags & SECTION_PT) pd |= pte_l1_s_cache_mode_pt; SLIST_FOREACH(l1, &l1_list, l1_link) { l1->l1_kva[L1_IDX(va)] = pd; PTE_SYNC(&l1->l1_kva[L1_IDX(va)]); } } /* * Make a temporary mapping for a physical address. This is only intended * to be used for panic dumps. */ void * pmap_kenter_temporary(vm_paddr_t pa, int i) { vm_offset_t va; va = (vm_offset_t)crashdumpmap + (i * PAGE_SIZE); pmap_kenter(va, pa); return ((void *)crashdumpmap); } /* * add a wired page to the kva * note that in order for the mapping to take effect -- you * should do a invltlb after doing the pmap_kenter... */ static PMAP_INLINE void pmap_kenter_internal(vm_offset_t va, vm_offset_t pa, int flags) { struct l2_bucket *l2b; pt_entry_t *pte; pt_entry_t opte; struct pv_entry *pve; vm_page_t m; PDEBUG(1, printf("pmap_kenter: va = %08x, pa = %08x\n", (uint32_t) va, (uint32_t) pa)); l2b = pmap_get_l2_bucket(kernel_pmap, va); if (l2b == NULL) l2b = pmap_grow_l2_bucket(kernel_pmap, va); KASSERT(l2b != NULL, ("No L2 Bucket")); pte = &l2b->l2b_kva[l2pte_index(va)]; opte = *pte; PDEBUG(1, printf("pmap_kenter: pte = %08x, opte = %08x, npte = %08x\n", (uint32_t) pte, opte, *pte)); if (l2pte_valid(opte)) { pmap_kremove(va); } else { if (opte == 0) l2b->l2b_occupancy++; } *pte = L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, VM_PROT_READ | VM_PROT_WRITE); if (flags & KENTER_CACHE) *pte |= pte_l2_s_cache_mode; if (flags & KENTER_USER) *pte |= L2_S_PROT_U; PTE_SYNC(pte); /* * A kernel mapping may not be the page's only mapping, so create a PV * entry to ensure proper caching. * * The existence test for the pvzone is used to delay the recording of * kernel mappings until the VM system is fully initialized. * * This expects the physical memory to have a vm_page_array entry. */ if (pvzone != NULL && (m = vm_phys_paddr_to_vm_page(pa)) != NULL) { rw_wlock(&pvh_global_lock); if (!TAILQ_EMPTY(&m->md.pv_list) || m->md.pv_kva != 0) { if ((pve = pmap_get_pv_entry()) == NULL) panic("pmap_kenter_internal: no pv entries"); PMAP_LOCK(kernel_pmap); pmap_enter_pv(m, pve, kernel_pmap, va, PVF_WRITE | PVF_UNMAN); pmap_fix_cache(m, kernel_pmap, va); PMAP_UNLOCK(kernel_pmap); } else { m->md.pv_kva = va; } rw_wunlock(&pvh_global_lock); } } void pmap_kenter(vm_offset_t va, vm_paddr_t pa) { pmap_kenter_internal(va, pa, KENTER_CACHE); } void pmap_kenter_nocache(vm_offset_t va, vm_paddr_t pa) { pmap_kenter_internal(va, pa, 0); } void pmap_kenter_device(vm_offset_t va, vm_size_t size, vm_paddr_t pa) { vm_offset_t sva; KASSERT((size & PAGE_MASK) == 0, ("%s: device mapping not page-sized", __func__)); sva = va; while (size != 0) { pmap_kenter_internal(va, pa, 0); va += PAGE_SIZE; pa += PAGE_SIZE; size -= PAGE_SIZE; } } void pmap_kremove_device(vm_offset_t va, vm_size_t size) { vm_offset_t sva; KASSERT((size & PAGE_MASK) == 0, ("%s: device mapping not page-sized", __func__)); sva = va; while (size != 0) { pmap_kremove(va); va += PAGE_SIZE; size -= PAGE_SIZE; } } void pmap_kenter_user(vm_offset_t va, vm_paddr_t pa) { pmap_kenter_internal(va, pa, KENTER_CACHE|KENTER_USER); /* * Call pmap_fault_fixup now, to make sure we'll have no exception * at the first use of the new address, or bad things will happen, * as we use one of these addresses in the exception handlers. */ pmap_fault_fixup(kernel_pmap, va, VM_PROT_READ|VM_PROT_WRITE, 1); } vm_paddr_t pmap_kextract(vm_offset_t va) { return (pmap_extract_locked(kernel_pmap, va)); } /* * remove a page from the kernel pagetables */ void pmap_kremove(vm_offset_t va) { struct l2_bucket *l2b; pt_entry_t *pte, opte; struct pv_entry *pve; vm_page_t m; vm_offset_t pa; l2b = pmap_get_l2_bucket(kernel_pmap, va); if (!l2b) return; KASSERT(l2b != NULL, ("No L2 Bucket")); pte = &l2b->l2b_kva[l2pte_index(va)]; opte = *pte; if (l2pte_valid(opte)) { /* pa = vtophs(va) taken from pmap_extract() */ if ((opte & L2_TYPE_MASK) == L2_TYPE_L) pa = (opte & L2_L_FRAME) | (va & L2_L_OFFSET); else pa = (opte & L2_S_FRAME) | (va & L2_S_OFFSET); /* note: should never have to remove an allocation * before the pvzone is initialized. */ rw_wlock(&pvh_global_lock); PMAP_LOCK(kernel_pmap); if (pvzone != NULL && (m = vm_phys_paddr_to_vm_page(pa)) && (pve = pmap_remove_pv(m, kernel_pmap, va))) pmap_free_pv_entry(pve); PMAP_UNLOCK(kernel_pmap); rw_wunlock(&pvh_global_lock); va = va & ~PAGE_MASK; cpu_dcache_wbinv_range(va, PAGE_SIZE); cpu_l2cache_wbinv_range(va, PAGE_SIZE); cpu_tlb_flushD_SE(va); cpu_cpwait(); *pte = 0; } } /* * Used to map a range of physical addresses into kernel * virtual address space. * * The value passed in '*virt' is a suggested virtual address for * the mapping. Architectures which can support a direct-mapped * physical to virtual region can return the appropriate address * within that region, leaving '*virt' unchanged. Other * architectures should map the pages starting at '*virt' and * update '*virt' with the first usable address after the mapped * region. */ vm_offset_t pmap_map(vm_offset_t *virt, vm_offset_t start, vm_offset_t end, int prot) { vm_offset_t sva = *virt; vm_offset_t va = sva; PDEBUG(1, printf("pmap_map: virt = %08x, start = %08x, end = %08x, " "prot = %d\n", (uint32_t) *virt, (uint32_t) start, (uint32_t) end, prot)); while (start < end) { pmap_kenter(va, start); va += PAGE_SIZE; start += PAGE_SIZE; } *virt = va; return (sva); } static void pmap_wb_page(vm_page_t m) { struct pv_entry *pv; TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) pmap_dcache_wb_range(pv->pv_pmap, pv->pv_va, PAGE_SIZE, FALSE, (pv->pv_flags & PVF_WRITE) == 0); } static void pmap_inv_page(vm_page_t m) { struct pv_entry *pv; TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) pmap_dcache_wb_range(pv->pv_pmap, pv->pv_va, PAGE_SIZE, TRUE, TRUE); } /* * Add a list of wired pages to the kva * this routine is only used for temporary * kernel mappings that do not need to have * page modification or references recorded. * Note that old mappings are simply written * over. The page *must* be wired. */ void pmap_qenter(vm_offset_t va, vm_page_t *m, int count) { int i; for (i = 0; i < count; i++) { pmap_wb_page(m[i]); pmap_kenter_internal(va, VM_PAGE_TO_PHYS(m[i]), KENTER_CACHE); va += PAGE_SIZE; } } /* * this routine jerks page mappings from the * kernel -- it is meant only for temporary mappings. */ void pmap_qremove(vm_offset_t va, int count) { vm_paddr_t pa; int i; for (i = 0; i < count; i++) { pa = vtophys(va); if (pa) { pmap_inv_page(PHYS_TO_VM_PAGE(pa)); pmap_kremove(va); } va += PAGE_SIZE; } } /* * pmap_object_init_pt preloads the ptes for a given object * into the specified pmap. This eliminates the blast of soft * faults on process startup and immediately after an mmap. */ void pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object, vm_pindex_t pindex, vm_size_t size) { VM_OBJECT_ASSERT_WLOCKED(object); KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, ("pmap_object_init_pt: non-device object")); } /* * pmap_is_prefaultable: * * Return whether or not the specified virtual address is elgible * for prefault. */ boolean_t pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr) { pd_entry_t *pde; pt_entry_t *pte; if (!pmap_get_pde_pte(pmap, addr, &pde, &pte)) return (FALSE); KASSERT(pte != NULL, ("Valid mapping but no pte ?")); if (*pte == 0) return (TRUE); return (FALSE); } /* * Fetch pointers to the PDE/PTE for the given pmap/VA pair. * Returns TRUE if the mapping exists, else FALSE. * * NOTE: This function is only used by a couple of arm-specific modules. * It is not safe to take any pmap locks here, since we could be right * in the middle of debugging the pmap anyway... * * It is possible for this routine to return FALSE even though a valid * mapping does exist. This is because we don't lock, so the metadata * state may be inconsistent. * * NOTE: We can return a NULL *ptp in the case where the L1 pde is * a "section" mapping. */ boolean_t pmap_get_pde_pte(pmap_t pm, vm_offset_t va, pd_entry_t **pdp, pt_entry_t **ptp) { struct l2_dtable *l2; pd_entry_t *pl1pd, l1pd; pt_entry_t *ptep; u_short l1idx; if (pm->pm_l1 == NULL) return (FALSE); l1idx = L1_IDX(va); *pdp = pl1pd = &pm->pm_l1->l1_kva[l1idx]; l1pd = *pl1pd; if (l1pte_section_p(l1pd)) { *ptp = NULL; return (TRUE); } if (pm->pm_l2 == NULL) return (FALSE); l2 = pm->pm_l2[L2_IDX(l1idx)]; if (l2 == NULL || (ptep = l2->l2_bucket[L2_BUCKET(l1idx)].l2b_kva) == NULL) { return (FALSE); } *ptp = &ptep[l2pte_index(va)]; return (TRUE); } /* * Routine: pmap_remove_all * Function: * Removes this physical page from * all physical maps in which it resides. * Reflects back modify bits to the pager. * * Notes: * Original versions of this routine were very * inefficient because they iteratively called * pmap_remove (slow...) */ void pmap_remove_all(vm_page_t m) { pv_entry_t pv; pt_entry_t *ptep; struct l2_bucket *l2b; boolean_t flush = FALSE; pmap_t curpm; int flags = 0; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_remove_all: page %p is not managed", m)); if (TAILQ_EMPTY(&m->md.pv_list)) return; rw_wlock(&pvh_global_lock); /* * XXX This call shouldn't exist. Iterating over the PV list twice, * once in pmap_clearbit() and again below, is both unnecessary and * inefficient. The below code should itself write back the cache * entry before it destroys the mapping. */ pmap_clearbit(m, PVF_WRITE); curpm = vmspace_pmap(curproc->p_vmspace); while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { if (flush == FALSE && (pv->pv_pmap == curpm || pv->pv_pmap == kernel_pmap)) flush = TRUE; PMAP_LOCK(pv->pv_pmap); /* * Cached contents were written-back in pmap_clearbit(), * but we still have to invalidate the cache entry to make * sure stale data are not retrieved when another page will be * mapped under this virtual address. */ if (pmap_is_current(pv->pv_pmap)) { cpu_dcache_inv_range(pv->pv_va, PAGE_SIZE); if (pmap_has_valid_mapping(pv->pv_pmap, pv->pv_va)) cpu_l2cache_inv_range(pv->pv_va, PAGE_SIZE); } if (pv->pv_flags & PVF_UNMAN) { /* remove the pv entry, but do not remove the mapping * and remember this is a kernel mapped page */ m->md.pv_kva = pv->pv_va; } else { /* remove the mapping and pv entry */ l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va); KASSERT(l2b != NULL, ("No l2 bucket")); ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; *ptep = 0; PTE_SYNC_CURRENT(pv->pv_pmap, ptep); pmap_free_l2_bucket(pv->pv_pmap, l2b, 1); pv->pv_pmap->pm_stats.resident_count--; flags |= pv->pv_flags; } pmap_nuke_pv(m, pv->pv_pmap, pv); PMAP_UNLOCK(pv->pv_pmap); pmap_free_pv_entry(pv); } if (flush) { if (PV_BEEN_EXECD(flags)) pmap_tlb_flushID(curpm); else pmap_tlb_flushD(curpm); } vm_page_aflag_clear(m, PGA_WRITEABLE); rw_wunlock(&pvh_global_lock); } /* * Set the physical protection on the * specified range of this map as requested. */ void pmap_protect(pmap_t pm, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) { struct l2_bucket *l2b; pt_entry_t *ptep, pte; vm_offset_t next_bucket; u_int flags; int flush; CTR4(KTR_PMAP, "pmap_protect: pmap %p sva 0x%08x eva 0x%08x prot %x", pm, sva, eva, prot); if ((prot & VM_PROT_READ) == 0) { pmap_remove(pm, sva, eva); return; } if (prot & VM_PROT_WRITE) { /* * If this is a read->write transition, just ignore it and let * vm_fault() take care of it later. */ return; } rw_wlock(&pvh_global_lock); PMAP_LOCK(pm); /* * OK, at this point, we know we're doing write-protect operation. * If the pmap is active, write-back the range. */ pmap_dcache_wb_range(pm, sva, eva - sva, FALSE, FALSE); flush = ((eva - sva) >= (PAGE_SIZE * 4)) ? 0 : -1; flags = 0; while (sva < eva) { next_bucket = L2_NEXT_BUCKET(sva); if (next_bucket > eva) next_bucket = eva; l2b = pmap_get_l2_bucket(pm, sva); if (l2b == NULL) { sva = next_bucket; continue; } ptep = &l2b->l2b_kva[l2pte_index(sva)]; while (sva < next_bucket) { if ((pte = *ptep) != 0 && (pte & L2_S_PROT_W) != 0) { struct vm_page *pg; u_int f; pg = PHYS_TO_VM_PAGE(l2pte_pa(pte)); pte &= ~L2_S_PROT_W; *ptep = pte; PTE_SYNC(ptep); if (!(pg->oflags & VPO_UNMANAGED)) { f = pmap_modify_pv(pg, pm, sva, PVF_WRITE, 0); if (f & PVF_WRITE) vm_page_dirty(pg); } else f = 0; if (flush >= 0) { flush++; flags |= f; } else if (PV_BEEN_EXECD(f)) pmap_tlb_flushID_SE(pm, sva); else if (PV_BEEN_REFD(f)) pmap_tlb_flushD_SE(pm, sva); } sva += PAGE_SIZE; ptep++; } } if (flush) { if (PV_BEEN_EXECD(flags)) pmap_tlb_flushID(pm); else if (PV_BEEN_REFD(flags)) pmap_tlb_flushD(pm); } rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(pm); } /* * Insert the given physical page (p) at * the specified virtual address (v) in the * target physical map with the protection requested. * * If specified, the page will be wired down, meaning * that the related pte can not be reclaimed. * * NB: This is the only routine which MAY NOT lazy-evaluate * or lose information. That is, this routine must actually * insert this page into the given map NOW. */ int pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, u_int flags, int8_t psind __unused) { int rv; rw_wlock(&pvh_global_lock); PMAP_LOCK(pmap); rv = pmap_enter_locked(pmap, va, m, prot, flags); rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(pmap); return (rv); } /* * The pvh global and pmap locks must be held. */ static int pmap_enter_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, u_int flags) { struct l2_bucket *l2b = NULL; struct vm_page *opg; struct pv_entry *pve = NULL; pt_entry_t *ptep, npte, opte; u_int nflags; u_int oflags; vm_paddr_t pa; PMAP_ASSERT_LOCKED(pmap); rw_assert(&pvh_global_lock, RA_WLOCKED); if (va == vector_page) { pa = systempage.pv_pa; m = NULL; } else { if ((m->oflags & VPO_UNMANAGED) == 0 && !vm_page_xbusied(m)) VM_OBJECT_ASSERT_LOCKED(m->object); pa = VM_PAGE_TO_PHYS(m); } nflags = 0; if (prot & VM_PROT_WRITE) nflags |= PVF_WRITE; if (prot & VM_PROT_EXECUTE) nflags |= PVF_EXEC; if ((flags & PMAP_ENTER_WIRED) != 0) nflags |= PVF_WIRED; PDEBUG(1, printf("pmap_enter: pmap = %08x, va = %08x, m = %08x, prot = %x, " "flags = %x\n", (uint32_t) pmap, va, (uint32_t) m, prot, flags)); if (pmap == kernel_pmap) { l2b = pmap_get_l2_bucket(pmap, va); if (l2b == NULL) l2b = pmap_grow_l2_bucket(pmap, va); } else { do_l2b_alloc: l2b = pmap_alloc_l2_bucket(pmap, va); if (l2b == NULL) { if ((flags & PMAP_ENTER_NOSLEEP) == 0) { PMAP_UNLOCK(pmap); rw_wunlock(&pvh_global_lock); VM_WAIT; rw_wlock(&pvh_global_lock); PMAP_LOCK(pmap); goto do_l2b_alloc; } return (KERN_RESOURCE_SHORTAGE); } } ptep = &l2b->l2b_kva[l2pte_index(va)]; opte = *ptep; npte = pa; oflags = 0; if (opte) { /* * There is already a mapping at this address. * If the physical address is different, lookup the * vm_page. */ if (l2pte_pa(opte) != pa) opg = PHYS_TO_VM_PAGE(l2pte_pa(opte)); else opg = m; } else opg = NULL; if ((prot & (VM_PROT_ALL)) || (!m || m->md.pvh_attrs & PVF_REF)) { /* * - The access type indicates that we don't need * to do referenced emulation. * OR * - The physical page has already been referenced * so no need to re-do referenced emulation here. */ npte |= L2_S_PROTO; nflags |= PVF_REF; if (m && ((prot & VM_PROT_WRITE) != 0 || (m->md.pvh_attrs & PVF_MOD))) { /* * This is a writable mapping, and the * page's mod state indicates it has * already been modified. Make it * writable from the outset. */ nflags |= PVF_MOD; if (!(m->md.pvh_attrs & PVF_MOD)) vm_page_dirty(m); } if (m && opte) vm_page_aflag_set(m, PGA_REFERENCED); } else { /* * Need to do page referenced emulation. */ npte |= L2_TYPE_INV; } if (prot & VM_PROT_WRITE) { npte |= L2_S_PROT_W; if (m != NULL && (m->oflags & VPO_UNMANAGED) == 0) vm_page_aflag_set(m, PGA_WRITEABLE); } if (m->md.pv_memattr != VM_MEMATTR_UNCACHEABLE) npte |= pte_l2_s_cache_mode; if (m && m == opg) { /* * We're changing the attrs of an existing mapping. */ oflags = pmap_modify_pv(m, pmap, va, PVF_WRITE | PVF_EXEC | PVF_WIRED | PVF_MOD | PVF_REF, nflags); /* * We may need to flush the cache if we're * doing rw-ro... */ if (pmap_is_current(pmap) && (oflags & PVF_NC) == 0 && (opte & L2_S_PROT_W) != 0 && (prot & VM_PROT_WRITE) == 0 && (opte & L2_TYPE_MASK) != L2_TYPE_INV) { cpu_dcache_wb_range(va, PAGE_SIZE); cpu_l2cache_wb_range(va, PAGE_SIZE); } } else { /* * New mapping, or changing the backing page * of an existing mapping. */ if (opg) { /* * Replacing an existing mapping with a new one. * It is part of our managed memory so we * must remove it from the PV list */ if ((pve = pmap_remove_pv(opg, pmap, va))) { /* note for patch: the oflags/invalidation was moved * because PG_FICTITIOUS pages could free the pve */ oflags = pve->pv_flags; /* * If the old mapping was valid (ref/mod * emulation creates 'invalid' mappings * initially) then make sure to frob * the cache. */ if ((oflags & PVF_NC) == 0 && l2pte_valid(opte)) { if (PV_BEEN_EXECD(oflags)) { pmap_idcache_wbinv_range(pmap, va, PAGE_SIZE); } else if (PV_BEEN_REFD(oflags)) { pmap_dcache_wb_range(pmap, va, PAGE_SIZE, TRUE, (oflags & PVF_WRITE) == 0); } } /* free/allocate a pv_entry for UNMANAGED pages if * this physical page is not/is already mapped. */ if (m && (m->oflags & VPO_UNMANAGED) && !m->md.pv_kva && TAILQ_EMPTY(&m->md.pv_list)) { pmap_free_pv_entry(pve); pve = NULL; } } else if (m && (!(m->oflags & VPO_UNMANAGED) || m->md.pv_kva || !TAILQ_EMPTY(&m->md.pv_list))) pve = pmap_get_pv_entry(); } else if (m && (!(m->oflags & VPO_UNMANAGED) || m->md.pv_kva || !TAILQ_EMPTY(&m->md.pv_list))) pve = pmap_get_pv_entry(); if (m) { if ((m->oflags & VPO_UNMANAGED)) { if (!TAILQ_EMPTY(&m->md.pv_list) || m->md.pv_kva) { KASSERT(pve != NULL, ("No pv")); nflags |= PVF_UNMAN; pmap_enter_pv(m, pve, pmap, va, nflags); } else m->md.pv_kva = va; } else { KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva, ("pmap_enter: managed mapping within the clean submap")); KASSERT(pve != NULL, ("No pv")); pmap_enter_pv(m, pve, pmap, va, nflags); } } } /* * Make sure userland mappings get the right permissions */ if (pmap != kernel_pmap && va != vector_page) { npte |= L2_S_PROT_U; } /* * Keep the stats up to date */ if (opte == 0) { l2b->l2b_occupancy++; pmap->pm_stats.resident_count++; } /* * If this is just a wiring change, the two PTEs will be * identical, so there's no need to update the page table. */ if (npte != opte) { boolean_t is_cached = pmap_is_current(pmap); *ptep = npte; if (is_cached) { /* * We only need to frob the cache/tlb if this pmap * is current */ PTE_SYNC(ptep); if (L1_IDX(va) != L1_IDX(vector_page) && l2pte_valid(npte)) { /* * This mapping is likely to be accessed as * soon as we return to userland. Fix up the * L1 entry to avoid taking another * page/domain fault. */ pd_entry_t *pl1pd, l1pd; pl1pd = &pmap->pm_l1->l1_kva[L1_IDX(va)]; l1pd = l2b->l2b_phys | L1_C_DOM(pmap->pm_domain) | L1_C_PROTO; if (*pl1pd != l1pd) { *pl1pd = l1pd; PTE_SYNC(pl1pd); } } } if (PV_BEEN_EXECD(oflags)) pmap_tlb_flushID_SE(pmap, va); else if (PV_BEEN_REFD(oflags)) pmap_tlb_flushD_SE(pmap, va); if (m) pmap_fix_cache(m, pmap, va); } return (KERN_SUCCESS); } /* * Maps a sequence of resident pages belonging to the same object. * The sequence begins with the given page m_start. This page is * mapped at the given virtual address start. Each subsequent page is * mapped at a virtual address that is offset from start by the same * amount as the page is offset from m_start within the object. The * last page in the sequence is the page with the largest offset from * m_start that can be mapped at a virtual address less than the given * virtual address end. Not every virtual page between start and end * is mapped; only those for which a resident page exists with the * corresponding offset from m_start are mapped. */ void pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, vm_page_t m_start, vm_prot_t prot) { vm_page_t m; vm_pindex_t diff, psize; VM_OBJECT_ASSERT_LOCKED(m_start->object); psize = atop(end - start); m = m_start; rw_wlock(&pvh_global_lock); PMAP_LOCK(pmap); while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { pmap_enter_locked(pmap, start + ptoa(diff), m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), PMAP_ENTER_NOSLEEP); m = TAILQ_NEXT(m, listq); } rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(pmap); } /* * this code makes some *MAJOR* assumptions: * 1. Current pmap & pmap exists. * 2. Not wired. * 3. Read access. * 4. No page table pages. * but is *MUCH* faster than pmap_enter... */ void pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot) { rw_wlock(&pvh_global_lock); PMAP_LOCK(pmap); pmap_enter_locked(pmap, va, m, prot & (VM_PROT_READ | VM_PROT_EXECUTE), PMAP_ENTER_NOSLEEP); rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(pmap); } /* * Clear the wired attribute from the mappings for the specified range of * addresses in the given pmap. Every valid mapping within that range * must have the wired attribute set. In contrast, invalid mappings * cannot have the wired attribute set, so they are ignored. * * XXX Wired mappings of unmanaged pages cannot be counted by this pmap * implementation. */ void pmap_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) { struct l2_bucket *l2b; pt_entry_t *ptep, pte; pv_entry_t pv; vm_offset_t next_bucket; vm_page_t m; rw_wlock(&pvh_global_lock); PMAP_LOCK(pmap); while (sva < eva) { next_bucket = L2_NEXT_BUCKET(sva); if (next_bucket > eva) next_bucket = eva; l2b = pmap_get_l2_bucket(pmap, sva); if (l2b == NULL) { sva = next_bucket; continue; } for (ptep = &l2b->l2b_kva[l2pte_index(sva)]; sva < next_bucket; sva += PAGE_SIZE, ptep++) { if ((pte = *ptep) == 0 || (m = PHYS_TO_VM_PAGE(l2pte_pa(pte))) == NULL || (m->oflags & VPO_UNMANAGED) != 0) continue; pv = pmap_find_pv(m, pmap, sva); if ((pv->pv_flags & PVF_WIRED) == 0) panic("pmap_unwire: pv %p isn't wired", pv); pv->pv_flags &= ~PVF_WIRED; pmap->pm_stats.wired_count--; } } rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(pmap); } /* * Copy the range specified by src_addr/len * from the source map to the range dst_addr/len * in the destination map. * * This routine is only advisory and need not do anything. */ void pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len, vm_offset_t src_addr) { } /* * Routine: pmap_extract * Function: * Extract the physical page address associated * with the given map/virtual_address pair. */ vm_paddr_t pmap_extract(pmap_t pmap, vm_offset_t va) { vm_paddr_t pa; PMAP_LOCK(pmap); pa = pmap_extract_locked(pmap, va); PMAP_UNLOCK(pmap); return (pa); } static vm_paddr_t pmap_extract_locked(pmap_t pmap, vm_offset_t va) { struct l2_dtable *l2; pd_entry_t l1pd; pt_entry_t *ptep, pte; vm_paddr_t pa; u_int l1idx; if (pmap != kernel_pmap) PMAP_ASSERT_LOCKED(pmap); l1idx = L1_IDX(va); l1pd = pmap->pm_l1->l1_kva[l1idx]; if (l1pte_section_p(l1pd)) { /* * These should only happen for the kernel pmap. */ KASSERT(pmap == kernel_pmap, ("unexpected section")); /* XXX: what to do about the bits > 32 ? */ if (l1pd & L1_S_SUPERSEC) pa = (l1pd & L1_SUP_FRAME) | (va & L1_SUP_OFFSET); else pa = (l1pd & L1_S_FRAME) | (va & L1_S_OFFSET); } else { /* * Note that we can't rely on the validity of the L1 * descriptor as an indication that a mapping exists. * We have to look it up in the L2 dtable. */ l2 = pmap->pm_l2[L2_IDX(l1idx)]; if (l2 == NULL || (ptep = l2->l2_bucket[L2_BUCKET(l1idx)].l2b_kva) == NULL) return (0); pte = ptep[l2pte_index(va)]; if (pte == 0) return (0); if ((pte & L2_TYPE_MASK) == L2_TYPE_L) pa = (pte & L2_L_FRAME) | (va & L2_L_OFFSET); else pa = (pte & L2_S_FRAME) | (va & L2_S_OFFSET); } return (pa); } /* * Atomically extract and hold the physical page with the given * pmap and virtual address pair if that mapping permits the given * protection. * */ vm_page_t pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot) { struct l2_dtable *l2; pd_entry_t l1pd; pt_entry_t *ptep, pte; vm_paddr_t pa, paddr; vm_page_t m = NULL; u_int l1idx; l1idx = L1_IDX(va); paddr = 0; PMAP_LOCK(pmap); retry: l1pd = pmap->pm_l1->l1_kva[l1idx]; if (l1pte_section_p(l1pd)) { /* * These should only happen for kernel_pmap */ KASSERT(pmap == kernel_pmap, ("huh")); /* XXX: what to do about the bits > 32 ? */ if (l1pd & L1_S_SUPERSEC) pa = (l1pd & L1_SUP_FRAME) | (va & L1_SUP_OFFSET); else pa = (l1pd & L1_S_FRAME) | (va & L1_S_OFFSET); if (vm_page_pa_tryrelock(pmap, pa & PG_FRAME, &paddr)) goto retry; if (l1pd & L1_S_PROT_W || (prot & VM_PROT_WRITE) == 0) { m = PHYS_TO_VM_PAGE(pa); vm_page_hold(m); } } else { /* * Note that we can't rely on the validity of the L1 * descriptor as an indication that a mapping exists. * We have to look it up in the L2 dtable. */ l2 = pmap->pm_l2[L2_IDX(l1idx)]; if (l2 == NULL || (ptep = l2->l2_bucket[L2_BUCKET(l1idx)].l2b_kva) == NULL) { PMAP_UNLOCK(pmap); return (NULL); } ptep = &ptep[l2pte_index(va)]; pte = *ptep; if (pte == 0) { PMAP_UNLOCK(pmap); return (NULL); } if (pte & L2_S_PROT_W || (prot & VM_PROT_WRITE) == 0) { if ((pte & L2_TYPE_MASK) == L2_TYPE_L) pa = (pte & L2_L_FRAME) | (va & L2_L_OFFSET); else pa = (pte & L2_S_FRAME) | (va & L2_S_OFFSET); if (vm_page_pa_tryrelock(pmap, pa & PG_FRAME, &paddr)) goto retry; m = PHYS_TO_VM_PAGE(pa); vm_page_hold(m); } } PMAP_UNLOCK(pmap); PA_UNLOCK_COND(paddr); return (m); } vm_paddr_t pmap_dump_kextract(vm_offset_t va, pt2_entry_t *pte2p) { struct l2_dtable *l2; pd_entry_t l1pd; pt_entry_t *ptep, pte; vm_paddr_t pa; u_int l1idx; l1idx = L1_IDX(va); l1pd = kernel_pmap->pm_l1->l1_kva[l1idx]; if (l1pte_section_p(l1pd)) { if (l1pd & L1_S_SUPERSEC) pa = (l1pd & L1_SUP_FRAME) | (va & L1_SUP_OFFSET); else pa = (l1pd & L1_S_FRAME) | (va & L1_S_OFFSET); pte = L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, VM_PROT_READ | VM_PROT_WRITE); } else { l2 = kernel_pmap->pm_l2[L2_IDX(l1idx)]; if (l2 == NULL || (ptep = l2->l2_bucket[L2_BUCKET(l1idx)].l2b_kva) == NULL) { pte = 0; pa = 0; goto out; } pte = ptep[l2pte_index(va)]; if (pte == 0) { pa = 0; goto out; } if ((pte & L2_TYPE_MASK) == L2_TYPE_L) pa = (pte & L2_L_FRAME) | (va & L2_L_OFFSET); else pa = (pte & L2_S_FRAME) | (va & L2_S_OFFSET); } out: if (pte2p != NULL) *pte2p = pte; return (pa); } /* * Initialize a preallocated and zeroed pmap structure, * such as one in a vmspace structure. */ int pmap_pinit(pmap_t pmap) { PDEBUG(1, printf("pmap_pinit: pmap = %08x\n", (uint32_t) pmap)); pmap_alloc_l1(pmap); bzero(pmap->pm_l2, sizeof(pmap->pm_l2)); CPU_ZERO(&pmap->pm_active); TAILQ_INIT(&pmap->pm_pvlist); bzero(&pmap->pm_stats, sizeof pmap->pm_stats); pmap->pm_stats.resident_count = 1; if (vector_page < KERNBASE) { pmap_enter(pmap, vector_page, PHYS_TO_VM_PAGE(systempage.pv_pa), VM_PROT_READ, PMAP_ENTER_WIRED | VM_PROT_READ, 0); } return (1); } /*************************************************** * page management routines. ***************************************************/ static void pmap_free_pv_entry(pv_entry_t pv) { pv_entry_count--; uma_zfree(pvzone, pv); } /* * get a new pv_entry, allocating a block from the system * when needed. * the memory allocation is performed bypassing the malloc code * because of the possibility of allocations at interrupt time. */ static pv_entry_t pmap_get_pv_entry(void) { pv_entry_t ret_value; pv_entry_count++; if (pv_entry_count > pv_entry_high_water) pagedaemon_wakeup(); ret_value = uma_zalloc(pvzone, M_NOWAIT); return ret_value; } /* * Remove the given range of addresses from the specified map. * * It is assumed that the start and end are properly * rounded to the page size. */ #define PMAP_REMOVE_CLEAN_LIST_SIZE 3 void pmap_remove(pmap_t pm, vm_offset_t sva, vm_offset_t eva) { struct l2_bucket *l2b; vm_offset_t next_bucket; pt_entry_t *ptep; u_int total; u_int mappings, is_exec, is_refd; int flushall = 0; /* * we lock in the pmap => pv_head direction */ rw_wlock(&pvh_global_lock); PMAP_LOCK(pm); total = 0; while (sva < eva) { /* * Do one L2 bucket's worth at a time. */ next_bucket = L2_NEXT_BUCKET(sva); if (next_bucket > eva) next_bucket = eva; l2b = pmap_get_l2_bucket(pm, sva); if (l2b == NULL) { sva = next_bucket; continue; } ptep = &l2b->l2b_kva[l2pte_index(sva)]; mappings = 0; while (sva < next_bucket) { struct vm_page *pg; pt_entry_t pte; vm_paddr_t pa; pte = *ptep; if (pte == 0) { /* * Nothing here, move along */ sva += PAGE_SIZE; ptep++; continue; } pm->pm_stats.resident_count--; pa = l2pte_pa(pte); is_exec = 0; is_refd = 1; /* * Update flags. In a number of circumstances, * we could cluster a lot of these and do a * number of sequential pages in one go. */ if ((pg = PHYS_TO_VM_PAGE(pa)) != NULL) { struct pv_entry *pve; pve = pmap_remove_pv(pg, pm, sva); if (pve) { is_exec = PV_BEEN_EXECD(pve->pv_flags); is_refd = PV_BEEN_REFD(pve->pv_flags); pmap_free_pv_entry(pve); } } if (l2pte_valid(pte) && pmap_is_current(pm)) { if (total < PMAP_REMOVE_CLEAN_LIST_SIZE) { total++; if (is_exec) { cpu_idcache_wbinv_range(sva, PAGE_SIZE); cpu_l2cache_wbinv_range(sva, PAGE_SIZE); cpu_tlb_flushID_SE(sva); } else if (is_refd) { cpu_dcache_wbinv_range(sva, PAGE_SIZE); cpu_l2cache_wbinv_range(sva, PAGE_SIZE); cpu_tlb_flushD_SE(sva); } } else if (total == PMAP_REMOVE_CLEAN_LIST_SIZE) { /* flushall will also only get set for * for a current pmap */ cpu_idcache_wbinv_all(); cpu_l2cache_wbinv_all(); flushall = 1; total++; } } *ptep = 0; PTE_SYNC(ptep); sva += PAGE_SIZE; ptep++; mappings++; } pmap_free_l2_bucket(pm, l2b, mappings); } rw_wunlock(&pvh_global_lock); if (flushall) cpu_tlb_flushID(); PMAP_UNLOCK(pm); } /* * pmap_zero_page() * * Zero a given physical page by mapping it at a page hook point. * In doing the zero page op, the page we zero is mapped cachable, as with * StrongARM accesses to non-cached pages are non-burst making writing * _any_ bulk data very slow. */ #if ARM_MMU_GENERIC != 0 || defined(CPU_XSCALE_CORE3) void pmap_zero_page_generic(vm_paddr_t phys, int off, int size) { if (_arm_bzero && size >= _min_bzero_size && _arm_bzero((void *)(phys + off), size, IS_PHYSICAL) == 0) return; mtx_lock(&cmtx); /* * Hook in the page, zero it, invalidate the TLB as needed. * * Note the temporary zero-page mapping must be a non-cached page in * order to work without corruption when write-allocate is enabled. */ *cdst_pte = L2_S_PROTO | phys | L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE); PTE_SYNC(cdst_pte); cpu_tlb_flushD_SE(cdstp); cpu_cpwait(); if (off || size != PAGE_SIZE) bzero((void *)(cdstp + off), size); else bzero_page(cdstp); mtx_unlock(&cmtx); } #endif /* ARM_MMU_GENERIC != 0 */ #if ARM_MMU_XSCALE == 1 void pmap_zero_page_xscale(vm_paddr_t phys, int off, int size) { if (_arm_bzero && size >= _min_bzero_size && _arm_bzero((void *)(phys + off), size, IS_PHYSICAL) == 0) return; mtx_lock(&cmtx); /* * Hook in the page, zero it, and purge the cache for that * zeroed page. Invalidate the TLB as needed. */ *cdst_pte = L2_S_PROTO | phys | L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ PTE_SYNC(cdst_pte); cpu_tlb_flushD_SE(cdstp); cpu_cpwait(); if (off || size != PAGE_SIZE) bzero((void *)(cdstp + off), size); else bzero_page(cdstp); mtx_unlock(&cmtx); xscale_cache_clean_minidata(); } /* * Change the PTEs for the specified kernel mappings such that they * will use the mini data cache instead of the main data cache. */ void pmap_use_minicache(vm_offset_t va, vm_size_t size) { struct l2_bucket *l2b; pt_entry_t *ptep, *sptep, pte; vm_offset_t next_bucket, eva; #if (ARM_NMMUS > 1) || defined(CPU_XSCALE_CORE3) if (xscale_use_minidata == 0) return; #endif eva = va + size; while (va < eva) { next_bucket = L2_NEXT_BUCKET(va); if (next_bucket > eva) next_bucket = eva; l2b = pmap_get_l2_bucket(kernel_pmap, va); sptep = ptep = &l2b->l2b_kva[l2pte_index(va)]; while (va < next_bucket) { pte = *ptep; if (!l2pte_minidata(pte)) { cpu_dcache_wbinv_range(va, PAGE_SIZE); cpu_tlb_flushD_SE(va); *ptep = pte & ~L2_B; } ptep++; va += PAGE_SIZE; } PTE_SYNC_RANGE(sptep, (u_int)(ptep - sptep)); } cpu_cpwait(); } #endif /* ARM_MMU_XSCALE == 1 */ /* * pmap_zero_page zeros the specified hardware page by mapping * the page into KVM and using bzero to clear its contents. */ void pmap_zero_page(vm_page_t m) { pmap_zero_page_func(VM_PAGE_TO_PHYS(m), 0, PAGE_SIZE); } /* * pmap_zero_page_area zeros the specified hardware page by mapping * the page into KVM and using bzero to clear its contents. * * off and size may not cover an area beyond a single hardware page. */ void pmap_zero_page_area(vm_page_t m, int off, int size) { pmap_zero_page_func(VM_PAGE_TO_PHYS(m), off, size); } /* * pmap_zero_page_idle zeros the specified hardware page by mapping * the page into KVM and using bzero to clear its contents. This * is intended to be called from the vm_pagezero process only and * outside of Giant. */ void pmap_zero_page_idle(vm_page_t m) { pmap_zero_page(m); } #if 0 /* * pmap_clean_page() * * This is a local function used to work out the best strategy to clean * a single page referenced by its entry in the PV table. It should be used by * pmap_copy_page, pmap_zero page and maybe some others later on. * * Its policy is effectively: * o If there are no mappings, we don't bother doing anything with the cache. * o If there is one mapping, we clean just that page. * o If there are multiple mappings, we clean the entire cache. * * So that some functions can be further optimised, it returns 0 if it didn't * clean the entire cache, or 1 if it did. * * XXX One bug in this routine is that if the pv_entry has a single page * mapped at 0x00000000 a whole cache clean will be performed rather than * just the 1 page. Since this should not occur in everyday use and if it does * it will just result in not the most efficient clean for the page. * * We don't yet use this function but may want to. */ static int pmap_clean_page(struct pv_entry *pv, boolean_t is_src) { pmap_t pm, pm_to_clean = NULL; struct pv_entry *npv; u_int cache_needs_cleaning = 0; u_int flags = 0; vm_offset_t page_to_clean = 0; if (pv == NULL) { /* nothing mapped in so nothing to flush */ return (0); } /* * Since we flush the cache each time we change to a different * user vmspace, we only need to flush the page if it is in the * current pmap. */ if (curthread) pm = vmspace_pmap(curproc->p_vmspace); else pm = kernel_pmap; for (npv = pv; npv; npv = TAILQ_NEXT(npv, pv_list)) { if (npv->pv_pmap == kernel_pmap || npv->pv_pmap == pm) { flags |= npv->pv_flags; /* * The page is mapped non-cacheable in * this map. No need to flush the cache. */ if (npv->pv_flags & PVF_NC) { #ifdef DIAGNOSTIC if (cache_needs_cleaning) panic("pmap_clean_page: " "cache inconsistency"); #endif break; } else if (is_src && (npv->pv_flags & PVF_WRITE) == 0) continue; if (cache_needs_cleaning) { page_to_clean = 0; break; } else { page_to_clean = npv->pv_va; pm_to_clean = npv->pv_pmap; } cache_needs_cleaning = 1; } } if (page_to_clean) { if (PV_BEEN_EXECD(flags)) pmap_idcache_wbinv_range(pm_to_clean, page_to_clean, PAGE_SIZE); else pmap_dcache_wb_range(pm_to_clean, page_to_clean, PAGE_SIZE, !is_src, (flags & PVF_WRITE) == 0); } else if (cache_needs_cleaning) { if (PV_BEEN_EXECD(flags)) pmap_idcache_wbinv_all(pm); else pmap_dcache_wbinv_all(pm); return (1); } return (0); } #endif /* * pmap_copy_page copies the specified (machine independent) * page by mapping the page into virtual memory and using * bcopy to copy the page, one machine dependent page at a * time. */ /* * pmap_copy_page() * * Copy one physical page into another, by mapping the pages into * hook points. The same comment regarding cachability as in * pmap_zero_page also applies here. */ #if ARM_MMU_GENERIC != 0 || defined (CPU_XSCALE_CORE3) void pmap_copy_page_generic(vm_paddr_t src, vm_paddr_t dst) { #if 0 struct vm_page *src_pg = PHYS_TO_VM_PAGE(src); #endif /* * Clean the source page. Hold the source page's lock for * the duration of the copy so that no other mappings can * be created while we have a potentially aliased mapping. */ #if 0 /* * XXX: Not needed while we call cpu_dcache_wbinv_all() in * pmap_copy_page(). */ (void) pmap_clean_page(TAILQ_FIRST(&src_pg->md.pv_list), TRUE); #endif /* * Map the pages into the page hook points, copy them, and purge * the cache for the appropriate page. Invalidate the TLB * as required. */ mtx_lock(&cmtx); *csrc_pte = L2_S_PROTO | src | L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | pte_l2_s_cache_mode; PTE_SYNC(csrc_pte); *cdst_pte = L2_S_PROTO | dst | L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; PTE_SYNC(cdst_pte); cpu_tlb_flushD_SE(csrcp); cpu_tlb_flushD_SE(cdstp); cpu_cpwait(); bcopy_page(csrcp, cdstp); mtx_unlock(&cmtx); cpu_dcache_inv_range(csrcp, PAGE_SIZE); cpu_dcache_wbinv_range(cdstp, PAGE_SIZE); cpu_l2cache_inv_range(csrcp, PAGE_SIZE); cpu_l2cache_wbinv_range(cdstp, PAGE_SIZE); } void pmap_copy_page_offs_generic(vm_paddr_t a_phys, vm_offset_t a_offs, vm_paddr_t b_phys, vm_offset_t b_offs, int cnt) { mtx_lock(&cmtx); *csrc_pte = L2_S_PROTO | a_phys | L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | pte_l2_s_cache_mode; PTE_SYNC(csrc_pte); *cdst_pte = L2_S_PROTO | b_phys | L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | pte_l2_s_cache_mode; PTE_SYNC(cdst_pte); cpu_tlb_flushD_SE(csrcp); cpu_tlb_flushD_SE(cdstp); cpu_cpwait(); bcopy((char *)csrcp + a_offs, (char *)cdstp + b_offs, cnt); mtx_unlock(&cmtx); cpu_dcache_inv_range(csrcp + a_offs, cnt); cpu_dcache_wbinv_range(cdstp + b_offs, cnt); cpu_l2cache_inv_range(csrcp + a_offs, cnt); cpu_l2cache_wbinv_range(cdstp + b_offs, cnt); } #endif /* ARM_MMU_GENERIC != 0 */ #if ARM_MMU_XSCALE == 1 void pmap_copy_page_xscale(vm_paddr_t src, vm_paddr_t dst) { #if 0 /* XXX: Only needed for pmap_clean_page(), which is commented out. */ struct vm_page *src_pg = PHYS_TO_VM_PAGE(src); #endif /* * Clean the source page. Hold the source page's lock for * the duration of the copy so that no other mappings can * be created while we have a potentially aliased mapping. */ #if 0 /* * XXX: Not needed while we call cpu_dcache_wbinv_all() in * pmap_copy_page(). */ (void) pmap_clean_page(TAILQ_FIRST(&src_pg->md.pv_list), TRUE); #endif /* * Map the pages into the page hook points, copy them, and purge * the cache for the appropriate page. Invalidate the TLB * as required. */ mtx_lock(&cmtx); *csrc_pte = L2_S_PROTO | src | L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ PTE_SYNC(csrc_pte); *cdst_pte = L2_S_PROTO | dst | L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); /* mini-data */ PTE_SYNC(cdst_pte); cpu_tlb_flushD_SE(csrcp); cpu_tlb_flushD_SE(cdstp); cpu_cpwait(); bcopy_page(csrcp, cdstp); mtx_unlock(&cmtx); xscale_cache_clean_minidata(); } void pmap_copy_page_offs_xscale(vm_paddr_t a_phys, vm_offset_t a_offs, vm_paddr_t b_phys, vm_offset_t b_offs, int cnt) { mtx_lock(&cmtx); *csrc_pte = L2_S_PROTO | a_phys | L2_S_PROT(PTE_KERNEL, VM_PROT_READ) | L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); PTE_SYNC(csrc_pte); *cdst_pte = L2_S_PROTO | b_phys | L2_S_PROT(PTE_KERNEL, VM_PROT_WRITE) | L2_C | L2_XSCALE_T_TEX(TEX_XSCALE_X); PTE_SYNC(cdst_pte); cpu_tlb_flushD_SE(csrcp); cpu_tlb_flushD_SE(cdstp); cpu_cpwait(); bcopy((char *)csrcp + a_offs, (char *)cdstp + b_offs, cnt); mtx_unlock(&cmtx); xscale_cache_clean_minidata(); } #endif /* ARM_MMU_XSCALE == 1 */ void pmap_copy_page(vm_page_t src, vm_page_t dst) { cpu_dcache_wbinv_all(); cpu_l2cache_wbinv_all(); if (_arm_memcpy && PAGE_SIZE >= _min_memcpy_size && _arm_memcpy((void *)VM_PAGE_TO_PHYS(dst), (void *)VM_PAGE_TO_PHYS(src), PAGE_SIZE, IS_PHYSICAL) == 0) return; pmap_copy_page_func(VM_PAGE_TO_PHYS(src), VM_PAGE_TO_PHYS(dst)); } /* * We have code to do unmapped I/O. However, it isn't quite right and * causes un-page-aligned I/O to devices to fail (most notably newfs * or fsck). We give up a little performance to not allow unmapped I/O * to gain stability. */ int unmapped_buf_allowed = 0; void pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[], vm_offset_t b_offset, int xfersize) { vm_page_t a_pg, b_pg; vm_offset_t a_pg_offset, b_pg_offset; int cnt; cpu_dcache_wbinv_all(); cpu_l2cache_wbinv_all(); while (xfersize > 0) { a_pg = ma[a_offset >> PAGE_SHIFT]; a_pg_offset = a_offset & PAGE_MASK; cnt = min(xfersize, PAGE_SIZE - a_pg_offset); b_pg = mb[b_offset >> PAGE_SHIFT]; b_pg_offset = b_offset & PAGE_MASK; cnt = min(cnt, PAGE_SIZE - b_pg_offset); pmap_copy_page_offs_func(VM_PAGE_TO_PHYS(a_pg), a_pg_offset, VM_PAGE_TO_PHYS(b_pg), b_pg_offset, cnt); xfersize -= cnt; a_offset += cnt; b_offset += cnt; } } vm_offset_t pmap_quick_enter_page(vm_page_t m) { /* * Don't bother with a PCPU pageframe, since we don't support * SMP for anything pre-armv7. Use pmap_kenter() to ensure * caching is handled correctly for multiple mappings of the * same physical page. */ mtx_assert(&qmap_mtx, MA_NOTOWNED); mtx_lock(&qmap_mtx); pmap_kenter(qmap_addr, VM_PAGE_TO_PHYS(m)); return (qmap_addr); } void pmap_quick_remove_page(vm_offset_t addr) { KASSERT(addr == qmap_addr, ("pmap_quick_remove_page: invalid address")); mtx_assert(&qmap_mtx, MA_OWNED); pmap_kremove(addr); mtx_unlock(&qmap_mtx); } /* * this routine returns true if a physical page resides * in the given pmap. */ boolean_t pmap_page_exists_quick(pmap_t pmap, vm_page_t m) { pv_entry_t pv; int loops = 0; boolean_t rv; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_page_exists_quick: page %p is not managed", m)); rv = FALSE; rw_wlock(&pvh_global_lock); TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { if (pv->pv_pmap == pmap) { rv = TRUE; break; } loops++; if (loops >= 16) break; } rw_wunlock(&pvh_global_lock); return (rv); } /* * pmap_page_wired_mappings: * * Return the number of managed mappings to the given physical page * that are wired. */ int pmap_page_wired_mappings(vm_page_t m) { pv_entry_t pv; int count; count = 0; if ((m->oflags & VPO_UNMANAGED) != 0) return (count); rw_wlock(&pvh_global_lock); TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) if ((pv->pv_flags & PVF_WIRED) != 0) count++; rw_wunlock(&pvh_global_lock); return (count); } /* * This function is advisory. */ void pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice) { } /* * pmap_ts_referenced: * * Return the count of reference bits for a page, clearing all of them. */ int pmap_ts_referenced(vm_page_t m) { KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_ts_referenced: page %p is not managed", m)); return (pmap_clearbit(m, PVF_REF)); } boolean_t pmap_is_modified(vm_page_t m) { KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_is_modified: page %p is not managed", m)); if (m->md.pvh_attrs & PVF_MOD) return (TRUE); return(FALSE); } /* * Clear the modify bits on the specified physical page. */ void pmap_clear_modify(vm_page_t m) { KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_clear_modify: page %p is not managed", m)); VM_OBJECT_ASSERT_WLOCKED(m->object); KASSERT(!vm_page_xbusied(m), ("pmap_clear_modify: page %p is exclusive busied", m)); /* * If the page is not PGA_WRITEABLE, then no mappings can be modified. * If the object containing the page is locked and the page is not * exclusive busied, then PGA_WRITEABLE cannot be concurrently set. */ if ((m->aflags & PGA_WRITEABLE) == 0) return; if (m->md.pvh_attrs & PVF_MOD) pmap_clearbit(m, PVF_MOD); } /* * pmap_is_referenced: * * Return whether or not the specified physical page was referenced * in any physical maps. */ boolean_t pmap_is_referenced(vm_page_t m) { KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_is_referenced: page %p is not managed", m)); return ((m->md.pvh_attrs & PVF_REF) != 0); } /* * Clear the write and modified bits in each of the given page's mappings. */ void pmap_remove_write(vm_page_t m) { KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_remove_write: page %p is not managed", m)); /* * If the page is not exclusive busied, then PGA_WRITEABLE cannot be * set by another thread while the object is locked. Thus, * if PGA_WRITEABLE is clear, no page table entries need updating. */ VM_OBJECT_ASSERT_WLOCKED(m->object); if (vm_page_xbusied(m) || (m->aflags & PGA_WRITEABLE) != 0) pmap_clearbit(m, PVF_WRITE); } /* * perform the pmap work for mincore */ int pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa) { struct l2_bucket *l2b; pt_entry_t *ptep, pte; vm_paddr_t pa; vm_page_t m; int val; boolean_t managed; PMAP_LOCK(pmap); retry: l2b = pmap_get_l2_bucket(pmap, addr); if (l2b == NULL) { val = 0; goto out; } ptep = &l2b->l2b_kva[l2pte_index(addr)]; pte = *ptep; if (!l2pte_valid(pte)) { val = 0; goto out; } val = MINCORE_INCORE; if (pte & L2_S_PROT_W) val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER; managed = false; pa = l2pte_pa(pte); m = PHYS_TO_VM_PAGE(pa); if (m != NULL && !(m->oflags & VPO_UNMANAGED)) managed = true; if (managed) { /* * The ARM pmap tries to maintain a per-mapping * reference bit. The trouble is that it's kept in * the PV entry, not the PTE, so it's costly to access * here. You would need to acquire the pvh global * lock, call pmap_find_pv(), and introduce a custom * version of vm_page_pa_tryrelock() that releases and * reacquires the pvh global lock. In the end, I * doubt it's worthwhile. This may falsely report * the given address as referenced. */ if ((m->md.pvh_attrs & PVF_REF) != 0) val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER; } if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) != (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) && managed) { /* Ensure that "PHYS_TO_VM_PAGE(pa)->object" doesn't change. */ if (vm_page_pa_tryrelock(pmap, pa, locked_pa)) goto retry; } else out: PA_UNLOCK_COND(*locked_pa); PMAP_UNLOCK(pmap); return (val); } void pmap_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz) { } /* * Increase the starting virtual address of the given mapping if a * different alignment might result in more superpage mappings. */ void pmap_align_superpage(vm_object_t object, vm_ooffset_t offset, vm_offset_t *addr, vm_size_t size) { } #define BOOTSTRAP_DEBUG /* * pmap_map_section: * * Create a single section mapping. */ void pmap_map_section(vm_offset_t l1pt, vm_offset_t va, vm_offset_t pa, int prot, int cache) { pd_entry_t *pde = (pd_entry_t *) l1pt; pd_entry_t fl; KASSERT(((va | pa) & L1_S_OFFSET) == 0, ("ouin2")); switch (cache) { case PTE_NOCACHE: default: fl = 0; break; case PTE_CACHE: fl = pte_l1_s_cache_mode; break; case PTE_PAGETABLE: fl = pte_l1_s_cache_mode_pt; break; } pde[va >> L1_S_SHIFT] = L1_S_PROTO | pa | L1_S_PROT(PTE_KERNEL, prot) | fl | L1_S_DOM(PMAP_DOMAIN_KERNEL); PTE_SYNC(&pde[va >> L1_S_SHIFT]); } /* * pmap_link_l2pt: * * Link the L2 page table specified by l2pv.pv_pa into the L1 * page table at the slot for "va". */ void pmap_link_l2pt(vm_offset_t l1pt, vm_offset_t va, struct pv_addr *l2pv) { pd_entry_t *pde = (pd_entry_t *) l1pt, proto; u_int slot = va >> L1_S_SHIFT; proto = L1_S_DOM(PMAP_DOMAIN_KERNEL) | L1_C_PROTO; #ifdef VERBOSE_INIT_ARM printf("pmap_link_l2pt: pa=0x%x va=0x%x\n", l2pv->pv_pa, l2pv->pv_va); #endif pde[slot + 0] = proto | (l2pv->pv_pa + 0x000); PTE_SYNC(&pde[slot]); SLIST_INSERT_HEAD(&kernel_pt_list, l2pv, pv_list); } /* * pmap_map_entry * * Create a single page mapping. */ void pmap_map_entry(vm_offset_t l1pt, vm_offset_t va, vm_offset_t pa, int prot, int cache) { pd_entry_t *pde = (pd_entry_t *) l1pt; pt_entry_t fl; pt_entry_t *pte; KASSERT(((va | pa) & PAGE_MASK) == 0, ("ouin")); switch (cache) { case PTE_NOCACHE: default: fl = 0; break; case PTE_CACHE: fl = pte_l2_s_cache_mode; break; case PTE_PAGETABLE: fl = pte_l2_s_cache_mode_pt; break; } if ((pde[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C) panic("pmap_map_entry: no L2 table for VA 0x%08x", va); pte = (pt_entry_t *) kernel_pt_lookup(pde[L1_IDX(va)] & L1_C_ADDR_MASK); if (pte == NULL) panic("pmap_map_entry: can't find L2 table for VA 0x%08x", va); pte[l2pte_index(va)] = L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot) | fl; PTE_SYNC(&pte[l2pte_index(va)]); } /* * pmap_map_chunk: * * Map a chunk of memory using the most efficient mappings * possible (section. large page, small page) into the * provided L1 and L2 tables at the specified virtual address. */ vm_size_t pmap_map_chunk(vm_offset_t l1pt, vm_offset_t va, vm_offset_t pa, vm_size_t size, int prot, int cache) { pd_entry_t *pde = (pd_entry_t *) l1pt; pt_entry_t *pte, f1, f2s, f2l; vm_size_t resid; int i; resid = roundup2(size, PAGE_SIZE); if (l1pt == 0) panic("pmap_map_chunk: no L1 table provided"); #ifdef VERBOSE_INIT_ARM printf("pmap_map_chunk: pa=0x%x va=0x%x size=0x%x resid=0x%x " "prot=0x%x cache=%d\n", pa, va, size, resid, prot, cache); #endif switch (cache) { case PTE_NOCACHE: default: f1 = 0; f2l = 0; f2s = 0; break; case PTE_CACHE: f1 = pte_l1_s_cache_mode; f2l = pte_l2_l_cache_mode; f2s = pte_l2_s_cache_mode; break; case PTE_PAGETABLE: f1 = pte_l1_s_cache_mode_pt; f2l = pte_l2_l_cache_mode_pt; f2s = pte_l2_s_cache_mode_pt; break; } size = resid; while (resid > 0) { /* See if we can use a section mapping. */ if (L1_S_MAPPABLE_P(va, pa, resid)) { #ifdef VERBOSE_INIT_ARM printf("S"); #endif pde[va >> L1_S_SHIFT] = L1_S_PROTO | pa | L1_S_PROT(PTE_KERNEL, prot) | f1 | L1_S_DOM(PMAP_DOMAIN_KERNEL); PTE_SYNC(&pde[va >> L1_S_SHIFT]); va += L1_S_SIZE; pa += L1_S_SIZE; resid -= L1_S_SIZE; continue; } /* * Ok, we're going to use an L2 table. Make sure * one is actually in the corresponding L1 slot * for the current VA. */ if ((pde[va >> L1_S_SHIFT] & L1_TYPE_MASK) != L1_TYPE_C) panic("pmap_map_chunk: no L2 table for VA 0x%08x", va); pte = (pt_entry_t *) kernel_pt_lookup( pde[L1_IDX(va)] & L1_C_ADDR_MASK); if (pte == NULL) panic("pmap_map_chunk: can't find L2 table for VA" "0x%08x", va); /* See if we can use a L2 large page mapping. */ if (L2_L_MAPPABLE_P(va, pa, resid)) { #ifdef VERBOSE_INIT_ARM printf("L"); #endif for (i = 0; i < 16; i++) { pte[l2pte_index(va) + i] = L2_L_PROTO | pa | L2_L_PROT(PTE_KERNEL, prot) | f2l; PTE_SYNC(&pte[l2pte_index(va) + i]); } va += L2_L_SIZE; pa += L2_L_SIZE; resid -= L2_L_SIZE; continue; } /* Use a small page mapping. */ #ifdef VERBOSE_INIT_ARM printf("P"); #endif pte[l2pte_index(va)] = L2_S_PROTO | pa | L2_S_PROT(PTE_KERNEL, prot) | f2s; PTE_SYNC(&pte[l2pte_index(va)]); va += PAGE_SIZE; pa += PAGE_SIZE; resid -= PAGE_SIZE; } #ifdef VERBOSE_INIT_ARM printf("\n"); #endif return (size); } void pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma) { /* * Remember the memattr in a field that gets used to set the appropriate * bits in the PTEs as mappings are established. */ m->md.pv_memattr = ma; /* * It appears that this function can only be called before any mappings * for the page are established on ARM. If this ever changes, this code * will need to walk the pv_list and make each of the existing mappings * uncacheable, being careful to sync caches and PTEs (and maybe * invalidate TLB?) for any current mapping it modifies. */ if (m->md.pv_kva != 0 || TAILQ_FIRST(&m->md.pv_list) != NULL) panic("Can't change memattr on page with existing mappings"); } Index: user/ngie/release-pkg-fix-tests/sys/arm/arm/pmap-v6.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/arm/pmap-v6.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/arm/pmap-v6.c (revision 299091) @@ -1,6805 +1,6805 @@ /*- * Copyright (c) 1991 Regents of the University of California. * Copyright (c) 1994 John S. Dyson * Copyright (c) 1994 David Greenman * Copyright (c) 2005-2010 Alan L. Cox * Copyright (c) 2014-2016 Svatopluk Kraus * Copyright (c) 2014-2016 Michal Meloun * All rights reserved. * * This code is derived from software contributed to Berkeley by * the Systems Programming Group of the University of Utah Computer * Science Department and William Jolitz of UUNET Technologies Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 */ /*- * Copyright (c) 2003 Networks Associates Technology, Inc. * All rights reserved. * * This software was developed for the FreeBSD Project by Jake Burkholder, * Safeport Network Services, and Network Associates Laboratories, the * Security Research Division of Network Associates, Inc. under * DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the DARPA * CHATS research program. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); /* * Manages physical address maps. * * Since the information managed by this module is * also stored by the logical address mapping module, * this module may throw away valid virtual-to-physical * mappings at almost any time. However, invalidations * of virtual-to-physical mappings must be done as * requested. * * In order to cope with hardware architectures which * make virtual-to-physical map invalidates expensive, * this module may delay invalidate or reduced protection * operations until such time as they are actually * necessary. This module is given full information as * to which processors are currently using which maps, * and to when physical maps must be made correct. */ #include "opt_vm.h" #include "opt_pmap.h" #include "opt_ddb.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef SMP #include #else #include #endif #ifdef DDB #include #endif #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef SMP #include #endif #ifndef PMAP_SHPGPERPROC #define PMAP_SHPGPERPROC 200 #endif #ifndef DIAGNOSTIC #define PMAP_INLINE __inline #else #define PMAP_INLINE #endif #ifdef PMAP_DEBUG static void pmap_zero_page_check(vm_page_t m); void pmap_debug(int level); int pmap_pid_dump(int pid); #define PDEBUG(_lev_,_stat_) \ if (pmap_debug_level >= (_lev_)) \ ((_stat_)) #define dprintf printf int pmap_debug_level = 1; #else /* PMAP_DEBUG */ #define PDEBUG(_lev_,_stat_) /* Nothing */ #define dprintf(x, arg...) #endif /* PMAP_DEBUG */ /* * Level 2 page tables map definion ('max' is excluded). */ #define PT2V_MIN_ADDRESS ((vm_offset_t)PT2MAP) #define PT2V_MAX_ADDRESS ((vm_offset_t)PT2MAP + PT2MAP_SIZE) #define UPT2V_MIN_ADDRESS ((vm_offset_t)PT2MAP) #define UPT2V_MAX_ADDRESS \ ((vm_offset_t)(PT2MAP + (KERNBASE >> PT2MAP_SHIFT))) /* * Promotion to a 1MB (PTE1) page mapping requires that the corresponding * 4KB (PTE2) page mappings have identical settings for the following fields: */ #define PTE2_PROMOTE (PTE2_V | PTE2_A | PTE2_NM | PTE2_S | PTE2_NG | \ PTE2_NX | PTE2_RO | PTE2_U | PTE2_W | \ PTE2_ATTR_MASK) #define PTE1_PROMOTE (PTE1_V | PTE1_A | PTE1_NM | PTE1_S | PTE1_NG | \ PTE1_NX | PTE1_RO | PTE1_U | PTE1_W | \ PTE1_ATTR_MASK) #define ATTR_TO_L1(l2_attr) ((((l2_attr) & L2_TEX0) ? L1_S_TEX0 : 0) | \ (((l2_attr) & L2_C) ? L1_S_C : 0) | \ (((l2_attr) & L2_B) ? L1_S_B : 0) | \ (((l2_attr) & PTE2_A) ? PTE1_A : 0) | \ (((l2_attr) & PTE2_NM) ? PTE1_NM : 0) | \ (((l2_attr) & PTE2_S) ? PTE1_S : 0) | \ (((l2_attr) & PTE2_NG) ? PTE1_NG : 0) | \ (((l2_attr) & PTE2_NX) ? PTE1_NX : 0) | \ (((l2_attr) & PTE2_RO) ? PTE1_RO : 0) | \ (((l2_attr) & PTE2_U) ? PTE1_U : 0) | \ (((l2_attr) & PTE2_W) ? PTE1_W : 0)) #define ATTR_TO_L2(l1_attr) ((((l1_attr) & L1_S_TEX0) ? L2_TEX0 : 0) | \ (((l1_attr) & L1_S_C) ? L2_C : 0) | \ (((l1_attr) & L1_S_B) ? L2_B : 0) | \ (((l1_attr) & PTE1_A) ? PTE2_A : 0) | \ (((l1_attr) & PTE1_NM) ? PTE2_NM : 0) | \ (((l1_attr) & PTE1_S) ? PTE2_S : 0) | \ (((l1_attr) & PTE1_NG) ? PTE2_NG : 0) | \ (((l1_attr) & PTE1_NX) ? PTE2_NX : 0) | \ (((l1_attr) & PTE1_RO) ? PTE2_RO : 0) | \ (((l1_attr) & PTE1_U) ? PTE2_U : 0) | \ (((l1_attr) & PTE1_W) ? PTE2_W : 0)) /* * PTE2 descriptors creation macros. */ #define PTE2_ATTR_DEFAULT vm_memattr_to_pte2(VM_MEMATTR_DEFAULT) #define PTE2_ATTR_PT vm_memattr_to_pte2(pt_memattr) #define PTE2_KPT(pa) PTE2_KERN(pa, PTE2_AP_KRW, PTE2_ATTR_PT) #define PTE2_KPT_NG(pa) PTE2_KERN_NG(pa, PTE2_AP_KRW, PTE2_ATTR_PT) #define PTE2_KRW(pa) PTE2_KERN(pa, PTE2_AP_KRW, PTE2_ATTR_DEFAULT) #define PTE2_KRO(pa) PTE2_KERN(pa, PTE2_AP_KR, PTE2_ATTR_DEFAULT) #define PV_STATS #ifdef PV_STATS #define PV_STAT(x) do { x ; } while (0) #else #define PV_STAT(x) do { } while (0) #endif /* * The boot_pt1 is used temporary in very early boot stage as L1 page table. * We can init many things with no memory allocation thanks to its static * allocation and this brings two main advantages: * (1) other cores can be started very simply, * (2) various boot loaders can be supported as its arguments can be processed * in virtual address space and can be moved to safe location before * first allocation happened. * Only disadvantage is that boot_pt1 is used only in very early boot stage. * However, the table is uninitialized and so lays in bss. Therefore kernel * image size is not influenced. * * QQQ: In the future, maybe, boot_pt1 can be used for soft reset and * CPU suspend/resume game. */ extern pt1_entry_t boot_pt1[]; vm_paddr_t base_pt1; pt1_entry_t *kern_pt1; pt2_entry_t *kern_pt2tab; pt2_entry_t *PT2MAP; static uint32_t ttb_flags; static vm_memattr_t pt_memattr; ttb_entry_t pmap_kern_ttb; struct pmap kernel_pmap_store; LIST_HEAD(pmaplist, pmap); static struct pmaplist allpmaps; static struct mtx allpmaps_lock; vm_offset_t virtual_avail; /* VA of first avail page (after kernel bss) */ vm_offset_t virtual_end; /* VA of last avail page (end of kernel AS) */ static vm_offset_t kernel_vm_end_new; vm_offset_t kernel_vm_end = KERNBASE + NKPT2PG * NPT2_IN_PG * PTE1_SIZE; vm_offset_t vm_max_kernel_address; vm_paddr_t kernel_l1pa; static struct rwlock __aligned(CACHE_LINE_SIZE) pvh_global_lock; /* * Data for the pv entry allocation mechanism */ static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks); static int pv_entry_count = 0, pv_entry_max = 0, pv_entry_high_water = 0; static struct md_page *pv_table; /* XXX: Is it used only the list in md_page? */ static int shpgperproc = PMAP_SHPGPERPROC; struct pv_chunk *pv_chunkbase; /* KVA block for pv_chunks */ int pv_maxchunks; /* How many chunks we have KVA for */ vm_offset_t pv_vafree; /* freelist stored in the PTE */ vm_paddr_t first_managed_pa; #define pa_to_pvh(pa) (&pv_table[pte1_index(pa - first_managed_pa)]) /* * All those kernel PT submaps that BSD is so fond of */ struct sysmaps { struct mtx lock; pt2_entry_t *CMAP1; pt2_entry_t *CMAP2; pt2_entry_t *CMAP3; caddr_t CADDR1; caddr_t CADDR2; caddr_t CADDR3; }; static struct sysmaps sysmaps_pcpu[MAXCPU]; static pt2_entry_t *CMAP3; static caddr_t CADDR3; caddr_t _tmppt = 0; struct msgbuf *msgbufp = NULL; /* XXX move it to machdep.c */ /* * Crashdump maps. */ static caddr_t crashdumpmap; static pt2_entry_t *PMAP1 = NULL, *PMAP2; static pt2_entry_t *PADDR1 = NULL, *PADDR2; #ifdef DDB static pt2_entry_t *PMAP3; static pt2_entry_t *PADDR3; static int PMAP3cpu __unused; /* for SMP only */ #endif #ifdef SMP static int PMAP1cpu; static int PMAP1changedcpu; SYSCTL_INT(_debug, OID_AUTO, PMAP1changedcpu, CTLFLAG_RD, &PMAP1changedcpu, 0, "Number of times pmap_pte2_quick changed CPU with same PMAP1"); #endif static int PMAP1changed; SYSCTL_INT(_debug, OID_AUTO, PMAP1changed, CTLFLAG_RD, &PMAP1changed, 0, "Number of times pmap_pte2_quick changed PMAP1"); static int PMAP1unchanged; SYSCTL_INT(_debug, OID_AUTO, PMAP1unchanged, CTLFLAG_RD, &PMAP1unchanged, 0, "Number of times pmap_pte2_quick didn't change PMAP1"); static struct mtx PMAP2mutex; static __inline void pt2_wirecount_init(vm_page_t m); static boolean_t pmap_demote_pte1(pmap_t pmap, pt1_entry_t *pte1p, vm_offset_t va); void cache_icache_sync_fresh(vm_offset_t va, vm_paddr_t pa, vm_size_t size); /* * Function to set the debug level of the pmap code. */ #ifdef PMAP_DEBUG void pmap_debug(int level) { pmap_debug_level = level; dprintf("pmap_debug: level=%d\n", pmap_debug_level); } #endif /* PMAP_DEBUG */ /* * This table must corespond with memory attribute configuration in vm.h. * First entry is used for normal system mapping. * * Device memory is always marked as shared. * Normal memory is shared only in SMP . * Not outer shareable bits are not used yet. * Class 6 cannot be used on ARM11. */ #define TEXDEF_TYPE_SHIFT 0 #define TEXDEF_TYPE_MASK 0x3 #define TEXDEF_INNER_SHIFT 2 #define TEXDEF_INNER_MASK 0x3 #define TEXDEF_OUTER_SHIFT 4 #define TEXDEF_OUTER_MASK 0x3 #define TEXDEF_NOS_SHIFT 6 #define TEXDEF_NOS_MASK 0x1 #define TEX(t, i, o, s) \ ((t) << TEXDEF_TYPE_SHIFT) | \ ((i) << TEXDEF_INNER_SHIFT) | \ ((o) << TEXDEF_OUTER_SHIFT | \ ((s) << TEXDEF_NOS_SHIFT)) static uint32_t tex_class[8] = { /* type inner cache outer cache */ TEX(PRRR_MEM, NMRR_WB_WA, NMRR_WB_WA, 0), /* 0 - ATTR_WB_WA */ TEX(PRRR_MEM, NMRR_NC, NMRR_NC, 0), /* 1 - ATTR_NOCACHE */ TEX(PRRR_DEV, NMRR_NC, NMRR_NC, 0), /* 2 - ATTR_DEVICE */ TEX(PRRR_SO, NMRR_NC, NMRR_NC, 0), /* 3 - ATTR_SO */ TEX(PRRR_MEM, NMRR_WT, NMRR_WT, 0), /* 4 - ATTR_WT */ TEX(PRRR_MEM, NMRR_NC, NMRR_NC, 0), /* 5 - NOT USED YET */ TEX(PRRR_MEM, NMRR_NC, NMRR_NC, 0), /* 6 - NOT USED YET */ TEX(PRRR_MEM, NMRR_NC, NMRR_NC, 0), /* 7 - NOT USED YET */ }; #undef TEX static uint32_t pte2_attr_tab[8] = { PTE2_ATTR_WB_WA, /* 0 - VM_MEMATTR_WB_WA */ PTE2_ATTR_NOCACHE, /* 1 - VM_MEMATTR_NOCACHE */ PTE2_ATTR_DEVICE, /* 2 - VM_MEMATTR_DEVICE */ PTE2_ATTR_SO, /* 3 - VM_MEMATTR_SO */ PTE2_ATTR_WT, /* 4 - VM_MEMATTR_WRITE_THROUGH */ 0, /* 5 - NOT USED YET */ 0, /* 6 - NOT USED YET */ 0 /* 7 - NOT USED YET */ }; CTASSERT(VM_MEMATTR_WB_WA == 0); CTASSERT(VM_MEMATTR_NOCACHE == 1); CTASSERT(VM_MEMATTR_DEVICE == 2); CTASSERT(VM_MEMATTR_SO == 3); CTASSERT(VM_MEMATTR_WRITE_THROUGH == 4); static inline uint32_t vm_memattr_to_pte2(vm_memattr_t ma) { KASSERT((u_int)ma < 5, ("%s: bad vm_memattr_t %d", __func__, ma)); return (pte2_attr_tab[(u_int)ma]); } static inline uint32_t vm_page_pte2_attr(vm_page_t m) { return (vm_memattr_to_pte2(m->md.pat_mode)); } /* * Convert TEX definition entry to TTB flags. */ static uint32_t encode_ttb_flags(int idx) { uint32_t inner, outer, nos, reg; inner = (tex_class[idx] >> TEXDEF_INNER_SHIFT) & TEXDEF_INNER_MASK; outer = (tex_class[idx] >> TEXDEF_OUTER_SHIFT) & TEXDEF_OUTER_MASK; nos = (tex_class[idx] >> TEXDEF_NOS_SHIFT) & TEXDEF_NOS_MASK; reg = nos << 5; reg |= outer << 3; if (cpuinfo.coherent_walk) reg |= (inner & 0x1) << 6; reg |= (inner & 0x2) >> 1; #ifdef SMP reg |= 1 << 1; #endif return reg; } /* * Set TEX remapping registers in current CPU. */ void pmap_set_tex(void) { uint32_t prrr, nmrr; uint32_t type, inner, outer, nos; int i; #ifdef PMAP_PTE_NOCACHE /* XXX fixme */ if (cpuinfo.coherent_walk) { pt_memattr = VM_MEMATTR_WB_WA; ttb_flags = encode_ttb_flags(0); } else { pt_memattr = VM_MEMATTR_NOCACHE; ttb_flags = encode_ttb_flags(1); } #else pt_memattr = VM_MEMATTR_WB_WA; ttb_flags = encode_ttb_flags(0); #endif prrr = 0; nmrr = 0; /* Build remapping register from TEX classes. */ for (i = 0; i < 8; i++) { type = (tex_class[i] >> TEXDEF_TYPE_SHIFT) & TEXDEF_TYPE_MASK; inner = (tex_class[i] >> TEXDEF_INNER_SHIFT) & TEXDEF_INNER_MASK; outer = (tex_class[i] >> TEXDEF_OUTER_SHIFT) & TEXDEF_OUTER_MASK; nos = (tex_class[i] >> TEXDEF_NOS_SHIFT) & TEXDEF_NOS_MASK; prrr |= type << (i * 2); prrr |= nos << (i + 24); nmrr |= inner << (i * 2); nmrr |= outer << (i * 2 + 16); } /* Add shareable bits for device memory. */ prrr |= PRRR_DS0 | PRRR_DS1; /* Add shareable bits for normal memory in SMP case. */ #ifdef SMP prrr |= PRRR_NS1; #endif cp15_prrr_set(prrr); cp15_nmrr_set(nmrr); /* Caches are disabled, so full TLB flush should be enough. */ tlb_flush_all_local(); } /* * KERNBASE must be multiple of NPT2_IN_PG * PTE1_SIZE. In other words, * KERNBASE is mapped by first L2 page table in L2 page table page. It * meets same constrain due to PT2MAP being placed just under KERNBASE. */ CTASSERT((KERNBASE & (NPT2_IN_PG * PTE1_SIZE - 1)) == 0); CTASSERT((KERNBASE - VM_MAXUSER_ADDRESS) >= PT2MAP_SIZE); /* * In crazy dreams, PAGE_SIZE could be a multiple of PTE2_SIZE in general. * For now, anyhow, the following check must be fulfilled. */ CTASSERT(PAGE_SIZE == PTE2_SIZE); /* * We don't want to mess up MI code with all MMU and PMAP definitions, * so some things, which depend on other ones, are defined independently. * Now, it is time to check that we don't screw up something. */ CTASSERT(PDRSHIFT == PTE1_SHIFT); /* * Check L1 and L2 page table entries definitions consistency. */ CTASSERT(NB_IN_PT1 == (sizeof(pt1_entry_t) * NPTE1_IN_PT1)); CTASSERT(NB_IN_PT2 == (sizeof(pt2_entry_t) * NPTE2_IN_PT2)); /* * Check L2 page tables page consistency. */ CTASSERT(PAGE_SIZE == (NPT2_IN_PG * NB_IN_PT2)); CTASSERT((1 << PT2PG_SHIFT) == NPT2_IN_PG); /* * Check PT2TAB consistency. * PT2TAB_ENTRIES is defined as a division of NPTE1_IN_PT1 by NPT2_IN_PG. * This should be done without remainder. */ CTASSERT(NPTE1_IN_PT1 == (PT2TAB_ENTRIES * NPT2_IN_PG)); /* * A PT2MAP magic. * * All level 2 page tables (PT2s) are mapped continuously and accordingly * into PT2MAP address space. As PT2 size is less than PAGE_SIZE, this can * be done only if PAGE_SIZE is a multiple of PT2 size. All PT2s in one page * must be used together, but not necessary at once. The first PT2 in a page * must map things on correctly aligned address and the others must follow * in right order. */ #define NB_IN_PT2TAB (PT2TAB_ENTRIES * sizeof(pt2_entry_t)) #define NPT2_IN_PT2TAB (NB_IN_PT2TAB / NB_IN_PT2) #define NPG_IN_PT2TAB (NB_IN_PT2TAB / PAGE_SIZE) /* * Check PT2TAB consistency. * NPT2_IN_PT2TAB is defined as a division of NB_IN_PT2TAB by NB_IN_PT2. * NPG_IN_PT2TAB is defined as a division of NB_IN_PT2TAB by PAGE_SIZE. * The both should be done without remainder. */ CTASSERT(NB_IN_PT2TAB == (NPT2_IN_PT2TAB * NB_IN_PT2)); CTASSERT(NB_IN_PT2TAB == (NPG_IN_PT2TAB * PAGE_SIZE)); /* * The implementation was made general, however, with the assumption * bellow in mind. In case of another value of NPG_IN_PT2TAB, * the code should be once more rechecked. */ CTASSERT(NPG_IN_PT2TAB == 1); /* * Get offset of PT2 in a page * associated with given PT1 index. */ static __inline u_int page_pt2off(u_int pt1_idx) { return ((pt1_idx & PT2PG_MASK) * NB_IN_PT2); } /* * Get physical address of PT2 * associated with given PT2s page and PT1 index. */ static __inline vm_paddr_t page_pt2pa(vm_paddr_t pgpa, u_int pt1_idx) { return (pgpa + page_pt2off(pt1_idx)); } /* * Get first entry of PT2 * associated with given PT2s page and PT1 index. */ static __inline pt2_entry_t * page_pt2(vm_offset_t pgva, u_int pt1_idx) { return ((pt2_entry_t *)(pgva + page_pt2off(pt1_idx))); } /* * Get virtual address of PT2s page (mapped in PT2MAP) * which holds PT2 which holds entry which maps given virtual address. */ static __inline vm_offset_t pt2map_pt2pg(vm_offset_t va) { va &= ~(NPT2_IN_PG * PTE1_SIZE - 1); return ((vm_offset_t)pt2map_entry(va)); } /***************************************************************************** * * THREE pmap initialization milestones exist: * * locore.S * -> fundamental init (including MMU) in ASM * * initarm() * -> fundamental init continues in C * -> first available physical address is known * * pmap_bootstrap_prepare() -> FIRST PMAP MILESTONE (first epoch begins) * -> basic (safe) interface for physical address allocation is made * -> basic (safe) interface for virtual mapping is made * -> limited not SMP coherent work is possible * * -> more fundamental init continues in C * -> locks and some more things are available * -> all fundamental allocations and mappings are done * * pmap_bootstrap() -> SECOND PMAP MILESTONE (second epoch begins) * -> phys_avail[] and virtual_avail is set * -> control is passed to vm subsystem * -> physical and virtual address allocation are off limit * -> low level mapping functions, some SMP coherent, * are available, which cannot be used before vm subsystem * is being inited * * mi_startup() * -> vm subsystem is being inited * * pmap_init() -> THIRD PMAP MILESTONE (third epoch begins) * -> pmap is fully inited * *****************************************************************************/ /***************************************************************************** * * PMAP first stage initialization and utility functions * for pre-bootstrap epoch. * * After pmap_bootstrap_prepare() is called, the following functions * can be used: * * (1) strictly only for this stage functions for physical page allocations, * virtual space allocations, and mappings: * * vm_paddr_t pmap_preboot_get_pages(u_int num); * void pmap_preboot_map_pages(vm_paddr_t pa, vm_offset_t va, u_int num); * vm_offset_t pmap_preboot_reserve_pages(u_int num); * vm_offset_t pmap_preboot_get_vpages(u_int num); * void pmap_preboot_map_attr(vm_paddr_t pa, vm_offset_t va, vm_size_t size, * vm_prot_t prot, vm_memattr_t attr); * * (2) for all stages: * * vm_paddr_t pmap_kextract(vm_offset_t va); * * NOTE: This is not SMP coherent stage. * *****************************************************************************/ #define KERNEL_P2V(pa) \ ((vm_offset_t)((pa) - arm_physmem_kernaddr + KERNVIRTADDR)) #define KERNEL_V2P(va) \ ((vm_paddr_t)((va) - KERNVIRTADDR + arm_physmem_kernaddr)) static vm_paddr_t last_paddr; /* * Pre-bootstrap epoch page allocator. */ vm_paddr_t pmap_preboot_get_pages(u_int num) { vm_paddr_t ret; ret = last_paddr; last_paddr += num * PAGE_SIZE; return (ret); } /* - * The fundamental initalization of PMAP stuff. + * The fundamental initialization of PMAP stuff. * * Some things already happened in locore.S and some things could happen * before pmap_bootstrap_prepare() is called, so let's recall what is done: * 1. Caches are disabled. * 2. We are running on virtual addresses already with 'boot_pt1' * as L1 page table. * 3. So far, all virtual addresses can be converted to physical ones and * vice versa by the following macros: * KERNEL_P2V(pa) .... physical to virtual ones, * KERNEL_V2P(va) .... virtual to physical ones. * * What is done herein: * 1. The 'boot_pt1' is replaced by real kernel L1 page table 'kern_pt1'. * 2. PT2MAP magic is brought to live. * 3. Basic preboot functions for page allocations and mappings can be used. * 4. Everything is prepared for L1 cache enabling. * * Variations: * 1. To use second TTB register, so kernel and users page tables will be * separated. This way process forking - pmap_pinit() - could be faster, * it saves physical pages and KVA per a process, and it's simple change. * However, it will lead, due to hardware matter, to the following: * (a) 2G space for kernel and 2G space for users. * (b) 1G space for kernel in low addresses and 3G for users above it. * A question is: Is the case (b) really an option? Note that case (b) * does save neither physical memory and KVA. */ void pmap_bootstrap_prepare(vm_paddr_t last) { vm_paddr_t pt2pg_pa, pt2tab_pa, pa, size; vm_offset_t pt2pg_va; pt1_entry_t *pte1p; pt2_entry_t *pte2p; u_int i; uint32_t actlr_mask, actlr_set, l1_attr; /* * Now, we are going to make real kernel mapping. Note that we are * already running on some mapping made in locore.S and we expect * that it's large enough to ensure nofault access to physical memory * allocated herein before switch. * * As kernel image and everything needed before are and will be mapped * by section mappings, we align last physical address to PTE1_SIZE. */ last_paddr = pte1_roundup(last); /* * Allocate and zero page(s) for kernel L1 page table. * * Note that it's first allocation on space which was PTE1_SIZE * aligned and as such base_pt1 is aligned to NB_IN_PT1 too. */ base_pt1 = pmap_preboot_get_pages(NPG_IN_PT1); kern_pt1 = (pt1_entry_t *)KERNEL_P2V(base_pt1); bzero((void*)kern_pt1, NB_IN_PT1); pte1_sync_range(kern_pt1, NB_IN_PT1); /* Allocate and zero page(s) for kernel PT2TAB. */ pt2tab_pa = pmap_preboot_get_pages(NPG_IN_PT2TAB); kern_pt2tab = (pt2_entry_t *)KERNEL_P2V(pt2tab_pa); bzero(kern_pt2tab, NB_IN_PT2TAB); pte2_sync_range(kern_pt2tab, NB_IN_PT2TAB); /* Allocate and zero page(s) for kernel L2 page tables. */ pt2pg_pa = pmap_preboot_get_pages(NKPT2PG); pt2pg_va = KERNEL_P2V(pt2pg_pa); size = NKPT2PG * PAGE_SIZE; bzero((void*)pt2pg_va, size); pte2_sync_range((pt2_entry_t *)pt2pg_va, size); /* * Add a physical memory segment (vm_phys_seg) corresponding to the * preallocated pages for kernel L2 page tables so that vm_page * structures representing these pages will be created. The vm_page * structures are required for promotion of the corresponding kernel * virtual addresses to section mappings. */ vm_phys_add_seg(pt2tab_pa, pmap_preboot_get_pages(0)); /* * Insert allocated L2 page table pages to PT2TAB and make * link to all PT2s in L1 page table. See how kernel_vm_end * is initialized. * * We play simple and safe. So every KVA will have underlaying * L2 page table, even kernel image mapped by sections. */ pte2p = kern_pt2tab_entry(KERNBASE); for (pa = pt2pg_pa; pa < pt2pg_pa + size; pa += PTE2_SIZE) pt2tab_store(pte2p++, PTE2_KPT(pa)); pte1p = kern_pte1(KERNBASE); for (pa = pt2pg_pa; pa < pt2pg_pa + size; pa += NB_IN_PT2) pte1_store(pte1p++, PTE1_LINK(pa)); /* Make section mappings for kernel. */ l1_attr = ATTR_TO_L1(PTE2_ATTR_DEFAULT); pte1p = kern_pte1(KERNBASE); for (pa = KERNEL_V2P(KERNBASE); pa < last; pa += PTE1_SIZE) pte1_store(pte1p++, PTE1_KERN(pa, PTE1_AP_KRW, l1_attr)); /* * Get free and aligned space for PT2MAP and make L1 page table links * to L2 page tables held in PT2TAB. * * Note that pages holding PT2s are stored in PT2TAB as pt2_entry_t * descriptors and PT2TAB page(s) itself is(are) used as PT2s. Thus * each entry in PT2TAB maps all PT2s in a page. This implies that * virtual address of PT2MAP must be aligned to NPT2_IN_PG * PTE1_SIZE. */ PT2MAP = (pt2_entry_t *)(KERNBASE - PT2MAP_SIZE); pte1p = kern_pte1((vm_offset_t)PT2MAP); for (pa = pt2tab_pa, i = 0; i < NPT2_IN_PT2TAB; i++, pa += NB_IN_PT2) { pte1_store(pte1p++, PTE1_LINK(pa)); } /* * Store PT2TAB in PT2TAB itself, i.e. self reference mapping. * Each pmap will hold own PT2TAB, so the mapping should be not global. */ pte2p = kern_pt2tab_entry((vm_offset_t)PT2MAP); for (pa = pt2tab_pa, i = 0; i < NPG_IN_PT2TAB; i++, pa += PTE2_SIZE) { pt2tab_store(pte2p++, PTE2_KPT_NG(pa)); } /* * Choose correct L2 page table and make mappings for allocations * made herein which replaces temporary locore.S mappings after a while. * Note that PT2MAP cannot be used until we switch to kern_pt1. * * Note, that these allocations started aligned on 1M section and * kernel PT1 was allocated first. Making of mappings must follow * order of physical allocations as we've used KERNEL_P2V() macro * for virtual addresses resolution. */ pte2p = kern_pt2tab_entry((vm_offset_t)kern_pt1); pt2pg_va = KERNEL_P2V(pte2_pa(pte2_load(pte2p))); pte2p = page_pt2(pt2pg_va, pte1_index((vm_offset_t)kern_pt1)); /* Make mapping for kernel L1 page table. */ for (pa = base_pt1, i = 0; i < NPG_IN_PT1; i++, pa += PTE2_SIZE) pte2_store(pte2p++, PTE2_KPT(pa)); /* Make mapping for kernel PT2TAB. */ for (pa = pt2tab_pa, i = 0; i < NPG_IN_PT2TAB; i++, pa += PTE2_SIZE) pte2_store(pte2p++, PTE2_KPT(pa)); /* Finally, switch from 'boot_pt1' to 'kern_pt1'. */ pmap_kern_ttb = base_pt1 | ttb_flags; cpuinfo_get_actlr_modifier(&actlr_mask, &actlr_set); reinit_mmu(pmap_kern_ttb, actlr_mask, actlr_set); /* * Initialize the first available KVA. As kernel image is mapped by * sections, we are leaving some gap behind. */ virtual_avail = (vm_offset_t)kern_pt2tab + NPG_IN_PT2TAB * PAGE_SIZE; } /* * Setup L2 page table page for given KVA. * Used in pre-bootstrap epoch. * * Note that we have allocated NKPT2PG pages for L2 page tables in advance * and used them for mapping KVA starting from KERNBASE. However, this is not * enough. Vectors and devices need L2 page tables too. Note that they are * even above VM_MAX_KERNEL_ADDRESS. */ static __inline vm_paddr_t pmap_preboot_pt2pg_setup(vm_offset_t va) { pt2_entry_t *pte2p, pte2; vm_paddr_t pt2pg_pa; /* Get associated entry in PT2TAB. */ pte2p = kern_pt2tab_entry(va); /* Just return, if PT2s page exists already. */ pte2 = pt2tab_load(pte2p); if (pte2_is_valid(pte2)) return (pte2_pa(pte2)); KASSERT(va >= VM_MAX_KERNEL_ADDRESS, ("%s: NKPT2PG too small", __func__)); /* * Allocate page for PT2s and insert it to PT2TAB. * In other words, map it into PT2MAP space. */ pt2pg_pa = pmap_preboot_get_pages(1); pt2tab_store(pte2p, PTE2_KPT(pt2pg_pa)); /* Zero all PT2s in allocated page. */ bzero((void*)pt2map_pt2pg(va), PAGE_SIZE); pte2_sync_range((pt2_entry_t *)pt2map_pt2pg(va), PAGE_SIZE); return (pt2pg_pa); } /* * Setup L2 page table for given KVA. * Used in pre-bootstrap epoch. */ static void pmap_preboot_pt2_setup(vm_offset_t va) { pt1_entry_t *pte1p; vm_paddr_t pt2pg_pa, pt2_pa; /* Setup PT2's page. */ pt2pg_pa = pmap_preboot_pt2pg_setup(va); pt2_pa = page_pt2pa(pt2pg_pa, pte1_index(va)); /* Insert PT2 to PT1. */ pte1p = kern_pte1(va); pte1_store(pte1p, PTE1_LINK(pt2_pa)); } /* * Get L2 page entry associated with given KVA. * Used in pre-bootstrap epoch. */ static __inline pt2_entry_t* pmap_preboot_vtopte2(vm_offset_t va) { pt1_entry_t *pte1p; /* Setup PT2 if needed. */ pte1p = kern_pte1(va); if (!pte1_is_valid(pte1_load(pte1p))) /* XXX - sections ?! */ pmap_preboot_pt2_setup(va); return (pt2map_entry(va)); } /* * Pre-bootstrap epoch page(s) mapping(s). */ void pmap_preboot_map_pages(vm_paddr_t pa, vm_offset_t va, u_int num) { u_int i; pt2_entry_t *pte2p; /* Map all the pages. */ for (i = 0; i < num; i++) { pte2p = pmap_preboot_vtopte2(va); pte2_store(pte2p, PTE2_KRW(pa)); va += PAGE_SIZE; pa += PAGE_SIZE; } } /* * Pre-bootstrap epoch virtual space alocator. */ vm_offset_t pmap_preboot_reserve_pages(u_int num) { u_int i; vm_offset_t start, va; pt2_entry_t *pte2p; /* Allocate virtual space. */ start = va = virtual_avail; virtual_avail += num * PAGE_SIZE; /* Zero the mapping. */ for (i = 0; i < num; i++) { pte2p = pmap_preboot_vtopte2(va); pte2_store(pte2p, 0); va += PAGE_SIZE; } return (start); } /* * Pre-bootstrap epoch page(s) allocation and mapping(s). */ vm_offset_t pmap_preboot_get_vpages(u_int num) { vm_paddr_t pa; vm_offset_t va; /* Allocate physical page(s). */ pa = pmap_preboot_get_pages(num); /* Allocate virtual space. */ va = virtual_avail; virtual_avail += num * PAGE_SIZE; /* Map and zero all. */ pmap_preboot_map_pages(pa, va, num); bzero((void *)va, num * PAGE_SIZE); return (va); } /* * Pre-bootstrap epoch page mapping(s) with attributes. */ void pmap_preboot_map_attr(vm_paddr_t pa, vm_offset_t va, vm_size_t size, vm_prot_t prot, vm_memattr_t attr) { u_int num; u_int l1_attr, l1_prot, l2_prot, l2_attr; pt1_entry_t *pte1p; pt2_entry_t *pte2p; l2_prot = prot & VM_PROT_WRITE ? PTE2_AP_KRW : PTE2_AP_KR; l2_prot |= (prot & VM_PROT_EXECUTE) ? PTE2_X : PTE2_NX; l2_attr = vm_memattr_to_pte2(attr); l1_prot = ATTR_TO_L1(l2_prot); l1_attr = ATTR_TO_L1(l2_attr); /* Map all the pages. */ num = round_page(size); while (num > 0) { if ((((va | pa) & PTE1_OFFSET) == 0) && (num >= PTE1_SIZE)) { pte1p = kern_pte1(va); pte1_store(pte1p, PTE1_KERN(pa, l1_prot, l1_attr)); va += PTE1_SIZE; pa += PTE1_SIZE; num -= PTE1_SIZE; } else { pte2p = pmap_preboot_vtopte2(va); pte2_store(pte2p, PTE2_KERN(pa, l2_prot, l2_attr)); va += PAGE_SIZE; pa += PAGE_SIZE; num -= PAGE_SIZE; } } } /* * Extract from the kernel page table the physical address * that is mapped by the given virtual address "va". */ vm_paddr_t pmap_kextract(vm_offset_t va) { vm_paddr_t pa; pt1_entry_t pte1; pt2_entry_t pte2; pte1 = pte1_load(kern_pte1(va)); if (pte1_is_section(pte1)) { pa = pte1_pa(pte1) | (va & PTE1_OFFSET); } else if (pte1_is_link(pte1)) { /* * We should beware of concurrent promotion that changes * pte1 at this point. However, it's not a problem as PT2 * page is preserved by promotion in PT2TAB. So even if * it happens, using of PT2MAP is still safe. * * QQQ: However, concurrent removing is a problem which * ends in abort on PT2MAP space. Locking must be used * to deal with this. */ pte2 = pte2_load(pt2map_entry(va)); pa = pte2_pa(pte2) | (va & PTE2_OFFSET); } else { panic("%s: va %#x pte1 %#x", __func__, va, pte1); } return (pa); } /* * Extract from the kernel page table the physical address * that is mapped by the given virtual address "va". Also * return L2 page table entry which maps the address. * * This is only intended to be used for panic dumps. */ vm_paddr_t pmap_dump_kextract(vm_offset_t va, pt2_entry_t *pte2p) { vm_paddr_t pa; pt1_entry_t pte1; pt2_entry_t pte2; pte1 = pte1_load(kern_pte1(va)); if (pte1_is_section(pte1)) { pa = pte1_pa(pte1) | (va & PTE1_OFFSET); pte2 = pa | ATTR_TO_L2(pte1) | PTE2_V; } else if (pte1_is_link(pte1)) { pte2 = pte2_load(pt2map_entry(va)); pa = pte2_pa(pte2); } else { pte2 = 0; pa = 0; } if (pte2p != NULL) *pte2p = pte2; return (pa); } /***************************************************************************** * * PMAP second stage initialization and utility functions * for bootstrap epoch. * * After pmap_bootstrap() is called, the following functions for * mappings can be used: * * void pmap_kenter(vm_offset_t va, vm_paddr_t pa); * void pmap_kremove(vm_offset_t va); * vm_offset_t pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, * int prot); * * NOTE: This is not SMP coherent stage. And physical page allocation is not * allowed during this stage. * *****************************************************************************/ /* * Initialize kernel PMAP locks and lists, kernel_pmap itself, and * reserve various virtual spaces for temporary mappings. */ void pmap_bootstrap(vm_offset_t firstaddr) { pt2_entry_t *unused __unused; struct sysmaps *sysmaps; u_int i; /* * Initialize the kernel pmap (which is statically allocated). */ PMAP_LOCK_INIT(kernel_pmap); kernel_l1pa = (vm_paddr_t)kern_pt1; /* for libkvm */ kernel_pmap->pm_pt1 = kern_pt1; kernel_pmap->pm_pt2tab = kern_pt2tab; CPU_FILL(&kernel_pmap->pm_active); /* don't allow deactivation */ TAILQ_INIT(&kernel_pmap->pm_pvchunk); /* * Initialize the global pv list lock. */ rw_init(&pvh_global_lock, "pmap pv global"); LIST_INIT(&allpmaps); /* * Request a spin mutex so that changes to allpmaps cannot be * preempted by smp_rendezvous_cpus(). */ mtx_init(&allpmaps_lock, "allpmaps", NULL, MTX_SPIN); mtx_lock_spin(&allpmaps_lock); LIST_INSERT_HEAD(&allpmaps, kernel_pmap, pm_list); mtx_unlock_spin(&allpmaps_lock); /* * Reserve some special page table entries/VA space for temporary * mapping of pages. */ #define SYSMAP(c, p, v, n) do { \ v = (c)pmap_preboot_reserve_pages(n); \ p = pt2map_entry((vm_offset_t)v); \ } while (0) /* * Local CMAP1/CMAP2 are used for zeroing and copying pages. * Local CMAP3 is used for data cache cleaning. * Global CMAP3 is used for the idle process page zeroing. */ for (i = 0; i < MAXCPU; i++) { sysmaps = &sysmaps_pcpu[i]; mtx_init(&sysmaps->lock, "SYSMAPS", NULL, MTX_DEF); SYSMAP(caddr_t, sysmaps->CMAP1, sysmaps->CADDR1, 1); SYSMAP(caddr_t, sysmaps->CMAP2, sysmaps->CADDR2, 1); SYSMAP(caddr_t, sysmaps->CMAP3, sysmaps->CADDR3, 1); } SYSMAP(caddr_t, CMAP3, CADDR3, 1); /* * Crashdump maps. */ SYSMAP(caddr_t, unused, crashdumpmap, MAXDUMPPGS); /* * _tmppt is used for reading arbitrary physical pages via /dev/mem. */ SYSMAP(caddr_t, unused, _tmppt, 1); /* * PADDR1 and PADDR2 are used by pmap_pte2_quick() and pmap_pte2(), * respectively. PADDR3 is used by pmap_pte2_ddb(). */ SYSMAP(pt2_entry_t *, PMAP1, PADDR1, 1); SYSMAP(pt2_entry_t *, PMAP2, PADDR2, 1); #ifdef DDB SYSMAP(pt2_entry_t *, PMAP3, PADDR3, 1); #endif mtx_init(&PMAP2mutex, "PMAP2", NULL, MTX_DEF); /* * Note that in very short time in initarm(), we are going to - * initialize phys_avail[] array and no futher page allocation + * initialize phys_avail[] array and no further page allocation * can happen after that until vm subsystem will be initialized. */ kernel_vm_end_new = kernel_vm_end; virtual_end = vm_max_kernel_address; } static void pmap_init_qpages(void) { struct pcpu *pc; int i; CPU_FOREACH(i) { pc = pcpu_find(i); pc->pc_qmap_addr = kva_alloc(PAGE_SIZE); if (pc->pc_qmap_addr == 0) panic("%s: unable to allocate KVA", __func__); } } SYSINIT(qpages_init, SI_SUB_CPU, SI_ORDER_ANY, pmap_init_qpages, NULL); /* * The function can already be use in second initialization stage. * As such, the function DOES NOT call pmap_growkernel() where PT2 * allocation can happen. So if used, be sure that PT2 for given * virtual address is allocated already! * * Add a wired page to the kva. * Note: not SMP coherent. */ static __inline void pmap_kenter_prot_attr(vm_offset_t va, vm_paddr_t pa, uint32_t prot, uint32_t attr) { pt1_entry_t *pte1p; pt2_entry_t *pte2p; pte1p = kern_pte1(va); if (!pte1_is_valid(pte1_load(pte1p))) { /* XXX - sections ?! */ /* * This is a very low level function, so PT2 and particularly * PT2PG associated with given virtual address must be already * allocated. It's a pain mainly during pmap initialization * stage. However, called after pmap initialization with * virtual address not under kernel_vm_end will lead to * the same misery. */ if (!pte2_is_valid(pte2_load(kern_pt2tab_entry(va)))) panic("%s: kernel PT2 not allocated!", __func__); } pte2p = pt2map_entry(va); pte2_store(pte2p, PTE2_KERN(pa, prot, attr)); } PMAP_INLINE void pmap_kenter(vm_offset_t va, vm_paddr_t pa) { pmap_kenter_prot_attr(va, pa, PTE2_AP_KRW, PTE2_ATTR_DEFAULT); } /* * Remove a page from the kernel pagetables. * Note: not SMP coherent. */ PMAP_INLINE void pmap_kremove(vm_offset_t va) { pt2_entry_t *pte2p; pte2p = pt2map_entry(va); pte2_clear(pte2p); } /* * Share new kernel PT2PG with all pmaps. * The caller is responsible for maintaining TLB consistency. */ static void pmap_kenter_pt2tab(vm_offset_t va, pt2_entry_t npte2) { pmap_t pmap; pt2_entry_t *pte2p; mtx_lock_spin(&allpmaps_lock); LIST_FOREACH(pmap, &allpmaps, pm_list) { pte2p = pmap_pt2tab_entry(pmap, va); pt2tab_store(pte2p, npte2); } mtx_unlock_spin(&allpmaps_lock); } /* * Share new kernel PTE1 with all pmaps. * The caller is responsible for maintaining TLB consistency. */ static void pmap_kenter_pte1(vm_offset_t va, pt1_entry_t npte1) { pmap_t pmap; pt1_entry_t *pte1p; mtx_lock_spin(&allpmaps_lock); LIST_FOREACH(pmap, &allpmaps, pm_list) { pte1p = pmap_pte1(pmap, va); pte1_store(pte1p, npte1); } mtx_unlock_spin(&allpmaps_lock); } /* * Used to map a range of physical addresses into kernel * virtual address space. * * The value passed in '*virt' is a suggested virtual address for * the mapping. Architectures which can support a direct-mapped * physical to virtual region can return the appropriate address * within that region, leaving '*virt' unchanged. Other * architectures should map the pages starting at '*virt' and * update '*virt' with the first usable address after the mapped * region. * * NOTE: Read the comments above pmap_kenter_prot_attr() as * the function is used herein! */ vm_offset_t pmap_map(vm_offset_t *virt, vm_paddr_t start, vm_paddr_t end, int prot) { vm_offset_t va, sva; vm_paddr_t pte1_offset; pt1_entry_t npte1; uint32_t l1prot, l2prot; uint32_t l1attr, l2attr; PDEBUG(1, printf("%s: virt = %#x, start = %#x, end = %#x (size = %#x)," " prot = %d\n", __func__, *virt, start, end, end - start, prot)); l2prot = (prot & VM_PROT_WRITE) ? PTE2_AP_KRW : PTE2_AP_KR; l2prot |= (prot & VM_PROT_EXECUTE) ? PTE2_X : PTE2_NX; l1prot = ATTR_TO_L1(l2prot); l2attr = PTE2_ATTR_DEFAULT; l1attr = ATTR_TO_L1(l2attr); va = *virt; /* * Does the physical address range's size and alignment permit at * least one section mapping to be created? */ pte1_offset = start & PTE1_OFFSET; if ((end - start) - ((PTE1_SIZE - pte1_offset) & PTE1_OFFSET) >= PTE1_SIZE) { /* * Increase the starting virtual address so that its alignment * does not preclude the use of section mappings. */ if ((va & PTE1_OFFSET) < pte1_offset) va = pte1_trunc(va) + pte1_offset; else if ((va & PTE1_OFFSET) > pte1_offset) va = pte1_roundup(va) + pte1_offset; } sva = va; while (start < end) { if ((start & PTE1_OFFSET) == 0 && end - start >= PTE1_SIZE) { KASSERT((va & PTE1_OFFSET) == 0, ("%s: misaligned va %#x", __func__, va)); npte1 = PTE1_KERN(start, l1prot, l1attr); pmap_kenter_pte1(va, npte1); va += PTE1_SIZE; start += PTE1_SIZE; } else { pmap_kenter_prot_attr(va, start, l2prot, l2attr); va += PAGE_SIZE; start += PAGE_SIZE; } } tlb_flush_range(sva, va - sva); *virt = va; return (sva); } /* * Make a temporary mapping for a physical address. * This is only intended to be used for panic dumps. */ void * pmap_kenter_temporary(vm_paddr_t pa, int i) { vm_offset_t va; /* QQQ: 'i' should be less or equal to MAXDUMPPGS. */ va = (vm_offset_t)crashdumpmap + (i * PAGE_SIZE); pmap_kenter(va, pa); tlb_flush_local(va); return ((void *)crashdumpmap); } /************************************* * * TLB & cache maintenance routines. * *************************************/ /* * We inline these within pmap.c for speed. */ PMAP_INLINE void pmap_tlb_flush(pmap_t pmap, vm_offset_t va) { if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active)) tlb_flush(va); } PMAP_INLINE void pmap_tlb_flush_range(pmap_t pmap, vm_offset_t sva, vm_size_t size) { if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active)) tlb_flush_range(sva, size); } /* * Abuse the pte2 nodes for unmapped kva to thread a kva freelist through. * Requirements: * - Must deal with pages in order to ensure that none of the PTE2_* bits * are ever set, PTE2_V in particular. * - Assumes we can write to pte2s without pte2_store() atomic ops. * - Assumes nothing will ever test these addresses for 0 to indicate * no mapping instead of correctly checking PTE2_V. * - Assumes a vm_offset_t will fit in a pte2 (true for arm). * Because PTE2_V is never set, there can be no mappings to invalidate. */ static vm_offset_t pmap_pte2list_alloc(vm_offset_t *head) { pt2_entry_t *pte2p; vm_offset_t va; va = *head; if (va == 0) panic("pmap_ptelist_alloc: exhausted ptelist KVA"); pte2p = pt2map_entry(va); *head = *pte2p; if (*head & PTE2_V) panic("%s: va with PTE2_V set!", __func__); *pte2p = 0; return (va); } static void pmap_pte2list_free(vm_offset_t *head, vm_offset_t va) { pt2_entry_t *pte2p; if (va & PTE2_V) panic("%s: freeing va with PTE2_V set!", __func__); pte2p = pt2map_entry(va); *pte2p = *head; /* virtual! PTE2_V is 0 though */ *head = va; } static void pmap_pte2list_init(vm_offset_t *head, void *base, int npages) { int i; vm_offset_t va; *head = 0; for (i = npages - 1; i >= 0; i--) { va = (vm_offset_t)base + i * PAGE_SIZE; pmap_pte2list_free(head, va); } } /***************************************************************************** * * PMAP third and final stage initialization. * * After pmap_init() is called, PMAP subsystem is fully initialized. * *****************************************************************************/ SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters"); SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_max, CTLFLAG_RD, &pv_entry_max, 0, "Max number of PV entries"); SYSCTL_INT(_vm_pmap, OID_AUTO, shpgperproc, CTLFLAG_RD, &shpgperproc, 0, "Page share factor per proc"); static u_long nkpt2pg = NKPT2PG; SYSCTL_ULONG(_vm_pmap, OID_AUTO, nkpt2pg, CTLFLAG_RD, &nkpt2pg, 0, "Pre-allocated pages for kernel PT2s"); static int sp_enabled = 1; SYSCTL_INT(_vm_pmap, OID_AUTO, sp_enabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &sp_enabled, 0, "Are large page mappings enabled?"); static SYSCTL_NODE(_vm_pmap, OID_AUTO, pte1, CTLFLAG_RD, 0, "1MB page mapping counters"); static u_long pmap_pte1_demotions; SYSCTL_ULONG(_vm_pmap_pte1, OID_AUTO, demotions, CTLFLAG_RD, &pmap_pte1_demotions, 0, "1MB page demotions"); static u_long pmap_pte1_mappings; SYSCTL_ULONG(_vm_pmap_pte1, OID_AUTO, mappings, CTLFLAG_RD, &pmap_pte1_mappings, 0, "1MB page mappings"); static u_long pmap_pte1_p_failures; SYSCTL_ULONG(_vm_pmap_pte1, OID_AUTO, p_failures, CTLFLAG_RD, &pmap_pte1_p_failures, 0, "1MB page promotion failures"); static u_long pmap_pte1_promotions; SYSCTL_ULONG(_vm_pmap_pte1, OID_AUTO, promotions, CTLFLAG_RD, &pmap_pte1_promotions, 0, "1MB page promotions"); static u_long pmap_pte1_kern_demotions; SYSCTL_ULONG(_vm_pmap_pte1, OID_AUTO, kern_demotions, CTLFLAG_RD, &pmap_pte1_kern_demotions, 0, "1MB page kernel demotions"); static u_long pmap_pte1_kern_promotions; SYSCTL_ULONG(_vm_pmap_pte1, OID_AUTO, kern_promotions, CTLFLAG_RD, &pmap_pte1_kern_promotions, 0, "1MB page kernel promotions"); static __inline ttb_entry_t pmap_ttb_get(pmap_t pmap) { return (vtophys(pmap->pm_pt1) | ttb_flags); } /* * Initialize a vm_page's machine-dependent fields. * * Variations: * 1. Pages for L2 page tables are always not managed. So, pv_list and * pt2_wirecount can share same physical space. However, proper * initialization on a page alloc for page tables and reinitialization * on the page free must be ensured. */ void pmap_page_init(vm_page_t m) { TAILQ_INIT(&m->md.pv_list); pt2_wirecount_init(m); m->md.pat_mode = VM_MEMATTR_DEFAULT; } /* * Virtualization for faster way how to zero whole page. */ static __inline void pagezero(void *page) { bzero(page, PAGE_SIZE); } /* * Zero L2 page table page. * Use same KVA as in pmap_zero_page(). */ static __inline vm_paddr_t pmap_pt2pg_zero(vm_page_t m) { vm_paddr_t pa; struct sysmaps *sysmaps; pa = VM_PAGE_TO_PHYS(m); /* * XXX: For now, we map whole page even if it's already zero, * to sync it even if the sync is only DSB. */ sched_pin(); sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)]; mtx_lock(&sysmaps->lock); if (pte2_load(sysmaps->CMAP2) != 0) panic("%s: CMAP2 busy", __func__); pte2_store(sysmaps->CMAP2, PTE2_KERN_NG(pa, PTE2_AP_KRW, vm_page_pte2_attr(m))); /* Even VM_ALLOC_ZERO request is only advisory. */ if ((m->flags & PG_ZERO) == 0) pagezero(sysmaps->CADDR2); pte2_sync_range((pt2_entry_t *)sysmaps->CADDR2, PAGE_SIZE); pte2_clear(sysmaps->CMAP2); tlb_flush((vm_offset_t)sysmaps->CADDR2); sched_unpin(); mtx_unlock(&sysmaps->lock); return (pa); } /* * Init just allocated page as L2 page table(s) holder * and return its physical address. */ static __inline vm_paddr_t pmap_pt2pg_init(pmap_t pmap, vm_offset_t va, vm_page_t m) { vm_paddr_t pa; pt2_entry_t *pte2p; /* Check page attributes. */ if (m->md.pat_mode != pt_memattr) pmap_page_set_memattr(m, pt_memattr); /* Zero page and init wire counts. */ pa = pmap_pt2pg_zero(m); pt2_wirecount_init(m); /* * Map page to PT2MAP address space for given pmap. * Note that PT2MAP space is shared with all pmaps. */ if (pmap == kernel_pmap) pmap_kenter_pt2tab(va, PTE2_KPT(pa)); else { pte2p = pmap_pt2tab_entry(pmap, va); pt2tab_store(pte2p, PTE2_KPT_NG(pa)); } return (pa); } /* * Initialize the pmap module. * Called by vm_init, to initialize any structures that the pmap * system needs to map virtual memory. */ void pmap_init(void) { vm_size_t s; pt2_entry_t *pte2p, pte2; u_int i, pte1_idx, pv_npg; PDEBUG(1, printf("%s: phys_start = %#x\n", __func__, PHYSADDR)); /* * Initialize the vm page array entries for kernel pmap's * L2 page table pages allocated in advance. */ pte1_idx = pte1_index(KERNBASE - PT2MAP_SIZE); pte2p = kern_pt2tab_entry(KERNBASE - PT2MAP_SIZE); for (i = 0; i < nkpt2pg + NPG_IN_PT2TAB; i++, pte2p++) { vm_paddr_t pa; vm_page_t m; pte2 = pte2_load(pte2p); KASSERT(pte2_is_valid(pte2), ("%s: no valid entry", __func__)); pa = pte2_pa(pte2); m = PHYS_TO_VM_PAGE(pa); KASSERT(m >= vm_page_array && m < &vm_page_array[vm_page_array_size], ("%s: L2 page table page is out of range", __func__)); m->pindex = pte1_idx; m->phys_addr = pa; pte1_idx += NPT2_IN_PG; } /* * Initialize the address space (zone) for the pv entries. Set a * high water mark so that the system can recover from excessive * numbers of pv entries. */ TUNABLE_INT_FETCH("vm.pmap.shpgperproc", &shpgperproc); pv_entry_max = shpgperproc * maxproc + vm_cnt.v_page_count; TUNABLE_INT_FETCH("vm.pmap.pv_entries", &pv_entry_max); pv_entry_max = roundup(pv_entry_max, _NPCPV); pv_entry_high_water = 9 * (pv_entry_max / 10); /* * Are large page mappings enabled? */ TUNABLE_INT_FETCH("vm.pmap.sp_enabled", &sp_enabled); if (sp_enabled) { KASSERT(MAXPAGESIZES > 1 && pagesizes[1] == 0, ("%s: can't assign to pagesizes[1]", __func__)); pagesizes[1] = PTE1_SIZE; } /* * Calculate the size of the pv head table for sections. * Handle the possibility that "vm_phys_segs[...].end" is zero. * Note that the table is only for sections which could be promoted. */ first_managed_pa = pte1_trunc(vm_phys_segs[0].start); pv_npg = (pte1_trunc(vm_phys_segs[vm_phys_nsegs - 1].end - PAGE_SIZE) - first_managed_pa) / PTE1_SIZE + 1; /* * Allocate memory for the pv head table for sections. */ s = (vm_size_t)(pv_npg * sizeof(struct md_page)); s = round_page(s); pv_table = (struct md_page *)kmem_malloc(kernel_arena, s, M_WAITOK | M_ZERO); for (i = 0; i < pv_npg; i++) TAILQ_INIT(&pv_table[i].pv_list); pv_maxchunks = MAX(pv_entry_max / _NPCPV, maxproc); pv_chunkbase = (struct pv_chunk *)kva_alloc(PAGE_SIZE * pv_maxchunks); if (pv_chunkbase == NULL) panic("%s: not enough kvm for pv chunks", __func__); pmap_pte2list_init(&pv_vafree, pv_chunkbase, pv_maxchunks); } /* * Add a list of wired pages to the kva * this routine is only used for temporary * kernel mappings that do not need to have * page modification or references recorded. * Note that old mappings are simply written * over. The page *must* be wired. * Note: SMP coherent. Uses a ranged shootdown IPI. */ void pmap_qenter(vm_offset_t sva, vm_page_t *ma, int count) { u_int anychanged; pt2_entry_t *epte2p, *pte2p, pte2; vm_page_t m; vm_paddr_t pa; anychanged = 0; pte2p = pt2map_entry(sva); epte2p = pte2p + count; while (pte2p < epte2p) { m = *ma++; pa = VM_PAGE_TO_PHYS(m); pte2 = pte2_load(pte2p); if ((pte2_pa(pte2) != pa) || (pte2_attr(pte2) != vm_page_pte2_attr(m))) { anychanged++; pte2_store(pte2p, PTE2_KERN(pa, PTE2_AP_KRW, vm_page_pte2_attr(m))); } pte2p++; } if (__predict_false(anychanged)) tlb_flush_range(sva, count * PAGE_SIZE); } /* * This routine tears out page mappings from the * kernel -- it is meant only for temporary mappings. * Note: SMP coherent. Uses a ranged shootdown IPI. */ void pmap_qremove(vm_offset_t sva, int count) { vm_offset_t va; va = sva; while (count-- > 0) { pmap_kremove(va); va += PAGE_SIZE; } tlb_flush_range(sva, va - sva); } /* * Are we current address space or kernel? */ static __inline int pmap_is_current(pmap_t pmap) { return (pmap == kernel_pmap || (pmap == vmspace_pmap(curthread->td_proc->p_vmspace))); } /* * If the given pmap is not the current or kernel pmap, the returned * pte2 must be released by passing it to pmap_pte2_release(). */ static pt2_entry_t * pmap_pte2(pmap_t pmap, vm_offset_t va) { pt1_entry_t pte1; vm_paddr_t pt2pg_pa; pte1 = pte1_load(pmap_pte1(pmap, va)); if (pte1_is_section(pte1)) panic("%s: attempt to map PTE1", __func__); if (pte1_is_link(pte1)) { /* Are we current address space or kernel? */ if (pmap_is_current(pmap)) return (pt2map_entry(va)); /* Note that L2 page table size is not equal to PAGE_SIZE. */ pt2pg_pa = trunc_page(pte1_link_pa(pte1)); mtx_lock(&PMAP2mutex); if (pte2_pa(pte2_load(PMAP2)) != pt2pg_pa) { pte2_store(PMAP2, PTE2_KPT(pt2pg_pa)); tlb_flush((vm_offset_t)PADDR2); } return (PADDR2 + (arm32_btop(va) & (NPTE2_IN_PG - 1))); } return (NULL); } /* * Releases a pte2 that was obtained from pmap_pte2(). * Be prepared for the pte2p being NULL. */ static __inline void pmap_pte2_release(pt2_entry_t *pte2p) { if ((pt2_entry_t *)(trunc_page((vm_offset_t)pte2p)) == PADDR2) { mtx_unlock(&PMAP2mutex); } } /* * Super fast pmap_pte2 routine best used when scanning * the pv lists. This eliminates many coarse-grained * invltlb calls. Note that many of the pv list * scans are across different pmaps. It is very wasteful * to do an entire tlb flush for checking a single mapping. * * If the given pmap is not the current pmap, pvh_global_lock * must be held and curthread pinned to a CPU. */ static pt2_entry_t * pmap_pte2_quick(pmap_t pmap, vm_offset_t va) { pt1_entry_t pte1; vm_paddr_t pt2pg_pa; pte1 = pte1_load(pmap_pte1(pmap, va)); if (pte1_is_section(pte1)) panic("%s: attempt to map PTE1", __func__); if (pte1_is_link(pte1)) { /* Are we current address space or kernel? */ if (pmap_is_current(pmap)) return (pt2map_entry(va)); rw_assert(&pvh_global_lock, RA_WLOCKED); KASSERT(curthread->td_pinned > 0, ("%s: curthread not pinned", __func__)); /* Note that L2 page table size is not equal to PAGE_SIZE. */ pt2pg_pa = trunc_page(pte1_link_pa(pte1)); if (pte2_pa(pte2_load(PMAP1)) != pt2pg_pa) { pte2_store(PMAP1, PTE2_KPT(pt2pg_pa)); #ifdef SMP PMAP1cpu = PCPU_GET(cpuid); #endif tlb_flush_local((vm_offset_t)PADDR1); PMAP1changed++; } else #ifdef SMP if (PMAP1cpu != PCPU_GET(cpuid)) { PMAP1cpu = PCPU_GET(cpuid); tlb_flush_local((vm_offset_t)PADDR1); PMAP1changedcpu++; } else #endif PMAP1unchanged++; return (PADDR1 + (arm32_btop(va) & (NPTE2_IN_PG - 1))); } return (NULL); } /* * Routine: pmap_extract * Function: * Extract the physical page address associated * with the given map/virtual_address pair. */ vm_paddr_t pmap_extract(pmap_t pmap, vm_offset_t va) { vm_paddr_t pa; pt1_entry_t pte1; pt2_entry_t *pte2p; PMAP_LOCK(pmap); pte1 = pte1_load(pmap_pte1(pmap, va)); if (pte1_is_section(pte1)) pa = pte1_pa(pte1) | (va & PTE1_OFFSET); else if (pte1_is_link(pte1)) { pte2p = pmap_pte2(pmap, va); pa = pte2_pa(pte2_load(pte2p)) | (va & PTE2_OFFSET); pmap_pte2_release(pte2p); } else pa = 0; PMAP_UNLOCK(pmap); return (pa); } /* * Routine: pmap_extract_and_hold * Function: * Atomically extract and hold the physical page * with the given pmap and virtual address pair * if that mapping permits the given protection. */ vm_page_t pmap_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot) { vm_paddr_t pa, lockpa; pt1_entry_t pte1; pt2_entry_t pte2, *pte2p; vm_page_t m; lockpa = 0; m = NULL; PMAP_LOCK(pmap); retry: pte1 = pte1_load(pmap_pte1(pmap, va)); if (pte1_is_section(pte1)) { if (!(pte1 & PTE1_RO) || !(prot & VM_PROT_WRITE)) { pa = pte1_pa(pte1) | (va & PTE1_OFFSET); if (vm_page_pa_tryrelock(pmap, pa, &lockpa)) goto retry; m = PHYS_TO_VM_PAGE(pa); vm_page_hold(m); } } else if (pte1_is_link(pte1)) { pte2p = pmap_pte2(pmap, va); pte2 = pte2_load(pte2p); pmap_pte2_release(pte2p); if (pte2_is_valid(pte2) && (!(pte2 & PTE2_RO) || !(prot & VM_PROT_WRITE))) { pa = pte2_pa(pte2); if (vm_page_pa_tryrelock(pmap, pa, &lockpa)) goto retry; m = PHYS_TO_VM_PAGE(pa); vm_page_hold(m); } } PA_UNLOCK_COND(lockpa); PMAP_UNLOCK(pmap); return (m); } /* * Grow the number of kernel L2 page table entries, if needed. */ void pmap_growkernel(vm_offset_t addr) { vm_page_t m; vm_paddr_t pt2pg_pa, pt2_pa; pt1_entry_t pte1; pt2_entry_t pte2; PDEBUG(1, printf("%s: addr = %#x\n", __func__, addr)); /* * All the time kernel_vm_end is first KVA for which underlying * L2 page table is either not allocated or linked from L1 page table * (not considering sections). Except for two possible cases: * * (1) in the very beginning as long as pmap_growkernel() was * not called, it could be first unused KVA (which is not * rounded up to PTE1_SIZE), * * (2) when all KVA space is mapped and kernel_map->max_offset * address is not rounded up to PTE1_SIZE. (For example, * it could be 0xFFFFFFFF.) */ kernel_vm_end = pte1_roundup(kernel_vm_end); mtx_assert(&kernel_map->system_mtx, MA_OWNED); addr = roundup2(addr, PTE1_SIZE); if (addr - 1 >= kernel_map->max_offset) addr = kernel_map->max_offset; while (kernel_vm_end < addr) { pte1 = pte1_load(kern_pte1(kernel_vm_end)); if (pte1_is_valid(pte1)) { kernel_vm_end += PTE1_SIZE; if (kernel_vm_end - 1 >= kernel_map->max_offset) { kernel_vm_end = kernel_map->max_offset; break; } continue; } /* * kernel_vm_end_new is used in pmap_pinit() when kernel * mappings are entered to new pmap all at once to avoid race * between pmap_kenter_pte1() and kernel_vm_end increase. * The same aplies to pmap_kenter_pt2tab(). */ kernel_vm_end_new = kernel_vm_end + PTE1_SIZE; pte2 = pt2tab_load(kern_pt2tab_entry(kernel_vm_end)); if (!pte2_is_valid(pte2)) { /* * Install new PT2s page into kernel PT2TAB. */ m = vm_page_alloc(NULL, pte1_index(kernel_vm_end) & ~PT2PG_MASK, VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO); if (m == NULL) panic("%s: no memory to grow kernel", __func__); /* * QQQ: To link all new L2 page tables from L1 page * table now and so pmap_kenter_pte1() them * at once together with pmap_kenter_pt2tab() * could be nice speed up. However, * pmap_growkernel() does not happen so often... * QQQ: The other TTBR is another option. */ pt2pg_pa = pmap_pt2pg_init(kernel_pmap, kernel_vm_end, m); } else pt2pg_pa = pte2_pa(pte2); pt2_pa = page_pt2pa(pt2pg_pa, pte1_index(kernel_vm_end)); pmap_kenter_pte1(kernel_vm_end, PTE1_LINK(pt2_pa)); kernel_vm_end = kernel_vm_end_new; if (kernel_vm_end - 1 >= kernel_map->max_offset) { kernel_vm_end = kernel_map->max_offset; break; } } } static int kvm_size(SYSCTL_HANDLER_ARGS) { unsigned long ksize = vm_max_kernel_address - KERNBASE; return (sysctl_handle_long(oidp, &ksize, 0, req)); } SYSCTL_PROC(_vm, OID_AUTO, kvm_size, CTLTYPE_LONG|CTLFLAG_RD, 0, 0, kvm_size, "IU", "Size of KVM"); static int kvm_free(SYSCTL_HANDLER_ARGS) { unsigned long kfree = vm_max_kernel_address - kernel_vm_end; return (sysctl_handle_long(oidp, &kfree, 0, req)); } SYSCTL_PROC(_vm, OID_AUTO, kvm_free, CTLTYPE_LONG|CTLFLAG_RD, 0, 0, kvm_free, "IU", "Amount of KVM free"); /*********************************************** * * Pmap allocation/deallocation routines. * ***********************************************/ /* * Initialize the pmap for the swapper process. */ void pmap_pinit0(pmap_t pmap) { PDEBUG(1, printf("%s: pmap = %p\n", __func__, pmap)); PMAP_LOCK_INIT(pmap); /* * Kernel page table directory and pmap stuff around is already * initialized, we are using it right now and here. So, finish * only PMAP structures initialization for process0 ... * * Since the L1 page table and PT2TAB is shared with the kernel pmap, * which is already included in the list "allpmaps", this pmap does * not need to be inserted into that list. */ pmap->pm_pt1 = kern_pt1; pmap->pm_pt2tab = kern_pt2tab; CPU_ZERO(&pmap->pm_active); PCPU_SET(curpmap, pmap); TAILQ_INIT(&pmap->pm_pvchunk); bzero(&pmap->pm_stats, sizeof pmap->pm_stats); CPU_SET(0, &pmap->pm_active); } static __inline void pte1_copy_nosync(pt1_entry_t *spte1p, pt1_entry_t *dpte1p, vm_offset_t sva, vm_offset_t eva) { u_int idx, count; idx = pte1_index(sva); count = (pte1_index(eva) - idx + 1) * sizeof(pt1_entry_t); bcopy(spte1p + idx, dpte1p + idx, count); } static __inline void pt2tab_copy_nosync(pt2_entry_t *spte2p, pt2_entry_t *dpte2p, vm_offset_t sva, vm_offset_t eva) { u_int idx, count; idx = pt2tab_index(sva); count = (pt2tab_index(eva) - idx + 1) * sizeof(pt2_entry_t); bcopy(spte2p + idx, dpte2p + idx, count); } /* * Initialize a preallocated and zeroed pmap structure, * such as one in a vmspace structure. */ int pmap_pinit(pmap_t pmap) { pt1_entry_t *pte1p; pt2_entry_t *pte2p; vm_paddr_t pa, pt2tab_pa; u_int i; PDEBUG(6, printf("%s: pmap = %p, pm_pt1 = %p\n", __func__, pmap, pmap->pm_pt1)); /* * No need to allocate L2 page table space yet but we do need * a valid L1 page table and PT2TAB table. * * Install shared kernel mappings to these tables. It's a little * tricky as some parts of KVA are reserved for vectors, devices, * and whatever else. These parts are supposed to be above * vm_max_kernel_address. Thus two regions should be installed: * * (1) . * * QQQ: The second region should be stable enough to be installed * only once in time when the tables are allocated. * QQQ: Maybe copy of both regions at once could be faster ... * QQQ: Maybe the other TTBR is an option. * * Finally, install own PT2TAB table to these tables. */ if (pmap->pm_pt1 == NULL) { pmap->pm_pt1 = (pt1_entry_t *)kmem_alloc_contig(kernel_arena, NB_IN_PT1, M_NOWAIT | M_ZERO, 0, -1UL, NB_IN_PT1, 0, pt_memattr); if (pmap->pm_pt1 == NULL) return (0); } if (pmap->pm_pt2tab == NULL) { /* * QQQ: (1) PT2TAB must be contiguous. If PT2TAB is one page * only, what should be the only size for 32 bit systems, * then we could allocate it with vm_page_alloc() and all * the stuff needed as other L2 page table pages. * (2) Note that a process PT2TAB is special L2 page table * page. Its mapping in kernel_arena is permanent and can * be used no matter which process is current. Its mapping * in PT2MAP can be used only for current process. */ pmap->pm_pt2tab = (pt2_entry_t *)kmem_alloc_attr(kernel_arena, NB_IN_PT2TAB, M_NOWAIT | M_ZERO, 0, -1UL, pt_memattr); if (pmap->pm_pt2tab == NULL) { /* * QQQ: As struct pmap is allocated from UMA with * UMA_ZONE_NOFREE flag, it's important to leave * no allocation in pmap if initialization failed. */ kmem_free(kernel_arena, (vm_offset_t)pmap->pm_pt1, NB_IN_PT1); pmap->pm_pt1 = NULL; return (0); } /* * QQQ: Each L2 page table page vm_page_t has pindex set to * pte1 index of virtual address mapped by this page. * It's not valid for non kernel PT2TABs themselves. * The pindex of these pages can not be altered because * of the way how they are allocated now. However, it * should not be a problem. */ } mtx_lock_spin(&allpmaps_lock); /* * To avoid race with pmap_kenter_pte1() and pmap_kenter_pt2tab(), * kernel_vm_end_new is used here instead of kernel_vm_end. */ pte1_copy_nosync(kern_pt1, pmap->pm_pt1, KERNBASE, kernel_vm_end_new - 1); pte1_copy_nosync(kern_pt1, pmap->pm_pt1, vm_max_kernel_address, 0xFFFFFFFF); pt2tab_copy_nosync(kern_pt2tab, pmap->pm_pt2tab, KERNBASE, kernel_vm_end_new - 1); pt2tab_copy_nosync(kern_pt2tab, pmap->pm_pt2tab, vm_max_kernel_address, 0xFFFFFFFF); LIST_INSERT_HEAD(&allpmaps, pmap, pm_list); mtx_unlock_spin(&allpmaps_lock); /* * Store PT2MAP PT2 pages (a.k.a. PT2TAB) in PT2TAB itself. * I.e. self reference mapping. The PT2TAB is private, however mapped * into shared PT2MAP space, so the mapping should be not global. */ pt2tab_pa = vtophys(pmap->pm_pt2tab); pte2p = pmap_pt2tab_entry(pmap, (vm_offset_t)PT2MAP); for (pa = pt2tab_pa, i = 0; i < NPG_IN_PT2TAB; i++, pa += PTE2_SIZE) { pt2tab_store(pte2p++, PTE2_KPT_NG(pa)); } /* Insert PT2MAP PT2s into pmap PT1. */ pte1p = pmap_pte1(pmap, (vm_offset_t)PT2MAP); for (pa = pt2tab_pa, i = 0; i < NPT2_IN_PT2TAB; i++, pa += NB_IN_PT2) { pte1_store(pte1p++, PTE1_LINK(pa)); } /* * Now synchronize new mapping which was made above. */ pte1_sync_range(pmap->pm_pt1, NB_IN_PT1); pte2_sync_range(pmap->pm_pt2tab, NB_IN_PT2TAB); CPU_ZERO(&pmap->pm_active); TAILQ_INIT(&pmap->pm_pvchunk); bzero(&pmap->pm_stats, sizeof pmap->pm_stats); return (1); } #ifdef INVARIANTS static boolean_t pt2tab_user_is_empty(pt2_entry_t *tab) { u_int i, end; end = pt2tab_index(VM_MAXUSER_ADDRESS); for (i = 0; i < end; i++) if (tab[i] != 0) return (FALSE); return (TRUE); } #endif /* * Release any resources held by the given physical map. * Called when a pmap initialized by pmap_pinit is being released. * Should only be called if the map contains no valid mappings. */ void pmap_release(pmap_t pmap) { #ifdef INVARIANTS vm_offset_t start, end; #endif KASSERT(pmap->pm_stats.resident_count == 0, ("%s: pmap resident count %ld != 0", __func__, pmap->pm_stats.resident_count)); KASSERT(pt2tab_user_is_empty(pmap->pm_pt2tab), ("%s: has allocated user PT2(s)", __func__)); KASSERT(CPU_EMPTY(&pmap->pm_active), ("%s: pmap %p is active on some CPU(s)", __func__, pmap)); mtx_lock_spin(&allpmaps_lock); LIST_REMOVE(pmap, pm_list); mtx_unlock_spin(&allpmaps_lock); #ifdef INVARIANTS start = pte1_index(KERNBASE) * sizeof(pt1_entry_t); end = (pte1_index(0xFFFFFFFF) + 1) * sizeof(pt1_entry_t); bzero((char *)pmap->pm_pt1 + start, end - start); start = pt2tab_index(KERNBASE) * sizeof(pt2_entry_t); end = (pt2tab_index(0xFFFFFFFF) + 1) * sizeof(pt2_entry_t); bzero((char *)pmap->pm_pt2tab + start, end - start); #endif /* * We are leaving PT1 and PT2TAB allocated on released pmap, * so hopefully UMA vmspace_zone will always be inited with * UMA_ZONE_NOFREE flag. */ } /********************************************************* * * L2 table pages and their pages management routines. * *********************************************************/ /* * Virtual interface for L2 page table wire counting. * * Each L2 page table in a page has own counter which counts a number of * valid mappings in a table. Global page counter counts mappings in all * tables in a page plus a single itself mapping in PT2TAB. * * During a promotion we leave the associated L2 page table counter * untouched, so the table (strictly speaking a page which holds it) * is never freed if promoted. * * If a page m->wire_count == 1 then no valid mappings exist in any L2 page * table in the page and the page itself is only mapped in PT2TAB. */ static __inline void pt2_wirecount_init(vm_page_t m) { u_int i; /* * Note: A page m is allocated with VM_ALLOC_WIRED flag and * m->wire_count should be already set correctly. * So, there is no need to set it again herein. */ for (i = 0; i < NPT2_IN_PG; i++) m->md.pt2_wirecount[i] = 0; } static __inline void pt2_wirecount_inc(vm_page_t m, uint32_t pte1_idx) { /* * Note: A just modificated pte2 (i.e. already allocated) * is acquiring one extra reference which must be * explicitly cleared. It influences the KASSERTs herein. * All L2 page tables in a page always belong to the same * pmap, so we allow only one extra reference for the page. */ KASSERT(m->md.pt2_wirecount[pte1_idx & PT2PG_MASK] < (NPTE2_IN_PT2 + 1), ("%s: PT2 is overflowing ...", __func__)); KASSERT(m->wire_count <= (NPTE2_IN_PG + 1), ("%s: PT2PG is overflowing ...", __func__)); m->wire_count++; m->md.pt2_wirecount[pte1_idx & PT2PG_MASK]++; } static __inline void pt2_wirecount_dec(vm_page_t m, uint32_t pte1_idx) { KASSERT(m->md.pt2_wirecount[pte1_idx & PT2PG_MASK] != 0, ("%s: PT2 is underflowing ...", __func__)); KASSERT(m->wire_count > 1, ("%s: PT2PG is underflowing ...", __func__)); m->wire_count--; m->md.pt2_wirecount[pte1_idx & PT2PG_MASK]--; } static __inline void pt2_wirecount_set(vm_page_t m, uint32_t pte1_idx, uint16_t count) { KASSERT(count <= NPTE2_IN_PT2, ("%s: invalid count %u", __func__, count)); KASSERT(m->wire_count > m->md.pt2_wirecount[pte1_idx & PT2PG_MASK], ("%s: PT2PG corrupting (%u, %u) ...", __func__, m->wire_count, m->md.pt2_wirecount[pte1_idx & PT2PG_MASK])); m->wire_count -= m->md.pt2_wirecount[pte1_idx & PT2PG_MASK]; m->wire_count += count; m->md.pt2_wirecount[pte1_idx & PT2PG_MASK] = count; KASSERT(m->wire_count <= (NPTE2_IN_PG + 1), ("%s: PT2PG is overflowed (%u) ...", __func__, m->wire_count)); } static __inline uint32_t pt2_wirecount_get(vm_page_t m, uint32_t pte1_idx) { return (m->md.pt2_wirecount[pte1_idx & PT2PG_MASK]); } static __inline boolean_t pt2_is_empty(vm_page_t m, vm_offset_t va) { return (m->md.pt2_wirecount[pte1_index(va) & PT2PG_MASK] == 0); } static __inline boolean_t pt2_is_full(vm_page_t m, vm_offset_t va) { return (m->md.pt2_wirecount[pte1_index(va) & PT2PG_MASK] == NPTE2_IN_PT2); } static __inline boolean_t pt2pg_is_empty(vm_page_t m) { return (m->wire_count == 1); } /* * This routine is called if the L2 page table * is not mapped correctly. */ static vm_page_t _pmap_allocpte2(pmap_t pmap, vm_offset_t va, u_int flags) { uint32_t pte1_idx; pt1_entry_t *pte1p; pt2_entry_t pte2; vm_page_t m; vm_paddr_t pt2pg_pa, pt2_pa; pte1_idx = pte1_index(va); pte1p = pmap->pm_pt1 + pte1_idx; KASSERT(pte1_load(pte1p) == 0, ("%s: pm_pt1[%#x] is not zero: %#x", __func__, pte1_idx, pte1_load(pte1p))); pte2 = pt2tab_load(pmap_pt2tab_entry(pmap, va)); if (!pte2_is_valid(pte2)) { /* * Install new PT2s page into pmap PT2TAB. */ m = vm_page_alloc(NULL, pte1_idx & ~PT2PG_MASK, VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO); if (m == NULL) { if ((flags & PMAP_ENTER_NOSLEEP) == 0) { PMAP_UNLOCK(pmap); rw_wunlock(&pvh_global_lock); VM_WAIT; rw_wlock(&pvh_global_lock); PMAP_LOCK(pmap); } /* * Indicate the need to retry. While waiting, * the L2 page table page may have been allocated. */ return (NULL); } pmap->pm_stats.resident_count++; pt2pg_pa = pmap_pt2pg_init(pmap, va, m); } else { pt2pg_pa = pte2_pa(pte2); m = PHYS_TO_VM_PAGE(pt2pg_pa); } pt2_wirecount_inc(m, pte1_idx); pt2_pa = page_pt2pa(pt2pg_pa, pte1_idx); pte1_store(pte1p, PTE1_LINK(pt2_pa)); return (m); } static vm_page_t pmap_allocpte2(pmap_t pmap, vm_offset_t va, u_int flags) { u_int pte1_idx; pt1_entry_t *pte1p, pte1; vm_page_t m; pte1_idx = pte1_index(va); retry: pte1p = pmap->pm_pt1 + pte1_idx; pte1 = pte1_load(pte1p); /* * This supports switching from a 1MB page to a * normal 4K page. */ if (pte1_is_section(pte1)) { (void)pmap_demote_pte1(pmap, pte1p, va); /* * Reload pte1 after demotion. * * Note: Demotion can even fail as either PT2 is not find for * the virtual address or PT2PG can not be allocated. */ pte1 = pte1_load(pte1p); } /* * If the L2 page table page is mapped, we just increment the * hold count, and activate it. */ if (pte1_is_link(pte1)) { m = PHYS_TO_VM_PAGE(pte1_link_pa(pte1)); pt2_wirecount_inc(m, pte1_idx); } else { /* * Here if the PT2 isn't mapped, or if it has * been deallocated. */ m = _pmap_allocpte2(pmap, va, flags); if (m == NULL && (flags & PMAP_ENTER_NOSLEEP) == 0) goto retry; } return (m); } static __inline void pmap_free_zero_pages(struct spglist *free) { vm_page_t m; while ((m = SLIST_FIRST(free)) != NULL) { SLIST_REMOVE_HEAD(free, plinks.s.ss); /* Preserve the page's PG_ZERO setting. */ vm_page_free_toq(m); } } /* * Schedule the specified unused L2 page table page to be freed. Specifically, * add the page to the specified list of pages that will be released to the * physical memory manager after the TLB has been updated. */ static __inline void pmap_add_delayed_free_list(vm_page_t m, struct spglist *free) { /* * Put page on a list so that it is released after * *ALL* TLB shootdown is done */ #ifdef PMAP_DEBUG pmap_zero_page_check(m); #endif m->flags |= PG_ZERO; SLIST_INSERT_HEAD(free, m, plinks.s.ss); } /* * Unwire L2 page tables page. */ static void pmap_unwire_pt2pg(pmap_t pmap, vm_offset_t va, vm_page_t m) { pt1_entry_t *pte1p, opte1 __unused; pt2_entry_t *pte2p; uint32_t i; KASSERT(pt2pg_is_empty(m), ("%s: pmap %p PT2PG %p wired", __func__, pmap, m)); /* * Unmap all L2 page tables in the page from L1 page table. * * QQQ: Individual L2 page tables (except the last one) can be unmapped * earlier. However, we are doing that this way. */ KASSERT(m->pindex == (pte1_index(va) & ~PT2PG_MASK), ("%s: pmap %p va %#x PT2PG %p bad index", __func__, pmap, va, m)); pte1p = pmap->pm_pt1 + m->pindex; for (i = 0; i < NPT2_IN_PG; i++, pte1p++) { KASSERT(m->md.pt2_wirecount[i] == 0, ("%s: pmap %p PT2 %u (PG %p) wired", __func__, pmap, i, m)); opte1 = pte1_load(pte1p); if (pte1_is_link(opte1)) { pte1_clear(pte1p); /* * Flush intermediate TLB cache. */ pmap_tlb_flush(pmap, (m->pindex + i) << PTE1_SHIFT); } #ifdef INVARIANTS else KASSERT((opte1 == 0) || pte1_is_section(opte1), ("%s: pmap %p va %#x bad pte1 %x at %u", __func__, pmap, va, opte1, i)); #endif } /* * Unmap the page from PT2TAB. */ pte2p = pmap_pt2tab_entry(pmap, va); (void)pt2tab_load_clear(pte2p); pmap_tlb_flush(pmap, pt2map_pt2pg(va)); m->wire_count = 0; pmap->pm_stats.resident_count--; /* * This is a release store so that the ordinary store unmapping * the L2 page table page is globally performed before TLB shoot- * down is begun. */ atomic_subtract_rel_int(&vm_cnt.v_wire_count, 1); } /* * Decrements a L2 page table page's wire count, which is used to record the * number of valid page table entries within the page. If the wire count * drops to zero, then the page table page is unmapped. Returns TRUE if the * page table page was unmapped and FALSE otherwise. */ static __inline boolean_t pmap_unwire_pt2(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) { pt2_wirecount_dec(m, pte1_index(va)); if (pt2pg_is_empty(m)) { /* * QQQ: Wire count is zero, so whole page should be zero and * we can set PG_ZERO flag to it. * Note that when promotion is enabled, it takes some * more efforts. See pmap_unwire_pt2_all() below. */ pmap_unwire_pt2pg(pmap, va, m); pmap_add_delayed_free_list(m, free); return (TRUE); } else return (FALSE); } /* * Drop a L2 page table page's wire count at once, which is used to record * the number of valid L2 page table entries within the page. If the wire * count drops to zero, then the L2 page table page is unmapped. */ static __inline void pmap_unwire_pt2_all(pmap_t pmap, vm_offset_t va, vm_page_t m, struct spglist *free) { u_int pte1_idx = pte1_index(va); KASSERT(m->pindex == (pte1_idx & ~PT2PG_MASK), ("%s: PT2 page's pindex is wrong", __func__)); KASSERT(m->wire_count > pt2_wirecount_get(m, pte1_idx), ("%s: bad pt2 wire count %u > %u", __func__, m->wire_count, pt2_wirecount_get(m, pte1_idx))); /* * It's possible that the L2 page table was never used. * It happened in case that a section was created without promotion. */ if (pt2_is_full(m, va)) { pt2_wirecount_set(m, pte1_idx, 0); /* * QQQ: We clear L2 page table now, so when L2 page table page * is going to be freed, we can set it PG_ZERO flag ... * This function is called only on section mappings, so * hopefully it's not to big overload. * * XXX: If pmap is current, existing PT2MAP mapping could be * used for zeroing. */ pmap_zero_page_area(m, page_pt2off(pte1_idx), NB_IN_PT2); } #ifdef INVARIANTS else KASSERT(pt2_is_empty(m, va), ("%s: PT2 is not empty (%u)", __func__, pt2_wirecount_get(m, pte1_idx))); #endif if (pt2pg_is_empty(m)) { pmap_unwire_pt2pg(pmap, va, m); pmap_add_delayed_free_list(m, free); } } /* * After removing a L2 page table entry, this routine is used to * conditionally free the page, and manage the hold/wire counts. */ static boolean_t pmap_unuse_pt2(pmap_t pmap, vm_offset_t va, struct spglist *free) { pt1_entry_t pte1; vm_page_t mpte; if (va >= VM_MAXUSER_ADDRESS) return (FALSE); pte1 = pte1_load(pmap_pte1(pmap, va)); mpte = PHYS_TO_VM_PAGE(pte1_link_pa(pte1)); return (pmap_unwire_pt2(pmap, va, mpte, free)); } /************************************* * * Page management routines. * *************************************/ CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE); CTASSERT(_NPCM == 11); CTASSERT(_NPCPV == 336); static __inline struct pv_chunk * pv_to_chunk(pv_entry_t pv) { return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK)); } #define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap) #define PC_FREE0_9 0xfffffffful /* Free values for index 0 through 9 */ #define PC_FREE10 0x0000fffful /* Free values for index 10 */ static const uint32_t pc_freemask[_NPCM] = { PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, PC_FREE0_9, PC_FREE10 }; SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_count, CTLFLAG_RD, &pv_entry_count, 0, "Current number of pv entries"); #ifdef PV_STATS static int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail; SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_count, CTLFLAG_RD, &pc_chunk_count, 0, "Current number of pv entry chunks"); SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_allocs, CTLFLAG_RD, &pc_chunk_allocs, 0, "Current number of pv entry chunks allocated"); SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_frees, CTLFLAG_RD, &pc_chunk_frees, 0, "Current number of pv entry chunks frees"); SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_tryfail, CTLFLAG_RD, &pc_chunk_tryfail, 0, "Number of times tried to get a chunk page but failed."); static long pv_entry_frees, pv_entry_allocs; static int pv_entry_spare; SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_frees, CTLFLAG_RD, &pv_entry_frees, 0, "Current number of pv entry frees"); SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_allocs, CTLFLAG_RD, &pv_entry_allocs, 0, "Current number of pv entry allocs"); SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_spare, CTLFLAG_RD, &pv_entry_spare, 0, "Current number of spare pv entries"); #endif /* * Is given page managed? */ static __inline boolean_t is_managed(vm_paddr_t pa) { vm_offset_t pgnum; vm_page_t m; pgnum = atop(pa); if (pgnum >= first_page) { m = PHYS_TO_VM_PAGE(pa); if (m == NULL) return (FALSE); if ((m->oflags & VPO_UNMANAGED) == 0) return (TRUE); } return (FALSE); } static __inline boolean_t pte1_is_managed(pt1_entry_t pte1) { return (is_managed(pte1_pa(pte1))); } static __inline boolean_t pte2_is_managed(pt2_entry_t pte2) { return (is_managed(pte2_pa(pte2))); } /* * We are in a serious low memory condition. Resort to * drastic measures to free some pages so we can allocate * another pv entry chunk. */ static vm_page_t pmap_pv_reclaim(pmap_t locked_pmap) { struct pch newtail; struct pv_chunk *pc; struct md_page *pvh; pt1_entry_t *pte1p; pmap_t pmap; pt2_entry_t *pte2p, tpte2; pv_entry_t pv; vm_offset_t va; vm_page_t m, m_pc; struct spglist free; uint32_t inuse; int bit, field, freed; PMAP_LOCK_ASSERT(locked_pmap, MA_OWNED); pmap = NULL; m_pc = NULL; SLIST_INIT(&free); TAILQ_INIT(&newtail); while ((pc = TAILQ_FIRST(&pv_chunks)) != NULL && (pv_vafree == 0 || SLIST_EMPTY(&free))) { TAILQ_REMOVE(&pv_chunks, pc, pc_lru); if (pmap != pc->pc_pmap) { if (pmap != NULL) { if (pmap != locked_pmap) PMAP_UNLOCK(pmap); } pmap = pc->pc_pmap; /* Avoid deadlock and lock recursion. */ if (pmap > locked_pmap) PMAP_LOCK(pmap); else if (pmap != locked_pmap && !PMAP_TRYLOCK(pmap)) { pmap = NULL; TAILQ_INSERT_TAIL(&newtail, pc, pc_lru); continue; } } /* * Destroy every non-wired, 4 KB page mapping in the chunk. */ freed = 0; for (field = 0; field < _NPCM; field++) { for (inuse = ~pc->pc_map[field] & pc_freemask[field]; inuse != 0; inuse &= ~(1UL << bit)) { bit = ffs(inuse) - 1; pv = &pc->pc_pventry[field * 32 + bit]; va = pv->pv_va; pte1p = pmap_pte1(pmap, va); if (pte1_is_section(pte1_load(pte1p))) continue; pte2p = pmap_pte2(pmap, va); tpte2 = pte2_load(pte2p); if ((tpte2 & PTE2_W) == 0) tpte2 = pte2_load_clear(pte2p); pmap_pte2_release(pte2p); if ((tpte2 & PTE2_W) != 0) continue; KASSERT(tpte2 != 0, ("pmap_pv_reclaim: pmap %p va %#x zero pte", pmap, va)); pmap_tlb_flush(pmap, va); m = PHYS_TO_VM_PAGE(pte2_pa(tpte2)); if (pte2_is_dirty(tpte2)) vm_page_dirty(m); if ((tpte2 & PTE2_A) != 0) vm_page_aflag_set(m, PGA_REFERENCED); TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); if (TAILQ_EMPTY(&m->md.pv_list) && (m->flags & PG_FICTITIOUS) == 0) { pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); if (TAILQ_EMPTY(&pvh->pv_list)) { vm_page_aflag_clear(m, PGA_WRITEABLE); } } pc->pc_map[field] |= 1UL << bit; pmap_unuse_pt2(pmap, va, &free); freed++; } } if (freed == 0) { TAILQ_INSERT_TAIL(&newtail, pc, pc_lru); continue; } /* Every freed mapping is for a 4 KB page. */ pmap->pm_stats.resident_count -= freed; PV_STAT(pv_entry_frees += freed); PV_STAT(pv_entry_spare += freed); pv_entry_count -= freed; TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); for (field = 0; field < _NPCM; field++) if (pc->pc_map[field] != pc_freemask[field]) { TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); TAILQ_INSERT_TAIL(&newtail, pc, pc_lru); /* * One freed pv entry in locked_pmap is * sufficient. */ if (pmap == locked_pmap) goto out; break; } if (field == _NPCM) { PV_STAT(pv_entry_spare -= _NPCPV); PV_STAT(pc_chunk_count--); PV_STAT(pc_chunk_frees++); /* Entire chunk is free; return it. */ m_pc = PHYS_TO_VM_PAGE(pmap_kextract((vm_offset_t)pc)); pmap_qremove((vm_offset_t)pc, 1); pmap_pte2list_free(&pv_vafree, (vm_offset_t)pc); break; } } out: TAILQ_CONCAT(&pv_chunks, &newtail, pc_lru); if (pmap != NULL) { if (pmap != locked_pmap) PMAP_UNLOCK(pmap); } if (m_pc == NULL && pv_vafree != 0 && SLIST_EMPTY(&free)) { m_pc = SLIST_FIRST(&free); SLIST_REMOVE_HEAD(&free, plinks.s.ss); /* Recycle a freed page table page. */ m_pc->wire_count = 1; atomic_add_int(&vm_cnt.v_wire_count, 1); } pmap_free_zero_pages(&free); return (m_pc); } static void free_pv_chunk(struct pv_chunk *pc) { vm_page_t m; TAILQ_REMOVE(&pv_chunks, pc, pc_lru); PV_STAT(pv_entry_spare -= _NPCPV); PV_STAT(pc_chunk_count--); PV_STAT(pc_chunk_frees++); /* entire chunk is free, return it */ m = PHYS_TO_VM_PAGE(pmap_kextract((vm_offset_t)pc)); pmap_qremove((vm_offset_t)pc, 1); vm_page_unwire(m, PQ_NONE); vm_page_free(m); pmap_pte2list_free(&pv_vafree, (vm_offset_t)pc); } /* * Free the pv_entry back to the free list. */ static void free_pv_entry(pmap_t pmap, pv_entry_t pv) { struct pv_chunk *pc; int idx, field, bit; rw_assert(&pvh_global_lock, RA_WLOCKED); PMAP_LOCK_ASSERT(pmap, MA_OWNED); PV_STAT(pv_entry_frees++); PV_STAT(pv_entry_spare++); pv_entry_count--; pc = pv_to_chunk(pv); idx = pv - &pc->pc_pventry[0]; field = idx / 32; bit = idx % 32; pc->pc_map[field] |= 1ul << bit; for (idx = 0; idx < _NPCM; idx++) if (pc->pc_map[idx] != pc_freemask[idx]) { /* * 98% of the time, pc is already at the head of the * list. If it isn't already, move it to the head. */ if (__predict_false(TAILQ_FIRST(&pmap->pm_pvchunk) != pc)) { TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); } return; } TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); free_pv_chunk(pc); } /* * Get a new pv_entry, allocating a block from the system * when needed. */ static pv_entry_t get_pv_entry(pmap_t pmap, boolean_t try) { static const struct timeval printinterval = { 60, 0 }; static struct timeval lastprint; int bit, field; pv_entry_t pv; struct pv_chunk *pc; vm_page_t m; rw_assert(&pvh_global_lock, RA_WLOCKED); PMAP_LOCK_ASSERT(pmap, MA_OWNED); PV_STAT(pv_entry_allocs++); pv_entry_count++; if (pv_entry_count > pv_entry_high_water) if (ratecheck(&lastprint, &printinterval)) printf("Approaching the limit on PV entries, consider " "increasing either the vm.pmap.shpgperproc or the " "vm.pmap.pv_entry_max tunable.\n"); retry: pc = TAILQ_FIRST(&pmap->pm_pvchunk); if (pc != NULL) { for (field = 0; field < _NPCM; field++) { if (pc->pc_map[field]) { bit = ffs(pc->pc_map[field]) - 1; break; } } if (field < _NPCM) { pv = &pc->pc_pventry[field * 32 + bit]; pc->pc_map[field] &= ~(1ul << bit); /* If this was the last item, move it to tail */ for (field = 0; field < _NPCM; field++) if (pc->pc_map[field] != 0) { PV_STAT(pv_entry_spare--); return (pv); /* not full, return */ } TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); PV_STAT(pv_entry_spare--); return (pv); } } /* * Access to the pte2list "pv_vafree" is synchronized by the pvh * global lock. If "pv_vafree" is currently non-empty, it will * remain non-empty until pmap_pte2list_alloc() completes. */ if (pv_vafree == 0 || (m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED)) == NULL) { if (try) { pv_entry_count--; PV_STAT(pc_chunk_tryfail++); return (NULL); } m = pmap_pv_reclaim(pmap); if (m == NULL) goto retry; } PV_STAT(pc_chunk_count++); PV_STAT(pc_chunk_allocs++); pc = (struct pv_chunk *)pmap_pte2list_alloc(&pv_vafree); pmap_qenter((vm_offset_t)pc, &m, 1); pc->pc_pmap = pmap; pc->pc_map[0] = pc_freemask[0] & ~1ul; /* preallocated bit 0 */ for (field = 1; field < _NPCM; field++) pc->pc_map[field] = pc_freemask[field]; TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru); pv = &pc->pc_pventry[0]; TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); PV_STAT(pv_entry_spare += _NPCPV - 1); return (pv); } /* * Create a pv entry for page at pa for * (pmap, va). */ static void pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m) { pv_entry_t pv; rw_assert(&pvh_global_lock, RA_WLOCKED); PMAP_LOCK_ASSERT(pmap, MA_OWNED); pv = get_pv_entry(pmap, FALSE); pv->pv_va = va; TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); } static __inline pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap, vm_offset_t va) { pv_entry_t pv; rw_assert(&pvh_global_lock, RA_WLOCKED); TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { if (pmap == PV_PMAP(pv) && va == pv->pv_va) { TAILQ_REMOVE(&pvh->pv_list, pv, pv_next); break; } } return (pv); } static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va) { pv_entry_t pv; pv = pmap_pvh_remove(pvh, pmap, va); KASSERT(pv != NULL, ("pmap_pvh_free: pv not found")); free_pv_entry(pmap, pv); } static void pmap_remove_entry(pmap_t pmap, vm_page_t m, vm_offset_t va) { struct md_page *pvh; rw_assert(&pvh_global_lock, RA_WLOCKED); pmap_pvh_free(&m->md, pmap, va); if (TAILQ_EMPTY(&m->md.pv_list) && (m->flags & PG_FICTITIOUS) == 0) { pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); if (TAILQ_EMPTY(&pvh->pv_list)) vm_page_aflag_clear(m, PGA_WRITEABLE); } } static void pmap_pv_demote_pte1(pmap_t pmap, vm_offset_t va, vm_paddr_t pa) { struct md_page *pvh; pv_entry_t pv; vm_offset_t va_last; vm_page_t m; rw_assert(&pvh_global_lock, RA_WLOCKED); KASSERT((pa & PTE1_OFFSET) == 0, ("pmap_pv_demote_pte1: pa is not 1mpage aligned")); /* * Transfer the 1mpage's pv entry for this mapping to the first * page's pv list. */ pvh = pa_to_pvh(pa); va = pte1_trunc(va); pv = pmap_pvh_remove(pvh, pmap, va); KASSERT(pv != NULL, ("pmap_pv_demote_pte1: pv not found")); m = PHYS_TO_VM_PAGE(pa); TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); /* Instantiate the remaining NPTE2_IN_PT2 - 1 pv entries. */ va_last = va + PTE1_SIZE - PAGE_SIZE; do { m++; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_pv_demote_pte1: page %p is not managed", m)); va += PAGE_SIZE; pmap_insert_entry(pmap, va, m); } while (va < va_last); } static void pmap_pv_promote_pte1(pmap_t pmap, vm_offset_t va, vm_paddr_t pa) { struct md_page *pvh; pv_entry_t pv; vm_offset_t va_last; vm_page_t m; rw_assert(&pvh_global_lock, RA_WLOCKED); KASSERT((pa & PTE1_OFFSET) == 0, ("pmap_pv_promote_pte1: pa is not 1mpage aligned")); /* * Transfer the first page's pv entry for this mapping to the * 1mpage's pv list. Aside from avoiding the cost of a call * to get_pv_entry(), a transfer avoids the possibility that * get_pv_entry() calls pmap_pv_reclaim() and that pmap_pv_reclaim() * removes one of the mappings that is being promoted. */ m = PHYS_TO_VM_PAGE(pa); va = pte1_trunc(va); pv = pmap_pvh_remove(&m->md, pmap, va); KASSERT(pv != NULL, ("pmap_pv_promote_pte1: pv not found")); pvh = pa_to_pvh(pa); TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next); /* Free the remaining NPTE2_IN_PT2 - 1 pv entries. */ va_last = va + PTE1_SIZE - PAGE_SIZE; do { m++; va += PAGE_SIZE; pmap_pvh_free(&m->md, pmap, va); } while (va < va_last); } /* * Conditionally create a pv entry. */ static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m) { pv_entry_t pv; rw_assert(&pvh_global_lock, RA_WLOCKED); PMAP_LOCK_ASSERT(pmap, MA_OWNED); if (pv_entry_count < pv_entry_high_water && (pv = get_pv_entry(pmap, TRUE)) != NULL) { pv->pv_va = va; TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); return (TRUE); } else return (FALSE); } /* * Create the pv entries for each of the pages within a section. */ static boolean_t pmap_pv_insert_pte1(pmap_t pmap, vm_offset_t va, vm_paddr_t pa) { struct md_page *pvh; pv_entry_t pv; rw_assert(&pvh_global_lock, RA_WLOCKED); if (pv_entry_count < pv_entry_high_water && (pv = get_pv_entry(pmap, TRUE)) != NULL) { pv->pv_va = va; pvh = pa_to_pvh(pa); TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next); return (TRUE); } else return (FALSE); } static inline void pmap_tlb_flush_pte1(pmap_t pmap, vm_offset_t va, pt1_entry_t npte1) { /* Kill all the small mappings or the big one only. */ if (pte1_is_section(npte1)) pmap_tlb_flush_range(pmap, pte1_trunc(va), PTE1_SIZE); else pmap_tlb_flush(pmap, pte1_trunc(va)); } /* * Update kernel pte1 on all pmaps. * * The following function is called only on one cpu with disabled interrupts. * In SMP case, smp_rendezvous_cpus() is used to stop other cpus. This way * nobody can invoke explicit hardware table walk during the update of pte1. * Unsolicited hardware table walk can still happen, invoked by speculative * data or instruction prefetch or even by speculative hardware table walk. * * The break-before-make approach should be implemented here. However, it's * not so easy to do that for kernel mappings as it would be unhappy to unmap * itself unexpectedly but voluntarily. */ static void pmap_update_pte1_kernel(vm_offset_t va, pt1_entry_t npte1) { pmap_t pmap; pt1_entry_t *pte1p; /* * Get current pmap. Interrupts should be disabled here * so PCPU_GET() is done atomically. */ pmap = PCPU_GET(curpmap); if (pmap == NULL) pmap = kernel_pmap; /* * (1) Change pte1 on current pmap. * (2) Flush all obsolete TLB entries on current CPU. * (3) Change pte1 on all pmaps. * (4) Flush all obsolete TLB entries on all CPUs in SMP case. */ pte1p = pmap_pte1(pmap, va); pte1_store(pte1p, npte1); /* Kill all the small mappings or the big one only. */ if (pte1_is_section(npte1)) { pmap_pte1_kern_promotions++; tlb_flush_range_local(pte1_trunc(va), PTE1_SIZE); } else { pmap_pte1_kern_demotions++; tlb_flush_local(pte1_trunc(va)); } /* * In SMP case, this function is called when all cpus are at smp * rendezvous, so there is no need to use 'allpmaps_lock' lock here. * In UP case, the function is called with this lock locked. */ LIST_FOREACH(pmap, &allpmaps, pm_list) { pte1p = pmap_pte1(pmap, va); pte1_store(pte1p, npte1); } #ifdef SMP /* Kill all the small mappings or the big one only. */ if (pte1_is_section(npte1)) tlb_flush_range(pte1_trunc(va), PTE1_SIZE); else tlb_flush(pte1_trunc(va)); #endif } #ifdef SMP struct pte1_action { vm_offset_t va; pt1_entry_t npte1; u_int update; /* CPU that updates the PTE1 */ }; static void pmap_update_pte1_action(void *arg) { struct pte1_action *act = arg; if (act->update == PCPU_GET(cpuid)) pmap_update_pte1_kernel(act->va, act->npte1); } /* * Change pte1 on current pmap. * Note that kernel pte1 must be changed on all pmaps. * * By ARM ARM manual, the behaviour is UNPREDICABLE when two or more TLB * entries map same VA. It's a problem when either promotion or demotion * is being done. The pte1 update and appropriate TLB flush must be done * atomically in general. */ static void pmap_change_pte1(pmap_t pmap, pt1_entry_t *pte1p, vm_offset_t va, pt1_entry_t npte1) { if (pmap == kernel_pmap) { struct pte1_action act; sched_pin(); act.va = va; act.npte1 = npte1; act.update = PCPU_GET(cpuid); smp_rendezvous_cpus(all_cpus, smp_no_rendevous_barrier, pmap_update_pte1_action, NULL, &act); sched_unpin(); } else { register_t cspr; /* * Use break-before-make approach for changing userland * mappings. It can cause L1 translation aborts on other * cores in SMP case. So, special treatment is implemented * in pmap_fault(). Interrups are disabled here to make it * without any interruption as quick as possible. */ cspr = disable_interrupts(PSR_I | PSR_F); pte1_clear(pte1p); pmap_tlb_flush_pte1(pmap, va, npte1); pte1_store(pte1p, npte1); restore_interrupts(cspr); } } #else static void pmap_change_pte1(pmap_t pmap, pt1_entry_t *pte1p, vm_offset_t va, pt1_entry_t npte1) { if (pmap == kernel_pmap) { mtx_lock_spin(&allpmaps_lock); pmap_update_pte1_kernel(va, npte1); mtx_unlock_spin(&allpmaps_lock); } else { register_t cspr; /* * Use break-before-make approach for changing userland * mappings. It's absolutely safe in UP case when interrupts * are disabled. */ cspr = disable_interrupts(PSR_I | PSR_F); pte1_clear(pte1p); pmap_tlb_flush_pte1(pmap, va, npte1); pte1_store(pte1p, npte1); restore_interrupts(cspr); } } #endif /* * Tries to promote the NPTE2_IN_PT2, contiguous 4KB page mappings that are * within a single page table page (PT2) to a single 1MB page mapping. * For promotion to occur, two conditions must be met: (1) the 4KB page * mappings must map aligned, contiguous physical memory and (2) the 4KB page * mappings must have identical characteristics. * * Managed (PG_MANAGED) mappings within the kernel address space are not * promoted. The reason is that kernel PTE1s are replicated in each pmap but * pmap_remove_write(), pmap_clear_modify(), and pmap_clear_reference() only * read the PTE1 from the kernel pmap. */ static void pmap_promote_pte1(pmap_t pmap, pt1_entry_t *pte1p, vm_offset_t va) { pt1_entry_t npte1; pt2_entry_t *fpte2p, fpte2, fpte2_fav; pt2_entry_t *pte2p, pte2; vm_offset_t pteva __unused; vm_page_t m __unused; PDEBUG(6, printf("%s(%p): try for va %#x pte1 %#x at %p\n", __func__, pmap, va, pte1_load(pte1p), pte1p)); PMAP_LOCK_ASSERT(pmap, MA_OWNED); /* * Examine the first PTE2 in the specified PT2. Abort if this PTE2 is * either invalid, unused, or does not map the first 4KB physical page * within a 1MB page. */ fpte2p = pmap_pte2_quick(pmap, pte1_trunc(va)); fpte2 = pte2_load(fpte2p); if ((fpte2 & ((PTE2_FRAME & PTE1_OFFSET) | PTE2_A | PTE2_V)) != (PTE2_A | PTE2_V)) { pmap_pte1_p_failures++; CTR3(KTR_PMAP, "%s: failure(1) for va %#x in pmap %p", __func__, va, pmap); return; } if (pte2_is_managed(fpte2) && pmap == kernel_pmap) { pmap_pte1_p_failures++; CTR3(KTR_PMAP, "%s: failure(2) for va %#x in pmap %p", __func__, va, pmap); return; } if ((fpte2 & (PTE2_NM | PTE2_RO)) == PTE2_NM) { /* * When page is not modified, PTE2_RO can be set without * a TLB invalidation. */ fpte2 |= PTE2_RO; pte2_store(fpte2p, fpte2); } /* * Examine each of the other PTE2s in the specified PT2. Abort if this * PTE2 maps an unexpected 4KB physical page or does not have identical * characteristics to the first PTE2. */ fpte2_fav = (fpte2 & (PTE2_FRAME | PTE2_A | PTE2_V)); fpte2_fav += PTE1_SIZE - PTE2_SIZE; /* examine from the end */ for (pte2p = fpte2p + NPTE2_IN_PT2 - 1; pte2p > fpte2p; pte2p--) { pte2 = pte2_load(pte2p); if ((pte2 & (PTE2_FRAME | PTE2_A | PTE2_V)) != fpte2_fav) { pmap_pte1_p_failures++; CTR3(KTR_PMAP, "%s: failure(3) for va %#x in pmap %p", __func__, va, pmap); return; } if ((pte2 & (PTE2_NM | PTE2_RO)) == PTE2_NM) { /* * When page is not modified, PTE2_RO can be set * without a TLB invalidation. See note above. */ pte2 |= PTE2_RO; pte2_store(pte2p, pte2); pteva = pte1_trunc(va) | (pte2 & PTE1_OFFSET & PTE2_FRAME); CTR3(KTR_PMAP, "%s: protect for va %#x in pmap %p", __func__, pteva, pmap); } if ((pte2 & PTE2_PROMOTE) != (fpte2 & PTE2_PROMOTE)) { pmap_pte1_p_failures++; CTR3(KTR_PMAP, "%s: failure(4) for va %#x in pmap %p", __func__, va, pmap); return; } fpte2_fav -= PTE2_SIZE; } /* * The page table page in its current state will stay in PT2TAB * until the PTE1 mapping the section is demoted by pmap_demote_pte1() * or destroyed by pmap_remove_pte1(). * * Note that L2 page table size is not equal to PAGE_SIZE. */ m = PHYS_TO_VM_PAGE(trunc_page(pte1_link_pa(pte1_load(pte1p)))); KASSERT(m >= vm_page_array && m < &vm_page_array[vm_page_array_size], ("%s: PT2 page is out of range", __func__)); KASSERT(m->pindex == (pte1_index(va) & ~PT2PG_MASK), ("%s: PT2 page's pindex is wrong", __func__)); /* * Get pte1 from pte2 format. */ npte1 = (fpte2 & PTE1_FRAME) | ATTR_TO_L1(fpte2) | PTE1_V; /* * Promote the pv entries. */ if (pte2_is_managed(fpte2)) pmap_pv_promote_pte1(pmap, va, pte1_pa(npte1)); /* * Promote the mappings. */ pmap_change_pte1(pmap, pte1p, va, npte1); pmap_pte1_promotions++; CTR3(KTR_PMAP, "%s: success for va %#x in pmap %p", __func__, va, pmap); PDEBUG(6, printf("%s(%p): success for va %#x pte1 %#x(%#x) at %p\n", __func__, pmap, va, npte1, pte1_load(pte1p), pte1p)); } /* * Zero L2 page table page. */ static __inline void pmap_clear_pt2(pt2_entry_t *fpte2p) { pt2_entry_t *pte2p; for (pte2p = fpte2p; pte2p < fpte2p + NPTE2_IN_PT2; pte2p++) pte2_clear(pte2p); } /* * Removes a 1MB page mapping from the kernel pmap. */ static void pmap_remove_kernel_pte1(pmap_t pmap, pt1_entry_t *pte1p, vm_offset_t va) { vm_page_t m; uint32_t pte1_idx; pt2_entry_t *fpte2p; vm_paddr_t pt2_pa; PMAP_LOCK_ASSERT(pmap, MA_OWNED); m = pmap_pt2_page(pmap, va); if (m == NULL) /* * QQQ: Is this function called only on promoted pte1? * We certainly do section mappings directly * (without promotion) in kernel !!! */ panic("%s: missing pt2 page", __func__); pte1_idx = pte1_index(va); /* * Initialize the L2 page table. */ fpte2p = page_pt2(pt2map_pt2pg(va), pte1_idx); pmap_clear_pt2(fpte2p); /* * Remove the mapping. */ pt2_pa = page_pt2pa(VM_PAGE_TO_PHYS(m), pte1_idx); pmap_kenter_pte1(va, PTE1_LINK(pt2_pa)); /* * QQQ: We do not need to invalidate PT2MAP mapping * as we did not change it. I.e. the L2 page table page * was and still is mapped the same way. */ } /* * Do the things to unmap a section in a process */ static void pmap_remove_pte1(pmap_t pmap, pt1_entry_t *pte1p, vm_offset_t sva, struct spglist *free) { pt1_entry_t opte1; struct md_page *pvh; vm_offset_t eva, va; vm_page_t m; PDEBUG(6, printf("%s(%p): va %#x pte1 %#x at %p\n", __func__, pmap, sva, pte1_load(pte1p), pte1p)); PMAP_LOCK_ASSERT(pmap, MA_OWNED); KASSERT((sva & PTE1_OFFSET) == 0, ("%s: sva is not 1mpage aligned", __func__)); /* * Clear and invalidate the mapping. It should occupy one and only TLB * entry. So, pmap_tlb_flush() called with aligned address should be * sufficient. */ opte1 = pte1_load_clear(pte1p); pmap_tlb_flush(pmap, sva); if (pte1_is_wired(opte1)) pmap->pm_stats.wired_count -= PTE1_SIZE / PAGE_SIZE; pmap->pm_stats.resident_count -= PTE1_SIZE / PAGE_SIZE; if (pte1_is_managed(opte1)) { pvh = pa_to_pvh(pte1_pa(opte1)); pmap_pvh_free(pvh, pmap, sva); eva = sva + PTE1_SIZE; for (va = sva, m = PHYS_TO_VM_PAGE(pte1_pa(opte1)); va < eva; va += PAGE_SIZE, m++) { if (pte1_is_dirty(opte1)) vm_page_dirty(m); if (opte1 & PTE1_A) vm_page_aflag_set(m, PGA_REFERENCED); if (TAILQ_EMPTY(&m->md.pv_list) && TAILQ_EMPTY(&pvh->pv_list)) vm_page_aflag_clear(m, PGA_WRITEABLE); } } if (pmap == kernel_pmap) { /* * L2 page table(s) can't be removed from kernel map as * kernel counts on it (stuff around pmap_growkernel()). */ pmap_remove_kernel_pte1(pmap, pte1p, sva); } else { /* * Get associated L2 page table page. * It's possible that the page was never allocated. */ m = pmap_pt2_page(pmap, sva); if (m != NULL) pmap_unwire_pt2_all(pmap, sva, m, free); } } /* * Fills L2 page table page with mappings to consecutive physical pages. */ static __inline void pmap_fill_pt2(pt2_entry_t *fpte2p, pt2_entry_t npte2) { pt2_entry_t *pte2p; for (pte2p = fpte2p; pte2p < fpte2p + NPTE2_IN_PT2; pte2p++) { pte2_store(pte2p, npte2); npte2 += PTE2_SIZE; } } /* * Tries to demote a 1MB page mapping. If demotion fails, the * 1MB page mapping is invalidated. */ static boolean_t pmap_demote_pte1(pmap_t pmap, pt1_entry_t *pte1p, vm_offset_t va) { pt1_entry_t opte1, npte1; pt2_entry_t *fpte2p, npte2; vm_paddr_t pt2pg_pa, pt2_pa; vm_page_t m; struct spglist free; uint32_t pte1_idx, isnew = 0; PDEBUG(6, printf("%s(%p): try for va %#x pte1 %#x at %p\n", __func__, pmap, va, pte1_load(pte1p), pte1p)); PMAP_LOCK_ASSERT(pmap, MA_OWNED); opte1 = pte1_load(pte1p); KASSERT(pte1_is_section(opte1), ("%s: opte1 not a section", __func__)); if ((opte1 & PTE1_A) == 0 || (m = pmap_pt2_page(pmap, va)) == NULL) { KASSERT(!pte1_is_wired(opte1), ("%s: PT2 page for a wired mapping is missing", __func__)); /* * Invalidate the 1MB page mapping and return * "failure" if the mapping was never accessed or the * allocation of the new page table page fails. */ if ((opte1 & PTE1_A) == 0 || (m = vm_page_alloc(NULL, pte1_index(va) & ~PT2PG_MASK, VM_ALLOC_NOOBJ | VM_ALLOC_NORMAL | VM_ALLOC_WIRED)) == NULL) { SLIST_INIT(&free); pmap_remove_pte1(pmap, pte1p, pte1_trunc(va), &free); pmap_free_zero_pages(&free); CTR3(KTR_PMAP, "%s: failure for va %#x in pmap %p", __func__, va, pmap); return (FALSE); } if (va < VM_MAXUSER_ADDRESS) pmap->pm_stats.resident_count++; isnew = 1; /* * We init all L2 page tables in the page even if * we are going to change everything for one L2 page * table in a while. */ pt2pg_pa = pmap_pt2pg_init(pmap, va, m); } else { if (va < VM_MAXUSER_ADDRESS) { if (pt2_is_empty(m, va)) isnew = 1; /* Demoting section w/o promotion. */ #ifdef INVARIANTS else KASSERT(pt2_is_full(m, va), ("%s: bad PT2 wire" " count %u", __func__, pt2_wirecount_get(m, pte1_index(va)))); #endif } } pt2pg_pa = VM_PAGE_TO_PHYS(m); pte1_idx = pte1_index(va); /* * If the pmap is current, then the PT2MAP can provide access to * the page table page (promoted L2 page tables are not unmapped). * Otherwise, temporarily map the L2 page table page (m) into * the kernel's address space at either PADDR1 or PADDR2. * * Note that L2 page table size is not equal to PAGE_SIZE. */ if (pmap_is_current(pmap)) fpte2p = page_pt2(pt2map_pt2pg(va), pte1_idx); else if (curthread->td_pinned > 0 && rw_wowned(&pvh_global_lock)) { if (pte2_pa(pte2_load(PMAP1)) != pt2pg_pa) { pte2_store(PMAP1, PTE2_KPT(pt2pg_pa)); #ifdef SMP PMAP1cpu = PCPU_GET(cpuid); #endif tlb_flush_local((vm_offset_t)PADDR1); PMAP1changed++; } else #ifdef SMP if (PMAP1cpu != PCPU_GET(cpuid)) { PMAP1cpu = PCPU_GET(cpuid); tlb_flush_local((vm_offset_t)PADDR1); PMAP1changedcpu++; } else #endif PMAP1unchanged++; fpte2p = page_pt2((vm_offset_t)PADDR1, pte1_idx); } else { mtx_lock(&PMAP2mutex); if (pte2_pa(pte2_load(PMAP2)) != pt2pg_pa) { pte2_store(PMAP2, PTE2_KPT(pt2pg_pa)); tlb_flush((vm_offset_t)PADDR2); } fpte2p = page_pt2((vm_offset_t)PADDR2, pte1_idx); } pt2_pa = page_pt2pa(pt2pg_pa, pte1_idx); npte1 = PTE1_LINK(pt2_pa); KASSERT((opte1 & PTE1_A) != 0, ("%s: opte1 is missing PTE1_A", __func__)); KASSERT((opte1 & (PTE1_NM | PTE1_RO)) != PTE1_NM, ("%s: opte1 has PTE1_NM", __func__)); /* * Get pte2 from pte1 format. */ npte2 = pte1_pa(opte1) | ATTR_TO_L2(opte1) | PTE2_V; /* * If the L2 page table page is new, initialize it. If the mapping * has changed attributes, update the page table entries. */ if (isnew != 0) { pt2_wirecount_set(m, pte1_idx, NPTE2_IN_PT2); pmap_fill_pt2(fpte2p, npte2); } else if ((pte2_load(fpte2p) & PTE2_PROMOTE) != (npte2 & PTE2_PROMOTE)) pmap_fill_pt2(fpte2p, npte2); KASSERT(pte2_pa(pte2_load(fpte2p)) == pte2_pa(npte2), ("%s: fpte2p and npte2 map different physical addresses", __func__)); if (fpte2p == PADDR2) mtx_unlock(&PMAP2mutex); /* * Demote the mapping. This pmap is locked. The old PTE1 has * PTE1_A set. If the old PTE1 has not PTE1_RO set, it also * has not PTE1_NM set. Thus, there is no danger of a race with * another processor changing the setting of PTE1_A and/or PTE1_NM * between the read above and the store below. */ pmap_change_pte1(pmap, pte1p, va, npte1); /* * Demote the pv entry. This depends on the earlier demotion * of the mapping. Specifically, the (re)creation of a per- * page pv entry might trigger the execution of pmap_pv_reclaim(), * which might reclaim a newly (re)created per-page pv entry * and destroy the associated mapping. In order to destroy * the mapping, the PTE1 must have already changed from mapping * the 1mpage to referencing the page table page. */ if (pte1_is_managed(opte1)) pmap_pv_demote_pte1(pmap, va, pte1_pa(opte1)); pmap_pte1_demotions++; CTR3(KTR_PMAP, "%s: success for va %#x in pmap %p", __func__, va, pmap); PDEBUG(6, printf("%s(%p): success for va %#x pte1 %#x(%#x) at %p\n", __func__, pmap, va, npte1, pte1_load(pte1p), pte1p)); return (TRUE); } /* * Insert the given physical page (p) at * the specified virtual address (v) in the * target physical map with the protection requested. * * If specified, the page will be wired down, meaning * that the related pte can not be reclaimed. * * NB: This is the only routine which MAY NOT lazy-evaluate * or lose information. That is, this routine must actually * insert this page into the given map NOW. */ int pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, u_int flags, int8_t psind) { pt1_entry_t *pte1p; pt2_entry_t *pte2p; pt2_entry_t npte2, opte2; pv_entry_t pv; vm_paddr_t opa, pa; vm_page_t mpte2, om; boolean_t wired; va = trunc_page(va); mpte2 = NULL; wired = (flags & PMAP_ENTER_WIRED) != 0; KASSERT(va <= vm_max_kernel_address, ("%s: toobig", __func__)); KASSERT(va < UPT2V_MIN_ADDRESS || va >= UPT2V_MAX_ADDRESS, ("%s: invalid to pmap_enter page table pages (va: 0x%x)", __func__, va)); if ((m->oflags & VPO_UNMANAGED) == 0 && !vm_page_xbusied(m)) VM_OBJECT_ASSERT_LOCKED(m->object); rw_wlock(&pvh_global_lock); PMAP_LOCK(pmap); sched_pin(); /* * In the case that a page table page is not * resident, we are creating it here. */ if (va < VM_MAXUSER_ADDRESS) { mpte2 = pmap_allocpte2(pmap, va, flags); if (mpte2 == NULL) { KASSERT((flags & PMAP_ENTER_NOSLEEP) != 0, ("pmap_allocpte2 failed with sleep allowed")); sched_unpin(); rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(pmap); return (KERN_RESOURCE_SHORTAGE); } } pte1p = pmap_pte1(pmap, va); if (pte1_is_section(pte1_load(pte1p))) panic("%s: attempted on 1MB page", __func__); pte2p = pmap_pte2_quick(pmap, va); if (pte2p == NULL) panic("%s: invalid L1 page table entry va=%#x", __func__, va); om = NULL; pa = VM_PAGE_TO_PHYS(m); opte2 = pte2_load(pte2p); opa = pte2_pa(opte2); /* * Mapping has not changed, must be protection or wiring change. */ if (pte2_is_valid(opte2) && (opa == pa)) { /* * Wiring change, just update stats. We don't worry about * wiring PT2 pages as they remain resident as long as there * are valid mappings in them. Hence, if a user page is wired, * the PT2 page will be also. */ if (wired && !pte2_is_wired(opte2)) pmap->pm_stats.wired_count++; else if (!wired && pte2_is_wired(opte2)) pmap->pm_stats.wired_count--; /* * Remove extra pte2 reference */ if (mpte2) pt2_wirecount_dec(mpte2, pte1_index(va)); if (pte2_is_managed(opte2)) om = m; goto validate; } /* * QQQ: We think that changing physical address on writeable mapping * is not safe. Well, maybe on kernel address space with correct * locking, it can make a sense. However, we have no idea why * anyone should do that on user address space. Are we wrong? */ KASSERT((opa == 0) || (opa == pa) || !pte2_is_valid(opte2) || ((opte2 & PTE2_RO) != 0), ("%s: pmap %p va %#x(%#x) opa %#x pa %#x - gotcha %#x %#x!", __func__, pmap, va, opte2, opa, pa, flags, prot)); pv = NULL; /* * Mapping has changed, invalidate old range and fall through to * handle validating new mapping. */ if (opa) { if (pte2_is_wired(opte2)) pmap->pm_stats.wired_count--; if (pte2_is_managed(opte2)) { om = PHYS_TO_VM_PAGE(opa); pv = pmap_pvh_remove(&om->md, pmap, va); } /* * Remove extra pte2 reference */ if (mpte2 != NULL) pt2_wirecount_dec(mpte2, va >> PTE1_SHIFT); } else pmap->pm_stats.resident_count++; /* * Enter on the PV list if part of our managed memory. */ if ((m->oflags & VPO_UNMANAGED) == 0) { KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva, ("%s: managed mapping within the clean submap", __func__)); if (pv == NULL) pv = get_pv_entry(pmap, FALSE); pv->pv_va = va; TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); } else if (pv != NULL) free_pv_entry(pmap, pv); /* * Increment counters */ if (wired) pmap->pm_stats.wired_count++; validate: /* * Now validate mapping with desired protection/wiring. */ npte2 = PTE2(pa, PTE2_NM, vm_page_pte2_attr(m)); if (prot & VM_PROT_WRITE) { if (pte2_is_managed(npte2)) vm_page_aflag_set(m, PGA_WRITEABLE); } else npte2 |= PTE2_RO; if ((prot & VM_PROT_EXECUTE) == 0) npte2 |= PTE2_NX; if (wired) npte2 |= PTE2_W; if (va < VM_MAXUSER_ADDRESS) npte2 |= PTE2_U; if (pmap != kernel_pmap) npte2 |= PTE2_NG; /* * If the mapping or permission bits are different, we need * to update the pte2. * * QQQ: Think again and again what to do * if the mapping is going to be changed! */ if ((opte2 & ~(PTE2_NM | PTE2_A)) != (npte2 & ~(PTE2_NM | PTE2_A))) { /* * Sync icache if exec permission and attribute VM_MEMATTR_WB_WA * is set. Do it now, before the mapping is stored and made * valid for hardware table walk. If done later, there is a race * for other threads of current process in lazy loading case. * Don't do it for kernel memory which is mapped with exec * permission even if the memory isn't going to hold executable * code. The only time when icache sync is needed is after * kernel module is loaded and the relocation info is processed. * And it's done in elf_cpu_load_file(). * * QQQ: (1) Does it exist any better way where * or how to sync icache? * (2) Now, we do it on a page basis. */ if ((prot & VM_PROT_EXECUTE) && pmap != kernel_pmap && m->md.pat_mode == VM_MEMATTR_WB_WA && (opa != pa || (opte2 & PTE2_NX))) cache_icache_sync_fresh(va, pa, PAGE_SIZE); npte2 |= PTE2_A; if (flags & VM_PROT_WRITE) npte2 &= ~PTE2_NM; if (opte2 & PTE2_V) { /* Change mapping with break-before-make approach. */ opte2 = pte2_load_clear(pte2p); pmap_tlb_flush(pmap, va); pte2_store(pte2p, npte2); if (opte2 & PTE2_A) { if (pte2_is_managed(opte2)) vm_page_aflag_set(om, PGA_REFERENCED); } if (pte2_is_dirty(opte2)) { if (pte2_is_managed(opte2)) vm_page_dirty(om); } if (pte2_is_managed(opte2) && TAILQ_EMPTY(&om->md.pv_list) && ((om->flags & PG_FICTITIOUS) != 0 || TAILQ_EMPTY(&pa_to_pvh(opa)->pv_list))) vm_page_aflag_clear(om, PGA_WRITEABLE); } else pte2_store(pte2p, npte2); } #if 0 else { /* * QQQ: In time when both access and not mofified bits are * emulated by software, this should not happen. Some * analysis is need, if this really happen. Missing * tlb flush somewhere could be the reason. */ panic("%s: pmap %p va %#x opte2 %x npte2 %x !!", __func__, pmap, va, opte2, npte2); } #endif /* * If both the L2 page table page and the reservation are fully * populated, then attempt promotion. */ if ((mpte2 == NULL || pt2_is_full(mpte2, va)) && sp_enabled && (m->flags & PG_FICTITIOUS) == 0 && vm_reserv_level_iffullpop(m) == 0) pmap_promote_pte1(pmap, pte1p, va); sched_unpin(); rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(pmap); return (KERN_SUCCESS); } /* * Do the things to unmap a page in a process. */ static int pmap_remove_pte2(pmap_t pmap, pt2_entry_t *pte2p, vm_offset_t va, struct spglist *free) { pt2_entry_t opte2; vm_page_t m; rw_assert(&pvh_global_lock, RA_WLOCKED); PMAP_LOCK_ASSERT(pmap, MA_OWNED); /* Clear and invalidate the mapping. */ opte2 = pte2_load_clear(pte2p); pmap_tlb_flush(pmap, va); KASSERT(pte2_is_valid(opte2), ("%s: pmap %p va %#x not link pte2 %#x", __func__, pmap, va, opte2)); if (opte2 & PTE2_W) pmap->pm_stats.wired_count -= 1; pmap->pm_stats.resident_count -= 1; if (pte2_is_managed(opte2)) { m = PHYS_TO_VM_PAGE(pte2_pa(opte2)); if (pte2_is_dirty(opte2)) vm_page_dirty(m); if (opte2 & PTE2_A) vm_page_aflag_set(m, PGA_REFERENCED); pmap_remove_entry(pmap, m, va); } return (pmap_unuse_pt2(pmap, va, free)); } /* * Remove a single page from a process address space. */ static void pmap_remove_page(pmap_t pmap, vm_offset_t va, struct spglist *free) { pt2_entry_t *pte2p; rw_assert(&pvh_global_lock, RA_WLOCKED); KASSERT(curthread->td_pinned > 0, ("%s: curthread not pinned", __func__)); PMAP_LOCK_ASSERT(pmap, MA_OWNED); if ((pte2p = pmap_pte2_quick(pmap, va)) == NULL || !pte2_is_valid(pte2_load(pte2p))) return; pmap_remove_pte2(pmap, pte2p, va, free); } /* * Remove the given range of addresses from the specified map. * * It is assumed that the start and end are properly * rounded to the page size. */ void pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) { vm_offset_t nextva; pt1_entry_t *pte1p, pte1; pt2_entry_t *pte2p, pte2; struct spglist free; /* * Perform an unsynchronized read. This is, however, safe. */ if (pmap->pm_stats.resident_count == 0) return; SLIST_INIT(&free); rw_wlock(&pvh_global_lock); sched_pin(); PMAP_LOCK(pmap); /* * Special handling of removing one page. A very common * operation and easy to short circuit some code. */ if (sva + PAGE_SIZE == eva) { pte1 = pte1_load(pmap_pte1(pmap, sva)); if (pte1_is_link(pte1)) { pmap_remove_page(pmap, sva, &free); goto out; } } for (; sva < eva; sva = nextva) { /* * Calculate address for next L2 page table. */ nextva = pte1_trunc(sva + PTE1_SIZE); if (nextva < sva) nextva = eva; if (pmap->pm_stats.resident_count == 0) break; pte1p = pmap_pte1(pmap, sva); pte1 = pte1_load(pte1p); /* * Weed out invalid mappings. Note: we assume that the L1 page * table is always allocated, and in kernel virtual. */ if (pte1 == 0) continue; if (pte1_is_section(pte1)) { /* * Are we removing the entire large page? If not, * demote the mapping and fall through. */ if (sva + PTE1_SIZE == nextva && eva >= nextva) { pmap_remove_pte1(pmap, pte1p, sva, &free); continue; } else if (!pmap_demote_pte1(pmap, pte1p, sva)) { /* The large page mapping was destroyed. */ continue; } #ifdef INVARIANTS else { /* Update pte1 after demotion. */ pte1 = pte1_load(pte1p); } #endif } KASSERT(pte1_is_link(pte1), ("%s: pmap %p va %#x pte1 %#x at %p" " is not link", __func__, pmap, sva, pte1, pte1p)); /* * Limit our scan to either the end of the va represented * by the current L2 page table page, or to the end of the * range being removed. */ if (nextva > eva) nextva = eva; for (pte2p = pmap_pte2_quick(pmap, sva); sva != nextva; pte2p++, sva += PAGE_SIZE) { pte2 = pte2_load(pte2p); if (!pte2_is_valid(pte2)) continue; if (pmap_remove_pte2(pmap, pte2p, sva, &free)) break; } } out: sched_unpin(); rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(pmap); pmap_free_zero_pages(&free); } /* * Routine: pmap_remove_all * Function: * Removes this physical page from * all physical maps in which it resides. * Reflects back modify bits to the pager. * * Notes: * Original versions of this routine were very * inefficient because they iteratively called * pmap_remove (slow...) */ void pmap_remove_all(vm_page_t m) { struct md_page *pvh; pv_entry_t pv; pmap_t pmap; pt2_entry_t *pte2p, opte2; pt1_entry_t *pte1p; vm_offset_t va; struct spglist free; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("%s: page %p is not managed", __func__, m)); SLIST_INIT(&free); rw_wlock(&pvh_global_lock); sched_pin(); if ((m->flags & PG_FICTITIOUS) != 0) goto small_mappings; pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); while ((pv = TAILQ_FIRST(&pvh->pv_list)) != NULL) { va = pv->pv_va; pmap = PV_PMAP(pv); PMAP_LOCK(pmap); pte1p = pmap_pte1(pmap, va); (void)pmap_demote_pte1(pmap, pte1p, va); PMAP_UNLOCK(pmap); } small_mappings: while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { pmap = PV_PMAP(pv); PMAP_LOCK(pmap); pmap->pm_stats.resident_count--; pte1p = pmap_pte1(pmap, pv->pv_va); KASSERT(!pte1_is_section(pte1_load(pte1p)), ("%s: found " "a 1mpage in page %p's pv list", __func__, m)); pte2p = pmap_pte2_quick(pmap, pv->pv_va); opte2 = pte2_load_clear(pte2p); pmap_tlb_flush(pmap, pv->pv_va); KASSERT(pte2_is_valid(opte2), ("%s: pmap %p va %x zero pte2", __func__, pmap, pv->pv_va)); if (pte2_is_wired(opte2)) pmap->pm_stats.wired_count--; if (opte2 & PTE2_A) vm_page_aflag_set(m, PGA_REFERENCED); /* * Update the vm_page_t clean and reference bits. */ if (pte2_is_dirty(opte2)) vm_page_dirty(m); pmap_unuse_pt2(pmap, pv->pv_va, &free); TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); free_pv_entry(pmap, pv); PMAP_UNLOCK(pmap); } vm_page_aflag_clear(m, PGA_WRITEABLE); sched_unpin(); rw_wunlock(&pvh_global_lock); pmap_free_zero_pages(&free); } /* * Just subroutine for pmap_remove_pages() to reasonably satisfy * good coding style, a.k.a. 80 character line width limit hell. */ static __inline void pmap_remove_pte1_quick(pmap_t pmap, pt1_entry_t pte1, pv_entry_t pv, struct spglist *free) { vm_paddr_t pa; vm_page_t m, mt, mpt2pg; struct md_page *pvh; pa = pte1_pa(pte1); m = PHYS_TO_VM_PAGE(pa); KASSERT(m->phys_addr == pa, ("%s: vm_page_t %p addr mismatch %#x %#x", __func__, m, m->phys_addr, pa)); KASSERT((m->flags & PG_FICTITIOUS) != 0 || m < &vm_page_array[vm_page_array_size], ("%s: bad pte1 %#x", __func__, pte1)); if (pte1_is_dirty(pte1)) { for (mt = m; mt < &m[PTE1_SIZE / PAGE_SIZE]; mt++) vm_page_dirty(mt); } pmap->pm_stats.resident_count -= PTE1_SIZE / PAGE_SIZE; pvh = pa_to_pvh(pa); TAILQ_REMOVE(&pvh->pv_list, pv, pv_next); if (TAILQ_EMPTY(&pvh->pv_list)) { for (mt = m; mt < &m[PTE1_SIZE / PAGE_SIZE]; mt++) if (TAILQ_EMPTY(&mt->md.pv_list)) vm_page_aflag_clear(mt, PGA_WRITEABLE); } mpt2pg = pmap_pt2_page(pmap, pv->pv_va); if (mpt2pg != NULL) pmap_unwire_pt2_all(pmap, pv->pv_va, mpt2pg, free); } /* * Just subroutine for pmap_remove_pages() to reasonably satisfy * good coding style, a.k.a. 80 character line width limit hell. */ static __inline void pmap_remove_pte2_quick(pmap_t pmap, pt2_entry_t pte2, pv_entry_t pv, struct spglist *free) { vm_paddr_t pa; vm_page_t m; struct md_page *pvh; pa = pte2_pa(pte2); m = PHYS_TO_VM_PAGE(pa); KASSERT(m->phys_addr == pa, ("%s: vm_page_t %p addr mismatch %#x %#x", __func__, m, m->phys_addr, pa)); KASSERT((m->flags & PG_FICTITIOUS) != 0 || m < &vm_page_array[vm_page_array_size], ("%s: bad pte2 %#x", __func__, pte2)); if (pte2_is_dirty(pte2)) vm_page_dirty(m); pmap->pm_stats.resident_count--; TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); if (TAILQ_EMPTY(&m->md.pv_list) && (m->flags & PG_FICTITIOUS) == 0) { pvh = pa_to_pvh(pa); if (TAILQ_EMPTY(&pvh->pv_list)) vm_page_aflag_clear(m, PGA_WRITEABLE); } pmap_unuse_pt2(pmap, pv->pv_va, free); } /* * Remove all pages from specified address space this aids process * exit speeds. Also, this code is special cased for current process * only, but can have the more generic (and slightly slower) mode enabled. * This is much faster than pmap_remove in the case of running down * an entire address space. */ void pmap_remove_pages(pmap_t pmap) { pt1_entry_t *pte1p, pte1; pt2_entry_t *pte2p, pte2; pv_entry_t pv; struct pv_chunk *pc, *npc; struct spglist free; int field, idx; int32_t bit; uint32_t inuse, bitmask; boolean_t allfree; /* * Assert that the given pmap is only active on the current * CPU. Unfortunately, we cannot block another CPU from * activating the pmap while this function is executing. */ KASSERT(pmap == vmspace_pmap(curthread->td_proc->p_vmspace), ("%s: non-current pmap %p", __func__, pmap)); #if defined(SMP) && defined(INVARIANTS) { cpuset_t other_cpus; sched_pin(); other_cpus = pmap->pm_active; CPU_CLR(PCPU_GET(cpuid), &other_cpus); sched_unpin(); KASSERT(CPU_EMPTY(&other_cpus), ("%s: pmap %p active on other cpus", __func__, pmap)); } #endif SLIST_INIT(&free); rw_wlock(&pvh_global_lock); PMAP_LOCK(pmap); sched_pin(); TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) { KASSERT(pc->pc_pmap == pmap, ("%s: wrong pmap %p %p", __func__, pmap, pc->pc_pmap)); allfree = TRUE; for (field = 0; field < _NPCM; field++) { inuse = (~(pc->pc_map[field])) & pc_freemask[field]; while (inuse != 0) { bit = ffs(inuse) - 1; bitmask = 1UL << bit; idx = field * 32 + bit; pv = &pc->pc_pventry[idx]; inuse &= ~bitmask; /* * Note that we cannot remove wired pages * from a process' mapping at this time */ pte1p = pmap_pte1(pmap, pv->pv_va); pte1 = pte1_load(pte1p); if (pte1_is_section(pte1)) { if (pte1_is_wired(pte1)) { allfree = FALSE; continue; } pte1_clear(pte1p); pmap_remove_pte1_quick(pmap, pte1, pv, &free); } else if (pte1_is_link(pte1)) { pte2p = pt2map_entry(pv->pv_va); pte2 = pte2_load(pte2p); if (!pte2_is_valid(pte2)) { printf("%s: pmap %p va %#x " "pte2 %#x\n", __func__, pmap, pv->pv_va, pte2); panic("bad pte2"); } if (pte2_is_wired(pte2)) { allfree = FALSE; continue; } pte2_clear(pte2p); pmap_remove_pte2_quick(pmap, pte2, pv, &free); } else { printf("%s: pmap %p va %#x pte1 %#x\n", __func__, pmap, pv->pv_va, pte1); panic("bad pte1"); } /* Mark free */ PV_STAT(pv_entry_frees++); PV_STAT(pv_entry_spare++); pv_entry_count--; pc->pc_map[field] |= bitmask; } } if (allfree) { TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); free_pv_chunk(pc); } } tlb_flush_all_ng_local(); sched_unpin(); rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(pmap); pmap_free_zero_pages(&free); } /* * This code makes some *MAJOR* assumptions: * 1. Current pmap & pmap exists. * 2. Not wired. * 3. Read access. * 4. No L2 page table pages. * but is *MUCH* faster than pmap_enter... */ static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, vm_page_t mpt2pg) { pt2_entry_t *pte2p, pte2; vm_paddr_t pa; struct spglist free; uint32_t l2prot; KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva || (m->oflags & VPO_UNMANAGED) != 0, ("%s: managed mapping within the clean submap", __func__)); rw_assert(&pvh_global_lock, RA_WLOCKED); PMAP_LOCK_ASSERT(pmap, MA_OWNED); /* * In the case that a L2 page table page is not * resident, we are creating it here. */ if (va < VM_MAXUSER_ADDRESS) { u_int pte1_idx; pt1_entry_t pte1, *pte1p; vm_paddr_t pt2_pa; /* * Get L1 page table things. */ pte1_idx = pte1_index(va); pte1p = pmap_pte1(pmap, va); pte1 = pte1_load(pte1p); if (mpt2pg && (mpt2pg->pindex == (pte1_idx & ~PT2PG_MASK))) { /* * Each of NPT2_IN_PG L2 page tables on the page can * come here. Make sure that associated L1 page table * link is established. * * QQQ: It comes that we don't establish all links to * L2 page tables for newly allocated L2 page * tables page. */ KASSERT(!pte1_is_section(pte1), ("%s: pte1 %#x is section", __func__, pte1)); if (!pte1_is_link(pte1)) { pt2_pa = page_pt2pa(VM_PAGE_TO_PHYS(mpt2pg), pte1_idx); pte1_store(pte1p, PTE1_LINK(pt2_pa)); } pt2_wirecount_inc(mpt2pg, pte1_idx); } else { /* * If the L2 page table page is mapped, we just * increment the hold count, and activate it. */ if (pte1_is_section(pte1)) { return (NULL); } else if (pte1_is_link(pte1)) { mpt2pg = PHYS_TO_VM_PAGE(pte1_link_pa(pte1)); pt2_wirecount_inc(mpt2pg, pte1_idx); } else { mpt2pg = _pmap_allocpte2(pmap, va, PMAP_ENTER_NOSLEEP); if (mpt2pg == NULL) return (NULL); } } } else { mpt2pg = NULL; } /* * This call to pt2map_entry() makes the assumption that we are * entering the page into the current pmap. In order to support * quick entry into any pmap, one would likely use pmap_pte2_quick(). * But that isn't as quick as pt2map_entry(). */ pte2p = pt2map_entry(va); pte2 = pte2_load(pte2p); if (pte2_is_valid(pte2)) { if (mpt2pg != NULL) { /* * Remove extra pte2 reference */ pt2_wirecount_dec(mpt2pg, pte1_index(va)); mpt2pg = NULL; } return (NULL); } /* * Enter on the PV list if part of our managed memory. */ if ((m->oflags & VPO_UNMANAGED) == 0 && !pmap_try_insert_pv_entry(pmap, va, m)) { if (mpt2pg != NULL) { SLIST_INIT(&free); if (pmap_unwire_pt2(pmap, va, mpt2pg, &free)) { pmap_tlb_flush(pmap, va); pmap_free_zero_pages(&free); } mpt2pg = NULL; } return (NULL); } /* * Increment counters */ pmap->pm_stats.resident_count++; /* * Now validate mapping with RO protection */ pa = VM_PAGE_TO_PHYS(m); l2prot = PTE2_RO | PTE2_NM; if (va < VM_MAXUSER_ADDRESS) l2prot |= PTE2_U | PTE2_NG; if ((prot & VM_PROT_EXECUTE) == 0) l2prot |= PTE2_NX; else if (m->md.pat_mode == VM_MEMATTR_WB_WA && pmap != kernel_pmap) { /* * Sync icache if exec permission and attribute VM_MEMATTR_WB_WA * is set. QQQ: For more info, see comments in pmap_enter(). */ cache_icache_sync_fresh(va, pa, PAGE_SIZE); } pte2_store(pte2p, PTE2(pa, l2prot, vm_page_pte2_attr(m))); return (mpt2pg); } void pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot) { rw_wlock(&pvh_global_lock); PMAP_LOCK(pmap); (void)pmap_enter_quick_locked(pmap, va, m, prot, NULL); rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(pmap); } /* * Tries to create 1MB page mapping. Returns TRUE if successful and * FALSE otherwise. Fails if (1) a page table page cannot be allocated without * blocking, (2) a mapping already exists at the specified virtual address, or * (3) a pv entry cannot be allocated without reclaiming another pv entry. */ static boolean_t pmap_enter_pte1(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot) { pt1_entry_t *pte1p; vm_paddr_t pa; uint32_t l1prot; rw_assert(&pvh_global_lock, RA_WLOCKED); PMAP_LOCK_ASSERT(pmap, MA_OWNED); pte1p = pmap_pte1(pmap, va); if (pte1_is_valid(pte1_load(pte1p))) { CTR3(KTR_PMAP, "%s: failure for va %#lx in pmap %p", __func__, va, pmap); return (FALSE); } if ((m->oflags & VPO_UNMANAGED) == 0) { /* * Abort this mapping if its PV entry could not be created. */ if (!pmap_pv_insert_pte1(pmap, va, VM_PAGE_TO_PHYS(m))) { CTR3(KTR_PMAP, "%s: failure for va %#lx in pmap %p", __func__, va, pmap); return (FALSE); } } /* * Increment counters. */ pmap->pm_stats.resident_count += PTE1_SIZE / PAGE_SIZE; /* * Map the section. * * QQQ: Why VM_PROT_WRITE is not evaluated and the mapping is * made readonly? */ pa = VM_PAGE_TO_PHYS(m); l1prot = PTE1_RO | PTE1_NM; if (va < VM_MAXUSER_ADDRESS) l1prot |= PTE1_U | PTE1_NG; if ((prot & VM_PROT_EXECUTE) == 0) l1prot |= PTE1_NX; else if (m->md.pat_mode == VM_MEMATTR_WB_WA && pmap != kernel_pmap) { /* * Sync icache if exec permission and attribute VM_MEMATTR_WB_WA * is set. QQQ: For more info, see comments in pmap_enter(). */ cache_icache_sync_fresh(va, pa, PTE1_SIZE); } pte1_store(pte1p, PTE1(pa, l1prot, ATTR_TO_L1(vm_page_pte2_attr(m)))); pmap_pte1_mappings++; CTR3(KTR_PMAP, "%s: success for va %#lx in pmap %p", __func__, va, pmap); return (TRUE); } /* * Maps a sequence of resident pages belonging to the same object. * The sequence begins with the given page m_start. This page is * mapped at the given virtual address start. Each subsequent page is * mapped at a virtual address that is offset from start by the same * amount as the page is offset from m_start within the object. The * last page in the sequence is the page with the largest offset from * m_start that can be mapped at a virtual address less than the given * virtual address end. Not every virtual page between start and end * is mapped; only those for which a resident page exists with the * corresponding offset from m_start are mapped. */ void pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, vm_page_t m_start, vm_prot_t prot) { vm_offset_t va; vm_page_t m, mpt2pg; vm_pindex_t diff, psize; PDEBUG(6, printf("%s: pmap %p start %#x end %#x m %p prot %#x\n", __func__, pmap, start, end, m_start, prot)); VM_OBJECT_ASSERT_LOCKED(m_start->object); psize = atop(end - start); mpt2pg = NULL; m = m_start; rw_wlock(&pvh_global_lock); PMAP_LOCK(pmap); while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) { va = start + ptoa(diff); if ((va & PTE1_OFFSET) == 0 && va + PTE1_SIZE <= end && m->psind == 1 && sp_enabled && pmap_enter_pte1(pmap, va, m, prot)) m = &m[PTE1_SIZE / PAGE_SIZE - 1]; else mpt2pg = pmap_enter_quick_locked(pmap, va, m, prot, mpt2pg); m = TAILQ_NEXT(m, listq); } rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(pmap); } /* * This code maps large physical mmap regions into the * processor address space. Note that some shortcuts * are taken, but the code works. */ void pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object, vm_pindex_t pindex, vm_size_t size) { pt1_entry_t *pte1p; vm_paddr_t pa, pte2_pa; vm_page_t p; vm_memattr_t pat_mode; u_int l1attr, l1prot; VM_OBJECT_ASSERT_WLOCKED(object); KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, ("%s: non-device object", __func__)); if ((addr & PTE1_OFFSET) == 0 && (size & PTE1_OFFSET) == 0) { if (!vm_object_populate(object, pindex, pindex + atop(size))) return; p = vm_page_lookup(object, pindex); KASSERT(p->valid == VM_PAGE_BITS_ALL, ("%s: invalid page %p", __func__, p)); pat_mode = p->md.pat_mode; /* * Abort the mapping if the first page is not physically * aligned to a 1MB page boundary. */ pte2_pa = VM_PAGE_TO_PHYS(p); if (pte2_pa & PTE1_OFFSET) return; /* * Skip the first page. Abort the mapping if the rest of * the pages are not physically contiguous or have differing * memory attributes. */ p = TAILQ_NEXT(p, listq); for (pa = pte2_pa + PAGE_SIZE; pa < pte2_pa + size; pa += PAGE_SIZE) { KASSERT(p->valid == VM_PAGE_BITS_ALL, ("%s: invalid page %p", __func__, p)); if (pa != VM_PAGE_TO_PHYS(p) || pat_mode != p->md.pat_mode) return; p = TAILQ_NEXT(p, listq); } /* * Map using 1MB pages. * * QQQ: Well, we are mapping a section, so same condition must * be hold like during promotion. It looks that only RW mapping * is done here, so readonly mapping must be done elsewhere. */ l1prot = PTE1_U | PTE1_NG | PTE1_RW | PTE1_M | PTE1_A; l1attr = ATTR_TO_L1(vm_memattr_to_pte2(pat_mode)); PMAP_LOCK(pmap); for (pa = pte2_pa; pa < pte2_pa + size; pa += PTE1_SIZE) { pte1p = pmap_pte1(pmap, addr); if (!pte1_is_valid(pte1_load(pte1p))) { pte1_store(pte1p, PTE1(pa, l1prot, l1attr)); pmap->pm_stats.resident_count += PTE1_SIZE / PAGE_SIZE; pmap_pte1_mappings++; } /* Else continue on if the PTE1 is already valid. */ addr += PTE1_SIZE; } PMAP_UNLOCK(pmap); } } /* * Do the things to protect a 1mpage in a process. */ static void pmap_protect_pte1(pmap_t pmap, pt1_entry_t *pte1p, vm_offset_t sva, vm_prot_t prot) { pt1_entry_t npte1, opte1; vm_offset_t eva, va; vm_page_t m; PMAP_LOCK_ASSERT(pmap, MA_OWNED); KASSERT((sva & PTE1_OFFSET) == 0, ("%s: sva is not 1mpage aligned", __func__)); opte1 = npte1 = pte1_load(pte1p); if (pte1_is_managed(opte1)) { eva = sva + PTE1_SIZE; for (va = sva, m = PHYS_TO_VM_PAGE(pte1_pa(opte1)); va < eva; va += PAGE_SIZE, m++) if (pte1_is_dirty(opte1)) vm_page_dirty(m); } if ((prot & VM_PROT_WRITE) == 0) npte1 |= PTE1_RO | PTE1_NM; if ((prot & VM_PROT_EXECUTE) == 0) npte1 |= PTE1_NX; /* * QQQ: Herein, execute permission is never set. * It only can be cleared. So, no icache * syncing is needed. */ if (npte1 != opte1) { pte1_store(pte1p, npte1); pmap_tlb_flush(pmap, sva); } } /* * Set the physical protection on the * specified range of this map as requested. */ void pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) { boolean_t pv_lists_locked; vm_offset_t nextva; pt1_entry_t *pte1p, pte1; pt2_entry_t *pte2p, opte2, npte2; KASSERT((prot & ~VM_PROT_ALL) == 0, ("invalid prot %x", prot)); if (prot == VM_PROT_NONE) { pmap_remove(pmap, sva, eva); return; } if ((prot & (VM_PROT_WRITE | VM_PROT_EXECUTE)) == (VM_PROT_WRITE | VM_PROT_EXECUTE)) return; if (pmap_is_current(pmap)) pv_lists_locked = FALSE; else { pv_lists_locked = TRUE; resume: rw_wlock(&pvh_global_lock); sched_pin(); } PMAP_LOCK(pmap); for (; sva < eva; sva = nextva) { /* * Calculate address for next L2 page table. */ nextva = pte1_trunc(sva + PTE1_SIZE); if (nextva < sva) nextva = eva; pte1p = pmap_pte1(pmap, sva); pte1 = pte1_load(pte1p); /* * Weed out invalid mappings. Note: we assume that L1 page * page table is always allocated, and in kernel virtual. */ if (pte1 == 0) continue; if (pte1_is_section(pte1)) { /* * Are we protecting the entire large page? If not, * demote the mapping and fall through. */ if (sva + PTE1_SIZE == nextva && eva >= nextva) { pmap_protect_pte1(pmap, pte1p, sva, prot); continue; } else { if (!pv_lists_locked) { pv_lists_locked = TRUE; if (!rw_try_wlock(&pvh_global_lock)) { PMAP_UNLOCK(pmap); goto resume; } sched_pin(); } if (!pmap_demote_pte1(pmap, pte1p, sva)) { /* * The large page mapping * was destroyed. */ continue; } #ifdef INVARIANTS else { /* Update pte1 after demotion */ pte1 = pte1_load(pte1p); } #endif } } KASSERT(pte1_is_link(pte1), ("%s: pmap %p va %#x pte1 %#x at %p" " is not link", __func__, pmap, sva, pte1, pte1p)); /* * Limit our scan to either the end of the va represented * by the current L2 page table page, or to the end of the * range being protected. */ if (nextva > eva) nextva = eva; for (pte2p = pmap_pte2_quick(pmap, sva); sva != nextva; pte2p++, sva += PAGE_SIZE) { vm_page_t m; opte2 = npte2 = pte2_load(pte2p); if (!pte2_is_valid(opte2)) continue; if ((prot & VM_PROT_WRITE) == 0) { if (pte2_is_managed(opte2) && pte2_is_dirty(opte2)) { m = PHYS_TO_VM_PAGE(pte2_pa(opte2)); vm_page_dirty(m); } npte2 |= PTE2_RO | PTE2_NM; } if ((prot & VM_PROT_EXECUTE) == 0) npte2 |= PTE2_NX; /* * QQQ: Herein, execute permission is never set. * It only can be cleared. So, no icache * syncing is needed. */ if (npte2 != opte2) { pte2_store(pte2p, npte2); pmap_tlb_flush(pmap, sva); } } } if (pv_lists_locked) { sched_unpin(); rw_wunlock(&pvh_global_lock); } PMAP_UNLOCK(pmap); } /* * pmap_pvh_wired_mappings: * * Return the updated number "count" of managed mappings that are wired. */ static int pmap_pvh_wired_mappings(struct md_page *pvh, int count) { pmap_t pmap; pt1_entry_t pte1; pt2_entry_t pte2; pv_entry_t pv; rw_assert(&pvh_global_lock, RA_WLOCKED); sched_pin(); TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { pmap = PV_PMAP(pv); PMAP_LOCK(pmap); pte1 = pte1_load(pmap_pte1(pmap, pv->pv_va)); if (pte1_is_section(pte1)) { if (pte1_is_wired(pte1)) count++; } else { KASSERT(pte1_is_link(pte1), ("%s: pte1 %#x is not link", __func__, pte1)); pte2 = pte2_load(pmap_pte2_quick(pmap, pv->pv_va)); if (pte2_is_wired(pte2)) count++; } PMAP_UNLOCK(pmap); } sched_unpin(); return (count); } /* * pmap_page_wired_mappings: * * Return the number of managed mappings to the given physical page * that are wired. */ int pmap_page_wired_mappings(vm_page_t m) { int count; count = 0; if ((m->oflags & VPO_UNMANAGED) != 0) return (count); rw_wlock(&pvh_global_lock); count = pmap_pvh_wired_mappings(&m->md, count); if ((m->flags & PG_FICTITIOUS) == 0) { count = pmap_pvh_wired_mappings(pa_to_pvh(VM_PAGE_TO_PHYS(m)), count); } rw_wunlock(&pvh_global_lock); return (count); } /* * Returns TRUE if any of the given mappings were used to modify * physical memory. Otherwise, returns FALSE. Both page and 1mpage * mappings are supported. */ static boolean_t pmap_is_modified_pvh(struct md_page *pvh) { pv_entry_t pv; pt1_entry_t pte1; pt2_entry_t pte2; pmap_t pmap; boolean_t rv; rw_assert(&pvh_global_lock, RA_WLOCKED); rv = FALSE; sched_pin(); TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { pmap = PV_PMAP(pv); PMAP_LOCK(pmap); pte1 = pte1_load(pmap_pte1(pmap, pv->pv_va)); if (pte1_is_section(pte1)) { rv = pte1_is_dirty(pte1); } else { KASSERT(pte1_is_link(pte1), ("%s: pte1 %#x is not link", __func__, pte1)); pte2 = pte2_load(pmap_pte2_quick(pmap, pv->pv_va)); rv = pte2_is_dirty(pte2); } PMAP_UNLOCK(pmap); if (rv) break; } sched_unpin(); return (rv); } /* * pmap_is_modified: * * Return whether or not the specified physical page was modified * in any physical maps. */ boolean_t pmap_is_modified(vm_page_t m) { boolean_t rv; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("%s: page %p is not managed", __func__, m)); /* * If the page is not exclusive busied, then PGA_WRITEABLE cannot be * concurrently set while the object is locked. Thus, if PGA_WRITEABLE * is clear, no PTE2s can have PG_M set. */ VM_OBJECT_ASSERT_WLOCKED(m->object); if (!vm_page_xbusied(m) && (m->aflags & PGA_WRITEABLE) == 0) return (FALSE); rw_wlock(&pvh_global_lock); rv = pmap_is_modified_pvh(&m->md) || ((m->flags & PG_FICTITIOUS) == 0 && pmap_is_modified_pvh(pa_to_pvh(VM_PAGE_TO_PHYS(m)))); rw_wunlock(&pvh_global_lock); return (rv); } /* * pmap_is_prefaultable: * * Return whether or not the specified virtual address is eligible * for prefault. */ boolean_t pmap_is_prefaultable(pmap_t pmap, vm_offset_t addr) { pt1_entry_t pte1; pt2_entry_t pte2; boolean_t rv; rv = FALSE; PMAP_LOCK(pmap); pte1 = pte1_load(pmap_pte1(pmap, addr)); if (pte1_is_link(pte1)) { pte2 = pte2_load(pt2map_entry(addr)); rv = !pte2_is_valid(pte2) ; } PMAP_UNLOCK(pmap); return (rv); } /* * Returns TRUE if any of the given mappings were referenced and FALSE * otherwise. Both page and 1mpage mappings are supported. */ static boolean_t pmap_is_referenced_pvh(struct md_page *pvh) { pv_entry_t pv; pt1_entry_t pte1; pt2_entry_t pte2; pmap_t pmap; boolean_t rv; rw_assert(&pvh_global_lock, RA_WLOCKED); rv = FALSE; sched_pin(); TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { pmap = PV_PMAP(pv); PMAP_LOCK(pmap); pte1 = pte1_load(pmap_pte1(pmap, pv->pv_va)); if (pte1_is_section(pte1)) { rv = (pte1 & (PTE1_A | PTE1_V)) == (PTE1_A | PTE1_V); } else { pte2 = pte2_load(pmap_pte2_quick(pmap, pv->pv_va)); rv = (pte2 & (PTE2_A | PTE2_V)) == (PTE2_A | PTE2_V); } PMAP_UNLOCK(pmap); if (rv) break; } sched_unpin(); return (rv); } /* * pmap_is_referenced: * * Return whether or not the specified physical page was referenced * in any physical maps. */ boolean_t pmap_is_referenced(vm_page_t m) { boolean_t rv; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("%s: page %p is not managed", __func__, m)); rw_wlock(&pvh_global_lock); rv = pmap_is_referenced_pvh(&m->md) || ((m->flags & PG_FICTITIOUS) == 0 && pmap_is_referenced_pvh(pa_to_pvh(VM_PAGE_TO_PHYS(m)))); rw_wunlock(&pvh_global_lock); return (rv); } #define PMAP_TS_REFERENCED_MAX 5 /* * pmap_ts_referenced: * * Return a count of reference bits for a page, clearing those bits. * It is not necessary for every reference bit to be cleared, but it * is necessary that 0 only be returned when there are truly no * reference bits set. * * XXX: The exact number of bits to check and clear is a matter that * should be tested and standardized at some point in the future for * optimal aging of shared pages. */ int pmap_ts_referenced(vm_page_t m) { struct md_page *pvh; pv_entry_t pv, pvf; pmap_t pmap; pt1_entry_t *pte1p, opte1; pt2_entry_t *pte2p; vm_paddr_t pa; int rtval = 0; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("%s: page %p is not managed", __func__, m)); pa = VM_PAGE_TO_PHYS(m); pvh = pa_to_pvh(pa); rw_wlock(&pvh_global_lock); sched_pin(); if ((m->flags & PG_FICTITIOUS) != 0 || (pvf = TAILQ_FIRST(&pvh->pv_list)) == NULL) goto small_mappings; pv = pvf; do { pmap = PV_PMAP(pv); PMAP_LOCK(pmap); pte1p = pmap_pte1(pmap, pv->pv_va); opte1 = pte1_load(pte1p); if ((opte1 & PTE1_A) != 0) { /* * Since this reference bit is shared by 256 4KB pages, * it should not be cleared every time it is tested. * Apply a simple "hash" function on the physical page * number, the virtual section number, and the pmap * address to select one 4KB page out of the 256 * on which testing the reference bit will result * in clearing that bit. This function is designed * to avoid the selection of the same 4KB page * for every 1MB page mapping. * * On demotion, a mapping that hasn't been referenced * is simply destroyed. To avoid the possibility of a * subsequent page fault on a demoted wired mapping, * always leave its reference bit set. Moreover, * since the section is wired, the current state of * its reference bit won't affect page replacement. */ if ((((pa >> PAGE_SHIFT) ^ (pv->pv_va >> PTE1_SHIFT) ^ (uintptr_t)pmap) & (NPTE2_IN_PG - 1)) == 0 && !pte1_is_wired(opte1)) { pte1_clear_bit(pte1p, PTE1_A); pmap_tlb_flush(pmap, pv->pv_va); } rtval++; } PMAP_UNLOCK(pmap); /* Rotate the PV list if it has more than one entry. */ if (TAILQ_NEXT(pv, pv_next) != NULL) { TAILQ_REMOVE(&pvh->pv_list, pv, pv_next); TAILQ_INSERT_TAIL(&pvh->pv_list, pv, pv_next); } if (rtval >= PMAP_TS_REFERENCED_MAX) goto out; } while ((pv = TAILQ_FIRST(&pvh->pv_list)) != pvf); small_mappings: if ((pvf = TAILQ_FIRST(&m->md.pv_list)) == NULL) goto out; pv = pvf; do { pmap = PV_PMAP(pv); PMAP_LOCK(pmap); pte1p = pmap_pte1(pmap, pv->pv_va); KASSERT(pte1_is_link(pte1_load(pte1p)), ("%s: not found a link in page %p's pv list", __func__, m)); pte2p = pmap_pte2_quick(pmap, pv->pv_va); if ((pte2_load(pte2p) & PTE2_A) != 0) { pte2_clear_bit(pte2p, PTE2_A); pmap_tlb_flush(pmap, pv->pv_va); rtval++; } PMAP_UNLOCK(pmap); /* Rotate the PV list if it has more than one entry. */ if (TAILQ_NEXT(pv, pv_next) != NULL) { TAILQ_REMOVE(&m->md.pv_list, pv, pv_next); TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_next); } } while ((pv = TAILQ_FIRST(&m->md.pv_list)) != pvf && rtval < PMAP_TS_REFERENCED_MAX); out: sched_unpin(); rw_wunlock(&pvh_global_lock); return (rtval); } /* * Clear the wired attribute from the mappings for the specified range of * addresses in the given pmap. Every valid mapping within that range * must have the wired attribute set. In contrast, invalid mappings * cannot have the wired attribute set, so they are ignored. * * The wired attribute of the page table entry is not a hardware feature, * so there is no need to invalidate any TLB entries. */ void pmap_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) { vm_offset_t nextva; pt1_entry_t *pte1p, pte1; pt2_entry_t *pte2p, pte2; boolean_t pv_lists_locked; if (pmap_is_current(pmap)) pv_lists_locked = FALSE; else { pv_lists_locked = TRUE; resume: rw_wlock(&pvh_global_lock); sched_pin(); } PMAP_LOCK(pmap); for (; sva < eva; sva = nextva) { nextva = pte1_trunc(sva + PTE1_SIZE); if (nextva < sva) nextva = eva; pte1p = pmap_pte1(pmap, sva); pte1 = pte1_load(pte1p); /* * Weed out invalid mappings. Note: we assume that L1 page * page table is always allocated, and in kernel virtual. */ if (pte1 == 0) continue; if (pte1_is_section(pte1)) { if (!pte1_is_wired(pte1)) panic("%s: pte1 %#x not wired", __func__, pte1); /* * Are we unwiring the entire large page? If not, * demote the mapping and fall through. */ if (sva + PTE1_SIZE == nextva && eva >= nextva) { pte1_clear_bit(pte1p, PTE1_W); pmap->pm_stats.wired_count -= PTE1_SIZE / PAGE_SIZE; continue; } else { if (!pv_lists_locked) { pv_lists_locked = TRUE; if (!rw_try_wlock(&pvh_global_lock)) { PMAP_UNLOCK(pmap); /* Repeat sva. */ goto resume; } sched_pin(); } if (!pmap_demote_pte1(pmap, pte1p, sva)) panic("%s: demotion failed", __func__); #ifdef INVARIANTS else { /* Update pte1 after demotion */ pte1 = pte1_load(pte1p); } #endif } } KASSERT(pte1_is_link(pte1), ("%s: pmap %p va %#x pte1 %#x at %p" " is not link", __func__, pmap, sva, pte1, pte1p)); /* * Limit our scan to either the end of the va represented * by the current L2 page table page, or to the end of the * range being protected. */ if (nextva > eva) nextva = eva; for (pte2p = pmap_pte2_quick(pmap, sva); sva != nextva; pte2p++, sva += PAGE_SIZE) { pte2 = pte2_load(pte2p); if (!pte2_is_valid(pte2)) continue; if (!pte2_is_wired(pte2)) panic("%s: pte2 %#x is missing PTE2_W", __func__, pte2); /* * PTE2_W must be cleared atomically. Although the pmap * lock synchronizes access to PTE2_W, another processor * could be changing PTE2_NM and/or PTE2_A concurrently. */ pte2_clear_bit(pte2p, PTE2_W); pmap->pm_stats.wired_count--; } } if (pv_lists_locked) { sched_unpin(); rw_wunlock(&pvh_global_lock); } PMAP_UNLOCK(pmap); } /* * Clear the write and modified bits in each of the given page's mappings. */ void pmap_remove_write(vm_page_t m) { struct md_page *pvh; pv_entry_t next_pv, pv; pmap_t pmap; pt1_entry_t *pte1p; pt2_entry_t *pte2p, opte2; vm_offset_t va; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("%s: page %p is not managed", __func__, m)); /* * If the page is not exclusive busied, then PGA_WRITEABLE cannot be * set by another thread while the object is locked. Thus, * if PGA_WRITEABLE is clear, no page table entries need updating. */ VM_OBJECT_ASSERT_WLOCKED(m->object); if (!vm_page_xbusied(m) && (m->aflags & PGA_WRITEABLE) == 0) return; rw_wlock(&pvh_global_lock); sched_pin(); if ((m->flags & PG_FICTITIOUS) != 0) goto small_mappings; pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_next, next_pv) { va = pv->pv_va; pmap = PV_PMAP(pv); PMAP_LOCK(pmap); pte1p = pmap_pte1(pmap, va); if (!(pte1_load(pte1p) & PTE1_RO)) (void)pmap_demote_pte1(pmap, pte1p, va); PMAP_UNLOCK(pmap); } small_mappings: TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { pmap = PV_PMAP(pv); PMAP_LOCK(pmap); pte1p = pmap_pte1(pmap, pv->pv_va); KASSERT(!pte1_is_section(pte1_load(pte1p)), ("%s: found" " a section in page %p's pv list", __func__, m)); pte2p = pmap_pte2_quick(pmap, pv->pv_va); opte2 = pte2_load(pte2p); if (!(opte2 & PTE2_RO)) { pte2_store(pte2p, opte2 | PTE2_RO | PTE2_NM); if (pte2_is_dirty(opte2)) vm_page_dirty(m); pmap_tlb_flush(pmap, pv->pv_va); } PMAP_UNLOCK(pmap); } vm_page_aflag_clear(m, PGA_WRITEABLE); sched_unpin(); rw_wunlock(&pvh_global_lock); } /* * Apply the given advice to the specified range of addresses within the * given pmap. Depending on the advice, clear the referenced and/or * modified flags in each mapping and set the mapped page's dirty field. */ void pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice) { pt1_entry_t *pte1p, opte1; pt2_entry_t *pte2p, pte2; vm_offset_t pdnxt; vm_page_t m; boolean_t pv_lists_locked; if (advice != MADV_DONTNEED && advice != MADV_FREE) return; if (pmap_is_current(pmap)) pv_lists_locked = FALSE; else { pv_lists_locked = TRUE; resume: rw_wlock(&pvh_global_lock); sched_pin(); } PMAP_LOCK(pmap); for (; sva < eva; sva = pdnxt) { pdnxt = pte1_trunc(sva + PTE1_SIZE); if (pdnxt < sva) pdnxt = eva; pte1p = pmap_pte1(pmap, sva); opte1 = pte1_load(pte1p); if (!pte1_is_valid(opte1)) /* XXX */ continue; else if (pte1_is_section(opte1)) { if (!pte1_is_managed(opte1)) continue; if (!pv_lists_locked) { pv_lists_locked = TRUE; if (!rw_try_wlock(&pvh_global_lock)) { PMAP_UNLOCK(pmap); goto resume; } sched_pin(); } if (!pmap_demote_pte1(pmap, pte1p, sva)) { /* * The large page mapping was destroyed. */ continue; } /* * Unless the page mappings are wired, remove the * mapping to a single page so that a subsequent * access may repromote. Since the underlying L2 page * table is fully populated, this removal never * frees a L2 page table page. */ if (!pte1_is_wired(opte1)) { pte2p = pmap_pte2_quick(pmap, sva); KASSERT(pte2_is_valid(pte2_load(pte2p)), ("%s: invalid PTE2", __func__)); pmap_remove_pte2(pmap, pte2p, sva, NULL); } } if (pdnxt > eva) pdnxt = eva; for (pte2p = pmap_pte2_quick(pmap, sva); sva != pdnxt; pte2p++, sva += PAGE_SIZE) { pte2 = pte2_load(pte2p); if (!pte2_is_valid(pte2) || !pte2_is_managed(pte2)) continue; else if (pte2_is_dirty(pte2)) { if (advice == MADV_DONTNEED) { /* * Future calls to pmap_is_modified() * can be avoided by making the page * dirty now. */ m = PHYS_TO_VM_PAGE(pte2_pa(pte2)); vm_page_dirty(m); } pte2_set_bit(pte2p, PTE2_NM); pte2_clear_bit(pte2p, PTE2_A); } else if ((pte2 & PTE2_A) != 0) pte2_clear_bit(pte2p, PTE2_A); else continue; pmap_tlb_flush(pmap, sva); } } if (pv_lists_locked) { sched_unpin(); rw_wunlock(&pvh_global_lock); } PMAP_UNLOCK(pmap); } /* * Clear the modify bits on the specified physical page. */ void pmap_clear_modify(vm_page_t m) { struct md_page *pvh; pv_entry_t next_pv, pv; pmap_t pmap; pt1_entry_t *pte1p, opte1; pt2_entry_t *pte2p, opte2; vm_offset_t va; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("%s: page %p is not managed", __func__, m)); VM_OBJECT_ASSERT_WLOCKED(m->object); KASSERT(!vm_page_xbusied(m), ("%s: page %p is exclusive busy", __func__, m)); /* * If the page is not PGA_WRITEABLE, then no PTE2s can have PTE2_NM * cleared. If the object containing the page is locked and the page * is not exclusive busied, then PGA_WRITEABLE cannot be concurrently * set. */ if ((m->flags & PGA_WRITEABLE) == 0) return; rw_wlock(&pvh_global_lock); sched_pin(); if ((m->flags & PG_FICTITIOUS) != 0) goto small_mappings; pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_next, next_pv) { va = pv->pv_va; pmap = PV_PMAP(pv); PMAP_LOCK(pmap); pte1p = pmap_pte1(pmap, va); opte1 = pte1_load(pte1p); if (!(opte1 & PTE1_RO)) { if (pmap_demote_pte1(pmap, pte1p, va) && !pte1_is_wired(opte1)) { /* * Write protect the mapping to a * single page so that a subsequent * write access may repromote. */ va += VM_PAGE_TO_PHYS(m) - pte1_pa(opte1); pte2p = pmap_pte2_quick(pmap, va); opte2 = pte2_load(pte2p); if ((opte2 & PTE2_V)) { pte2_set_bit(pte2p, PTE2_NM | PTE2_RO); vm_page_dirty(m); pmap_tlb_flush(pmap, va); } } } PMAP_UNLOCK(pmap); } small_mappings: TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { pmap = PV_PMAP(pv); PMAP_LOCK(pmap); pte1p = pmap_pte1(pmap, pv->pv_va); KASSERT(!pte1_is_section(pte1_load(pte1p)), ("%s: found" " a section in page %p's pv list", __func__, m)); pte2p = pmap_pte2_quick(pmap, pv->pv_va); if (pte2_is_dirty(pte2_load(pte2p))) { pte2_set_bit(pte2p, PTE2_NM); pmap_tlb_flush(pmap, pv->pv_va); } PMAP_UNLOCK(pmap); } sched_unpin(); rw_wunlock(&pvh_global_lock); } /* * Sets the memory attribute for the specified page. */ void pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma) { struct sysmaps *sysmaps; vm_memattr_t oma; vm_paddr_t pa; oma = m->md.pat_mode; m->md.pat_mode = ma; CTR5(KTR_PMAP, "%s: page %p - 0x%08X oma: %d, ma: %d", __func__, m, VM_PAGE_TO_PHYS(m), oma, ma); if ((m->flags & PG_FICTITIOUS) != 0) return; #if 0 /* * If "m" is a normal page, flush it from the cache. * * First, try to find an existing mapping of the page by sf * buffer. sf_buf_invalidate_cache() modifies mapping and * flushes the cache. */ if (sf_buf_invalidate_cache(m, oma)) return; #endif /* * If page is not mapped by sf buffer, map the page * transient and do invalidation. */ if (ma != oma) { pa = VM_PAGE_TO_PHYS(m); sched_pin(); sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)]; mtx_lock(&sysmaps->lock); if (*sysmaps->CMAP2) panic("%s: CMAP2 busy", __func__); pte2_store(sysmaps->CMAP2, PTE2_KERN_NG(pa, PTE2_AP_KRW, vm_memattr_to_pte2(ma))); dcache_wbinv_poc((vm_offset_t)sysmaps->CADDR2, pa, PAGE_SIZE); pte2_clear(sysmaps->CMAP2); tlb_flush((vm_offset_t)sysmaps->CADDR2); sched_unpin(); mtx_unlock(&sysmaps->lock); } } /* * Miscellaneous support routines follow */ /* * Returns TRUE if the given page is mapped individually or as part of * a 1mpage. Otherwise, returns FALSE. */ boolean_t pmap_page_is_mapped(vm_page_t m) { boolean_t rv; if ((m->oflags & VPO_UNMANAGED) != 0) return (FALSE); rw_wlock(&pvh_global_lock); rv = !TAILQ_EMPTY(&m->md.pv_list) || ((m->flags & PG_FICTITIOUS) == 0 && !TAILQ_EMPTY(&pa_to_pvh(VM_PAGE_TO_PHYS(m))->pv_list)); rw_wunlock(&pvh_global_lock); return (rv); } /* * Returns true if the pmap's pv is one of the first * 16 pvs linked to from this page. This count may * be changed upwards or downwards in the future; it * is only necessary that true be returned for a small * subset of pmaps for proper page aging. */ boolean_t pmap_page_exists_quick(pmap_t pmap, vm_page_t m) { struct md_page *pvh; pv_entry_t pv; int loops = 0; boolean_t rv; KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("%s: page %p is not managed", __func__, m)); rv = FALSE; rw_wlock(&pvh_global_lock); TAILQ_FOREACH(pv, &m->md.pv_list, pv_next) { if (PV_PMAP(pv) == pmap) { rv = TRUE; break; } loops++; if (loops >= 16) break; } if (!rv && loops < 16 && (m->flags & PG_FICTITIOUS) == 0) { pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); TAILQ_FOREACH(pv, &pvh->pv_list, pv_next) { if (PV_PMAP(pv) == pmap) { rv = TRUE; break; } loops++; if (loops >= 16) break; } } rw_wunlock(&pvh_global_lock); return (rv); } /* * pmap_zero_page zeros the specified hardware page by mapping * the page into KVM and using bzero to clear its contents. */ void pmap_zero_page(vm_page_t m) { struct sysmaps *sysmaps; sched_pin(); sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)]; mtx_lock(&sysmaps->lock); if (pte2_load(sysmaps->CMAP2) != 0) panic("%s: CMAP2 busy", __func__); pte2_store(sysmaps->CMAP2, PTE2_KERN_NG(VM_PAGE_TO_PHYS(m), PTE2_AP_KRW, vm_page_pte2_attr(m))); pagezero(sysmaps->CADDR2); pte2_clear(sysmaps->CMAP2); tlb_flush((vm_offset_t)sysmaps->CADDR2); sched_unpin(); mtx_unlock(&sysmaps->lock); } /* * pmap_zero_page_area zeros the specified hardware page by mapping * the page into KVM and using bzero to clear its contents. * * off and size may not cover an area beyond a single hardware page. */ void pmap_zero_page_area(vm_page_t m, int off, int size) { struct sysmaps *sysmaps; sched_pin(); sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)]; mtx_lock(&sysmaps->lock); if (pte2_load(sysmaps->CMAP2) != 0) panic("%s: CMAP2 busy", __func__); pte2_store(sysmaps->CMAP2, PTE2_KERN_NG(VM_PAGE_TO_PHYS(m), PTE2_AP_KRW, vm_page_pte2_attr(m))); if (off == 0 && size == PAGE_SIZE) pagezero(sysmaps->CADDR2); else bzero(sysmaps->CADDR2 + off, size); pte2_clear(sysmaps->CMAP2); tlb_flush((vm_offset_t)sysmaps->CADDR2); sched_unpin(); mtx_unlock(&sysmaps->lock); } /* * pmap_zero_page_idle zeros the specified hardware page by mapping * the page into KVM and using bzero to clear its contents. This * is intended to be called from the vm_pagezero process only and * outside of Giant. */ void pmap_zero_page_idle(vm_page_t m) { if (pte2_load(CMAP3) != 0) panic("%s: CMAP3 busy", __func__); sched_pin(); pte2_store(CMAP3, PTE2_KERN_NG(VM_PAGE_TO_PHYS(m), PTE2_AP_KRW, vm_page_pte2_attr(m))); pagezero(CADDR3); pte2_clear(CMAP3); tlb_flush((vm_offset_t)CADDR3); sched_unpin(); } /* * pmap_copy_page copies the specified (machine independent) * page by mapping the page into virtual memory and using * bcopy to copy the page, one machine dependent page at a * time. */ void pmap_copy_page(vm_page_t src, vm_page_t dst) { struct sysmaps *sysmaps; sched_pin(); sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)]; mtx_lock(&sysmaps->lock); if (pte2_load(sysmaps->CMAP1) != 0) panic("%s: CMAP1 busy", __func__); if (pte2_load(sysmaps->CMAP2) != 0) panic("%s: CMAP2 busy", __func__); pte2_store(sysmaps->CMAP1, PTE2_KERN_NG(VM_PAGE_TO_PHYS(src), PTE2_AP_KR | PTE2_NM, vm_page_pte2_attr(src))); pte2_store(sysmaps->CMAP2, PTE2_KERN_NG(VM_PAGE_TO_PHYS(dst), PTE2_AP_KRW, vm_page_pte2_attr(dst))); bcopy(sysmaps->CADDR1, sysmaps->CADDR2, PAGE_SIZE); pte2_clear(sysmaps->CMAP1); tlb_flush((vm_offset_t)sysmaps->CADDR1); pte2_clear(sysmaps->CMAP2); tlb_flush((vm_offset_t)sysmaps->CADDR2); sched_unpin(); mtx_unlock(&sysmaps->lock); } int unmapped_buf_allowed = 1; void pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[], vm_offset_t b_offset, int xfersize) { struct sysmaps *sysmaps; vm_page_t a_pg, b_pg; char *a_cp, *b_cp; vm_offset_t a_pg_offset, b_pg_offset; int cnt; sched_pin(); sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)]; mtx_lock(&sysmaps->lock); if (*sysmaps->CMAP1 != 0) panic("pmap_copy_pages: CMAP1 busy"); if (*sysmaps->CMAP2 != 0) panic("pmap_copy_pages: CMAP2 busy"); while (xfersize > 0) { a_pg = ma[a_offset >> PAGE_SHIFT]; a_pg_offset = a_offset & PAGE_MASK; cnt = min(xfersize, PAGE_SIZE - a_pg_offset); b_pg = mb[b_offset >> PAGE_SHIFT]; b_pg_offset = b_offset & PAGE_MASK; cnt = min(cnt, PAGE_SIZE - b_pg_offset); pte2_store(sysmaps->CMAP1, PTE2_KERN_NG(VM_PAGE_TO_PHYS(a_pg), PTE2_AP_KR | PTE2_NM, vm_page_pte2_attr(a_pg))); tlb_flush_local((vm_offset_t)sysmaps->CADDR1); pte2_store(sysmaps->CMAP2, PTE2_KERN_NG(VM_PAGE_TO_PHYS(b_pg), PTE2_AP_KRW, vm_page_pte2_attr(b_pg))); tlb_flush_local((vm_offset_t)sysmaps->CADDR2); a_cp = sysmaps->CADDR1 + a_pg_offset; b_cp = sysmaps->CADDR2 + b_pg_offset; bcopy(a_cp, b_cp, cnt); a_offset += cnt; b_offset += cnt; xfersize -= cnt; } pte2_clear(sysmaps->CMAP1); tlb_flush((vm_offset_t)sysmaps->CADDR1); pte2_clear(sysmaps->CMAP2); tlb_flush((vm_offset_t)sysmaps->CADDR2); sched_unpin(); mtx_unlock(&sysmaps->lock); } vm_offset_t pmap_quick_enter_page(vm_page_t m) { pt2_entry_t *pte2p; vm_offset_t qmap_addr; critical_enter(); qmap_addr = PCPU_GET(qmap_addr); pte2p = pt2map_entry(qmap_addr); KASSERT(pte2_load(pte2p) == 0, ("%s: PTE2 busy", __func__)); pte2_store(pte2p, PTE2_KERN_NG(VM_PAGE_TO_PHYS(m), PTE2_AP_KRW, vm_page_pte2_attr(m))); return (qmap_addr); } void pmap_quick_remove_page(vm_offset_t addr) { pt2_entry_t *pte2p; vm_offset_t qmap_addr; qmap_addr = PCPU_GET(qmap_addr); pte2p = pt2map_entry(qmap_addr); KASSERT(addr == qmap_addr, ("%s: invalid address", __func__)); KASSERT(pte2_load(pte2p) != 0, ("%s: PTE2 not in use", __func__)); pte2_clear(pte2p); tlb_flush(qmap_addr); critical_exit(); } /* * Copy the range specified by src_addr/len * from the source map to the range dst_addr/len * in the destination map. * * This routine is only advisory and need not do anything. */ void pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len, vm_offset_t src_addr) { struct spglist free; vm_offset_t addr; vm_offset_t end_addr = src_addr + len; vm_offset_t nextva; if (dst_addr != src_addr) return; if (!pmap_is_current(src_pmap)) return; rw_wlock(&pvh_global_lock); if (dst_pmap < src_pmap) { PMAP_LOCK(dst_pmap); PMAP_LOCK(src_pmap); } else { PMAP_LOCK(src_pmap); PMAP_LOCK(dst_pmap); } sched_pin(); for (addr = src_addr; addr < end_addr; addr = nextva) { pt2_entry_t *src_pte2p, *dst_pte2p; vm_page_t dst_mpt2pg, src_mpt2pg; pt1_entry_t src_pte1; u_int pte1_idx; KASSERT(addr < VM_MAXUSER_ADDRESS, ("%s: invalid to pmap_copy page tables", __func__)); nextva = pte1_trunc(addr + PTE1_SIZE); if (nextva < addr) nextva = end_addr; pte1_idx = pte1_index(addr); src_pte1 = src_pmap->pm_pt1[pte1_idx]; if (pte1_is_section(src_pte1)) { if ((addr & PTE1_OFFSET) != 0 || (addr + PTE1_SIZE) > end_addr) continue; if (dst_pmap->pm_pt1[pte1_idx] == 0 && (!pte1_is_managed(src_pte1) || pmap_pv_insert_pte1(dst_pmap, addr, pte1_pa(src_pte1)))) { dst_pmap->pm_pt1[pte1_idx] = src_pte1 & ~PTE1_W; dst_pmap->pm_stats.resident_count += PTE1_SIZE / PAGE_SIZE; pmap_pte1_mappings++; } continue; } else if (!pte1_is_link(src_pte1)) continue; src_mpt2pg = PHYS_TO_VM_PAGE(pte1_link_pa(src_pte1)); /* * We leave PT2s to be linked from PT1 even if they are not * referenced until all PT2s in a page are without reference. * * QQQ: It could be changed ... */ #if 0 /* single_pt2_link_is_cleared */ KASSERT(pt2_wirecount_get(src_mpt2pg, pte1_idx) > 0, ("%s: source page table page is unused", __func__)); #else if (pt2_wirecount_get(src_mpt2pg, pte1_idx) == 0) continue; #endif if (nextva > end_addr) nextva = end_addr; src_pte2p = pt2map_entry(addr); while (addr < nextva) { pt2_entry_t temp_pte2; temp_pte2 = pte2_load(src_pte2p); /* * we only virtual copy managed pages */ if (pte2_is_managed(temp_pte2)) { dst_mpt2pg = pmap_allocpte2(dst_pmap, addr, PMAP_ENTER_NOSLEEP); if (dst_mpt2pg == NULL) goto out; dst_pte2p = pmap_pte2_quick(dst_pmap, addr); if (!pte2_is_valid(pte2_load(dst_pte2p)) && pmap_try_insert_pv_entry(dst_pmap, addr, PHYS_TO_VM_PAGE(pte2_pa(temp_pte2)))) { /* * Clear the wired, modified, and * accessed (referenced) bits * during the copy. */ temp_pte2 &= ~(PTE2_W | PTE2_A); temp_pte2 |= PTE2_NM; pte2_store(dst_pte2p, temp_pte2); dst_pmap->pm_stats.resident_count++; } else { SLIST_INIT(&free); if (pmap_unwire_pt2(dst_pmap, addr, dst_mpt2pg, &free)) { pmap_tlb_flush(dst_pmap, addr); pmap_free_zero_pages(&free); } goto out; } if (pt2_wirecount_get(dst_mpt2pg, pte1_idx) >= pt2_wirecount_get(src_mpt2pg, pte1_idx)) break; } addr += PAGE_SIZE; src_pte2p++; } } out: sched_unpin(); rw_wunlock(&pvh_global_lock); PMAP_UNLOCK(src_pmap); PMAP_UNLOCK(dst_pmap); } /* * Increase the starting virtual address of the given mapping if a * different alignment might result in more section mappings. */ void pmap_align_superpage(vm_object_t object, vm_ooffset_t offset, vm_offset_t *addr, vm_size_t size) { vm_offset_t pte1_offset; if (size < PTE1_SIZE) return; if (object != NULL && (object->flags & OBJ_COLORED) != 0) offset += ptoa(object->pg_color); pte1_offset = offset & PTE1_OFFSET; if (size - ((PTE1_SIZE - pte1_offset) & PTE1_OFFSET) < PTE1_SIZE || (*addr & PTE1_OFFSET) == pte1_offset) return; if ((*addr & PTE1_OFFSET) < pte1_offset) *addr = pte1_trunc(*addr) + pte1_offset; else *addr = pte1_roundup(*addr) + pte1_offset; } void pmap_activate(struct thread *td) { pmap_t pmap, oldpmap; u_int cpuid, ttb; PDEBUG(9, printf("%s: td = %08x\n", __func__, (uint32_t)td)); critical_enter(); pmap = vmspace_pmap(td->td_proc->p_vmspace); oldpmap = PCPU_GET(curpmap); cpuid = PCPU_GET(cpuid); #if defined(SMP) CPU_CLR_ATOMIC(cpuid, &oldpmap->pm_active); CPU_SET_ATOMIC(cpuid, &pmap->pm_active); #else CPU_CLR(cpuid, &oldpmap->pm_active); CPU_SET(cpuid, &pmap->pm_active); #endif ttb = pmap_ttb_get(pmap); /* * pmap_activate is for the current thread on the current cpu */ td->td_pcb->pcb_pagedir = ttb; cp15_ttbr_set(ttb); PCPU_SET(curpmap, pmap); critical_exit(); } /* * Perform the pmap work for mincore. */ int pmap_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *locked_pa) { pt1_entry_t *pte1p, pte1; pt2_entry_t *pte2p, pte2; vm_paddr_t pa; boolean_t managed; int val; PMAP_LOCK(pmap); retry: pte1p = pmap_pte1(pmap, addr); pte1 = pte1_load(pte1p); if (pte1_is_section(pte1)) { pa = trunc_page(pte1_pa(pte1) | (addr & PTE1_OFFSET)); managed = pte1_is_managed(pte1); val = MINCORE_SUPER | MINCORE_INCORE; if (pte1_is_dirty(pte1)) val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER; if (pte1 & PTE1_A) val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER; } else if (pte1_is_link(pte1)) { pte2p = pmap_pte2(pmap, addr); pte2 = pte2_load(pte2p); pmap_pte2_release(pte2p); pa = pte2_pa(pte2); managed = pte2_is_managed(pte2); val = MINCORE_INCORE; if (pte2_is_dirty(pte2)) val |= MINCORE_MODIFIED | MINCORE_MODIFIED_OTHER; if (pte2 & PTE2_A) val |= MINCORE_REFERENCED | MINCORE_REFERENCED_OTHER; } else { managed = FALSE; val = 0; } if ((val & (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER)) != (MINCORE_MODIFIED_OTHER | MINCORE_REFERENCED_OTHER) && managed) { /* Ensure that "PHYS_TO_VM_PAGE(pa)->object" doesn't change. */ if (vm_page_pa_tryrelock(pmap, pa, locked_pa)) goto retry; } else PA_UNLOCK_COND(*locked_pa); PMAP_UNLOCK(pmap); return (val); } void pmap_kenter_device(vm_offset_t va, vm_size_t size, vm_paddr_t pa) { vm_offset_t sva; uint32_t l2attr; KASSERT((size & PAGE_MASK) == 0, ("%s: device mapping not page-sized", __func__)); sva = va; l2attr = vm_memattr_to_pte2(VM_MEMATTR_DEVICE); while (size != 0) { pmap_kenter_prot_attr(va, pa, PTE2_AP_KRW, l2attr); va += PAGE_SIZE; pa += PAGE_SIZE; size -= PAGE_SIZE; } tlb_flush_range(sva, va - sva); } void pmap_kremove_device(vm_offset_t va, vm_size_t size) { vm_offset_t sva; KASSERT((size & PAGE_MASK) == 0, ("%s: device mapping not page-sized", __func__)); sva = va; while (size != 0) { pmap_kremove(va); va += PAGE_SIZE; size -= PAGE_SIZE; } tlb_flush_range(sva, va - sva); } void pmap_set_pcb_pagedir(pmap_t pmap, struct pcb *pcb) { pcb->pcb_pagedir = pmap_ttb_get(pmap); } /* * Clean L1 data cache range by physical address. * The range must be within a single page. */ static void pmap_dcache_wb_pou(vm_paddr_t pa, vm_size_t size, uint32_t attr) { struct sysmaps *sysmaps; KASSERT(((pa & PAGE_MASK) + size) <= PAGE_SIZE, ("%s: not on single page", __func__)); sched_pin(); sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)]; mtx_lock(&sysmaps->lock); if (*sysmaps->CMAP3) panic("%s: CMAP3 busy", __func__); pte2_store(sysmaps->CMAP3, PTE2_KERN_NG(pa, PTE2_AP_KRW, attr)); dcache_wb_pou((vm_offset_t)sysmaps->CADDR3 + (pa & PAGE_MASK), size); pte2_clear(sysmaps->CMAP3); tlb_flush((vm_offset_t)sysmaps->CADDR3); sched_unpin(); mtx_unlock(&sysmaps->lock); } /* * Sync instruction cache range which is not mapped yet. */ void cache_icache_sync_fresh(vm_offset_t va, vm_paddr_t pa, vm_size_t size) { uint32_t len, offset; vm_page_t m; /* Write back d-cache on given address range. */ offset = pa & PAGE_MASK; for ( ; size != 0; size -= len, pa += len, offset = 0) { len = min(PAGE_SIZE - offset, size); m = PHYS_TO_VM_PAGE(pa); KASSERT(m != NULL, ("%s: vm_page_t is null for %#x", __func__, pa)); pmap_dcache_wb_pou(pa, len, vm_page_pte2_attr(m)); } /* * I-cache is VIPT. Only way how to flush all virtual mappings * on given physical address is to invalidate all i-cache. */ icache_inv_all(); } void pmap_sync_icache(pmap_t pmap, vm_offset_t va, vm_size_t size) { /* Write back d-cache on given address range. */ if (va >= VM_MIN_KERNEL_ADDRESS) { dcache_wb_pou(va, size); } else { uint32_t len, offset; vm_paddr_t pa; vm_page_t m; offset = va & PAGE_MASK; for ( ; size != 0; size -= len, va += len, offset = 0) { pa = pmap_extract(pmap, va); /* offset is preserved */ len = min(PAGE_SIZE - offset, size); m = PHYS_TO_VM_PAGE(pa); KASSERT(m != NULL, ("%s: vm_page_t is null for %#x", __func__, pa)); pmap_dcache_wb_pou(pa, len, vm_page_pte2_attr(m)); } } /* * I-cache is VIPT. Only way how to flush all virtual mappings * on given physical address is to invalidate all i-cache. */ icache_inv_all(); } /* * The implementation of pmap_fault() uses IN_RANGE2() macro which * depends on the fact that given range size is a power of 2. */ CTASSERT(powerof2(NB_IN_PT1)); CTASSERT(powerof2(PT2MAP_SIZE)); #define IN_RANGE2(addr, start, size) \ ((vm_offset_t)(start) == ((vm_offset_t)(addr) & ~((size) - 1))) /* * Handle access and R/W emulation faults. */ int pmap_fault(pmap_t pmap, vm_offset_t far, uint32_t fsr, int idx, bool usermode) { pt1_entry_t *pte1p, pte1; pt2_entry_t *pte2p, pte2; if (pmap == NULL) pmap = kernel_pmap; /* * In kernel, we should never get abort with FAR which is in range of * pmap->pm_pt1 or PT2MAP address spaces. If it happens, stop here * and print out a useful abort message and even get to the debugger * otherwise it likely ends with never ending loop of aborts. */ if (__predict_false(IN_RANGE2(far, pmap->pm_pt1, NB_IN_PT1))) { /* * All L1 tables should always be mapped and present. * However, we check only current one herein. For user mode, * only permission abort from malicious user is not fatal. * And alignment abort as it may have higher priority. */ if (!usermode || (idx != FAULT_ALIGN && idx != FAULT_PERM_L2)) { CTR4(KTR_PMAP, "%s: pmap %#x pm_pt1 %#x far %#x", __func__, pmap, pmap->pm_pt1, far); panic("%s: pm_pt1 abort", __func__); } return (KERN_INVALID_ADDRESS); } if (__predict_false(IN_RANGE2(far, PT2MAP, PT2MAP_SIZE))) { /* * PT2MAP should be always mapped and present in current * L1 table. However, only existing L2 tables are mapped * in PT2MAP. For user mode, only L2 translation abort and * permission abort from malicious user is not fatal. * And alignment abort as it may have higher priority. */ if (!usermode || (idx != FAULT_ALIGN && idx != FAULT_TRAN_L2 && idx != FAULT_PERM_L2)) { CTR4(KTR_PMAP, "%s: pmap %#x PT2MAP %#x far %#x", __func__, pmap, PT2MAP, far); panic("%s: PT2MAP abort", __func__); } return (KERN_INVALID_ADDRESS); } /* * A pmap lock is used below for handling of access and R/W emulation * aborts. They were handled by atomic operations before so some * analysis of new situation is needed to answer the following question: * Is it safe to use the lock even for these aborts? * * There may happen two cases in general: * * (1) Aborts while the pmap lock is locked already - this should not * happen as pmap lock is not recursive. However, under pmap lock only * internal kernel data should be accessed and such data should be * mapped with A bit set and NM bit cleared. If double abort happens, * then a mapping of data which has caused it must be fixed. Further, * all new mappings are always made with A bit set and the bit can be * cleared only on managed mappings. * * (2) Aborts while another lock(s) is/are locked - this already can * happen. However, there is no difference here if it's either access or * R/W emulation abort, or if it's some other abort. */ PMAP_LOCK(pmap); #ifdef SMP /* * Special treatment due to break-before-make approach done when * pte1 is updated for userland mapping during section promotion or * demotion. If not catched here, pmap_enter() can find a section * mapping on faulting address. That is not allowed. */ if (idx == FAULT_TRAN_L1 && usermode && cp15_ats1cur_check(far) == 0) { PMAP_UNLOCK(pmap); return (KERN_SUCCESS); } #endif /* * Accesss bits for page and section. Note that the entry * is not in TLB yet, so TLB flush is not necessary. * * QQQ: This is hardware emulation, we do not call userret() * for aborts from user mode. */ if (idx == FAULT_ACCESS_L2) { pte2p = pt2map_entry(far); pte2 = pte2_load(pte2p); if (pte2_is_valid(pte2)) { pte2_store(pte2p, pte2 | PTE2_A); PMAP_UNLOCK(pmap); return (KERN_SUCCESS); } } if (idx == FAULT_ACCESS_L1) { pte1p = pmap_pte1(pmap, far); pte1 = pte1_load(pte1p); if (pte1_is_section(pte1)) { pte1_store(pte1p, pte1 | PTE1_A); PMAP_UNLOCK(pmap); return (KERN_SUCCESS); } } /* * Handle modify bits for page and section. Note that the modify * bit is emulated by software. So PTEx_RO is software read only * bit and PTEx_NM flag is real hardware read only bit. * * QQQ: This is hardware emulation, we do not call userret() * for aborts from user mode. */ if ((fsr & FSR_WNR) && (idx == FAULT_PERM_L2)) { pte2p = pt2map_entry(far); pte2 = pte2_load(pte2p); if (pte2_is_valid(pte2) && !(pte2 & PTE2_RO) && (pte2 & PTE2_NM)) { pte2_store(pte2p, pte2 & ~PTE2_NM); tlb_flush(trunc_page(far)); PMAP_UNLOCK(pmap); return (KERN_SUCCESS); } } if ((fsr & FSR_WNR) && (idx == FAULT_PERM_L1)) { pte1p = pmap_pte1(pmap, far); pte1 = pte1_load(pte1p); if (pte1_is_section(pte1) && !(pte1 & PTE1_RO) && (pte1 & PTE1_NM)) { pte1_store(pte1p, pte1 & ~PTE1_NM); tlb_flush(pte1_trunc(far)); PMAP_UNLOCK(pmap); return (KERN_SUCCESS); } } /* * QQQ: The previous code, mainly fast handling of access and * modify bits aborts, could be moved to ASM. Now we are * starting to deal with not fast aborts. */ #ifdef INVARIANTS /* * Read an entry in PT2TAB associated with both pmap and far. * It's safe because PT2TAB is always mapped. */ pte2 = pt2tab_load(pmap_pt2tab_entry(pmap, far)); if (pte2_is_valid(pte2)) { /* * Now, when we know that L2 page table is allocated, * we can use PT2MAP to get L2 page table entry. */ pte2 = pte2_load(pt2map_entry(far)); if (pte2_is_valid(pte2)) { /* * If L2 page table entry is valid, make sure that * L1 page table entry is valid too. Note that we * leave L2 page entries untouched when promoted. */ pte1 = pte1_load(pmap_pte1(pmap, far)); if (!pte1_is_valid(pte1)) { panic("%s: missing L1 page entry (%p, %#x)", __func__, pmap, far); } } } #endif PMAP_UNLOCK(pmap); return (KERN_FAILURE); } #if defined(PMAP_DEBUG) /* * Reusing of KVA used in pmap_zero_page function !!! */ static void pmap_zero_page_check(vm_page_t m) { uint32_t *p, *end; struct sysmaps *sysmaps; sched_pin(); sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)]; mtx_lock(&sysmaps->lock); if (pte2_load(sysmaps->CMAP2) != 0) panic("%s: CMAP2 busy", __func__); pte2_store(sysmaps->CMAP2, PTE2_KERN_NG(VM_PAGE_TO_PHYS(m), PTE2_AP_KRW, vm_page_pte2_attr(m))); end = (uint32_t*)(sysmaps->CADDR2 + PAGE_SIZE); for (p = (uint32_t*)sysmaps->CADDR2; p < end; p++) if (*p != 0) panic("%s: page %p not zero, va: %p", __func__, m, sysmaps->CADDR2); pte2_clear(sysmaps->CMAP2); tlb_flush((vm_offset_t)sysmaps->CADDR2); sched_unpin(); mtx_unlock(&sysmaps->lock); } int pmap_pid_dump(int pid) { pmap_t pmap; struct proc *p; int npte2 = 0; int i, j, index; sx_slock(&allproc_lock); FOREACH_PROC_IN_SYSTEM(p) { if (p->p_pid != pid || p->p_vmspace == NULL) continue; index = 0; pmap = vmspace_pmap(p->p_vmspace); for (i = 0; i < NPTE1_IN_PT1; i++) { pt1_entry_t pte1; pt2_entry_t *pte2p, pte2; vm_offset_t base, va; vm_paddr_t pa; vm_page_t m; base = i << PTE1_SHIFT; pte1 = pte1_load(&pmap->pm_pt1[i]); if (pte1_is_section(pte1)) { /* * QQQ: Do something here! */ } else if (pte1_is_link(pte1)) { for (j = 0; j < NPTE2_IN_PT2; j++) { va = base + (j << PAGE_SHIFT); if (va >= VM_MIN_KERNEL_ADDRESS) { if (index) { index = 0; printf("\n"); } sx_sunlock(&allproc_lock); return (npte2); } pte2p = pmap_pte2(pmap, va); pte2 = pte2_load(pte2p); pmap_pte2_release(pte2p); if (!pte2_is_valid(pte2)) continue; pa = pte2_pa(pte2); m = PHYS_TO_VM_PAGE(pa); printf("va: 0x%x, pa: 0x%x, h: %d, w:" " %d, f: 0x%x", va, pa, m->hold_count, m->wire_count, m->flags); npte2++; index++; if (index >= 2) { index = 0; printf("\n"); } else { printf(" "); } } } } } sx_sunlock(&allproc_lock); return (npte2); } #endif #ifdef DDB static pt2_entry_t * pmap_pte2_ddb(pmap_t pmap, vm_offset_t va) { pt1_entry_t pte1; vm_paddr_t pt2pg_pa; pte1 = pte1_load(pmap_pte1(pmap, va)); if (!pte1_is_link(pte1)) return (NULL); if (pmap_is_current(pmap)) return (pt2map_entry(va)); /* Note that L2 page table size is not equal to PAGE_SIZE. */ pt2pg_pa = trunc_page(pte1_link_pa(pte1)); if (pte2_pa(pte2_load(PMAP3)) != pt2pg_pa) { pte2_store(PMAP3, PTE2_KPT(pt2pg_pa)); #ifdef SMP PMAP3cpu = PCPU_GET(cpuid); #endif tlb_flush_local((vm_offset_t)PADDR3); } #ifdef SMP else if (PMAP3cpu != PCPU_GET(cpuid)) { PMAP3cpu = PCPU_GET(cpuid); tlb_flush_local((vm_offset_t)PADDR3); } #endif return (PADDR3 + (arm32_btop(va) & (NPTE2_IN_PG - 1))); } static void dump_pmap(pmap_t pmap) { printf("pmap %p\n", pmap); printf(" pm_pt1: %p\n", pmap->pm_pt1); printf(" pm_pt2tab: %p\n", pmap->pm_pt2tab); printf(" pm_active: 0x%08lX\n", pmap->pm_active.__bits[0]); } DB_SHOW_COMMAND(pmaps, pmap_list_pmaps) { pmap_t pmap; LIST_FOREACH(pmap, &allpmaps, pm_list) { dump_pmap(pmap); } } static int pte2_class(pt2_entry_t pte2) { int cls; cls = (pte2 >> 2) & 0x03; cls |= (pte2 >> 4) & 0x04; return (cls); } static void dump_section(pmap_t pmap, uint32_t pte1_idx) { } static void dump_link(pmap_t pmap, uint32_t pte1_idx, boolean_t invalid_ok) { uint32_t i; vm_offset_t va; pt2_entry_t *pte2p, pte2; vm_page_t m; va = pte1_idx << PTE1_SHIFT; pte2p = pmap_pte2_ddb(pmap, va); for (i = 0; i < NPTE2_IN_PT2; i++, pte2p++, va += PAGE_SIZE) { pte2 = pte2_load(pte2p); if (pte2 == 0) continue; if (!pte2_is_valid(pte2)) { printf(" 0x%08X: 0x%08X", va, pte2); if (!invalid_ok) printf(" - not valid !!!"); printf("\n"); continue; } m = PHYS_TO_VM_PAGE(pte2_pa(pte2)); printf(" 0x%08X: 0x%08X, TEX%d, s:%d, g:%d, m:%p", va , pte2, pte2_class(pte2), !!(pte2 & PTE2_S), !(pte2 & PTE2_NG), m); if (m != NULL) { printf(" v:%d h:%d w:%d f:0x%04X\n", m->valid, m->hold_count, m->wire_count, m->flags); } else { printf("\n"); } } } static __inline boolean_t is_pv_chunk_space(vm_offset_t va) { if ((((vm_offset_t)pv_chunkbase) <= va) && (va < ((vm_offset_t)pv_chunkbase + PAGE_SIZE * pv_maxchunks))) return (TRUE); return (FALSE); } DB_SHOW_COMMAND(pmap, pmap_pmap_print) { /* XXX convert args. */ pmap_t pmap = (pmap_t)addr; pt1_entry_t pte1; pt2_entry_t pte2; vm_offset_t va, eva; vm_page_t m; uint32_t i; boolean_t invalid_ok, dump_link_ok, dump_pv_chunk; if (have_addr) { pmap_t pm; LIST_FOREACH(pm, &allpmaps, pm_list) if (pm == pmap) break; if (pm == NULL) { printf("given pmap %p is not in allpmaps list\n", pmap); return; } } else pmap = PCPU_GET(curpmap); eva = (modif[0] == 'u') ? VM_MAXUSER_ADDRESS : 0xFFFFFFFF; dump_pv_chunk = FALSE; /* XXX evaluate from modif[] */ printf("pmap: 0x%08X\n", (uint32_t)pmap); printf("PT2MAP: 0x%08X\n", (uint32_t)PT2MAP); printf("pt2tab: 0x%08X\n", (uint32_t)pmap->pm_pt2tab); for(i = 0; i < NPTE1_IN_PT1; i++) { pte1 = pte1_load(&pmap->pm_pt1[i]); if (pte1 == 0) continue; va = i << PTE1_SHIFT; if (va >= eva) break; if (pte1_is_section(pte1)) { printf("0x%08X: Section 0x%08X, s:%d g:%d\n", va, pte1, !!(pte1 & PTE1_S), !(pte1 & PTE1_NG)); dump_section(pmap, i); } else if (pte1_is_link(pte1)) { dump_link_ok = TRUE; invalid_ok = FALSE; pte2 = pte2_load(pmap_pt2tab_entry(pmap, va)); m = PHYS_TO_VM_PAGE(pte1_link_pa(pte1)); printf("0x%08X: Link 0x%08X, pt2tab: 0x%08X m: %p", va, pte1, pte2, m); if (is_pv_chunk_space(va)) { printf(" - pv_chunk space"); if (dump_pv_chunk) invalid_ok = TRUE; else dump_link_ok = FALSE; } else if (m != NULL) printf(" w:%d w2:%u", m->wire_count, pt2_wirecount_get(m, pte1_index(va))); if (pte2 == 0) printf(" !!! pt2tab entry is ZERO"); else if (pte2_pa(pte1) != pte2_pa(pte2)) printf(" !!! pt2tab entry is DIFFERENT - m: %p", PHYS_TO_VM_PAGE(pte2_pa(pte2))); printf("\n"); if (dump_link_ok) dump_link(pmap, i, invalid_ok); } else printf("0x%08X: Invalid entry 0x%08X\n", va, pte1); } } static void dump_pt2tab(pmap_t pmap) { uint32_t i; pt2_entry_t pte2; vm_offset_t va; vm_paddr_t pa; vm_page_t m; printf("PT2TAB:\n"); for (i = 0; i < PT2TAB_ENTRIES; i++) { pte2 = pte2_load(&pmap->pm_pt2tab[i]); if (!pte2_is_valid(pte2)) continue; va = i << PT2TAB_SHIFT; pa = pte2_pa(pte2); m = PHYS_TO_VM_PAGE(pa); printf(" 0x%08X: 0x%08X, TEX%d, s:%d, m:%p", va, pte2, pte2_class(pte2), !!(pte2 & PTE2_S), m); if (m != NULL) printf(" , h: %d, w: %d, f: 0x%04X pidx: %lld", m->hold_count, m->wire_count, m->flags, m->pindex); printf("\n"); } } DB_SHOW_COMMAND(pmap_pt2tab, pmap_pt2tab_print) { /* XXX convert args. */ pmap_t pmap = (pmap_t)addr; pt1_entry_t pte1; pt2_entry_t pte2; vm_offset_t va; uint32_t i, start; if (have_addr) { printf("supported only on current pmap\n"); return; } pmap = PCPU_GET(curpmap); printf("curpmap: 0x%08X\n", (uint32_t)pmap); printf("PT2MAP: 0x%08X\n", (uint32_t)PT2MAP); printf("pt2tab: 0x%08X\n", (uint32_t)pmap->pm_pt2tab); start = pte1_index((vm_offset_t)PT2MAP); for (i = start; i < (start + NPT2_IN_PT2TAB); i++) { pte1 = pte1_load(&pmap->pm_pt1[i]); if (pte1 == 0) continue; va = i << PTE1_SHIFT; if (pte1_is_section(pte1)) { printf("0x%08X: Section 0x%08X, s:%d\n", va, pte1, !!(pte1 & PTE1_S)); dump_section(pmap, i); } else if (pte1_is_link(pte1)) { pte2 = pte2_load(pmap_pt2tab_entry(pmap, va)); printf("0x%08X: Link 0x%08X, pt2tab: 0x%08X\n", va, pte1, pte2); if (pte2 == 0) printf(" !!! pt2tab entry is ZERO\n"); } else printf("0x%08X: Invalid entry 0x%08X\n", va, pte1); } dump_pt2tab(pmap); } #endif Index: user/ngie/release-pkg-fix-tests/sys/arm/arm/swtch-v4.S =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/arm/swtch-v4.S (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/arm/swtch-v4.S (revision 299091) @@ -1,377 +1,377 @@ /* $NetBSD: cpuswitch.S,v 1.41 2003/11/15 08:44:18 scw Exp $ */ /*- * Copyright 2003 Wasabi Systems, Inc. * All rights reserved. * * Written by Steve C. Woodford for Wasabi Systems, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed for the NetBSD Project by * Wasabi Systems, Inc. * 4. The name of Wasabi Systems, Inc. may not be used to endorse * or promote products derived from this software without specific prior * written permission. * * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /*- * Copyright (c) 1994-1998 Mark Brinicombe. * Copyright (c) 1994 Brini. * All rights reserved. * * This code is derived from software written for Brini by Mark Brinicombe * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Brini. * 4. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * RiscBSD kernel project * * cpuswitch.S * * cpu switching functions * * Created : 15/10/94 * */ #include "assym.s" #include "opt_sched.h" #include #include #include #include #include __FBSDID("$FreeBSD$"); #define GET_PCPU(tmp, tmp2) \ ldr tmp, .Lcurpcpu #ifdef VFP .fpu vfp /* allow VFP instructions */ #endif .Lcurpcpu: .word _C_LABEL(__pcpu) .Lblocked_lock: .word _C_LABEL(blocked_lock) #define DOMAIN_CLIENT 0x01 .Lcpufuncs: .word _C_LABEL(cpufuncs) /* * cpu_throw(oldtd, newtd) * * Remove current thread state, then select the next thread to run * and load its state. * r0 = oldtd * r1 = newtd */ ENTRY(cpu_throw) mov r5, r1 /* * r0 = oldtd * r5 = newtd */ #ifdef VFP /* This thread is dying, disable */ bl _C_LABEL(vfp_discard) /* VFP without preserving state. */ #endif GET_PCPU(r7, r9) ldr r7, [r5, #(TD_PCB)] /* r7 = new thread's PCB */ /* Switch to lwp0 context */ ldr r9, .Lcpufuncs mov lr, pc ldr pc, [r9, #CF_IDCACHE_WBINV_ALL] ldr r0, [r7, #(PCB_PL1VEC)] ldr r1, [r7, #(PCB_DACR)] /* * r0 = Pointer to L1 slot for vector_page (or NULL) * r1 = lwp0's DACR * r5 = lwp0 * r7 = lwp0's PCB * r9 = cpufuncs */ /* * Ensure the vector table is accessible by fixing up lwp0's L1 */ cmp r0, #0 /* No need to fixup vector table? */ ldrne r3, [r0] /* But if yes, fetch current value */ ldrne r2, [r7, #(PCB_L1VEC)] /* Fetch new vector_page value */ mcr p15, 0, r1, c3, c0, 0 /* Update DACR for lwp0's context */ cmpne r3, r2 /* Stuffing the same value? */ strne r2, [r0] /* Store if not. */ #ifdef PMAP_INCLUDE_PTE_SYNC /* * Need to sync the cache to make sure that last store is * visible to the MMU. */ movne r1, #4 movne lr, pc ldrne pc, [r9, #CF_DCACHE_WB_RANGE] #endif /* PMAP_INCLUDE_PTE_SYNC */ /* * Note: We don't do the same optimisation as cpu_switch() with * respect to avoiding flushing the TLB if we're switching to * the same L1 since this process' VM space may be about to go * away, so we don't want *any* turds left in the TLB. */ /* Switch the memory to the new process */ ldr r0, [r7, #(PCB_PAGEDIR)] mov lr, pc ldr pc, [r9, #CF_CONTEXT_SWITCH] GET_PCPU(r6, r4) /* Hook in a new pcb */ str r7, [r6, #PC_CURPCB] /* We have a new curthread now so make a note it */ str r5, [r6, #PC_CURTHREAD] /* Set the new tp */ ldr r6, [r5, #(TD_MD + MD_TP)] ldr r4, =ARM_TP_ADDRESS str r6, [r4] ldr r6, [r5, #(TD_MD + MD_RAS_START)] str r6, [r4, #4] /* ARM_RAS_START */ ldr r6, [r5, #(TD_MD + MD_RAS_END)] str r6, [r4, #8] /* ARM_RAS_END */ /* Restore all the saved registers and exit */ add r3, r7, #PCB_R4 ldmia r3, {r4-r12, sp, pc} END(cpu_throw) /* * cpu_switch(oldtd, newtd, lock) * * Save the current thread state, then select the next thread to run * and load its state. * r0 = oldtd * r1 = newtd * r2 = lock (new lock for old thread) */ ENTRY(cpu_switch) /* Interrupts are disabled. */ /* Save all the registers in the old thread's pcb. */ ldr r3, [r0, #(TD_PCB)] /* Restore all the saved registers and exit */ add r3, #(PCB_R4) stmia r3, {r4-r12, sp, lr, pc} mov r6, r2 /* Save the mutex */ /* rem: r0 = old lwp */ /* rem: interrupts are disabled */ /* Process is now on a processor. */ /* We have a new curthread now so make a note it */ GET_PCPU(r7, r2) str r1, [r7, #PC_CURTHREAD] /* Hook in a new pcb */ ldr r2, [r1, #TD_PCB] str r2, [r7, #PC_CURPCB] /* Stage two : Save old context */ /* Get the user structure for the old thread. */ ldr r2, [r0, #(TD_PCB)] mov r4, r0 /* Save the old thread. */ /* Store the old tp; userland can change it on armv4. */ ldr r3, =ARM_TP_ADDRESS ldr r9, [r3] str r9, [r0, #(TD_MD + MD_TP)] ldr r9, [r3, #4] str r9, [r0, #(TD_MD + MD_RAS_START)] ldr r9, [r3, #8] str r9, [r0, #(TD_MD + MD_RAS_END)] /* Set the new tp */ ldr r9, [r1, #(TD_MD + MD_TP)] str r9, [r3] ldr r9, [r1, #(TD_MD + MD_RAS_START)] str r9, [r3, #4] ldr r9, [r1, #(TD_MD + MD_RAS_END)] str r9, [r3, #8] /* Get the user structure for the new process in r9 */ ldr r9, [r1, #(TD_PCB)] /* rem: r2 = old PCB */ /* rem: r9 = new PCB */ /* rem: interrupts are enabled */ #ifdef VFP fmrx r0, fpexc /* If the VFP is enabled */ tst r0, #(VFPEXC_EN) /* the current thread has */ movne r1, #1 /* used it, so go save */ addne r0, r2, #(PCB_VFPSTATE) /* the state into the PCB */ blne _C_LABEL(vfp_store) /* and disable the VFP. */ #endif /* r0-r3 now free! */ /* Third phase : restore saved context */ /* rem: r2 = old PCB */ /* rem: r9 = new PCB */ ldr r5, [r9, #(PCB_DACR)] /* r5 = new DACR */ mov r2, #DOMAIN_CLIENT cmp r5, r2, lsl #(PMAP_DOMAIN_KERNEL * 2) /* Sw to kernel thread? */ beq .Lcs_context_switched /* Yup. Don't flush cache */ mrc p15, 0, r0, c3, c0, 0 /* r0 = old DACR */ /* * Get the new L1 table pointer into r11. If we're switching to * an LWP with the same address space as the outgoing one, we can * skip the cache purge and the TTB load. * * To avoid data dep stalls that would happen anyway, we try * and get some useful work done in the mean time. */ mrc p15, 0, r10, c2, c0, 0 /* r10 = old L1 */ ldr r11, [r9, #(PCB_PAGEDIR)] /* r11 = new L1 */ teq r10, r11 /* Same L1? */ cmpeq r0, r5 /* Same DACR? */ beq .Lcs_context_switched /* yes! */ /* - * Definately need to flush the cache. + * Definitely need to flush the cache. */ ldr r1, .Lcpufuncs mov lr, pc ldr pc, [r1, #CF_IDCACHE_WBINV_ALL] .Lcs_cache_purge_skipped: /* rem: r6 = lock */ /* rem: r9 = new PCB */ /* rem: r10 = old L1 */ /* rem: r11 = new L1 */ mov r2, #0x00000000 ldr r7, [r9, #(PCB_PL1VEC)] /* * Ensure the vector table is accessible by fixing up the L1 */ cmp r7, #0 /* No need to fixup vector table? */ ldrne r2, [r7] /* But if yes, fetch current value */ ldrne r0, [r9, #(PCB_L1VEC)] /* Fetch new vector_page value */ mcr p15, 0, r5, c3, c0, 0 /* Update DACR for new context */ cmpne r2, r0 /* Stuffing the same value? */ #ifndef PMAP_INCLUDE_PTE_SYNC strne r0, [r7] /* Nope, update it */ #else beq .Lcs_same_vector str r0, [r7] /* Otherwise, update it */ /* * Need to sync the cache to make sure that last store is * visible to the MMU. */ ldr r2, .Lcpufuncs mov r0, r7 mov r1, #4 mov lr, pc ldr pc, [r2, #CF_DCACHE_WB_RANGE] .Lcs_same_vector: #endif /* PMAP_INCLUDE_PTE_SYNC */ cmp r10, r11 /* Switching to the same L1? */ ldr r10, .Lcpufuncs beq .Lcs_same_l1 /* Yup. */ /* * Do a full context switch, including full TLB flush. */ mov r0, r11 mov lr, pc ldr pc, [r10, #CF_CONTEXT_SWITCH] b .Lcs_context_switched /* * We're switching to a different process in the same L1. * In this situation, we only need to flush the TLB for the * vector_page mapping, and even then only if r7 is non-NULL. */ .Lcs_same_l1: cmp r7, #0 movne r0, #0 /* We *know* vector_page's VA is 0x0 */ movne lr, pc ldrne pc, [r10, #CF_TLB_FLUSHID_SE] .Lcs_context_switched: /* Release the old thread */ str r6, [r4, #TD_LOCK] /* XXXSCW: Safe to re-enable FIQs here */ /* rem: r9 = new PCB */ /* Restore all the saved registers and exit */ add r3, r9, #PCB_R4 ldmia r3, {r4-r12, sp, pc} END(cpu_switch) Index: user/ngie/release-pkg-fix-tests/sys/arm/at91/at91_cfata.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/at91/at91_cfata.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/at91/at91_cfata.c (revision 299091) @@ -1,281 +1,281 @@ /*- * Copyright (c) 2008 Deglitch Networks, Stanislav Sedov * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * Driver for the AT91RM9200 CompactFlash controller operating in a * common memory mode. Interrupts are driven by polling. The driver * implements an ATA bridge and attached ATA channel driver on top * of it. - * NOTE WELL: this driver uses polling mode. To achive an acceptable + * NOTE WELL: this driver uses polling mode. To achieve an acceptable * operating speed you will probably want to use HZ=2000 in kernel * config. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include struct at91_cfata_softc { device_t dev; struct resource *mem_res; struct resource irq; void (*isr_cb)(void *); void *isr_arg; struct callout tick; }; static int at91_cfata_detach(device_t dev); static void at91_cfata_callout(void *arg); static int at91_cfata_probe(device_t dev) { device_set_desc_copy(dev, "AT91RM9200 CompactFlash controller"); return (0); } static int at91_cfata_attach(device_t dev) { struct at91_cfata_softc *sc; int rid, error; sc = device_get_softc(dev); sc->dev = dev; rid = 0; error = 0; sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->mem_res == NULL) return (ENOMEM); /* XXX: init CF controller? */ callout_init(&sc->tick, 1); /* Callout to poll the device. */ device_add_child(dev, "ata", -1); bus_generic_attach(dev); return (0); } static int at91_cfata_detach(device_t dev) { struct at91_cfata_softc *sc; sc = device_get_softc(dev); bus_generic_detach(sc->dev); if (sc->mem_res != NULL) { bus_release_resource(dev, SYS_RES_MEMORY, rman_get_rid(sc->mem_res), sc->mem_res); sc->mem_res = NULL; } return (0); } static struct resource * ata_at91_alloc_resource(device_t dev, device_t child, int type, int *rid, u_long start, u_long end, u_long count, u_int flags) { struct at91_cfata_softc *sc = device_get_softc(dev); KASSERT(type == SYS_RES_IRQ && *rid == ATA_IRQ_RID, ("[at91_cfata, %d]: illegal resource request (type %u rid %u)", __LINE__, type, *rid)); return (&sc->irq); } static int ata_at91_release_resource(device_t dev, device_t child, int type, int rid, struct resource *r) { KASSERT(type == SYS_RES_IRQ && rid == ATA_IRQ_RID, ("[at91_cfata, %d]: illegal resource request (type %u rid %u)", __LINE__, type, rid)); return (0); } static int ata_at91_setup_intr(device_t dev, device_t child, struct resource *irq, int flags, driver_filter_t *filt, driver_intr_t *function, void *argument, void **cookiep) { struct at91_cfata_softc *sc = device_get_softc(dev); KASSERT(sc->isr_cb == NULL, ("[at91_cfata, %d]: overwriting the old handler", __LINE__)); sc->isr_cb = function; sc->isr_arg = argument; *cookiep = sc; callout_reset(&sc->tick, 1, at91_cfata_callout, sc); return (0); } static int ata_at91_teardown_intr(device_t dev, device_t child, struct resource *irq, void *cookie) { struct at91_cfata_softc *sc = device_get_softc(dev); sc->isr_cb = NULL; sc->isr_arg = NULL; return (0); } static void at91_cfata_callout(void *arg) { struct at91_cfata_softc *sc; sc = (struct at91_cfata_softc *)arg; if (sc->isr_cb != NULL) sc->isr_cb(sc->isr_arg); callout_reset(&sc->tick, 1, at91_cfata_callout, sc); } static int at91_channel_probe(device_t dev) { struct ata_channel *ch = device_get_softc(dev); ch->unit = 0; ch->flags |= ATA_USE_16BIT | ATA_NO_SLAVE; device_set_desc_copy(dev, "ATA channel 0"); return (ata_probe(dev)); } static int at91_channel_attach(device_t dev) { struct at91_cfata_softc *sc = device_get_softc(device_get_parent(dev)); struct ata_channel *ch = device_get_softc(dev); int i; for (i = 0; i < ATA_MAX_RES; i++) ch->r_io[i].res = sc->mem_res; /* * CF+ Specification. * 6.1.3 Memory Mapped Addressing. */ ch->r_io[ATA_DATA].offset = 0x00; ch->r_io[ATA_FEATURE].offset = 0x01; ch->r_io[ATA_COUNT].offset = 0x02; ch->r_io[ATA_SECTOR].offset = 0x03; ch->r_io[ATA_CYL_LSB].offset = 0x04; ch->r_io[ATA_CYL_MSB].offset = 0x05; ch->r_io[ATA_DRIVE].offset = 0x06; ch->r_io[ATA_COMMAND].offset = 0x07; ch->r_io[ATA_ERROR].offset = 0x01; ch->r_io[ATA_IREASON].offset = 0x02; ch->r_io[ATA_STATUS].offset = 0x07; ch->r_io[ATA_ALTSTAT].offset = 0x0e; ch->r_io[ATA_CONTROL].offset = 0x0e; /* Should point at the base of registers. */ ch->r_io[ATA_IDX_ADDR].offset = 0x0; ata_generic_hw(dev); return (ata_attach(dev)); } static device_method_t at91_cfata_methods[] = { /* Device interface. */ DEVMETHOD(device_probe, at91_cfata_probe), DEVMETHOD(device_attach, at91_cfata_attach), DEVMETHOD(device_detach, at91_cfata_detach), DEVMETHOD(device_shutdown, bus_generic_shutdown), DEVMETHOD(device_suspend, bus_generic_suspend), DEVMETHOD(device_resume, bus_generic_resume), /* ATA bus methods. */ DEVMETHOD(bus_alloc_resource, ata_at91_alloc_resource), DEVMETHOD(bus_release_resource, ata_at91_release_resource), DEVMETHOD(bus_activate_resource, bus_generic_activate_resource), DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource), DEVMETHOD(bus_setup_intr, ata_at91_setup_intr), DEVMETHOD(bus_teardown_intr, ata_at91_teardown_intr), { 0, 0 } }; devclass_t at91_cfata_devclass; static driver_t at91_cfata_driver = { "at91_cfata", at91_cfata_methods, sizeof(struct at91_cfata_softc), }; DRIVER_MODULE(at91_cfata, atmelarm, at91_cfata_driver, at91_cfata_devclass, 0, 0); MODULE_VERSION(at91_cfata, 1); MODULE_DEPEND(at91_cfata, ata, 1, 1, 1); /* * ATA channel driver. */ static device_method_t at91_channel_methods[] = { /* device interface */ DEVMETHOD(device_probe, at91_channel_probe), DEVMETHOD(device_attach, at91_channel_attach), DEVMETHOD(device_detach, ata_detach), DEVMETHOD(device_shutdown, bus_generic_shutdown), DEVMETHOD(device_suspend, ata_suspend), DEVMETHOD(device_resume, ata_resume), { 0, 0 } }; driver_t at91_channel_driver = { "ata", at91_channel_methods, sizeof(struct ata_channel), }; DRIVER_MODULE(ata, at91_cfata, at91_channel_driver, ata_devclass, 0, 0); Index: user/ngie/release-pkg-fix-tests/sys/arm/at91/at91_machdep.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/at91/at91_machdep.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/at91/at91_machdep.c (revision 299091) @@ -1,685 +1,685 @@ /*- * Copyright (c) 1994-1998 Mark Brinicombe. * Copyright (c) 1994 Brini. * All rights reserved. * * This code is derived from software written for Brini by Mark Brinicombe * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Brini. * 4. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * RiscBSD kernel project * * machdep.c * - * Machine dependant functions for kernel setup + * Machine dependent functions for kernel setup * * This file needs a lot of work. * * Created : 17/09/94 */ #include "opt_kstack_pages.h" #include "opt_platform.h" #include __FBSDID("$FreeBSD$"); #define _ARM32_BUS_DMA_PRIVATE #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifndef MAXCPU #define MAXCPU 1 #endif /* Page table for mapping proc0 zero page */ #define KERNEL_PT_SYS 0 #define KERNEL_PT_KERN 1 #define KERNEL_PT_KERN_NUM 22 /* L2 table for mapping after kernel */ #define KERNEL_PT_AFKERNEL KERNEL_PT_KERN + KERNEL_PT_KERN_NUM #define KERNEL_PT_AFKERNEL_NUM 5 /* this should be evenly divisable by PAGE_SIZE / L2_TABLE_SIZE_REAL (or 4) */ #define NUM_KERNEL_PTS (KERNEL_PT_AFKERNEL + KERNEL_PT_AFKERNEL_NUM) struct pv_addr kernel_pt_table[NUM_KERNEL_PTS]; /* Static device mappings. */ const struct devmap_entry at91_devmap[] = { /* * Map the critical on-board devices. The interrupt vector at * 0xffff0000 makes it impossible to map them PA == VA, so we map all * 0xfffxxxxx addresses to 0xdffxxxxx. This covers all critical devices * on all members of the AT91SAM9 and AT91RM9200 families. */ { 0xdff00000, 0xfff00000, 0x00100000, }, /* There's a notion that we should do the rest of these lazily. */ /* * We can't just map the OHCI registers VA == PA, because * AT91xx_xxx_BASE belongs to the userland address space. * We could just choose a different virtual address, but a better * solution would probably be to just use pmap_mapdev() to allocate * KVA, as we don't need the OHCI controller before the vm * initialization is done. However, the AT91 resource allocation * system doesn't know how to use pmap_mapdev() yet. * Care must be taken to ensure PA and VM address do not overlap * between entries. */ { /* * Add the ohci controller, and anything else that might be * on this chip select for a VA/PA mapping. */ /* Internal Memory 1MB */ AT91RM92_OHCI_VA_BASE, AT91RM92_OHCI_BASE, 0x00100000, }, { /* CompactFlash controller. Portion of EBI CS4 1MB */ AT91RM92_CF_VA_BASE, AT91RM92_CF_BASE, 0x00100000, }, /* * The next two should be good for the 9260, 9261 and 9G20 since * addresses mapping is the same. */ { /* Internal Memory 1MB */ AT91SAM9G20_OHCI_VA_BASE, AT91SAM9G20_OHCI_BASE, 0x00100000, }, { /* EBI CS3 256MB */ AT91SAM9G20_NAND_VA_BASE, AT91SAM9G20_NAND_BASE, AT91SAM9G20_NAND_SIZE, }, /* * The next should be good for the 9G45. */ { /* Internal Memory 1MB */ AT91SAM9G45_OHCI_VA_BASE, AT91SAM9G45_OHCI_BASE, 0x00100000, }, { 0, 0, 0, } }; #ifdef LINUX_BOOT_ABI extern int membanks; extern int memstart[]; extern int memsize[]; #endif long at91_ramsize(void) { uint32_t cr, mdr, mr, *SDRAMC; int banks, rows, cols, bw; #ifdef LINUX_BOOT_ABI /* * If we found any ATAGs that were for memory, return the first bank. */ if (membanks > 0) return (memsize[0]); #endif if (at91_is_rm92()) { SDRAMC = (uint32_t *)(AT91_BASE + AT91RM92_SDRAMC_BASE); cr = SDRAMC[AT91RM92_SDRAMC_CR / 4]; mr = SDRAMC[AT91RM92_SDRAMC_MR / 4]; banks = (cr & AT91RM92_SDRAMC_CR_NB_4) ? 2 : 1; rows = ((cr & AT91RM92_SDRAMC_CR_NR_MASK) >> 2) + 11; cols = (cr & AT91RM92_SDRAMC_CR_NC_MASK) + 8; bw = (mr & AT91RM92_SDRAMC_MR_DBW_16) ? 1 : 2; } else if (at91_cpu_is(AT91_T_SAM9G45)) { SDRAMC = (uint32_t *)(AT91_BASE + AT91SAM9G45_DDRSDRC0_BASE); cr = SDRAMC[AT91SAM9G45_DDRSDRC_CR / 4]; mdr = SDRAMC[AT91SAM9G45_DDRSDRC_MDR / 4]; banks = 0; rows = ((cr & AT91SAM9G45_DDRSDRC_CR_NR_MASK) >> 2) + 11; cols = (cr & AT91SAM9G45_DDRSDRC_CR_NC_MASK) + 8; bw = (mdr & AT91SAM9G45_DDRSDRC_MDR_DBW_16) ? 1 : 2; /* Fix the calculation for DDR memory */ mdr &= AT91SAM9G45_DDRSDRC_MDR_MASK; if (mdr & AT91SAM9G45_DDRSDRC_MDR_LPDDR1 || mdr & AT91SAM9G45_DDRSDRC_MDR_DDR2) { /* The cols value is 1 higher for DDR */ cols += 1; /* DDR has 4 internal banks. */ banks = 2; } } else { /* * This should be good for the 9260, 9261, 9G20, 9G35 and 9X25 * as addresses and registers are the same. */ SDRAMC = (uint32_t *)(AT91_BASE + AT91SAM9G20_SDRAMC_BASE); cr = SDRAMC[AT91SAM9G20_SDRAMC_CR / 4]; mr = SDRAMC[AT91SAM9G20_SDRAMC_MR / 4]; banks = (cr & AT91SAM9G20_SDRAMC_CR_NB_4) ? 2 : 1; rows = ((cr & AT91SAM9G20_SDRAMC_CR_NR_MASK) >> 2) + 11; cols = (cr & AT91SAM9G20_SDRAMC_CR_NC_MASK) + 8; bw = (cr & AT91SAM9G20_SDRAMC_CR_DBW_16) ? 1 : 2; } return (1 << (cols + rows + banks + bw)); } static const char *soc_type_name[] = { [AT91_T_CAP9] = "at91cap9", [AT91_T_RM9200] = "at91rm9200", [AT91_T_SAM9260] = "at91sam9260", [AT91_T_SAM9261] = "at91sam9261", [AT91_T_SAM9263] = "at91sam9263", [AT91_T_SAM9G10] = "at91sam9g10", [AT91_T_SAM9G20] = "at91sam9g20", [AT91_T_SAM9G45] = "at91sam9g45", [AT91_T_SAM9N12] = "at91sam9n12", [AT91_T_SAM9RL] = "at91sam9rl", [AT91_T_SAM9X5] = "at91sam9x5", [AT91_T_NONE] = "UNKNOWN" }; static const char *soc_subtype_name[] = { [AT91_ST_NONE] = "UNKNOWN", [AT91_ST_RM9200_BGA] = "at91rm9200_bga", [AT91_ST_RM9200_PQFP] = "at91rm9200_pqfp", [AT91_ST_SAM9XE] = "at91sam9xe", [AT91_ST_SAM9G45] = "at91sam9g45", [AT91_ST_SAM9M10] = "at91sam9m10", [AT91_ST_SAM9G46] = "at91sam9g46", [AT91_ST_SAM9M11] = "at91sam9m11", [AT91_ST_SAM9G15] = "at91sam9g15", [AT91_ST_SAM9G25] = "at91sam9g25", [AT91_ST_SAM9G35] = "at91sam9g35", [AT91_ST_SAM9X25] = "at91sam9x25", [AT91_ST_SAM9X35] = "at91sam9x35", }; struct at91_soc_info soc_info; /* * Read the SoC ID from the CIDR register and try to match it against the * values we know. If we find a good one, we return true. If not, we * return false. When we find a good one, we also find the subtype * and CPU family. */ static int at91_try_id(uint32_t dbgu_base) { uint32_t socid; soc_info.cidr = *(volatile uint32_t *)(AT91_BASE + dbgu_base + DBGU_C1R); socid = soc_info.cidr & ~AT91_CPU_VERSION_MASK; soc_info.type = AT91_T_NONE; soc_info.subtype = AT91_ST_NONE; soc_info.family = (soc_info.cidr & AT91_CPU_FAMILY_MASK) >> 20; soc_info.exid = *(volatile uint32_t *)(AT91_BASE + dbgu_base + DBGU_C2R); switch (socid) { case AT91_CPU_CAP9: soc_info.type = AT91_T_CAP9; break; case AT91_CPU_RM9200: soc_info.type = AT91_T_RM9200; break; case AT91_CPU_SAM9XE128: case AT91_CPU_SAM9XE256: case AT91_CPU_SAM9XE512: case AT91_CPU_SAM9260: soc_info.type = AT91_T_SAM9260; if (soc_info.family == AT91_FAMILY_SAM9XE) soc_info.subtype = AT91_ST_SAM9XE; break; case AT91_CPU_SAM9261: soc_info.type = AT91_T_SAM9261; break; case AT91_CPU_SAM9263: soc_info.type = AT91_T_SAM9263; break; case AT91_CPU_SAM9G10: soc_info.type = AT91_T_SAM9G10; break; case AT91_CPU_SAM9G20: soc_info.type = AT91_T_SAM9G20; break; case AT91_CPU_SAM9G45: soc_info.type = AT91_T_SAM9G45; break; case AT91_CPU_SAM9N12: soc_info.type = AT91_T_SAM9N12; break; case AT91_CPU_SAM9RL64: soc_info.type = AT91_T_SAM9RL; break; case AT91_CPU_SAM9X5: soc_info.type = AT91_T_SAM9X5; break; default: return (0); } switch (soc_info.type) { case AT91_T_SAM9G45: switch (soc_info.exid) { case AT91_EXID_SAM9G45: soc_info.subtype = AT91_ST_SAM9G45; break; case AT91_EXID_SAM9G46: soc_info.subtype = AT91_ST_SAM9G46; break; case AT91_EXID_SAM9M10: soc_info.subtype = AT91_ST_SAM9M10; break; case AT91_EXID_SAM9M11: soc_info.subtype = AT91_ST_SAM9M11; break; } break; case AT91_T_SAM9X5: switch (soc_info.exid) { case AT91_EXID_SAM9G15: soc_info.subtype = AT91_ST_SAM9G15; break; case AT91_EXID_SAM9G25: soc_info.subtype = AT91_ST_SAM9G25; break; case AT91_EXID_SAM9G35: soc_info.subtype = AT91_ST_SAM9G35; break; case AT91_EXID_SAM9X25: soc_info.subtype = AT91_ST_SAM9X25; break; case AT91_EXID_SAM9X35: soc_info.subtype = AT91_ST_SAM9X35; break; } break; default: break; } /* * Disable interrupts in the DBGU unit... */ *(volatile uint32_t *)(AT91_BASE + dbgu_base + USART_IDR) = 0xffffffff; /* * Save the name for later... */ snprintf(soc_info.name, sizeof(soc_info.name), "%s%s%s", soc_type_name[soc_info.type], soc_info.subtype == AT91_ST_NONE ? "" : " subtype ", soc_info.subtype == AT91_ST_NONE ? "" : soc_subtype_name[soc_info.subtype]); /* * try to get the matching CPU support. */ soc_info.soc_data = at91_match_soc(soc_info.type, soc_info.subtype); soc_info.dbgu_base = AT91_BASE + dbgu_base; return (1); } void at91_soc_id(void) { if (!at91_try_id(AT91_DBGU0)) at91_try_id(AT91_DBGU1); } #ifdef ARM_MANY_BOARD /* likely belongs in arm/arm/machdep.c, but since board_init is still at91 only... */ SET_DECLARE(arm_board_set, const struct arm_board); /* Not yet fully functional, but enough to build ATMEL config */ static long board_init(void) { return -1; } #endif #ifndef FDT /* Physical and virtual addresses for some global pages */ struct pv_addr msgbufpv; struct pv_addr kernelstack; struct pv_addr systempage; struct pv_addr irqstack; struct pv_addr abtstack; struct pv_addr undstack; void * initarm(struct arm_boot_params *abp) { struct pv_addr kernel_l1pt; struct pv_addr dpcpu; int i; u_int l1pagetable; vm_offset_t freemempos; vm_offset_t afterkern; uint32_t memsize; vm_offset_t lastaddr; lastaddr = parse_boot_param(abp); arm_physmem_kernaddr = abp->abp_physaddr; set_cpufuncs(); pcpu0_init(); /* Do basic tuning, hz etc */ init_param1(); freemempos = (lastaddr + PAGE_MASK) & ~PAGE_MASK; /* Define a macro to simplify memory allocation */ #define valloc_pages(var, np) \ alloc_pages((var).pv_va, (np)); \ (var).pv_pa = (var).pv_va + (abp->abp_physaddr - KERNVIRTADDR); #define alloc_pages(var, np) \ (var) = freemempos; \ freemempos += (np * PAGE_SIZE); \ memset((char *)(var), 0, ((np) * PAGE_SIZE)); while (((freemempos - L1_TABLE_SIZE) & (L1_TABLE_SIZE - 1)) != 0) freemempos += PAGE_SIZE; valloc_pages(kernel_l1pt, L1_TABLE_SIZE / PAGE_SIZE); for (i = 0; i < NUM_KERNEL_PTS; ++i) { if (!(i % (PAGE_SIZE / L2_TABLE_SIZE_REAL))) { valloc_pages(kernel_pt_table[i], L2_TABLE_SIZE / PAGE_SIZE); } else { kernel_pt_table[i].pv_va = freemempos - (i % (PAGE_SIZE / L2_TABLE_SIZE_REAL)) * L2_TABLE_SIZE_REAL; kernel_pt_table[i].pv_pa = kernel_pt_table[i].pv_va - KERNVIRTADDR + abp->abp_physaddr; } } /* * Allocate a page for the system page mapped to 0x00000000 * or 0xffff0000. This page will just contain the system vectors * and can be shared by all processes. */ valloc_pages(systempage, 1); /* Allocate dynamic per-cpu area. */ valloc_pages(dpcpu, DPCPU_SIZE / PAGE_SIZE); dpcpu_init((void *)dpcpu.pv_va, 0); /* Allocate stacks for all modes */ valloc_pages(irqstack, IRQ_STACK_SIZE * MAXCPU); valloc_pages(abtstack, ABT_STACK_SIZE * MAXCPU); valloc_pages(undstack, UND_STACK_SIZE * MAXCPU); valloc_pages(kernelstack, kstack_pages * MAXCPU); valloc_pages(msgbufpv, round_page(msgbufsize) / PAGE_SIZE); /* * Now we start construction of the L1 page table * We start by mapping the L2 page tables into the L1. * This means that we can replace L1 mappings later on if necessary */ l1pagetable = kernel_l1pt.pv_va; /* Map the L2 pages tables in the L1 page table */ pmap_link_l2pt(l1pagetable, ARM_VECTORS_HIGH, &kernel_pt_table[KERNEL_PT_SYS]); for (i = 0; i < KERNEL_PT_KERN_NUM; i++) pmap_link_l2pt(l1pagetable, KERNBASE + i * L1_S_SIZE, &kernel_pt_table[KERNEL_PT_KERN + i]); pmap_map_chunk(l1pagetable, KERNBASE, PHYSADDR, rounddown2(((uint32_t)lastaddr - KERNBASE) + PAGE_SIZE, PAGE_SIZE), VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); afterkern = round_page(rounddown2(lastaddr + L1_S_SIZE, L1_S_SIZE)); for (i = 0; i < KERNEL_PT_AFKERNEL_NUM; i++) { pmap_link_l2pt(l1pagetable, afterkern + i * L1_S_SIZE, &kernel_pt_table[KERNEL_PT_AFKERNEL + i]); } /* Map the vector page. */ pmap_map_entry(l1pagetable, ARM_VECTORS_HIGH, systempage.pv_pa, VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); /* Map the DPCPU pages */ pmap_map_chunk(l1pagetable, dpcpu.pv_va, dpcpu.pv_pa, DPCPU_SIZE, VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); /* Map the stack pages */ pmap_map_chunk(l1pagetable, irqstack.pv_va, irqstack.pv_pa, IRQ_STACK_SIZE * PAGE_SIZE, VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); pmap_map_chunk(l1pagetable, abtstack.pv_va, abtstack.pv_pa, ABT_STACK_SIZE * PAGE_SIZE, VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); pmap_map_chunk(l1pagetable, undstack.pv_va, undstack.pv_pa, UND_STACK_SIZE * PAGE_SIZE, VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); pmap_map_chunk(l1pagetable, kernelstack.pv_va, kernelstack.pv_pa, kstack_pages * PAGE_SIZE, VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); pmap_map_chunk(l1pagetable, kernel_l1pt.pv_va, kernel_l1pt.pv_pa, L1_TABLE_SIZE, VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE); pmap_map_chunk(l1pagetable, msgbufpv.pv_va, msgbufpv.pv_pa, msgbufsize, VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); for (i = 0; i < NUM_KERNEL_PTS; ++i) { pmap_map_chunk(l1pagetable, kernel_pt_table[i].pv_va, kernel_pt_table[i].pv_pa, L2_TABLE_SIZE, VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE); } devmap_bootstrap(l1pagetable, at91_devmap); cpu_domains((DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL * 2)) | DOMAIN_CLIENT); cpu_setttb(kernel_l1pt.pv_pa); cpu_tlb_flushID(); cpu_domains(DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL * 2)); at91_soc_id(); /* * Initialize all the clocks, so that the console can work. We can only * do this if at91_soc_id() was able to fill in the support data. Even * if we can't init the clocks, still try to do a console init so we can * try to print the error message about missing soc support. There's a * chance the printf will work if the bootloader set up the DBGU. */ if (soc_info.soc_data != NULL) { soc_info.soc_data->soc_clock_init(); at91_pmc_init_clock(); } cninit(); if (soc_info.soc_data == NULL) printf("Warning: No soc support for %s found.\n", soc_info.name); memsize = board_init(); if (memsize == -1) { printf("board_init() failed, cannot determine ram size; " "assuming 16MB\n"); memsize = 16 * 1024 * 1024; } /* * Pages were allocated during the secondary bootstrap for the * stacks for different CPU modes. * We must now set the r13 registers in the different CPU modes to * point to these stacks. * Since the ARM stacks use STMFD etc. we must set r13 to the top end * of the stack memory. */ cpu_control(CPU_CONTROL_MMU_ENABLE, CPU_CONTROL_MMU_ENABLE); cpu_setup(); set_stackptrs(0); /* * We must now clean the cache again.... * Cleaning may be done by reading new data to displace any * dirty data in the cache. This will have happened in cpu_setttb() * but since we are boot strapping the addresses used for the read * may have just been remapped and thus the cache could be out * of sync. A re-clean after the switch will cure this. * After booting there are no gross relocations of the kernel thus * this problem will not occur after initarm(). */ cpu_idcache_wbinv_all(); undefined_init(); init_proc0(kernelstack.pv_va); arm_vector_init(ARM_VECTORS_HIGH, ARM_VEC_ALL); pmap_curmaxkvaddr = afterkern + L1_S_SIZE * (KERNEL_PT_KERN_NUM - 1); /* Always use the 256MB of KVA we have available between the kernel and devices */ vm_max_kernel_address = KERNVIRTADDR + (256 << 20); pmap_bootstrap(freemempos, &kernel_l1pt); msgbufp = (void*)msgbufpv.pv_va; msgbufinit(msgbufp, msgbufsize); mutex_init(); /* * Add the physical ram we have available. * * Exclude the kernel, and all the things we allocated which immediately * follow the kernel, from the VM allocation pool but not from crash * dumps. virtual_avail is a global variable which tracks the kva we've * "allocated" while setting up pmaps. * * Prepare the list of physical memory available to the vm subsystem. */ arm_physmem_hardware_region(PHYSADDR, memsize); arm_physmem_exclude_region(abp->abp_physaddr, virtual_avail - KERNVIRTADDR, EXFLAG_NOALLOC); arm_physmem_init_kernel_globals(); init_param2(physmem); kdb_init(); return ((void *)(kernelstack.pv_va + USPACE_SVC_STACK_TOP - sizeof(struct pcb))); } #endif /* * These functions are handled elsewhere, so make them nops here. */ void cpu_startprofclock(void) { } void cpu_stopprofclock(void) { } void cpu_initclocks(void) { } void DELAY(int n) { if (soc_info.soc_data) soc_info.soc_data->soc_delay(n); } void cpu_reset(void) { if (soc_info.soc_data) soc_info.soc_data->soc_reset(); while (1) continue; } Index: user/ngie/release-pkg-fix-tests/sys/arm/at91/at91_mci.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/at91/at91_mci.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/at91/at91_mci.c (revision 299091) @@ -1,1415 +1,1415 @@ /*- * Copyright (c) 2006 Bernd Walter. All rights reserved. * Copyright (c) 2006 M. Warner Losh. All rights reserved. * Copyright (c) 2010 Greg Ansley. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include "opt_platform.h" #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef FDT #include #include #include #endif #include "mmcbr_if.h" #include "opt_at91.h" /* * About running the MCI bus above 25MHz * * Historically, the MCI bus has been run at 30MHz on systems with a 60MHz * master clock, in part due to a bug in dev/mmc.c making always request * 30MHz, and in part over clocking the bus because 15MHz was too slow. * Fixing that bug causes the mmc driver to request a 25MHz clock (as it * should) and the logic in at91_mci_update_ios() picks the highest speed that * doesn't exceed that limit. With a 60MHz MCK that would be 15MHz, and * that's a real performance buzzkill when you've been getting away with 30MHz * all along. * * By defining AT91_MCI_ALLOW_OVERCLOCK (or setting the allow_overclock=1 * device hint or sysctl) you can enable logic in at91_mci_update_ios() to * overlcock the SD bus a little by running it at MCK / 2 when the requested * speed is 25MHz and the next highest speed is 15MHz or less. This appears * to work on virtually all SD cards, since it is what this driver has been * doing prior to the introduction of this option, where the overclocking vs - * underclocking decision was automaticly "overclock". Modern SD cards can + * underclocking decision was automatically "overclock". Modern SD cards can * run at 45mhz/1-bit in standard mode (high speed mode enable commands not * sent) without problems. * * Speaking of high-speed mode, the rm9200 manual says the MCI device supports * the SD v1.0 specification and can run up to 50MHz. This is interesting in * that the SD v1.0 spec caps the speed at 25MHz; high speed mode was added in * the v1.10 spec. Furthermore, high speed mode doesn't just crank up the * clock, it alters the signal timing. The rm9200 MCI device doesn't support * these altered timings. So while speeds over 25MHz may work, they only work * in what the SD spec calls "default" speed mode, and it amounts to violating * the spec by overclocking the bus. * * If you also enable 4-wire mode it's possible transfers faster than 25MHz * will fail. On the AT91RM9200, due to bugs in the bus contention logic, if * you have the USB host device and OHCI driver enabled will fail. Even * underclocking to 15MHz, intermittant overrun and underrun errors occur. * Note that you don't even need to have usb devices attached to the system, * the errors begin to occur as soon as the OHCI driver sets the register bit * to enable periodic transfers. It appears (based on brief investigation) * that the usb host controller uses so much ASB bandwidth that sometimes the * DMA for MCI transfers doesn't get a bus grant in time and data gets * dropped. Adding even a modicum of network activity changes the symptom * from intermittant to very frequent. Members of the AT91SAM9 family have * corrected this problem, or are at least better about their use of the bus. */ #ifndef AT91_MCI_ALLOW_OVERCLOCK #define AT91_MCI_ALLOW_OVERCLOCK 1 #endif /* * Allocate 2 bounce buffers we'll use to endian-swap the data due to the rm9200 * erratum. We use a pair of buffers because when reading that lets us begin * endian-swapping the data in the first buffer while the DMA is reading into * the second buffer. (We can't use the same trick for writing because we might * not get all the data in the 2nd buffer swapped before the hardware needs it; * dealing with that would add complexity to the driver.) * * The buffers are sized at 16K each due to the way the busdma cache sync * operations work on arm. A dcache_inv_range() operation on a range larger * than 16K gets turned into a dcache_wbinv_all(). That needlessly flushes the * entire data cache, impacting overall system performance. */ #define BBCOUNT 2 #define BBSIZE (16*1024) #define MAX_BLOCKS ((BBSIZE*BBCOUNT)/512) static int mci_debug; struct at91_mci_softc { void *intrhand; /* Interrupt handle */ device_t dev; int sc_cap; #define CAP_HAS_4WIRE 1 /* Has 4 wire bus */ #define CAP_NEEDS_BYTESWAP 2 /* broken hardware needing bounce */ #define CAP_MCI1_REV2XX 4 /* MCI 1 rev 2.x */ int flags; #define PENDING_CMD 0x01 #define PENDING_STOP 0x02 #define CMD_MULTIREAD 0x10 #define CMD_MULTIWRITE 0x20 int has_4wire; int allow_overclock; struct resource *irq_res; /* IRQ resource */ struct resource *mem_res; /* Memory resource */ struct mtx sc_mtx; bus_dma_tag_t dmatag; struct mmc_host host; int bus_busy; struct mmc_request *req; struct mmc_command *curcmd; bus_dmamap_t bbuf_map[BBCOUNT]; char * bbuf_vaddr[BBCOUNT]; /* bounce bufs in KVA space */ uint32_t bbuf_len[BBCOUNT]; /* len currently queued for bounce buf */ uint32_t bbuf_curidx; /* which bbuf is the active DMA buffer */ uint32_t xfer_offset; /* offset so far into caller's buf */ }; /* bus entry points */ static int at91_mci_probe(device_t dev); static int at91_mci_attach(device_t dev); static int at91_mci_detach(device_t dev); static void at91_mci_intr(void *); /* helper routines */ static int at91_mci_activate(device_t dev); static void at91_mci_deactivate(device_t dev); static int at91_mci_is_mci1rev2xx(void); #define AT91_MCI_LOCK(_sc) mtx_lock(&(_sc)->sc_mtx) #define AT91_MCI_UNLOCK(_sc) mtx_unlock(&(_sc)->sc_mtx) #define AT91_MCI_LOCK_INIT(_sc) \ mtx_init(&_sc->sc_mtx, device_get_nameunit(_sc->dev), \ "mci", MTX_DEF) #define AT91_MCI_LOCK_DESTROY(_sc) mtx_destroy(&_sc->sc_mtx); #define AT91_MCI_ASSERT_LOCKED(_sc) mtx_assert(&_sc->sc_mtx, MA_OWNED); #define AT91_MCI_ASSERT_UNLOCKED(_sc) mtx_assert(&_sc->sc_mtx, MA_NOTOWNED); static inline uint32_t RD4(struct at91_mci_softc *sc, bus_size_t off) { return (bus_read_4(sc->mem_res, off)); } static inline void WR4(struct at91_mci_softc *sc, bus_size_t off, uint32_t val) { bus_write_4(sc->mem_res, off, val); } static void at91_bswap_buf(struct at91_mci_softc *sc, void * dptr, void * sptr, uint32_t memsize) { uint32_t * dst = (uint32_t *)dptr; uint32_t * src = (uint32_t *)sptr; uint32_t i; /* * If the hardware doesn't need byte-swapping, let bcopy() do the * work. Use bounce buffer even if we don't need byteswap, since - * buffer may straddle a page boundry, and we don't handle + * buffer may straddle a page boundary, and we don't handle * multi-segment transfers in hardware. Seen from 'bsdlabel -w' which * uses raw geom access to the volume. Greg Ansley (gja (at) * ansley.com) */ if (!(sc->sc_cap & CAP_NEEDS_BYTESWAP)) { memcpy(dptr, sptr, memsize); return; } /* * Nice performance boost for slightly unrolling this loop. * (But very little extra boost for further unrolling it.) */ for (i = 0; i < memsize; i += 16) { *dst++ = bswap32(*src++); *dst++ = bswap32(*src++); *dst++ = bswap32(*src++); *dst++ = bswap32(*src++); } /* Mop up the last 1-3 words, if any. */ for (i = 0; i < (memsize & 0x0F); i += 4) { *dst++ = bswap32(*src++); } } static void at91_mci_getaddr(void *arg, bus_dma_segment_t *segs, int nsegs, int error) { if (error != 0) return; *(bus_addr_t *)arg = segs[0].ds_addr; } static void at91_mci_pdc_disable(struct at91_mci_softc *sc) { WR4(sc, PDC_PTCR, PDC_PTCR_TXTDIS | PDC_PTCR_RXTDIS); WR4(sc, PDC_RPR, 0); WR4(sc, PDC_RCR, 0); WR4(sc, PDC_RNPR, 0); WR4(sc, PDC_RNCR, 0); WR4(sc, PDC_TPR, 0); WR4(sc, PDC_TCR, 0); WR4(sc, PDC_TNPR, 0); WR4(sc, PDC_TNCR, 0); } /* * Reset the controller, then restore most of the current state. * * This is called after detecting an error. It's also called after stopping a * multi-block write, to un-wedge the device so that it will handle the NOTBUSY * signal correctly. See comments in at91_mci_stop_done() for more details. */ static void at91_mci_reset(struct at91_mci_softc *sc) { uint32_t mr; uint32_t sdcr; uint32_t dtor; uint32_t imr; at91_mci_pdc_disable(sc); /* save current state */ imr = RD4(sc, MCI_IMR); mr = RD4(sc, MCI_MR) & 0x7fff; sdcr = RD4(sc, MCI_SDCR); dtor = RD4(sc, MCI_DTOR); /* reset the controller */ WR4(sc, MCI_IDR, 0xffffffff); WR4(sc, MCI_CR, MCI_CR_MCIDIS | MCI_CR_SWRST); /* restore state */ WR4(sc, MCI_CR, MCI_CR_MCIEN|MCI_CR_PWSEN); WR4(sc, MCI_MR, mr); WR4(sc, MCI_SDCR, sdcr); WR4(sc, MCI_DTOR, dtor); WR4(sc, MCI_IER, imr); /* * Make sure sdio interrupts will fire. Not sure why reading * SR ensures that, but this is in the linux driver. */ RD4(sc, MCI_SR); } static void at91_mci_init(device_t dev) { struct at91_mci_softc *sc = device_get_softc(dev); uint32_t val; WR4(sc, MCI_CR, MCI_CR_MCIDIS | MCI_CR_SWRST); /* device into reset */ WR4(sc, MCI_IDR, 0xffffffff); /* Turn off interrupts */ WR4(sc, MCI_DTOR, MCI_DTOR_DTOMUL_1M | 1); val = MCI_MR_PDCMODE; val |= 0x34a; /* PWSDIV = 3; CLKDIV = 74 */ // if (sc->sc_cap & CAP_MCI1_REV2XX) // val |= MCI_MR_RDPROOF | MCI_MR_WRPROOF; WR4(sc, MCI_MR, val); #ifndef AT91_MCI_SLOT_B WR4(sc, MCI_SDCR, 0); /* SLOT A, 1 bit bus */ #else /* * XXX Really should add second "unit" but nobody using using * a two slot card that we know of. XXX */ WR4(sc, MCI_SDCR, 1); /* SLOT B, 1 bit bus */ #endif /* * Enable controller, including power-save. The slower clock * of the power-save mode is only in effect when there is no * transfer in progress, so it can be left in this mode all * the time. */ WR4(sc, MCI_CR, MCI_CR_MCIEN|MCI_CR_PWSEN); } static void at91_mci_fini(device_t dev) { struct at91_mci_softc *sc = device_get_softc(dev); WR4(sc, MCI_IDR, 0xffffffff); /* Turn off interrupts */ at91_mci_pdc_disable(sc); WR4(sc, MCI_CR, MCI_CR_MCIDIS | MCI_CR_SWRST); /* device into reset */ } static int at91_mci_probe(device_t dev) { #ifdef FDT if (!ofw_bus_is_compatible(dev, "atmel,hsmci")) return (ENXIO); #endif device_set_desc(dev, "MCI mmc/sd host bridge"); return (0); } static int at91_mci_attach(device_t dev) { struct at91_mci_softc *sc = device_get_softc(dev); struct sysctl_ctx_list *sctx; struct sysctl_oid *soid; device_t child; int err, i; sctx = device_get_sysctl_ctx(dev); soid = device_get_sysctl_tree(dev); sc->dev = dev; sc->sc_cap = 0; if (at91_is_rm92()) sc->sc_cap |= CAP_NEEDS_BYTESWAP; /* * MCI1 Rev 2 controllers need some workarounds, flag if so. */ if (at91_mci_is_mci1rev2xx()) sc->sc_cap |= CAP_MCI1_REV2XX; err = at91_mci_activate(dev); if (err) goto out; AT91_MCI_LOCK_INIT(sc); at91_mci_fini(dev); at91_mci_init(dev); /* * Allocate DMA tags and maps and bounce buffers. * * The parms in the tag_create call cause the dmamem_alloc call to * create each bounce buffer as a single contiguous buffer of BBSIZE * bytes aligned to a 4096 byte boundary. * * Do not use DMA_COHERENT for these buffers because that maps the * memory as non-cachable, which prevents cache line burst fills/writes, * which is something we need since we're trying to overlap the * byte-swapping with the DMA operations. */ err = bus_dma_tag_create(bus_get_dma_tag(dev), 4096, 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, BBSIZE, 1, BBSIZE, 0, NULL, NULL, &sc->dmatag); if (err != 0) goto out; for (i = 0; i < BBCOUNT; ++i) { err = bus_dmamem_alloc(sc->dmatag, (void **)&sc->bbuf_vaddr[i], BUS_DMA_NOWAIT, &sc->bbuf_map[i]); if (err != 0) goto out; } /* * Activate the interrupt */ err = bus_setup_intr(dev, sc->irq_res, INTR_TYPE_MISC | INTR_MPSAFE, NULL, at91_mci_intr, sc, &sc->intrhand); if (err) { AT91_MCI_LOCK_DESTROY(sc); goto out; } /* * Allow 4-wire to be initially set via #define. * Allow a device hint to override that. * Allow a sysctl to override that. */ #if defined(AT91_MCI_HAS_4WIRE) && AT91_MCI_HAS_4WIRE != 0 sc->has_4wire = 1; #endif resource_int_value(device_get_name(dev), device_get_unit(dev), "4wire", &sc->has_4wire); SYSCTL_ADD_UINT(sctx, SYSCTL_CHILDREN(soid), OID_AUTO, "4wire", CTLFLAG_RW, &sc->has_4wire, 0, "has 4 wire SD Card bus"); if (sc->has_4wire) sc->sc_cap |= CAP_HAS_4WIRE; sc->allow_overclock = AT91_MCI_ALLOW_OVERCLOCK; resource_int_value(device_get_name(dev), device_get_unit(dev), "allow_overclock", &sc->allow_overclock); SYSCTL_ADD_UINT(sctx, SYSCTL_CHILDREN(soid), OID_AUTO, "allow_overclock", CTLFLAG_RW, &sc->allow_overclock, 0, "Allow up to 30MHz clock for 25MHz request when next highest speed 15MHz or less."); SYSCTL_ADD_UINT(sctx, SYSCTL_CHILDREN(soid), OID_AUTO, "debug", CTLFLAG_RWTUN, &mci_debug, 0, "enable debug output"); /* * Our real min freq is master_clock/512, but upper driver layers are * going to set the min speed during card discovery, and the right speed * for that is 400kHz, so advertise a safe value just under that. * * For max speed, while the rm9200 manual says the max is 50mhz, it also * says it supports only the SD v1.0 spec, which means the real limit is * 25mhz. On the other hand, historical use has been to slightly violate * the standard by running the bus at 30MHz. For more information on * that, see the comments at the top of this file. */ sc->host.f_min = 375000; sc->host.f_max = at91_master_clock / 2; if (sc->host.f_max > 25000000) sc->host.f_max = 25000000; sc->host.host_ocr = MMC_OCR_320_330 | MMC_OCR_330_340; sc->host.caps = 0; if (sc->sc_cap & CAP_HAS_4WIRE) sc->host.caps |= MMC_CAP_4_BIT_DATA; child = device_add_child(dev, "mmc", 0); device_set_ivars(dev, &sc->host); err = bus_generic_attach(dev); out: if (err) at91_mci_deactivate(dev); return (err); } static int at91_mci_detach(device_t dev) { struct at91_mci_softc *sc = device_get_softc(dev); at91_mci_fini(dev); at91_mci_deactivate(dev); bus_dmamem_free(sc->dmatag, sc->bbuf_vaddr[0], sc->bbuf_map[0]); bus_dmamem_free(sc->dmatag, sc->bbuf_vaddr[1], sc->bbuf_map[1]); bus_dma_tag_destroy(sc->dmatag); return (EBUSY); /* XXX */ } static int at91_mci_activate(device_t dev) { struct at91_mci_softc *sc; int rid; sc = device_get_softc(dev); rid = 0; sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->mem_res == NULL) goto errout; rid = 0; sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE); if (sc->irq_res == NULL) goto errout; return (0); errout: at91_mci_deactivate(dev); return (ENOMEM); } static void at91_mci_deactivate(device_t dev) { struct at91_mci_softc *sc; sc = device_get_softc(dev); if (sc->intrhand) bus_teardown_intr(dev, sc->irq_res, sc->intrhand); sc->intrhand = NULL; bus_generic_detach(sc->dev); if (sc->mem_res) bus_release_resource(dev, SYS_RES_MEMORY, rman_get_rid(sc->mem_res), sc->mem_res); sc->mem_res = NULL; if (sc->irq_res) bus_release_resource(dev, SYS_RES_IRQ, rman_get_rid(sc->irq_res), sc->irq_res); sc->irq_res = NULL; return; } static int at91_mci_is_mci1rev2xx(void) { switch (soc_info.type) { case AT91_T_SAM9260: case AT91_T_SAM9263: case AT91_T_CAP9: case AT91_T_SAM9G10: case AT91_T_SAM9G20: case AT91_T_SAM9RL: return(1); default: return (0); } } static int at91_mci_update_ios(device_t brdev, device_t reqdev) { struct at91_mci_softc *sc; struct mmc_ios *ios; uint32_t clkdiv; uint32_t freq; sc = device_get_softc(brdev); ios = &sc->host.ios; /* * Calculate our closest available clock speed that doesn't exceed the * requested speed. * * When overclocking is allowed, the requested clock is 25MHz, the * computed frequency is 15MHz or smaller and clockdiv is 1, use * clockdiv of 0 to double that. If less than 12.5MHz, double * regardless of the overclocking setting. * * Whatever we come up with, store it back into ios->clock so that the * upper layer drivers can report the actual speed of the bus. */ if (ios->clock == 0) { WR4(sc, MCI_CR, MCI_CR_MCIDIS); clkdiv = 0; } else { WR4(sc, MCI_CR, MCI_CR_MCIEN|MCI_CR_PWSEN); if ((at91_master_clock % (ios->clock * 2)) == 0) clkdiv = ((at91_master_clock / ios->clock) / 2) - 1; else clkdiv = (at91_master_clock / ios->clock) / 2; freq = at91_master_clock / ((clkdiv+1) * 2); if (clkdiv == 1 && ios->clock == 25000000 && freq <= 15000000) { if (sc->allow_overclock || freq <= 12500000) { clkdiv = 0; freq = at91_master_clock / ((clkdiv+1) * 2); } } ios->clock = freq; } if (ios->bus_width == bus_width_4) WR4(sc, MCI_SDCR, RD4(sc, MCI_SDCR) | MCI_SDCR_SDCBUS); else WR4(sc, MCI_SDCR, RD4(sc, MCI_SDCR) & ~MCI_SDCR_SDCBUS); WR4(sc, MCI_MR, (RD4(sc, MCI_MR) & ~MCI_MR_CLKDIV) | clkdiv); /* Do we need a settle time here? */ /* XXX We need to turn the device on/off here with a GPIO pin */ return (0); } static void at91_mci_start_cmd(struct at91_mci_softc *sc, struct mmc_command *cmd) { uint32_t cmdr, mr; struct mmc_data *data; sc->curcmd = cmd; data = cmd->data; /* XXX Upper layers don't always set this */ cmd->mrq = sc->req; /* Begin setting up command register. */ cmdr = cmd->opcode; if (sc->host.ios.bus_mode == opendrain) cmdr |= MCI_CMDR_OPDCMD; /* Set up response handling. Allow max timeout for responses. */ if (MMC_RSP(cmd->flags) == MMC_RSP_NONE) cmdr |= MCI_CMDR_RSPTYP_NO; else { cmdr |= MCI_CMDR_MAXLAT; if (cmd->flags & MMC_RSP_136) cmdr |= MCI_CMDR_RSPTYP_136; else cmdr |= MCI_CMDR_RSPTYP_48; } /* * If there is no data transfer, just set up the right interrupt mask * and start the command. * * The interrupt mask needs to be CMDRDY plus all non-data-transfer * errors. It's important to leave the transfer-related errors out, to * avoid spurious timeout or crc errors on a STOP command following a * multiblock read. When a multiblock read is in progress, sending a * STOP in the middle of a block occasionally triggers such errors, but * we're totally disinterested in them because we've already gotten all * the data we wanted without error before sending the STOP command. */ if (data == NULL) { uint32_t ier = MCI_SR_CMDRDY | MCI_SR_RTOE | MCI_SR_RENDE | MCI_SR_RCRCE | MCI_SR_RDIRE | MCI_SR_RINDE; at91_mci_pdc_disable(sc); if (cmd->opcode == MMC_STOP_TRANSMISSION) cmdr |= MCI_CMDR_TRCMD_STOP; /* Ignore response CRC on CMD2 and ACMD41, per standard. */ if (cmd->opcode == MMC_SEND_OP_COND || cmd->opcode == ACMD_SD_SEND_OP_COND) ier &= ~MCI_SR_RCRCE; if (mci_debug) printf("CMDR %x (opcode %d) ARGR %x no data\n", cmdr, cmd->opcode, cmd->arg); WR4(sc, MCI_ARGR, cmd->arg); WR4(sc, MCI_CMDR, cmdr); WR4(sc, MCI_IDR, 0xffffffff); WR4(sc, MCI_IER, ier); return; } /* There is data, set up the transfer-related parts of the command. */ if (data->flags & MMC_DATA_READ) cmdr |= MCI_CMDR_TRDIR; if (data->flags & (MMC_DATA_READ | MMC_DATA_WRITE)) cmdr |= MCI_CMDR_TRCMD_START; if (data->flags & MMC_DATA_STREAM) cmdr |= MCI_CMDR_TRTYP_STREAM; else if (data->flags & MMC_DATA_MULTI) { cmdr |= MCI_CMDR_TRTYP_MULTIPLE; sc->flags |= (data->flags & MMC_DATA_READ) ? CMD_MULTIREAD : CMD_MULTIWRITE; } /* * Disable PDC until we're ready. * * Set block size and turn on PDC mode for dma xfer. * Note that the block size is the smaller of the amount of data to be * transferred, or 512 bytes. The 512 size is fixed by the standard; * smaller blocks are possible, but never larger. */ WR4(sc, PDC_PTCR, PDC_PTCR_RXTDIS | PDC_PTCR_TXTDIS); mr = RD4(sc,MCI_MR) & ~MCI_MR_BLKLEN; mr |= min(data->len, 512) << 16; WR4(sc, MCI_MR, mr | MCI_MR_PDCMODE|MCI_MR_PDCPADV); /* * Set up DMA. * * Use bounce buffers even if we don't need to byteswap, because doing * multi-block IO with large DMA buffers is way fast (compared to * single-block IO), even after incurring the overhead of also copying * from/to the caller's buffers (which may be in non-contiguous physical * pages). * * In an ideal non-byteswap world we could create a dma tag that allows * for discontiguous segments and do the IO directly from/to the * caller's buffer(s), using ENDRX/ENDTX interrupts to chain the * discontiguous buffers through the PDC. Someday. * * If a read is bigger than 2k, split it in half so that we can start * byte-swapping the first half while the second half is on the wire. * It would be best if we could split it into 8k chunks, but we can't * always keep up with the byte-swapping due to other system activity, * and if an RXBUFF interrupt happens while we're still handling the * byte-swap from the prior buffer (IE, we haven't returned from * handling the prior interrupt yet), then data will get dropped on the * floor and we can't easily recover from that. The right fix for that * would be to have the interrupt handling only keep the DMA flowing and * enqueue filled buffers to be byte-swapped in a non-interrupt context. * Even that won't work on the write side of things though; in that * context we have to have all the data ready to go before starting the * dma. * * XXX what about stream transfers? */ sc->xfer_offset = 0; sc->bbuf_curidx = 0; if (data->flags & (MMC_DATA_READ | MMC_DATA_WRITE)) { uint32_t len; uint32_t remaining = data->len; bus_addr_t paddr; int err; if (remaining > (BBCOUNT*BBSIZE)) panic("IO read size exceeds MAXDATA\n"); if (data->flags & MMC_DATA_READ) { if (remaining > 2048) // XXX len = remaining / 2; else len = remaining; err = bus_dmamap_load(sc->dmatag, sc->bbuf_map[0], sc->bbuf_vaddr[0], len, at91_mci_getaddr, &paddr, BUS_DMA_NOWAIT); if (err != 0) panic("IO read dmamap_load failed\n"); bus_dmamap_sync(sc->dmatag, sc->bbuf_map[0], BUS_DMASYNC_PREREAD); WR4(sc, PDC_RPR, paddr); WR4(sc, PDC_RCR, len / 4); sc->bbuf_len[0] = len; remaining -= len; if (remaining == 0) { sc->bbuf_len[1] = 0; } else { len = remaining; err = bus_dmamap_load(sc->dmatag, sc->bbuf_map[1], sc->bbuf_vaddr[1], len, at91_mci_getaddr, &paddr, BUS_DMA_NOWAIT); if (err != 0) panic("IO read dmamap_load failed\n"); bus_dmamap_sync(sc->dmatag, sc->bbuf_map[1], BUS_DMASYNC_PREREAD); WR4(sc, PDC_RNPR, paddr); WR4(sc, PDC_RNCR, len / 4); sc->bbuf_len[1] = len; remaining -= len; } WR4(sc, PDC_PTCR, PDC_PTCR_RXTEN); } else { len = min(BBSIZE, remaining); at91_bswap_buf(sc, sc->bbuf_vaddr[0], data->data, len); err = bus_dmamap_load(sc->dmatag, sc->bbuf_map[0], sc->bbuf_vaddr[0], len, at91_mci_getaddr, &paddr, BUS_DMA_NOWAIT); if (err != 0) panic("IO write dmamap_load failed\n"); bus_dmamap_sync(sc->dmatag, sc->bbuf_map[0], BUS_DMASYNC_PREWRITE); /* * Erratum workaround: PDC transfer length on a write * must not be smaller than 12 bytes (3 words); only * blklen bytes (set above) are actually transferred. */ WR4(sc, PDC_TPR,paddr); WR4(sc, PDC_TCR, (len < 12) ? 3 : len / 4); sc->bbuf_len[0] = len; remaining -= len; if (remaining == 0) { sc->bbuf_len[1] = 0; } else { len = remaining; at91_bswap_buf(sc, sc->bbuf_vaddr[1], ((char *)data->data)+BBSIZE, len); err = bus_dmamap_load(sc->dmatag, sc->bbuf_map[1], sc->bbuf_vaddr[1], len, at91_mci_getaddr, &paddr, BUS_DMA_NOWAIT); if (err != 0) panic("IO write dmamap_load failed\n"); bus_dmamap_sync(sc->dmatag, sc->bbuf_map[1], BUS_DMASYNC_PREWRITE); WR4(sc, PDC_TNPR, paddr); WR4(sc, PDC_TNCR, (len < 12) ? 3 : len / 4); sc->bbuf_len[1] = len; remaining -= len; } /* do not enable PDC xfer until CMDRDY asserted */ } data->xfer_len = 0; /* XXX what's this? appears to be unused. */ } if (mci_debug) printf("CMDR %x (opcode %d) ARGR %x with data len %d\n", cmdr, cmd->opcode, cmd->arg, cmd->data->len); WR4(sc, MCI_ARGR, cmd->arg); WR4(sc, MCI_CMDR, cmdr); WR4(sc, MCI_IER, MCI_SR_ERROR | MCI_SR_CMDRDY); } static void at91_mci_next_operation(struct at91_mci_softc *sc) { struct mmc_request *req; req = sc->req; if (req == NULL) return; if (sc->flags & PENDING_CMD) { sc->flags &= ~PENDING_CMD; at91_mci_start_cmd(sc, req->cmd); return; } else if (sc->flags & PENDING_STOP) { sc->flags &= ~PENDING_STOP; at91_mci_start_cmd(sc, req->stop); return; } WR4(sc, MCI_IDR, 0xffffffff); sc->req = NULL; sc->curcmd = NULL; //printf("req done\n"); req->done(req); } static int at91_mci_request(device_t brdev, device_t reqdev, struct mmc_request *req) { struct at91_mci_softc *sc = device_get_softc(brdev); AT91_MCI_LOCK(sc); if (sc->req != NULL) { AT91_MCI_UNLOCK(sc); return (EBUSY); } //printf("new req\n"); sc->req = req; sc->flags = PENDING_CMD; if (sc->req->stop) sc->flags |= PENDING_STOP; at91_mci_next_operation(sc); AT91_MCI_UNLOCK(sc); return (0); } static int at91_mci_get_ro(device_t brdev, device_t reqdev) { return (0); } static int at91_mci_acquire_host(device_t brdev, device_t reqdev) { struct at91_mci_softc *sc = device_get_softc(brdev); int err = 0; AT91_MCI_LOCK(sc); while (sc->bus_busy) msleep(sc, &sc->sc_mtx, PZERO, "mciah", hz / 5); sc->bus_busy++; AT91_MCI_UNLOCK(sc); return (err); } static int at91_mci_release_host(device_t brdev, device_t reqdev) { struct at91_mci_softc *sc = device_get_softc(brdev); AT91_MCI_LOCK(sc); sc->bus_busy--; wakeup(sc); AT91_MCI_UNLOCK(sc); return (0); } static void at91_mci_read_done(struct at91_mci_softc *sc, uint32_t sr) { struct mmc_command *cmd = sc->curcmd; char * dataptr = (char *)cmd->data->data; uint32_t curidx = sc->bbuf_curidx; uint32_t len = sc->bbuf_len[curidx]; /* * We arrive here when a DMA transfer for a read is done, whether it's * a single or multi-block read. * * We byte-swap the buffer that just completed, and if that is the * last buffer that's part of this read then we move on to the next * operation, otherwise we wait for another ENDRX for the next bufer. */ bus_dmamap_sync(sc->dmatag, sc->bbuf_map[curidx], BUS_DMASYNC_POSTREAD); bus_dmamap_unload(sc->dmatag, sc->bbuf_map[curidx]); at91_bswap_buf(sc, dataptr + sc->xfer_offset, sc->bbuf_vaddr[curidx], len); if (mci_debug) { printf("read done sr %x curidx %d len %d xfer_offset %d\n", sr, curidx, len, sc->xfer_offset); } sc->xfer_offset += len; sc->bbuf_curidx = !curidx; /* swap buffers */ /* * If we've transferred all the data, move on to the next operation. * * If we're still transferring the last buffer, RNCR is already zero but * we have to write a zero anyway to clear the ENDRX status so we don't * re-interrupt until the last buffer is done. */ if (sc->xfer_offset == cmd->data->len) { WR4(sc, PDC_PTCR, PDC_PTCR_RXTDIS | PDC_PTCR_TXTDIS); cmd->error = MMC_ERR_NONE; at91_mci_next_operation(sc); } else { WR4(sc, PDC_RNCR, 0); WR4(sc, MCI_IER, MCI_SR_ERROR | MCI_SR_ENDRX); } } static void at91_mci_write_done(struct at91_mci_softc *sc, uint32_t sr) { struct mmc_command *cmd = sc->curcmd; /* * We arrive here when the entire DMA transfer for a write is done, * whether it's a single or multi-block write. If it's multi-block we * have to immediately move on to the next operation which is to send * the stop command. If it's a single-block transfer we need to wait * for NOTBUSY, but if that's already asserted we can avoid another * interrupt and just move on to completing the request right away. */ WR4(sc, PDC_PTCR, PDC_PTCR_RXTDIS | PDC_PTCR_TXTDIS); bus_dmamap_sync(sc->dmatag, sc->bbuf_map[sc->bbuf_curidx], BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(sc->dmatag, sc->bbuf_map[sc->bbuf_curidx]); if ((cmd->data->flags & MMC_DATA_MULTI) || (sr & MCI_SR_NOTBUSY)) { cmd->error = MMC_ERR_NONE; at91_mci_next_operation(sc); } else { WR4(sc, MCI_IER, MCI_SR_ERROR | MCI_SR_NOTBUSY); } } static void at91_mci_notbusy(struct at91_mci_softc *sc) { struct mmc_command *cmd = sc->curcmd; /* * We arrive here by either completion of a single-block write, or * completion of the stop command that ended a multi-block write (and, * I suppose, after a card-select or erase, but I haven't tested * those). Anyway, we're done and it's time to move on to the next * command. */ cmd->error = MMC_ERR_NONE; at91_mci_next_operation(sc); } static void at91_mci_stop_done(struct at91_mci_softc *sc, uint32_t sr) { struct mmc_command *cmd = sc->curcmd; /* * We arrive here after receiving CMDRDY for a MMC_STOP_TRANSMISSION * command. Depending on the operation being stopped, we may have to * do some unusual things to work around hardware bugs. */ /* * This is known to be true of at91rm9200 hardware; it may or may not * apply to more recent chips: * * After stopping a multi-block write, the NOTBUSY bit in MCI_SR does * not properly reflect the actual busy state of the card as signaled * on the DAT0 line; it always claims the card is not-busy. If we * believe that and let operations continue, following commands will * fail with response timeouts (except of course MMC_SEND_STATUS -- it * indicates the card is busy in the PRG state, which was the smoking * gun that showed MCI_SR NOTBUSY was not tracking DAT0 correctly). * * The atmel docs are emphatic: "This flag [NOTBUSY] must be used only * for Write Operations." I guess technically since we sent a stop * it's not a write operation anymore. But then just what did they * think it meant for the stop command to have "...an optional busy * signal transmitted on the data line" according to the SD spec? * * I tried a variety of things to un-wedge the MCI and get the status * register to reflect NOTBUSY correctly again, but the only thing * that worked was a full device reset. It feels like an awfully big * hammer, but doing a full reset after every multiblock write is * still faster than doing single-block IO (by almost two orders of * magnitude: 20KB/sec improves to about 1.8MB/sec best case). * * After doing the reset, wait for a NOTBUSY interrupt before * continuing with the next operation. * * This workaround breaks multiwrite on the rev2xx parts, but some other * workaround is needed. */ if ((sc->flags & CMD_MULTIWRITE) && (sc->sc_cap & CAP_NEEDS_BYTESWAP)) { at91_mci_reset(sc); WR4(sc, MCI_IER, MCI_SR_ERROR | MCI_SR_NOTBUSY); return; } /* * This is known to be true of at91rm9200 hardware; it may or may not * apply to more recent chips: * * After stopping a multi-block read, loop to read and discard any * data that coasts in after we sent the stop command. The docs don't * say anything about it, but empirical testing shows that 1-3 * additional words of data get buffered up in some unmentioned * internal fifo and if we don't read and discard them here they end * up on the front of the next read DMA transfer we do. * * This appears to be unnecessary for rev2xx parts. */ if ((sc->flags & CMD_MULTIREAD) && (sc->sc_cap & CAP_NEEDS_BYTESWAP)) { uint32_t sr; int count = 0; do { sr = RD4(sc, MCI_SR); if (sr & MCI_SR_RXRDY) { RD4(sc, MCI_RDR); ++count; } } while (sr & MCI_SR_RXRDY); at91_mci_reset(sc); } cmd->error = MMC_ERR_NONE; at91_mci_next_operation(sc); } static void at91_mci_cmdrdy(struct at91_mci_softc *sc, uint32_t sr) { struct mmc_command *cmd = sc->curcmd; int i; if (cmd == NULL) return; /* * We get here at the end of EVERY command. We retrieve the command * response (if any) then decide what to do next based on the command. */ if (cmd->flags & MMC_RSP_PRESENT) { for (i = 0; i < ((cmd->flags & MMC_RSP_136) ? 4 : 1); i++) { cmd->resp[i] = RD4(sc, MCI_RSPR + i * 4); if (mci_debug) printf("RSPR[%d] = %x sr=%x\n", i, cmd->resp[i], sr); } } /* * If this was a stop command, go handle the various special * conditions (read: bugs) that have to be dealt with following a stop. */ if (cmd->opcode == MMC_STOP_TRANSMISSION) { at91_mci_stop_done(sc, sr); return; } /* * If this command can continue to assert BUSY beyond the response then * we need to wait for NOTBUSY before the command is really done. * * Note that this may not work properly on the at91rm9200. It certainly * doesn't work for the STOP command that follows a multi-block write, * so post-stop CMDRDY is handled separately; see the special handling * in at91_mci_stop_done(). * * Beside STOP, there are other R1B-type commands that use the busy * signal after CMDRDY: CMD7 (card select), CMD28-29 (write protect), * CMD38 (erase). I haven't tested any of them, but I rather expect * them all to have the same sort of problem with MCI_SR not actually * reflecting the state of the DAT0-line busy indicator. So this code * may need to grow some sort of special handling for them too. (This * just in: CMD7 isn't a problem right now because dev/mmc.c incorrectly * sets the response flags to R1 rather than R1B.) XXX */ if ((cmd->flags & MMC_RSP_BUSY)) { WR4(sc, MCI_IER, MCI_SR_ERROR | MCI_SR_NOTBUSY); return; } /* * If there is a data transfer with this command, then... * - If it's a read, we need to wait for ENDRX. * - If it's a write, now is the time to enable the PDC, and we need * to wait for a BLKE that follows a TXBUFE, because if we're doing * a split transfer we get a BLKE after the first half (when TPR/TCR * get loaded from TNPR/TNCR). So first we wait for the TXBUFE, and * the handling for that interrupt will then invoke the wait for the * subsequent BLKE which indicates actual completion. */ if (cmd->data) { uint32_t ier; if (cmd->data->flags & MMC_DATA_READ) { ier = MCI_SR_ENDRX; } else { ier = MCI_SR_TXBUFE; WR4(sc, PDC_PTCR, PDC_PTCR_TXTEN); } WR4(sc, MCI_IER, MCI_SR_ERROR | ier); return; } /* * If we made it to here, we don't need to wait for anything more for * the current command, move on to the next command (will complete the * request if there is no next command). */ cmd->error = MMC_ERR_NONE; at91_mci_next_operation(sc); } static void at91_mci_intr(void *arg) { struct at91_mci_softc *sc = (struct at91_mci_softc*)arg; struct mmc_command *cmd = sc->curcmd; uint32_t sr, isr; AT91_MCI_LOCK(sc); sr = RD4(sc, MCI_SR); isr = sr & RD4(sc, MCI_IMR); if (mci_debug) printf("i 0x%x sr 0x%x\n", isr, sr); /* * All interrupts are one-shot; disable it now. * The next operation will re-enable whatever interrupts it wants. */ WR4(sc, MCI_IDR, isr); if (isr & MCI_SR_ERROR) { if (isr & (MCI_SR_RTOE | MCI_SR_DTOE)) cmd->error = MMC_ERR_TIMEOUT; else if (isr & (MCI_SR_RCRCE | MCI_SR_DCRCE)) cmd->error = MMC_ERR_BADCRC; else if (isr & (MCI_SR_OVRE | MCI_SR_UNRE)) cmd->error = MMC_ERR_FIFO; else cmd->error = MMC_ERR_FAILED; /* * CMD8 is used to probe for SDHC cards, a standard SD card * will get a response timeout; don't report it because it's a * normal and expected condition. One might argue that all * error reporting should be left to higher levels, but when * they report at all it's always EIO, which isn't very * helpful. XXX bootverbose? */ if (cmd->opcode != 8) { device_printf(sc->dev, "IO error; status MCI_SR = 0x%b cmd opcode = %d%s\n", sr, MCI_SR_BITSTRING, cmd->opcode, (cmd->opcode != 12) ? "" : (sc->flags & CMD_MULTIREAD) ? " after read" : " after write"); /* XXX not sure RTOE needs a full reset, just a retry */ at91_mci_reset(sc); } at91_mci_next_operation(sc); } else { if (isr & MCI_SR_TXBUFE) { // printf("TXBUFE\n"); /* * We need to wait for a BLKE that follows TXBUFE * (intermediate BLKEs might happen after ENDTXes if * we're chaining multiple buffers). If BLKE is also * asserted at the time we get TXBUFE, we can avoid * another interrupt and process it right away, below. */ if (sr & MCI_SR_BLKE) isr |= MCI_SR_BLKE; else WR4(sc, MCI_IER, MCI_SR_BLKE); } if (isr & MCI_SR_RXBUFF) { // printf("RXBUFF\n"); } if (isr & MCI_SR_ENDTX) { // printf("ENDTX\n"); } if (isr & MCI_SR_ENDRX) { // printf("ENDRX\n"); at91_mci_read_done(sc, sr); } if (isr & MCI_SR_NOTBUSY) { // printf("NOTBUSY\n"); at91_mci_notbusy(sc); } if (isr & MCI_SR_DTIP) { // printf("Data transfer in progress\n"); } if (isr & MCI_SR_BLKE) { // printf("Block transfer end\n"); at91_mci_write_done(sc, sr); } if (isr & MCI_SR_TXRDY) { // printf("Ready to transmit\n"); } if (isr & MCI_SR_RXRDY) { // printf("Ready to receive\n"); } if (isr & MCI_SR_CMDRDY) { // printf("Command ready\n"); at91_mci_cmdrdy(sc, sr); } } AT91_MCI_UNLOCK(sc); } static int at91_mci_read_ivar(device_t bus, device_t child, int which, uintptr_t *result) { struct at91_mci_softc *sc = device_get_softc(bus); switch (which) { default: return (EINVAL); case MMCBR_IVAR_BUS_MODE: *(int *)result = sc->host.ios.bus_mode; break; case MMCBR_IVAR_BUS_WIDTH: *(int *)result = sc->host.ios.bus_width; break; case MMCBR_IVAR_CHIP_SELECT: *(int *)result = sc->host.ios.chip_select; break; case MMCBR_IVAR_CLOCK: *(int *)result = sc->host.ios.clock; break; case MMCBR_IVAR_F_MIN: *(int *)result = sc->host.f_min; break; case MMCBR_IVAR_F_MAX: *(int *)result = sc->host.f_max; break; case MMCBR_IVAR_HOST_OCR: *(int *)result = sc->host.host_ocr; break; case MMCBR_IVAR_MODE: *(int *)result = sc->host.mode; break; case MMCBR_IVAR_OCR: *(int *)result = sc->host.ocr; break; case MMCBR_IVAR_POWER_MODE: *(int *)result = sc->host.ios.power_mode; break; case MMCBR_IVAR_VDD: *(int *)result = sc->host.ios.vdd; break; case MMCBR_IVAR_CAPS: if (sc->has_4wire) { sc->sc_cap |= CAP_HAS_4WIRE; sc->host.caps |= MMC_CAP_4_BIT_DATA; } else { sc->sc_cap &= ~CAP_HAS_4WIRE; sc->host.caps &= ~MMC_CAP_4_BIT_DATA; } *(int *)result = sc->host.caps; break; case MMCBR_IVAR_MAX_DATA: /* * Something is wrong with the 2x parts and multiblock, so * just do 1 block at a time for now, which really kills * performance. */ if (sc->sc_cap & CAP_MCI1_REV2XX) *(int *)result = 1; else *(int *)result = MAX_BLOCKS; break; } return (0); } static int at91_mci_write_ivar(device_t bus, device_t child, int which, uintptr_t value) { struct at91_mci_softc *sc = device_get_softc(bus); switch (which) { default: return (EINVAL); case MMCBR_IVAR_BUS_MODE: sc->host.ios.bus_mode = value; break; case MMCBR_IVAR_BUS_WIDTH: sc->host.ios.bus_width = value; break; case MMCBR_IVAR_CHIP_SELECT: sc->host.ios.chip_select = value; break; case MMCBR_IVAR_CLOCK: sc->host.ios.clock = value; break; case MMCBR_IVAR_MODE: sc->host.mode = value; break; case MMCBR_IVAR_OCR: sc->host.ocr = value; break; case MMCBR_IVAR_POWER_MODE: sc->host.ios.power_mode = value; break; case MMCBR_IVAR_VDD: sc->host.ios.vdd = value; break; /* These are read-only */ case MMCBR_IVAR_CAPS: case MMCBR_IVAR_HOST_OCR: case MMCBR_IVAR_F_MIN: case MMCBR_IVAR_F_MAX: case MMCBR_IVAR_MAX_DATA: return (EINVAL); } return (0); } static device_method_t at91_mci_methods[] = { /* device_if */ DEVMETHOD(device_probe, at91_mci_probe), DEVMETHOD(device_attach, at91_mci_attach), DEVMETHOD(device_detach, at91_mci_detach), /* Bus interface */ DEVMETHOD(bus_read_ivar, at91_mci_read_ivar), DEVMETHOD(bus_write_ivar, at91_mci_write_ivar), /* mmcbr_if */ DEVMETHOD(mmcbr_update_ios, at91_mci_update_ios), DEVMETHOD(mmcbr_request, at91_mci_request), DEVMETHOD(mmcbr_get_ro, at91_mci_get_ro), DEVMETHOD(mmcbr_acquire_host, at91_mci_acquire_host), DEVMETHOD(mmcbr_release_host, at91_mci_release_host), DEVMETHOD_END }; static driver_t at91_mci_driver = { "at91_mci", at91_mci_methods, sizeof(struct at91_mci_softc), }; static devclass_t at91_mci_devclass; #ifdef FDT DRIVER_MODULE(at91_mci, simplebus, at91_mci_driver, at91_mci_devclass, NULL, NULL); #else DRIVER_MODULE(at91_mci, atmelarm, at91_mci_driver, at91_mci_devclass, NULL, NULL); #endif DRIVER_MODULE(mmc, at91_mci, mmc_driver, mmc_devclass, NULL, NULL); MODULE_DEPEND(at91_mci, mmc, 1, 1, 1); Index: user/ngie/release-pkg-fix-tests/sys/arm/at91/at91_reset.S =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/at91/at91_reset.S (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/at91/at91_reset.S (revision 299091) @@ -1,57 +1,57 @@ #include #include #include #include __FBSDID("$FreeBSD$"); #define SDRAM_TR (AT91_BASE + \ AT91SAM9G20_SDRAMC_BASE + AT91SAM9G20_SDRAMC_TR) #define SDRAM_LPR (AT91_BASE + \ AT91SAM9G20_SDRAMC_BASE + AT91SAM9G20_SDRAMC_LPR) #define RSTC_RCR (AT91_BASE + \ AT91SAM9G20_RSTC_BASE + RST_CR) /* * From AT91SAM9G20 Datasheet errata 44:3.5: * - * When User Reset occurs durring SDRAM read acces, eh SDRAM clock is turned + * When User Reset occurs during SDRAM read access, the SDRAM clock is turned * off while data are ready to be read on the data bus. The SDRAM maintains * the data until the clock restarts. * * If the User reset is programed to assert a general reset, the data * maintained by the SDRAM leads to a data bus conflict and adversly affects * the boot memories connected to the EBI: * + NAND Flash boot functionality, if the system boots out of internal ROM. * + NOR Flash boot, if the system boots on an external memory connected to * the EBI CS0. * * Assembly code is mandatory for the following sequnce as ARM * instructions need to be piplined. * */ ENTRY(cpu_reset_sam9g20) /* Disable IRQs */ mrs r0, cpsr orr r0, r0, #0x80 msr cpsr_c, r0 /* Change Refresh to block all data access */ ldr r0, =SDRAM_TR ldr r1, =1 str r1, [r0] /* Prepare power down command */ ldr r0, =SDRAM_LPR ldr r1, =2 /* Prepare proc_reset and periph reset */ ldr r2, =RSTC_RCR ldr r3, =0xA5000005 /* perform power down command */ str r1, [r0] /* Perfom proc_reset and periph reset (in the ARM pipeline) */ str r3, [r2] Index: user/ngie/release-pkg-fix-tests/sys/arm/at91/at91reg.h =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/at91/at91reg.h (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/at91/at91reg.h (revision 299091) @@ -1,90 +1,90 @@ /*- * Copyright (c) 2009 Greg Ansley All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * $FreeBSD$ */ #ifndef _AT91REG_H_ #define _AT91REG_H_ #include "opt_at91.h" /* Where builtin peripherals start in KVM */ #define AT91_BASE 0xd0000000 /* Where builtin peripherals start PA */ #define AT91_PA_BASE 0xf0000000 /* A few things that we count on being the same - * throught the whole family of SOCs */ + * throughout the whole family of SOCs */ -/* SYSC System Controler */ +/* SYSC System Controller */ /* System Registers */ #define AT91_SYS_BASE 0xffff000 #define AT91_SYS_SIZE 0x1000 #define AT91_DBGU0 0x0ffff200 /* Most */ #define AT91_DBGU1 0x0fffee00 /* SAM9263, CAP9, and SAM9G45 */ #define AT91_DBGU_SIZE 0x200 #define DBGU_C1R (64) /* Chip ID1 Register */ #define DBGU_C2R (68) /* Chip ID2 Register */ #define DBGU_FNTR (72) /* Force NTRST Register */ #define AT91_CPU_VERSION_MASK 0x0000001f #define AT91_CPU_FAMILY_MASK 0x0ff00000 #define AT91_CPU_RM9200 0x09290780 #define AT91_CPU_SAM9260 0x019803a0 #define AT91_CPU_SAM9261 0x019703a0 #define AT91_CPU_SAM9263 0x019607a0 #define AT91_CPU_SAM9G10 0x819903a0 #define AT91_CPU_SAM9G20 0x019905a0 #define AT91_CPU_SAM9G45 0x819b05a0 #define AT91_CPU_SAM9N12 0x819a07a0 #define AT91_CPU_SAM9RL64 0x019b03a0 #define AT91_CPU_SAM9X5 0x819a05a0 #define AT91_CPU_SAM9XE128 0x329973a0 #define AT91_CPU_SAM9XE256 0x329a93a0 #define AT91_CPU_SAM9XE512 0x329aa3a0 #define AT91_CPU_CAP9 0x039a03a0 #define AT91_EXID_SAM9M11 0x00000001 #define AT91_EXID_SAM9M10 0x00000002 #define AT91_EXID_SAM9G46 0x00000003 #define AT91_EXID_SAM9G45 0x00000004 #define AT91_EXID_SAM9G15 0x00000000 #define AT91_EXID_SAM9G35 0x00000001 #define AT91_EXID_SAM9X35 0x00000002 #define AT91_EXID_SAM9G25 0x00000003 #define AT91_EXID_SAM9X25 0x00000004 #define AT91_IRQ_SYSTEM 1 #endif /* _AT91REG_H_ */ Index: user/ngie/release-pkg-fix-tests/sys/arm/at91/at91sam9260.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/at91/at91sam9260.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/at91/at91sam9260.c (revision 299091) @@ -1,218 +1,218 @@ /*- * Copyright (c) 2005 Olivier Houchard. All rights reserved. * Copyright (c) 2010 Greg Ansley. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #define _ARM32_BUS_DMA_PRIVATE #include #include #include #include #include #include #include #include #include #include /* * Standard priority levels for the system. 0 is lowest and 7 is highest. * These values are the ones Atmel uses for its Linux port */ static const int at91_irq_prio[32] = { 7, /* Advanced Interrupt Controller */ 7, /* System Peripherals */ 1, /* Parallel IO Controller A */ 1, /* Parallel IO Controller B */ 1, /* Parallel IO Controller C */ 0, /* Analog-to-Digital Converter */ 5, /* USART 0 */ 5, /* USART 1 */ 5, /* USART 2 */ 0, /* Multimedia Card Interface */ 2, /* USB Device Port */ 6, /* Two-Wire Interface */ 5, /* Serial Peripheral Interface 0 */ 5, /* Serial Peripheral Interface 1 */ 5, /* Serial Synchronous Controller */ 0, /* (reserved) */ 0, /* (reserved) */ 0, /* Timer Counter 0 */ 0, /* Timer Counter 1 */ 0, /* Timer Counter 2 */ 2, /* USB Host port */ 3, /* Ethernet */ 0, /* Image Sensor Interface */ 5, /* USART 3 */ 5, /* USART 4 */ 5, /* USART 5 */ 0, /* Timer Counter 3 */ 0, /* Timer Counter 4 */ 0, /* Timer Counter 5 */ 0, /* Advanced Interrupt Controller IRQ0 */ 0, /* Advanced Interrupt Controller IRQ1 */ 0, /* Advanced Interrupt Controller IRQ2 */ }; static const uint32_t at91_pio_base[] = { AT91SAM9260_PIOA_BASE, AT91SAM9260_PIOB_BASE, AT91SAM9260_PIOC_BASE, }; #define DEVICE(_name, _id, _unit) \ { \ _name, _unit, \ AT91SAM9260_ ## _id ##_BASE, \ AT91SAM9260_ ## _id ## _SIZE, \ AT91SAM9260_IRQ_ ## _id \ } static const struct cpu_devs at91_devs[] = { DEVICE("at91_aic", AIC, 0), DEVICE("at91_pmc", PMC, 0), DEVICE("at91_wdt", WDT, 0), DEVICE("at91_rst", RSTC, 0), DEVICE("at91_pit", PIT, 0), DEVICE("at91_pio", PIOA, 0), DEVICE("at91_pio", PIOB, 1), DEVICE("at91_pio", PIOC, 2), DEVICE("at91_twi", TWI, 0), DEVICE("at91_mci", MCI, 0), DEVICE("uart", DBGU, 0), DEVICE("uart", USART0, 1), DEVICE("uart", USART1, 2), DEVICE("uart", USART2, 3), DEVICE("uart", USART3, 4), DEVICE("uart", USART4, 5), DEVICE("uart", USART5, 6), DEVICE("spi", SPI0, 0), DEVICE("spi", SPI1, 1), DEVICE("ate", EMAC, 0), DEVICE("macb", EMAC, 0), DEVICE("nand", NAND, 0), DEVICE("ohci", OHCI, 0), { 0, 0, 0, 0, 0 } }; /* * The following is unused currently since we don't ever set the PLLA * frequency of the device. */ static uint32_t at91_pll_outa(int freq) { uint32_t outa = 0; /* * Set OUTA, per the data sheet. See Table 40-15 titled * PLLA Characteristics in the SAM9260 doc. */ if (freq > 155000000) outa = 2 << 14; return ((1 << 29) | outa); } static uint32_t at91_pll_outb(int freq) { return (1 << 14); } static void at91_clock_init(void) { struct at91_pmc_clock *clk; /* Update USB device port clock info */ clk = at91_pmc_clock_ref("udpck"); clk->pmc_mask = PMC_SCER_UDP_SAM9; at91_pmc_clock_deref(clk); /* Update USB host port clock info */ clk = at91_pmc_clock_ref("uhpck"); clk->pmc_mask = PMC_SCER_UHP_SAM9; at91_pmc_clock_deref(clk); /* Each SOC has different PLL contraints */ clk = at91_pmc_clock_ref("plla"); clk->pll_min_in = SAM9260_PLL_A_MIN_IN_FREQ; /* 1 MHz */ clk->pll_max_in = SAM9260_PLL_A_MAX_IN_FREQ; /* 32 MHz */ clk->pll_min_out = SAM9260_PLL_A_MIN_OUT_FREQ; /* 80 MHz */ clk->pll_max_out = SAM9260_PLL_A_MAX_OUT_FREQ; /* 240 MHz */ clk->pll_mul_shift = SAM9260_PLL_A_MUL_SHIFT; clk->pll_mul_mask = SAM9260_PLL_A_MUL_MASK; clk->pll_div_shift = SAM9260_PLL_A_DIV_SHIFT; clk->pll_div_mask = SAM9260_PLL_A_DIV_MASK; clk->set_outb = at91_pll_outa; at91_pmc_clock_deref(clk); /* * Fudge MAX pll in frequence down below 3.0 MHz to ensure * PMC alogrithm choose the divisor that causes the input clock * to be near the optimal 2 MHz per datasheet. We know * we are going to be using this for the USB clock at 96 MHz. - * Causes no extra frequency deviation for all recomended crystal + * Causes no extra frequency deviation for all recommended crystal * values. See Note 1, table 40-16 SAM9260 doc. */ clk = at91_pmc_clock_ref("pllb"); clk->pll_min_in = SAM9260_PLL_B_MIN_IN_FREQ; /* 1 MHz */ clk->pll_max_in = SAM9260_PLL_B_MAX_IN_FREQ; /* 5 MHz */ clk->pll_max_in = 2999999; /* ~3 MHz */ clk->pll_min_out = SAM9260_PLL_B_MIN_OUT_FREQ; /* 70 MHz */ clk->pll_max_out = SAM9260_PLL_B_MAX_OUT_FREQ; /* 130 MHz */ clk->pll_mul_shift = SAM9260_PLL_B_MUL_SHIFT; clk->pll_mul_mask = SAM9260_PLL_B_MUL_MASK; clk->pll_div_shift = SAM9260_PLL_B_DIV_SHIFT; clk->pll_div_mask = SAM9260_PLL_B_DIV_MASK; clk->set_outb = at91_pll_outb; at91_pmc_clock_deref(clk); } static struct at91_soc_data soc_data = { .soc_delay = at91_pit_delay, .soc_reset = at91_rst_cpu_reset, .soc_clock_init = at91_clock_init, .soc_irq_prio = at91_irq_prio, .soc_children = at91_devs, .soc_pio_base = at91_pio_base, .soc_pio_count = nitems(at91_pio_base), }; AT91_SOC(AT91_T_SAM9260, &soc_data); Index: user/ngie/release-pkg-fix-tests/sys/arm/at91/if_ate.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/at91/if_ate.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/at91/if_ate.c (revision 299091) @@ -1,1531 +1,1531 @@ /*- * Copyright (c) 2006 M. Warner Losh. All rights reserved. * Copyright (c) 2009 Greg Ansley. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* TODO * * 1) Turn on the clock in pmc? Turn off? * 2) GPIO initializtion in board setup code. */ #include "opt_platform.h" #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef INET #include #include #include #include #endif #include #include #include #include #include "opt_at91.h" #include #include #include #ifdef FDT #include #include #include #endif #include "miibus_if.h" /* * Driver-specific flags. */ #define ATE_FLAG_DETACHING 0x01 #define ATE_FLAG_MULTICAST 0x02 /* * Old EMAC assumes whole packet fits in one buffer; * new EBACB assumes all receive buffers are 128 bytes */ #define RX_BUF_SIZE(sc) (sc->is_emacb ? 128 : MCLBYTES) /* * EMACB has an 11 bit counter for Rx/Tx Descriptors * for max total of 1024 decriptors each. */ #define ATE_MAX_RX_DESCR 1024 #define ATE_MAX_TX_DESCR 1024 /* How many buffers to allocate */ #define ATE_MAX_TX_BUFFERS 4 /* We have ping-pong tx buffers */ /* How much memory to use for rx buffers */ #define ATE_RX_MEMORY (ATE_MAX_RX_DESCR * 128) /* Actual number of descriptors we allocate */ #define ATE_NUM_RX_DESCR ATE_MAX_RX_DESCR #define ATE_NUM_TX_DESCR ATE_MAX_TX_BUFFERS #if ATE_NUM_TX_DESCR > ATE_MAX_TX_DESCR #error "Can't have more TX buffers that descriptors" #endif #if ATE_NUM_RX_DESCR > ATE_MAX_RX_DESCR #error "Can't have more RX buffers that descriptors" #endif /* Wrap indexes the same way the hardware does */ #define NEXT_RX_IDX(sc, cur) \ ((sc->rx_descs[cur].addr & ETH_WRAP_BIT) ? 0 : (cur + 1)) #define NEXT_TX_IDX(sc, cur) \ ((sc->tx_descs[cur].status & ETHB_TX_WRAP) ? 0 : (cur + 1)) struct ate_softc { struct ifnet *ifp; /* ifnet pointer */ struct mtx sc_mtx; /* Basically a perimeter lock */ device_t dev; /* Myself */ device_t miibus; /* My child miibus */ struct resource *irq_res; /* IRQ resource */ struct resource *mem_res; /* Memory resource */ struct callout tick_ch; /* Tick callout */ struct ifmib_iso_8802_3 mibdata; /* Stuff for network mgmt */ bus_dma_tag_t mtag; /* bus dma tag for mbufs */ bus_dma_tag_t rx_tag; bus_dma_tag_t rx_desc_tag; bus_dmamap_t rx_desc_map; bus_dmamap_t rx_map[ATE_MAX_RX_DESCR]; bus_addr_t rx_desc_phys; /* PA of rx descriptors */ eth_rx_desc_t *rx_descs; /* VA of rx descriptors */ void *rx_buf[ATE_NUM_RX_DESCR]; /* RX buffer space */ int rxhead; /* Current RX map/desc index */ uint32_t rx_buf_size; /* Size of Rx buffers */ bus_dma_tag_t tx_desc_tag; bus_dmamap_t tx_desc_map; bus_dmamap_t tx_map[ATE_MAX_TX_BUFFERS]; bus_addr_t tx_desc_phys; /* PA of tx descriptors */ eth_tx_desc_t *tx_descs; /* VA of tx descriptors */ int txhead; /* Current TX map/desc index */ int txtail; /* Current TX map/desc index */ struct mbuf *sent_mbuf[ATE_MAX_TX_BUFFERS]; /* Sent mbufs */ void *intrhand; /* Interrupt handle */ int flags; int if_flags; int use_rmii; int is_emacb; /* SAM9x hardware version */ }; static inline uint32_t RD4(struct ate_softc *sc, bus_size_t off) { return (bus_read_4(sc->mem_res, off)); } static inline void WR4(struct ate_softc *sc, bus_size_t off, uint32_t val) { bus_write_4(sc->mem_res, off, val); } static inline void BARRIER(struct ate_softc *sc, bus_size_t off, bus_size_t len, int flags) { bus_barrier(sc->mem_res, off, len, flags); } #define ATE_LOCK(_sc) mtx_lock(&(_sc)->sc_mtx) #define ATE_UNLOCK(_sc) mtx_unlock(&(_sc)->sc_mtx) #define ATE_LOCK_INIT(_sc) \ mtx_init(&_sc->sc_mtx, device_get_nameunit(_sc->dev), \ MTX_NETWORK_LOCK, MTX_DEF) #define ATE_LOCK_DESTROY(_sc) mtx_destroy(&_sc->sc_mtx); #define ATE_ASSERT_LOCKED(_sc) mtx_assert(&_sc->sc_mtx, MA_OWNED); #define ATE_ASSERT_UNLOCKED(_sc) mtx_assert(&_sc->sc_mtx, MA_NOTOWNED); static devclass_t ate_devclass; /* * ifnet entry points. */ static void ateinit_locked(void *); static void atestart_locked(struct ifnet *); static void ateinit(void *); static void atestart(struct ifnet *); static void atestop(struct ate_softc *); static int ateioctl(struct ifnet * ifp, u_long, caddr_t); /* * Bus entry points. */ static int ate_probe(device_t dev); static int ate_attach(device_t dev); static int ate_detach(device_t dev); static void ate_intr(void *); /* * Helper routines. */ static int ate_activate(device_t dev); static void ate_deactivate(struct ate_softc *sc); static int ate_ifmedia_upd(struct ifnet *ifp); static void ate_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr); static int ate_get_mac(struct ate_softc *sc, u_char *eaddr); static void ate_set_mac(struct ate_softc *sc, u_char *eaddr); static void ate_rxfilter(struct ate_softc *sc); static int ate_miibus_readreg(device_t dev, int phy, int reg); static int ate_miibus_writereg(device_t dev, int phy, int reg, int data); /* * The AT91 family of products has the ethernet interface called EMAC. * However, it isn't self identifying. It is anticipated that the parent bus * code will take care to only add ate devices where they really are. As * such, we do nothing here to identify the device and just set its name. * However, FDT makes it self-identifying. */ static int ate_probe(device_t dev) { #ifdef FDT if (!ofw_bus_is_compatible(dev, "cdns,at91rm9200-emac") && !ofw_bus_is_compatible(dev, "cdns,emac") && !ofw_bus_is_compatible(dev, "cdns,at32ap7000-macb")) return (ENXIO); #endif device_set_desc(dev, "EMAC"); return (0); } #ifdef FDT /* * We have to know if we're using MII or RMII attachment * for the MACB to talk to the PHY correctly. With FDT, * we must use rmii if there's a proprety phy-mode * equal to "rmii". Otherwise we MII mode is used. */ static void ate_set_rmii(struct ate_softc *sc) { phandle_t node; char prop[10]; ssize_t len; node = ofw_bus_get_node(sc->dev); memset(prop, 0 ,sizeof(prop)); len = OF_getproplen(node, "phy-mode"); if (len != 4) return; if (OF_getprop(node, "phy-mode", prop, len) != len) return; if (strncmp(prop, "rmii", 4) == 0) sc->use_rmii = 1; } #else /* * We have to know if we're using MII or RMII attachment * for the MACB to talk to the PHY correctly. Without FDT, * there's no good way to do this. So, if the config file * has 'option AT91_ATE_USE_RMII', then we'll force RMII. * Otherwise, we'll use what the bootloader setup. Either * it setup RMII or MII, in which case we'll get it right, * or it did nothing, and we'll fall back to MII and the * option would override if present. */ static void ate_set_rmii(struct ate_softc *sc) { /* Default to what boot rom did */ if (!sc->is_emacb) sc->use_rmii = (RD4(sc, ETH_CFG) & ETH_CFG_RMII) == ETH_CFG_RMII; else sc->use_rmii = (RD4(sc, ETHB_UIO) & ETHB_UIO_RMII) == ETHB_UIO_RMII; #ifdef AT91_ATE_USE_RMII /* Compile time override */ sc->use_rmii = 1; #endif } #endif static int ate_attach(device_t dev) { struct ate_softc *sc; struct ifnet *ifp = NULL; struct sysctl_ctx_list *sctx; struct sysctl_oid *soid; u_char eaddr[ETHER_ADDR_LEN]; uint32_t rnd; int rid, err; sc = device_get_softc(dev); sc->dev = dev; ATE_LOCK_INIT(sc); rid = 0; sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->mem_res == NULL) { device_printf(dev, "could not allocate memory resources.\n"); err = ENOMEM; goto out; } rid = 0; sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE); if (sc->irq_res == NULL) { device_printf(dev, "could not allocate interrupt resources.\n"); err = ENOMEM; goto out; } /* New or old version, chooses buffer size. */ #ifdef FDT sc->is_emacb = ofw_bus_is_compatible(dev, "cdns,at32ap7000-macb"); #else sc->is_emacb = at91_is_sam9() || at91_is_sam9xe(); #endif sc->rx_buf_size = RX_BUF_SIZE(sc); err = ate_activate(dev); if (err) goto out; ate_set_rmii(sc); /* Sysctls */ sctx = device_get_sysctl_ctx(dev); soid = device_get_sysctl_tree(dev); SYSCTL_ADD_UINT(sctx, SYSCTL_CHILDREN(soid), OID_AUTO, "rmii", CTLFLAG_RW, &sc->use_rmii, 0, "rmii in use"); /* Calling atestop before ifp is set is OK. */ ATE_LOCK(sc); atestop(sc); ATE_UNLOCK(sc); callout_init_mtx(&sc->tick_ch, &sc->sc_mtx, 0); if ((err = ate_get_mac(sc, eaddr)) != 0) { /* No MAC address configured. Generate the random one. */ if (bootverbose) device_printf(dev, "Generating random ethernet address.\n"); rnd = arc4random(); /* * Set OUI to convenient locally assigned address. 'b' * is 0x62, which has the locally assigned bit set, and * the broadcast/multicast bit clear. */ eaddr[0] = 'b'; eaddr[1] = 's'; eaddr[2] = 'd'; eaddr[3] = (rnd >> 16) & 0xff; eaddr[4] = (rnd >> 8) & 0xff; eaddr[5] = (rnd >> 0) & 0xff; } sc->ifp = ifp = if_alloc(IFT_ETHER); err = mii_attach(dev, &sc->miibus, ifp, ate_ifmedia_upd, ate_ifmedia_sts, BMSR_DEFCAPMASK, MII_PHY_ANY, MII_OFFSET_ANY, 0); if (err != 0) { device_printf(dev, "attaching PHYs failed\n"); goto out; } /* * XXX: Clear the isolate bit, or we won't get up, * at least on the HL201 */ ate_miibus_writereg(dev, 0, 0, 0x3000); ifp->if_softc = sc; if_initname(ifp, device_get_name(dev), device_get_unit(dev)); ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; ifp->if_capabilities |= IFCAP_VLAN_MTU; ifp->if_capenable |= IFCAP_VLAN_MTU; /* The hw bits already set. */ ifp->if_start = atestart; ifp->if_ioctl = ateioctl; ifp->if_init = ateinit; ifp->if_baudrate = 10000000; IFQ_SET_MAXLEN(&ifp->if_snd, IFQ_MAXLEN); ifp->if_snd.ifq_drv_maxlen = IFQ_MAXLEN; IFQ_SET_READY(&ifp->if_snd); ifp->if_linkmib = &sc->mibdata; ifp->if_linkmiblen = sizeof(sc->mibdata); sc->mibdata.dot3Compliance = DOT3COMPLIANCE_COLLS; sc->if_flags = ifp->if_flags; ether_ifattach(ifp, eaddr); /* Activate the interrupt. */ err = bus_setup_intr(dev, sc->irq_res, INTR_TYPE_NET | INTR_MPSAFE, NULL, ate_intr, sc, &sc->intrhand); if (err) { device_printf(dev, "could not establish interrupt handler.\n"); ether_ifdetach(ifp); goto out; } out: if (err) ate_detach(dev); return (err); } static int ate_detach(device_t dev) { struct ate_softc *sc; struct ifnet *ifp; sc = device_get_softc(dev); KASSERT(sc != NULL, ("[ate: %d]: sc is NULL", __LINE__)); ifp = sc->ifp; if (device_is_attached(dev)) { ATE_LOCK(sc); sc->flags |= ATE_FLAG_DETACHING; atestop(sc); ATE_UNLOCK(sc); callout_drain(&sc->tick_ch); ether_ifdetach(ifp); } if (sc->miibus != NULL) { device_delete_child(dev, sc->miibus); sc->miibus = NULL; } bus_generic_detach(sc->dev); ate_deactivate(sc); if (sc->intrhand != NULL) { bus_teardown_intr(dev, sc->irq_res, sc->intrhand); sc->intrhand = NULL; } if (ifp != NULL) { if_free(ifp); sc->ifp = NULL; } if (sc->mem_res != NULL) { bus_release_resource(dev, SYS_RES_IOPORT, rman_get_rid(sc->mem_res), sc->mem_res); sc->mem_res = NULL; } if (sc->irq_res != NULL) { bus_release_resource(dev, SYS_RES_IRQ, rman_get_rid(sc->irq_res), sc->irq_res); sc->irq_res = NULL; } ATE_LOCK_DESTROY(sc); return (0); } static void ate_getaddr(void *arg, bus_dma_segment_t *segs, int nsegs, int error) { if (error != 0) return; *(bus_addr_t *)arg = segs[0].ds_addr; } static void ate_load_rx_buf(void *arg, bus_dma_segment_t *segs, int nsegs, int error) { struct ate_softc *sc; if (error != 0) return; sc = (struct ate_softc *)arg; bus_dmamap_sync(sc->rx_desc_tag, sc->rx_desc_map, BUS_DMASYNC_PREWRITE); sc->rx_descs[sc->rxhead].addr = segs[0].ds_addr; sc->rx_descs[sc->rxhead].status = 0; bus_dmamap_sync(sc->rx_desc_tag, sc->rx_desc_map, BUS_DMASYNC_POSTWRITE); } static uint32_t ate_mac_hash(const uint8_t *buf) { uint32_t index = 0; for (int i = 0; i < 48; i++) { index ^= ((buf[i >> 3] >> (i & 7)) & 1) << (i % 6); } return (index); } /* * Compute the multicast filter for this device. */ static int ate_setmcast(struct ate_softc *sc) { uint32_t index; uint32_t mcaf[2]; u_char *af = (u_char *) mcaf; struct ifmultiaddr *ifma; struct ifnet *ifp; ifp = sc->ifp; if ((ifp->if_flags & IFF_PROMISC) != 0) return (0); if ((ifp->if_flags & IFF_ALLMULTI) != 0) { WR4(sc, ETH_HSL, 0xffffffff); WR4(sc, ETH_HSH, 0xffffffff); return (1); } /* Compute the multicast hash. */ mcaf[0] = 0; mcaf[1] = 0; if_maddr_rlock(ifp); TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_LINK) continue; index = ate_mac_hash(LLADDR((struct sockaddr_dl *) ifma->ifma_addr)); af[index >> 3] |= 1 << (index & 7); } if_maddr_runlock(ifp); /* * Write the hash to the hash register. This card can also * accept unicast packets as well as multicast packets using this * register for easier bridging operations, but we don't take * advantage of that. Locks here are to avoid LOR with the * if_maddr_rlock, but might not be strictly necessary. */ WR4(sc, ETH_HSL, mcaf[0]); WR4(sc, ETH_HSH, mcaf[1]); return (mcaf[0] || mcaf[1]); } static int ate_activate(device_t dev) { struct ate_softc *sc; int i; sc = device_get_softc(dev); /* Allocate DMA tags and maps for TX mbufs */ if (bus_dma_tag_create(bus_get_dma_tag(dev), 1, 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, MCLBYTES, 1, MCLBYTES, 0, busdma_lock_mutex, &sc->sc_mtx, &sc->mtag)) goto errout; for (i = 0; i < ATE_MAX_TX_BUFFERS; i++) { if ( bus_dmamap_create(sc->mtag, 0, &sc->tx_map[i])) goto errout; } /* DMA tag and map for the RX descriptors. */ if (bus_dma_tag_create(bus_get_dma_tag(dev), sizeof(eth_rx_desc_t), 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, ATE_NUM_RX_DESCR * sizeof(eth_rx_desc_t), 1, ATE_NUM_RX_DESCR * sizeof(eth_rx_desc_t), 0, busdma_lock_mutex, &sc->sc_mtx, &sc->rx_desc_tag)) goto errout; if (bus_dmamem_alloc(sc->rx_desc_tag, (void **)&sc->rx_descs, BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &sc->rx_desc_map) != 0) goto errout; if (bus_dmamap_load(sc->rx_desc_tag, sc->rx_desc_map, sc->rx_descs, ATE_NUM_RX_DESCR * sizeof(eth_rx_desc_t), ate_getaddr, &sc->rx_desc_phys, 0) != 0) goto errout; /* Allocate DMA tags and maps for RX. buffers */ if (bus_dma_tag_create(bus_get_dma_tag(dev), 1, 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, sc->rx_buf_size, 1, sc->rx_buf_size, 0, busdma_lock_mutex, &sc->sc_mtx, &sc->rx_tag)) goto errout; /* * Allocate our RX buffers. * This chip has a RX structure that's filled in. * XXX On MACB (SAM9 part) we should receive directly into mbuf * to avoid the copy. XXX */ sc->rxhead = 0; for (sc->rxhead = 0; sc->rxhead < ATE_RX_MEMORY/sc->rx_buf_size; sc->rxhead++) { if (bus_dmamem_alloc(sc->rx_tag, (void **)&sc->rx_buf[sc->rxhead], BUS_DMA_NOWAIT, &sc->rx_map[sc->rxhead]) != 0) goto errout; if (bus_dmamap_load(sc->rx_tag, sc->rx_map[sc->rxhead], sc->rx_buf[sc->rxhead], sc->rx_buf_size, ate_load_rx_buf, sc, 0) != 0) { printf("bus_dmamem_load\n"); goto errout; } bus_dmamap_sync(sc->rx_tag, sc->rx_map[sc->rxhead], BUS_DMASYNC_PREREAD); } /* * For the last buffer, set the wrap bit so the controller * restarts from the first descriptor. */ sc->rx_descs[--sc->rxhead].addr |= ETH_WRAP_BIT; sc->rxhead = 0; /* Flush the memory for the EMAC rx descriptor. */ bus_dmamap_sync(sc->rx_desc_tag, sc->rx_desc_map, BUS_DMASYNC_PREWRITE); /* Write the descriptor queue address. */ WR4(sc, ETH_RBQP, sc->rx_desc_phys); /* * DMA tag and map for the TX descriptors. */ if (bus_dma_tag_create(bus_get_dma_tag(dev), sizeof(eth_tx_desc_t), 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, ATE_MAX_TX_BUFFERS * sizeof(eth_tx_desc_t), 1, ATE_MAX_TX_BUFFERS * sizeof(eth_tx_desc_t), 0, busdma_lock_mutex, &sc->sc_mtx, &sc->tx_desc_tag) != 0) goto errout; if (bus_dmamem_alloc(sc->tx_desc_tag, (void **)&sc->tx_descs, BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &sc->tx_desc_map) != 0) goto errout; if (bus_dmamap_load(sc->tx_desc_tag, sc->tx_desc_map, sc->tx_descs, ATE_MAX_TX_BUFFERS * sizeof(eth_tx_desc_t), ate_getaddr, &sc->tx_desc_phys, 0) != 0) goto errout; - /* Initilize descriptors; mark all empty */ + /* Initialize descriptors; mark all empty */ for (i = 0; i < ATE_MAX_TX_BUFFERS; i++) { sc->tx_descs[i].addr =0; sc->tx_descs[i].status = ETHB_TX_USED; sc->sent_mbuf[i] = NULL; } /* Mark last entry to cause wrap when indexing through */ sc->tx_descs[ATE_MAX_TX_BUFFERS - 1].status = ETHB_TX_WRAP | ETHB_TX_USED; /* Flush the memory for the EMAC tx descriptor. */ bus_dmamap_sync(sc->tx_desc_tag, sc->tx_desc_map, BUS_DMASYNC_PREWRITE); sc->txhead = sc->txtail = 0; if (sc->is_emacb) { /* Write the descriptor queue address. */ WR4(sc, ETHB_TBQP, sc->tx_desc_phys); /* EMACB: Enable transceiver input clock */ WR4(sc, ETHB_UIO, RD4(sc, ETHB_UIO) | ETHB_UIO_CLKE); } return (0); errout: return (ENOMEM); } static void ate_deactivate(struct ate_softc *sc) { int i; KASSERT(sc != NULL, ("[ate, %d]: sc is NULL!", __LINE__)); if (sc->mtag != NULL) { for (i = 0; i < ATE_MAX_TX_BUFFERS; i++) { if (sc->sent_mbuf[i] != NULL) { bus_dmamap_sync(sc->mtag, sc->tx_map[i], BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(sc->mtag, sc->tx_map[i]); m_freem(sc->sent_mbuf[i]); } bus_dmamap_destroy(sc->mtag, sc->tx_map[i]); sc->sent_mbuf[i] = NULL; sc->tx_map[i] = NULL; } bus_dma_tag_destroy(sc->mtag); } if (sc->rx_desc_tag != NULL) { if (sc->rx_descs != NULL) { if (sc->rx_desc_phys != 0) { bus_dmamap_sync(sc->rx_desc_tag, sc->rx_desc_map, BUS_DMASYNC_POSTREAD); bus_dmamap_unload(sc->rx_desc_tag, sc->rx_desc_map); sc->rx_desc_phys = 0; } } } if (sc->rx_tag != NULL) { for (i = 0; sc->rx_buf[i] != NULL; i++) { if (sc->rx_descs[i].addr != 0) { bus_dmamap_sync(sc->rx_tag, sc->rx_map[i], BUS_DMASYNC_POSTREAD); bus_dmamap_unload(sc->rx_tag, sc->rx_map[i]); sc->rx_descs[i].addr = 0; } bus_dmamem_free(sc->rx_tag, sc->rx_buf[i], sc->rx_map[i]); sc->rx_buf[i] = NULL; } bus_dma_tag_destroy(sc->rx_tag); } if (sc->rx_desc_tag != NULL) { if (sc->rx_descs != NULL) bus_dmamem_free(sc->rx_desc_tag, sc->rx_descs, sc->rx_desc_map); bus_dma_tag_destroy(sc->rx_desc_tag); sc->rx_descs = NULL; sc->rx_desc_tag = NULL; } if (sc->is_emacb) WR4(sc, ETHB_UIO, RD4(sc, ETHB_UIO) & ~ETHB_UIO_CLKE); } /* * Change media according to request. */ static int ate_ifmedia_upd(struct ifnet *ifp) { struct ate_softc *sc = ifp->if_softc; struct mii_data *mii; mii = device_get_softc(sc->miibus); ATE_LOCK(sc); mii_mediachg(mii); ATE_UNLOCK(sc); return (0); } /* * Notify the world which media we're using. */ static void ate_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr) { struct ate_softc *sc = ifp->if_softc; struct mii_data *mii; mii = device_get_softc(sc->miibus); ATE_LOCK(sc); mii_pollstat(mii); ifmr->ifm_active = mii->mii_media_active; ifmr->ifm_status = mii->mii_media_status; ATE_UNLOCK(sc); } static void ate_stat_update(struct ate_softc *sc, int active) { uint32_t reg; /* * The speed and full/half-duplex state needs to be reflected * in the ETH_CFG register. */ reg = RD4(sc, ETH_CFG); reg &= ~(ETH_CFG_SPD | ETH_CFG_FD); if (IFM_SUBTYPE(active) != IFM_10_T) reg |= ETH_CFG_SPD; if (active & IFM_FDX) reg |= ETH_CFG_FD; WR4(sc, ETH_CFG, reg); } static void ate_tick(void *xsc) { struct ate_softc *sc = xsc; struct ifnet *ifp = sc->ifp; struct mii_data *mii; int active; uint32_t c; /* * The KB920x boot loader tests ETH_SR & ETH_SR_LINK and will ask * the MII if there's a link if this bit is clear. Not sure if we * should do the same thing here or not. */ ATE_ASSERT_LOCKED(sc); if (sc->miibus != NULL) { mii = device_get_softc(sc->miibus); active = mii->mii_media_active; mii_tick(mii); if (mii->mii_media_status & IFM_ACTIVE && active != mii->mii_media_active) ate_stat_update(sc, mii->mii_media_active); } /* * Update the stats as best we can. When we're done, clear * the status counters and start over. We're supposed to read these * registers often enough that they won't overflow. Hopefully * once a second is often enough. Some don't map well to * the dot3Stats mib, so for those we just count them as general * errors. Stats for iframes, ibutes, oframes and obytes are * collected elsewhere. These registers zero on a read to prevent * races. For all the collision stats, also update the collision * stats for the interface. */ sc->mibdata.dot3StatsAlignmentErrors += RD4(sc, ETH_ALE); sc->mibdata.dot3StatsFCSErrors += RD4(sc, ETH_SEQE); c = RD4(sc, ETH_SCOL); if_inc_counter(ifp, IFCOUNTER_COLLISIONS, c); sc->mibdata.dot3StatsSingleCollisionFrames += c; c = RD4(sc, ETH_MCOL); sc->mibdata.dot3StatsMultipleCollisionFrames += c; if_inc_counter(ifp, IFCOUNTER_COLLISIONS, c); sc->mibdata.dot3StatsSQETestErrors += RD4(sc, ETH_SQEE); sc->mibdata.dot3StatsDeferredTransmissions += RD4(sc, ETH_DTE); c = RD4(sc, ETH_LCOL); sc->mibdata.dot3StatsLateCollisions += c; if_inc_counter(ifp, IFCOUNTER_COLLISIONS, c); c = RD4(sc, ETH_ECOL); sc->mibdata.dot3StatsExcessiveCollisions += c; if_inc_counter(ifp, IFCOUNTER_COLLISIONS, c); sc->mibdata.dot3StatsCarrierSenseErrors += RD4(sc, ETH_CSE); sc->mibdata.dot3StatsFrameTooLongs += RD4(sc, ETH_ELR); sc->mibdata.dot3StatsInternalMacReceiveErrors += RD4(sc, ETH_DRFC); /* * Not sure where to lump these, so count them against the errors * for the interface. */ if_inc_counter(sc->ifp, IFCOUNTER_OERRORS, RD4(sc, ETH_TUE)); if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, RD4(sc, ETH_CDE) + RD4(sc, ETH_RJB) + RD4(sc, ETH_USF)); /* Schedule another timeout one second from now. */ callout_reset(&sc->tick_ch, hz, ate_tick, sc); } static void ate_set_mac(struct ate_softc *sc, u_char *eaddr) { WR4(sc, ETH_SA1L, (eaddr[3] << 24) | (eaddr[2] << 16) | (eaddr[1] << 8) | eaddr[0]); WR4(sc, ETH_SA1H, (eaddr[5] << 8) | (eaddr[4])); } static int ate_get_mac(struct ate_softc *sc, u_char *eaddr) { bus_size_t sa_low_reg[] = { ETH_SA1L, ETH_SA2L, ETH_SA3L, ETH_SA4L }; bus_size_t sa_high_reg[] = { ETH_SA1H, ETH_SA2H, ETH_SA3H, ETH_SA4H }; uint32_t low, high; int i; /* * The boot loader may setup the MAC with an address(es), grab the * first MAC address from the SA[1-4][HL] registers. */ for (i = 0; i < 4; i++) { low = RD4(sc, sa_low_reg[i]); high = RD4(sc, sa_high_reg[i]); if ((low | (high & 0xffff)) != 0) { eaddr[0] = low & 0xff; eaddr[1] = (low >> 8) & 0xff; eaddr[2] = (low >> 16) & 0xff; eaddr[3] = (low >> 24) & 0xff; eaddr[4] = high & 0xff; eaddr[5] = (high >> 8) & 0xff; return (0); } } return (ENXIO); } static void ate_intr(void *xsc) { struct ate_softc *sc = xsc; struct ifnet *ifp = sc->ifp; struct mbuf *mb; eth_rx_desc_t *rxdhead; uint32_t status, reg, idx; int remain, count, done; status = RD4(sc, ETH_ISR); if (status == 0) return; if (status & ETH_ISR_RCOM) { bus_dmamap_sync(sc->rx_desc_tag, sc->rx_desc_map, BUS_DMASYNC_POSTREAD); rxdhead = &sc->rx_descs[sc->rxhead]; while (rxdhead->addr & ETH_CPU_OWNER) { if (!sc->is_emacb) { /* * Simulate SAM9 FIRST/LAST bits for RM9200. * RM9200 EMAC has only on Rx buffer per packet. - * But sometime we are handed a zero lenght packet. + * But sometime we are handed a zero length packet. */ if ((rxdhead->status & ETH_LEN_MASK) == 0) rxdhead->status = 0; /* Mark error */ else rxdhead->status |= ETH_BUF_FIRST | ETH_BUF_LAST; } if ((rxdhead->status & ETH_BUF_FIRST) == 0) { /* Something went wrong during RX so release back to EMAC all buffers of invalid packets. */ rxdhead->status = 0; rxdhead->addr &= ~ETH_CPU_OWNER; sc->rxhead = NEXT_RX_IDX(sc, sc->rxhead); rxdhead = &sc->rx_descs[sc->rxhead]; continue; } /* Find end of packet or start of next */ idx = sc->rxhead; if ((sc->rx_descs[idx].status & ETH_BUF_LAST) == 0) { idx = NEXT_RX_IDX(sc, idx); while ((sc->rx_descs[idx].addr & ETH_CPU_OWNER) && ((sc->rx_descs[idx].status & (ETH_BUF_FIRST|ETH_BUF_LAST))== 0)) idx = NEXT_RX_IDX(sc, idx); } /* Packet NOT yet completely in memory; we are done */ if ((sc->rx_descs[idx].addr & ETH_CPU_OWNER) == 0 || ((sc->rx_descs[idx].status & (ETH_BUF_FIRST|ETH_BUF_LAST))== 0)) break; /* Packets with no end descriptor are invalid. */ if ((sc->rx_descs[idx].status & ETH_BUF_LAST) == 0) { rxdhead->status &= ~ETH_BUF_FIRST; continue; } /* FCS is not coppied into mbuf. */ remain = (sc->rx_descs[idx].status & ETH_LEN_MASK) - 4; /* Get an appropriately sized mbuf. */ mb = m_get2(remain + ETHER_ALIGN, M_NOWAIT, MT_DATA, M_PKTHDR); if (mb == NULL) { if_inc_counter(sc->ifp, IFCOUNTER_IQDROPS, 1); rxdhead->status = 0; continue; } mb->m_data += ETHER_ALIGN; mb->m_pkthdr.rcvif = ifp; WR4(sc, ETH_RSR, RD4(sc, ETH_RSR)); /* Reset status */ /* Now we process the buffers that make up the packet */ do { /* Last buffer may just be 1-4 bytes of FCS so remain - * may be zero for last decriptor. */ + * may be zero for last descriptor. */ if (remain > 0) { /* Make sure we get the current bytes */ bus_dmamap_sync(sc->rx_tag, sc->rx_map[sc->rxhead], BUS_DMASYNC_POSTREAD); count = MIN(remain, sc->rx_buf_size); /* XXX Performance robbing copy. Could - * recieve directly to mbufs if not an + * receive directly to mbufs if not an * RM9200. And even then we could likely * copy just the protocol headers. XXX */ m_append(mb, count, sc->rx_buf[sc->rxhead]); remain -= count; } done = (rxdhead->status & ETH_BUF_LAST) != 0; /* Return the descriptor to the EMAC */ rxdhead->status = 0; rxdhead->addr &= ~ETH_CPU_OWNER; bus_dmamap_sync(sc->rx_desc_tag, sc->rx_desc_map, BUS_DMASYNC_PREWRITE); /* Move on to next descriptor with wrap */ sc->rxhead = NEXT_RX_IDX(sc, sc->rxhead); rxdhead = &sc->rx_descs[sc->rxhead]; } while (!done); if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1); (*ifp->if_input)(ifp, mb); } } if (status & ETH_ISR_TCOM) { bus_dmamap_sync(sc->tx_desc_tag, sc->tx_desc_map, BUS_DMASYNC_POSTREAD); ATE_LOCK(sc); /* XXX TSR register should be cleared */ if (!sc->is_emacb) { /* Simulate Transmit descriptor table */ /* First packet done */ if (sc->txtail < sc->txhead) sc->tx_descs[sc->txtail].status |= ETHB_TX_USED; /* Second Packet done */ if (sc->txtail + 1 < sc->txhead && RD4(sc, ETH_TSR) & ETH_TSR_IDLE) sc->tx_descs[sc->txtail + 1].status |= ETHB_TX_USED; } while ((sc->tx_descs[sc->txtail].status & ETHB_TX_USED) && sc->sent_mbuf[sc->txtail] != NULL) { bus_dmamap_sync(sc->mtag, sc->tx_map[sc->txtail], BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(sc->mtag, sc->tx_map[sc->txtail]); m_freem(sc->sent_mbuf[sc->txtail]); sc->tx_descs[sc->txtail].addr = 0; sc->sent_mbuf[sc->txtail] = NULL; if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); sc->txtail = NEXT_TX_IDX(sc, sc->txtail); } /* Flush descriptors to EMAC */ bus_dmamap_sync(sc->tx_desc_tag, sc->tx_desc_map, BUS_DMASYNC_PREWRITE); /* * We're no longer busy, so clear the busy flag and call the * start routine to xmit more packets. */ sc->ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; atestart_locked(sc->ifp); ATE_UNLOCK(sc); } if (status & ETH_ISR_RBNA) { /* Workaround RM9200 Errata #11 */ if (bootverbose) device_printf(sc->dev, "RBNA workaround\n"); reg = RD4(sc, ETH_CTL); WR4(sc, ETH_CTL, reg & ~ETH_CTL_RE); BARRIER(sc, ETH_CTL, 4, BUS_SPACE_BARRIER_WRITE); WR4(sc, ETH_CTL, reg | ETH_CTL_RE); } /* XXX need to work around SAM9260 errata 43.2.4.1: * disable the mac, reset tx buffer, enable mac on TUND */ } /* * Reset and initialize the chip. */ static void ateinit_locked(void *xsc) { struct ate_softc *sc = xsc; struct ifnet *ifp = sc->ifp; struct mii_data *mii; uint8_t eaddr[ETHER_ADDR_LEN]; uint32_t reg; ATE_ASSERT_LOCKED(sc); /* * XXX TODO(3) * we need to turn on the EMAC clock in the pmc. With the * default boot loader, this is already turned on. However, we * need to think about how best to turn it on/off as the interface * is brought up/down, as well as dealing with the mii bus... * * We also need to multiplex the pins correctly (in board_xxx.c). */ /* * There are two different ways that the mii bus is connected * to this chip mii or rmii. */ if (!sc->is_emacb) { /* RM9200 */ reg = RD4(sc, ETH_CFG); if (sc->use_rmii) reg |= ETH_CFG_RMII; else reg &= ~ETH_CFG_RMII; WR4(sc, ETH_CFG, reg); } else { /* SAM9 */ reg = ETHB_UIO_CLKE; reg |= (sc->use_rmii) ? ETHB_UIO_RMII : 0; WR4(sc, ETHB_UIO, reg); } ate_rxfilter(sc); /* * Set the chip MAC address. */ bcopy(IF_LLADDR(ifp), eaddr, ETHER_ADDR_LEN); ate_set_mac(sc, eaddr); /* Make sure we know state of TX queue */ sc->txhead = sc->txtail = 0; if (sc->is_emacb) { /* Write the descriptor queue address. */ WR4(sc, ETHB_TBQP, sc->tx_desc_phys); } /* * Turn on MACs and interrupt processing. */ WR4(sc, ETH_CTL, RD4(sc, ETH_CTL) | ETH_CTL_TE | ETH_CTL_RE); WR4(sc, ETH_IER, ETH_ISR_RCOM | ETH_ISR_TCOM | ETH_ISR_RBNA); /* Enable big packets. */ WR4(sc, ETH_CFG, RD4(sc, ETH_CFG) | ETH_CFG_BIG); /* * Set 'running' flag, and clear output active flag * and attempt to start the output. */ ifp->if_drv_flags |= IFF_DRV_RUNNING; ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; mii = device_get_softc(sc->miibus); mii_pollstat(mii); ate_stat_update(sc, mii->mii_media_active); atestart_locked(ifp); callout_reset(&sc->tick_ch, hz, ate_tick, sc); } /* * Dequeue packets and transmit. */ static void atestart_locked(struct ifnet *ifp) { struct ate_softc *sc = ifp->if_softc; struct mbuf *m, *mdefrag; bus_dma_segment_t segs[1]; int nseg, e; ATE_ASSERT_LOCKED(sc); if (ifp->if_drv_flags & IFF_DRV_OACTIVE) return; while (sc->tx_descs[sc->txhead].status & ETHB_TX_USED) { /* * Check to see if there's room to put another packet into the * xmit queue. The old EMAC version has a ping-pong buffer for * xmit packets. We use OACTIVE to indicate "we can stuff more * into our buffers (clear) or not (set)." */ /* RM9200 has only two hardware entries */ if (!sc->is_emacb && (RD4(sc, ETH_TSR) & ETH_TSR_BNQ) == 0) { ifp->if_drv_flags |= IFF_DRV_OACTIVE; return; } IFQ_DRV_DEQUEUE(&ifp->if_snd, m); if (m == NULL) break; e = bus_dmamap_load_mbuf_sg(sc->mtag, sc->tx_map[sc->txhead], m, segs, &nseg, 0); if (e == EFBIG) { mdefrag = m_defrag(m, M_NOWAIT); if (mdefrag == NULL) { IFQ_DRV_PREPEND(&ifp->if_snd, m); return; } m = mdefrag; e = bus_dmamap_load_mbuf_sg(sc->mtag, sc->tx_map[sc->txhead], m, segs, &nseg, 0); } if (e != 0) { m_freem(m); continue; } /* * There's a small race between the loop in ate_intr finishing * and the check above to see if the packet was finished, as well * as when atestart gets called via other paths. Lose the race * gracefully and free the mbuf... */ if (sc->sent_mbuf[sc->txhead] != NULL) { bus_dmamap_sync(sc->mtag, sc->tx_map[sc->txtail], BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(sc->mtag, sc->tx_map[sc->txtail]); m_free(sc->sent_mbuf[sc->txhead]); if_inc_counter(ifp, IFCOUNTER_OPACKETS, 1); } sc->sent_mbuf[sc->txhead] = m; bus_dmamap_sync(sc->mtag, sc->tx_map[sc->txhead], BUS_DMASYNC_PREWRITE); /* Tell the hardware to xmit the packet. */ if (!sc->is_emacb) { WR4(sc, ETH_TAR, segs[0].ds_addr); BARRIER(sc, ETH_TAR, 4, BUS_SPACE_BARRIER_WRITE); WR4(sc, ETH_TCR, segs[0].ds_len); } else { bus_dmamap_sync(sc->tx_desc_tag, sc->tx_desc_map, BUS_DMASYNC_POSTWRITE); sc->tx_descs[sc->txhead].addr = segs[0].ds_addr; sc->tx_descs[sc->txhead].status = segs[0].ds_len | (sc->tx_descs[sc->txhead].status & ETHB_TX_WRAP) | ETHB_TX_BUF_LAST; bus_dmamap_sync(sc->tx_desc_tag, sc->tx_desc_map, BUS_DMASYNC_PREWRITE); WR4(sc, ETH_CTL, RD4(sc, ETH_CTL) | ETHB_CTL_TGO); } sc->txhead = NEXT_TX_IDX(sc, sc->txhead); /* Tap off here if there is a bpf listener. */ BPF_MTAP(ifp, m); } if ((sc->tx_descs[sc->txhead].status & ETHB_TX_USED) == 0) ifp->if_drv_flags |= IFF_DRV_OACTIVE; } static void ateinit(void *xsc) { struct ate_softc *sc = xsc; ATE_LOCK(sc); ateinit_locked(sc); ATE_UNLOCK(sc); } static void atestart(struct ifnet *ifp) { struct ate_softc *sc = ifp->if_softc; ATE_LOCK(sc); atestart_locked(ifp); ATE_UNLOCK(sc); } /* * Turn off interrupts, and stop the NIC. Can be called with sc->ifp NULL, * so be careful. */ static void atestop(struct ate_softc *sc) { struct ifnet *ifp; int i; ATE_ASSERT_LOCKED(sc); ifp = sc->ifp; if (ifp) { //ifp->if_timer = 0; ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); } callout_stop(&sc->tick_ch); /* * Enable some parts of the MAC that are needed always (like the * MII bus. This turns off the RE and TE bits, which will remain * off until ateinit() is called to turn them on. With RE and TE * turned off, there's no DMA to worry about after this write. */ WR4(sc, ETH_CTL, ETH_CTL_MPE); /* * Turn off all the configured options and revert to defaults. */ /* Make sure thate the MDIO clk is less than * 2.5 Mhz. Can no longer default to /32 since * SAM9 family may have MCK > 80 Mhz */ if (at91_master_clock <= 2000000) WR4(sc, ETH_CFG, ETH_CFG_CLK_8); else if (at91_master_clock <= 4000000) WR4(sc, ETH_CFG, ETH_CFG_CLK_16); else if (at91_master_clock <= 800000) WR4(sc, ETH_CFG, ETH_CFG_CLK_32); else WR4(sc, ETH_CFG, ETH_CFG_CLK_64); /* * Turn off all the interrupts, and ack any pending ones by reading * the ISR. */ WR4(sc, ETH_IDR, 0xffffffff); RD4(sc, ETH_ISR); /* * Clear out the Transmit and Receiver Status registers of any * errors they may be reporting */ WR4(sc, ETH_TSR, 0xffffffff); WR4(sc, ETH_RSR, 0xffffffff); /* Release TX resources. */ for (i = 0; i < ATE_MAX_TX_BUFFERS; i++) { if (sc->sent_mbuf[i] != NULL) { bus_dmamap_sync(sc->mtag, sc->tx_map[i], BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(sc->mtag, sc->tx_map[i]); m_freem(sc->sent_mbuf[i]); sc->sent_mbuf[i] = NULL; } } /* Turn off transeiver input clock */ if (sc->is_emacb) WR4(sc, ETHB_UIO, RD4(sc, ETHB_UIO) & ~ETHB_UIO_CLKE); /* * XXX we should power down the EMAC if it isn't in use, after * putting it into loopback mode. This saves about 400uA according * to the datasheet. */ } static void ate_rxfilter(struct ate_softc *sc) { struct ifnet *ifp; uint32_t reg; int enabled; KASSERT(sc != NULL, ("[ate, %d]: sc is NULL!", __LINE__)); ATE_ASSERT_LOCKED(sc); ifp = sc->ifp; /* Wipe out old filter settings. */ reg = RD4(sc, ETH_CFG); reg &= ~(ETH_CFG_CAF | ETH_CFG_MTI | ETH_CFG_UNI); reg |= ETH_CFG_NBC; sc->flags &= ~ATE_FLAG_MULTICAST; /* Set new parameters. */ if ((ifp->if_flags & IFF_BROADCAST) != 0) reg &= ~ETH_CFG_NBC; if ((ifp->if_flags & IFF_PROMISC) != 0) { reg |= ETH_CFG_CAF; } else { enabled = ate_setmcast(sc); if (enabled != 0) { reg |= ETH_CFG_MTI; sc->flags |= ATE_FLAG_MULTICAST; } } WR4(sc, ETH_CFG, reg); } static int ateioctl(struct ifnet *ifp, u_long cmd, caddr_t data) { struct ate_softc *sc = ifp->if_softc; struct mii_data *mii; struct ifreq *ifr = (struct ifreq *)data; int drv_flags, flags; int mask, error, enabled; error = 0; flags = ifp->if_flags; drv_flags = ifp->if_drv_flags; switch (cmd) { case SIOCSIFFLAGS: ATE_LOCK(sc); if ((flags & IFF_UP) != 0) { if ((drv_flags & IFF_DRV_RUNNING) != 0) { if (((flags ^ sc->if_flags) & (IFF_PROMISC | IFF_ALLMULTI)) != 0) ate_rxfilter(sc); } else { if ((sc->flags & ATE_FLAG_DETACHING) == 0) ateinit_locked(sc); } } else if ((drv_flags & IFF_DRV_RUNNING) != 0) { ifp->if_drv_flags &= ~IFF_DRV_RUNNING; atestop(sc); } sc->if_flags = flags; ATE_UNLOCK(sc); break; case SIOCADDMULTI: case SIOCDELMULTI: if ((drv_flags & IFF_DRV_RUNNING) != 0) { ATE_LOCK(sc); enabled = ate_setmcast(sc); if (enabled != (sc->flags & ATE_FLAG_MULTICAST)) ate_rxfilter(sc); ATE_UNLOCK(sc); } break; case SIOCSIFMEDIA: case SIOCGIFMEDIA: mii = device_get_softc(sc->miibus); error = ifmedia_ioctl(ifp, ifr, &mii->mii_media, cmd); break; case SIOCSIFCAP: mask = ifp->if_capenable ^ ifr->ifr_reqcap; if (mask & IFCAP_VLAN_MTU) { ATE_LOCK(sc); if (ifr->ifr_reqcap & IFCAP_VLAN_MTU) { WR4(sc, ETH_CFG, RD4(sc, ETH_CFG) | ETH_CFG_BIG); ifp->if_capenable |= IFCAP_VLAN_MTU; } else { WR4(sc, ETH_CFG, RD4(sc, ETH_CFG) & ~ETH_CFG_BIG); ifp->if_capenable &= ~IFCAP_VLAN_MTU; } ATE_UNLOCK(sc); } default: error = ether_ioctl(ifp, cmd, data); break; } return (error); } static void ate_child_detached(device_t dev, device_t child) { struct ate_softc *sc; sc = device_get_softc(dev); if (child == sc->miibus) sc->miibus = NULL; } /* * MII bus support routines. */ static int ate_miibus_readreg(device_t dev, int phy, int reg) { struct ate_softc *sc; int val; /* - * XXX if we implement agressive power savings, then we need + * XXX if we implement aggressive power savings, then we need * XXX to make sure that the clock to the emac is on here */ sc = device_get_softc(dev); DELAY(1); /* Hangs w/o this delay really 30.5us atm */ WR4(sc, ETH_MAN, ETH_MAN_REG_RD(phy, reg)); while ((RD4(sc, ETH_SR) & ETH_SR_IDLE) == 0) continue; val = RD4(sc, ETH_MAN) & ETH_MAN_VALUE_MASK; return (val); } static int ate_miibus_writereg(device_t dev, int phy, int reg, int data) { struct ate_softc *sc; /* - * XXX if we implement agressive power savings, then we need + * XXX if we implement aggressive power savings, then we need * XXX to make sure that the clock to the emac is on here */ sc = device_get_softc(dev); WR4(sc, ETH_MAN, ETH_MAN_REG_WR(phy, reg, data)); while ((RD4(sc, ETH_SR) & ETH_SR_IDLE) == 0) continue; return (0); } static device_method_t ate_methods[] = { /* Device interface */ DEVMETHOD(device_probe, ate_probe), DEVMETHOD(device_attach, ate_attach), DEVMETHOD(device_detach, ate_detach), /* Bus interface */ DEVMETHOD(bus_child_detached, ate_child_detached), /* MII interface */ DEVMETHOD(miibus_readreg, ate_miibus_readreg), DEVMETHOD(miibus_writereg, ate_miibus_writereg), DEVMETHOD_END }; static driver_t ate_driver = { "ate", ate_methods, sizeof(struct ate_softc), }; #ifdef FDT DRIVER_MODULE(ate, simplebus, ate_driver, ate_devclass, NULL, NULL); #else DRIVER_MODULE(ate, atmelarm, ate_driver, ate_devclass, NULL, NULL); #endif DRIVER_MODULE(miibus, ate, miibus_driver, miibus_devclass, NULL, NULL); MODULE_DEPEND(ate, miibus, 1, 1, 1); MODULE_DEPEND(ate, ether, 1, 1, 1); Index: user/ngie/release-pkg-fix-tests/sys/arm/at91/if_atereg.h =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/at91/if_atereg.h (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/at91/if_atereg.h (revision 299091) @@ -1,214 +1,214 @@ /*- * Copyright (c) 2006 M. Warner Losh. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* $FreeBSD$ */ #ifndef ARM_AT91_IF_ATEREG_H #define ARM_AT91_IF_ATEREG_H -/* deines begining ETHB_ are EMACB (newer SAM9 hardware) versions only */ +/* Defines beginning ETHB_ are EMACB (newer SAM9 hardware) versions only. */ #define ETH_CTL 0x00 /* EMAC Control Register */ #define ETH_CFG 0x04 /* EMAC Configuration Register */ #define ETH_SR 0x08 /* EMAC STatus Register */ #define ETH_TAR 0x0c /* EMAC Transmit Address Register */ #define ETH_TCR 0x10 /* EMAC Transmit Control Register */ #define ETH_TSR 0x14 /* EMAC Transmit Status Register */ #define ETH_RBQP 0x18 /* EMAC Receive Buffer Queue Pointer */ #define ETHB_TBQP 0x1c /* reserved */ #define ETH_RSR 0x20 /* EMAC Receive Status Register */ #define ETH_ISR 0x24 /* EMAC Interrupt Status Register */ #define ETH_IER 0x28 /* EMAC Interrupt Enable Register */ #define ETH_IDR 0x2c /* EMAC Interrupt Disable Register */ #define ETH_IMR 0x30 /* EMAC Interrupt Mask Register */ #define ETH_MAN 0x34 /* EMAC PHY Maintenance Register */ /* 0x38 reserved */ /* 0x3c reserved */ #define ETH_FRA 0x40 /* Frames Transmitted OK Register */ #define ETH_SCOL 0x44 /* Single Collision Frame Register */ #define ETH_MCOL 0x48 /* Multiple Collision Frame Register */ #define ETH_OK 0x4c /* Frames Received OK Register */ #define ETH_SEQE 0x50 /* Frame Check Sequence Error Reg */ #define ETH_ALE 0x54 /* Alignment Error Register */ #define ETH_DTE 0x58 /* Deferred Transmittion Frame Reg */ #define ETH_LCOL 0x5c /* Late Collision Register */ #define ETH_ECOL 0x60 /* Excessive Collision Register */ #define ETH_TUE 0x64 /* Transmit Underrun Error Register */ #define ETH_CSE 0x68 /* Carrier Sense Error Register */ #define ETH_DRFC 0x6c /* Discarded RX Frame Register */ #define ETH_ROV 0x68 /* Receive Overrun Register */ #define ETH_CDE 0x64 /* Code Error Register */ #define ETH_ELR 0x78 /* Excessive Length Error Register */ #define ETH_RJB 0x7c /* Receive Jabber Register */ #define ETH_USF 0x80 /* Undersize Frame Register */ #define ETH_SQEE 0x84 /* SQE Test Error Register */ /* 0x88 reserved */ /* 0x8c reserved */ #define ETH_HSL 0x90 /* EMAC Hash Address Low [31:0] */ #define ETH_HSH 0x94 /* EMAC Hash Address High [63:32] */ #define ETH_SA1L 0x98 /* EMAC Specific Address 1 Low */ #define ETH_SA1H 0x9c /* EMAC Specific Address 1 High */ #define ETH_SA2L 0xa0 /* EMAC Specific Address 2 Low */ #define ETH_SA2H 0xa4 /* EMAC Specific Address 2 High */ #define ETH_SA3L 0xa8 /* EMAC Specific Address 3 Low */ #define ETH_SA3H 0xac /* EMAC Specific Address 3 High */ #define ETH_SA4L 0xb0 /* EMAC Specific Address 4 Low */ #define ETH_SA4H 0xb4 /* EMAC Specific Address 4 High */ #define ETHB_TID 0xb8 /* EMAC Type ID Checking */ #define ETHB_UIO 0xC0 /* EMAC User I/O Reg */ /* ETH_CTL */ #define ETH_CTL_LB (1U << 0) /* LB: Loopback */ #define ETH_CTL_LBL (1U << 1) /* LBL: Loopback Local */ #define ETH_CTL_RE (1U << 2) /* RE: Receive Enable */ #define ETH_CTL_TE (1U << 3) /* TE: Transmit Enable */ #define ETH_CTL_MPE (1U << 4) /* MPE: Management Port Enable */ #define ETH_CTL_CSR (1U << 5) /* CSR: Clear Statistics Registers */ #define ETH_CTL_ISR (1U << 6) /* ISR: Incremenet Statistics Regs */ #define ETH_CTL_WES (1U << 7) /* WES: Write Enable Statistics regs */ #define ETH_CTL_BP (1U << 8) /* BP: Back Pressure */ #define ETHB_CTL_TGO (1U << 9) /* TGO: Transmitter Start */ #define ETHB_CTL_TSTP (1U << 10) /* TSTP: Transmitter Stop */ /* ETH_CFG */ #define ETH_CFG_SPD (1U << 0) /* SPD: Speed 1 == 100: 0 == 10 */ #define ETH_CFG_FD (1U << 1) /* FD: Full duplex */ #define ETH_CFG_BR (1U << 2) /* BR: Bit Rate (optional?) */ /* bit 3 reserved */ #define ETH_CFG_CAF (1U << 4) /* CAF: Copy All Frames */ #define ETH_CFG_NBC (1U << 5) /* NBC: No Broadcast */ #define ETH_CFG_MTI (1U << 6) /* MTI: Multicast Hash Enable */ #define ETH_CFG_UNI (1U << 7) /* UNI: Unicast Hash Enable */ #define ETH_CFG_BIG (1U << 8) /* BIG: Receive 1522 Bytes */ #define ETH_CFG_EAE (1U << 9) /* EAE: External Address Match En */ #define ETH_CFG_CLK_8 (0U << 10) /* CLK: Clock / 8 */ #define ETH_CFG_CLK_16 (1U << 10) /* CLK: Clock / 16 */ #define ETH_CFG_CLK_32 (2U << 10) /* CLK: Clock / 32 */ #define ETH_CFG_CLK_64 (3U << 10) /* CLK: Clock / 64 */ #define ETH_CFG_RTY (1U << 12) /* RTY: Retry Test*/ #define ETH_CFG_RMII (1U << 13) /* RMII: Reduce MII */ #define ETHB_CFG_JBO (1U << 3) /* JBO: Jumbo Frames */ #define ETHB_CFG_PAE (1U << 13) /* PAE: Pause Enable */ #define ETHB_CFG_RBOF_0 (0U << 14) /* RBOF: Rx Buffer Offset */ #define ETHB_CFG_RBOF_1 (1U << 14) /* RBOF: Rx Buffer Offset */ #define ETHB_CFG_RBOF_2 (3U << 14) /* RBOF: Rx Buffer Offset */ #define ETHB_CFG_RBOF_3 (3U << 14) /* RBOF: Rx Buffer Offset */ #define ETHB_CFG_RCLE (1U << 16) /* RCLE: Rx Length Check Enable */ #define ETHB_CFG_DRFC (1U << 17) /* DRFC: Discard Rx FCS */ #define ETHB_CFG_RHD (1U << 18) /* RHD: RX TX'ed frame in half-duplex */ #define ETHB_CFG_IFCS (1U << 19) /* IFCS: Ignore bad RX FCS */ /* ETH_SR */ #define ETH_SR_LINK (1U << 0) /* Reserved! */ #define ETH_SR_MDIO (1U << 1) /* MDIO pin status */ #define ETH_SR_IDLE (1U << 2) /* IDLE (PHY logic) */ /* ETH_TCR */ #define ETH_TCR_NCRC (1U << 15) /* NCRC: No CRC */ /* ETH_TSR */ #define ETH_TSR_OVR (1U << 0) /* OVR: Ethernet Transmit Overrun */ #define ETH_TSR_COL (1U << 1) /* COL: Collision Occurred */ #define ETH_TSR_RLE (1U << 2) /* RLE: Retry Limit Exceeded */ #define ETH_TSR_IDLE (1U << 3) /* IDLE: Transmitter Idle */ #define ETH_TSR_BNQ (1U << 4) /* BNQ: Enet Tran Buff not Queued */ #define ETH_TSR_COMP (1U << 5) /* COMP: Transmit Complete */ #define ETH_TSR_UND (1U << 6) /* UND: Transmit Underrun */ #define ETH_TSR_WR_MASK (0x67) /* write 1 to clear bits */ /* ETH_RSR */ #define ETH_RSR_BNA (1U << 0) /* BNA: Buffer Not Available */ #define ETH_RSR_REC (1U << 1) /* REC: Frame Received */ #define ETH_RSR_OVR (1U << 2) /* OVR: RX Overrun */ /* ETH_ISR */ #define ETH_ISR_DONE (1U << 0) /* DONE: Management Done */ #define ETH_ISR_RCOM (1U << 1) /* RCOM: Receive Complete */ #define ETH_ISR_RBNA (1U << 2) /* RBNA: Receive Buffer Not Avail */ #define ETH_ISR_TOVR (1U << 3) /* TOVR: Transmit Buffer Overrun */ #define ETH_ISR_TUND (1U << 4) /* TUND: Transmit Buffer Underrun */ #define ETH_ISR_RTRY (1U << 5) /* RTRY: Retry Limit */ #define ETH_ISR_TBRE (1U << 6) /* TBRE: Trasnmit Buffer Reg empty */ #define ETH_ISR_TCOM (1U << 7) /* TCOM: Transmit Complete */ #define ETH_ISR_TIDLE (1U << 8) /* TIDLE: Transmit Idle */ #define ETH_ISR_LINK (1U << 9) /* LINK: Link pin delta (optional) */ #define ETH_ISR_ROVR (1U << 10) /* ROVR: RX Overrun */ #define ETH_ISR_ABT (1U << 11) /* ABT: Abort */ /* ETHB_UIO */ #define ETHB_UIO_RMII (1U << 0) /* RMII: Reduce MII */ #define ETHB_UIO_CLKE (1U << 1) /* CLKE: Clock Enable */ /* ETH_MAN */ #define ETH_MAN_BITS 0x40020000 /* HIGH and CODE bits */ #define ETH_MAN_READ (2U << 28) #define ETH_MAN_WRITE (1U << 28) #define ETH_MAN_PHYA_BIT 23 #define ETH_MAN_REGA_BIT 18 #define ETH_MAN_VALUE_MASK 0xffffU #define ETH_MAN_REG_WR(phy, reg, val) \ (ETH_MAN_BITS | ETH_MAN_WRITE | ((phy) << ETH_MAN_PHYA_BIT) | \ ((reg) << ETH_MAN_REGA_BIT) | ((val) & ETH_MAN_VALUE_MASK)) #define ETH_MAN_REG_RD(phy, reg) \ (ETH_MAN_BITS | ETH_MAN_READ | ((phy) << ETH_MAN_PHYA_BIT) | \ ((reg) << ETH_MAN_REGA_BIT)) typedef struct { uint32_t addr; #define ETH_CPU_OWNER (1U << 0) #define ETH_WRAP_BIT (1U << 1) #define ETH_ADR_MASK ~(EHT_CPU_OWNER | ETH_WRAP_BIT) uint32_t status; #define ETH_LEN_MASK 0x7ff #define ETH_BUF_FIRST (1U << 14) /* Packet matched addr 4 */ #define ETH_BUF_LAST (1U << 15) /* Packet matched addr 4 */ #define ETH_MAC_LOCAL_4 (1U << 23) /* Packet matched addr 4 */ #define ETH_MAC_LOCAL_3 (1U << 24) /* Packet matched addr 3 */ #define ETH_MAC_LOCAL_2 (1U << 25) /* Packet matched addr 2 */ #define ETH_MAC_LOCAL_1 (1U << 26) /* Packet matched addr 1 */ -#define ETH_MAC_UNK (1U << 27) /* Unkown source address RFU */ +#define ETH_MAC_UNK (1U << 27) /* Unknown source address RFU */ #define ETH_MAC_EXT (1U << 28) /* External Address */ #define ETH_MAC_UCAST (1U << 29) /* Unicast hash match */ #define ETH_MAC_MCAST (1U << 30) /* Multicast hash match */ #define ETH_MAC_ONES (1U << 31) /* Global all ones bcast addr */ } eth_rx_desc_t; typedef struct { uint32_t addr; uint32_t status; #define ETHB_TX_LEN_MASK 0x7ff #define ETHB_TX_BUF_LAST (1U << 15) /* Last buffer in packet */ #define ETHB_TX_NOCRC (1U << 16) /* Don't xmit CRC*/ #define ETHB_TX_BUFE (1U << 27) /* Buffers exhausted mid frame */ #define ETHB_TX_TUND (1U << 28) /* Transmit Underrun */ #define ETHB_TX_RTRYE (1U << 29) /* Re-try limit exceeded */ #define ETHB_TX_WRAP (1U << 30) /* Last descritor in list */ #define ETHB_TX_USED (1U << 31) /* Packet Transmitted */ } eth_tx_desc_t; #endif /* ARM_AT91_IF_ATEREG_H */ Index: user/ngie/release-pkg-fix-tests/sys/arm/broadcom/bcm2835/bcm2835_audio.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/broadcom/bcm2835/bcm2835_audio.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/broadcom/bcm2835/bcm2835_audio.c (revision 299091) @@ -1,895 +1,895 @@ /*- * Copyright (c) 2015 Oleksandr Tymoshenko * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifdef HAVE_KERNEL_OPTION_HEADERS #include "opt_snd.h" #endif #include #include #include "mixer_if.h" #include "interface/compat/vchi_bsd.h" #include "interface/vchi/vchi.h" #include "interface/vchiq_arm/vchiq.h" #include "vc_vchi_audioserv_defs.h" SND_DECLARE_FILE("$FreeBSD$"); #define DEST_AUTO 0 #define DEST_HEADPHONES 1 #define DEST_HDMI 2 #define VCHIQ_AUDIO_PACKET_SIZE 4000 #define VCHIQ_AUDIO_BUFFER_SIZE 128000 #define VCHIQ_AUDIO_PREBUFFER 10 /* Number of pre-buffered audio messages */ #define VCHIQ_AUDIO_MAX_VOLUME /* volume in terms of 0.01dB */ #define VCHIQ_AUDIO_VOLUME_MIN -10239 #define VCHIQ_AUDIO_VOLUME(db100) (uint32_t)(-((db100) << 8)/100) /* dB levels with 5% volume step */ static int db_levels[] = { VCHIQ_AUDIO_VOLUME_MIN, -4605, -3794, -3218, -2772, -2407, -2099, -1832, -1597, -1386, -1195, -1021, -861, -713, -575, -446, -325, -210, -102, 0, }; static uint32_t bcm2835_audio_playfmt[] = { SND_FORMAT(AFMT_U8, 1, 0), SND_FORMAT(AFMT_U8, 2, 0), SND_FORMAT(AFMT_S8, 1, 0), SND_FORMAT(AFMT_S8, 2, 0), SND_FORMAT(AFMT_S16_LE, 1, 0), SND_FORMAT(AFMT_S16_LE, 2, 0), SND_FORMAT(AFMT_U16_LE, 1, 0), SND_FORMAT(AFMT_U16_LE, 2, 0), 0 }; static struct pcmchan_caps bcm2835_audio_playcaps = {8000, 48000, bcm2835_audio_playfmt, 0}; struct bcm2835_audio_info; #define PLAYBACK_IDLE 0 #define PLAYBACK_STARTING 1 #define PLAYBACK_PLAYING 2 #define PLAYBACK_STOPPING 3 struct bcm2835_audio_chinfo { struct bcm2835_audio_info *parent; struct pcm_channel *channel; struct snd_dbuf *buffer; uint32_t fmt, spd, blksz; uint32_t complete_pos; uint32_t free_buffer; uint32_t buffered_ptr; int playback_state; int prebuffered; }; struct bcm2835_audio_info { device_t dev; unsigned int bufsz; struct bcm2835_audio_chinfo pch; uint32_t dest, volume; struct mtx *lock; struct intr_config_hook intr_hook; /* VCHI data */ struct mtx vchi_lock; VCHI_INSTANCE_T vchi_instance; VCHI_CONNECTION_T *vchi_connection; VCHI_SERVICE_HANDLE_T vchi_handle; struct mtx data_lock; struct cv data_cv; /* Unloadign module */ int unloading; }; #define bcm2835_audio_lock(_ess) snd_mtxlock((_ess)->lock) #define bcm2835_audio_unlock(_ess) snd_mtxunlock((_ess)->lock) #define bcm2835_audio_lock_assert(_ess) snd_mtxassert((_ess)->lock) #define VCHIQ_VCHI_LOCK(sc) mtx_lock(&(sc)->vchi_lock) #define VCHIQ_VCHI_UNLOCK(sc) mtx_unlock(&(sc)->vchi_lock) static const char * dest_description(uint32_t dest) { switch (dest) { case DEST_AUTO: return "AUTO"; break; case DEST_HEADPHONES: return "HEADPHONES"; break; case DEST_HDMI: return "HDMI"; break; default: return "UNKNOWN"; break; } } static void bcm2835_audio_callback(void *param, const VCHI_CALLBACK_REASON_T reason, void *msg_handle) { struct bcm2835_audio_info *sc = (struct bcm2835_audio_info *)param; int32_t status; uint32_t msg_len; VC_AUDIO_MSG_T m; if (reason != VCHI_CALLBACK_MSG_AVAILABLE) return; status = vchi_msg_dequeue(sc->vchi_handle, &m, sizeof m, &msg_len, VCHI_FLAGS_NONE); if (m.type == VC_AUDIO_MSG_TYPE_RESULT) { if (m.u.result.success) { device_printf(sc->dev, "msg type %08x failed\n", m.type); } } else if (m.type == VC_AUDIO_MSG_TYPE_COMPLETE) { struct bcm2835_audio_chinfo *ch = m.u.complete.cookie; int count = m.u.complete.count & 0xffff; int perr = (m.u.complete.count & (1U << 30)) != 0; ch->complete_pos = (ch->complete_pos + count) % sndbuf_getsize(ch->buffer); ch->free_buffer += count; chn_intr(sc->pch.channel); if (perr || ch->free_buffer >= VCHIQ_AUDIO_PACKET_SIZE) cv_signal(&sc->data_cv); } else printf("%s: unknown m.type: %d\n", __func__, m.type); } /* VCHIQ stuff */ static void bcm2835_audio_init(struct bcm2835_audio_info *sc) { int status; /* Initialize and create a VCHI connection */ status = vchi_initialise(&sc->vchi_instance); if (status != 0) { printf("vchi_initialise failed: %d\n", status); return; } status = vchi_connect(NULL, 0, sc->vchi_instance); if (status != 0) { printf("vchi_connect failed: %d\n", status); return; } SERVICE_CREATION_T params = { VCHI_VERSION_EX(VC_AUDIOSERV_VER, VC_AUDIOSERV_MIN_VER), VC_AUDIO_SERVER_NAME, /* 4cc service code */ sc->vchi_connection, /* passed in fn pointers */ 0, /* rx fifo size */ 0, /* tx fifo size */ bcm2835_audio_callback, /* service callback */ sc, /* service callback parameter */ 1, 1, 0 /* want crc check on bulk transfers */ }; status = vchi_service_open(sc->vchi_instance, ¶ms, &sc->vchi_handle); if (status == 0) /* Finished with the service for now */ vchi_service_release(sc->vchi_handle); else sc->vchi_handle = VCHIQ_SERVICE_HANDLE_INVALID; } static void bcm2835_audio_release(struct bcm2835_audio_info *sc) { int success; if (sc->vchi_handle != VCHIQ_SERVICE_HANDLE_INVALID) { vchi_service_use(sc->vchi_handle); success = vchi_service_close(sc->vchi_handle); if (success != 0) printf("vchi_service_close failed: %d\n", success); sc->vchi_handle = VCHIQ_SERVICE_HANDLE_INVALID; } vchi_disconnect(sc->vchi_instance); } static void bcm2835_audio_reset_channel(struct bcm2835_audio_chinfo *ch) { ch->free_buffer = VCHIQ_AUDIO_BUFFER_SIZE; ch->playback_state = 0; ch->buffered_ptr = 0; ch->complete_pos = 0; ch->prebuffered = 0; sndbuf_reset(ch->buffer); } static void bcm2835_audio_start(struct bcm2835_audio_chinfo *ch) { VC_AUDIO_MSG_T m; int ret; struct bcm2835_audio_info *sc = ch->parent; VCHIQ_VCHI_LOCK(sc); if (sc->vchi_handle != VCHIQ_SERVICE_HANDLE_INVALID) { vchi_service_use(sc->vchi_handle); bcm2835_audio_reset_channel(ch); m.type = VC_AUDIO_MSG_TYPE_START; ret = vchi_msg_queue(sc->vchi_handle, &m, sizeof m, VCHI_FLAGS_BLOCK_UNTIL_QUEUED, NULL); if (ret != 0) printf("%s: vchi_msg_queue failed (err %d)\n", __func__, ret); vchi_service_release(sc->vchi_handle); } VCHIQ_VCHI_UNLOCK(sc); } static void bcm2835_audio_stop(struct bcm2835_audio_chinfo *ch) { VC_AUDIO_MSG_T m; int ret; struct bcm2835_audio_info *sc = ch->parent; VCHIQ_VCHI_LOCK(sc); if (sc->vchi_handle != VCHIQ_SERVICE_HANDLE_INVALID) { vchi_service_use(sc->vchi_handle); m.type = VC_AUDIO_MSG_TYPE_STOP; m.u.stop.draining = 0; ret = vchi_msg_queue(sc->vchi_handle, &m, sizeof m, VCHI_FLAGS_BLOCK_UNTIL_QUEUED, NULL); if (ret != 0) printf("%s: vchi_msg_queue failed (err %d)\n", __func__, ret); vchi_service_release(sc->vchi_handle); } VCHIQ_VCHI_UNLOCK(sc); } static void bcm2835_audio_open(struct bcm2835_audio_info *sc) { VC_AUDIO_MSG_T m; int ret; VCHIQ_VCHI_LOCK(sc); if (sc->vchi_handle != VCHIQ_SERVICE_HANDLE_INVALID) { vchi_service_use(sc->vchi_handle); m.type = VC_AUDIO_MSG_TYPE_OPEN; ret = vchi_msg_queue(sc->vchi_handle, &m, sizeof m, VCHI_FLAGS_BLOCK_UNTIL_QUEUED, NULL); if (ret != 0) printf("%s: vchi_msg_queue failed (err %d)\n", __func__, ret); vchi_service_release(sc->vchi_handle); } VCHIQ_VCHI_UNLOCK(sc); } static void bcm2835_audio_update_controls(struct bcm2835_audio_info *sc) { VC_AUDIO_MSG_T m; int ret, db; VCHIQ_VCHI_LOCK(sc); if (sc->vchi_handle != VCHIQ_SERVICE_HANDLE_INVALID) { vchi_service_use(sc->vchi_handle); m.type = VC_AUDIO_MSG_TYPE_CONTROL; m.u.control.dest = sc->dest; if (sc->volume > 99) sc->volume = 99; db = db_levels[sc->volume/5]; m.u.control.volume = VCHIQ_AUDIO_VOLUME(db); ret = vchi_msg_queue(sc->vchi_handle, &m, sizeof m, VCHI_FLAGS_BLOCK_UNTIL_QUEUED, NULL); if (ret != 0) printf("%s: vchi_msg_queue failed (err %d)\n", __func__, ret); vchi_service_release(sc->vchi_handle); } VCHIQ_VCHI_UNLOCK(sc); } static void bcm2835_audio_update_params(struct bcm2835_audio_info *sc, struct bcm2835_audio_chinfo *ch) { VC_AUDIO_MSG_T m; int ret; VCHIQ_VCHI_LOCK(sc); if (sc->vchi_handle != VCHIQ_SERVICE_HANDLE_INVALID) { vchi_service_use(sc->vchi_handle); m.type = VC_AUDIO_MSG_TYPE_CONFIG; m.u.config.channels = AFMT_CHANNEL(ch->fmt); m.u.config.samplerate = ch->spd; m.u.config.bps = AFMT_BIT(ch->fmt); ret = vchi_msg_queue(sc->vchi_handle, &m, sizeof m, VCHI_FLAGS_BLOCK_UNTIL_QUEUED, NULL); if (ret != 0) printf("%s: vchi_msg_queue failed (err %d)\n", __func__, ret); vchi_service_release(sc->vchi_handle); } VCHIQ_VCHI_UNLOCK(sc); } static __inline uint32_t vchiq_unbuffered_bytes(struct bcm2835_audio_chinfo *ch) { uint32_t size, ready, readyptr, readyend; size = sndbuf_getsize(ch->buffer); readyptr = sndbuf_getreadyptr(ch->buffer); ready = sndbuf_getready(ch->buffer); readyend = readyptr + ready; /* Normal case */ if (ch->buffered_ptr >= readyptr) { if (readyend > ch->buffered_ptr) return readyend - ch->buffered_ptr; else return 0; } else { /* buffered_ptr overflow */ if (readyend > ch->buffered_ptr + size) return readyend - ch->buffered_ptr - size; else return 0; } } static void bcm2835_audio_write_samples(struct bcm2835_audio_chinfo *ch) { struct bcm2835_audio_info *sc = ch->parent; VC_AUDIO_MSG_T m; void *buf; uint32_t count, size; int ret; VCHIQ_VCHI_LOCK(sc); if (sc->vchi_handle == VCHIQ_SERVICE_HANDLE_INVALID) { VCHIQ_VCHI_UNLOCK(sc); return; } vchi_service_use(sc->vchi_handle); size = sndbuf_getsize(ch->buffer); count = vchiq_unbuffered_bytes(ch); buf = (uint8_t*)sndbuf_getbuf(ch->buffer) + ch->buffered_ptr; if (ch->buffered_ptr + count > size) count = size - ch->buffered_ptr; if (count < VCHIQ_AUDIO_PACKET_SIZE) goto done; count = min(count, ch->free_buffer); count -= count % VCHIQ_AUDIO_PACKET_SIZE; m.type = VC_AUDIO_MSG_TYPE_WRITE; m.u.write.count = count; m.u.write.max_packet = VCHIQ_AUDIO_PACKET_SIZE; m.u.write.callback = NULL; m.u.write.cookie = ch; if (buf) m.u.write.silence = 0; else m.u.write.silence = 1; ret = vchi_msg_queue(sc->vchi_handle, &m, sizeof m, VCHI_FLAGS_BLOCK_UNTIL_QUEUED, NULL); if (ret != 0) printf("%s: vchi_msg_queue failed (err %d)\n", __func__, ret); if (buf) { while (count > 0) { int bytes = MIN((int)m.u.write.max_packet, (int)count); ch->free_buffer -= bytes; ch->buffered_ptr += bytes; ch->buffered_ptr = ch->buffered_ptr % size; ret = vchi_msg_queue(sc->vchi_handle, buf, bytes, VCHI_FLAGS_BLOCK_UNTIL_QUEUED, NULL); if (ret != 0) printf("%s: vchi_msg_queue failed: %d\n", __func__, ret); buf = (char *)buf + bytes; count -= bytes; } } done: vchi_service_release(sc->vchi_handle); VCHIQ_VCHI_UNLOCK(sc); } static void bcm2835_audio_worker(void *data) { struct bcm2835_audio_info *sc = (struct bcm2835_audio_info *)data; struct bcm2835_audio_chinfo *ch = &sc->pch; mtx_lock(&sc->data_lock); while(1) { if (sc->unloading) break; if (ch->playback_state == PLAYBACK_IDLE) { cv_wait_sig(&sc->data_cv, &sc->data_lock); continue; } if (ch->playback_state == PLAYBACK_STOPPING) { bcm2835_audio_reset_channel(&sc->pch); ch->playback_state = PLAYBACK_IDLE; continue; } if (ch->free_buffer < vchiq_unbuffered_bytes(ch)) { cv_timedwait_sig(&sc->data_cv, &sc->data_lock, 10); continue; } bcm2835_audio_write_samples(ch); if (ch->playback_state == PLAYBACK_STARTING) { ch->prebuffered++; if (ch->prebuffered == VCHIQ_AUDIO_PREBUFFER) { bcm2835_audio_start(ch); ch->playback_state = PLAYBACK_PLAYING; } } } mtx_unlock(&sc->data_lock); kproc_exit(0); } static void bcm2835_audio_create_worker(struct bcm2835_audio_info *sc) { struct proc *newp; if (kproc_create(bcm2835_audio_worker, (void*)sc, &newp, 0, 0, "bcm2835_audio_worker") != 0) { printf("failed to create bcm2835_audio_worker\n"); } } /* -------------------------------------------------------------------- */ /* channel interface for VCHI audio */ static void * bcmchan_init(kobj_t obj, void *devinfo, struct snd_dbuf *b, struct pcm_channel *c, int dir) { struct bcm2835_audio_info *sc = devinfo; struct bcm2835_audio_chinfo *ch = &sc->pch; void *buffer; if (dir == PCMDIR_REC) return NULL; ch->parent = sc; ch->channel = c; ch->buffer = b; /* default values */ ch->spd = 44100; ch->fmt = SND_FORMAT(AFMT_S16_LE, 2, 0); ch->blksz = VCHIQ_AUDIO_PACKET_SIZE; buffer = malloc(sc->bufsz, M_DEVBUF, M_WAITOK | M_ZERO); if (sndbuf_setup(ch->buffer, buffer, sc->bufsz) != 0) { free(buffer, M_DEVBUF); return NULL; } bcm2835_audio_update_params(sc, ch); return ch; } static int bcmchan_free(kobj_t obj, void *data) { struct bcm2835_audio_chinfo *ch = data; void *buffer; buffer = sndbuf_getbuf(ch->buffer); if (buffer) free(buffer, M_DEVBUF); return (0); } static int bcmchan_setformat(kobj_t obj, void *data, uint32_t format) { struct bcm2835_audio_chinfo *ch = data; struct bcm2835_audio_info *sc = ch->parent; bcm2835_audio_lock(sc); ch->fmt = format; bcm2835_audio_update_params(sc, ch); bcm2835_audio_unlock(sc); return 0; } static uint32_t bcmchan_setspeed(kobj_t obj, void *data, uint32_t speed) { struct bcm2835_audio_chinfo *ch = data; struct bcm2835_audio_info *sc = ch->parent; bcm2835_audio_lock(sc); ch->spd = speed; bcm2835_audio_update_params(sc, ch); bcm2835_audio_unlock(sc); return ch->spd; } static uint32_t bcmchan_setblocksize(kobj_t obj, void *data, uint32_t blocksize) { struct bcm2835_audio_chinfo *ch = data; return ch->blksz; } static int bcmchan_trigger(kobj_t obj, void *data, int go) { struct bcm2835_audio_chinfo *ch = data; struct bcm2835_audio_info *sc = ch->parent; if (!PCMTRIG_COMMON(go)) return (0); bcm2835_audio_lock(sc); switch (go) { case PCMTRIG_START: ch->playback_state = PLAYBACK_STARTING; /* wakeup worker thread */ cv_signal(&sc->data_cv); break; case PCMTRIG_STOP: case PCMTRIG_ABORT: ch->playback_state = PLAYBACK_STOPPING; bcm2835_audio_stop(ch); break; default: break; } bcm2835_audio_unlock(sc); return 0; } static uint32_t bcmchan_getptr(kobj_t obj, void *data) { struct bcm2835_audio_chinfo *ch = data; struct bcm2835_audio_info *sc = ch->parent; uint32_t ret; bcm2835_audio_lock(sc); ret = ch->complete_pos - (ch->complete_pos % VCHIQ_AUDIO_PACKET_SIZE); bcm2835_audio_unlock(sc); return ret; } static struct pcmchan_caps * bcmchan_getcaps(kobj_t obj, void *data) { return &bcm2835_audio_playcaps; } static kobj_method_t bcmchan_methods[] = { KOBJMETHOD(channel_init, bcmchan_init), KOBJMETHOD(channel_free, bcmchan_free), KOBJMETHOD(channel_setformat, bcmchan_setformat), KOBJMETHOD(channel_setspeed, bcmchan_setspeed), KOBJMETHOD(channel_setblocksize, bcmchan_setblocksize), KOBJMETHOD(channel_trigger, bcmchan_trigger), KOBJMETHOD(channel_getptr, bcmchan_getptr), KOBJMETHOD(channel_getcaps, bcmchan_getcaps), KOBJMETHOD_END }; CHANNEL_DECLARE(bcmchan); /************************************************************/ static int bcmmix_init(struct snd_mixer *m) { mix_setdevs(m, SOUND_MASK_VOLUME); return (0); } static int bcmmix_set(struct snd_mixer *m, unsigned dev, unsigned left, unsigned right) { struct bcm2835_audio_info *sc = mix_getdevinfo(m); switch (dev) { case SOUND_MIXER_VOLUME: sc->volume = left; bcm2835_audio_update_controls(sc); break; default: break; } return left | (left << 8); } static kobj_method_t bcmmixer_methods[] = { KOBJMETHOD(mixer_init, bcmmix_init), KOBJMETHOD(mixer_set, bcmmix_set), KOBJMETHOD_END }; MIXER_DECLARE(bcmmixer); static int sysctl_bcm2835_audio_dest(SYSCTL_HANDLER_ARGS) { struct bcm2835_audio_info *sc = arg1; int val; int err; val = sc->dest; err = sysctl_handle_int(oidp, &val, 0, req); if (err || !req->newptr) /* error || read request */ return (err); if ((val < 0) || (val > 2)) return (EINVAL); sc->dest = val; device_printf(sc->dev, "destination set to %s\n", dest_description(val)); bcm2835_audio_update_controls(sc); return (0); } static void vchi_audio_sysctl_init(struct bcm2835_audio_info *sc) { struct sysctl_ctx_list *ctx; struct sysctl_oid *tree_node; struct sysctl_oid_list *tree; /* * Add system sysctl tree/handlers. */ ctx = device_get_sysctl_ctx(sc->dev); tree_node = device_get_sysctl_tree(sc->dev); tree = SYSCTL_CHILDREN(tree_node); SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "dest", CTLFLAG_RW | CTLTYPE_UINT, sc, sizeof(*sc), sysctl_bcm2835_audio_dest, "IU", "audio destination, " "0 - auto, 1 - headphones, 2 - HDMI"); } static void bcm2835_audio_identify(driver_t *driver, device_t parent) { BUS_ADD_CHILD(parent, 0, "pcm", 0); } static int bcm2835_audio_probe(device_t dev) { device_set_desc(dev, "VCHIQ audio"); return (BUS_PROBE_DEFAULT); } static void bcm2835_audio_delayed_init(void *xsc) { struct bcm2835_audio_info *sc; char status[SND_STATUSLEN]; sc = xsc; config_intrhook_disestablish(&sc->intr_hook); bcm2835_audio_init(sc); bcm2835_audio_open(sc); sc->volume = 75; sc->dest = DEST_AUTO; if (mixer_init(sc->dev, &bcmmixer_class, sc)) { device_printf(sc->dev, "mixer_init failed\n"); goto no; } if (pcm_register(sc->dev, sc, 1, 1)) { device_printf(sc->dev, "pcm_register failed\n"); goto no; } pcm_addchan(sc->dev, PCMDIR_PLAY, &bcmchan_class, sc); snprintf(status, SND_STATUSLEN, "at VCHIQ"); pcm_setstatus(sc->dev, status); bcm2835_audio_reset_channel(&sc->pch); bcm2835_audio_create_worker(sc); vchi_audio_sysctl_init(sc); no: ; } static int bcm2835_audio_attach(device_t dev) { struct bcm2835_audio_info *sc; sc = malloc(sizeof(*sc), M_DEVBUF, M_WAITOK | M_ZERO); sc->dev = dev; sc->bufsz = VCHIQ_AUDIO_BUFFER_SIZE; sc->lock = snd_mtxcreate(device_get_nameunit(dev), "bcm2835_audio softc"); mtx_init(&sc->vchi_lock, "bcm2835_audio", "vchi_lock", MTX_DEF); mtx_init(&sc->data_lock, "data_mtx", "data_mtx", MTX_DEF); cv_init(&sc->data_cv, "data_cv"); sc->vchi_handle = VCHIQ_SERVICE_HANDLE_INVALID; /* * We need interrupts enabled for VCHI to work properly, - * so delay intialization until it happens + * so delay initialization until it happens. */ sc->intr_hook.ich_func = bcm2835_audio_delayed_init; sc->intr_hook.ich_arg = sc; if (config_intrhook_establish(&sc->intr_hook) != 0) goto no; return 0; no: return ENXIO; } static int bcm2835_audio_detach(device_t dev) { int r; struct bcm2835_audio_info *sc; sc = pcm_getdevinfo(dev); /* Stop worker thread */ sc->unloading = 1; cv_signal(&sc->data_cv); r = pcm_unregister(dev); if (r) return r; mtx_destroy(&sc->vchi_lock); mtx_destroy(&sc->data_lock); cv_destroy(&sc->data_cv); bcm2835_audio_release(sc); if (sc->lock) { snd_mtxfree(sc->lock); sc->lock = NULL; } free(sc, M_DEVBUF); return 0; } static device_method_t bcm2835_audio_methods[] = { /* Device interface */ DEVMETHOD(device_identify, bcm2835_audio_identify), DEVMETHOD(device_probe, bcm2835_audio_probe), DEVMETHOD(device_attach, bcm2835_audio_attach), DEVMETHOD(device_detach, bcm2835_audio_detach), { 0, 0 } }; static driver_t bcm2835_audio_driver = { "pcm", bcm2835_audio_methods, PCM_SOFTC_SIZE, }; DRIVER_MODULE(bcm2835_audio, vchiq, bcm2835_audio_driver, pcm_devclass, 0, 0); MODULE_DEPEND(bcm2835_audio, sound, SOUND_MINVER, SOUND_PREFVER, SOUND_MAXVER); MODULE_DEPEND(bcm2835_audio, vchiq, 1, 1, 1); MODULE_VERSION(bcm2835_audio, 1); Index: user/ngie/release-pkg-fix-tests/sys/arm/broadcom/bcm2835/bcm2835_dma.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/broadcom/bcm2835/bcm2835_dma.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/broadcom/bcm2835/bcm2835_dma.c (revision 299091) @@ -1,766 +1,766 @@ /* * Copyright (c) 2013 Daisuke Aoyama * Copyright (c) 2013 Oleksandr Tymoshenko * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "bcm2835_dma.h" #include "bcm2835_vcbus.h" #define MAX_REG 9 /* private flags */ #define BCM_DMA_CH_USED 0x00000001 #define BCM_DMA_CH_FREE 0x40000000 #define BCM_DMA_CH_UNMAP 0x80000000 /* Register Map (4.2.1.2) */ #define BCM_DMA_CS(n) (0x100*(n) + 0x00) #define CS_ACTIVE (1 << 0) #define CS_END (1 << 1) #define CS_INT (1 << 2) #define CS_DREQ (1 << 3) #define CS_ISPAUSED (1 << 4) #define CS_ISHELD (1 << 5) #define CS_ISWAIT (1 << 6) #define CS_ERR (1 << 8) #define CS_WAITWRT (1 << 28) #define CS_DISDBG (1 << 29) #define CS_ABORT (1 << 30) #define CS_RESET (1U << 31) #define BCM_DMA_CBADDR(n) (0x100*(n) + 0x04) #define BCM_DMA_INFO(n) (0x100*(n) + 0x08) #define INFO_INT_EN (1 << 0) #define INFO_TDMODE (1 << 1) #define INFO_WAIT_RESP (1 << 3) #define INFO_D_INC (1 << 4) #define INFO_D_WIDTH (1 << 5) #define INFO_D_DREQ (1 << 6) #define INFO_S_INC (1 << 8) #define INFO_S_WIDTH (1 << 9) #define INFO_S_DREQ (1 << 10) #define INFO_WAITS_SHIFT (21) #define INFO_PERMAP_SHIFT (16) #define INFO_PERMAP_MASK (0x1f << INFO_PERMAP_SHIFT) #define BCM_DMA_SRC(n) (0x100*(n) + 0x0C) #define BCM_DMA_DST(n) (0x100*(n) + 0x10) #define BCM_DMA_LEN(n) (0x100*(n) + 0x14) #define BCM_DMA_STRIDE(n) (0x100*(n) + 0x18) #define BCM_DMA_CBNEXT(n) (0x100*(n) + 0x1C) #define BCM_DMA_DEBUG(n) (0x100*(n) + 0x20) #define DEBUG_ERROR_MASK (7) #define BCM_DMA_INT_STATUS 0xfe0 #define BCM_DMA_ENABLE 0xff0 /* relative offset from BCM_VC_DMA0_BASE (p.39) */ #define BCM_DMA_CH(n) (0x100*(n)) /* channels used by GPU */ #define BCM_DMA_CH_BULK 0 #define BCM_DMA_CH_FAST1 2 #define BCM_DMA_CH_FAST2 3 #define BCM_DMA_CH_GPU_MASK ((1 << BCM_DMA_CH_BULK) | \ (1 << BCM_DMA_CH_FAST1) | \ (1 << BCM_DMA_CH_FAST2)) /* DMA Control Block - 256bit aligned (p.40) */ struct bcm_dma_cb { uint32_t info; /* Transfer Information */ uint32_t src; /* Source Address */ uint32_t dst; /* Destination Address */ uint32_t len; /* Transfer Length */ uint32_t stride; /* 2D Mode Stride */ uint32_t next; /* Next Control Block Address */ uint32_t rsvd1; /* Reserved */ uint32_t rsvd2; /* Reserved */ }; #ifdef DEBUG static void bcm_dma_cb_dump(struct bcm_dma_cb *cb); static void bcm_dma_reg_dump(int ch); #endif /* DMA channel private info */ struct bcm_dma_ch { int ch; uint32_t flags; struct bcm_dma_cb * cb; uint32_t vc_cb; bus_dmamap_t dma_map; void (*intr_func)(int, void *); void * intr_arg; }; struct bcm_dma_softc { device_t sc_dev; struct mtx sc_mtx; struct resource * sc_mem; struct resource * sc_irq[BCM_DMA_CH_MAX]; void * sc_intrhand[BCM_DMA_CH_MAX]; struct bcm_dma_ch sc_dma_ch[BCM_DMA_CH_MAX]; bus_dma_tag_t sc_dma_tag; }; static struct bcm_dma_softc *bcm_dma_sc = NULL; static uint32_t bcm_dma_channel_mask; static void bcm_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int err) { bus_addr_t *addr; if (err) return; addr = (bus_addr_t*)arg; *addr = PHYS_TO_VCBUS(segs[0].ds_addr); } static void bcm_dma_reset(device_t dev, int ch) { struct bcm_dma_softc *sc = device_get_softc(dev); struct bcm_dma_cb *cb; uint32_t cs; int count; if (ch < 0 || ch >= BCM_DMA_CH_MAX) return; cs = bus_read_4(sc->sc_mem, BCM_DMA_CS(ch)); if (cs & CS_ACTIVE) { /* pause current task */ bus_write_4(sc->sc_mem, BCM_DMA_CS(ch), 0); count = 1000; do { cs = bus_read_4(sc->sc_mem, BCM_DMA_CS(ch)); } while (!(cs & CS_ISPAUSED) && (count-- > 0)); if (!(cs & CS_ISPAUSED)) { device_printf(dev, "Can't abort DMA transfer at channel %d\n", ch); } bus_write_4(sc->sc_mem, BCM_DMA_CBNEXT(ch), 0); /* Complete everything, clear interrupt */ bus_write_4(sc->sc_mem, BCM_DMA_CS(ch), CS_ABORT | CS_INT | CS_END| CS_ACTIVE); } /* clear control blocks */ bus_write_4(sc->sc_mem, BCM_DMA_CBADDR(ch), 0); bus_write_4(sc->sc_mem, BCM_DMA_CBNEXT(ch), 0); /* Reset control block */ cb = sc->sc_dma_ch[ch].cb; bzero(cb, sizeof(*cb)); cb->info = INFO_WAIT_RESP; } static int bcm_dma_init(device_t dev) { struct bcm_dma_softc *sc = device_get_softc(dev); uint32_t reg; struct bcm_dma_ch *ch; void *cb_virt; vm_paddr_t cb_phys; int err; int i; /* * Only channels set in bcm_dma_channel_mask can be controlled by us. * The others are out of our control as well as the corresponding bits * in both BCM_DMA_ENABLE and BCM_DMA_INT_STATUS global registers. As * these registers are RW ones, there is no safe way how to write only * the bits which can be controlled by us. * * Fortunately, after reset, all channels are enabled in BCM_DMA_ENABLE * register and all statuses are cleared in BCM_DMA_INT_STATUS one. * Not touching these registers is a trade off between correct * initialization which does not count on anything and not messing up * something we have no control over. */ reg = bus_read_4(sc->sc_mem, BCM_DMA_ENABLE); if ((reg & bcm_dma_channel_mask) != bcm_dma_channel_mask) device_printf(dev, "channels are not enabled\n"); reg = bus_read_4(sc->sc_mem, BCM_DMA_INT_STATUS); if ((reg & bcm_dma_channel_mask) != 0) device_printf(dev, "statuses are not cleared\n"); /* Allocate DMA chunks control blocks */ /* p.40 of spec - control block should be 32-bit aligned */ err = bus_dma_tag_create(bus_get_dma_tag(dev), 1, 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, sizeof(struct bcm_dma_cb), 1, sizeof(struct bcm_dma_cb), BUS_DMA_ALLOCNOW, NULL, NULL, &sc->sc_dma_tag); if (err) { device_printf(dev, "failed allocate DMA tag\n"); return (err); } /* setup initial settings */ for (i = 0; i < BCM_DMA_CH_MAX; i++) { ch = &sc->sc_dma_ch[i]; bzero(ch, sizeof(struct bcm_dma_ch)); ch->ch = i; ch->flags = BCM_DMA_CH_UNMAP; if ((bcm_dma_channel_mask & (1 << i)) == 0) continue; err = bus_dmamem_alloc(sc->sc_dma_tag, &cb_virt, BUS_DMA_WAITOK | BUS_DMA_COHERENT | BUS_DMA_ZERO, &ch->dma_map); if (err) { device_printf(dev, "cannot allocate DMA memory\n"); break; } /* * Least alignment for busdma-allocated stuff is cache - * line size, so just make sure nothing stupid happend + * line size, so just make sure nothing stupid happened * and we got properly aligned address */ if ((uintptr_t)cb_virt & 0x1f) { device_printf(dev, "DMA address is not 32-bytes aligned: %p\n", (void*)cb_virt); break; } err = bus_dmamap_load(sc->sc_dma_tag, ch->dma_map, cb_virt, sizeof(struct bcm_dma_cb), bcm_dmamap_cb, &cb_phys, BUS_DMA_WAITOK); if (err) { device_printf(dev, "cannot load DMA memory\n"); break; } ch->cb = cb_virt; ch->vc_cb = cb_phys; ch->flags = BCM_DMA_CH_FREE; ch->cb->info = INFO_WAIT_RESP; /* reset DMA engine */ bus_write_4(sc->sc_mem, BCM_DMA_CS(i), CS_RESET); } return (0); } /* * Allocate DMA channel for further use, returns channel # or * BCM_DMA_CH_INVALID */ int bcm_dma_allocate(int req_ch) { struct bcm_dma_softc *sc = bcm_dma_sc; int ch = BCM_DMA_CH_INVALID; int i; if (req_ch >= BCM_DMA_CH_MAX) return (BCM_DMA_CH_INVALID); /* Auto(req_ch < 0) or CH specified */ mtx_lock(&sc->sc_mtx); if (req_ch < 0) { for (i = 0; i < BCM_DMA_CH_MAX; i++) { if (sc->sc_dma_ch[i].flags & BCM_DMA_CH_FREE) { ch = i; sc->sc_dma_ch[ch].flags &= ~BCM_DMA_CH_FREE; sc->sc_dma_ch[ch].flags |= BCM_DMA_CH_USED; break; } } } else { if (sc->sc_dma_ch[req_ch].flags & BCM_DMA_CH_FREE) { ch = req_ch; sc->sc_dma_ch[ch].flags &= ~BCM_DMA_CH_FREE; sc->sc_dma_ch[ch].flags |= BCM_DMA_CH_USED; } } mtx_unlock(&sc->sc_mtx); return (ch); } /* * Frees allocated channel. Returns 0 on success, -1 otherwise */ int bcm_dma_free(int ch) { struct bcm_dma_softc *sc = bcm_dma_sc; if (ch < 0 || ch >= BCM_DMA_CH_MAX) return (-1); mtx_lock(&sc->sc_mtx); if (sc->sc_dma_ch[ch].flags & BCM_DMA_CH_USED) { sc->sc_dma_ch[ch].flags |= BCM_DMA_CH_FREE; sc->sc_dma_ch[ch].flags &= ~BCM_DMA_CH_USED; sc->sc_dma_ch[ch].intr_func = NULL; sc->sc_dma_ch[ch].intr_arg = NULL; /* reset DMA engine */ bcm_dma_reset(sc->sc_dev, ch); } mtx_unlock(&sc->sc_mtx); return (0); } /* * Assign handler function for channel interrupt * Returns 0 on success, -1 otherwise */ int bcm_dma_setup_intr(int ch, void (*func)(int, void *), void *arg) { struct bcm_dma_softc *sc = bcm_dma_sc; struct bcm_dma_cb *cb; if (ch < 0 || ch >= BCM_DMA_CH_MAX) return (-1); if (!(sc->sc_dma_ch[ch].flags & BCM_DMA_CH_USED)) return (-1); sc->sc_dma_ch[ch].intr_func = func; sc->sc_dma_ch[ch].intr_arg = arg; cb = sc->sc_dma_ch[ch].cb; cb->info |= INFO_INT_EN; return (0); } /* * Setup DMA source parameters * ch - channel number * dreq - hardware DREQ # or BCM_DMA_DREQ_NONE if * source is physical memory * inc_addr - BCM_DMA_INC_ADDR if source address * should be increased after each access or * BCM_DMA_SAME_ADDR if address should remain * the same * width - size of read operation, BCM_DMA_32BIT * for 32bit bursts, BCM_DMA_128BIT for 128 bits * * Returns 0 on success, -1 otherwise */ int bcm_dma_setup_src(int ch, int dreq, int inc_addr, int width) { struct bcm_dma_softc *sc = bcm_dma_sc; uint32_t info; if (ch < 0 || ch >= BCM_DMA_CH_MAX) return (-1); if (!(sc->sc_dma_ch[ch].flags & BCM_DMA_CH_USED)) return (-1); info = sc->sc_dma_ch[ch].cb->info; info &= ~INFO_PERMAP_MASK; info |= (dreq << INFO_PERMAP_SHIFT) & INFO_PERMAP_MASK; if (dreq) info |= INFO_S_DREQ; else info &= ~INFO_S_DREQ; if (width == BCM_DMA_128BIT) info |= INFO_S_WIDTH; else info &= ~INFO_S_WIDTH; if (inc_addr == BCM_DMA_INC_ADDR) info |= INFO_S_INC; else info &= ~INFO_S_INC; sc->sc_dma_ch[ch].cb->info = info; return (0); } /* * Setup DMA destination parameters * ch - channel number * dreq - hardware DREQ # or BCM_DMA_DREQ_NONE if * destination is physical memory * inc_addr - BCM_DMA_INC_ADDR if source address * should be increased after each access or * BCM_DMA_SAME_ADDR if address should remain * the same * width - size of write operation, BCM_DMA_32BIT * for 32bit bursts, BCM_DMA_128BIT for 128 bits * * Returns 0 on success, -1 otherwise */ int bcm_dma_setup_dst(int ch, int dreq, int inc_addr, int width) { struct bcm_dma_softc *sc = bcm_dma_sc; uint32_t info; if (ch < 0 || ch >= BCM_DMA_CH_MAX) return (-1); if (!(sc->sc_dma_ch[ch].flags & BCM_DMA_CH_USED)) return (-1); info = sc->sc_dma_ch[ch].cb->info; info &= ~INFO_PERMAP_MASK; info |= (dreq << INFO_PERMAP_SHIFT) & INFO_PERMAP_MASK; if (dreq) info |= INFO_D_DREQ; else info &= ~INFO_D_DREQ; if (width == BCM_DMA_128BIT) info |= INFO_D_WIDTH; else info &= ~INFO_D_WIDTH; if (inc_addr == BCM_DMA_INC_ADDR) info |= INFO_D_INC; else info &= ~INFO_D_INC; sc->sc_dma_ch[ch].cb->info = info; return (0); } #ifdef DEBUG void bcm_dma_cb_dump(struct bcm_dma_cb *cb) { printf("DMA CB "); printf("INFO: %8.8x ", cb->info); printf("SRC: %8.8x ", cb->src); printf("DST: %8.8x ", cb->dst); printf("LEN: %8.8x ", cb->len); printf("\n"); printf("STRIDE: %8.8x ", cb->stride); printf("NEXT: %8.8x ", cb->next); printf("RSVD1: %8.8x ", cb->rsvd1); printf("RSVD2: %8.8x ", cb->rsvd2); printf("\n"); } void bcm_dma_reg_dump(int ch) { struct bcm_dma_softc *sc = bcm_dma_sc; int i; uint32_t reg; if (ch < 0 || ch >= BCM_DMA_CH_MAX) return; printf("DMA%d: ", ch); for (i = 0; i < MAX_REG; i++) { reg = bus_read_4(sc->sc_mem, BCM_DMA_CH(ch) + i*4); printf("%8.8x ", reg); } printf("\n"); } #endif /* * Start DMA transaction * ch - channel number * src, dst - source and destination address in * ARM physical memory address space. - * len - amount of bytes to be transfered + * len - amount of bytes to be transferred * * Returns 0 on success, -1 otherwise */ int bcm_dma_start(int ch, vm_paddr_t src, vm_paddr_t dst, int len) { struct bcm_dma_softc *sc = bcm_dma_sc; struct bcm_dma_cb *cb; if (ch < 0 || ch >= BCM_DMA_CH_MAX) return (-1); if (!(sc->sc_dma_ch[ch].flags & BCM_DMA_CH_USED)) return (-1); cb = sc->sc_dma_ch[ch].cb; if (BCM2835_ARM_IS_IO(src)) cb->src = IO_TO_VCBUS(src); else cb->src = PHYS_TO_VCBUS(src); if (BCM2835_ARM_IS_IO(dst)) cb->dst = IO_TO_VCBUS(dst); else cb->dst = PHYS_TO_VCBUS(dst); cb->len = len; bus_dmamap_sync(sc->sc_dma_tag, sc->sc_dma_ch[ch].dma_map, BUS_DMASYNC_PREWRITE); bus_write_4(sc->sc_mem, BCM_DMA_CBADDR(ch), sc->sc_dma_ch[ch].vc_cb); bus_write_4(sc->sc_mem, BCM_DMA_CS(ch), CS_ACTIVE); #ifdef DEBUG bcm_dma_cb_dump(sc->sc_dma_ch[ch].cb); bcm_dma_reg_dump(ch); #endif return (0); } /* * Get length requested for DMA transaction * ch - channel number * * Returns size of transaction, 0 if channel is invalid */ uint32_t bcm_dma_length(int ch) { struct bcm_dma_softc *sc = bcm_dma_sc; struct bcm_dma_cb *cb; if (ch < 0 || ch >= BCM_DMA_CH_MAX) return (0); if (!(sc->sc_dma_ch[ch].flags & BCM_DMA_CH_USED)) return (0); cb = sc->sc_dma_ch[ch].cb; return (cb->len); } static void bcm_dma_intr(void *arg) { struct bcm_dma_softc *sc = bcm_dma_sc; struct bcm_dma_ch *ch = (struct bcm_dma_ch *)arg; uint32_t cs, debug; /* my interrupt? */ cs = bus_read_4(sc->sc_mem, BCM_DMA_CS(ch->ch)); if (!(cs & (CS_INT | CS_ERR))) { device_printf(sc->sc_dev, "unexpected DMA intr CH=%d, CS=%x\n", ch->ch, cs); return; } /* running? */ if (!(ch->flags & BCM_DMA_CH_USED)) { device_printf(sc->sc_dev, "unused DMA intr CH=%d, CS=%x\n", ch->ch, cs); return; } if (cs & CS_ERR) { debug = bus_read_4(sc->sc_mem, BCM_DMA_DEBUG(ch->ch)); device_printf(sc->sc_dev, "DMA error %d on CH%d\n", debug & DEBUG_ERROR_MASK, ch->ch); bus_write_4(sc->sc_mem, BCM_DMA_DEBUG(ch->ch), debug & DEBUG_ERROR_MASK); bcm_dma_reset(sc->sc_dev, ch->ch); } if (cs & CS_INT) { /* acknowledge interrupt */ bus_write_4(sc->sc_mem, BCM_DMA_CS(ch->ch), CS_INT | CS_END); /* Prepare for possible access to len field */ bus_dmamap_sync(sc->sc_dma_tag, ch->dma_map, BUS_DMASYNC_POSTWRITE); /* save callback function and argument */ if (ch->intr_func) ch->intr_func(ch->ch, ch->intr_arg); } } static int bcm_dma_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, "broadcom,bcm2835-dma")) return (ENXIO); device_set_desc(dev, "BCM2835 DMA Controller"); return (BUS_PROBE_DEFAULT); } static int bcm_dma_attach(device_t dev) { struct bcm_dma_softc *sc = device_get_softc(dev); phandle_t node; int rid, err = 0; int i; sc->sc_dev = dev; if (bcm_dma_sc) return (ENXIO); for (i = 0; i < BCM_DMA_CH_MAX; i++) { sc->sc_irq[i] = NULL; sc->sc_intrhand[i] = NULL; } /* Get DMA channel mask. */ node = ofw_bus_get_node(sc->sc_dev); if (OF_getencprop(node, "brcm,dma-channel-mask", &bcm_dma_channel_mask, sizeof(bcm_dma_channel_mask)) == -1 && OF_getencprop(node, "broadcom,channels", &bcm_dma_channel_mask, sizeof(bcm_dma_channel_mask)) == -1) { device_printf(dev, "could not get channel mask property\n"); return (ENXIO); } /* Mask out channels used by GPU. */ bcm_dma_channel_mask &= ~BCM_DMA_CH_GPU_MASK; /* DMA0 - DMA14 */ rid = 0; sc->sc_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->sc_mem == NULL) { device_printf(dev, "could not allocate memory resource\n"); return (ENXIO); } /* IRQ DMA0 - DMA11 XXX NOT USE DMA12(spurious?) */ for (rid = 0; rid < BCM_DMA_CH_MAX; rid++) { if ((bcm_dma_channel_mask & (1 << rid)) == 0) continue; sc->sc_irq[rid] = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE); if (sc->sc_irq[rid] == NULL) { device_printf(dev, "cannot allocate interrupt\n"); err = ENXIO; goto fail; } if (bus_setup_intr(dev, sc->sc_irq[rid], INTR_TYPE_MISC | INTR_MPSAFE, NULL, bcm_dma_intr, &sc->sc_dma_ch[rid], &sc->sc_intrhand[rid])) { device_printf(dev, "cannot setup interrupt handler\n"); err = ENXIO; goto fail; } } mtx_init(&sc->sc_mtx, "bcmdma", "bcmdma", MTX_DEF); bcm_dma_sc = sc; err = bcm_dma_init(dev); if (err) goto fail; return (err); fail: if (sc->sc_mem) bus_release_resource(dev, SYS_RES_MEMORY, 0, sc->sc_mem); for (i = 0; i < BCM_DMA_CH_MAX; i++) { if (sc->sc_intrhand[i]) bus_teardown_intr(dev, sc->sc_irq[i], sc->sc_intrhand[i]); if (sc->sc_irq[i]) bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq[i]); } return (err); } static device_method_t bcm_dma_methods[] = { DEVMETHOD(device_probe, bcm_dma_probe), DEVMETHOD(device_attach, bcm_dma_attach), { 0, 0 } }; static driver_t bcm_dma_driver = { "bcm_dma", bcm_dma_methods, sizeof(struct bcm_dma_softc), }; static devclass_t bcm_dma_devclass; DRIVER_MODULE(bcm_dma, simplebus, bcm_dma_driver, bcm_dma_devclass, 0, 0); MODULE_VERSION(bcm_dma, 1); Index: user/ngie/release-pkg-fix-tests/sys/arm/cavium/cns11xx/if_ece.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/cavium/cns11xx/if_ece.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/cavium/cns11xx/if_ece.c (revision 299091) @@ -1,1955 +1,1955 @@ /*- * Copyright (c) 2009 Yohanes Nugroho * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef INET #include #include #include #include #endif #include #include #include #include #include #include #include #include #include /* "device miibus" required. See GENERIC if you get errors here. */ #include "miibus_if.h" static uint8_t vlan0_mac[ETHER_ADDR_LEN] = {0x00, 0xaa, 0xbb, 0xcc, 0xdd, 0x19}; /* * Boot loader expects the hardware state to be the same when we * restart the device (warm boot), so we need to save the initial * config values. */ int initial_switch_config; int initial_cpu_config; int initial_port0_config; int initial_port1_config; static inline uint32_t read_4(struct ece_softc *sc, bus_size_t off) { return (bus_read_4(sc->mem_res, off)); } static inline void write_4(struct ece_softc *sc, bus_size_t off, uint32_t val) { bus_write_4(sc->mem_res, off, val); } #define ECE_LOCK(_sc) mtx_lock(&(_sc)->sc_mtx) #define ECE_UNLOCK(_sc) mtx_unlock(&(_sc)->sc_mtx) #define ECE_LOCK_INIT(_sc) \ mtx_init(&_sc->sc_mtx, device_get_nameunit(_sc->dev), \ MTX_NETWORK_LOCK, MTX_DEF) #define ECE_TXLOCK(_sc) mtx_lock(&(_sc)->sc_mtx_tx) #define ECE_TXUNLOCK(_sc) mtx_unlock(&(_sc)->sc_mtx_tx) #define ECE_TXLOCK_INIT(_sc) \ mtx_init(&_sc->sc_mtx_tx, device_get_nameunit(_sc->dev), \ "ECE TX Lock", MTX_DEF) #define ECE_CLEANUPLOCK(_sc) mtx_lock(&(_sc)->sc_mtx_cleanup) #define ECE_CLEANUPUNLOCK(_sc) mtx_unlock(&(_sc)->sc_mtx_cleanup) #define ECE_CLEANUPLOCK_INIT(_sc) \ mtx_init(&_sc->sc_mtx_cleanup, device_get_nameunit(_sc->dev), \ "ECE cleanup Lock", MTX_DEF) #define ECE_RXLOCK(_sc) mtx_lock(&(_sc)->sc_mtx_rx) #define ECE_RXUNLOCK(_sc) mtx_unlock(&(_sc)->sc_mtx_rx) #define ECE_RXLOCK_INIT(_sc) \ mtx_init(&_sc->sc_mtx_rx, device_get_nameunit(_sc->dev), \ "ECE RX Lock", MTX_DEF) #define ECE_LOCK_DESTROY(_sc) mtx_destroy(&_sc->sc_mtx); #define ECE_TXLOCK_DESTROY(_sc) mtx_destroy(&_sc->sc_mtx_tx); #define ECE_RXLOCK_DESTROY(_sc) mtx_destroy(&_sc->sc_mtx_rx); #define ECE_CLEANUPLOCK_DESTROY(_sc) \ mtx_destroy(&_sc->sc_mtx_cleanup); #define ECE_ASSERT_LOCKED(_sc) mtx_assert(&_sc->sc_mtx, MA_OWNED); #define ECE_ASSERT_UNLOCKED(_sc) mtx_assert(&_sc->sc_mtx, MA_NOTOWNED); static devclass_t ece_devclass; /* ifnet entry points */ static void eceinit_locked(void *); static void ecestart_locked(struct ifnet *); static void eceinit(void *); static void ecestart(struct ifnet *); static void ecestop(struct ece_softc *); static int eceioctl(struct ifnet * ifp, u_long, caddr_t); /* bus entry points */ static int ece_probe(device_t dev); static int ece_attach(device_t dev); static int ece_detach(device_t dev); static void ece_intr(void *); static void ece_intr_qf(void *); static void ece_intr_status(void *xsc); /* helper routines */ static int ece_activate(device_t dev); static void ece_deactivate(device_t dev); static int ece_ifmedia_upd(struct ifnet *ifp); static void ece_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr); static int ece_get_mac(struct ece_softc *sc, u_char *eaddr); static void ece_set_mac(struct ece_softc *sc, u_char *eaddr); static int configure_cpu_port(struct ece_softc *sc); static int configure_lan_port(struct ece_softc *sc, int phy_type); static void set_pvid(struct ece_softc *sc, int port0, int port1, int cpu); static void set_vlan_vid(struct ece_softc *sc, int vlan); static void set_vlan_member(struct ece_softc *sc, int vlan); static void set_vlan_tag(struct ece_softc *sc, int vlan); static int hardware_init(struct ece_softc *sc); static void ece_intr_rx_locked(struct ece_softc *sc, int count); static void ece_free_desc_dma_tx(struct ece_softc *sc); static void ece_free_desc_dma_rx(struct ece_softc *sc); static void ece_intr_task(void *arg, int pending __unused); static void ece_tx_task(void *arg, int pending __unused); static void ece_cleanup_task(void *arg, int pending __unused); static int ece_allocate_dma(struct ece_softc *sc); static void ece_intr_tx(void *xsc); static void clear_mac_entries(struct ece_softc *ec, int include_this_mac); static uint32_t read_mac_entry(struct ece_softc *ec, uint8_t *mac_result, int first); /*PHY related functions*/ static inline int phy_read(struct ece_softc *sc, int phy, int reg) { int val; int ii; int status; write_4(sc, PHY_CONTROL, PHY_RW_OK); write_4(sc, PHY_CONTROL, (PHY_ADDRESS(phy)|PHY_READ_COMMAND | PHY_REGISTER(reg))); for (ii = 0; ii < 0x1000; ii++) { status = read_4(sc, PHY_CONTROL); if (status & PHY_RW_OK) { /* Clear the rw_ok status, and clear other * bits value. */ write_4(sc, PHY_CONTROL, PHY_RW_OK); val = PHY_GET_DATA(status); return (val); } } return (0); } static inline void phy_write(struct ece_softc *sc, int phy, int reg, int data) { int ii; write_4(sc, PHY_CONTROL, PHY_RW_OK); write_4(sc, PHY_CONTROL, PHY_ADDRESS(phy) | PHY_REGISTER(reg) | PHY_WRITE_COMMAND | PHY_DATA(data)); for (ii = 0; ii < 0x1000; ii++) { if (read_4(sc, PHY_CONTROL) & PHY_RW_OK) { /* Clear the rw_ok status, and clear other * bits value. */ write_4(sc, PHY_CONTROL, PHY_RW_OK); return; } } } static int get_phy_type(struct ece_softc *sc) { uint16_t phy0_id = 0, phy1_id = 0; /* * Use SMI (MDC/MDIO) to read Link Partner's PHY Identifier * Register 1. */ phy0_id = phy_read(sc, 0, 0x2); phy1_id = phy_read(sc, 1, 0x2); if ((phy0_id == 0xFFFF) && (phy1_id == 0x000F)) return (ASIX_GIGA_PHY); else if ((phy0_id == 0x0243) && (phy1_id == 0x0243)) return (TWO_SINGLE_PHY); else if ((phy0_id == 0xFFFF) && (phy1_id == 0x0007)) return (VSC8601_GIGA_PHY); else if ((phy0_id == 0x0243) && (phy1_id == 0xFFFF)) return (IC_PLUS_PHY); return (NOT_FOUND_PHY); } static int ece_probe(device_t dev) { device_set_desc(dev, "Econa Ethernet Controller"); return (0); } static int ece_attach(device_t dev) { struct ece_softc *sc; struct ifnet *ifp = NULL; struct sysctl_ctx_list *sctx; struct sysctl_oid *soid; u_char eaddr[ETHER_ADDR_LEN]; int err; int i, rid; uint32_t rnd; err = 0; sc = device_get_softc(dev); sc->dev = dev; rid = 0; sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->mem_res == NULL) goto out; power_on_network_interface(); rid = 0; sc->irq_res_status = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE); if (sc->irq_res_status == NULL) goto out; rid = 1; /*TSTC: Fm-Switch-Tx-Complete*/ sc->irq_res_tx = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE); if (sc->irq_res_tx == NULL) goto out; rid = 2; /*FSRC: Fm-Switch-Rx-Complete*/ sc->irq_res_rec = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE); if (sc->irq_res_rec == NULL) goto out; rid = 4; /*FSQF: Fm-Switch-Queue-Full*/ sc->irq_res_qf = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE); if (sc->irq_res_qf == NULL) goto out; err = ece_activate(dev); if (err) goto out; /* Sysctls */ sctx = device_get_sysctl_ctx(dev); soid = device_get_sysctl_tree(dev); ECE_LOCK_INIT(sc); callout_init_mtx(&sc->tick_ch, &sc->sc_mtx, 0); if ((err = ece_get_mac(sc, eaddr)) != 0) { /* No MAC address configured. Generate the random one. */ if (bootverbose) device_printf(dev, "Generating random ethernet address.\n"); rnd = arc4random(); /*from if_ae.c/if_ate.c*/ /* * Set OUI to convenient locally assigned address. 'b' * is 0x62, which has the locally assigned bit set, and * the broadcast/multicast bit clear. */ eaddr[0] = 'b'; eaddr[1] = 's'; eaddr[2] = 'd'; eaddr[3] = (rnd >> 16) & 0xff; eaddr[4] = (rnd >> 8) & 0xff; eaddr[5] = rnd & 0xff; for (i = 0; i < ETHER_ADDR_LEN; i++) eaddr[i] = vlan0_mac[i]; } ece_set_mac(sc, eaddr); sc->ifp = ifp = if_alloc(IFT_ETHER); /* Only one PHY at address 0 in this device. */ err = mii_attach(dev, &sc->miibus, ifp, ece_ifmedia_upd, ece_ifmedia_sts, BMSR_DEFCAPMASK, 0, MII_OFFSET_ANY, 0); if (err != 0) { device_printf(dev, "attaching PHYs failed\n"); goto out; } ifp->if_softc = sc; if_initname(ifp, device_get_name(dev), device_get_unit(dev)); ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST; ifp->if_capabilities = IFCAP_HWCSUM; ifp->if_hwassist = (CSUM_IP | CSUM_TCP | CSUM_UDP); ifp->if_capenable = ifp->if_capabilities; ifp->if_start = ecestart; ifp->if_ioctl = eceioctl; ifp->if_init = eceinit; ifp->if_snd.ifq_drv_maxlen = ECE_MAX_TX_BUFFERS - 1; IFQ_SET_MAXLEN(&ifp->if_snd, ECE_MAX_TX_BUFFERS - 1); IFQ_SET_READY(&ifp->if_snd); /* Create local taskq. */ TASK_INIT(&sc->sc_intr_task, 0, ece_intr_task, sc); TASK_INIT(&sc->sc_tx_task, 1, ece_tx_task, ifp); TASK_INIT(&sc->sc_cleanup_task, 2, ece_cleanup_task, sc); sc->sc_tq = taskqueue_create_fast("ece_taskq", M_WAITOK, taskqueue_thread_enqueue, &sc->sc_tq); if (sc->sc_tq == NULL) { device_printf(sc->dev, "could not create taskqueue\n"); goto out; } ether_ifattach(ifp, eaddr); /* * Activate interrupts */ err = bus_setup_intr(dev, sc->irq_res_rec, INTR_TYPE_NET | INTR_MPSAFE, NULL, ece_intr, sc, &sc->intrhand); if (err) { ether_ifdetach(ifp); ECE_LOCK_DESTROY(sc); goto out; } err = bus_setup_intr(dev, sc->irq_res_status, INTR_TYPE_NET | INTR_MPSAFE, NULL, ece_intr_status, sc, &sc->intrhand_status); if (err) { ether_ifdetach(ifp); ECE_LOCK_DESTROY(sc); goto out; } err = bus_setup_intr(dev, sc->irq_res_qf, INTR_TYPE_NET | INTR_MPSAFE, NULL,ece_intr_qf, sc, &sc->intrhand_qf); if (err) { ether_ifdetach(ifp); ECE_LOCK_DESTROY(sc); goto out; } err = bus_setup_intr(dev, sc->irq_res_tx, INTR_TYPE_NET | INTR_MPSAFE, NULL, ece_intr_tx, sc, &sc->intrhand_tx); if (err) { ether_ifdetach(ifp); ECE_LOCK_DESTROY(sc); goto out; } ECE_TXLOCK_INIT(sc); ECE_RXLOCK_INIT(sc); ECE_CLEANUPLOCK_INIT(sc); /* Enable all interrupt sources. */ write_4(sc, INTERRUPT_MASK, 0x00000000); /* Enable port 0. */ write_4(sc, PORT_0_CONFIG, read_4(sc, PORT_0_CONFIG) & ~(PORT_DISABLE)); taskqueue_start_threads(&sc->sc_tq, 1, PI_NET, "%s taskq", device_get_nameunit(sc->dev)); out: if (err) ece_deactivate(dev); if (err && ifp) if_free(ifp); return (err); } static int ece_detach(device_t dev) { struct ece_softc *sc = device_get_softc(dev); struct ifnet *ifp = sc->ifp; ecestop(sc); if (ifp != NULL) { ether_ifdetach(ifp); if_free(ifp); } ece_deactivate(dev); return (0); } static void ece_getaddr(void *arg, bus_dma_segment_t *segs, int nsegs, int error) { u_int32_t *paddr; KASSERT(nsegs == 1, ("wrong number of segments, should be 1")); paddr = arg; *paddr = segs->ds_addr; } static int ece_alloc_desc_dma_tx(struct ece_softc *sc) { int i; int error; /* Allocate a busdma tag and DMA safe memory for TX/RX descriptors. */ error = bus_dma_tag_create(sc->sc_parent_tag, /* parent */ 16, 0, /* alignment, boundary */ BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filtfunc, filtfuncarg */ sizeof(eth_tx_desc_t)*ECE_MAX_TX_BUFFERS, /* max size */ 1, /*nsegments */ sizeof(eth_tx_desc_t)*ECE_MAX_TX_BUFFERS, 0, /* flags */ NULL, NULL, /* lockfunc, lockfuncarg */ &sc->dmatag_data_tx); /* dmat */ /* Allocate memory for TX ring. */ error = bus_dmamem_alloc(sc->dmatag_data_tx, (void**)&(sc->desc_tx), BUS_DMA_NOWAIT | BUS_DMA_ZERO | BUS_DMA_COHERENT, &(sc->dmamap_ring_tx)); if (error) { if_printf(sc->ifp, "failed to allocate DMA memory\n"); bus_dma_tag_destroy(sc->dmatag_data_tx); sc->dmatag_data_tx = 0; return (ENXIO); } /* Load Ring DMA. */ error = bus_dmamap_load(sc->dmatag_data_tx, sc->dmamap_ring_tx, sc->desc_tx, sizeof(eth_tx_desc_t)*ECE_MAX_TX_BUFFERS, ece_getaddr, &(sc->ring_paddr_tx), BUS_DMA_NOWAIT); if (error) { if_printf(sc->ifp, "can't load descriptor\n"); bus_dmamem_free(sc->dmatag_data_tx, sc->desc_tx, sc->dmamap_ring_tx); sc->desc_tx = NULL; bus_dma_tag_destroy(sc->dmatag_data_tx); sc->dmatag_data_tx = 0; return (ENXIO); } /* Allocate a busdma tag for mbufs. Alignment is 2 bytes */ error = bus_dma_tag_create(sc->sc_parent_tag, /* parent */ 1, 0, /* alignment, boundary */ BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filtfunc, filtfuncarg */ MCLBYTES*MAX_FRAGMENT, /* maxsize */ MAX_FRAGMENT, /* nsegments */ MCLBYTES, 0, /* maxsegsz, flags */ NULL, NULL, /* lockfunc, lockfuncarg */ &sc->dmatag_ring_tx); /* dmat */ if (error) { if_printf(sc->ifp, "failed to create busdma tag for mbufs\n"); return (ENXIO); } for (i = 0; i < ECE_MAX_TX_BUFFERS; i++) { /* Create dma map for each descriptor. */ error = bus_dmamap_create(sc->dmatag_ring_tx, 0, &(sc->tx_desc[i].dmamap)); if (error) { if_printf(sc->ifp, "failed to create map for mbuf\n"); return (ENXIO); } } return (0); } static void ece_free_desc_dma_tx(struct ece_softc *sc) { int i; for (i = 0; i < ECE_MAX_TX_BUFFERS; i++) { if (sc->tx_desc[i].buff) { m_freem(sc->tx_desc[i].buff); sc->tx_desc[i].buff= 0; } } if (sc->ring_paddr_tx) { bus_dmamap_unload(sc->dmatag_data_tx, sc->dmamap_ring_tx); sc->ring_paddr_tx = 0; } if (sc->desc_tx) { bus_dmamem_free(sc->dmatag_data_tx, sc->desc_tx, sc->dmamap_ring_tx); sc->desc_tx = NULL; } if (sc->dmatag_data_tx) { bus_dma_tag_destroy(sc->dmatag_data_tx); sc->dmatag_data_tx = 0; } if (sc->dmatag_ring_tx) { for (i = 0; idmatag_ring_tx, sc->tx_desc[i].dmamap); sc->tx_desc[i].dmamap = 0; } bus_dma_tag_destroy(sc->dmatag_ring_tx); sc->dmatag_ring_tx = 0; } } static int ece_alloc_desc_dma_rx(struct ece_softc *sc) { int error; int i; /* Allocate a busdma tag and DMA safe memory for RX descriptors. */ error = bus_dma_tag_create(sc->sc_parent_tag, /* parent */ 16, 0, /* alignment, boundary */ BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filtfunc, filtfuncarg */ /* maxsize, nsegments */ sizeof(eth_rx_desc_t)*ECE_MAX_RX_BUFFERS, 1, /* maxsegsz, flags */ sizeof(eth_rx_desc_t)*ECE_MAX_RX_BUFFERS, 0, NULL, NULL, /* lockfunc, lockfuncarg */ &sc->dmatag_data_rx); /* dmat */ /* Allocate RX ring. */ error = bus_dmamem_alloc(sc->dmatag_data_rx, (void**)&(sc->desc_rx), BUS_DMA_NOWAIT | BUS_DMA_ZERO | BUS_DMA_COHERENT, &(sc->dmamap_ring_rx)); if (error) { if_printf(sc->ifp, "failed to allocate DMA memory\n"); return (ENXIO); } /* Load dmamap. */ error = bus_dmamap_load(sc->dmatag_data_rx, sc->dmamap_ring_rx, sc->desc_rx, sizeof(eth_rx_desc_t)*ECE_MAX_RX_BUFFERS, ece_getaddr, &(sc->ring_paddr_rx), BUS_DMA_NOWAIT); if (error) { if_printf(sc->ifp, "can't load descriptor\n"); bus_dmamem_free(sc->dmatag_data_rx, sc->desc_rx, sc->dmamap_ring_rx); bus_dma_tag_destroy(sc->dmatag_data_rx); sc->desc_rx = NULL; return (ENXIO); } /* Allocate a busdma tag for mbufs. */ error = bus_dma_tag_create(sc->sc_parent_tag,/* parent */ 16, 0, /* alignment, boundary */ BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filtfunc, filtfuncarg */ MCLBYTES, 1, /* maxsize, nsegments */ MCLBYTES, 0, /* maxsegsz, flags */ NULL, NULL, /* lockfunc, lockfuncarg */ &sc->dmatag_ring_rx); /* dmat */ if (error) { if_printf(sc->ifp, "failed to create busdma tag for mbufs\n"); return (ENXIO); } for (i = 0; idmatag_ring_rx, 0, &sc->rx_desc[i].dmamap); if (error) { if_printf(sc->ifp, "failed to create map for mbuf\n"); return (ENXIO); } } error = bus_dmamap_create(sc->dmatag_ring_rx, 0, &sc->rx_sparemap); if (error) { if_printf(sc->ifp, "failed to create spare map\n"); return (ENXIO); } return (0); } static void ece_free_desc_dma_rx(struct ece_softc *sc) { int i; for (i = 0; i < ECE_MAX_RX_BUFFERS; i++) { if (sc->rx_desc[i].buff) { m_freem(sc->rx_desc[i].buff); sc->rx_desc[i].buff = NULL; } } if (sc->ring_paddr_rx) { bus_dmamap_unload(sc->dmatag_data_rx, sc->dmamap_ring_rx); sc->ring_paddr_rx = 0; } if (sc->desc_rx) { bus_dmamem_free(sc->dmatag_data_rx, sc->desc_rx, sc->dmamap_ring_rx); sc->desc_rx = NULL; } if (sc->dmatag_data_rx) { bus_dma_tag_destroy(sc->dmatag_data_rx); sc->dmatag_data_rx = NULL; } if (sc->dmatag_ring_rx) { for (i = 0; i < ECE_MAX_RX_BUFFERS; i++) bus_dmamap_destroy(sc->dmatag_ring_rx, sc->rx_desc[i].dmamap); bus_dmamap_destroy(sc->dmatag_ring_rx, sc->rx_sparemap); bus_dma_tag_destroy(sc->dmatag_ring_rx); sc->dmatag_ring_rx = NULL; } } static int ece_new_rxbuf(struct ece_softc *sc, struct rx_desc_info* descinfo) { struct mbuf *new_mbuf; bus_dma_segment_t seg[1]; bus_dmamap_t map; int error; int nsegs; bus_dma_tag_t tag; tag = sc->dmatag_ring_rx; new_mbuf = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); if (new_mbuf == NULL) return (ENOBUFS); new_mbuf->m_len = new_mbuf->m_pkthdr.len = MCLBYTES; error = bus_dmamap_load_mbuf_sg(tag, sc->rx_sparemap, new_mbuf, seg, &nsegs, BUS_DMA_NOWAIT); KASSERT(nsegs == 1, ("Too many segments returned!")); if (nsegs != 1 || error) { m_free(new_mbuf); return (ENOBUFS); } if (descinfo->buff != NULL) { bus_dmamap_sync(tag, descinfo->dmamap, BUS_DMASYNC_POSTREAD); bus_dmamap_unload(tag, descinfo->dmamap); } map = descinfo->dmamap; descinfo->dmamap = sc->rx_sparemap; sc->rx_sparemap = map; bus_dmamap_sync(tag, descinfo->dmamap, BUS_DMASYNC_PREREAD); descinfo->buff = new_mbuf; descinfo->desc->data_ptr = seg->ds_addr; descinfo->desc->length = seg->ds_len - 2; return (0); } static int ece_allocate_dma(struct ece_softc *sc) { eth_tx_desc_t *desctx; eth_rx_desc_t *descrx; int i; int error; /* Create parent tag for tx and rx */ error = bus_dma_tag_create( bus_get_dma_tag(sc->dev),/* parent */ 1, 0, /* alignment, boundary */ BUS_SPACE_MAXADDR, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ BUS_SPACE_MAXSIZE_32BIT, 0,/* maxsize, nsegments */ BUS_SPACE_MAXSIZE_32BIT, /* maxsegsize */ 0, /* flags */ NULL, NULL, /* lockfunc, lockarg */ &sc->sc_parent_tag); ece_alloc_desc_dma_tx(sc); for (i = 0; i < ECE_MAX_TX_BUFFERS; i++) { desctx = (eth_tx_desc_t *)(&sc->desc_tx[i]); memset(desctx, 0, sizeof(eth_tx_desc_t)); desctx->length = MAX_PACKET_LEN; desctx->cown = 1; if (i == ECE_MAX_TX_BUFFERS - 1) desctx->eor = 1; } ece_alloc_desc_dma_rx(sc); for (i = 0; i < ECE_MAX_RX_BUFFERS; i++) { descrx = &(sc->desc_rx[i]); memset(descrx, 0, sizeof(eth_rx_desc_t)); sc->rx_desc[i].desc = descrx; sc->rx_desc[i].buff = 0; ece_new_rxbuf(sc, &(sc->rx_desc[i])); if (i == ECE_MAX_RX_BUFFERS - 1) descrx->eor = 1; } sc->tx_prod = 0; sc->tx_cons = 0; sc->last_rx = 0; sc->desc_curr_tx = 0; return (0); } static int ece_activate(device_t dev) { struct ece_softc *sc; int err; uint32_t mac_port_config; struct ifnet *ifp; sc = device_get_softc(dev); ifp = sc->ifp; initial_switch_config = read_4(sc, SWITCH_CONFIG); initial_cpu_config = read_4(sc, CPU_PORT_CONFIG); initial_port0_config = read_4(sc, MAC_PORT_0_CONFIG); initial_port1_config = read_4(sc, MAC_PORT_1_CONFIG); /* Disable Port 0 */ mac_port_config = read_4(sc, MAC_PORT_0_CONFIG); mac_port_config |= (PORT_DISABLE); write_4(sc, MAC_PORT_0_CONFIG, mac_port_config); /* Disable Port 1 */ mac_port_config = read_4(sc, MAC_PORT_1_CONFIG); mac_port_config |= (PORT_DISABLE); write_4(sc, MAC_PORT_1_CONFIG, mac_port_config); err = ece_allocate_dma(sc); if (err) { if_printf(sc->ifp, "failed allocating dma\n"); goto out; } write_4(sc, TS_DESCRIPTOR_POINTER, sc->ring_paddr_tx); write_4(sc, TS_DESCRIPTOR_BASE_ADDR, sc->ring_paddr_tx); write_4(sc, FS_DESCRIPTOR_POINTER, sc->ring_paddr_rx); write_4(sc, FS_DESCRIPTOR_BASE_ADDR, sc->ring_paddr_rx); write_4(sc, FS_DMA_CONTROL, 1); return (0); out: return (ENXIO); } static void ece_deactivate(device_t dev) { struct ece_softc *sc; sc = device_get_softc(dev); if (sc->intrhand) bus_teardown_intr(dev, sc->irq_res_rec, sc->intrhand); sc->intrhand = 0; if (sc->intrhand_qf) bus_teardown_intr(dev, sc->irq_res_qf, sc->intrhand_qf); sc->intrhand_qf = 0; bus_generic_detach(sc->dev); if (sc->miibus) device_delete_child(sc->dev, sc->miibus); if (sc->mem_res) bus_release_resource(dev, SYS_RES_IOPORT, rman_get_rid(sc->mem_res), sc->mem_res); sc->mem_res = 0; if (sc->irq_res_rec) bus_release_resource(dev, SYS_RES_IRQ, rman_get_rid(sc->irq_res_rec), sc->irq_res_rec); if (sc->irq_res_qf) bus_release_resource(dev, SYS_RES_IRQ, rman_get_rid(sc->irq_res_qf), sc->irq_res_qf); if (sc->irq_res_qf) bus_release_resource(dev, SYS_RES_IRQ, rman_get_rid(sc->irq_res_status), sc->irq_res_status); sc->irq_res_rec = 0; sc->irq_res_qf = 0; sc->irq_res_status = 0; ECE_TXLOCK_DESTROY(sc); ECE_RXLOCK_DESTROY(sc); ece_free_desc_dma_tx(sc); ece_free_desc_dma_rx(sc); return; } /* * Change media according to request. */ static int ece_ifmedia_upd(struct ifnet *ifp) { struct ece_softc *sc = ifp->if_softc; struct mii_data *mii; int error; mii = device_get_softc(sc->miibus); ECE_LOCK(sc); error = mii_mediachg(mii); ECE_UNLOCK(sc); return (error); } /* * Notify the world which media we're using. */ static void ece_ifmedia_sts(struct ifnet *ifp, struct ifmediareq *ifmr) { struct ece_softc *sc = ifp->if_softc; struct mii_data *mii; mii = device_get_softc(sc->miibus); ECE_LOCK(sc); mii_pollstat(mii); ifmr->ifm_active = mii->mii_media_active; ifmr->ifm_status = mii->mii_media_status; ECE_UNLOCK(sc); } static void ece_tick(void *xsc) { struct ece_softc *sc = xsc; struct mii_data *mii; int active; mii = device_get_softc(sc->miibus); active = mii->mii_media_active; mii_tick(mii); /* * Schedule another timeout one second from now. */ callout_reset(&sc->tick_ch, hz, ece_tick, sc); } static uint32_t read_mac_entry(struct ece_softc *ec, uint8_t *mac_result, int first) { uint32_t ii; struct arl_table_entry_t entry; uint32_t *entry_val; write_4(ec, ARL_TABLE_ACCESS_CONTROL_0, 0); write_4(ec, ARL_TABLE_ACCESS_CONTROL_1, 0); write_4(ec, ARL_TABLE_ACCESS_CONTROL_2, 0); if (first) write_4(ec, ARL_TABLE_ACCESS_CONTROL_0, 0x1); else write_4(ec, ARL_TABLE_ACCESS_CONTROL_0, 0x2); for (ii = 0; ii < 0x1000; ii++) if (read_4(ec, ARL_TABLE_ACCESS_CONTROL_1) & (0x1)) break; entry_val = (uint32_t*) (&entry); entry_val[0] = read_4(ec, ARL_TABLE_ACCESS_CONTROL_1); entry_val[1] = read_4(ec, ARL_TABLE_ACCESS_CONTROL_2); if (mac_result) memcpy(mac_result, entry.mac_addr, ETHER_ADDR_LEN); return (entry.table_end); } static uint32_t write_arl_table_entry(struct ece_softc *ec, uint32_t filter, uint32_t vlan_mac, uint32_t vlan_gid, uint32_t age_field, uint32_t port_map, const uint8_t *mac_addr) { uint32_t ii; uint32_t *entry_val; struct arl_table_entry_t entry; memset(&entry, 0, sizeof(entry)); entry.filter = filter; entry.vlan_mac = vlan_mac; entry.vlan_gid = vlan_gid; entry.age_field = age_field; entry.port_map = port_map; memcpy(entry.mac_addr, mac_addr, ETHER_ADDR_LEN); entry_val = (uint32_t*) (&entry); write_4(ec, ARL_TABLE_ACCESS_CONTROL_0, 0); write_4(ec, ARL_TABLE_ACCESS_CONTROL_1, 0); write_4(ec, ARL_TABLE_ACCESS_CONTROL_2, 0); write_4(ec, ARL_TABLE_ACCESS_CONTROL_1, entry_val[0]); write_4(ec, ARL_TABLE_ACCESS_CONTROL_2, entry_val[1]); write_4(ec, ARL_TABLE_ACCESS_CONTROL_0, ARL_WRITE_COMMAND); for (ii = 0; ii < 0x1000; ii++) if (read_4(ec, ARL_TABLE_ACCESS_CONTROL_1) & ARL_COMMAND_COMPLETE) return (1); /* Write OK. */ /* Write failed. */ return (0); } static void remove_mac_entry(struct ece_softc *sc, uint8_t *mac) { /* Invalid age_field mean erase this entry. */ write_arl_table_entry(sc, 0, 1, VLAN0_GROUP_ID, INVALID_ENTRY, VLAN0_GROUP, mac); } static void add_mac_entry(struct ece_softc *sc, uint8_t *mac) { write_arl_table_entry(sc, 0, 1, VLAN0_GROUP_ID, NEW_ENTRY, VLAN0_GROUP, mac); } /** * The behavior of ARL table reading and deletion is not well defined * in the documentation. To be safe, all mac addresses are put to a * list, then deleted. * */ static void clear_mac_entries(struct ece_softc *ec, int include_this_mac) { int table_end; struct mac_list * temp; struct mac_list * mac_list_header; struct mac_list * current; char mac[ETHER_ADDR_LEN]; current = NULL; mac_list_header = NULL; table_end = read_mac_entry(ec, mac, 1); while (!table_end) { if (!include_this_mac && memcmp(mac, vlan0_mac, ETHER_ADDR_LEN) == 0) { /* Read next entry. */ table_end = read_mac_entry(ec, mac, 0); continue; } temp = (struct mac_list*)malloc(sizeof(struct mac_list), M_DEVBUF, M_NOWAIT | M_ZERO); memcpy(temp->mac_addr, mac, ETHER_ADDR_LEN); temp->next = 0; if (mac_list_header) { current->next = temp; current = temp; } else { mac_list_header = temp; current = temp; } /* Read next Entry */ table_end = read_mac_entry(ec, mac, 0); } current = mac_list_header; while (current) { remove_mac_entry(ec, current->mac_addr); temp = current; current = current->next; free(temp, M_DEVBUF); } } static int configure_lan_port(struct ece_softc *sc, int phy_type) { uint32_t sw_config; uint32_t mac_port_config; /* * Configure switch */ sw_config = read_4(sc, SWITCH_CONFIG); /* Enable fast aging. */ sw_config |= FAST_AGING; /* Enable IVL learning. */ sw_config |= IVL_LEARNING; /* Disable hardware NAT. */ sw_config &= ~(HARDWARE_NAT); sw_config |= SKIP_L2_LOOKUP_PORT_0 | SKIP_L2_LOOKUP_PORT_1| NIC_MODE; write_4(sc, SWITCH_CONFIG, sw_config); sw_config = read_4(sc, SWITCH_CONFIG); mac_port_config = read_4(sc, MAC_PORT_0_CONFIG); if (!(mac_port_config & 0x1) || (mac_port_config & 0x2)) if_printf(sc->ifp, "Link Down\n"); else write_4(sc, MAC_PORT_0_CONFIG, mac_port_config); return (0); } static void set_pvid(struct ece_softc *sc, int port0, int port1, int cpu) { uint32_t val; val = read_4(sc, VLAN_PORT_PVID) & (~(0x7 << 0)); write_4(sc, VLAN_PORT_PVID, val); val = read_4(sc, VLAN_PORT_PVID) | ((port0) & 0x07); write_4(sc, VLAN_PORT_PVID, val); val = read_4(sc, VLAN_PORT_PVID) & (~(0x7 << 4)); write_4(sc, VLAN_PORT_PVID, val); val = read_4(sc, VLAN_PORT_PVID) | (((port1) & 0x07) << 4); write_4(sc, VLAN_PORT_PVID, val); val = read_4(sc, VLAN_PORT_PVID) & (~(0x7 << 8)); write_4(sc, VLAN_PORT_PVID, val); val = read_4(sc, VLAN_PORT_PVID) | (((cpu) & 0x07) << 8); write_4(sc, VLAN_PORT_PVID, val); } /* VLAN related functions */ static void set_vlan_vid(struct ece_softc *sc, int vlan) { const uint32_t regs[] = { VLAN_VID_0_1, VLAN_VID_0_1, VLAN_VID_2_3, VLAN_VID_2_3, VLAN_VID_4_5, VLAN_VID_4_5, VLAN_VID_6_7, VLAN_VID_6_7 }; const int vids[] = { VLAN0_VID, VLAN1_VID, VLAN2_VID, VLAN3_VID, VLAN4_VID, VLAN5_VID, VLAN6_VID, VLAN7_VID }; uint32_t val; uint32_t reg; int vid; reg = regs[vlan]; vid = vids[vlan]; if (vlan & 1) { val = read_4(sc, reg); write_4(sc, reg, val & (~(0xFFF << 0))); val = read_4(sc, reg); write_4(sc, reg, val|((vid & 0xFFF) << 0)); } else { val = read_4(sc, reg); write_4(sc, reg, val & (~(0xFFF << 12))); val = read_4(sc, reg); write_4(sc, reg, val|((vid & 0xFFF) << 12)); } } static void set_vlan_member(struct ece_softc *sc, int vlan) { unsigned char shift; uint32_t val; int group; const int groups[] = { VLAN0_GROUP, VLAN1_GROUP, VLAN2_GROUP, VLAN3_GROUP, VLAN4_GROUP, VLAN5_GROUP, VLAN6_GROUP, VLAN7_GROUP }; group = groups[vlan]; shift = vlan*3; val = read_4(sc, VLAN_MEMBER_PORT_MAP) & (~(0x7 << shift)); write_4(sc, VLAN_MEMBER_PORT_MAP, val); val = read_4(sc, VLAN_MEMBER_PORT_MAP); write_4(sc, VLAN_MEMBER_PORT_MAP, val | ((group & 0x7) << shift)); } static void set_vlan_tag(struct ece_softc *sc, int vlan) { unsigned char shift; uint32_t val; int tag = 0; shift = vlan*3; val = read_4(sc, VLAN_TAG_PORT_MAP) & (~(0x7 << shift)); write_4(sc, VLAN_TAG_PORT_MAP, val); val = read_4(sc, VLAN_TAG_PORT_MAP); write_4(sc, VLAN_TAG_PORT_MAP, val | ((tag & 0x7) << shift)); } static int configure_cpu_port(struct ece_softc *sc) { uint32_t cpu_port_config; int i; cpu_port_config = read_4(sc, CPU_PORT_CONFIG); /* SA learning Disable */ cpu_port_config |= (SA_LEARNING_DISABLE); /* set data offset + 2 */ cpu_port_config &= ~(1U << 31); write_4(sc, CPU_PORT_CONFIG, cpu_port_config); if (!write_arl_table_entry(sc, 0, 1, VLAN0_GROUP_ID, STATIC_ENTRY, VLAN0_GROUP, vlan0_mac)) return (1); set_pvid(sc, PORT0_PVID, PORT1_PVID, CPU_PORT_PVID); for (i = 0; i < 8; i++) { set_vlan_vid(sc, i); set_vlan_member(sc, i); set_vlan_tag(sc, i); } /* disable all interrupt status sources */ write_4(sc, INTERRUPT_MASK, 0xffff1fff); /* clear previous interrupt sources */ write_4(sc, INTERRUPT_STATUS, 0x00001FFF); write_4(sc, TS_DMA_CONTROL, 0); write_4(sc, FS_DMA_CONTROL, 0); return (0); } static int hardware_init(struct ece_softc *sc) { int status = 0; static int gw_phy_type; gw_phy_type = get_phy_type(sc); /* Currently only ic_plus phy is supported. */ if (gw_phy_type != IC_PLUS_PHY) { device_printf(sc->dev, "PHY type is not supported (%d)\n", gw_phy_type); return (-1); } status = configure_lan_port(sc, gw_phy_type); configure_cpu_port(sc); return (0); } static void set_mac_address(struct ece_softc *sc, const char *mac, int mac_len) { /* Invalid age_field mean erase this entry. */ write_arl_table_entry(sc, 0, 1, VLAN0_GROUP_ID, INVALID_ENTRY, VLAN0_GROUP, mac); memcpy(vlan0_mac, mac, ETHER_ADDR_LEN); write_arl_table_entry(sc, 0, 1, VLAN0_GROUP_ID, STATIC_ENTRY, VLAN0_GROUP, mac); } static void ece_set_mac(struct ece_softc *sc, u_char *eaddr) { memcpy(vlan0_mac, eaddr, ETHER_ADDR_LEN); set_mac_address(sc, eaddr, ETHER_ADDR_LEN); } /* * TODO: the device doesn't have MAC stored, we should read the * configuration stored in FLASH, but the format depends on the * bootloader used.* */ static int ece_get_mac(struct ece_softc *sc, u_char *eaddr) { return (ENXIO); } static void ece_intr_rx_locked(struct ece_softc *sc, int count) { struct ifnet *ifp = sc->ifp; struct mbuf *mb; struct rx_desc_info *rxdesc; eth_rx_desc_t *desc; int fssd_curr; int fssd; int i; int idx; int rxcount; uint32_t status; fssd_curr = read_4(sc, FS_DESCRIPTOR_POINTER); fssd = (fssd_curr - (uint32_t)sc->ring_paddr_rx)>>4; desc = sc->rx_desc[sc->last_rx].desc; /* Prepare to read the data in the ring. */ bus_dmamap_sync(sc->dmatag_ring_rx, sc->dmamap_ring_rx, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); if (fssd > sc->last_rx) rxcount = fssd - sc->last_rx; else if (fssd < sc->last_rx) rxcount = (ECE_MAX_RX_BUFFERS - sc->last_rx) + fssd; else { if (desc->cown == 0) return; else rxcount = ECE_MAX_RX_BUFFERS; } for (i= 0; i < rxcount; i++) { status = desc->cown; if (!status) break; idx = sc->last_rx; rxdesc = &sc->rx_desc[idx]; mb = rxdesc->buff; if (desc->length < ETHER_MIN_LEN - ETHER_CRC_LEN || desc->length > ETHER_MAX_LEN - ETHER_CRC_LEN + ETHER_VLAN_ENCAP_LEN) { if_inc_counter(ifp, IFCOUNTER_IERRORS, 1); desc->cown = 0; desc->length = MCLBYTES - 2; /* Invalid packet, skip and process next * packet. */ continue; } if (ece_new_rxbuf(sc, rxdesc) != 0) { if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1); desc->cown = 0; desc->length = MCLBYTES - 2; break; } /** * The device will write to addrress + 2 So we need to adjust * the address after the packet is received. */ mb->m_data += 2; mb->m_len = mb->m_pkthdr.len = desc->length; mb->m_flags |= M_PKTHDR; mb->m_pkthdr.rcvif = ifp; if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) { /*check for valid checksum*/ if ( (!desc->l4f) && (desc->prot != 3)) { mb->m_pkthdr.csum_flags |= CSUM_IP_CHECKED; mb->m_pkthdr.csum_flags |= CSUM_IP_VALID; mb->m_pkthdr.csum_data = 0xffff; } } ECE_RXUNLOCK(sc); (*ifp->if_input)(ifp, mb); ECE_RXLOCK(sc); desc->cown = 0; desc->length = MCLBYTES - 2; bus_dmamap_sync(sc->dmatag_ring_rx, sc->dmamap_ring_rx, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); if (sc->last_rx == ECE_MAX_RX_BUFFERS - 1) sc->last_rx = 0; else sc->last_rx++; desc = sc->rx_desc[sc->last_rx].desc; } /* Sync updated flags. */ bus_dmamap_sync(sc->dmatag_ring_rx, sc->dmamap_ring_rx, BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE); return; } static void ece_intr_task(void *arg, int pending __unused) { struct ece_softc *sc = arg; ECE_RXLOCK(sc); ece_intr_rx_locked(sc, -1); ECE_RXUNLOCK(sc); } static void ece_intr(void *xsc) { struct ece_softc *sc = xsc; struct ifnet *ifp = sc->ifp; if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { write_4(sc, FS_DMA_CONTROL, 0); return; } taskqueue_enqueue(sc->sc_tq, &sc->sc_intr_task); if (!IFQ_DRV_IS_EMPTY(&ifp->if_snd)) taskqueue_enqueue(sc->sc_tq, &sc->sc_tx_task); } static void ece_intr_status(void *xsc) { struct ece_softc *sc = xsc; struct ifnet *ifp = sc->ifp; int stat; stat = read_4(sc, INTERRUPT_STATUS); write_4(sc, INTERRUPT_STATUS, stat); if ((ifp->if_drv_flags & IFF_DRV_RUNNING) != 0) { if ((stat & ERROR_MASK) != 0) if_inc_counter(ifp, IFCOUNTER_IQDROPS, 1); } } static void ece_cleanup_locked(struct ece_softc *sc) { eth_tx_desc_t *desc; if (sc->tx_cons == sc->tx_prod) return; /* Prepare to read the ring (owner bit). */ bus_dmamap_sync(sc->dmatag_ring_tx, sc->dmamap_ring_tx, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); while (sc->tx_cons != sc->tx_prod) { desc = sc->tx_desc[sc->tx_cons].desc; if (desc->cown != 0) { struct tx_desc_info *td = &(sc->tx_desc[sc->tx_cons]); /* We are finished with this descriptor ... */ bus_dmamap_sync(sc->dmatag_data_tx, td->dmamap, BUS_DMASYNC_POSTWRITE); /* ... and unload, so we can reuse. */ bus_dmamap_unload(sc->dmatag_data_tx, td->dmamap); m_freem(td->buff); td->buff = 0; sc->tx_cons = (sc->tx_cons + 1) % ECE_MAX_TX_BUFFERS; } else { break; } } } static void ece_cleanup_task(void *arg, int pending __unused) { struct ece_softc *sc = arg; ECE_CLEANUPLOCK(sc); ece_cleanup_locked(sc); ECE_CLEANUPUNLOCK(sc); } static void ece_intr_tx(void *xsc) { struct ece_softc *sc = xsc; struct ifnet *ifp = sc->ifp; if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { /* This should not happen, stop DMA. */ write_4(sc, FS_DMA_CONTROL, 0); return; } taskqueue_enqueue(sc->sc_tq, &sc->sc_cleanup_task); } static void ece_intr_qf(void *xsc) { struct ece_softc *sc = xsc; struct ifnet *ifp = sc->ifp; if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { /* This should not happen, stop DMA. */ write_4(sc, FS_DMA_CONTROL, 0); return; } taskqueue_enqueue(sc->sc_tq, &sc->sc_intr_task); write_4(sc, FS_DMA_CONTROL, 1); } /* * Reset and initialize the chip */ static void eceinit_locked(void *xsc) { struct ece_softc *sc = xsc; struct ifnet *ifp = sc->ifp; struct mii_data *mii; uint32_t cfg_reg; uint32_t cpu_port_config; uint32_t mac_port_config; while (1) { cfg_reg = read_4(sc, BIST_RESULT_TEST_0); if ((cfg_reg & (1<<17))) break; DELAY(100); } /* Set to default values. */ write_4(sc, SWITCH_CONFIG, 0x007AA7A1); write_4(sc, MAC_PORT_0_CONFIG, 0x00423D00); write_4(sc, MAC_PORT_1_CONFIG, 0x00423D80); write_4(sc, CPU_PORT_CONFIG, 0x004C0000); hardware_init(sc); mac_port_config = read_4(sc, MAC_PORT_0_CONFIG); /* Enable Port 0 */ mac_port_config &= (~(PORT_DISABLE)); write_4(sc, MAC_PORT_0_CONFIG, mac_port_config); cpu_port_config = read_4(sc, CPU_PORT_CONFIG); /* Enable CPU. */ cpu_port_config &= ~(PORT_DISABLE); write_4(sc, CPU_PORT_CONFIG, cpu_port_config); /* * Set 'running' flag, and clear output active flag * and attempt to start the output */ ifp->if_drv_flags |= IFF_DRV_RUNNING; ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; mii = device_get_softc(sc->miibus); mii_pollstat(mii); /* Enable DMA. */ write_4(sc, FS_DMA_CONTROL, 1); callout_reset(&sc->tick_ch, hz, ece_tick, sc); } static inline int ece_encap(struct ece_softc *sc, struct mbuf *m0) { struct ifnet *ifp; bus_dma_segment_t segs[MAX_FRAGMENT]; bus_dmamap_t mapp; eth_tx_desc_t *desc = NULL; int csum_flags; int desc_no; int error; int nsegs; int seg; ifp = sc->ifp; /* Fetch unused map */ mapp = sc->tx_desc[sc->tx_prod].dmamap; error = bus_dmamap_load_mbuf_sg(sc->dmatag_ring_tx, mapp, m0, segs, &nsegs, BUS_DMA_NOWAIT); if (error != 0) { bus_dmamap_unload(sc->dmatag_ring_tx, mapp); return ((error != 0) ? error : -1); } desc = &(sc->desc_tx[sc->desc_curr_tx]); sc->tx_desc[sc->tx_prod].desc = desc; sc->tx_desc[sc->tx_prod].buff = m0; desc_no = sc->desc_curr_tx; for (seg = 0; seg < nsegs; seg++) { if (desc->cown == 0 ) { if_printf(ifp, "ERROR: descriptor is still used\n"); return (-1); } desc->length = segs[seg].ds_len; desc->data_ptr = segs[seg].ds_addr; if (seg == 0) { desc->fs = 1; } else { desc->fs = 0; } if (seg == nsegs - 1) { desc->ls = 1; } else { desc->ls = 0; } csum_flags = m0->m_pkthdr.csum_flags; desc->fr = 1; desc->pmap = 1; desc->insv = 0; desc->ico = 0; desc->tco = 0; desc->uco = 0; desc->interrupt = 1; if (csum_flags & CSUM_IP) { desc->ico = 1; if (csum_flags & CSUM_TCP) desc->tco = 1; if (csum_flags & CSUM_UDP) desc->uco = 1; } desc++; sc->desc_curr_tx = (sc->desc_curr_tx + 1) % ECE_MAX_TX_BUFFERS; if (sc->desc_curr_tx == 0) { desc = (eth_tx_desc_t *)&(sc->desc_tx[0]); } } desc = sc->tx_desc[sc->tx_prod].desc; sc->tx_prod = (sc->tx_prod + 1) % ECE_MAX_TX_BUFFERS; /* * After all descriptors are set, we set the flags to start the - * sending proces. + * sending process. */ for (seg = 0; seg < nsegs; seg++) { desc->cown = 0; desc++; desc_no = (desc_no + 1) % ECE_MAX_TX_BUFFERS; if (desc_no == 0) desc = (eth_tx_desc_t *)&(sc->desc_tx[0]); } bus_dmamap_sync(sc->dmatag_data_tx, mapp, BUS_DMASYNC_PREWRITE); return (0); } /* * dequeu packets and transmit */ static void ecestart_locked(struct ifnet *ifp) { struct ece_softc *sc; struct mbuf *m0; uint32_t queued = 0; sc = ifp->if_softc; if ((ifp->if_drv_flags & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) != IFF_DRV_RUNNING) return; bus_dmamap_sync(sc->dmatag_ring_tx, sc->dmamap_ring_tx, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); for (;;) { /* Get packet from the queue */ IF_DEQUEUE(&ifp->if_snd, m0); if (m0 == NULL) break; if (ece_encap(sc, m0)) { IF_PREPEND(&ifp->if_snd, m0); ifp->if_drv_flags |= IFF_DRV_OACTIVE; break; } queued++; BPF_MTAP(ifp, m0); } if (queued) { bus_dmamap_sync(sc->dmatag_ring_tx, sc->dmamap_ring_tx, BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); write_4(sc, TS_DMA_CONTROL, 1); } } static void eceinit(void *xsc) { struct ece_softc *sc = xsc; ECE_LOCK(sc); eceinit_locked(sc); ECE_UNLOCK(sc); } static void ece_tx_task(void *arg, int pending __unused) { struct ifnet *ifp; ifp = (struct ifnet *)arg; ecestart(ifp); } static void ecestart(struct ifnet *ifp) { struct ece_softc *sc = ifp->if_softc; ECE_TXLOCK(sc); ecestart_locked(ifp); ECE_TXUNLOCK(sc); } /* * Turn off interrupts, and stop the nic. Can be called with sc->ifp * NULL so be careful. */ static void ecestop(struct ece_softc *sc) { struct ifnet *ifp = sc->ifp; uint32_t mac_port_config; write_4(sc, TS_DMA_CONTROL, 0); write_4(sc, FS_DMA_CONTROL, 0); if (ifp) ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE); callout_stop(&sc->tick_ch); /*Disable Port 0 */ mac_port_config = read_4(sc, MAC_PORT_0_CONFIG); mac_port_config |= (PORT_DISABLE); write_4(sc, MAC_PORT_0_CONFIG, mac_port_config); /*Disable Port 1 */ mac_port_config = read_4(sc, MAC_PORT_1_CONFIG); mac_port_config |= (PORT_DISABLE); write_4(sc, MAC_PORT_1_CONFIG, mac_port_config); /* Disable all interrupt status sources. */ write_4(sc, INTERRUPT_MASK, 0x00001FFF); /* Clear previous interrupt sources. */ write_4(sc, INTERRUPT_STATUS, 0x00001FFF); write_4(sc, SWITCH_CONFIG, initial_switch_config); write_4(sc, CPU_PORT_CONFIG, initial_cpu_config); write_4(sc, MAC_PORT_0_CONFIG, initial_port0_config); write_4(sc, MAC_PORT_1_CONFIG, initial_port1_config); clear_mac_entries(sc, 1); } static void ece_restart(struct ece_softc *sc) { struct ifnet *ifp = sc->ifp; ifp->if_drv_flags |= IFF_DRV_RUNNING; ifp->if_drv_flags &= ~IFF_DRV_OACTIVE; /* Enable port 0. */ write_4(sc, PORT_0_CONFIG, read_4(sc, PORT_0_CONFIG) & ~(PORT_DISABLE)); write_4(sc, INTERRUPT_MASK, 0x00000000); write_4(sc, FS_DMA_CONTROL, 1); callout_reset(&sc->tick_ch, hz, ece_tick, sc); } static void set_filter(struct ece_softc *sc) { struct ifnet *ifp; struct ifmultiaddr *ifma; uint32_t mac_port_config; ifp = sc->ifp; clear_mac_entries(sc, 0); if (ifp->if_flags & IFF_ALLMULTI || ifp->if_flags & IFF_PROMISC) { mac_port_config = read_4(sc, MAC_PORT_0_CONFIG); mac_port_config &= ~(DISABLE_BROADCAST_PACKET); mac_port_config &= ~(DISABLE_MULTICAST_PACKET); write_4(sc, MAC_PORT_0_CONFIG, mac_port_config); return; } if_maddr_rlock(ifp); TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_LINK) continue; add_mac_entry(sc, LLADDR((struct sockaddr_dl *)ifma->ifma_addr)); } if_maddr_runlock(ifp); } static int eceioctl(struct ifnet *ifp, u_long cmd, caddr_t data) { struct ece_softc *sc = ifp->if_softc; struct mii_data *mii; struct ifreq *ifr = (struct ifreq *)data; int mask, error = 0; switch (cmd) { case SIOCSIFFLAGS: ECE_LOCK(sc); if ((ifp->if_flags & IFF_UP) == 0 && ifp->if_drv_flags & IFF_DRV_RUNNING) { ifp->if_drv_flags &= ~IFF_DRV_RUNNING; ecestop(sc); } else { /* Reinitialize card on any parameter change. */ if ((ifp->if_flags & IFF_UP) && !(ifp->if_drv_flags & IFF_DRV_RUNNING)) ece_restart(sc); } ECE_UNLOCK(sc); break; case SIOCADDMULTI: case SIOCDELMULTI: ECE_LOCK(sc); set_filter(sc); ECE_UNLOCK(sc); break; case SIOCSIFMEDIA: case SIOCGIFMEDIA: mii = device_get_softc(sc->miibus); error = ifmedia_ioctl(ifp, ifr, &mii->mii_media, cmd); break; case SIOCSIFCAP: mask = ifp->if_capenable ^ ifr->ifr_reqcap; if (mask & IFCAP_VLAN_MTU) { ECE_LOCK(sc); ECE_UNLOCK(sc); } default: error = ether_ioctl(ifp, cmd, data); break; } return (error); } static void ece_child_detached(device_t dev, device_t child) { struct ece_softc *sc; sc = device_get_softc(dev); if (child == sc->miibus) sc->miibus = NULL; } /* * MII bus support routines. */ static int ece_miibus_readreg(device_t dev, int phy, int reg) { struct ece_softc *sc; sc = device_get_softc(dev); return (phy_read(sc, phy, reg)); } static int ece_miibus_writereg(device_t dev, int phy, int reg, int data) { struct ece_softc *sc; sc = device_get_softc(dev); phy_write(sc, phy, reg, data); return (0); } static device_method_t ece_methods[] = { /* Device interface */ DEVMETHOD(device_probe, ece_probe), DEVMETHOD(device_attach, ece_attach), DEVMETHOD(device_detach, ece_detach), /* Bus interface */ DEVMETHOD(bus_child_detached, ece_child_detached), /* MII interface */ DEVMETHOD(miibus_readreg, ece_miibus_readreg), DEVMETHOD(miibus_writereg, ece_miibus_writereg), { 0, 0 } }; static driver_t ece_driver = { "ece", ece_methods, sizeof(struct ece_softc), }; DRIVER_MODULE(ece, econaarm, ece_driver, ece_devclass, 0, 0); DRIVER_MODULE(miibus, ece, miibus_driver, miibus_devclass, 0, 0); MODULE_DEPEND(ece, miibus, 1, 1, 1); MODULE_DEPEND(ece, ether, 1, 1, 1); Index: user/ngie/release-pkg-fix-tests/sys/arm/freescale/imx/imx6_ipu.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/freescale/imx/imx6_ipu.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/freescale/imx/imx6_ipu.c (revision 299091) @@ -1,1203 +1,1203 @@ /*- * Copyright 2015 Oleksandr Tymoshenko * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "fb_if.h" #include "hdmi_if.h" #define EDID_DEBUG_not static int have_ipu = 0; #define LDB_CLOCK_RATE 280000000 #define MODE_HBP(mode) ((mode)->htotal - (mode)->hsync_end) #define MODE_HFP(mode) ((mode)->hsync_start - (mode)->hdisplay) #define MODE_HSW(mode) ((mode)->hsync_end - (mode)->hsync_start) #define MODE_VBP(mode) ((mode)->vtotal - (mode)->vsync_end) #define MODE_VFP(mode) ((mode)->vsync_start - (mode)->vdisplay) #define MODE_VSW(mode) ((mode)->vsync_end - (mode)->vsync_start) #define MODE_BPP 16 #define MODE_PIXEL_CLOCK_INVERT 1 #define M(nm,hr,vr,clk,hs,he,ht,vs,ve,vt,f) \ { clk, hr, hs, he, ht, vr, vs, ve, vt, f, nm } static struct videomode mode1024x768 = M("1024x768x60",1024,768,65000,1048,1184,1344,771,777,806,VID_NHSYNC|VID_PHSYNC); #define DMA_CHANNEL 23 #define DC_CHAN5 5 #define DI_PORT 0 #define IPU_LOCK(_sc) mtx_lock(&(_sc)->sc_mtx) #define IPU_UNLOCK(_sc) mtx_unlock(&(_sc)->sc_mtx) #define IPU_LOCK_INIT(_sc) mtx_init(&(_sc)->sc_mtx, \ device_get_nameunit(_sc->sc_dev), "ipu", MTX_DEF) #define IPU_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->sc_mtx) #define IPU_READ4(_sc, reg) bus_read_4((_sc)->sc_mem_res, (reg)) #define IPU_WRITE4(_sc, reg, value) \ bus_write_4((_sc)->sc_mem_res, (reg), (value)) #define CPMEM_BASE 0x300000 #define DC_TEMPL_BASE 0x380000 /* Microcode */ /* Word 1 */ #define TEMPLATE_SYNC(v) ((v) << 0) #define TEMPLATE_GLUELOGIC(v) ((v) << 4) #define TEMPLATE_MAPPING(v) ((v) << 15) #define TEMPLATE_WAVEFORM(v) ((v) << 11) #define GLUELOGIC_KEEP_ASSERTED (1 << 3) #define GLUELOGIC_KEEP_NEGATED (1 << 2) /* Word 2 */ #define TEMPLATE_OPCODE(v) ((v) << 4) #define OPCODE_WROD 0x18 #define TEMPLATE_STOP (1 << 9) #define IPU_CONF 0x200000 #define IPU_CONF_DMFC_EN (1 << 10) #define IPU_CONF_DC_EN (1 << 9) #define IPU_CONF_DI1_EN (1 << 7) #define IPU_CONF_DI0_EN (1 << 6) #define IPU_CONF_DP_EN (1 << 5) #define IPU_DISP_GEN 0x2000C4 #define DISP_GEN_DI1_CNTR_RELEASE (1 << 25) #define DISP_GEN_DI0_CNTR_RELEASE (1 << 24) #define DISP_GEN_MCU_MAX_BURST_STOP (1 << 22) #define DISP_GEN_MCU_T_SHIFT 18 #define IPU_MEM_RST 0x2000DC #define IPU_MEM_RST_START (1 << 31) #define IPU_MEM_RST_ALL 0x807FFFFF #define IPU_CH_DB_MODE_SEL_0 0x200150 #define IPU_CH_DB_MODE_SEL_1 0x200154 #define IPU_CUR_BUF_0 0x20023C #define IPU_CUR_BUF_1 0x200240 #define IPU_IDMAC_CH_EN_1 0x208004 #define IPU_IDMAC_CH_EN_2 0x208008 #define IPU_IDMAC_CH_PRI_1 0x208014 #define IPU_IDMAC_CH_PRI_2 0x208018 #define IPU_DI0_GENERAL 0x240000 #define DI_CLOCK_EXTERNAL (1 << 20) #define DI_GENERAL_POL_CLK (1 << 17) #define DI_GENERAL_POLARITY_3 (1 << 2) #define DI_GENERAL_POLARITY_2 (1 << 1) #define IPU_DI0_BS_CLKGEN0 0x240004 #define DI_BS_CLKGEN0(_int, _frac) (((_int) << 4) | (_frac)) #define IPU_DI0_BS_CLKGEN1 0x240008 #define DI_BS_CLKGEN1_DOWN(_int, _frac) ((((_int) << 1) | (_frac)) << 16) #define IPU_DI0_SW_GEN0_1 0x24000C #define DI_RUN_VALUE_M1(v) ((v) << 19) #define DI_RUN_RESOLUTION(v) ((v) << 16) #define DI_OFFSET_VALUE(v) ((v) << 3) #define IPU_DI0_SW_GEN1_1 0x240030 #define DI_CNT_POLARITY_GEN_EN(v) ((v) << 29) #define DI_CNT_AUTO_RELOAD (1 << 28) #define DI_CNT_CLR_SEL(v) ((v) << 25) #define DI_CNT_DOWN(v) ((v) << 16) #define DI_CNT_POLARITY_TRIGGER_SEL(v) ((v) << 12) #define DI_CNT_POLARITY_CLR_SEL(v) ((v) << 9) #define IPU_DI0_SYNC_AS_GEN 0x240054 #define SYNC_AS_GEN_VSYNC_SEL(v) ((v) << 13) #define SYNC_AS_GEN_SYNC_START(v) ((v) << 0) #define IPU_DI0_DW_GEN_0 0x240058 #define DW_GEN_DI_ACCESS_SIZE(v) ((v) << 24) #define DW_GEN_DI_COMPONENT_SIZE(v) ((v) << 16) #define DW_GEN_DI_SET_MASK 3 #define DW_GEN_DI_PIN_15_SET(v) ((v) << 8) #define IPU_DI0_DW_SET3_0 0x240118 #define DW_SET_DATA_CNT_DOWN(v) ((v) << 16) #define DW_SET_DATA_CNT_UP(v) ((v) << 0) #define IPU_DI0_STP_REP 0x240148 #define IPU_DI0_POL 0x240164 #define DI_POL_DRDY_POLARITY_15 (1 << 4) #define IPU_DI0_SCR_CONF 0x240170 #define IPU_DI1_GENERAL 0x248000 #define IPU_DI1_BS_CLKGEN0 0x248004 #define IPU_DI1_BS_CLKGEN1 0x248008 #define IPU_DI1_SW_GEN0_1 0x24800C #define IPU_DI1_SW_GEN1_1 0x248030 #define IPU_DI1_SYNC_AS_GEN 0x248054 #define IPU_DI1_DW_GEN_0 0x248058 #define IPU_DI1_POL 0x248164 #define IPU_DI1_DW_SET3_0 0x248118 #define IPU_DI1_STP_REP 0x248148 #define IPU_DI1_SCR_CONF 0x248170 #define DMFC_RD_CHAN 0x260000 #define DMFC_WR_CHAN 0x260004 #define DMFC_WR_CHAN_BURST_SIZE_32 (0 << 6) #define DMFC_WR_CHAN_BURST_SIZE_16 (1 << 6) #define DMFC_WR_CHAN_BURST_SIZE_8 (2 << 6) #define DMFC_WR_CHAN_BURST_SIZE_4 (3 << 6) #define DMFC_WR_CHAN_BURST_SIZE_4 (3 << 6) #define DMFC_WR_CHAN_FIFO_SIZE_128 (2 << 3) #define DMFC_WR_CHAN_DEF 0x260008 #define DMFC_WR_CHAN_DEF_WM_CLR_2C(v) ((v) << 29) #define DMFC_WR_CHAN_DEF_WM_CLR_1C(v) ((v) << 21) #define DMFC_WR_CHAN_DEF_WM_CLR_2(v) ((v) << 13) #define DMFC_WR_CHAN_DEF_WM_CLR_1(v) ((v) << 5) #define DMFC_WR_CHAN_DEF_WM_SET_1(v) ((v) << 2) #define DMFC_WR_CHAN_DEF_WM_EN_1 (1 << 1) #define DMFC_DP_CHAN 0x26000C #define DMFC_DP_CHAN_BURST_SIZE_8 2 #define DMFC_DP_CHAN_FIFO_SIZE_256 1 #define DMFC_DP_CHAN_FIFO_SIZE_128 2 #define DMFC_DP_CHAN_BURST_SIZE_5F(v) ((v) << 14) #define DMFC_DP_CHAN_FIFO_SIZE_5F(v) ((v) << 11) #define DMFC_DP_CHAN_ST_ADDR_SIZE_5F(v) ((v) << 8) #define DMFC_DP_CHAN_BURST_SIZE_5B(v) ((v) << 6) #define DMFC_DP_CHAN_FIFO_SIZE_5B(v) ((v) << 3) #define DMFC_DP_CHAN_ST_ADDR_SIZE_5B(v) ((v) << 0) #define DMFC_DP_CHAN_DEF 0x260010 #define DMFC_DP_CHAN_DEF_WM_CLR_6F(v) ((v) << 29) #define DMFC_DP_CHAN_DEF_WM_CLR_6B(v) ((v) << 21) #define DMFC_DP_CHAN_DEF_WM_CLR_5F(v) ((v) << 13) #define DMFC_DP_CHAN_DEF_WM_SET_5F(v) ((v) << 10) #define DMFC_DP_CHAN_DEF_WM_EN_5F (1 << 9) #define DMFC_DP_CHAN_DEF_WM_CLR_5B(v) ((v) << 5) #define DMFC_DP_CHAN_DEF_WM_SET_5B(v) ((v) << 2) #define DMFC_DP_CHAN_DEF_WM_EN_5B (1 << 1) #define DMFC_GENERAL_1 0x260014 #define DMFC_GENERAL_1_WAIT4EOT_5B (1 << 20) #define DMFC_IC_CTRL 0x26001C #define DMFC_IC_CTRL_DISABLED 0x2 #define DC_WRITE_CH_CONF_1 0x0025801C #define WRITE_CH_CONF_PROG_CHAN_TYP_MASK (7 << 5) #define WRITE_CH_CONF_PROG_CHAN_NORMAL (4 << 5) #define DC_WRITE_CH_ADDR_1 0x00258020 #define DC_WRITE_CH_CONF_5 0x0025805C #define WRITE_CH_CONF_PROG_DISP_ID(v) ((v) << 3) #define WRITE_CH_CONF_PROG_DI_ID(v) ((v) << 2) #define WRITE_CH_CONF_PROG_W_SIZE(v) (v) #define DC_WRITE_CH_ADDR_5 0x00258060 #define DC_RL0_CH_5 0x00258064 #define DC_GEN 0x002580D4 #define DC_GEN_SYNC_PRIORITY (1 << 7) #define DC_GEN_ASYNC (0 << 1) #define DC_GEN_SYNC (2 << 1) #define DC_DISP_CONF2(di) (0x002580E8 + (di) * 4) #define DC_MAP_CONF_0 0x00258108 #define DC_MAP_CONF_15 0x00258144 #define DC_MAP_CONF_VAL(map) (DC_MAP_CONF_15 + ((map) / 2) * sizeof(uint32_t)) #define MAP_CONF_VAL_MASK 0xffff #define DC_MAP_CONF_PTR(ptr) (DC_MAP_CONF_0 + ((ptr) / 2) * sizeof(uint32_t)) #define MAP_CONF_PTR_MASK 0x1f #define DI_COUNTER_INT_HSYNC 1 #define DI_COUNTER_HSYNC 2 #define DI_COUNTER_VSYNC 3 #define DI_COUNTER_AD_0 4 #define DI_COUNTER_AD_1 5 #define DI_SYNC_NONE 0 #define DI_SYNC_CLK 1 #define DI_SYNC_COUNTER(c) ((c) + 1) struct ipu_cpmem_word { uint32_t data[5]; uint32_t padding[3]; }; struct ipu_cpmem_ch_param { struct ipu_cpmem_word word[2]; }; #define CH_PARAM_RESET(param) memset(param, 0, sizeof(*param)) #define IPU_READ_CH_PARAM(_sc, ch, param) bus_read_region_4( \ (_sc)->sc_mem_res, CPMEM_BASE + ch * (sizeof(*param)),\ (uint32_t*)param, sizeof(*param) / 4) #define IPU_WRITE_CH_PARAM(_sc, ch, param) bus_write_region_4( \ (_sc)->sc_mem_res, CPMEM_BASE + ch * (sizeof(*param)),\ (uint32_t*)param, sizeof(*param) / 4) #define CH_PARAM_SET_FW(param, v) ipu_ch_param_set_value((param), \ 0, 125, 13, (v)) #define CH_PARAM_SET_FH(param, v) ipu_ch_param_set_value((param), \ 0, 138, 12, (v)) #define CH_PARAM_SET_SLY(param, v) ipu_ch_param_set_value((param), \ 1, 102, 14, (v)) #define CH_PARAM_SET_EBA0(param, v) ipu_ch_param_set_value((param), \ 1, 0, 29, (v)) #define CH_PARAM_SET_EBA1(param, v) ipu_ch_param_set_value((param), \ 1, 29, 29, (v)) #define CH_PARAM_SET_BPP(param, v) ipu_ch_param_set_value((param), \ 0, 107, 3, (v)) #define CH_PARAM_SET_PFS(param, v) ipu_ch_param_set_value((param), \ 1, 85, 4, (v)) #define CH_PARAM_SET_NPB(param, v) ipu_ch_param_set_value((param), \ 1, 78, 7, (v)) #define CH_PARAM_SET_UBO(param, v) ipu_ch_param_set_value((param), \ 0, 46, 22, (v)) #define CH_PARAM_SET_VBO(param, v) ipu_ch_param_set_value((param), \ 0, 68, 22, (v)) #define CH_PARAM_SET_RED_WIDTH(param, v) ipu_ch_param_set_value((param), \ 1, 116, 3, (v)) #define CH_PARAM_SET_RED_OFFSET(param, v) ipu_ch_param_set_value((param), \ 1, 128, 5, (v)) #define CH_PARAM_SET_GREEN_WIDTH(param, v) ipu_ch_param_set_value((param), \ 1, 119, 3, (v)) #define CH_PARAM_SET_GREEN_OFFSET(param, v) ipu_ch_param_set_value((param), \ 1, 133, 5, (v)) #define CH_PARAM_SET_BLUE_WIDTH(param, v) ipu_ch_param_set_value((param), \ 1, 122, 3, (v)) #define CH_PARAM_SET_BLUE_OFFSET(param, v) ipu_ch_param_set_value((param), \ 1, 138, 5, (v)) #define CH_PARAM_SET_ALPHA_WIDTH(param, v) ipu_ch_param_set_value((param), \ 1, 125, 3, (v)) #define CH_PARAM_SET_ALPHA_OFFSET(param, v) ipu_ch_param_set_value((param), \ 1, 143, 5, (v)) #define CH_PARAM_GET_FW(param) ipu_ch_param_get_value((param), \ 0, 125, 13) #define CH_PARAM_GET_FH(param) ipu_ch_param_get_value((param), \ 0, 138, 12) #define CH_PARAM_GET_SLY(param) ipu_ch_param_get_value((param), \ 1, 102, 14) #define CH_PARAM_GET_EBA0(param) ipu_ch_param_get_value((param), \ 1, 0, 29) #define CH_PARAM_GET_EBA1(param) ipu_ch_param_get_value((param), \ 1, 29, 29) #define CH_PARAM_GET_BPP(param) ipu_ch_param_get_value((param), \ 0, 107, 3) #define CH_PARAM_GET_PFS(param) ipu_ch_param_get_value((param), \ 1, 85, 4) #define CH_PARAM_GET_NPB(param) ipu_ch_param_get_value((param), \ 1, 78, 7) #define CH_PARAM_GET_UBO(param) ipu_ch_param_get_value((param), \ 0, 46, 22) #define CH_PARAM_GET_VBO(param) ipu_ch_param_get_value((param), \ 0, 68, 22) #define CH_PARAM_GET_RED_WIDTH(param) ipu_ch_param_get_value((param), \ 1, 116, 3) #define CH_PARAM_GET_RED_OFFSET(param) ipu_ch_param_get_value((param), \ 1, 128, 5) #define CH_PARAM_GET_GREEN_WIDTH(param) ipu_ch_param_get_value((param), \ 1, 119, 3) #define CH_PARAM_GET_GREEN_OFFSET(param) ipu_ch_param_get_value((param), \ 1, 133, 5) #define CH_PARAM_GET_BLUE_WIDTH(param) ipu_ch_param_get_value((param), \ 1, 122, 3) #define CH_PARAM_GET_BLUE_OFFSET(param) ipu_ch_param_get_value((param), \ 1, 138, 5) #define CH_PARAM_GET_ALPHA_WIDTH(param) ipu_ch_param_get_value((param), \ 1, 125, 3) #define CH_PARAM_GET_ALPHA_OFFSET(param) ipu_ch_param_get_value((param), \ 1, 143, 5) #define IPU_PIX_FORMAT_BPP_32 0 #define IPU_PIX_FORMAT_BPP_24 1 #define IPU_PIX_FORMAT_BPP_18 2 #define IPU_PIX_FORMAT_BPP_16 3 #define IPU_PIX_FORMAT_BPP_12 4 #define IPU_PIX_FORMAT_BPP_8 5 #define IPU_PIX_FORMAT_BPP_ #define IPU_PIX_FORMAT_RGB 7 enum dc_event_t { DC_EVENT_NF = 0, DC_EVENT_NL, DC_EVENT_EOF, DC_EVENT_NFIELD, DC_EVENT_EOL, DC_EVENT_EOFIELD, DC_EVENT_NEW_ADDR, DC_EVENT_NEW_CHAN, DC_EVENT_NEW_DATA }; struct ipu_softc { device_t sc_dev; struct resource *sc_mem_res; int sc_mem_rid; struct resource *sc_irq_res; int sc_irq_rid; void *sc_intr_hl; struct mtx sc_mtx; struct fb_info sc_fb_info; struct videomode *sc_mode; /* Framebuffer */ bus_dma_tag_t sc_dma_tag; bus_dmamap_t sc_dma_map; size_t sc_fb_size; bus_addr_t sc_fb_phys; uint8_t *sc_fb_base; /* HDMI */ eventhandler_tag sc_hdmi_evh; }; static void ipu_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int err) { bus_addr_t *addr; if (err) return; addr = (bus_addr_t*)arg; *addr = segs[0].ds_addr; } static void ipu_ch_param_set_value(struct ipu_cpmem_ch_param *param, int word, unsigned int offset, int len, uint32_t value) { uint32_t datapos, bitpos, mask; uint32_t data, data2; KASSERT((len <= 32), ("%s: field len is more than 32", __func__)); datapos = offset / 32; bitpos = offset % 32; mask = (1 << len) - 1; data = param->word[word].data[datapos]; data &= ~(mask << bitpos); data |= (value << bitpos); param->word[word].data[datapos] = data; if ((bitpos + len) > 32) { len = bitpos + len - 32; mask = (1UL << len) - 1; data2 = param->word[word].data[datapos + 1]; data2 &= mask; data2 |= (value >> (32 - bitpos)); param->word[word].data[datapos + 1] = data2; } } #ifdef DEBUG static uint32_t ipu_ch_param_get_value(struct ipu_cpmem_ch_param *param, int word, unsigned int offset, int len) { uint32_t datapos, bitpos, mask; uint32_t data, data2; KASSERT((len <= 32), ("%s: field len is more than 32", __func__)); datapos = offset / 32; bitpos = offset % 32; mask = (1UL << len) - 1; data = param->word[word].data[datapos]; data = data >> bitpos; data &= mask; if ((bitpos + len) > 32) { len = bitpos + len - 32; mask = (1UL << len) - 1; data2 = param->word[word].data[datapos + 1]; data2 &= mask; data |= (data2 << (32 - bitpos)); } return (data); } static void ipu_print_channel(struct ipu_cpmem_ch_param *param) { int offset0[] = {0, 10, 19, 32, 44, 45, 46, 68, 90, 94, 95, 113, 114, 117, 119, 120, 121, 122, 123, 124, 125, 138, 150, 151, -1}; int offset1[] = {0, 29, 58, 78, 85, 89, 90, 93, 95, 102, 116, 119, 122, 125, 128, 133, 138, 143, 148, 149, 150, -1}; printf("WORD0: %08x %08x %08x %08x %08x\n", param->word[0].data[0], param->word[0].data[1], param->word[0].data[2], param->word[0].data[3], param->word[0].data[4]); printf("WORD1: %08x %08x %08x %08x %08x\n", param->word[1].data[0], param->word[1].data[1], param->word[1].data[2], param->word[1].data[3], param->word[1].data[4]); for (int i = 0; offset0[i + 1] != -1; i++) { int len = offset0[i + 1] - offset0[i]; printf("W0[%d:%d] = %d\n", offset0[i], offset0[i] + len - 1, ipu_ch_param_get_value(param, 0, offset0[i], len) ); } for (int i = 0; offset1[i + 1] != -1; i++) { int len = offset1[i + 1] - offset1[i]; printf("W1[%d:%d] = %d\n", offset1[i], offset1[i] + len - 1, ipu_ch_param_get_value(param, 1, offset1[i], len) ); } printf("FW: %d\n", CH_PARAM_GET_FW(param)); printf("FH: %d\n", CH_PARAM_GET_FH(param)); printf("SLY: %d\n", CH_PARAM_GET_SLY(param)); printf("EBA0: 0x%08x\n", CH_PARAM_GET_EBA0(param)); printf("EBA1: 0x%08x\n", CH_PARAM_GET_EBA1(param)); printf("BPP: %d\n", CH_PARAM_GET_BPP(param)); printf("PFS: %d\n", CH_PARAM_GET_PFS(param)); printf("NPB: %d\n", CH_PARAM_GET_NPB(param)); printf("UBO: %d\n", CH_PARAM_GET_UBO(param)); printf("VBO: %d\n", CH_PARAM_GET_VBO(param)); printf("RED: %d bits @%d\n", CH_PARAM_GET_RED_WIDTH(param) + 1, CH_PARAM_GET_RED_OFFSET(param)); printf("GREEN: %d bits @%d\n", CH_PARAM_GET_GREEN_WIDTH(param) + 1, CH_PARAM_GET_GREEN_OFFSET(param)); printf("BLUE: %d bits @%d\n", CH_PARAM_GET_BLUE_WIDTH(param) + 1, CH_PARAM_GET_BLUE_OFFSET(param)); printf("ALPHA: %d bits @%d\n", CH_PARAM_GET_ALPHA_WIDTH(param) + 1, CH_PARAM_GET_ALPHA_OFFSET(param)); } #endif static void ipu_di_enable(struct ipu_softc *sc, int di) { uint32_t flag, reg; flag = di ? DISP_GEN_DI1_CNTR_RELEASE : DISP_GEN_DI0_CNTR_RELEASE; reg = IPU_READ4(sc, IPU_DISP_GEN); reg |= flag; IPU_WRITE4(sc, IPU_DISP_GEN, reg); } static void ipu_config_wave_gen_0(struct ipu_softc *sc, int di, int wave_gen, int run_value, int run_res, int offset_value, int offset_res) { uint32_t addr, reg; addr = (di ? IPU_DI1_SW_GEN0_1 : IPU_DI0_SW_GEN0_1) + (wave_gen - 1) * sizeof(uint32_t); reg = DI_RUN_VALUE_M1(run_value) | DI_RUN_RESOLUTION(run_res) | DI_OFFSET_VALUE(offset_value) | offset_res; IPU_WRITE4(sc, addr, reg); } static void ipu_config_wave_gen_1(struct ipu_softc *sc, int di, int wave_gen, int repeat_count, int cnt_clr_src, int cnt_polarity_gen_en, int cnt_polarity_clr_src, int cnt_polarity_trigger_src, int cnt_up, int cnt_down) { uint32_t addr, reg; addr = (di ? IPU_DI1_SW_GEN1_1 : IPU_DI0_SW_GEN1_1) + (wave_gen - 1) * sizeof(uint32_t); reg = DI_CNT_POLARITY_GEN_EN(cnt_polarity_gen_en) | DI_CNT_CLR_SEL(cnt_clr_src) | DI_CNT_POLARITY_TRIGGER_SEL(cnt_polarity_trigger_src) | DI_CNT_POLARITY_CLR_SEL(cnt_polarity_clr_src); reg |= DI_CNT_DOWN(cnt_down) | cnt_up; if (repeat_count == 0) reg |= DI_CNT_AUTO_RELOAD; IPU_WRITE4(sc, addr, reg); addr = (di ? IPU_DI1_STP_REP : IPU_DI0_STP_REP) + (wave_gen - 1) / 2 * sizeof(uint32_t); reg = IPU_READ4(sc, addr); if (wave_gen % 2) { reg &= ~(0xffff); reg |= repeat_count; } else { reg &= ~(0xffff << 16); reg |= (repeat_count << 16); } IPU_WRITE4(sc, addr, reg); } static void ipu_reset_wave_gen(struct ipu_softc *sc, int di, int wave_gen) { uint32_t addr, reg; addr = (di ? IPU_DI1_SW_GEN0_1 : IPU_DI0_SW_GEN0_1) + (wave_gen - 1) * sizeof(uint32_t); IPU_WRITE4(sc, addr, 0); addr = (di ? IPU_DI1_SW_GEN1_1 : IPU_DI0_SW_GEN1_1) + (wave_gen - 1) * sizeof(uint32_t); IPU_WRITE4(sc, addr, 0); addr = (di ? IPU_DI1_STP_REP : IPU_DI0_STP_REP) + (wave_gen - 1) / 2 * sizeof(uint32_t); reg = IPU_READ4(sc, addr); if (wave_gen % 2) reg &= ~(0xffff); else reg &= ~(0xffff << 16); IPU_WRITE4(sc, addr, reg); } static void ipu_init_microcode_template(struct ipu_softc *sc, int di, int map) { uint32_t addr; uint32_t w1, w2; int i, word; int glue; word = di ? 2 : 5; for (i = 0; i < 3; i++) { if (i == 0) glue = GLUELOGIC_KEEP_ASSERTED; else if (i == 1) glue = GLUELOGIC_KEEP_NEGATED; else if (i == 2) glue = 0; w1 = TEMPLATE_SYNC(5) | TEMPLATE_GLUELOGIC(glue) | TEMPLATE_WAVEFORM(1) | /* wave unit 0 */ TEMPLATE_MAPPING(map + 1); /* operand is zero */ /* Write data to DI and Hold data in register */ w2 = TEMPLATE_OPCODE(OPCODE_WROD) | TEMPLATE_STOP; addr = DC_TEMPL_BASE + (word + i) * 2 * sizeof(uint32_t); IPU_WRITE4(sc, addr, w1); IPU_WRITE4(sc, addr + sizeof(uint32_t), w2); } } static void ipu_config_timing(struct ipu_softc *sc, int di) { int div; uint32_t di_scr_conf; uint32_t gen_offset, gen; uint32_t as_gen_offset, as_gen; uint32_t dw_gen_offset, dw_gen; uint32_t dw_set_offset, dw_set; uint32_t bs_clkgen_offset; int map; /* TODO: check mode restrictions / fixup */ /* TODO: enable timers, get divisors */ div = 1; map = 0; bs_clkgen_offset = di ? IPU_DI1_BS_CLKGEN0 : IPU_DI0_BS_CLKGEN0; IPU_WRITE4(sc, bs_clkgen_offset, DI_BS_CLKGEN0(div, 0)); /* half of the divider */ IPU_WRITE4(sc, bs_clkgen_offset + 4, DI_BS_CLKGEN1_DOWN(div / 2, div % 2)); /* * TODO: Configure LLDB clock by changing following fields * in CCM fields: * CS2CDR_LDB_DI0_CLK_SEL * CSCMR2_LDB_DI0_IPU_DIV * CBCDR_MMDC_CH1_AXI_PODF */ /* Setup wave generator */ dw_gen_offset = di ? IPU_DI1_DW_GEN_0 : IPU_DI0_DW_GEN_0; dw_gen = DW_GEN_DI_ACCESS_SIZE(div - 1) | DW_GEN_DI_COMPONENT_SIZE(div - 1); dw_gen &= ~DW_GEN_DI_PIN_15_SET(DW_GEN_DI_SET_MASK); dw_gen |= DW_GEN_DI_PIN_15_SET(3); /* set 3*/ IPU_WRITE4(sc, dw_gen_offset, dw_gen); dw_set_offset = di ? IPU_DI1_DW_SET3_0 : IPU_DI0_DW_SET3_0; dw_set = DW_SET_DATA_CNT_DOWN(div * 2) | DW_SET_DATA_CNT_UP(0); IPU_WRITE4(sc, dw_set_offset, dw_set); /* DI_COUNTER_INT_HSYNC */ ipu_config_wave_gen_0(sc, di, DI_COUNTER_INT_HSYNC, sc->sc_mode->htotal - 1, DI_SYNC_CLK, 0, DI_SYNC_NONE); ipu_config_wave_gen_1(sc, di, DI_COUNTER_INT_HSYNC, 0, DI_SYNC_NONE, 0, DI_SYNC_NONE, DI_SYNC_NONE, 0, 0); /* DI_COUNTER_HSYNC */ ipu_config_wave_gen_0(sc, di, DI_COUNTER_HSYNC, sc->sc_mode->htotal - 1, DI_SYNC_CLK, 0, DI_SYNC_CLK); ipu_config_wave_gen_1(sc, di, DI_COUNTER_HSYNC, 0, DI_SYNC_NONE, 1, DI_SYNC_NONE, DI_SYNC_CLK, 0, MODE_HSW(sc->sc_mode) * 2); /* DI_COUNTER_VSYNC */ ipu_config_wave_gen_0(sc, di, DI_COUNTER_VSYNC, sc->sc_mode->vtotal - 1, DI_SYNC_COUNTER(DI_COUNTER_INT_HSYNC), 0, DI_SYNC_NONE); ipu_config_wave_gen_1(sc, di, DI_COUNTER_VSYNC, 0, DI_SYNC_NONE, 1, DI_SYNC_NONE, DI_SYNC_COUNTER(DI_COUNTER_INT_HSYNC), 0, MODE_VSW(sc->sc_mode) * 2); di_scr_conf = di ? IPU_DI1_SCR_CONF : IPU_DI0_SCR_CONF; IPU_WRITE4(sc, di_scr_conf, sc->sc_mode->vtotal - 1); /* TODO: update DI_SCR_CONF */ /* Active Data 0 */ ipu_config_wave_gen_0(sc, di, DI_COUNTER_AD_0, 0, DI_SYNC_COUNTER(DI_COUNTER_HSYNC), MODE_VSW(sc->sc_mode) + MODE_VFP(sc->sc_mode), DI_SYNC_COUNTER(DI_COUNTER_HSYNC)); ipu_config_wave_gen_1(sc, di, DI_COUNTER_AD_0, sc->sc_mode->vdisplay, DI_SYNC_COUNTER(DI_COUNTER_VSYNC), 0, DI_SYNC_NONE, DI_SYNC_NONE, 0, 0); ipu_config_wave_gen_0(sc, di, DI_COUNTER_AD_1, 0, DI_SYNC_CLK, MODE_HSW(sc->sc_mode) + MODE_HFP(sc->sc_mode), DI_SYNC_CLK); ipu_config_wave_gen_1(sc, di, DI_COUNTER_AD_1, sc->sc_mode->hdisplay, DI_SYNC_COUNTER(DI_COUNTER_AD_0), 0, DI_SYNC_NONE, DI_SYNC_NONE, 0, 0); ipu_reset_wave_gen(sc, di, 6); ipu_reset_wave_gen(sc, di, 7); ipu_reset_wave_gen(sc, di, 8); ipu_reset_wave_gen(sc, di, 9); ipu_init_microcode_template(sc, di, map); gen_offset = di ? IPU_DI1_GENERAL : IPU_DI0_GENERAL; gen = IPU_READ4(sc, gen_offset); if (sc->sc_mode->flags & VID_NHSYNC) gen &= ~DI_GENERAL_POLARITY_2; else /* active high */ gen |= DI_GENERAL_POLARITY_2; if (sc->sc_mode->flags & VID_NVSYNC) gen &= ~DI_GENERAL_POLARITY_3; else /* active high */ gen |= DI_GENERAL_POLARITY_3; if (MODE_PIXEL_CLOCK_INVERT) gen &= ~DI_GENERAL_POL_CLK; else gen |= DI_GENERAL_POL_CLK; /* Use LDB clock to drive pixel clock */ gen |= DI_CLOCK_EXTERNAL; IPU_WRITE4(sc, gen_offset, gen); as_gen_offset = di ? IPU_DI1_SYNC_AS_GEN : IPU_DI0_SYNC_AS_GEN; as_gen = SYNC_AS_GEN_VSYNC_SEL(DI_COUNTER_VSYNC - 1) | SYNC_AS_GEN_SYNC_START(2); IPU_WRITE4(sc, as_gen_offset, as_gen); IPU_WRITE4(sc, (di ? IPU_DI1_POL : IPU_DI0_POL), DI_POL_DRDY_POLARITY_15); IPU_WRITE4(sc, DC_DISP_CONF2(di), sc->sc_mode->hdisplay); } static void ipu_dc_enable(struct ipu_softc *sc) { uint32_t conf; /* channel 1 uses DI1 */ IPU_WRITE4(sc, DC_WRITE_CH_CONF_1, WRITE_CH_CONF_PROG_DI_ID(1)); conf = IPU_READ4(sc, DC_WRITE_CH_CONF_5); conf &= ~WRITE_CH_CONF_PROG_CHAN_TYP_MASK; conf |= WRITE_CH_CONF_PROG_CHAN_NORMAL; IPU_WRITE4(sc, DC_WRITE_CH_CONF_5, conf); /* TODO: enable clock */ } static void ipu_dc_link_event(struct ipu_softc *sc, int event, int addr, int priority) { uint32_t reg; int offset; int shift; if (event % 2) shift = 16; else shift = 0; offset = DC_RL0_CH_5 + (event / 2) * sizeof(uint32_t); reg = IPU_READ4(sc, offset); reg &= ~(0xFFFF << shift); reg |= ((addr << 8) | priority) << shift; IPU_WRITE4(sc, offset, reg); } static void ipu_dc_setup_map(struct ipu_softc *sc, int map, int byte, int offset, int mask) { uint32_t reg, shift, ptr; ptr = map * 3 + byte; reg = IPU_READ4(sc, DC_MAP_CONF_VAL(ptr)); if (ptr & 1) shift = 16; else shift = 0; reg &= ~(0xffff << shift); reg |= ((offset << 8) | mask) << shift; IPU_WRITE4(sc, DC_MAP_CONF_VAL(ptr), reg); reg = IPU_READ4(sc, DC_MAP_CONF_PTR(map)); if (map & 1) shift = 16 + 5 * byte; else shift = 5 * byte; reg &= ~(MAP_CONF_PTR_MASK << shift); reg |= (ptr) << shift; IPU_WRITE4(sc, DC_MAP_CONF_PTR(map), reg); } static void ipu_dc_reset_map(struct ipu_softc *sc, int map) { uint32_t reg, shift; reg = IPU_READ4(sc, DC_MAP_CONF_VAL(map)); if (map & 1) shift = 16; else shift = 0; reg &= ~(MAP_CONF_VAL_MASK << shift); IPU_WRITE4(sc, DC_MAP_CONF_VAL(map), reg); } static void ipu_dc_init(struct ipu_softc *sc, int di_port) { int addr; uint32_t conf; if (di_port) addr = 2; else addr = 5; ipu_dc_link_event(sc, DC_EVENT_NL, addr, 3); ipu_dc_link_event(sc, DC_EVENT_EOL, addr + 1, 2); ipu_dc_link_event(sc, DC_EVENT_NEW_DATA, addr + 2, 1); ipu_dc_link_event(sc, DC_EVENT_NF, 0, 0); ipu_dc_link_event(sc, DC_EVENT_NFIELD, 0, 0); ipu_dc_link_event(sc, DC_EVENT_EOF, 0, 0); ipu_dc_link_event(sc, DC_EVENT_EOFIELD, 0, 0); ipu_dc_link_event(sc, DC_EVENT_NEW_CHAN, 0, 0); ipu_dc_link_event(sc, DC_EVENT_NEW_ADDR, 0, 0); conf = WRITE_CH_CONF_PROG_W_SIZE(0x02) | WRITE_CH_CONF_PROG_DISP_ID(DI_PORT) | WRITE_CH_CONF_PROG_DI_ID(DI_PORT); IPU_WRITE4(sc, DC_WRITE_CH_CONF_5, conf); IPU_WRITE4(sc, DC_WRITE_CH_ADDR_5, 0x00000000); IPU_WRITE4(sc, DC_GEN, DC_GEN_SYNC_PRIORITY | DC_GEN_SYNC); /* High priority, sync */ } static void ipu_init_buffer(struct ipu_softc *sc) { struct ipu_cpmem_ch_param param; uint32_t stride; uint32_t reg, db_mode_sel, cur_buf; stride = sc->sc_mode->hdisplay * MODE_BPP / 8; - /* init channel paramters */ + /* init channel parameters */ CH_PARAM_RESET(¶m); /* XXX: interlaced modes are not supported yet */ CH_PARAM_SET_FW(¶m, sc->sc_mode->hdisplay - 1); CH_PARAM_SET_FH(¶m, sc->sc_mode->vdisplay - 1); CH_PARAM_SET_SLY(¶m, stride - 1); CH_PARAM_SET_EBA0(¶m, (sc->sc_fb_phys >> 3)); CH_PARAM_SET_EBA1(¶m, (sc->sc_fb_phys >> 3)); CH_PARAM_SET_BPP(¶m, IPU_PIX_FORMAT_BPP_16); CH_PARAM_SET_PFS(¶m, IPU_PIX_FORMAT_RGB); /* 16 pixels per burst access */ CH_PARAM_SET_NPB(¶m, 16 - 1); CH_PARAM_SET_RED_OFFSET(¶m, 0); CH_PARAM_SET_RED_WIDTH(¶m, 5 - 1); CH_PARAM_SET_GREEN_OFFSET(¶m, 5); CH_PARAM_SET_GREEN_WIDTH(¶m, 6 - 1); CH_PARAM_SET_BLUE_OFFSET(¶m, 11); CH_PARAM_SET_BLUE_WIDTH(¶m, 5 - 1); CH_PARAM_SET_ALPHA_OFFSET(¶m, 16); CH_PARAM_SET_ALPHA_WIDTH(¶m, 8 - 1); CH_PARAM_SET_UBO(¶m, 0); CH_PARAM_SET_VBO(¶m, 0); IPU_WRITE_CH_PARAM(sc, DMA_CHANNEL, ¶m); #ifdef DEBUG ipu_print_channel(¶m); #endif /* init DMFC */ IPU_WRITE4(sc, DMFC_IC_CTRL, DMFC_IC_CTRL_DISABLED); /* High resolution DP */ IPU_WRITE4(sc, DMFC_WR_CHAN, DMFC_WR_CHAN_BURST_SIZE_8 | DMFC_WR_CHAN_FIFO_SIZE_128); IPU_WRITE4(sc, DMFC_WR_CHAN_DEF, DMFC_WR_CHAN_DEF_WM_CLR_2C(1) | DMFC_WR_CHAN_DEF_WM_CLR_1C(1) | DMFC_WR_CHAN_DEF_WM_CLR_2(1) | DMFC_WR_CHAN_DEF_WM_CLR_1(7) | DMFC_WR_CHAN_DEF_WM_SET_1(5) | DMFC_WR_CHAN_DEF_WM_EN_1); IPU_WRITE4(sc, DMFC_DP_CHAN, DMFC_DP_CHAN_BURST_SIZE_5F(DMFC_DP_CHAN_BURST_SIZE_8) | DMFC_DP_CHAN_FIFO_SIZE_5F(DMFC_DP_CHAN_FIFO_SIZE_128) | DMFC_DP_CHAN_ST_ADDR_SIZE_5F(6) /* segment 6 */ | DMFC_DP_CHAN_BURST_SIZE_5B(DMFC_DP_CHAN_BURST_SIZE_8) | DMFC_DP_CHAN_FIFO_SIZE_5B(DMFC_DP_CHAN_FIFO_SIZE_256) | DMFC_DP_CHAN_ST_ADDR_SIZE_5B(2) /* segment 2 */); IPU_WRITE4(sc, DMFC_DP_CHAN_DEF, DMFC_DP_CHAN_DEF_WM_CLR_6F(1) | DMFC_DP_CHAN_DEF_WM_CLR_6B(1) | DMFC_DP_CHAN_DEF_WM_CLR_5F(7) | DMFC_DP_CHAN_DEF_WM_SET_5F(5) | DMFC_DP_CHAN_DEF_WM_EN_5F | DMFC_DP_CHAN_DEF_WM_CLR_5B(7) | DMFC_DP_CHAN_DEF_WM_SET_5B(5) | DMFC_DP_CHAN_DEF_WM_EN_5B); reg = IPU_READ4(sc, DMFC_GENERAL_1); reg &= ~(DMFC_GENERAL_1_WAIT4EOT_5B); IPU_WRITE4(sc, DMFC_GENERAL_1, reg); /* XXX: set priority? */ /* Set single buffer mode */ if (DMA_CHANNEL < 32) { db_mode_sel = IPU_CH_DB_MODE_SEL_0; cur_buf = IPU_CUR_BUF_0; } else { db_mode_sel = IPU_CH_DB_MODE_SEL_1; cur_buf = IPU_CUR_BUF_1; } reg = IPU_READ4(sc, db_mode_sel); reg |= (1UL << (DMA_CHANNEL & 0x1f)); IPU_WRITE4(sc, db_mode_sel, reg); IPU_WRITE4(sc, cur_buf, (1UL << (DMA_CHANNEL & 0x1f))); } static int ipu_init(struct ipu_softc *sc) { uint32_t reg, off; int i, err; size_t dma_size; IPU_WRITE4(sc, IPU_CONF, DI_PORT ? IPU_CONF_DI1_EN : IPU_CONF_DI0_EN); IPU_WRITE4(sc, IPU_MEM_RST, IPU_MEM_RST_ALL); i = 1000; while (i-- > 0) { if (!(IPU_READ4(sc, IPU_MEM_RST) & IPU_MEM_RST_START)) break; DELAY(1); } if (i <= 0) { err = ETIMEDOUT; device_printf(sc->sc_dev, "timeout while resetting memory\n"); goto fail; } ipu_dc_reset_map(sc, 0); ipu_dc_setup_map(sc, 0, 0, 7, 0xff); ipu_dc_setup_map(sc, 0, 1, 15, 0xff); ipu_dc_setup_map(sc, 0, 2, 23, 0xff); dma_size = round_page(sc->sc_mode->hdisplay * sc->sc_mode->vdisplay * (MODE_BPP / 8)); /* * Now allocate framebuffer memory */ err = bus_dma_tag_create( bus_get_dma_tag(sc->sc_dev), 4, 0, /* alignment, boundary */ BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ dma_size, 1, /* maxsize, nsegments */ dma_size, 0, /* maxsegsize, flags */ NULL, NULL, /* lockfunc, lockarg */ &sc->sc_dma_tag); if (err) goto fail; err = bus_dmamem_alloc(sc->sc_dma_tag, (void **)&sc->sc_fb_base, BUS_DMA_COHERENT | BUS_DMA_ZERO, &sc->sc_dma_map); if (err) { device_printf(sc->sc_dev, "cannot allocate framebuffer\n"); goto fail; } err = bus_dmamap_load(sc->sc_dma_tag, sc->sc_dma_map, sc->sc_fb_base, dma_size, ipu_dmamap_cb, &sc->sc_fb_phys, BUS_DMA_NOWAIT); if (err) { device_printf(sc->sc_dev, "cannot load DMA map\n"); goto fail; } /* Calculate actual FB Size */ sc->sc_fb_size = sc->sc_mode->hdisplay * sc->sc_mode->vdisplay * MODE_BPP / 8; ipu_dc_init(sc, DI_PORT); reg = IPU_READ4(sc, IPU_CONF); reg |= IPU_CONF_DMFC_EN | IPU_CONF_DC_EN | IPU_CONF_DP_EN; IPU_WRITE4(sc, IPU_CONF, reg); ipu_config_timing(sc, DI_PORT); ipu_init_buffer(sc); ipu_di_enable(sc, DI_PORT); /* Enable DMA channel */ off = (DMA_CHANNEL > 31) ? IPU_IDMAC_CH_EN_2 : IPU_IDMAC_CH_EN_1; reg = IPU_READ4(sc, off); reg |= (1 << (DMA_CHANNEL & 0x1f)); IPU_WRITE4(sc, off, reg); ipu_dc_enable(sc); sc->sc_fb_info.fb_name = device_get_nameunit(sc->sc_dev); sc->sc_fb_info.fb_vbase = (intptr_t)sc->sc_fb_base; sc->sc_fb_info.fb_pbase = sc->sc_fb_phys; sc->sc_fb_info.fb_size = sc->sc_fb_size; sc->sc_fb_info.fb_bpp = sc->sc_fb_info.fb_depth = MODE_BPP; sc->sc_fb_info.fb_stride = sc->sc_mode->hdisplay * MODE_BPP / 8; sc->sc_fb_info.fb_width = sc->sc_mode->hdisplay; sc->sc_fb_info.fb_height = sc->sc_mode->vdisplay; device_t fbd = device_add_child(sc->sc_dev, "fbd", device_get_unit(sc->sc_dev)); if (fbd == NULL) { device_printf(sc->sc_dev, "Failed to add fbd child\n"); goto fail; } if (device_probe_and_attach(fbd) != 0) { device_printf(sc->sc_dev, "Failed to attach fbd device\n"); goto fail; } return (0); fail: return (err); } static void ipu_hdmi_event(void *arg, device_t hdmi_dev) { struct ipu_softc *sc; uint8_t *edid; uint32_t edid_len; #ifdef EDID_DEBUG struct edid_info ei; #endif const struct videomode *videomode; sc = arg; edid = NULL; edid_len = 0; if (HDMI_GET_EDID(hdmi_dev, &edid, &edid_len) != 0) { device_printf(sc->sc_dev, "failed to get EDID info from HDMI framer\n"); } videomode = NULL; #ifdef EDID_DEBUG if ( edid && (edid_parse(edid, &ei) == 0)) { edid_print(&ei); } else device_printf(sc->sc_dev, "failed to parse EDID\n"); #endif sc->sc_mode = &mode1024x768; ipu_init(sc); HDMI_SET_VIDEOMODE(hdmi_dev, sc->sc_mode); } static int ipu_probe(device_t dev) { if (have_ipu) return (ENXIO); if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, "fsl,imx6q-ipu")) return (ENXIO); device_set_desc(dev, "Freescale IPU"); return (BUS_PROBE_DEFAULT); } static int ipu_attach(device_t dev) { struct ipu_softc *sc; if (have_ipu) return (ENXIO); sc = device_get_softc(dev); sc->sc_dev = dev; sc->sc_mem_rid = 0; sc->sc_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->sc_mem_rid, RF_ACTIVE); if (!sc->sc_mem_res) { device_printf(dev, "cannot allocate memory window\n"); return (ENXIO); } sc->sc_irq_rid = 0; sc->sc_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->sc_irq_rid, RF_ACTIVE); if (!sc->sc_irq_res) { bus_release_resource(dev, SYS_RES_MEMORY, sc->sc_mem_rid, sc->sc_mem_res); device_printf(dev, "cannot allocate interrupt\n"); return (ENXIO); } /* Enable IPU1 */ imx_ccm_ipu_enable(1); if (src_reset_ipu() != 0) { device_printf(dev, "failed to reset IPU\n"); return (ENXIO); } IPU_LOCK_INIT(sc); sc->sc_hdmi_evh = EVENTHANDLER_REGISTER(hdmi_event, ipu_hdmi_event, sc, 0); have_ipu = 1; return (0); } static int ipu_detach(device_t dev) { /* Do not let unload driver */ return (EBUSY); } static struct fb_info * ipu_fb_getinfo(device_t dev) { struct ipu_softc *sc; sc = device_get_softc(dev); return (&sc->sc_fb_info); } static device_method_t ipu_methods[] = { DEVMETHOD(device_probe, ipu_probe), DEVMETHOD(device_attach, ipu_attach), DEVMETHOD(device_detach, ipu_detach), /* Framebuffer service methods */ DEVMETHOD(fb_getinfo, ipu_fb_getinfo), DEVMETHOD_END }; static driver_t ipu_driver = { "fb", ipu_methods, sizeof(struct ipu_softc), }; static devclass_t ipu_devclass; DRIVER_MODULE(ipu, simplebus, ipu_driver, ipu_devclass, 0, 0); MODULE_VERSION(ipu, 1); MODULE_DEPEND(ipu, simplebus, 1, 1, 1); Index: user/ngie/release-pkg-fix-tests/sys/arm/freescale/imx/imx6_ssi.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/freescale/imx/imx6_ssi.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/freescale/imx/imx6_ssi.c (revision 299091) @@ -1,854 +1,854 @@ /*- * Copyright (c) 2015 Ruslan Bukin * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * i.MX6 Synchronous Serial Interface (SSI) * * Chapter 61, i.MX 6Dual/6Quad Applications Processor Reference Manual, * Rev. 1, 04/2013 */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define READ4(_sc, _reg) \ bus_space_read_4(_sc->bst, _sc->bsh, _reg) #define WRITE4(_sc, _reg, _val) \ bus_space_write_4(_sc->bst, _sc->bsh, _reg, _val) #define SSI_NCHANNELS 1 /* i.MX6 SSI registers */ #define SSI_STX0 0x00 /* Transmit Data Register n */ #define SSI_STX1 0x04 /* Transmit Data Register n */ #define SSI_SRX0 0x08 /* Receive Data Register n */ #define SSI_SRX1 0x0C /* Receive Data Register n */ #define SSI_SCR 0x10 /* Control Register */ #define SCR_I2S_MODE_S 5 /* I2S Mode Select. */ #define SCR_I2S_MODE_M 0x3 #define SCR_SYN (1 << 4) #define SCR_NET (1 << 3) /* Network mode */ #define SCR_RE (1 << 2) /* Receive Enable. */ #define SCR_TE (1 << 1) /* Transmit Enable. */ #define SCR_SSIEN (1 << 0) /* SSI Enable */ #define SSI_SISR 0x14 /* Interrupt Status Register */ #define SSI_SIER 0x18 /* Interrupt Enable Register */ #define SIER_RDMAE (1 << 22) /* Receive DMA Enable. */ #define SIER_RIE (1 << 21) /* Receive Interrupt Enable. */ #define SIER_TDMAE (1 << 20) /* Transmit DMA Enable. */ #define SIER_TIE (1 << 19) /* Transmit Interrupt Enable. */ #define SIER_TDE0IE (1 << 12) /* Transmit Data Register Empty 0. */ #define SIER_TUE0IE (1 << 8) /* Transmitter Underrun Error 0. */ #define SIER_TFE0IE (1 << 0) /* Transmit FIFO Empty 0 IE. */ #define SSI_STCR 0x1C /* Transmit Configuration Register */ #define STCR_TXBIT0 (1 << 9) /* Transmit Bit 0 shift MSB/LSB */ #define STCR_TFEN1 (1 << 8) /* Transmit FIFO Enable 1. */ #define STCR_TFEN0 (1 << 7) /* Transmit FIFO Enable 0. */ #define STCR_TFDIR (1 << 6) /* Transmit Frame Direction. */ #define STCR_TXDIR (1 << 5) /* Transmit Clock Direction. */ #define STCR_TSHFD (1 << 4) /* Transmit Shift Direction. */ #define STCR_TSCKP (1 << 3) /* Transmit Clock Polarity. */ #define STCR_TFSI (1 << 2) /* Transmit Frame Sync Invert. */ #define STCR_TFSL (1 << 1) /* Transmit Frame Sync Length. */ #define STCR_TEFS (1 << 0) /* Transmit Early Frame Sync. */ #define SSI_SRCR 0x20 /* Receive Configuration Register */ #define SSI_STCCR 0x24 /* Transmit Clock Control Register */ #define STCCR_DIV2 (1 << 18) /* Divide By 2. */ #define STCCR_PSR (1 << 17) /* Divide clock by 8. */ #define WL3_WL0_S 13 #define WL3_WL0_M 0xf #define DC4_DC0_S 8 #define DC4_DC0_M 0x1f #define PM7_PM0_S 0 #define PM7_PM0_M 0xff #define SSI_SRCCR 0x28 /* Receive Clock Control Register */ #define SSI_SFCSR 0x2C /* FIFO Control/Status Register */ #define SFCSR_RFWM1_S 20 /* Receive FIFO Empty WaterMark 1 */ #define SFCSR_RFWM1_M 0xf #define SFCSR_TFWM1_S 16 /* Transmit FIFO Empty WaterMark 1 */ #define SFCSR_TFWM1_M 0xf #define SFCSR_RFWM0_S 4 /* Receive FIFO Empty WaterMark 0 */ #define SFCSR_RFWM0_M 0xf #define SFCSR_TFWM0_S 0 /* Transmit FIFO Empty WaterMark 0 */ #define SFCSR_TFWM0_M 0xf #define SSI_SACNT 0x38 /* AC97 Control Register */ #define SSI_SACADD 0x3C /* AC97 Command Address Register */ #define SSI_SACDAT 0x40 /* AC97 Command Data Register */ #define SSI_SATAG 0x44 /* AC97 Tag Register */ #define SSI_STMSK 0x48 /* Transmit Time Slot Mask Register */ #define SSI_SRMSK 0x4C /* Receive Time Slot Mask Register */ #define SSI_SACCST 0x50 /* AC97 Channel Status Register */ #define SSI_SACCEN 0x54 /* AC97 Channel Enable Register */ #define SSI_SACCDIS 0x58 /* AC97 Channel Disable Register */ static MALLOC_DEFINE(M_SSI, "ssi", "ssi audio"); uint32_t ssi_dma_intr(void *arg, int chn); struct ssi_rate { uint32_t speed; uint32_t mfi; /* PLL4 Multiplication Factor Integer */ uint32_t mfn; /* PLL4 Multiplication Factor Numerator */ uint32_t mfd; /* PLL4 Multiplication Factor Denominator */ /* More dividers to configure can be added here */ }; static struct ssi_rate rate_map[] = { { 192000, 49, 152, 1000 }, /* PLL4 49.152 Mhz */ /* TODO: add more frequences */ { 0, 0 }, }; /* * i.MX6 example bit clock formula * * BCLK = 2 channels * 192000 hz * 24 bit = 9216000 hz = * (24000000 * (49 + 152/1000.0) / 4 / 4 / 2 / 2 / 2 / 1 / 1) * ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ ^ * | | | | | | | | | | | * Fref ------/ | | | | | | | | | | * PLL4 div select -/ | | | | | | | | | * PLL4 num --------------/ | | | | | | | | * PLL4 denom -------------------/ | | | | | | | * PLL4 post div ---------------------/ | | | | | | * CCM ssi pre div (CCM_CS1CDR) ----------/ | | | | | * CCM ssi post div (CCM_CS1CDR) -------------/ | | | | * SSI PM7_PM0_S ---------------------------------/ | | | * SSI Fixed divider ---------------------------------/ | | * SSI DIV2 ----------------------------------------------/ | * SSI PSR (prescaler /1 or /8) ------------------------------/ * * MCLK (Master clock) depends on DAC, usually BCLK * 4 */ struct sc_info { struct resource *res[2]; bus_space_tag_t bst; bus_space_handle_t bsh; device_t dev; struct mtx *lock; void *ih; int pos; int dma_size; bus_dma_tag_t dma_tag; bus_dmamap_t dma_map; bus_addr_t buf_base_phys; uint32_t *buf_base; struct sdma_conf *conf; struct ssi_rate *sr; struct sdma_softc *sdma_sc; int sdma_ev_rx; int sdma_ev_tx; int sdma_channel; }; /* Channel registers */ struct sc_chinfo { struct snd_dbuf *buffer; struct pcm_channel *channel; struct sc_pcminfo *parent; /* Channel information */ uint32_t dir; uint32_t format; /* Flags */ uint32_t run; }; /* PCM device private data */ struct sc_pcminfo { device_t dev; uint32_t (*ih)(struct sc_pcminfo *scp); uint32_t chnum; struct sc_chinfo chan[SSI_NCHANNELS]; struct sc_info *sc; }; static struct resource_spec ssi_spec[] = { { SYS_RES_MEMORY, 0, RF_ACTIVE }, { SYS_RES_IRQ, 0, RF_ACTIVE }, { -1, 0 } }; static int setup_dma(struct sc_pcminfo *scp); static void setup_ssi(struct sc_info *); static void ssi_configure_clock(struct sc_info *); /* * Mixer interface. */ static int ssimixer_init(struct snd_mixer *m) { struct sc_pcminfo *scp; struct sc_info *sc; int mask; scp = mix_getdevinfo(m); sc = scp->sc; if (sc == NULL) return -1; mask = SOUND_MASK_PCM; mask |= SOUND_MASK_VOLUME; snd_mtxlock(sc->lock); pcm_setflags(scp->dev, pcm_getflags(scp->dev) | SD_F_SOFTPCMVOL); mix_setdevs(m, mask); snd_mtxunlock(sc->lock); return (0); } static int ssimixer_set(struct snd_mixer *m, unsigned dev, unsigned left, unsigned right) { struct sc_pcminfo *scp; scp = mix_getdevinfo(m); /* Here we can configure hardware volume on our DAC */ #if 1 device_printf(scp->dev, "ssimixer_set() %d %d\n", left, right); #endif return (0); } static kobj_method_t ssimixer_methods[] = { KOBJMETHOD(mixer_init, ssimixer_init), KOBJMETHOD(mixer_set, ssimixer_set), KOBJMETHOD_END }; MIXER_DECLARE(ssimixer); /* * Channel interface. */ static void * ssichan_init(kobj_t obj, void *devinfo, struct snd_dbuf *b, struct pcm_channel *c, int dir) { struct sc_pcminfo *scp; struct sc_chinfo *ch; struct sc_info *sc; scp = (struct sc_pcminfo *)devinfo; sc = scp->sc; snd_mtxlock(sc->lock); ch = &scp->chan[0]; ch->dir = dir; ch->run = 0; ch->buffer = b; ch->channel = c; ch->parent = scp; snd_mtxunlock(sc->lock); if (sndbuf_setup(ch->buffer, sc->buf_base, sc->dma_size) != 0) { device_printf(scp->dev, "Can't setup sndbuf.\n"); return NULL; } return ch; } static int ssichan_free(kobj_t obj, void *data) { struct sc_chinfo *ch = data; struct sc_pcminfo *scp = ch->parent; struct sc_info *sc = scp->sc; #if 0 device_printf(scp->dev, "ssichan_free()\n"); #endif snd_mtxlock(sc->lock); /* TODO: free channel buffer */ snd_mtxunlock(sc->lock); return (0); } static int ssichan_setformat(kobj_t obj, void *data, uint32_t format) { struct sc_chinfo *ch = data; ch->format = format; return (0); } static uint32_t ssichan_setspeed(kobj_t obj, void *data, uint32_t speed) { struct sc_pcminfo *scp; struct sc_chinfo *ch; struct ssi_rate *sr; struct sc_info *sc; int threshold; int i; ch = data; scp = ch->parent; sc = scp->sc; sr = NULL; /* First look for equal frequency. */ for (i = 0; rate_map[i].speed != 0; i++) { if (rate_map[i].speed == speed) sr = &rate_map[i]; } /* If no match, just find nearest. */ if (sr == NULL) { for (i = 0; rate_map[i].speed != 0; i++) { sr = &rate_map[i]; threshold = sr->speed + ((rate_map[i + 1].speed != 0) ? ((rate_map[i + 1].speed - sr->speed) >> 1) : 0); if (speed < threshold) break; } } sc->sr = sr; ssi_configure_clock(sc); return (sr->speed); } static void ssi_configure_clock(struct sc_info *sc) { struct ssi_rate *sr; sr = sc->sr; pll4_configure_output(sr->mfi, sr->mfn, sr->mfd); /* Configure other dividers here, if any */ } static uint32_t ssichan_setblocksize(kobj_t obj, void *data, uint32_t blocksize) { struct sc_chinfo *ch = data; struct sc_pcminfo *scp = ch->parent; struct sc_info *sc = scp->sc; sndbuf_resize(ch->buffer, sc->dma_size / blocksize, blocksize); setup_dma(scp); return (sndbuf_getblksz(ch->buffer)); } uint32_t ssi_dma_intr(void *arg, int chn) { struct sc_pcminfo *scp; struct sdma_conf *conf; struct sc_chinfo *ch; struct sc_info *sc; int bufsize; scp = arg; ch = &scp->chan[0]; sc = scp->sc; conf = sc->conf; bufsize = sndbuf_getsize(ch->buffer); sc->pos += conf->period; if (sc->pos >= bufsize) sc->pos -= bufsize; if (ch->run) chn_intr(ch->channel); return (0); } static int find_sdma_controller(struct sc_info *sc) { struct sdma_softc *sdma_sc; phandle_t node, sdma_node; device_t sdma_dev; int dts_value[8]; int len; if ((node = ofw_bus_get_node(sc->dev)) == -1) return (ENXIO); if ((len = OF_getproplen(node, "dmas")) <= 0) return (ENXIO); OF_getprop(node, "dmas", &dts_value, len); sc->sdma_ev_rx = fdt32_to_cpu(dts_value[1]); sc->sdma_ev_tx = fdt32_to_cpu(dts_value[5]); sdma_node = OF_node_from_xref(fdt32_to_cpu(dts_value[0])); sdma_sc = NULL; sdma_dev = devclass_get_device(devclass_find("sdma"), 0); if (sdma_dev) sdma_sc = device_get_softc(sdma_dev); if (sdma_sc == NULL) { device_printf(sc->dev, "No sDMA found. Can't operate\n"); return (ENXIO); } sc->sdma_sc = sdma_sc; return (0); }; static int setup_dma(struct sc_pcminfo *scp) { struct sdma_conf *conf; struct sc_chinfo *ch; struct sc_info *sc; int fmt; ch = &scp->chan[0]; sc = scp->sc; conf = sc->conf; conf->ih = ssi_dma_intr; conf->ih_user = scp; conf->saddr = sc->buf_base_phys; conf->daddr = rman_get_start(sc->res[0]) + SSI_STX0; conf->event = sc->sdma_ev_tx; /* SDMA TX event */ conf->period = sndbuf_getblksz(ch->buffer); conf->num_bd = sndbuf_getblkcnt(ch->buffer); /* * Word Length * Can be 32, 24, 16 or 8 for sDMA. * * SSI supports 24 at max. */ fmt = sndbuf_getfmt(ch->buffer); if (fmt & AFMT_16BIT) { conf->word_length = 16; conf->command = CMD_2BYTES; } else if (fmt & AFMT_24BIT) { conf->word_length = 24; conf->command = CMD_3BYTES; } else { device_printf(sc->dev, "Unknown format\n"); return (-1); } return (0); } static int ssi_start(struct sc_pcminfo *scp) { struct sc_info *sc; int reg; sc = scp->sc; if (sdma_configure(sc->sdma_channel, sc->conf) != 0) { device_printf(sc->dev, "Can't configure sDMA\n"); return (-1); } /* Enable DMA interrupt */ reg = (SIER_TDMAE); WRITE4(sc, SSI_SIER, reg); sdma_start(sc->sdma_channel); return (0); } static int ssi_stop(struct sc_pcminfo *scp) { struct sc_info *sc; int reg; sc = scp->sc; reg = READ4(sc, SSI_SIER); reg &= ~(SIER_TDMAE); WRITE4(sc, SSI_SIER, reg); sdma_stop(sc->sdma_channel); bzero(sc->buf_base, sc->dma_size); return (0); } static int ssichan_trigger(kobj_t obj, void *data, int go) { struct sc_pcminfo *scp; struct sc_chinfo *ch; struct sc_info *sc; ch = data; scp = ch->parent; sc = scp->sc; snd_mtxlock(sc->lock); switch (go) { case PCMTRIG_START: #if 0 device_printf(scp->dev, "trigger start\n"); #endif ch->run = 1; ssi_start(scp); break; case PCMTRIG_STOP: case PCMTRIG_ABORT: #if 0 device_printf(scp->dev, "trigger stop or abort\n"); #endif ch->run = 0; ssi_stop(scp); break; } snd_mtxunlock(sc->lock); return (0); } static uint32_t ssichan_getptr(kobj_t obj, void *data) { struct sc_pcminfo *scp; struct sc_chinfo *ch; struct sc_info *sc; ch = data; scp = ch->parent; sc = scp->sc; return (sc->pos); } static uint32_t ssi_pfmt[] = { SND_FORMAT(AFMT_S24_LE, 2, 0), 0 }; static struct pcmchan_caps ssi_pcaps = {44100, 192000, ssi_pfmt, 0}; static struct pcmchan_caps * ssichan_getcaps(kobj_t obj, void *data) { return (&ssi_pcaps); } static kobj_method_t ssichan_methods[] = { KOBJMETHOD(channel_init, ssichan_init), KOBJMETHOD(channel_free, ssichan_free), KOBJMETHOD(channel_setformat, ssichan_setformat), KOBJMETHOD(channel_setspeed, ssichan_setspeed), KOBJMETHOD(channel_setblocksize, ssichan_setblocksize), KOBJMETHOD(channel_trigger, ssichan_trigger), KOBJMETHOD(channel_getptr, ssichan_getptr), KOBJMETHOD(channel_getcaps, ssichan_getcaps), KOBJMETHOD_END }; CHANNEL_DECLARE(ssichan); static int ssi_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, "fsl,imx6q-ssi")) return (ENXIO); device_set_desc(dev, "i.MX6 Synchronous Serial Interface (SSI)"); return (BUS_PROBE_DEFAULT); } static void ssi_intr(void *arg) { struct sc_pcminfo *scp; struct sc_chinfo *ch; struct sc_info *sc; scp = arg; sc = scp->sc; ch = &scp->chan[0]; /* We don't use SSI interrupt */ #if 0 device_printf(sc->dev, "SSI Intr 0x%08x\n", READ4(sc, SSI_SISR)); #endif } static void setup_ssi(struct sc_info *sc) { int reg; reg = READ4(sc, SSI_STCCR); reg &= ~(WL3_WL0_M << WL3_WL0_S); reg |= (0xb << WL3_WL0_S); /* 24 bit */ reg &= ~(DC4_DC0_M << DC4_DC0_S); reg |= (1 << DC4_DC0_S); /* 2 words per frame */ reg &= ~(STCCR_DIV2); /* Divide by 1 */ reg &= ~(STCCR_PSR); /* Divide by 1 */ reg &= ~(PM7_PM0_M << PM7_PM0_S); reg |= (1 << PM7_PM0_S); /* Divide by 2 */ WRITE4(sc, SSI_STCCR, reg); reg = READ4(sc, SSI_SFCSR); reg &= ~(SFCSR_TFWM0_M << SFCSR_TFWM0_S); reg |= (8 << SFCSR_TFWM0_S); /* empty slots */ WRITE4(sc, SSI_SFCSR, reg); reg = READ4(sc, SSI_STCR); reg |= (STCR_TFEN0); reg &= ~(STCR_TFEN1); reg &= ~(STCR_TSHFD); /* MSB */ reg |= (STCR_TXBIT0); reg |= (STCR_TXDIR | STCR_TFDIR); reg |= (STCR_TSCKP); /* falling edge */ reg |= (STCR_TFSI); reg &= ~(STCR_TFSI); /* active high frame sync */ reg &= ~(STCR_TFSL); reg |= STCR_TEFS; WRITE4(sc, SSI_STCR, reg); reg = READ4(sc, SSI_SCR); reg &= ~(SCR_I2S_MODE_M << SCR_I2S_MODE_S); /* Not master */ reg |= (SCR_SSIEN | SCR_TE); reg |= (SCR_NET); reg |= (SCR_SYN); WRITE4(sc, SSI_SCR, reg); } static void ssi_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int err) { bus_addr_t *addr; if (err) return; addr = (bus_addr_t*)arg; *addr = segs[0].ds_addr; } static int ssi_attach(device_t dev) { char status[SND_STATUSLEN]; struct sc_pcminfo *scp; struct sc_info *sc; int err; sc = malloc(sizeof(*sc), M_DEVBUF, M_WAITOK | M_ZERO); sc->dev = dev; sc->sr = &rate_map[0]; sc->pos = 0; sc->conf = malloc(sizeof(struct sdma_conf), M_DEVBUF, M_WAITOK | M_ZERO); sc->lock = snd_mtxcreate(device_get_nameunit(dev), "ssi softc"); if (sc->lock == NULL) { - device_printf(dev, "Cant create mtx\n"); + device_printf(dev, "Can't create mtx\n"); return (ENXIO); } if (bus_alloc_resources(dev, ssi_spec, sc->res)) { device_printf(dev, "could not allocate resources\n"); return (ENXIO); } /* Memory interface */ sc->bst = rman_get_bustag(sc->res[0]); sc->bsh = rman_get_bushandle(sc->res[0]); /* SDMA */ if (find_sdma_controller(sc)) { device_printf(dev, "could not find active SDMA\n"); return (ENXIO); } /* Setup PCM */ scp = malloc(sizeof(struct sc_pcminfo), M_DEVBUF, M_NOWAIT | M_ZERO); scp->sc = sc; scp->dev = dev; /* * Maximum possible DMA buffer. - * Will be used partialy to match 24 bit word. + * Will be used partially to match 24 bit word. */ sc->dma_size = 131072; /* * Must use dma_size boundary as modulo feature required. * Modulo feature allows setup circular buffer. */ err = bus_dma_tag_create( bus_get_dma_tag(sc->dev), 4, sc->dma_size, /* alignment, boundary */ BUS_SPACE_MAXADDR_32BIT, /* lowaddr */ BUS_SPACE_MAXADDR, /* highaddr */ NULL, NULL, /* filter, filterarg */ sc->dma_size, 1, /* maxsize, nsegments */ sc->dma_size, 0, /* maxsegsize, flags */ NULL, NULL, /* lockfunc, lockarg */ &sc->dma_tag); err = bus_dmamem_alloc(sc->dma_tag, (void **)&sc->buf_base, BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &sc->dma_map); if (err) { device_printf(dev, "cannot allocate framebuffer\n"); return (ENXIO); } err = bus_dmamap_load(sc->dma_tag, sc->dma_map, sc->buf_base, sc->dma_size, ssi_dmamap_cb, &sc->buf_base_phys, BUS_DMA_NOWAIT); if (err) { device_printf(dev, "cannot load DMA map\n"); return (ENXIO); } bzero(sc->buf_base, sc->dma_size); /* Setup interrupt handler */ err = bus_setup_intr(dev, sc->res[1], INTR_MPSAFE | INTR_TYPE_AV, NULL, ssi_intr, scp, &sc->ih); if (err) { device_printf(dev, "Unable to alloc interrupt resource.\n"); return (ENXIO); } pcm_setflags(dev, pcm_getflags(dev) | SD_F_MPSAFE); err = pcm_register(dev, scp, 1, 0); if (err) { device_printf(dev, "Can't register pcm.\n"); return (ENXIO); } scp->chnum = 0; pcm_addchan(dev, PCMDIR_PLAY, &ssichan_class, scp); scp->chnum++; snprintf(status, SND_STATUSLEN, "at simplebus"); pcm_setstatus(dev, status); mixer_init(dev, &ssimixer_class, scp); setup_ssi(sc); imx_ccm_ssi_configure(dev); sc->sdma_channel = sdma_alloc(); if (sc->sdma_channel < 0) { device_printf(sc->dev, "Can't get sDMA channel\n"); return (1); } return (0); } static device_method_t ssi_pcm_methods[] = { DEVMETHOD(device_probe, ssi_probe), DEVMETHOD(device_attach, ssi_attach), { 0, 0 } }; static driver_t ssi_pcm_driver = { "pcm", ssi_pcm_methods, PCM_SOFTC_SIZE, }; DRIVER_MODULE(ssi, simplebus, ssi_pcm_driver, pcm_devclass, 0, 0); MODULE_DEPEND(ssi, sound, SOUND_MINVER, SOUND_PREFVER, SOUND_MAXVER); MODULE_VERSION(ssi, 1); Index: user/ngie/release-pkg-fix-tests/sys/arm/freescale/vybrid/vf_uart.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/freescale/vybrid/vf_uart.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/freescale/vybrid/vf_uart.c (revision 299091) @@ -1,516 +1,516 @@ /*- * Copyright (c) 2013 Ruslan Bukin * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * Vybrid Family Universal Asynchronous Receiver/Transmitter * Chapter 49, Vybrid Reference Manual, Rev. 5, 07/2013 */ #include __FBSDID("$FreeBSD$"); #include "opt_ddb.h" #include #include #include #include #include #include #include #include #include #include #include "uart_if.h" #define UART_BDH 0x00 /* Baud Rate Registers: High */ #define UART_BDL 0x01 /* Baud Rate Registers: Low */ #define UART_C1 0x02 /* Control Register 1 */ #define UART_C2 0x03 /* Control Register 2 */ #define UART_S1 0x04 /* Status Register 1 */ #define UART_S2 0x05 /* Status Register 2 */ #define UART_C3 0x06 /* Control Register 3 */ #define UART_D 0x07 /* Data Register */ #define UART_MA1 0x08 /* Match Address Registers 1 */ #define UART_MA2 0x09 /* Match Address Registers 2 */ #define UART_C4 0x0A /* Control Register 4 */ #define UART_C5 0x0B /* Control Register 5 */ #define UART_ED 0x0C /* Extended Data Register */ #define UART_MODEM 0x0D /* Modem Register */ #define UART_IR 0x0E /* Infrared Register */ #define UART_PFIFO 0x10 /* FIFO Parameters */ #define UART_CFIFO 0x11 /* FIFO Control Register */ #define UART_SFIFO 0x12 /* FIFO Status Register */ #define UART_TWFIFO 0x13 /* FIFO Transmit Watermark */ #define UART_TCFIFO 0x14 /* FIFO Transmit Count */ #define UART_RWFIFO 0x15 /* FIFO Receive Watermark */ #define UART_RCFIFO 0x16 /* FIFO Receive Count */ #define UART_C7816 0x18 /* 7816 Control Register */ #define UART_IE7816 0x19 /* 7816 Interrupt Enable Register */ #define UART_IS7816 0x1A /* 7816 Interrupt Status Register */ #define UART_WP7816T0 0x1B /* 7816 Wait Parameter Register */ #define UART_WP7816T1 0x1B /* 7816 Wait Parameter Register */ #define UART_WN7816 0x1C /* 7816 Wait N Register */ #define UART_WF7816 0x1D /* 7816 Wait FD Register */ #define UART_ET7816 0x1E /* 7816 Error Threshold Register */ #define UART_TL7816 0x1F /* 7816 Transmit Length Register */ #define UART_C6 0x21 /* CEA709.1-B Control Register 6 */ #define UART_PCTH 0x22 /* CEA709.1-B Packet Cycle Time Counter High */ #define UART_PCTL 0x23 /* CEA709.1-B Packet Cycle Time Counter Low */ #define UART_B1T 0x24 /* CEA709.1-B Beta1 Timer */ #define UART_SDTH 0x25 /* CEA709.1-B Secondary Delay Timer High */ #define UART_SDTL 0x26 /* CEA709.1-B Secondary Delay Timer Low */ #define UART_PRE 0x27 /* CEA709.1-B Preamble */ #define UART_TPL 0x28 /* CEA709.1-B Transmit Packet Length */ #define UART_IE 0x29 /* CEA709.1-B Interrupt Enable Register */ #define UART_WB 0x2A /* CEA709.1-B WBASE */ #define UART_S3 0x2B /* CEA709.1-B Status Register */ #define UART_S4 0x2C /* CEA709.1-B Status Register */ #define UART_RPL 0x2D /* CEA709.1-B Received Packet Length */ #define UART_RPREL 0x2E /* CEA709.1-B Received Preamble Length */ #define UART_CPW 0x2F /* CEA709.1-B Collision Pulse Width */ #define UART_RIDT 0x30 /* CEA709.1-B Receive Indeterminate Time */ #define UART_TIDT 0x31 /* CEA709.1-B Transmit Indeterminate Time */ #define UART_C2_TE (1 << 3) /* Transmitter Enable */ #define UART_C2_TIE (1 << 7) /* Transmitter Interrupt Enable */ #define UART_C2_RE (1 << 2) /* Receiver Enable */ #define UART_C2_RIE (1 << 5) /* Receiver Interrupt Enable */ #define UART_S1_TDRE (1 << 7) /* Transmit Data Register Empty Flag */ #define UART_S1_RDRF (1 << 5) /* Receive Data Register Full Flag */ #define UART_S2_LBKDIF (1 << 7) /* LIN Break Detect Interrupt Flag */ #define UART_C4_BRFA 0x1f /* Baud Rate Fine Adjust */ #define UART_BDH_SBR 0x1f /* UART Baud Rate Bits */ /* * Low-level UART interface. */ static int vf_uart_probe(struct uart_bas *bas); static void vf_uart_init(struct uart_bas *bas, int, int, int, int); static void vf_uart_term(struct uart_bas *bas); static void vf_uart_putc(struct uart_bas *bas, int); static int vf_uart_rxready(struct uart_bas *bas); static int vf_uart_getc(struct uart_bas *bas, struct mtx *); void uart_reinit(struct uart_softc *,int,int); static struct uart_ops uart_vybrid_ops = { .probe = vf_uart_probe, .init = vf_uart_init, .term = vf_uart_term, .putc = vf_uart_putc, .rxready = vf_uart_rxready, .getc = vf_uart_getc, }; static int vf_uart_probe(struct uart_bas *bas) { return (0); } static void vf_uart_init(struct uart_bas *bas, int baudrate, int databits, int stopbits, int parity) { } static void vf_uart_term(struct uart_bas *bas) { } static void vf_uart_putc(struct uart_bas *bas, int c) { while (!(uart_getreg(bas, UART_S1) & UART_S1_TDRE)) ; uart_setreg(bas, UART_D, c); } static int vf_uart_rxready(struct uart_bas *bas) { int usr1; usr1 = uart_getreg(bas, UART_S1); if (usr1 & UART_S1_RDRF) { return (1); } return (0); } static int vf_uart_getc(struct uart_bas *bas, struct mtx *hwmtx) { int c; uart_lock(hwmtx); while (!(uart_getreg(bas, UART_S1) & UART_S1_RDRF)) ; c = uart_getreg(bas, UART_D); uart_unlock(hwmtx); return (c & 0xff); } /* * High-level UART interface. */ struct vf_uart_softc { struct uart_softc base; }; void uart_reinit(struct uart_softc *sc, int clkspeed, int baud) { struct uart_bas *bas; int sbr; int brfa; int reg; bas = &sc->sc_bas; if (!bas) { - printf("Error: cant reconfigure bas\n"); + printf("Error: can't reconfigure bas\n"); return; } uart_setreg(bas, UART_MODEM, 0x00); /* * Disable transmitter and receiver * for a while. */ reg = uart_getreg(bas, UART_C2); reg &= ~(UART_C2_RE | UART_C2_TE); uart_setreg(bas, UART_C2, 0x00); uart_setreg(bas, UART_C1, 0x00); sbr = (uint16_t) (clkspeed / (baud * 16)); brfa = (clkspeed / baud) - (sbr * 16); reg = uart_getreg(bas, UART_BDH); reg &= ~UART_BDH_SBR; reg |= ((sbr & 0x1f00) >> 8); uart_setreg(bas, UART_BDH, reg); reg = sbr & 0x00ff; uart_setreg(bas, UART_BDL, reg); reg = uart_getreg(bas, UART_C4); reg &= ~UART_C4_BRFA; reg |= (brfa & UART_C4_BRFA); uart_setreg(bas, UART_C4, reg); reg = uart_getreg(bas, UART_C2); reg |= (UART_C2_RE | UART_C2_TE); uart_setreg(bas, UART_C2, reg); } static int vf_uart_bus_attach(struct uart_softc *); static int vf_uart_bus_detach(struct uart_softc *); static int vf_uart_bus_flush(struct uart_softc *, int); static int vf_uart_bus_getsig(struct uart_softc *); static int vf_uart_bus_ioctl(struct uart_softc *, int, intptr_t); static int vf_uart_bus_ipend(struct uart_softc *); static int vf_uart_bus_param(struct uart_softc *, int, int, int, int); static int vf_uart_bus_probe(struct uart_softc *); static int vf_uart_bus_receive(struct uart_softc *); static int vf_uart_bus_setsig(struct uart_softc *, int); static int vf_uart_bus_transmit(struct uart_softc *); static kobj_method_t vf_uart_methods[] = { KOBJMETHOD(uart_attach, vf_uart_bus_attach), KOBJMETHOD(uart_detach, vf_uart_bus_detach), KOBJMETHOD(uart_flush, vf_uart_bus_flush), KOBJMETHOD(uart_getsig, vf_uart_bus_getsig), KOBJMETHOD(uart_ioctl, vf_uart_bus_ioctl), KOBJMETHOD(uart_ipend, vf_uart_bus_ipend), KOBJMETHOD(uart_param, vf_uart_bus_param), KOBJMETHOD(uart_probe, vf_uart_bus_probe), KOBJMETHOD(uart_receive, vf_uart_bus_receive), KOBJMETHOD(uart_setsig, vf_uart_bus_setsig), KOBJMETHOD(uart_transmit, vf_uart_bus_transmit), { 0, 0 } }; static struct uart_class uart_vybrid_class = { "vybrid", vf_uart_methods, sizeof(struct vf_uart_softc), .uc_ops = &uart_vybrid_ops, .uc_range = 0x100, .uc_rclk = 24000000, /* TODO: get value from CCM */ .uc_rshift = 0 }; static struct ofw_compat_data compat_data[] = { {"fsl,mvf600-uart", (uintptr_t)&uart_vybrid_class}, {NULL, (uintptr_t)NULL}, }; UART_FDT_CLASS_AND_DEVICE(compat_data); static int vf_uart_bus_attach(struct uart_softc *sc) { struct uart_bas *bas; int reg; bas = &sc->sc_bas; sc->sc_hwiflow = 0; sc->sc_hwoflow = 0; uart_reinit(sc, 66000000, 115200); reg = uart_getreg(bas, UART_C2); if (sc->sc_sysdev != NULL && sc->sc_sysdev->type == UART_DEV_CONSOLE) { reg &= ~UART_C2_RIE; } else { reg |= UART_C2_RIE; } uart_setreg(bas, UART_C2, reg); return (0); } static int vf_uart_bus_detach(struct uart_softc *sc) { /* TODO */ return (0); } static int vf_uart_bus_flush(struct uart_softc *sc, int what) { /* TODO */ return (0); } static int vf_uart_bus_getsig(struct uart_softc *sc) { /* TODO */ return (0); } static int vf_uart_bus_ioctl(struct uart_softc *sc, int request, intptr_t data) { struct uart_bas *bas; int error; bas = &sc->sc_bas; error = 0; uart_lock(sc->sc_hwmtx); switch (request) { case UART_IOCTL_BREAK: /* TODO */ break; case UART_IOCTL_BAUD: /* TODO */ *(int*)data = 115200; break; default: error = EINVAL; break; } uart_unlock(sc->sc_hwmtx); return (error); } static int vf_uart_bus_ipend(struct uart_softc *sc) { struct uart_bas *bas; int ipend; uint32_t usr1, usr2; int reg; int sfifo; bas = &sc->sc_bas; ipend = 0; uart_lock(sc->sc_hwmtx); usr1 = uart_getreg(bas, UART_S1); usr2 = uart_getreg(bas, UART_S2); sfifo = uart_getreg(bas, UART_SFIFO); /* ack usr2 */ uart_setreg(bas, UART_S2, usr2); if (usr1 & UART_S1_TDRE) { reg = uart_getreg(bas, UART_C2); reg &= ~(UART_C2_TIE); uart_setreg(bas, UART_C2, reg); if (sc->sc_txbusy != 0) { ipend |= SER_INT_TXIDLE; } } if (usr1 & UART_S1_RDRF) { reg = uart_getreg(bas, UART_C2); reg &= ~(UART_C2_RIE); uart_setreg(bas, UART_C2, reg); ipend |= SER_INT_RXREADY; } if (usr2 & UART_S2_LBKDIF) { ipend |= SER_INT_BREAK; } uart_unlock(sc->sc_hwmtx); return (ipend); } static int vf_uart_bus_param(struct uart_softc *sc, int baudrate, int databits, int stopbits, int parity) { uart_lock(sc->sc_hwmtx); vf_uart_init(&sc->sc_bas, baudrate, databits, stopbits, parity); uart_unlock(sc->sc_hwmtx); return (0); } static int vf_uart_bus_probe(struct uart_softc *sc) { int error; error = vf_uart_probe(&sc->sc_bas); if (error) return (error); sc->sc_rxfifosz = 1; sc->sc_txfifosz = 1; device_set_desc(sc->sc_dev, "Vybrid Family UART"); return (0); } static int vf_uart_bus_receive(struct uart_softc *sc) { struct uart_bas *bas; int reg; int c; bas = &sc->sc_bas; uart_lock(sc->sc_hwmtx); /* Read FIFO */ while (uart_getreg(bas, UART_S1) & UART_S1_RDRF) { if (uart_rx_full(sc)) { /* No space left in input buffer */ sc->sc_rxbuf[sc->sc_rxput] = UART_STAT_OVERRUN; break; } c = uart_getreg(bas, UART_D); uart_rx_put(sc, c); } /* Reenable Data Ready interrupt */ reg = uart_getreg(bas, UART_C2); reg |= (UART_C2_RIE); uart_setreg(bas, UART_C2, reg); uart_unlock(sc->sc_hwmtx); return (0); } static int vf_uart_bus_setsig(struct uart_softc *sc, int sig) { struct uart_bas *bas; int reg; /* TODO: implement (?) */ /* XXX workaround to have working console on mount prompt */ /* Enable RX interrupt */ bas = &sc->sc_bas; if (sc->sc_sysdev != NULL && sc->sc_sysdev->type == UART_DEV_CONSOLE) { reg = uart_getreg(bas, UART_C2); reg |= (UART_C2_RIE); uart_setreg(bas, UART_C2, reg); } return (0); } static int vf_uart_bus_transmit(struct uart_softc *sc) { struct uart_bas *bas = &sc->sc_bas; int i; int reg; bas = &sc->sc_bas; uart_lock(sc->sc_hwmtx); /* Fill TX FIFO */ for (i = 0; i < sc->sc_txdatasz; i++) { uart_setreg(bas, UART_D, sc->sc_txbuf[i] & 0xff); uart_barrier(&sc->sc_bas); } sc->sc_txbusy = 1; /* Call me when ready */ reg = uart_getreg(bas, UART_C2); reg |= (UART_C2_TIE); uart_setreg(bas, UART_C2, reg); uart_unlock(sc->sc_hwmtx); return (0); } Index: user/ngie/release-pkg-fix-tests/sys/arm/mv/mpic.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/mv/mpic.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/mv/mpic.c (revision 299091) @@ -1,631 +1,631 @@ /*- * Copyright (c) 2006 Benno Rice. * Copyright (C) 2007-2011 MARVELL INTERNATIONAL LTD. * Copyright (c) 2012 Semihalf. * All rights reserved. * * Developed by Semihalf. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * from: FreeBSD: //depot/projects/arm/src/sys/arm/xscale/pxa2x0/pxa2x0_icu.c, rev 1 * from: FreeBSD: src/sys/arm/mv/ic.c,v 1.5 2011/02/08 01:49:30 */ #include __FBSDID("$FreeBSD$"); #include "opt_platform.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef INTRNG #include "pic_if.h" #endif #ifdef DEBUG #define debugf(fmt, args...) do { printf("%s(): ", __func__); \ printf(fmt,##args); } while (0) #else #define debugf(fmt, args...) #endif #define MPIC_INT_ERR 4 #define MPIC_INT_MSI 96 #define MPIC_IRQ_MASK 0x3ff #define MPIC_CTRL 0x0 #define MPIC_SOFT_INT 0x4 #define MPIC_SOFT_INT_DRBL1 (1 << 5) #define MPIC_ERR_CAUSE 0x20 #define MPIC_ISE 0x30 #define MPIC_ICE 0x34 #define MPIC_INT_CTL(irq) (0x100 + (irq)*4) #define MPIC_INT_IRQ_FIQ_MASK(cpuid) (0x101 << (cpuid)) #define MPIC_CTRL_NIRQS(ctrl) (((ctrl) >> 2) & 0x3ff) #define MPIC_IN_DRBL 0x08 #define MPIC_IN_DRBL_MASK 0x0c #define MPIC_PPI_CAUSE 0x10 #define MPIC_CTP 0x40 #define MPIC_IIACK 0x44 #define MPIC_ISM 0x48 #define MPIC_ICM 0x4c #define MPIC_ERR_MASK 0xe50 #define MPIC_PPI 32 #ifdef INTRNG struct mv_mpic_irqsrc { struct intr_irqsrc mmi_isrc; u_int mmi_irq; }; #endif struct mv_mpic_softc { device_t sc_dev; struct resource * mpic_res[4]; bus_space_tag_t mpic_bst; bus_space_handle_t mpic_bsh; bus_space_tag_t cpu_bst; bus_space_handle_t cpu_bsh; bus_space_tag_t drbl_bst; bus_space_handle_t drbl_bsh; struct mtx mtx; #ifdef INTRNG struct mv_mpic_irqsrc * mpic_isrcs; #endif int nirqs; void * intr_hand; }; static struct resource_spec mv_mpic_spec[] = { { SYS_RES_MEMORY, 0, RF_ACTIVE }, { SYS_RES_MEMORY, 1, RF_ACTIVE }, { SYS_RES_MEMORY, 2, RF_ACTIVE | RF_OPTIONAL }, { SYS_RES_IRQ, 0, RF_ACTIVE | RF_OPTIONAL }, { -1, 0 } }; static struct ofw_compat_data compat_data[] = { {"mrvl,mpic", true}, {"marvell,mpic", true}, {NULL, false} }; static struct mv_mpic_softc *mv_mpic_sc = NULL; void mpic_send_ipi(int cpus, u_int ipi); static int mv_mpic_probe(device_t); static int mv_mpic_attach(device_t); uint32_t mv_mpic_get_cause(void); uint32_t mv_mpic_get_cause_err(void); uint32_t mv_mpic_get_msi(void); static void mpic_unmask_irq(uintptr_t nb); static void mpic_mask_irq(uintptr_t nb); static void mpic_mask_irq_err(uintptr_t nb); static void mpic_unmask_irq_err(uintptr_t nb); static int mpic_intr(void *arg); static void mpic_unmask_msi(void); #ifndef INTRNG static void arm_mask_irq_err(uintptr_t); static void arm_unmask_irq_err(uintptr_t); #endif #define MPIC_WRITE(softc, reg, val) \ bus_space_write_4((softc)->mpic_bst, (softc)->mpic_bsh, (reg), (val)) #define MPIC_READ(softc, reg) \ bus_space_read_4((softc)->mpic_bst, (softc)->mpic_bsh, (reg)) #define MPIC_CPU_WRITE(softc, reg, val) \ bus_space_write_4((softc)->cpu_bst, (softc)->cpu_bsh, (reg), (val)) #define MPIC_CPU_READ(softc, reg) \ bus_space_read_4((softc)->cpu_bst, (softc)->cpu_bsh, (reg)) #define MPIC_DRBL_WRITE(softc, reg, val) \ bus_space_write_4((softc)->drbl_bst, (softc)->drbl_bsh, (reg), (val)) #define MPIC_DRBL_READ(softc, reg) \ bus_space_read_4((softc)->drbl_bst, (softc)->drbl_bsh, (reg)) static int mv_mpic_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_search_compatible(dev, compat_data)->ocd_data) return (ENXIO); device_set_desc(dev, "Marvell Integrated Interrupt Controller"); return (0); } #ifdef INTRNG static int mv_mpic_register_isrcs(struct mv_mpic_softc *sc) { int error; uint32_t irq; struct intr_irqsrc *isrc; const char *name; sc->mpic_isrcs = malloc(sc->nirqs * sizeof (*sc->mpic_isrcs), M_DEVBUF, M_WAITOK | M_ZERO); name = device_get_nameunit(sc->sc_dev); for (irq = 0; irq < sc->nirqs; irq++) { sc->mpic_isrcs[irq].mmi_irq = irq; isrc = &sc->mpic_isrcs[irq].mmi_isrc; if (irq < MPIC_PPI) { error = intr_isrc_register(isrc, sc->sc_dev, INTR_ISRCF_PPI, "%s", name); } else { error = intr_isrc_register(isrc, sc->sc_dev, 0, "%s", name); } if (error != 0) { /* XXX call intr_isrc_deregister() */ device_printf(sc->sc_dev, "%s failed", __func__); return (error); } } return (0); } #endif static int mv_mpic_attach(device_t dev) { struct mv_mpic_softc *sc; int error; uint32_t val; sc = (struct mv_mpic_softc *)device_get_softc(dev); if (mv_mpic_sc != NULL) return (ENXIO); mv_mpic_sc = sc; sc->sc_dev = dev; mtx_init(&sc->mtx, "MPIC lock", NULL, MTX_SPIN); error = bus_alloc_resources(dev, mv_mpic_spec, sc->mpic_res); if (error) { device_printf(dev, "could not allocate resources\n"); return (ENXIO); } #ifdef INTRNG if (sc->mpic_res[3] == NULL) device_printf(dev, "No interrupt to use.\n"); else bus_setup_intr(dev, sc->mpic_res[3], INTR_TYPE_CLK, mpic_intr, NULL, sc, &sc->intr_hand); #endif sc->mpic_bst = rman_get_bustag(sc->mpic_res[0]); sc->mpic_bsh = rman_get_bushandle(sc->mpic_res[0]); sc->cpu_bst = rman_get_bustag(sc->mpic_res[1]); sc->cpu_bsh = rman_get_bushandle(sc->mpic_res[1]); if (sc->mpic_res[2] != NULL) { /* This is required only if MSIs are used. */ sc->drbl_bst = rman_get_bustag(sc->mpic_res[2]); sc->drbl_bsh = rman_get_bushandle(sc->mpic_res[2]); } bus_space_write_4(mv_mpic_sc->mpic_bst, mv_mpic_sc->mpic_bsh, MPIC_CTRL, 1); MPIC_CPU_WRITE(mv_mpic_sc, MPIC_CTP, 0); val = MPIC_READ(mv_mpic_sc, MPIC_CTRL); sc->nirqs = MPIC_CTRL_NIRQS(val); #ifdef INTRNG if (mv_mpic_register_isrcs(sc) != 0) { device_printf(dev, "could not register PIC ISRCs\n"); bus_release_resources(dev, mv_mpic_spec, sc->mpic_res); return (ENXIO); } if (intr_pic_register(dev, OF_xref_from_device(dev)) != 0) { device_printf(dev, "could not register PIC\n"); bus_release_resources(dev, mv_mpic_spec, sc->mpic_res); return (ENXIO); } #endif mpic_unmask_msi(); return (0); } #ifdef INTRNG static int mpic_intr(void *arg) { struct mv_mpic_softc *sc; uint32_t cause, irqsrc; unsigned int irq; u_int cpuid; sc = arg; cpuid = PCPU_GET(cpuid); irq = 0; for (cause = MPIC_CPU_READ(sc, MPIC_PPI_CAUSE); cause > 0; cause >>= 1, irq++) { if (cause & 1) { irqsrc = MPIC_READ(sc, MPIC_INT_CTL(irq)); if ((irqsrc & MPIC_INT_IRQ_FIQ_MASK(cpuid)) == 0) continue; if (intr_isrc_dispatch(&sc->mpic_isrcs[irq].mmi_isrc, curthread->td_intr_frame) != 0) { mpic_mask_irq(irq); device_printf(sc->sc_dev, "Stray irq %u " "disabled\n", irq); } } } return (FILTER_HANDLED); } static void mpic_disable_intr(device_t dev, struct intr_irqsrc *isrc) { u_int irq; irq = ((struct mv_mpic_irqsrc *)isrc)->mmi_irq; mpic_mask_irq(irq); } static void mpic_enable_intr(device_t dev, struct intr_irqsrc *isrc) { u_int irq; irq = ((struct mv_mpic_irqsrc *)isrc)->mmi_irq; mpic_unmask_irq(irq); } static int mpic_map_intr(device_t dev, struct intr_map_data *data, struct intr_irqsrc **isrcp) { struct mv_mpic_softc *sc; sc = device_get_softc(dev); if (data->type != INTR_MAP_DATA_FDT || data->fdt.ncells !=1 || data->fdt.cells[0] >= sc->nirqs) return (EINVAL); *isrcp = &sc->mpic_isrcs[data->fdt.cells[0]].mmi_isrc; return (0); } static void mpic_pre_ithread(device_t dev, struct intr_irqsrc *isrc) { mpic_disable_intr(dev, isrc); } static void mpic_post_ithread(device_t dev, struct intr_irqsrc *isrc) { mpic_enable_intr(dev, isrc); } #endif static device_method_t mv_mpic_methods[] = { DEVMETHOD(device_probe, mv_mpic_probe), DEVMETHOD(device_attach, mv_mpic_attach), #ifdef INTRNG DEVMETHOD(pic_disable_intr, mpic_disable_intr), DEVMETHOD(pic_enable_intr, mpic_enable_intr), DEVMETHOD(pic_map_intr, mpic_map_intr), DEVMETHOD(pic_post_ithread, mpic_post_ithread), DEVMETHOD(pic_pre_ithread, mpic_pre_ithread), #endif { 0, 0 } }; static driver_t mv_mpic_driver = { "mpic", mv_mpic_methods, sizeof(struct mv_mpic_softc), }; static devclass_t mv_mpic_devclass; EARLY_DRIVER_MODULE(mpic, simplebus, mv_mpic_driver, mv_mpic_devclass, 0, 0, BUS_PASS_INTERRUPT); #ifndef INTRNG int arm_get_next_irq(int last) { u_int irq, next = -1; irq = mv_mpic_get_cause() & MPIC_IRQ_MASK; CTR2(KTR_INTR, "%s: irq:%#x", __func__, irq); if (irq != MPIC_IRQ_MASK) { if (irq == MPIC_INT_ERR) irq = mv_mpic_get_cause_err(); if (irq == MPIC_INT_MSI) irq = mv_mpic_get_msi(); next = irq; } CTR3(KTR_INTR, "%s: last=%d, next=%d", __func__, last, next); return (next); } /* * XXX We can make arm_enable_irq to operate on ICE and then mask/unmask only * by ISM/ICM and remove access to ICE in masking operation */ void arm_mask_irq(uintptr_t nb) { mpic_mask_irq(nb); } static void arm_mask_irq_err(uintptr_t nb) { mpic_mask_irq_err(nb); } void arm_unmask_irq(uintptr_t nb) { mpic_unmask_irq(nb); } void arm_unmask_irq_err(uintptr_t nb) { mpic_unmask_irq_err(nb); } #endif static void mpic_unmask_msi(void) { mpic_unmask_irq(MPIC_INT_MSI); } static void mpic_unmask_irq_err(uintptr_t nb) { uint32_t mask; uint8_t bit_off; bus_space_write_4(mv_mpic_sc->mpic_bst, mv_mpic_sc->mpic_bsh, MPIC_ISE, MPIC_INT_ERR); MPIC_CPU_WRITE(mv_mpic_sc, MPIC_ICM, MPIC_INT_ERR); bit_off = nb - ERR_IRQ; mask = MPIC_CPU_READ(mv_mpic_sc, MPIC_ERR_MASK); mask |= (1 << bit_off); MPIC_CPU_WRITE(mv_mpic_sc, MPIC_ERR_MASK, mask); } static void mpic_mask_irq_err(uintptr_t nb) { uint32_t mask; uint8_t bit_off; bit_off = nb - ERR_IRQ; mask = MPIC_CPU_READ(mv_mpic_sc, MPIC_ERR_MASK); mask &= ~(1 << bit_off); MPIC_CPU_WRITE(mv_mpic_sc, MPIC_ERR_MASK, mask); } static void mpic_unmask_irq(uintptr_t nb) { if (nb < ERR_IRQ) { bus_space_write_4(mv_mpic_sc->mpic_bst, mv_mpic_sc->mpic_bsh, MPIC_ISE, nb); MPIC_CPU_WRITE(mv_mpic_sc, MPIC_ICM, nb); } else if (nb < MSI_IRQ) mpic_unmask_irq_err(nb); if (nb == 0) MPIC_CPU_WRITE(mv_mpic_sc, MPIC_IN_DRBL_MASK, 0xffffffff); } static void mpic_mask_irq(uintptr_t nb) { if (nb < ERR_IRQ) { bus_space_write_4(mv_mpic_sc->mpic_bst, mv_mpic_sc->mpic_bsh, MPIC_ICE, nb); MPIC_CPU_WRITE(mv_mpic_sc, MPIC_ISM, nb); } else if (nb < MSI_IRQ) mpic_mask_irq_err(nb); } uint32_t mv_mpic_get_cause(void) { return (MPIC_CPU_READ(mv_mpic_sc, MPIC_IIACK)); } uint32_t mv_mpic_get_cause_err(void) { uint32_t err_cause; uint8_t bit_off; err_cause = bus_space_read_4(mv_mpic_sc->mpic_bst, mv_mpic_sc->mpic_bsh, MPIC_ERR_CAUSE); if (err_cause) bit_off = ffs(err_cause) - 1; else return (-1); debugf("%s: irq:%x cause:%x\n", __func__, bit_off, err_cause); return (ERR_IRQ + bit_off); } uint32_t mv_mpic_get_msi(void) { uint32_t cause; uint8_t bit_off; KASSERT(mv_mpic_sc->drbl_bst != NULL, ("No doorbell in mv_mpic_get_msi")); cause = MPIC_DRBL_READ(mv_mpic_sc, 0); if (cause) bit_off = ffs(cause) - 1; else return (-1); debugf("%s: irq:%x cause:%x\n", __func__, bit_off, cause); cause &= ~(1 << bit_off); MPIC_DRBL_WRITE(mv_mpic_sc, 0, cause); return (MSI_IRQ + bit_off); } int mv_msi_data(int irq, uint64_t *addr, uint32_t *data) { u_long phys, base, size; phandle_t node; int error; node = ofw_bus_get_node(mv_mpic_sc->sc_dev); - /* Get physical addres of register space */ + /* Get physical address of register space */ error = fdt_get_range(OF_parent(node), 0, &phys, &size); if (error) { printf("%s: Cannot get register physical address, err:%d", __func__, error); return (error); } /* Get offset of MPIC register space */ error = fdt_regsize(node, &base, &size); if (error) { printf("%s: Cannot get MPIC register offset, err:%d", __func__, error); return (error); } *addr = phys + base + MPIC_SOFT_INT; *data = MPIC_SOFT_INT_DRBL1 | irq; return (0); } #if defined(SMP) && defined(SOC_MV_ARMADAXP) void intr_pic_init_secondary(void) { } void pic_ipi_send(cpuset_t cpus, u_int ipi) { uint32_t val, i; val = 0x00000000; for (i = 0; i < MAXCPU; i++) if (CPU_ISSET(i, &cpus)) val |= (1 << (8 + i)); val |= ipi; bus_space_write_4(mv_mpic_sc->mpic_bst, mv_mpic_sc->mpic_bsh, MPIC_SOFT_INT, val); } int pic_ipi_read(int i __unused) { uint32_t val; int ipi; val = MPIC_CPU_READ(mv_mpic_sc, MPIC_IN_DRBL); if (val) { ipi = ffs(val) - 1; MPIC_CPU_WRITE(mv_mpic_sc, MPIC_IN_DRBL, ~(1 << ipi)); return (ipi); } return (0x3ff); } void pic_ipi_clear(int ipi) { } #endif Index: user/ngie/release-pkg-fix-tests/sys/arm/mv/mv_common.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/mv/mv_common.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/mv/mv_common.c (revision 299091) @@ -1,2334 +1,2334 @@ /*- * Copyright (C) 2008-2011 MARVELL INTERNATIONAL LTD. * All rights reserved. * * Developed by Semihalf. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of MARVELL nor the names of contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include MALLOC_DEFINE(M_IDMA, "idma", "idma dma test memory"); #define IDMA_DEBUG #undef IDMA_DEBUG #define MAX_CPU_WIN 5 #ifdef DEBUG #define debugf(fmt, args...) do { printf("%s(): ", __func__); \ printf(fmt,##args); } while (0) #else #define debugf(fmt, args...) #endif #ifdef DEBUG #define MV_DUMP_WIN 1 #else #define MV_DUMP_WIN 0 #endif static int win_eth_can_remap(int i); #ifndef SOC_MV_FREY static int decode_win_cpu_valid(void); #endif static int decode_win_usb_valid(void); static int decode_win_usb3_valid(void); static int decode_win_eth_valid(void); static int decode_win_pcie_valid(void); static int decode_win_sata_valid(void); static int decode_win_idma_valid(void); static int decode_win_xor_valid(void); #ifndef SOC_MV_FREY static void decode_win_cpu_setup(void); #endif #ifdef SOC_MV_ARMADAXP static int decode_win_sdram_fixup(void); #endif static void decode_win_usb_setup(u_long); static void decode_win_usb3_setup(u_long); static void decode_win_eth_setup(u_long); static void decode_win_sata_setup(u_long); static void decode_win_idma_setup(u_long); static void decode_win_xor_setup(u_long); static void decode_win_usb_dump(u_long); static void decode_win_usb3_dump(u_long); static void decode_win_eth_dump(u_long base); static void decode_win_idma_dump(u_long base); static void decode_win_xor_dump(u_long base); static int fdt_get_ranges(const char *, void *, int, int *, int *); #ifdef SOC_MV_ARMADA38X int gic_decode_fdt(phandle_t iparent, pcell_t *intr, int *interrupt, int *trig, int *pol); #endif static int win_cpu_from_dt(void); static int fdt_win_setup(void); static uint32_t dev_mask = 0; static int cpu_wins_no = 0; static int eth_port = 0; static int usb_port = 0; static struct decode_win cpu_win_tbl[MAX_CPU_WIN]; const struct decode_win *cpu_wins = cpu_win_tbl; typedef void (*decode_win_setup_t)(u_long); typedef void (*dump_win_t)(u_long); struct soc_node_spec { const char *compat; decode_win_setup_t decode_handler; dump_win_t dump_handler; }; static struct soc_node_spec soc_nodes[] = { { "mrvl,ge", &decode_win_eth_setup, &decode_win_eth_dump }, { "mrvl,usb-ehci", &decode_win_usb_setup, &decode_win_usb_dump }, { "marvell,armada-380-xhci", &decode_win_usb3_setup, &decode_win_usb3_dump }, { "mrvl,sata", &decode_win_sata_setup, NULL }, { "mrvl,xor", &decode_win_xor_setup, &decode_win_xor_dump }, { "mrvl,idma", &decode_win_idma_setup, &decode_win_idma_dump }, { "mrvl,pcie", &decode_win_pcie_setup, NULL }, { NULL, NULL, NULL }, }; struct fdt_pm_mask_entry fdt_pm_mask_table[] = { { "mrvl,ge", CPU_PM_CTRL_GE(0) }, { "mrvl,ge", CPU_PM_CTRL_GE(1) }, { "mrvl,usb-ehci", CPU_PM_CTRL_USB(0) }, { "mrvl,usb-ehci", CPU_PM_CTRL_USB(1) }, { "mrvl,usb-ehci", CPU_PM_CTRL_USB(2) }, { "mrvl,xor", CPU_PM_CTRL_XOR }, { "mrvl,sata", CPU_PM_CTRL_SATA }, { NULL, 0 } }; static __inline int pm_is_disabled(uint32_t mask) { #if defined(SOC_MV_KIRKWOOD) return (soc_power_ctrl_get(mask) == mask); #else return (soc_power_ctrl_get(mask) == mask ? 0 : 1); #endif } /* * Disable device using power management register. * 1 - Device Power On * 0 - Device Power Off * Mask can be set in loader. * EXAMPLE: * loader> set hw.pm-disable-mask=0x2 * * Common mask: * |-------------------------------| * | Device | Kirkwood | Discovery | * |-------------------------------| * | USB0 | 0x00008 | 0x020000 | * |-------------------------------| * | USB1 | - | 0x040000 | * |-------------------------------| * | USB2 | - | 0x080000 | * |-------------------------------| * | GE0 | 0x00001 | 0x000002 | * |-------------------------------| * | GE1 | - | 0x000004 | * |-------------------------------| * | IDMA | - | 0x100000 | * |-------------------------------| * | XOR | 0x10000 | 0x200000 | * |-------------------------------| * | CESA | 0x20000 | 0x400000 | * |-------------------------------| * | SATA | 0x04000 | 0x004000 | * --------------------------------| * This feature can be used only on Kirkwood and Discovery * machines. */ static __inline void pm_disable_device(int mask) { #ifdef DIAGNOSTIC uint32_t reg; reg = soc_power_ctrl_get(CPU_PM_CTRL_ALL); printf("Power Management Register: 0%x\n", reg); reg &= ~mask; soc_power_ctrl_set(reg); printf("Device %x is disabled\n", mask); reg = soc_power_ctrl_get(CPU_PM_CTRL_ALL); printf("Power Management Register: 0%x\n", reg); #endif } int fdt_pm(phandle_t node) { uint32_t cpu_pm_ctrl; int i, ena, compat; ena = 1; cpu_pm_ctrl = read_cpu_ctrl(CPU_PM_CTRL); for (i = 0; fdt_pm_mask_table[i].compat != NULL; i++) { if (dev_mask & (1 << i)) continue; compat = fdt_is_compatible(node, fdt_pm_mask_table[i].compat); #if defined(SOC_MV_KIRKWOOD) if (compat && (cpu_pm_ctrl & fdt_pm_mask_table[i].mask)) { dev_mask |= (1 << i); ena = 0; break; } else if (compat) { dev_mask |= (1 << i); break; } #else if (compat && (~cpu_pm_ctrl & fdt_pm_mask_table[i].mask)) { dev_mask |= (1 << i); ena = 0; break; } else if (compat) { dev_mask |= (1 << i); break; } #endif } return (ena); } uint32_t read_cpu_ctrl(uint32_t reg) { return (bus_space_read_4(fdtbus_bs_tag, MV_CPU_CONTROL_BASE, reg)); } void write_cpu_ctrl(uint32_t reg, uint32_t val) { bus_space_write_4(fdtbus_bs_tag, MV_CPU_CONTROL_BASE, reg, val); } #if defined(SOC_MV_ARMADAXP) || defined(SOC_MV_ARMADA38X) uint32_t read_cpu_mp_clocks(uint32_t reg) { return (bus_space_read_4(fdtbus_bs_tag, MV_MP_CLOCKS_BASE, reg)); } void write_cpu_mp_clocks(uint32_t reg, uint32_t val) { bus_space_write_4(fdtbus_bs_tag, MV_MP_CLOCKS_BASE, reg, val); } uint32_t read_cpu_misc(uint32_t reg) { return (bus_space_read_4(fdtbus_bs_tag, MV_MISC_BASE, reg)); } void write_cpu_misc(uint32_t reg, uint32_t val) { bus_space_write_4(fdtbus_bs_tag, MV_MISC_BASE, reg, val); } #endif void cpu_reset(void) { #if defined(SOC_MV_ARMADAXP) || defined (SOC_MV_ARMADA38X) write_cpu_misc(RSTOUTn_MASK, SOFT_RST_OUT_EN); write_cpu_misc(SYSTEM_SOFT_RESET, SYS_SOFT_RST); #else write_cpu_ctrl(RSTOUTn_MASK, SOFT_RST_OUT_EN); write_cpu_ctrl(SYSTEM_SOFT_RESET, SYS_SOFT_RST); #endif while (1); } uint32_t cpu_extra_feat(void) { uint32_t dev, rev; uint32_t ef = 0; soc_id(&dev, &rev); switch (dev) { case MV_DEV_88F6281: case MV_DEV_88F6282: case MV_DEV_88RC8180: case MV_DEV_MV78100_Z0: case MV_DEV_MV78100: __asm __volatile("mrc p15, 1, %0, c15, c1, 0" : "=r" (ef)); break; case MV_DEV_88F5182: case MV_DEV_88F5281: __asm __volatile("mrc p15, 0, %0, c14, c0, 0" : "=r" (ef)); break; default: if (bootverbose) printf("This ARM Core does not support any extra features\n"); } return (ef); } /* * Get the power status of device. This feature is only supported on * Kirkwood and Discovery SoCs. */ uint32_t soc_power_ctrl_get(uint32_t mask) { #if !defined(SOC_MV_ORION) && !defined(SOC_MV_LOKIPLUS) && !defined(SOC_MV_FREY) if (mask != CPU_PM_CTRL_NONE) mask &= read_cpu_ctrl(CPU_PM_CTRL); return (mask); #else return (mask); #endif } /* * Set the power status of device. This feature is only supported on * Kirkwood and Discovery SoCs. */ void soc_power_ctrl_set(uint32_t mask) { #if !defined(SOC_MV_ORION) && !defined(SOC_MV_LOKIPLUS) if (mask != CPU_PM_CTRL_NONE) write_cpu_ctrl(CPU_PM_CTRL, mask); #endif } void soc_id(uint32_t *dev, uint32_t *rev) { /* * Notice: system identifiers are available in the registers range of * PCIE controller, so using this function is only allowed (and * possible) after the internal registers range has been mapped in via * devmap_bootstrap(). */ *dev = bus_space_read_4(fdtbus_bs_tag, MV_PCIE_BASE, 0) >> 16; *rev = bus_space_read_4(fdtbus_bs_tag, MV_PCIE_BASE, 8) & 0xff; } static void soc_identify(void) { uint32_t d, r, size, mode; const char *dev; const char *rev; soc_id(&d, &r); printf("SOC: "); if (bootverbose) printf("(0x%4x:0x%02x) ", d, r); rev = ""; switch (d) { case MV_DEV_88F5181: dev = "Marvell 88F5181"; if (r == 3) rev = "B1"; break; case MV_DEV_88F5182: dev = "Marvell 88F5182"; if (r == 2) rev = "A2"; break; case MV_DEV_88F5281: dev = "Marvell 88F5281"; if (r == 4) rev = "D0"; else if (r == 5) rev = "D1"; else if (r == 6) rev = "D2"; break; case MV_DEV_88F6281: dev = "Marvell 88F6281"; if (r == 0) rev = "Z0"; else if (r == 2) rev = "A0"; else if (r == 3) rev = "A1"; break; case MV_DEV_88RC8180: dev = "Marvell 88RC8180"; break; case MV_DEV_88RC9480: dev = "Marvell 88RC9480"; break; case MV_DEV_88RC9580: dev = "Marvell 88RC9580"; break; case MV_DEV_88F6781: dev = "Marvell 88F6781"; if (r == 2) rev = "Y0"; break; case MV_DEV_88F6282: dev = "Marvell 88F6282"; if (r == 0) rev = "A0"; else if (r == 1) rev = "A1"; break; case MV_DEV_88F6828: dev = "Marvell 88F6828"; break; case MV_DEV_88F6820: dev = "Marvell 88F6820"; break; case MV_DEV_88F6810: dev = "Marvell 88F6810"; break; case MV_DEV_MV78100_Z0: dev = "Marvell MV78100 Z0"; break; case MV_DEV_MV78100: dev = "Marvell MV78100"; break; case MV_DEV_MV78160: dev = "Marvell MV78160"; break; case MV_DEV_MV78260: dev = "Marvell MV78260"; break; case MV_DEV_MV78460: dev = "Marvell MV78460"; break; default: dev = "UNKNOWN"; break; } printf("%s", dev); if (*rev != '\0') printf(" rev %s", rev); printf(", TClock %dMHz\n", get_tclk() / 1000 / 1000); mode = read_cpu_ctrl(CPU_CONFIG); printf(" Instruction cache prefetch %s, data cache prefetch %s\n", (mode & CPU_CONFIG_IC_PREF) ? "enabled" : "disabled", (mode & CPU_CONFIG_DC_PREF) ? "enabled" : "disabled"); switch (d) { case MV_DEV_88F6281: case MV_DEV_88F6282: mode = read_cpu_ctrl(CPU_L2_CONFIG) & CPU_L2_CONFIG_MODE; printf(" 256KB 4-way set-associative %s unified L2 cache\n", mode ? "write-through" : "write-back"); break; case MV_DEV_MV78100: mode = read_cpu_ctrl(CPU_CONTROL); size = mode & CPU_CONTROL_L2_SIZE; mode = mode & CPU_CONTROL_L2_MODE; printf(" %s set-associative %s unified L2 cache\n", size ? "256KB 4-way" : "512KB 8-way", mode ? "write-through" : "write-back"); break; default: break; } } static void platform_identify(void *dummy) { soc_identify(); /* * XXX Board identification e.g. read out from FPGA or similar should * go here */ } SYSINIT(platform_identify, SI_SUB_CPU, SI_ORDER_SECOND, platform_identify, NULL); #ifdef KDB static void mv_enter_debugger(void *dummy) { if (boothowto & RB_KDB) kdb_enter(KDB_WHY_BOOTFLAGS, "Boot flags requested debugger"); } SYSINIT(mv_enter_debugger, SI_SUB_CPU, SI_ORDER_ANY, mv_enter_debugger, NULL); #endif int soc_decode_win(void) { uint32_t dev, rev; int mask, err; mask = 0; TUNABLE_INT_FETCH("hw.pm-disable-mask", &mask); if (mask != 0) pm_disable_device(mask); /* Retrieve data about physical addresses from device tree. */ if ((err = win_cpu_from_dt()) != 0) return (err); /* Retrieve our ID: some windows facilities vary between SoC models */ soc_id(&dev, &rev); #ifdef SOC_MV_ARMADAXP if ((err = decode_win_sdram_fixup()) != 0) return(err); #endif #ifndef SOC_MV_FREY if (!decode_win_cpu_valid() || !decode_win_usb_valid() || !decode_win_eth_valid() || !decode_win_idma_valid() || !decode_win_pcie_valid() || !decode_win_sata_valid() || !decode_win_xor_valid() || !decode_win_usb3_valid()) return (EINVAL); decode_win_cpu_setup(); #else if (!decode_win_usb_valid() || !decode_win_eth_valid() || !decode_win_idma_valid() || !decode_win_pcie_valid() || !decode_win_sata_valid() || !decode_win_xor_valid() || !decode_win_usb3_valid()) return (EINVAL); #endif if (MV_DUMP_WIN) soc_dump_decode_win(); eth_port = 0; usb_port = 0; if ((err = fdt_win_setup()) != 0) return (err); return (0); } /************************************************************************** * Decode windows registers accessors **************************************************************************/ #if !defined(SOC_MV_FREY) WIN_REG_IDX_RD(win_cpu, cr, MV_WIN_CPU_CTRL, MV_MBUS_BRIDGE_BASE) WIN_REG_IDX_RD(win_cpu, br, MV_WIN_CPU_BASE, MV_MBUS_BRIDGE_BASE) WIN_REG_IDX_RD(win_cpu, remap_l, MV_WIN_CPU_REMAP_LO, MV_MBUS_BRIDGE_BASE) WIN_REG_IDX_RD(win_cpu, remap_h, MV_WIN_CPU_REMAP_HI, MV_MBUS_BRIDGE_BASE) WIN_REG_IDX_WR(win_cpu, cr, MV_WIN_CPU_CTRL, MV_MBUS_BRIDGE_BASE) WIN_REG_IDX_WR(win_cpu, br, MV_WIN_CPU_BASE, MV_MBUS_BRIDGE_BASE) WIN_REG_IDX_WR(win_cpu, remap_l, MV_WIN_CPU_REMAP_LO, MV_MBUS_BRIDGE_BASE) WIN_REG_IDX_WR(win_cpu, remap_h, MV_WIN_CPU_REMAP_HI, MV_MBUS_BRIDGE_BASE) #endif WIN_REG_BASE_IDX_RD(win_usb, cr, MV_WIN_USB_CTRL) WIN_REG_BASE_IDX_RD(win_usb, br, MV_WIN_USB_BASE) WIN_REG_BASE_IDX_WR(win_usb, cr, MV_WIN_USB_CTRL) WIN_REG_BASE_IDX_WR(win_usb, br, MV_WIN_USB_BASE) #ifdef SOC_MV_ARMADA38X WIN_REG_BASE_IDX_RD(win_usb3, cr, MV_WIN_USB3_CTRL) WIN_REG_BASE_IDX_RD(win_usb3, br, MV_WIN_USB3_BASE) WIN_REG_BASE_IDX_WR(win_usb3, cr, MV_WIN_USB3_CTRL) WIN_REG_BASE_IDX_WR(win_usb3, br, MV_WIN_USB3_BASE) #endif WIN_REG_BASE_IDX_RD(win_eth, br, MV_WIN_ETH_BASE) WIN_REG_BASE_IDX_RD(win_eth, sz, MV_WIN_ETH_SIZE) WIN_REG_BASE_IDX_RD(win_eth, har, MV_WIN_ETH_REMAP) WIN_REG_BASE_IDX_WR(win_eth, br, MV_WIN_ETH_BASE) WIN_REG_BASE_IDX_WR(win_eth, sz, MV_WIN_ETH_SIZE) WIN_REG_BASE_IDX_WR(win_eth, har, MV_WIN_ETH_REMAP) WIN_REG_BASE_IDX_RD2(win_xor, br, MV_WIN_XOR_BASE) WIN_REG_BASE_IDX_RD2(win_xor, sz, MV_WIN_XOR_SIZE) WIN_REG_BASE_IDX_RD2(win_xor, har, MV_WIN_XOR_REMAP) WIN_REG_BASE_IDX_RD2(win_xor, ctrl, MV_WIN_XOR_CTRL) WIN_REG_BASE_IDX_WR2(win_xor, br, MV_WIN_XOR_BASE) WIN_REG_BASE_IDX_WR2(win_xor, sz, MV_WIN_XOR_SIZE) WIN_REG_BASE_IDX_WR2(win_xor, har, MV_WIN_XOR_REMAP) WIN_REG_BASE_IDX_WR2(win_xor, ctrl, MV_WIN_XOR_CTRL) WIN_REG_BASE_RD(win_eth, bare, 0x290) WIN_REG_BASE_RD(win_eth, epap, 0x294) WIN_REG_BASE_WR(win_eth, bare, 0x290) WIN_REG_BASE_WR(win_eth, epap, 0x294) WIN_REG_BASE_IDX_RD(win_pcie, cr, MV_WIN_PCIE_CTRL); WIN_REG_BASE_IDX_RD(win_pcie, br, MV_WIN_PCIE_BASE); WIN_REG_BASE_IDX_RD(win_pcie, remap, MV_WIN_PCIE_REMAP); WIN_REG_BASE_IDX_WR(win_pcie, cr, MV_WIN_PCIE_CTRL); WIN_REG_BASE_IDX_WR(win_pcie, br, MV_WIN_PCIE_BASE); WIN_REG_BASE_IDX_WR(win_pcie, remap, MV_WIN_PCIE_REMAP); WIN_REG_BASE_IDX_RD(pcie_bar, br, MV_PCIE_BAR_BASE); WIN_REG_BASE_IDX_WR(pcie_bar, br, MV_PCIE_BAR_BASE); WIN_REG_BASE_IDX_WR(pcie_bar, brh, MV_PCIE_BAR_BASE_H); WIN_REG_BASE_IDX_WR(pcie_bar, cr, MV_PCIE_BAR_CTRL); WIN_REG_BASE_IDX_RD(win_idma, br, MV_WIN_IDMA_BASE) WIN_REG_BASE_IDX_RD(win_idma, sz, MV_WIN_IDMA_SIZE) WIN_REG_BASE_IDX_RD(win_idma, har, MV_WIN_IDMA_REMAP) WIN_REG_BASE_IDX_RD(win_idma, cap, MV_WIN_IDMA_CAP) WIN_REG_BASE_IDX_WR(win_idma, br, MV_WIN_IDMA_BASE) WIN_REG_BASE_IDX_WR(win_idma, sz, MV_WIN_IDMA_SIZE) WIN_REG_BASE_IDX_WR(win_idma, har, MV_WIN_IDMA_REMAP) WIN_REG_BASE_IDX_WR(win_idma, cap, MV_WIN_IDMA_CAP) WIN_REG_BASE_RD(win_idma, bare, 0xa80) WIN_REG_BASE_WR(win_idma, bare, 0xa80) WIN_REG_BASE_IDX_RD(win_sata, cr, MV_WIN_SATA_CTRL); WIN_REG_BASE_IDX_RD(win_sata, br, MV_WIN_SATA_BASE); WIN_REG_BASE_IDX_WR(win_sata, cr, MV_WIN_SATA_CTRL); WIN_REG_BASE_IDX_WR(win_sata, br, MV_WIN_SATA_BASE); #ifndef SOC_MV_DOVE WIN_REG_IDX_RD(ddr, br, MV_WIN_DDR_BASE, MV_DDR_CADR_BASE) WIN_REG_IDX_RD(ddr, sz, MV_WIN_DDR_SIZE, MV_DDR_CADR_BASE) WIN_REG_IDX_WR(ddr, br, MV_WIN_DDR_BASE, MV_DDR_CADR_BASE) WIN_REG_IDX_WR(ddr, sz, MV_WIN_DDR_SIZE, MV_DDR_CADR_BASE) #else /* * On 88F6781 (Dove) SoC DDR Controller is accessed through * single MBUS <-> AXI bridge. In this case we provide emulated * ddr_br_read() and ddr_sz_read() functions to keep compatibility * with common decoding windows setup code. */ static inline uint32_t ddr_br_read(int i) { uint32_t mmap; /* Read Memory Address Map Register for CS i */ mmap = bus_space_read_4(fdtbus_bs_tag, MV_DDR_CADR_BASE + (i * 0x10), 0); /* Return CS i base address */ return (mmap & 0xFF000000); } static inline uint32_t ddr_sz_read(int i) { uint32_t mmap, size; /* Read Memory Address Map Register for CS i */ mmap = bus_space_read_4(fdtbus_bs_tag, MV_DDR_CADR_BASE + (i * 0x10), 0); /* Extract size of CS space in 64kB units */ size = (1 << ((mmap >> 16) & 0x0F)); /* Return CS size and enable/disable status */ return (((size - 1) << 16) | (mmap & 0x01)); } #endif #if !defined(SOC_MV_FREY) /************************************************************************** * Decode windows helper routines **************************************************************************/ void soc_dump_decode_win(void) { uint32_t dev, rev; int i; soc_id(&dev, &rev); for (i = 0; i < MV_WIN_CPU_MAX; i++) { printf("CPU window#%d: c 0x%08x, b 0x%08x", i, win_cpu_cr_read(i), win_cpu_br_read(i)); if (win_cpu_can_remap(i)) printf(", rl 0x%08x, rh 0x%08x", win_cpu_remap_l_read(i), win_cpu_remap_h_read(i)); printf("\n"); } printf("Internal regs base: 0x%08x\n", bus_space_read_4(fdtbus_bs_tag, MV_INTREGS_BASE, 0)); for (i = 0; i < MV_WIN_DDR_MAX; i++) printf("DDR CS#%d: b 0x%08x, s 0x%08x\n", i, ddr_br_read(i), ddr_sz_read(i)); } /************************************************************************** * CPU windows routines **************************************************************************/ int win_cpu_can_remap(int i) { uint32_t dev, rev; soc_id(&dev, &rev); /* Depending on the SoC certain windows have remap capability */ if ((dev == MV_DEV_88F5182 && i < 2) || (dev == MV_DEV_88F5281 && i < 4) || (dev == MV_DEV_88F6281 && i < 4) || (dev == MV_DEV_88F6282 && i < 4) || (dev == MV_DEV_88F6828 && i < 20) || (dev == MV_DEV_88F6820 && i < 20) || (dev == MV_DEV_88F6810 && i < 20) || (dev == MV_DEV_88RC8180 && i < 2) || (dev == MV_DEV_88F6781 && i < 4) || (dev == MV_DEV_MV78100_Z0 && i < 8) || ((dev & MV_DEV_FAMILY_MASK) == MV_DEV_DISCOVERY && i < 8)) return (1); return (0); } /* XXX This should check for overlapping remap fields too.. */ int decode_win_overlap(int win, int win_no, const struct decode_win *wintab) { const struct decode_win *tab; int i; tab = wintab; for (i = 0; i < win_no; i++, tab++) { if (i == win) /* Skip self */ continue; if ((tab->base + tab->size - 1) < (wintab + win)->base) continue; else if (((wintab + win)->base + (wintab + win)->size - 1) < tab->base) continue; else return (i); } return (-1); } static int decode_win_cpu_valid(void) { int i, j, rv; uint32_t b, e, s; if (cpu_wins_no > MV_WIN_CPU_MAX) { printf("CPU windows: too many entries: %d\n", cpu_wins_no); return (0); } rv = 1; for (i = 0; i < cpu_wins_no; i++) { if (cpu_wins[i].target == 0) { printf("CPU window#%d: DDR target window is not " "supposed to be reprogrammed!\n", i); rv = 0; } if (cpu_wins[i].remap != ~0 && win_cpu_can_remap(i) != 1) { printf("CPU window#%d: not capable of remapping, but " "val 0x%08x defined\n", i, cpu_wins[i].remap); rv = 0; } s = cpu_wins[i].size; b = cpu_wins[i].base; e = b + s - 1; if (s > (0xFFFFFFFF - b + 1)) { /* * XXX this boundary check should account for 64bit * and remapping.. */ printf("CPU window#%d: no space for size 0x%08x at " "0x%08x\n", i, s, b); rv = 0; continue; } if (b != rounddown2(b, s)) { printf("CPU window#%d: address 0x%08x is not aligned " "to 0x%08x\n", i, b, s); rv = 0; continue; } j = decode_win_overlap(i, cpu_wins_no, &cpu_wins[0]); if (j >= 0) { printf("CPU window#%d: (0x%08x - 0x%08x) overlaps " "with #%d (0x%08x - 0x%08x)\n", i, b, e, j, cpu_wins[j].base, cpu_wins[j].base + cpu_wins[j].size - 1); rv = 0; } } return (rv); } int decode_win_cpu_set(int target, int attr, vm_paddr_t base, uint32_t size, vm_paddr_t remap) { uint32_t br, cr; int win, i; if (remap == ~0) { win = MV_WIN_CPU_MAX - 1; i = -1; } else { win = 0; i = 1; } while ((win >= 0) && (win < MV_WIN_CPU_MAX)) { cr = win_cpu_cr_read(win); if ((cr & MV_WIN_CPU_ENABLE_BIT) == 0) break; if ((cr & ((0xff << MV_WIN_CPU_ATTR_SHIFT) | (0x1f << MV_WIN_CPU_TARGET_SHIFT))) == ((attr << MV_WIN_CPU_ATTR_SHIFT) | (target << MV_WIN_CPU_TARGET_SHIFT))) break; win += i; } if ((win < 0) || (win >= MV_WIN_CPU_MAX) || ((remap != ~0) && (win_cpu_can_remap(win) == 0))) return (-1); br = base & 0xffff0000; win_cpu_br_write(win, br); if (win_cpu_can_remap(win)) { if (remap != ~0) { win_cpu_remap_l_write(win, remap & 0xffff0000); win_cpu_remap_h_write(win, 0); } else { /* * Remap function is not used for a given window * (capable of remapping) - set remap field with the * same value as base. */ win_cpu_remap_l_write(win, base & 0xffff0000); win_cpu_remap_h_write(win, 0); } } cr = ((size - 1) & 0xffff0000) | (attr << MV_WIN_CPU_ATTR_SHIFT) | (target << MV_WIN_CPU_TARGET_SHIFT) | MV_WIN_CPU_ENABLE_BIT; win_cpu_cr_write(win, cr); return (0); } static void decode_win_cpu_setup(void) { int i; /* Disable all CPU windows */ for (i = 0; i < MV_WIN_CPU_MAX; i++) { win_cpu_cr_write(i, 0); win_cpu_br_write(i, 0); if (win_cpu_can_remap(i)) { win_cpu_remap_l_write(i, 0); win_cpu_remap_h_write(i, 0); } } for (i = 0; i < cpu_wins_no; i++) if (cpu_wins[i].target > 0) decode_win_cpu_set(cpu_wins[i].target, cpu_wins[i].attr, cpu_wins[i].base, cpu_wins[i].size, cpu_wins[i].remap); } #endif #ifdef SOC_MV_ARMADAXP static int decode_win_sdram_fixup(void) { struct mem_region mr[FDT_MEM_REGIONS]; uint8_t window_valid[MV_WIN_DDR_MAX]; int mr_cnt, err, i, j; uint32_t valid_win_num = 0; /* Grab physical memory regions information from device tree. */ err = fdt_get_mem_regions(mr, &mr_cnt, NULL); if (err != 0) return (err); for (i = 0; i < MV_WIN_DDR_MAX; i++) window_valid[i] = 0; /* Try to match entries from device tree with settings from u-boot */ for (i = 0; i < mr_cnt; i++) { for (j = 0; j < MV_WIN_DDR_MAX; j++) { if (ddr_is_active(j) && (ddr_base(j) == mr[i].mr_start) && (ddr_size(j) == mr[i].mr_size)) { window_valid[j] = 1; valid_win_num++; } } } if (mr_cnt != valid_win_num) return (EINVAL); /* Destroy windows without corresponding device tree entry */ for (j = 0; j < MV_WIN_DDR_MAX; j++) { if (ddr_is_active(j) && (window_valid[j] != 1)) { printf("Disabling SDRAM decoding window: %d\n", j); ddr_disable(j); } } return (0); } #endif /* * Check if we're able to cover all active DDR banks. */ static int decode_win_can_cover_ddr(int max) { int i, c; c = 0; for (i = 0; i < MV_WIN_DDR_MAX; i++) if (ddr_is_active(i)) c++; if (c > max) { printf("Unable to cover all active DDR banks: " "%d, available windows: %d\n", c, max); return (0); } return (1); } /************************************************************************** * DDR windows routines **************************************************************************/ int ddr_is_active(int i) { if (ddr_sz_read(i) & 0x1) return (1); return (0); } void ddr_disable(int i) { ddr_sz_write(i, 0); ddr_br_write(i, 0); } uint32_t ddr_base(int i) { return (ddr_br_read(i) & 0xff000000); } uint32_t ddr_size(int i) { return ((ddr_sz_read(i) | 0x00ffffff) + 1); } uint32_t ddr_attr(int i) { uint32_t dev, rev; soc_id(&dev, &rev); if (dev == MV_DEV_88RC8180) return ((ddr_sz_read(i) & 0xf0) >> 4); if (dev == MV_DEV_88F6781) return (0); return (i == 0 ? 0xe : (i == 1 ? 0xd : (i == 2 ? 0xb : (i == 3 ? 0x7 : 0xff)))); } uint32_t ddr_target(int i) { uint32_t dev, rev; soc_id(&dev, &rev); if (dev == MV_DEV_88RC8180) { i = (ddr_sz_read(i) & 0xf0) >> 4; return (i == 0xe ? 0xc : (i == 0xd ? 0xd : (i == 0xb ? 0xe : (i == 0x7 ? 0xf : 0xc)))); } /* * On SOCs other than 88RC8180 Mbus unit ID for * DDR SDRAM controller is always 0x0. */ return (0); } /************************************************************************** * USB windows routines **************************************************************************/ static int decode_win_usb_valid(void) { return (decode_win_can_cover_ddr(MV_WIN_USB_MAX)); } static void decode_win_usb_dump(u_long base) { int i; if (pm_is_disabled(CPU_PM_CTRL_USB(usb_port - 1))) return; for (i = 0; i < MV_WIN_USB_MAX; i++) printf("USB window#%d: c 0x%08x, b 0x%08x\n", i, win_usb_cr_read(base, i), win_usb_br_read(base, i)); } /* * Set USB decode windows. */ static void decode_win_usb_setup(u_long base) { uint32_t br, cr; int i, j; if (pm_is_disabled(CPU_PM_CTRL_USB(usb_port))) return; usb_port++; for (i = 0; i < MV_WIN_USB_MAX; i++) { win_usb_cr_write(base, i, 0); win_usb_br_write(base, i, 0); } /* Only access to active DRAM banks is required */ for (i = 0; i < MV_WIN_DDR_MAX; i++) { if (ddr_is_active(i)) { br = ddr_base(i); /* * XXX for 6281 we should handle Mbus write * burst limit field in the ctrl reg */ cr = (((ddr_size(i) - 1) & 0xffff0000) | (ddr_attr(i) << 8) | (ddr_target(i) << 4) | 1); /* Set the first free USB window */ for (j = 0; j < MV_WIN_USB_MAX; j++) { if (win_usb_cr_read(base, j) & 0x1) continue; win_usb_br_write(base, j, br); win_usb_cr_write(base, j, cr); break; } } } } /************************************************************************** * USB3 windows routines **************************************************************************/ #ifdef SOC_MV_ARMADA38X static int decode_win_usb3_valid(void) { return (decode_win_can_cover_ddr(MV_WIN_USB3_MAX)); } static void decode_win_usb3_dump(u_long base) { int i; for (i = 0; i < MV_WIN_USB3_MAX; i++) printf("USB3.0 window#%d: c 0x%08x, b 0x%08x\n", i, win_usb3_cr_read(base, i), win_usb3_br_read(base, i)); } /* * Set USB3 decode windows */ static void decode_win_usb3_setup(u_long base) { uint32_t br, cr; int i, j; for (i = 0; i < MV_WIN_USB3_MAX; i++) { win_usb3_cr_write(base, i, 0); win_usb3_br_write(base, i, 0); } /* Only access to active DRAM banks is required */ for (i = 0; i < MV_WIN_DDR_MAX; i++) { if (ddr_is_active(i)) { br = ddr_base(i); cr = (((ddr_size(i) - 1) & (IO_WIN_SIZE_MASK << IO_WIN_SIZE_SHIFT)) | (ddr_attr(i) << IO_WIN_ATTR_SHIFT) | (ddr_target(i) << IO_WIN_TGT_SHIFT) | IO_WIN_ENA_MASK); /* Set the first free USB3.0 window */ for (j = 0; j < MV_WIN_USB3_MAX; j++) { if (win_usb3_cr_read(base, j) & IO_WIN_ENA_MASK) continue; win_usb3_br_write(base, j, br); win_usb3_cr_write(base, j, cr); break; } } } } #else /* * Provide dummy functions to satisfy the build * for SoCs not equipped with USB3 */ static int decode_win_usb3_valid(void) { return (1); } static void decode_win_usb3_setup(u_long base) { } static void decode_win_usb3_dump(u_long base) { } #endif /************************************************************************** * ETH windows routines **************************************************************************/ static int win_eth_can_remap(int i) { /* ETH encode windows 0-3 have remap capability */ if (i < 4) return (1); return (0); } static int eth_bare_read(uint32_t base, int i) { uint32_t v; v = win_eth_bare_read(base); v &= (1 << i); return (v >> i); } static void eth_bare_write(uint32_t base, int i, int val) { uint32_t v; v = win_eth_bare_read(base); v &= ~(1 << i); v |= (val << i); win_eth_bare_write(base, v); } static void eth_epap_write(uint32_t base, int i, int val) { uint32_t v; v = win_eth_epap_read(base); v &= ~(0x3 << (i * 2)); v |= (val << (i * 2)); win_eth_epap_write(base, v); } static void decode_win_eth_dump(u_long base) { int i; if (pm_is_disabled(CPU_PM_CTRL_GE(eth_port - 1))) return; for (i = 0; i < MV_WIN_ETH_MAX; i++) { printf("ETH window#%d: b 0x%08x, s 0x%08x", i, win_eth_br_read(base, i), win_eth_sz_read(base, i)); if (win_eth_can_remap(i)) printf(", ha 0x%08x", win_eth_har_read(base, i)); printf("\n"); } printf("ETH windows: bare 0x%08x, epap 0x%08x\n", win_eth_bare_read(base), win_eth_epap_read(base)); } #if defined(SOC_MV_LOKIPLUS) #define MV_WIN_ETH_DDR_TRGT(n) 0 #else #define MV_WIN_ETH_DDR_TRGT(n) ddr_target(n) #endif static void decode_win_eth_setup(u_long base) { uint32_t br, sz; int i, j; if (pm_is_disabled(CPU_PM_CTRL_GE(eth_port))) return; eth_port++; /* Disable, clear and revoke protection for all ETH windows */ for (i = 0; i < MV_WIN_ETH_MAX; i++) { eth_bare_write(base, i, 1); eth_epap_write(base, i, 0); win_eth_br_write(base, i, 0); win_eth_sz_write(base, i, 0); if (win_eth_can_remap(i)) win_eth_har_write(base, i, 0); } /* Only access to active DRAM banks is required */ for (i = 0; i < MV_WIN_DDR_MAX; i++) if (ddr_is_active(i)) { br = ddr_base(i) | (ddr_attr(i) << 8) | MV_WIN_ETH_DDR_TRGT(i); sz = ((ddr_size(i) - 1) & 0xffff0000); /* Set the first free ETH window */ for (j = 0; j < MV_WIN_ETH_MAX; j++) { if (eth_bare_read(base, j) == 0) continue; win_eth_br_write(base, j, br); win_eth_sz_write(base, j, sz); /* XXX remapping ETH windows not supported */ /* Set protection RW */ eth_epap_write(base, j, 0x3); /* Enable window */ eth_bare_write(base, j, 0); break; } } } static int decode_win_eth_valid(void) { return (decode_win_can_cover_ddr(MV_WIN_ETH_MAX)); } /************************************************************************** * PCIE windows routines **************************************************************************/ void decode_win_pcie_setup(u_long base) { uint32_t size = 0, ddrbase = ~0; uint32_t cr, br; int i, j; for (i = 0; i < MV_PCIE_BAR_MAX; i++) { pcie_bar_br_write(base, i, MV_PCIE_BAR_64BIT | MV_PCIE_BAR_PREFETCH_EN); if (i < 3) pcie_bar_brh_write(base, i, 0); if (i > 0) pcie_bar_cr_write(base, i, 0); } for (i = 0; i < MV_WIN_PCIE_MAX; i++) { win_pcie_cr_write(base, i, 0); win_pcie_br_write(base, i, 0); win_pcie_remap_write(base, i, 0); } /* On End-Point only set BAR size to 1MB regardless of DDR size */ if ((bus_space_read_4(fdtbus_bs_tag, base, MV_PCIE_CONTROL) & MV_PCIE_ROOT_CMPLX) == 0) { pcie_bar_cr_write(base, 1, 0xf0000 | 1); return; } for (i = 0; i < MV_WIN_DDR_MAX; i++) { if (ddr_is_active(i)) { /* Map DDR to BAR 1 */ cr = (ddr_size(i) - 1) & 0xffff0000; size += ddr_size(i) & 0xffff0000; cr |= (ddr_attr(i) << 8) | (ddr_target(i) << 4) | 1; br = ddr_base(i); if (br < ddrbase) ddrbase = br; /* Use the first available PCIE window */ for (j = 0; j < MV_WIN_PCIE_MAX; j++) { if (win_pcie_cr_read(base, j) != 0) continue; win_pcie_br_write(base, j, br); win_pcie_cr_write(base, j, cr); break; } } } /* * Upper 16 bits in BAR register is interpreted as BAR size - * (in 64 kB units) plus 64kB, so substract 0x10000 + * (in 64 kB units) plus 64kB, so subtract 0x10000 * form value passed to register to get correct value. */ size -= 0x10000; pcie_bar_cr_write(base, 1, size | 1); pcie_bar_br_write(base, 1, ddrbase | MV_PCIE_BAR_64BIT | MV_PCIE_BAR_PREFETCH_EN); pcie_bar_br_write(base, 0, fdt_immr_pa | MV_PCIE_BAR_64BIT | MV_PCIE_BAR_PREFETCH_EN); } static int decode_win_pcie_valid(void) { return (decode_win_can_cover_ddr(MV_WIN_PCIE_MAX)); } /************************************************************************** * IDMA windows routines **************************************************************************/ #if defined(SOC_MV_ORION) || defined(SOC_MV_DISCOVERY) static int idma_bare_read(u_long base, int i) { uint32_t v; v = win_idma_bare_read(base); v &= (1 << i); return (v >> i); } static void idma_bare_write(u_long base, int i, int val) { uint32_t v; v = win_idma_bare_read(base); v &= ~(1 << i); v |= (val << i); win_idma_bare_write(base, v); } /* * Sets channel protection 'val' for window 'w' on channel 'c' */ static void idma_cap_write(u_long base, int c, int w, int val) { uint32_t v; v = win_idma_cap_read(base, c); v &= ~(0x3 << (w * 2)); v |= (val << (w * 2)); win_idma_cap_write(base, c, v); } /* * Set protection 'val' on all channels for window 'w' */ static void idma_set_prot(u_long base, int w, int val) { int c; for (c = 0; c < MV_IDMA_CHAN_MAX; c++) idma_cap_write(base, c, w, val); } static int win_idma_can_remap(int i) { /* IDMA decode windows 0-3 have remap capability */ if (i < 4) return (1); return (0); } void decode_win_idma_setup(u_long base) { uint32_t br, sz; int i, j; if (pm_is_disabled(CPU_PM_CTRL_IDMA)) return; /* * Disable and clear all IDMA windows, revoke protection for all channels */ for (i = 0; i < MV_WIN_IDMA_MAX; i++) { idma_bare_write(base, i, 1); win_idma_br_write(base, i, 0); win_idma_sz_write(base, i, 0); if (win_idma_can_remap(i) == 1) win_idma_har_write(base, i, 0); } for (i = 0; i < MV_IDMA_CHAN_MAX; i++) win_idma_cap_write(base, i, 0); /* * Set up access to all active DRAM banks */ for (i = 0; i < MV_WIN_DDR_MAX; i++) if (ddr_is_active(i)) { br = ddr_base(i) | (ddr_attr(i) << 8) | ddr_target(i); sz = ((ddr_size(i) - 1) & 0xffff0000); /* Place DDR entries in non-remapped windows */ for (j = 0; j < MV_WIN_IDMA_MAX; j++) if (win_idma_can_remap(j) != 1 && idma_bare_read(base, j) == 1) { /* Configure window */ win_idma_br_write(base, j, br); win_idma_sz_write(base, j, sz); /* Set protection RW on all channels */ idma_set_prot(base, j, 0x3); /* Enable window */ idma_bare_write(base, j, 0); break; } } /* * Remaining targets -- from statically defined table */ for (i = 0; i < idma_wins_no; i++) if (idma_wins[i].target > 0) { br = (idma_wins[i].base & 0xffff0000) | (idma_wins[i].attr << 8) | idma_wins[i].target; sz = ((idma_wins[i].size - 1) & 0xffff0000); /* Set the first free IDMA window */ for (j = 0; j < MV_WIN_IDMA_MAX; j++) { if (idma_bare_read(base, j) == 0) continue; /* Configure window */ win_idma_br_write(base, j, br); win_idma_sz_write(base, j, sz); if (win_idma_can_remap(j) && idma_wins[j].remap >= 0) win_idma_har_write(base, j, idma_wins[j].remap); /* Set protection RW on all channels */ idma_set_prot(base, j, 0x3); /* Enable window */ idma_bare_write(base, j, 0); break; } } } int decode_win_idma_valid(void) { const struct decode_win *wintab; int c, i, j, rv; uint32_t b, e, s; if (idma_wins_no > MV_WIN_IDMA_MAX) { printf("IDMA windows: too many entries: %d\n", idma_wins_no); return (0); } for (i = 0, c = 0; i < MV_WIN_DDR_MAX; i++) if (ddr_is_active(i)) c++; if (idma_wins_no > (MV_WIN_IDMA_MAX - c)) { printf("IDMA windows: too many entries: %d, available: %d\n", idma_wins_no, MV_WIN_IDMA_MAX - c); return (0); } wintab = idma_wins; rv = 1; for (i = 0; i < idma_wins_no; i++, wintab++) { if (wintab->target == 0) { printf("IDMA window#%d: DDR target window is not " "supposed to be reprogrammed!\n", i); rv = 0; } if (wintab->remap >= 0 && win_cpu_can_remap(i) != 1) { printf("IDMA window#%d: not capable of remapping, but " "val 0x%08x defined\n", i, wintab->remap); rv = 0; } s = wintab->size; b = wintab->base; e = b + s - 1; if (s > (0xFFFFFFFF - b + 1)) { /* XXX this boundary check should account for 64bit and * remapping.. */ printf("IDMA window#%d: no space for size 0x%08x at " "0x%08x\n", i, s, b); rv = 0; continue; } j = decode_win_overlap(i, idma_wins_no, &idma_wins[0]); if (j >= 0) { printf("IDMA window#%d: (0x%08x - 0x%08x) overlaps " "with #%d (0x%08x - 0x%08x)\n", i, b, e, j, idma_wins[j].base, idma_wins[j].base + idma_wins[j].size - 1); rv = 0; } } return (rv); } void decode_win_idma_dump(u_long base) { int i; if (pm_is_disabled(CPU_PM_CTRL_IDMA)) return; for (i = 0; i < MV_WIN_IDMA_MAX; i++) { printf("IDMA window#%d: b 0x%08x, s 0x%08x", i, win_idma_br_read(base, i), win_idma_sz_read(base, i)); if (win_idma_can_remap(i)) printf(", ha 0x%08x", win_idma_har_read(base, i)); printf("\n"); } for (i = 0; i < MV_IDMA_CHAN_MAX; i++) printf("IDMA channel#%d: ap 0x%08x\n", i, win_idma_cap_read(base, i)); printf("IDMA windows: bare 0x%08x\n", win_idma_bare_read(base)); } #else /* Provide dummy functions to satisfy the build for SoCs not equipped with IDMA */ int decode_win_idma_valid(void) { return (1); } void decode_win_idma_setup(u_long base) { } void decode_win_idma_dump(u_long base) { } #endif /************************************************************************** * XOR windows routines **************************************************************************/ #if defined(SOC_MV_KIRKWOOD) || defined(SOC_MV_DISCOVERY) static int xor_ctrl_read(u_long base, int i, int c, int e) { uint32_t v; v = win_xor_ctrl_read(base, c, e); v &= (1 << i); return (v >> i); } static void xor_ctrl_write(u_long base, int i, int c, int e, int val) { uint32_t v; v = win_xor_ctrl_read(base, c, e); v &= ~(1 << i); v |= (val << i); win_xor_ctrl_write(base, c, e, v); } /* * Set channel protection 'val' for window 'w' on channel 'c' */ static void xor_chan_write(u_long base, int c, int e, int w, int val) { uint32_t v; v = win_xor_ctrl_read(base, c, e); v &= ~(0x3 << (w * 2 + 16)); v |= (val << (w * 2 + 16)); win_xor_ctrl_write(base, c, e, v); } /* * Set protection 'val' on all channels for window 'w' on engine 'e' */ static void xor_set_prot(u_long base, int w, int e, int val) { int c; for (c = 0; c < MV_XOR_CHAN_MAX; c++) xor_chan_write(base, c, e, w, val); } static int win_xor_can_remap(int i) { /* XOR decode windows 0-3 have remap capability */ if (i < 4) return (1); return (0); } static int xor_max_eng(void) { uint32_t dev, rev; soc_id(&dev, &rev); switch (dev) { case MV_DEV_88F6281: case MV_DEV_88F6282: case MV_DEV_MV78130: case MV_DEV_MV78160: case MV_DEV_MV78230: case MV_DEV_MV78260: case MV_DEV_MV78460: return (2); case MV_DEV_MV78100: case MV_DEV_MV78100_Z0: return (1); default: return (0); } } static void xor_active_dram(u_long base, int c, int e, int *window) { uint32_t br, sz; int i, m, w; /* * Set up access to all active DRAM banks */ m = xor_max_eng(); for (i = 0; i < m; i++) if (ddr_is_active(i)) { br = ddr_base(i) | (ddr_attr(i) << 8) | ddr_target(i); sz = ((ddr_size(i) - 1) & 0xffff0000); /* Place DDR entries in non-remapped windows */ for (w = 0; w < MV_WIN_XOR_MAX; w++) if (win_xor_can_remap(w) != 1 && (xor_ctrl_read(base, w, c, e) == 0) && w > *window) { /* Configure window */ win_xor_br_write(base, w, e, br); win_xor_sz_write(base, w, e, sz); /* Set protection RW on all channels */ xor_set_prot(base, w, e, 0x3); /* Enable window */ xor_ctrl_write(base, w, c, e, 1); (*window)++; break; } } } void decode_win_xor_setup(u_long base) { uint32_t br, sz; int i, j, z, e = 1, m, window; if (pm_is_disabled(CPU_PM_CTRL_XOR)) return; /* * Disable and clear all XOR windows, revoke protection for all * channels */ m = xor_max_eng(); for (j = 0; j < m; j++, e--) { /* Number of non-remaped windows */ window = MV_XOR_NON_REMAP - 1; for (i = 0; i < MV_WIN_XOR_MAX; i++) { win_xor_br_write(base, i, e, 0); win_xor_sz_write(base, i, e, 0); } if (win_xor_can_remap(i) == 1) win_xor_har_write(base, i, e, 0); for (i = 0; i < MV_XOR_CHAN_MAX; i++) { win_xor_ctrl_write(base, i, e, 0); xor_active_dram(base, i, e, &window); } /* * Remaining targets -- from a statically defined table */ for (i = 0; i < xor_wins_no; i++) if (xor_wins[i].target > 0) { br = (xor_wins[i].base & 0xffff0000) | (xor_wins[i].attr << 8) | xor_wins[i].target; sz = ((xor_wins[i].size - 1) & 0xffff0000); /* Set the first free XOR window */ for (z = 0; z < MV_WIN_XOR_MAX; z++) { if (xor_ctrl_read(base, z, 0, e) && xor_ctrl_read(base, z, 1, e)) continue; /* Configure window */ win_xor_br_write(base, z, e, br); win_xor_sz_write(base, z, e, sz); if (win_xor_can_remap(z) && xor_wins[z].remap >= 0) win_xor_har_write(base, z, e, xor_wins[z].remap); /* Set protection RW on all channels */ xor_set_prot(base, z, e, 0x3); /* Enable window */ xor_ctrl_write(base, z, 0, e, 1); xor_ctrl_write(base, z, 1, e, 1); break; } } } } int decode_win_xor_valid(void) { const struct decode_win *wintab; int c, i, j, rv; uint32_t b, e, s; if (xor_wins_no > MV_WIN_XOR_MAX) { printf("XOR windows: too many entries: %d\n", xor_wins_no); return (0); } for (i = 0, c = 0; i < MV_WIN_DDR_MAX; i++) if (ddr_is_active(i)) c++; if (xor_wins_no > (MV_WIN_XOR_MAX - c)) { printf("XOR windows: too many entries: %d, available: %d\n", xor_wins_no, MV_WIN_IDMA_MAX - c); return (0); } wintab = xor_wins; rv = 1; for (i = 0; i < xor_wins_no; i++, wintab++) { if (wintab->target == 0) { printf("XOR window#%d: DDR target window is not " "supposed to be reprogrammed!\n", i); rv = 0; } if (wintab->remap >= 0 && win_cpu_can_remap(i) != 1) { printf("XOR window#%d: not capable of remapping, but " "val 0x%08x defined\n", i, wintab->remap); rv = 0; } s = wintab->size; b = wintab->base; e = b + s - 1; if (s > (0xFFFFFFFF - b + 1)) { /* * XXX this boundary check should account for 64bit * and remapping.. */ printf("XOR window#%d: no space for size 0x%08x at " "0x%08x\n", i, s, b); rv = 0; continue; } j = decode_win_overlap(i, xor_wins_no, &xor_wins[0]); if (j >= 0) { printf("XOR window#%d: (0x%08x - 0x%08x) overlaps " "with #%d (0x%08x - 0x%08x)\n", i, b, e, j, xor_wins[j].base, xor_wins[j].base + xor_wins[j].size - 1); rv = 0; } } return (rv); } void decode_win_xor_dump(u_long base) { int i, j; int e = 1; if (pm_is_disabled(CPU_PM_CTRL_XOR)) return; for (j = 0; j < xor_max_eng(); j++, e--) { for (i = 0; i < MV_WIN_XOR_MAX; i++) { printf("XOR window#%d: b 0x%08x, s 0x%08x", i, win_xor_br_read(base, i, e), win_xor_sz_read(base, i, e)); if (win_xor_can_remap(i)) printf(", ha 0x%08x", win_xor_har_read(base, i, e)); printf("\n"); } for (i = 0; i < MV_XOR_CHAN_MAX; i++) printf("XOR control#%d: 0x%08x\n", i, win_xor_ctrl_read(base, i, e)); } } #else /* Provide dummy functions to satisfy the build for SoCs not equipped with XOR */ static int decode_win_xor_valid(void) { return (1); } static void decode_win_xor_setup(u_long base) { } static void decode_win_xor_dump(u_long base) { } #endif /************************************************************************** * SATA windows routines **************************************************************************/ static void decode_win_sata_setup(u_long base) { uint32_t cr, br; int i, j; if (pm_is_disabled(CPU_PM_CTRL_SATA)) return; for (i = 0; i < MV_WIN_SATA_MAX; i++) { win_sata_cr_write(base, i, 0); win_sata_br_write(base, i, 0); } for (i = 0; i < MV_WIN_DDR_MAX; i++) if (ddr_is_active(i)) { cr = ((ddr_size(i) - 1) & 0xffff0000) | (ddr_attr(i) << 8) | (ddr_target(i) << 4) | 1; br = ddr_base(i); /* Use the first available SATA window */ for (j = 0; j < MV_WIN_SATA_MAX; j++) { if ((win_sata_cr_read(base, j) & 1) != 0) continue; win_sata_br_write(base, j, br); win_sata_cr_write(base, j, cr); break; } } } static int decode_win_sata_valid(void) { uint32_t dev, rev; soc_id(&dev, &rev); if (dev == MV_DEV_88F5281) return (1); return (decode_win_can_cover_ddr(MV_WIN_SATA_MAX)); } /************************************************************************** * FDT parsing routines. **************************************************************************/ static int fdt_get_ranges(const char *nodename, void *buf, int size, int *tuples, int *tuplesize) { phandle_t node; pcell_t addr_cells, par_addr_cells, size_cells; int len, tuple_size, tuples_count; node = OF_finddevice(nodename); if (node == -1) return (EINVAL); if ((fdt_addrsize_cells(node, &addr_cells, &size_cells)) != 0) return (ENXIO); par_addr_cells = fdt_parent_addr_cells(node); if (par_addr_cells > 2) return (ERANGE); tuple_size = sizeof(pcell_t) * (addr_cells + par_addr_cells + size_cells); /* Note the OF_getprop_alloc() cannot be used at this early stage. */ len = OF_getprop(node, "ranges", buf, size); /* * XXX this does not handle the empty 'ranges;' case, which is * legitimate and should be allowed. */ tuples_count = len / tuple_size; if (tuples_count <= 0) return (ERANGE); if (par_addr_cells > 2 || addr_cells > 2 || size_cells > 2) return (ERANGE); *tuples = tuples_count; *tuplesize = tuple_size; return (0); } static int win_cpu_from_dt(void) { pcell_t ranges[48]; phandle_t node; int i, entry_size, err, t, tuple_size, tuples; u_long sram_base, sram_size; t = 0; /* Retrieve 'ranges' property of '/localbus' node. */ if ((err = fdt_get_ranges("/localbus", ranges, sizeof(ranges), &tuples, &tuple_size)) == 0) { /* * Fill CPU decode windows table. */ bzero((void *)&cpu_win_tbl, sizeof(cpu_win_tbl)); entry_size = tuple_size / sizeof(pcell_t); cpu_wins_no = tuples; for (i = 0, t = 0; t < tuples; i += entry_size, t++) { cpu_win_tbl[t].target = 1; cpu_win_tbl[t].attr = fdt32_to_cpu(ranges[i + 1]); cpu_win_tbl[t].base = fdt32_to_cpu(ranges[i + 2]); cpu_win_tbl[t].size = fdt32_to_cpu(ranges[i + 3]); cpu_win_tbl[t].remap = ~0; debugf("target = 0x%0x attr = 0x%0x base = 0x%0x " "size = 0x%0x remap = 0x%0x\n", cpu_win_tbl[t].target, cpu_win_tbl[t].attr, cpu_win_tbl[t].base, cpu_win_tbl[t].size, cpu_win_tbl[t].remap); } } /* * Retrieve CESA SRAM data. */ if ((node = OF_finddevice("sram")) != -1) if (fdt_is_compatible(node, "mrvl,cesa-sram")) goto moveon; if ((node = OF_finddevice("/")) == 0) return (ENXIO); if ((node = fdt_find_compatible(node, "mrvl,cesa-sram", 0)) == 0) /* SRAM block is not always present. */ return (0); moveon: sram_base = sram_size = 0; if (fdt_regsize(node, &sram_base, &sram_size) != 0) return (EINVAL); cpu_win_tbl[t].target = MV_WIN_CESA_TARGET; cpu_win_tbl[t].attr = MV_WIN_CESA_ATTR(1); cpu_win_tbl[t].base = sram_base; cpu_win_tbl[t].size = sram_size; cpu_win_tbl[t].remap = ~0; cpu_wins_no++; debugf("sram: base = 0x%0lx size = 0x%0lx\n", sram_base, sram_size); return (0); } static int fdt_win_setup(void) { phandle_t node, child; struct soc_node_spec *soc_node; u_long size, base; int err, i; node = OF_finddevice("/"); if (node == -1) panic("fdt_win_setup: no root node"); /* * Traverse through all children of root and simple-bus nodes. * For each found device retrieve decode windows data (if applicable). */ child = OF_child(node); while (child != 0) { for (i = 0; soc_nodes[i].compat != NULL; i++) { soc_node = &soc_nodes[i]; if (!fdt_is_compatible(child, soc_node->compat)) continue; err = fdt_regsize(child, &base, &size); if (err != 0) return (err); base = (base & 0x000fffff) | fdt_immr_va; if (soc_node->decode_handler != NULL) soc_node->decode_handler(base); else return (ENXIO); if (MV_DUMP_WIN && (soc_node->dump_handler != NULL)) soc_node->dump_handler(base); } /* * Once done with root-level children let's move down to * simple-bus and its children. */ child = OF_peer(child); if ((child == 0) && (node == OF_finddevice("/"))) { node = fdt_find_compatible(node, "simple-bus", 0); if (node == 0) return (ENXIO); child = OF_child(node); } /* * Next, move one more level down to internal-regs node (if * it is present) and its children. This node also have * "simple-bus" compatible. */ if ((child == 0) && (node == OF_finddevice("simple-bus"))) { node = fdt_find_compatible(node, "simple-bus", 0); if (node == 0) return (0); child = OF_child(node); } } return (0); } static void fdt_fixup_busfreq(phandle_t root) { phandle_t sb; pcell_t freq; freq = cpu_to_fdt32(get_tclk()); /* * Fix bus speed in cpu node */ if ((sb = OF_finddevice("cpu")) != 0) if (fdt_is_compatible_strict(sb, "ARM,88VS584")) OF_setprop(sb, "bus-frequency", (void *)&freq, sizeof(freq)); /* * This fixup sets the simple-bus bus-frequency property. */ if ((sb = fdt_find_compatible(root, "simple-bus", 1)) != 0) OF_setprop(sb, "bus-frequency", (void *)&freq, sizeof(freq)); } static void fdt_fixup_ranges(phandle_t root) { phandle_t node; pcell_t par_addr_cells, addr_cells, size_cells; pcell_t ranges[3], reg[2], *rangesptr; int len, tuple_size, tuples_count; uint32_t base; /* Fix-up SoC ranges according to real fdt_immr_pa */ if ((node = fdt_find_compatible(root, "simple-bus", 1)) != 0) { if (fdt_addrsize_cells(node, &addr_cells, &size_cells) == 0 && (par_addr_cells = fdt_parent_addr_cells(node) <= 2)) { tuple_size = sizeof(pcell_t) * (par_addr_cells + addr_cells + size_cells); len = OF_getprop(node, "ranges", ranges, sizeof(ranges)); tuples_count = len / tuple_size; /* Unexpected settings are not supported */ if (tuples_count != 1) goto fixup_failed; rangesptr = &ranges[0]; rangesptr += par_addr_cells; base = fdt_data_get((void *)rangesptr, addr_cells); *rangesptr = cpu_to_fdt32(fdt_immr_pa); if (OF_setprop(node, "ranges", (void *)&ranges[0], sizeof(ranges)) < 0) goto fixup_failed; } } /* Fix-up PCIe reg according to real PCIe registers' PA */ if ((node = fdt_find_compatible(root, "mrvl,pcie", 1)) != 0) { if (fdt_addrsize_cells(OF_parent(node), &par_addr_cells, &size_cells) == 0) { tuple_size = sizeof(pcell_t) * (par_addr_cells + size_cells); len = OF_getprop(node, "reg", reg, sizeof(reg)); tuples_count = len / tuple_size; /* Unexpected settings are not supported */ if (tuples_count != 1) goto fixup_failed; base = fdt_data_get((void *)®[0], par_addr_cells); base &= ~0xFF000000; base |= fdt_immr_pa; reg[0] = cpu_to_fdt32(base); if (OF_setprop(node, "reg", (void *)®[0], sizeof(reg)) < 0) goto fixup_failed; } } /* Fix-up succeeded. May return and continue */ return; fixup_failed: while (1) { /* * In case of any error while fixing ranges just hang. * 1. No message can be displayed yet since console * is not initialized. * 2. Going further will cause failure on bus_space_map() * relying on the wrong ranges or data abort when * accessing PCIe registers. */ } } struct fdt_fixup_entry fdt_fixup_table[] = { { "mrvl,DB-88F6281", &fdt_fixup_busfreq }, { "mrvl,DB-78460", &fdt_fixup_busfreq }, { "mrvl,DB-78460", &fdt_fixup_ranges }, { NULL, NULL } }; #ifndef INTRNG static int fdt_pic_decode_ic(phandle_t node, pcell_t *intr, int *interrupt, int *trig, int *pol) { if (!fdt_is_compatible(node, "mrvl,pic") && !fdt_is_compatible(node, "mrvl,mpic")) return (ENXIO); *interrupt = fdt32_to_cpu(intr[0]); *trig = INTR_TRIGGER_CONFORM; *pol = INTR_POLARITY_CONFORM; return (0); } fdt_pic_decode_t fdt_pic_table[] = { #ifdef SOC_MV_ARMADA38X &gic_decode_fdt, #endif &fdt_pic_decode_ic, NULL }; #endif uint64_t get_sar_value(void) { uint32_t sar_low, sar_high; #if defined(SOC_MV_ARMADAXP) sar_high = bus_space_read_4(fdtbus_bs_tag, MV_MISC_BASE, SAMPLE_AT_RESET_HI); sar_low = bus_space_read_4(fdtbus_bs_tag, MV_MISC_BASE, SAMPLE_AT_RESET_LO); #elif defined(SOC_MV_ARMADA38X) sar_high = 0; sar_low = bus_space_read_4(fdtbus_bs_tag, MV_MISC_BASE, SAMPLE_AT_RESET); #else /* * TODO: Add getting proper values for other SoC configurations */ sar_high = 0; sar_low = 0; #endif return (((uint64_t)sar_high << 32) | sar_low); } Index: user/ngie/release-pkg-fix-tests/sys/arm/qemu/virt_machdep.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/qemu/virt_machdep.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/qemu/virt_machdep.c (revision 299091) @@ -1,99 +1,116 @@ /*- * Copyright (c) 2015 Andrew Turner * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ #include "opt_platform.h" #include __FBSDID("$FreeBSD$"); #define _ARM32_BUS_DMA_PRIVATE #include #include #include #include #include #include #include #include #include "platform_if.h" struct arm32_dma_range * bus_dma_get_range(void) { return (NULL); } int bus_dma_get_range_nb(void) { return (0); } void cpu_reset(void) { while (1); } static vm_offset_t virt_lastaddr(platform_t plat) { return (devmap_lastaddr()); } /* * Set up static device mappings. */ static int virt_devmap_init(platform_t plat) { devmap_add_entry(0x09000000, 0x100000); /* Uart */ return (0); } static platform_method_t virt_methods[] = { PLATFORMMETHOD(platform_devmap_init, virt_devmap_init), PLATFORMMETHOD(platform_lastaddr, virt_lastaddr), #ifdef SMP PLATFORMMETHOD(platform_mp_start_ap, virt_mp_start_ap), PLATFORMMETHOD(platform_mp_setmaxid, virt_mp_setmaxid), #endif PLATFORMMETHOD_END, }; FDT_PLATFORM_DEF(virt, "virt", 0, "linux,dummy-virt", 1); + +static int +gem5_devmap_init(platform_t plat) +{ + + devmap_add_entry(0x1c090000, 0x100000); /* Uart */ + return (0); +} + +static platform_method_t gem5_methods[] = { + PLATFORMMETHOD(platform_devmap_init, gem5_devmap_init), + PLATFORMMETHOD(platform_lastaddr, virt_lastaddr), + + PLATFORMMETHOD_END, +}; + +FDT_PLATFORM_DEF(gem5, "gem5", 0, "arm,vexpress", 1); Index: user/ngie/release-pkg-fix-tests/sys/arm/samsung/exynos/exynos5_usb_phy.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/samsung/exynos/exynos5_usb_phy.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/samsung/exynos/exynos5_usb_phy.c (revision 299091) @@ -1,271 +1,271 @@ /*- * Copyright (c) 2014 Ruslan Bukin * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * DWC3 USB 3.0 DRD (dual role device) PHY */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "gpio_if.h" #define USB_DRD_LINKSYSTEM 0x04 #define LINKSYSTEM_FLADJ_MASK (0x3f << 1) #define LINKSYSTEM_FLADJ(x) ((x) << 1) #define LINKSYSTEM_XHCI_VERSION_CTRL (1 << 27) #define USB_DRD_PHYUTMI 0x08 #define PHYUTMI_OTGDISABLE (1 << 6) #define PHYUTMI_FORCESUSPEND (1 << 1) #define PHYUTMI_FORCESLEEP (1 << 0) #define USB_DRD_PHYPIPE 0x0c #define USB_DRD_PHYCLKRST 0x10 #define PHYCLKRST_PORTRESET (1 << 1) #define PHYCLKRST_COMMONONN (1 << 0) #define PHYCLKRST_EN_UTMISUSPEND (1 << 31) #define PHYCLKRST_SSC_REFCLKSEL_MASK (0xff << 23) #define PHYCLKRST_SSC_REFCLKSEL(x) ((x) << 23) #define PHYCLKRST_SSC_RANGE_MASK (0x03 << 21) #define PHYCLKRST_SSC_RANGE(x) ((x) << 21) #define PHYCLKRST_SSC_EN (1 << 20) #define PHYCLKRST_REF_SSP_EN (1 << 19) #define PHYCLKRST_REF_CLKDIV2 (1 << 18) #define PHYCLKRST_MPLL_MLTPR_MASK (0x7f << 11) #define PHYCLKRST_MPLL_MLTPR_100MHZ (0x19 << 11) #define PHYCLKRST_MPLL_MLTPR_50M (0x32 << 11) #define PHYCLKRST_MPLL_MLTPR_24MHZ (0x68 << 11) #define PHYCLKRST_MPLL_MLTPR_20MHZ (0x7d << 11) #define PHYCLKRST_MPLL_MLTPR_19200KHZ (0x02 << 11) #define PHYCLKRST_FSEL_UTMI_MASK (0x7 << 5) #define PHYCLKRST_FSEL_PIPE_MASK (0x7 << 8) #define PHYCLKRST_FSEL(x) ((x) << 5) #define PHYCLKRST_FSEL_9MHZ6 0x0 #define PHYCLKRST_FSEL_10MHZ 0x1 #define PHYCLKRST_FSEL_12MHZ 0x2 #define PHYCLKRST_FSEL_19MHZ2 0x3 #define PHYCLKRST_FSEL_20MHZ 0x4 #define PHYCLKRST_FSEL_24MHZ 0x5 #define PHYCLKRST_FSEL_50MHZ 0x7 #define PHYCLKRST_RETENABLEN (1 << 4) #define PHYCLKRST_REFCLKSEL_MASK (0x03 << 2) #define PHYCLKRST_REFCLKSEL_PAD_REFCLK (0x2 << 2) #define PHYCLKRST_REFCLKSEL_EXT_REFCLK (0x3 << 2) #define USB_DRD_PHYREG0 0x14 #define USB_DRD_PHYREG1 0x18 #define USB_DRD_PHYPARAM0 0x1c #define PHYPARAM0_REF_USE_PAD (1 << 31) #define PHYPARAM0_REF_LOSLEVEL_MASK (0x1f << 26) #define PHYPARAM0_REF_LOSLEVEL (0x9 << 26) #define USB_DRD_PHYPARAM1 0x20 #define PHYPARAM1_PCS_TXDEEMPH_MASK (0x1f << 0) #define PHYPARAM1_PCS_TXDEEMPH (0x1c) #define USB_DRD_PHYTERM 0x24 #define USB_DRD_PHYTEST 0x28 #define PHYTEST_POWERDOWN_SSP (1 << 3) #define PHYTEST_POWERDOWN_HSP (1 << 2) #define USB_DRD_PHYADP 0x2c #define USB_DRD_PHYUTMICLKSEL 0x30 #define PHYUTMICLKSEL_UTMI_CLKSEL (1 << 2) #define USB_DRD_PHYRESUME 0x34 #define USB_DRD_LINKPORT 0x44 struct usb_phy_softc { struct resource *res[1]; bus_space_tag_t bst; bus_space_handle_t bsh; device_t dev; }; static struct resource_spec usb_phy_spec[] = { { SYS_RES_MEMORY, 0, RF_ACTIVE }, { -1, 0 } }; static int usb_phy_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, "samsung,exynos5420-usbdrd-phy")) return (ENXIO); device_set_desc(dev, "Samsung Exynos 5 USB PHY"); return (BUS_PROBE_DEFAULT); } static int vbus_on(struct usb_phy_softc *sc) { pcell_t dts_value[3]; device_t gpio_dev; phandle_t node; pcell_t pin; int len; if ((node = ofw_bus_get_node(sc->dev)) == -1) return (-1); /* Power pin */ if ((len = OF_getproplen(node, "vbus-supply")) <= 0) return (-1); OF_getencprop(node, "vbus-supply", dts_value, len); pin = dts_value[0]; gpio_dev = devclass_get_device(devclass_find("gpio"), 0); if (gpio_dev == NULL) { - device_printf(sc->dev, "cant find gpio_dev\n"); + device_printf(sc->dev, "can't find gpio_dev\n"); return (1); } GPIO_PIN_SETFLAGS(gpio_dev, pin, GPIO_PIN_OUTPUT); GPIO_PIN_SET(gpio_dev, pin, GPIO_PIN_HIGH); return (0); } static int usb3_phy_init(struct usb_phy_softc *sc) { int reg; /* Reset USB 3.0 PHY */ WRITE4(sc, USB_DRD_PHYREG0, 0); reg = READ4(sc, USB_DRD_PHYPARAM0); /* PHY CLK src */ reg &= ~(PHYPARAM0_REF_USE_PAD); reg &= ~(PHYPARAM0_REF_LOSLEVEL_MASK); reg |= (PHYPARAM0_REF_LOSLEVEL); WRITE4(sc, USB_DRD_PHYPARAM0, reg); WRITE4(sc, USB_DRD_PHYRESUME, 0); reg = (LINKSYSTEM_XHCI_VERSION_CTRL | LINKSYSTEM_FLADJ(0x20)); WRITE4(sc, USB_DRD_LINKSYSTEM, reg); reg = READ4(sc, USB_DRD_PHYPARAM1); reg &= ~(PHYPARAM1_PCS_TXDEEMPH_MASK); reg |= (PHYPARAM1_PCS_TXDEEMPH); WRITE4(sc, USB_DRD_PHYPARAM1, reg); reg = READ4(sc, USB_DRD_PHYUTMICLKSEL); reg |= (PHYUTMICLKSEL_UTMI_CLKSEL); WRITE4(sc, USB_DRD_PHYUTMICLKSEL, reg); reg = READ4(sc, USB_DRD_PHYTEST); reg &= ~(PHYTEST_POWERDOWN_HSP); reg &= ~(PHYTEST_POWERDOWN_SSP); WRITE4(sc, USB_DRD_PHYTEST, reg); WRITE4(sc, USB_DRD_PHYUTMI, PHYUTMI_OTGDISABLE); /* Clock */ reg = (PHYCLKRST_REFCLKSEL_EXT_REFCLK); reg |= (PHYCLKRST_FSEL(PHYCLKRST_FSEL_24MHZ)); reg |= (PHYCLKRST_MPLL_MLTPR_24MHZ); reg |= (PHYCLKRST_SSC_REFCLKSEL(0x88)); reg |= (PHYCLKRST_RETENABLEN | PHYCLKRST_REF_SSP_EN | /* Super speed */ PHYCLKRST_SSC_EN | /* Spread spectrum */ PHYCLKRST_COMMONONN | PHYCLKRST_PORTRESET); WRITE4(sc, USB_DRD_PHYCLKRST, reg); DELAY(50000); reg &= ~PHYCLKRST_PORTRESET; WRITE4(sc, USB_DRD_PHYCLKRST, reg); return (0); } static int usb_phy_attach(device_t dev) { struct usb_phy_softc *sc; sc = device_get_softc(dev); sc->dev = dev; if (bus_alloc_resources(dev, usb_phy_spec, sc->res)) { device_printf(dev, "could not allocate resources\n"); return (ENXIO); } /* Memory interface */ sc->bst = rman_get_bustag(sc->res[0]); sc->bsh = rman_get_bushandle(sc->res[0]); vbus_on(sc); usbdrd_phy_power_on(); DELAY(100); usb3_phy_init(sc); return (0); } static device_method_t usb_phy_methods[] = { DEVMETHOD(device_probe, usb_phy_probe), DEVMETHOD(device_attach, usb_phy_attach), { 0, 0 } }; static driver_t usb_phy_driver = { "usb_phy", usb_phy_methods, sizeof(struct usb_phy_softc), }; static devclass_t usb_phy_devclass; DRIVER_MODULE(usb_phy, simplebus, usb_phy_driver, usb_phy_devclass, 0, 0); Index: user/ngie/release-pkg-fix-tests/sys/arm/ti/omap4/omap4_prcm_clks.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/ti/omap4/omap4_prcm_clks.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/ti/omap4/omap4_prcm_clks.c (revision 299091) @@ -1,1496 +1,1496 @@ /*- * Copyright (c) 2011 * Ben Gray . * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY BEN GRAY ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL BEN GRAY BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * This file defines the clock configuration for the OMAP4xxx series of * devices. * * How This is Suppose to Work * =========================== * - There is a top level omap_prcm module that defines all OMAP SoC drivers * should use to enable/disable the system clocks regardless of the version * of OMAP device they are running on. This top level PRCM module is just * a thin shim to chip specific functions that perform the donkey work of * configuring the clock - this file is the 'donkey' for OMAP44xx devices. * * - The key bit in this file is the omap_clk_devmap array, it's * used by the omap_prcm driver to determine what clocks are valid and which * functions to call to manipulate them. * * - In essence you just need to define some callbacks for each of the * clocks and then you're done. * * - The other thing that is worth noting is that when the omap_prcm device * is registered you typically pass in some memory ranges which are the * SYS_MEMORY resources. These resources are in turn allocated using * bus_allocate_resources(...) and the resource handles are passed to all * individual clock callback handlers. * * * * OMAP4 devices are different from the previous OMAP3 devices in that there * is no longer a separate functional and interface clock for each module, * instead there is typically an interface clock that spans many modules. */ #define FREQ_96MHZ 96000000 #define FREQ_64MHZ 64000000 #define FREQ_48MHZ 48000000 #define FREQ_32KHZ 32000 #define PRM_INSTANCE 1 #define CM1_INSTANCE 2 #define CM2_INSTANCE 3 /** * Address offsets from the PRM memory region to the top level clock control * registers. */ #define CKGEN_PRM_OFFSET 0x00000100UL #define MPU_PRM_OFFSET 0x00000300UL #define DSP_PRM_OFFSET 0x00000400UL #define ABE_PRM_OFFSET 0x00000500UL #define ALWAYS_ON_PRM_OFFSET 0x00000600UL #define CORE_PRM_OFFSET 0x00000700UL #define IVAHD_PRM_OFFSET 0x00000F00UL #define CAM_PRM_OFFSET 0x00001000UL #define DSS_PRM_OFFSET 0x00001100UL #define SGX_PRM_OFFSET 0x00001200UL #define L3INIT_PRM_OFFSET 0x00001300UL #define L4PER_PRM_OFFSET 0x00001400UL #define WKUP_PRM_OFFSET 0x00001700UL #define WKUP_CM_OFFSET 0x00001800UL #define EMU_PRM_OFFSET 0x00001900UL #define EMU_CM_OFFSET 0x00001A00UL #define DEVICE_PRM_OFFSET 0x00001B00UL #define INSTR_PRM_OFFSET 0x00001F00UL #define CM_ABE_DSS_SYS_CLKSEL_OFFSET (CKGEN_PRM_OFFSET + 0x0000UL) #define CM_L4_WKUP_CLKSELL_OFFSET (CKGEN_PRM_OFFSET + 0x0008UL) #define CM_ABE_PLL_REF_CLKSEL_OFFSET (CKGEN_PRM_OFFSET + 0x000CUL) #define CM_SYS_CLKSEL_OFFSET (CKGEN_PRM_OFFSET + 0x0010UL) /** * Address offsets from the CM1 memory region to the top level clock control * registers. */ #define CKGEN_CM1_OFFSET 0x00000100UL #define MPU_CM1_OFFSET 0x00000300UL #define DSP_CM1_OFFSET 0x00000400UL #define ABE_CM1_OFFSET 0x00000500UL #define RESTORE_CM1_OFFSET 0x00000E00UL #define INSTR_CM1_OFFSET 0x00000F00UL #define CM_CLKSEL_DPLL_MPU (CKGEN_CM1_OFFSET + 0x006CUL) /** * Address offsets from the CM2 memory region to the top level clock control * registers. */ #define INTRCONN_SOCKET_CM2_OFFSET 0x00000000UL #define CKGEN_CM2_OFFSET 0x00000100UL #define ALWAYS_ON_CM2_OFFSET 0x00000600UL #define CORE_CM2_OFFSET 0x00000700UL #define IVAHD_CM2_OFFSET 0x00000F00UL #define CAM_CM2_OFFSET 0x00001000UL #define DSS_CM2_OFFSET 0x00001100UL #define SGX_CM2_OFFSET 0x00001200UL #define L3INIT_CM2_OFFSET 0x00001300UL #define L4PER_CM2_OFFSET 0x00001400UL #define RESTORE_CM2_OFFSET 0x00001E00UL #define INSTR_CM2_OFFSET 0x00001F00UL #define CLKCTRL_MODULEMODE_MASK 0x00000003UL #define CLKCTRL_MODULEMODE_DISABLE 0x00000000UL #define CLKCTRL_MODULEMODE_AUTO 0x00000001UL #define CLKCTRL_MODULEMODE_ENABLE 0x00000001UL #define CLKCTRL_IDLEST_MASK 0x00030000UL #define CLKCTRL_IDLEST_ENABLED 0x00000000UL #define CLKCTRL_IDLEST_WAKING 0x00010000UL #define CLKCTRL_IDLEST_IDLE 0x00020000UL #define CLKCTRL_IDLEST_DISABLED 0x00030000UL static struct ofw_compat_data compat_data[] = { {"ti,omap4-cm1", (uintptr_t)CM1_INSTANCE}, {"ti,omap4-cm2", (uintptr_t)CM2_INSTANCE}, {"ti,omap4-prm", (uintptr_t)PRM_INSTANCE}, {NULL, (uintptr_t)0}, }; struct omap4_prcm_softc { struct resource *sc_res; int sc_rid; int sc_instance; }; static int omap4_clk_generic_activate(struct ti_clock_dev *clkdev); static int omap4_clk_generic_deactivate(struct ti_clock_dev *clkdev); static int omap4_clk_generic_accessible(struct ti_clock_dev *clkdev); static int omap4_clk_generic_set_source(struct ti_clock_dev *clkdev, clk_src_t clksrc); static int omap4_clk_generic_get_source_freq(struct ti_clock_dev *clkdev, unsigned int *freq); static int omap4_clk_gptimer_set_source(struct ti_clock_dev *clkdev, clk_src_t clksrc); static int omap4_clk_gptimer_get_source_freq(struct ti_clock_dev *clkdev, unsigned int *freq); static int omap4_clk_hsmmc_set_source(struct ti_clock_dev *clkdev, clk_src_t clksrc); static int omap4_clk_hsmmc_get_source_freq(struct ti_clock_dev *clkdev, unsigned int *freq); static int omap4_clk_hsusbhost_set_source(struct ti_clock_dev *clkdev, clk_src_t clksrc); static int omap4_clk_hsusbhost_activate(struct ti_clock_dev *clkdev); static int omap4_clk_hsusbhost_deactivate(struct ti_clock_dev *clkdev); static int omap4_clk_hsusbhost_accessible(struct ti_clock_dev *clkdev); static int omap4_clk_get_sysclk_freq(struct ti_clock_dev *clkdev, unsigned int *freq); static int omap4_clk_get_arm_fclk_freq(struct ti_clock_dev *clkdev, unsigned int *freq); /** * omap_clk_devmap - Array of clock devices available on OMAP4xxx devices * * This map only defines which clocks are valid and the callback functions * for clock activate, deactivate, etc. It is used by the top level omap_prcm * driver. * * The actual details of the clocks (config registers, bit fields, sources, * etc) are in the private g_omap3_clk_details array below. * */ #define OMAP4_GENERIC_CLOCK_DEV(i) \ { .id = (i), \ .clk_activate = omap4_clk_generic_activate, \ .clk_deactivate = omap4_clk_generic_deactivate, \ .clk_set_source = omap4_clk_generic_set_source, \ .clk_accessible = omap4_clk_generic_accessible, \ .clk_get_source_freq = omap4_clk_generic_get_source_freq, \ .clk_set_source_freq = NULL \ } #define OMAP4_GPTIMER_CLOCK_DEV(i) \ { .id = (i), \ .clk_activate = omap4_clk_generic_activate, \ .clk_deactivate = omap4_clk_generic_deactivate, \ .clk_set_source = omap4_clk_gptimer_set_source, \ .clk_accessible = omap4_clk_generic_accessible, \ .clk_get_source_freq = omap4_clk_gptimer_get_source_freq, \ .clk_set_source_freq = NULL \ } #define OMAP4_HSMMC_CLOCK_DEV(i) \ { .id = (i), \ .clk_activate = omap4_clk_generic_activate, \ .clk_deactivate = omap4_clk_generic_deactivate, \ .clk_set_source = omap4_clk_hsmmc_set_source, \ .clk_accessible = omap4_clk_generic_accessible, \ .clk_get_source_freq = omap4_clk_hsmmc_get_source_freq, \ .clk_set_source_freq = NULL \ } #define OMAP4_HSUSBHOST_CLOCK_DEV(i) \ { .id = (i), \ .clk_activate = omap4_clk_hsusbhost_activate, \ .clk_deactivate = omap4_clk_hsusbhost_deactivate, \ .clk_set_source = omap4_clk_hsusbhost_set_source, \ .clk_accessible = omap4_clk_hsusbhost_accessible, \ .clk_get_source_freq = NULL, \ .clk_set_source_freq = NULL \ } struct ti_clock_dev ti_omap4_clk_devmap[] = { /* System clocks */ { .id = SYS_CLK, .clk_activate = NULL, .clk_deactivate = NULL, .clk_set_source = NULL, .clk_accessible = NULL, .clk_get_source_freq = omap4_clk_get_sysclk_freq, .clk_set_source_freq = NULL, }, /* MPU (ARM) core clocks */ { .id = MPU_CLK, .clk_activate = NULL, .clk_deactivate = NULL, .clk_set_source = NULL, .clk_accessible = NULL, .clk_get_source_freq = omap4_clk_get_arm_fclk_freq, .clk_set_source_freq = NULL, }, /* UART device clocks */ OMAP4_GENERIC_CLOCK_DEV(UART1_CLK), OMAP4_GENERIC_CLOCK_DEV(UART2_CLK), OMAP4_GENERIC_CLOCK_DEV(UART3_CLK), OMAP4_GENERIC_CLOCK_DEV(UART4_CLK), /* Timer device source clocks */ OMAP4_GPTIMER_CLOCK_DEV(TIMER1_CLK), OMAP4_GPTIMER_CLOCK_DEV(TIMER2_CLK), OMAP4_GPTIMER_CLOCK_DEV(TIMER3_CLK), OMAP4_GPTIMER_CLOCK_DEV(TIMER4_CLK), OMAP4_GPTIMER_CLOCK_DEV(TIMER5_CLK), OMAP4_GPTIMER_CLOCK_DEV(TIMER6_CLK), OMAP4_GPTIMER_CLOCK_DEV(TIMER7_CLK), OMAP4_GPTIMER_CLOCK_DEV(TIMER8_CLK), OMAP4_GPTIMER_CLOCK_DEV(TIMER9_CLK), OMAP4_GPTIMER_CLOCK_DEV(TIMER10_CLK), OMAP4_GPTIMER_CLOCK_DEV(TIMER11_CLK), /* MMC device clocks (MMC1 and MMC2 can have different input clocks) */ OMAP4_HSMMC_CLOCK_DEV(MMC1_CLK), OMAP4_HSMMC_CLOCK_DEV(MMC2_CLK), OMAP4_GENERIC_CLOCK_DEV(MMC3_CLK), OMAP4_GENERIC_CLOCK_DEV(MMC4_CLK), OMAP4_GENERIC_CLOCK_DEV(MMC5_CLK), /* USB HS (high speed TLL, EHCI and OHCI) */ OMAP4_HSUSBHOST_CLOCK_DEV(USBTLL_CLK), OMAP4_HSUSBHOST_CLOCK_DEV(USBHSHOST_CLK), OMAP4_HSUSBHOST_CLOCK_DEV(USBFSHOST_CLK), OMAP4_HSUSBHOST_CLOCK_DEV(USBP1_PHY_CLK), OMAP4_HSUSBHOST_CLOCK_DEV(USBP2_PHY_CLK), OMAP4_HSUSBHOST_CLOCK_DEV(USBP1_UTMI_CLK), OMAP4_HSUSBHOST_CLOCK_DEV(USBP2_UTMI_CLK), OMAP4_HSUSBHOST_CLOCK_DEV(USBP1_HSIC_CLK), OMAP4_HSUSBHOST_CLOCK_DEV(USBP2_HSIC_CLK), /* GPIO */ OMAP4_GENERIC_CLOCK_DEV(GPIO1_CLK), OMAP4_GENERIC_CLOCK_DEV(GPIO2_CLK), OMAP4_GENERIC_CLOCK_DEV(GPIO3_CLK), OMAP4_GENERIC_CLOCK_DEV(GPIO4_CLK), OMAP4_GENERIC_CLOCK_DEV(GPIO5_CLK), OMAP4_GENERIC_CLOCK_DEV(GPIO6_CLK), /* sDMA */ OMAP4_GENERIC_CLOCK_DEV(SDMA_CLK), /* I2C */ OMAP4_GENERIC_CLOCK_DEV(I2C1_CLK), OMAP4_GENERIC_CLOCK_DEV(I2C2_CLK), OMAP4_GENERIC_CLOCK_DEV(I2C3_CLK), OMAP4_GENERIC_CLOCK_DEV(I2C4_CLK), { INVALID_CLK_IDENT, NULL, NULL, NULL, NULL } }; /** * omap4_clk_details - Stores details for all the different clocks supported * * Whenever an operation on a clock is being performed (activated, deactivated, * etc) this array is looked up to find the correct register and bit(s) we * should be modifying. * */ struct omap4_clk_details { clk_ident_t id; uint32_t instance; uint32_t clksel_reg; int32_t src_freq; uint32_t enable_mode; }; #define OMAP4_GENERIC_CLOCK_DETAILS(i, f, di, r, e) \ { .id = (i), \ .instance = (di), \ .clksel_reg = (r), \ .src_freq = (f), \ .enable_mode = (e), \ } static struct omap4_clk_details g_omap4_clk_details[] = { /* UART */ OMAP4_GENERIC_CLOCK_DETAILS(UART1_CLK, FREQ_48MHZ, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x0140), CLKCTRL_MODULEMODE_ENABLE), OMAP4_GENERIC_CLOCK_DETAILS(UART2_CLK, FREQ_48MHZ, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x0148), CLKCTRL_MODULEMODE_ENABLE), OMAP4_GENERIC_CLOCK_DETAILS(UART3_CLK, FREQ_48MHZ, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x0150), CLKCTRL_MODULEMODE_ENABLE), OMAP4_GENERIC_CLOCK_DETAILS(UART4_CLK, FREQ_48MHZ, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x0158), CLKCTRL_MODULEMODE_ENABLE), /* General purpose timers */ OMAP4_GENERIC_CLOCK_DETAILS(TIMER1_CLK, -1, PRM_INSTANCE, (WKUP_CM_OFFSET + 0x040), CLKCTRL_MODULEMODE_ENABLE), OMAP4_GENERIC_CLOCK_DETAILS(TIMER2_CLK, -1, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x038), CLKCTRL_MODULEMODE_ENABLE), OMAP4_GENERIC_CLOCK_DETAILS(TIMER3_CLK, -1, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x040), CLKCTRL_MODULEMODE_ENABLE), OMAP4_GENERIC_CLOCK_DETAILS(TIMER4_CLK, -1, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x048), CLKCTRL_MODULEMODE_ENABLE), OMAP4_GENERIC_CLOCK_DETAILS(TIMER5_CLK, -1, CM1_INSTANCE, (ABE_CM1_OFFSET + 0x068), CLKCTRL_MODULEMODE_ENABLE), OMAP4_GENERIC_CLOCK_DETAILS(TIMER6_CLK, -1, CM1_INSTANCE, (ABE_CM1_OFFSET + 0x070), CLKCTRL_MODULEMODE_ENABLE), OMAP4_GENERIC_CLOCK_DETAILS(TIMER7_CLK, -1, CM1_INSTANCE, (ABE_CM1_OFFSET + 0x078), CLKCTRL_MODULEMODE_ENABLE), OMAP4_GENERIC_CLOCK_DETAILS(TIMER8_CLK, -1, CM1_INSTANCE, (ABE_CM1_OFFSET + 0x080), CLKCTRL_MODULEMODE_ENABLE), OMAP4_GENERIC_CLOCK_DETAILS(TIMER9_CLK, -1, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x050), CLKCTRL_MODULEMODE_ENABLE), OMAP4_GENERIC_CLOCK_DETAILS(TIMER10_CLK, -1, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x028), CLKCTRL_MODULEMODE_ENABLE), OMAP4_GENERIC_CLOCK_DETAILS(TIMER11_CLK, -1, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x030), CLKCTRL_MODULEMODE_ENABLE), /* HSMMC (MMC1 and MMC2 can have different input clocks) */ OMAP4_GENERIC_CLOCK_DETAILS(MMC1_CLK, -1, CM2_INSTANCE, (L3INIT_CM2_OFFSET + 0x028), /*CLKCTRL_MODULEMODE_ENABLE*/2), OMAP4_GENERIC_CLOCK_DETAILS(MMC2_CLK, -1, CM2_INSTANCE, (L3INIT_CM2_OFFSET + 0x030), /*CLKCTRL_MODULEMODE_ENABLE*/2), OMAP4_GENERIC_CLOCK_DETAILS(MMC3_CLK, FREQ_48MHZ, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x120), /*CLKCTRL_MODULEMODE_ENABLE*/2), OMAP4_GENERIC_CLOCK_DETAILS(MMC4_CLK, FREQ_48MHZ, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x128), /*CLKCTRL_MODULEMODE_ENABLE*/2), OMAP4_GENERIC_CLOCK_DETAILS(MMC5_CLK, FREQ_48MHZ, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x160), /*CLKCTRL_MODULEMODE_ENABLE*/1), /* GPIO modules */ OMAP4_GENERIC_CLOCK_DETAILS(GPIO1_CLK, -1, PRM_INSTANCE, (WKUP_CM_OFFSET + 0x038), CLKCTRL_MODULEMODE_AUTO), OMAP4_GENERIC_CLOCK_DETAILS(GPIO2_CLK, -1, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x060), CLKCTRL_MODULEMODE_AUTO), OMAP4_GENERIC_CLOCK_DETAILS(GPIO3_CLK, -1, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x068), CLKCTRL_MODULEMODE_AUTO), OMAP4_GENERIC_CLOCK_DETAILS(GPIO4_CLK, -1, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x070), CLKCTRL_MODULEMODE_AUTO), OMAP4_GENERIC_CLOCK_DETAILS(GPIO5_CLK, -1, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x078), CLKCTRL_MODULEMODE_AUTO), OMAP4_GENERIC_CLOCK_DETAILS(GPIO6_CLK, -1, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x080), CLKCTRL_MODULEMODE_AUTO), /* sDMA block */ OMAP4_GENERIC_CLOCK_DETAILS(SDMA_CLK, -1, CM2_INSTANCE, (CORE_CM2_OFFSET + 0x300), CLKCTRL_MODULEMODE_AUTO), /* I2C modules */ OMAP4_GENERIC_CLOCK_DETAILS(I2C1_CLK, -1, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x0A0), CLKCTRL_MODULEMODE_ENABLE), OMAP4_GENERIC_CLOCK_DETAILS(I2C2_CLK, -1, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x0A8), CLKCTRL_MODULEMODE_ENABLE), OMAP4_GENERIC_CLOCK_DETAILS(I2C3_CLK, -1, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x0B0), CLKCTRL_MODULEMODE_ENABLE), OMAP4_GENERIC_CLOCK_DETAILS(I2C4_CLK, -1, CM2_INSTANCE, (L4PER_CM2_OFFSET + 0x0B8), CLKCTRL_MODULEMODE_ENABLE), { INVALID_CLK_IDENT, 0, 0, 0, 0 }, }; /** * MAX_MODULE_ENABLE_WAIT - the number of loops to wait for the module to come * alive. * */ #define MAX_MODULE_ENABLE_WAIT 100 /** * ARRAY_SIZE - Macro to return the number of elements in a static const array. * */ #define ARRAY_SIZE(x) (sizeof(x)/sizeof(x[0])) /** * omap4_clk_details - writes a 32-bit value to one of the timer registers * @timer: Timer device context * @off: The offset of a register from the timer register address range * @val: The value to write into the register * * * RETURNS: * nothing */ static struct omap4_clk_details* omap4_clk_details(clk_ident_t id) { struct omap4_clk_details *walker; for (walker = g_omap4_clk_details; walker->id != INVALID_CLK_IDENT; walker++) { if (id == walker->id) return (walker); } return NULL; } static struct omap4_prcm_softc * omap4_prcm_get_instance_softc(int module_instance) { int i, maxunit; devclass_t prcm_devclass; device_t dev; struct omap4_prcm_softc *sc; prcm_devclass = devclass_find("omap4_prcm"); maxunit = devclass_get_maxunit(prcm_devclass); for (i = 0; i < maxunit; i++) { dev = devclass_get_device(prcm_devclass, i); sc = device_get_softc(dev); if (sc->sc_instance == module_instance) return (sc); } return (NULL); } /** * omap4_clk_generic_activate - checks if a module is accessible * @module: identifier for the module to check, see omap3_prcm.h for a list * of possible modules. * Example: OMAP3_MODULE_MMC1 * * * * LOCKING: * Inherits the locks from the omap_prcm driver, no internal locking. * * RETURNS: * Returns 0 on success or a positive error code on failure. */ static int omap4_clk_generic_activate(struct ti_clock_dev *clkdev) { struct omap4_prcm_softc *sc; struct omap4_clk_details* clk_details; struct resource* clk_mem_res; uint32_t clksel; unsigned int i; clk_details = omap4_clk_details(clkdev->id); if (clk_details == NULL) return (ENXIO); sc = omap4_prcm_get_instance_softc(clk_details->instance); if (sc == NULL) return ENXIO; clk_mem_res = sc->sc_res; if (clk_mem_res == NULL) return (EINVAL); /* All the 'generic' clocks have a CLKCTRL register which is more or less * generic - the have at least two fielda called MODULEMODE and IDLEST. */ clksel = bus_read_4(clk_mem_res, clk_details->clksel_reg); clksel &= ~CLKCTRL_MODULEMODE_MASK; clksel |= clk_details->enable_mode; bus_write_4(clk_mem_res, clk_details->clksel_reg, clksel); /* Now poll on the IDLEST register to tell us if the module has come up. * TODO: We need to take into account the parent clocks. */ /* Try MAX_MODULE_ENABLE_WAIT number of times to check if enabled */ for (i = 0; i < MAX_MODULE_ENABLE_WAIT; i++) { clksel = bus_read_4(clk_mem_res, clk_details->clksel_reg); if ((clksel & CLKCTRL_IDLEST_MASK) == CLKCTRL_IDLEST_ENABLED) break; DELAY(10); } /* Check the enabled state */ if ((clksel & CLKCTRL_IDLEST_MASK) != CLKCTRL_IDLEST_ENABLED) { printf("Error: failed to enable module with clock %d\n", clkdev->id); printf("Error: 0x%08x => 0x%08x\n", clk_details->clksel_reg, clksel); return (ETIMEDOUT); } return (0); } /** * omap4_clk_generic_deactivate - checks if a module is accessible * @module: identifier for the module to check, see omap3_prcm.h for a list * of possible modules. * Example: OMAP3_MODULE_MMC1 * * * * LOCKING: * Inherits the locks from the omap_prcm driver, no internal locking. * * RETURNS: * Returns 0 on success or a positive error code on failure. */ static int omap4_clk_generic_deactivate(struct ti_clock_dev *clkdev) { struct omap4_prcm_softc *sc; struct omap4_clk_details* clk_details; struct resource* clk_mem_res; uint32_t clksel; clk_details = omap4_clk_details(clkdev->id); if (clk_details == NULL) return (ENXIO); sc = omap4_prcm_get_instance_softc(clk_details->instance); if (sc == NULL) return ENXIO; clk_mem_res = sc->sc_res; if (clk_mem_res == NULL) return (EINVAL); /* All the 'generic' clocks have a CLKCTRL register which is more or less * generic - the have at least two fielda called MODULEMODE and IDLEST. */ clksel = bus_read_4(clk_mem_res, clk_details->clksel_reg); clksel &= ~CLKCTRL_MODULEMODE_MASK; clksel |= CLKCTRL_MODULEMODE_DISABLE; bus_write_4(clk_mem_res, clk_details->clksel_reg, clksel); return (0); } /** * omap4_clk_generic_set_source - checks if a module is accessible * @module: identifier for the module to check, see omap3_prcm.h for a list * of possible modules. * Example: OMAP3_MODULE_MMC1 * * * * LOCKING: * Inherits the locks from the omap_prcm driver, no internal locking. * * RETURNS: * Returns 0 on success or a positive error code on failure. */ static int omap4_clk_generic_set_source(struct ti_clock_dev *clkdev, clk_src_t clksrc) { return (0); } /** * omap4_clk_generic_accessible - checks if a module is accessible * @module: identifier for the module to check, see omap3_prcm.h for a list * of possible modules. * Example: OMAP3_MODULE_MMC1 * * * * LOCKING: * Inherits the locks from the omap_prcm driver, no internal locking. * * RETURNS: * Returns 0 on success or a negative error code on failure. */ static int omap4_clk_generic_accessible(struct ti_clock_dev *clkdev) { struct omap4_prcm_softc *sc; struct omap4_clk_details* clk_details; struct resource* clk_mem_res; uint32_t clksel; clk_details = omap4_clk_details(clkdev->id); if (clk_details == NULL) return (ENXIO); sc = omap4_prcm_get_instance_softc(clk_details->instance); if (sc == NULL) return ENXIO; clk_mem_res = sc->sc_res; if (clk_mem_res == NULL) return (EINVAL); clksel = bus_read_4(clk_mem_res, clk_details->clksel_reg); /* Check the enabled state */ if ((clksel & CLKCTRL_IDLEST_MASK) != CLKCTRL_IDLEST_ENABLED) return (0); return (1); } /** * omap4_clk_generic_get_source_freq - checks if a module is accessible * @module: identifier for the module to check, see omap3_prcm.h for a list * of possible modules. * Example: OMAP3_MODULE_MMC1 * * * * LOCKING: * Inherits the locks from the omap_prcm driver, no internal locking. * * RETURNS: * Returns 0 on success or a negative error code on failure. */ static int omap4_clk_generic_get_source_freq(struct ti_clock_dev *clkdev, unsigned int *freq ) { struct omap4_clk_details* clk_details = omap4_clk_details(clkdev->id); if (clk_details == NULL) return (ENXIO); /* Simply return the stored frequency */ if (freq) *freq = (unsigned int)clk_details->src_freq; return (0); } /** * omap4_clk_gptimer_set_source - checks if a module is accessible * @module: identifier for the module to check, see omap3_prcm.h for a list * of possible modules. * Example: OMAP3_MODULE_MMC1 * * * * LOCKING: * Inherits the locks from the omap_prcm driver, no internal locking. * * RETURNS: * Returns 0 on success or a negative error code on failure. */ static int omap4_clk_gptimer_set_source(struct ti_clock_dev *clkdev, clk_src_t clksrc) { struct omap4_prcm_softc *sc; struct omap4_clk_details* clk_details; struct resource* clk_mem_res; clk_details = omap4_clk_details(clkdev->id); if (clk_details == NULL) return (ENXIO); sc = omap4_prcm_get_instance_softc(clk_details->instance); if (sc == NULL) return ENXIO; clk_mem_res = sc->sc_res; if (clk_mem_res == NULL) return (EINVAL); /* TODO: Implement */ return (0); } /** * omap4_clk_gptimer_get_source_freq - checks if a module is accessible * @module: identifier for the module to check, see omap3_prcm.h for a list * of possible modules. * Example: OMAP3_MODULE_MMC1 * * * * LOCKING: * Inherits the locks from the omap_prcm driver, no internal locking. * * RETURNS: * Returns 0 on success or a negative error code on failure. */ static int omap4_clk_gptimer_get_source_freq(struct ti_clock_dev *clkdev, unsigned int *freq ) { struct omap4_prcm_softc *sc; struct omap4_clk_details* clk_details; struct resource* clk_mem_res; uint32_t clksel; unsigned int src_freq; clk_details = omap4_clk_details(clkdev->id); if (clk_details == NULL) return (ENXIO); sc = omap4_prcm_get_instance_softc(clk_details->instance); if (sc == NULL) return ENXIO; clk_mem_res = sc->sc_res; if (clk_mem_res == NULL) return (EINVAL); /* Need to read the CLKSEL field to determine the clock source */ clksel = bus_read_4(clk_mem_res, clk_details->clksel_reg); if (clksel & (0x1UL << 24)) src_freq = FREQ_32KHZ; else omap4_clk_get_sysclk_freq(NULL, &src_freq); /* Return the frequency */ if (freq) *freq = src_freq; return (0); } /** * omap4_clk_hsmmc_set_source - sets the source clock (freq) * @clkdev: pointer to the clockdev structure (id field will contain clock id) * * The MMC 1 and 2 clocks can be source from either a 64MHz or 96MHz clock. * * LOCKING: * Inherits the locks from the omap_prcm driver, no internal locking. * * RETURNS: * Returns 0 on success or a negative error code on failure. */ static int omap4_clk_hsmmc_set_source(struct ti_clock_dev *clkdev, clk_src_t clksrc) { struct omap4_prcm_softc *sc; struct omap4_clk_details* clk_details; struct resource* clk_mem_res; uint32_t clksel; clk_details = omap4_clk_details(clkdev->id); if (clk_details == NULL) return (ENXIO); sc = omap4_prcm_get_instance_softc(clk_details->instance); if (sc == NULL) return ENXIO; clk_mem_res = sc->sc_res; if (clk_mem_res == NULL) return (EINVAL); /* For MMC modules 3, 4 & 5 you can't change the freq, it's always 48MHz */ if ((clkdev->id == MMC3_CLK) || (clkdev->id == MMC4_CLK) || (clkdev->id == MMC5_CLK)) { if (clksrc != F48MHZ_CLK) return (EINVAL); return 0; } clksel = bus_read_4(clk_mem_res, clk_details->clksel_reg); /* Bit 24 is set if 96MHz clock or cleared for 64MHz clock */ if (clksrc == F64MHZ_CLK) clksel &= ~(0x1UL << 24); else if (clksrc == F96MHZ_CLK) clksel |= (0x1UL << 24); else return (EINVAL); bus_write_4(clk_mem_res, clk_details->clksel_reg, clksel); return (0); } /** * omap4_clk_hsmmc_get_source_freq - checks if a module is accessible * @clkdev: pointer to the clockdev structure (id field will contain clock id) * * * * LOCKING: * Inherits the locks from the omap_prcm driver, no internal locking. * * RETURNS: * Returns 0 on success or a negative error code on failure. */ static int omap4_clk_hsmmc_get_source_freq(struct ti_clock_dev *clkdev, unsigned int *freq ) { struct omap4_prcm_softc *sc; struct omap4_clk_details* clk_details; struct resource* clk_mem_res; uint32_t clksel; unsigned int src_freq; clk_details = omap4_clk_details(clkdev->id); if (clk_details == NULL) return (ENXIO); sc = omap4_prcm_get_instance_softc(clk_details->instance); if (sc == NULL) return ENXIO; clk_mem_res = sc->sc_res; if (clk_mem_res == NULL) return (EINVAL); switch (clkdev->id) { case MMC1_CLK: case MMC2_CLK: /* Need to read the CLKSEL field to determine the clock source */ clksel = bus_read_4(clk_mem_res, clk_details->clksel_reg); if (clksel & (0x1UL << 24)) src_freq = FREQ_96MHZ; else src_freq = FREQ_64MHZ; break; case MMC3_CLK: case MMC4_CLK: case MMC5_CLK: src_freq = FREQ_48MHZ; break; default: return (EINVAL); } /* Return the frequency */ if (freq) *freq = src_freq; return (0); } /** * omap4_clk_get_sysclk_freq - gets the sysclk frequency * @sc: pointer to the clk module/device context * * Read the clocking information from the power-control/boot-strap registers, * and stored in two global variables. * * RETURNS: * nothing, values are saved in global variables */ static int omap4_clk_get_sysclk_freq(struct ti_clock_dev *clkdev, unsigned int *freq) { uint32_t clksel; uint32_t sysclk; struct omap4_prcm_softc *sc; sc = omap4_prcm_get_instance_softc(PRM_INSTANCE); if (sc == NULL) return ENXIO; /* Read the input clock freq from the configuration register (CM_SYS_CLKSEL) */ clksel = bus_read_4(sc->sc_res, CM_SYS_CLKSEL_OFFSET); switch (clksel & 0x7) { case 0x1: /* 12Mhz */ sysclk = 12000000; break; case 0x3: /* 16.8Mhz */ sysclk = 16800000; break; case 0x4: /* 19.2Mhz */ sysclk = 19200000; break; case 0x5: /* 26Mhz */ sysclk = 26000000; break; case 0x7: /* 38.4Mhz */ sysclk = 38400000; break; default: panic("%s: Invalid clock freq", __func__); } /* Return the value */ if (freq) *freq = sysclk; return (0); } /** * omap4_clk_get_arm_fclk_freq - gets the MPU clock frequency * @clkdev: ignored * @freq: pointer which upon return will contain the freq in hz * @mem_res: array of allocated memory resources * * Reads the frequency setting information registers and returns the value * in the freq variable. * * RETURNS: * returns 0 on success, a positive error code on failure. */ static int omap4_clk_get_arm_fclk_freq(struct ti_clock_dev *clkdev, unsigned int *freq) { uint32_t clksel; uint32_t pll_mult, pll_div; uint32_t mpuclk, sysclk; struct omap4_prcm_softc *sc; sc = omap4_prcm_get_instance_softc(CM1_INSTANCE); if (sc == NULL) return ENXIO; /* Read the clksel register which contains the DPLL multiple and divide * values. These are applied to the sysclk. */ clksel = bus_read_4(sc->sc_res, CM_CLKSEL_DPLL_MPU); pll_mult = ((clksel >> 8) & 0x7ff); pll_div = (clksel & 0x7f) + 1; /* Get the system clock freq */ omap4_clk_get_sysclk_freq(NULL, &sysclk); /* Calculate the MPU freq */ mpuclk = ((uint64_t)sysclk * pll_mult) / pll_div; /* Return the value */ if (freq) *freq = mpuclk; return (0); } /** * omap4_clk_hsusbhost_activate - activates the USB clocks for the given module * @clkdev: pointer to the clock device structure. * @mem_res: array of memory resources allocated by the top level PRCM driver. * * The USB clocking setup seems to be a bit more tricky than the other modules, * to start with the clocking diagram for the HS host module shows 13 different * clocks. So to try and make it easier to follow the clocking activation * and deactivation is handled in it's own set of callbacks. * * LOCKING: * Inherits the locks from the omap_prcm driver, no internal locking. * * RETURNS: * Returns 0 on success or a positive error code on failure. */ struct dpll_param { unsigned int m; unsigned int n; unsigned int m2; unsigned int m3; unsigned int m4; unsigned int m5; unsigned int m6; unsigned int m7; }; /* USB parameters */ struct dpll_param usb_dpll_param[7] = { /* 12M values */ {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, /* 13M values */ {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, /* 16.8M values */ {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, /* 19.2M values */ {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, /* 26M values */ {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, /* 27M values */ {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, /* 38.4M values */ #ifdef CONFIG_OMAP4_SDC {0x32, 0x1, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0}, #else {0x32, 0x1, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0}, #endif }; static int omap4_clk_hsusbhost_activate(struct ti_clock_dev *clkdev) { struct omap4_prcm_softc *sc; struct resource* clk_mem_res; uint32_t clksel_reg_off; uint32_t clksel; unsigned int i; sc = omap4_prcm_get_instance_softc(CM2_INSTANCE); if (sc == NULL) return ENXIO; switch (clkdev->id) { case USBTLL_CLK: /* For the USBTLL module we need to enable the following clocks: * - INIT_L4_ICLK (will be enabled by bootloader) * - TLL_CH0_FCLK * - TLL_CH1_FCLK */ /* We need the CM_L3INIT_HSUSBTLL_CLKCTRL register in CM2 register set */ clk_mem_res = sc->sc_res; clksel_reg_off = L3INIT_CM2_OFFSET + 0x68; /* Enable the module and also enable the optional func clocks for * channels 0 & 1 (is this needed ?) */ clksel = bus_read_4(clk_mem_res, clksel_reg_off); clksel &= ~CLKCTRL_MODULEMODE_MASK; clksel |= CLKCTRL_MODULEMODE_ENABLE; clksel |= (0x1 << 8); /* USB-HOST optional clock: USB_CH0_CLK */ clksel |= (0x1 << 9); /* USB-HOST optional clock: USB_CH1_CLK */ break; case USBHSHOST_CLK: case USBP1_PHY_CLK: case USBP2_PHY_CLK: case USBP1_UTMI_CLK: case USBP2_UTMI_CLK: case USBP1_HSIC_CLK: case USBP2_HSIC_CLK: /* For the USB HS HOST module we need to enable the following clocks: * - INIT_L4_ICLK (will be enabled by bootloader) * - INIT_L3_ICLK (will be enabled by bootloader) * - INIT_48MC_FCLK * - UTMI_ROOT_GFCLK (UTMI only, create a new clock for that ?) * - UTMI_P1_FCLK (UTMI only, create a new clock for that ?) * - UTMI_P2_FCLK (UTMI only, create a new clock for that ?) * - HSIC_P1_60 (HSIC only, create a new clock for that ?) * - HSIC_P1_480 (HSIC only, create a new clock for that ?) * - HSIC_P2_60 (HSIC only, create a new clock for that ?) * - HSIC_P2_480 (HSIC only, create a new clock for that ?) */ /* We need the CM_L3INIT_HSUSBHOST_CLKCTRL register in CM2 register set */ clk_mem_res = sc->sc_res; clksel_reg_off = L3INIT_CM2_OFFSET + 0x58; clksel = bus_read_4(clk_mem_res, clksel_reg_off); /* Enable the module and also enable the optional func clocks */ if (clkdev->id == USBHSHOST_CLK) { clksel &= ~CLKCTRL_MODULEMODE_MASK; clksel |= /*CLKCTRL_MODULEMODE_ENABLE*/2; clksel |= (0x1 << 15); /* USB-HOST clock control: FUNC48MCLK */ } else if (clkdev->id == USBP1_UTMI_CLK) clksel |= (0x1 << 8); /* UTMI_P1_CLK */ else if (clkdev->id == USBP2_UTMI_CLK) clksel |= (0x1 << 9); /* UTMI_P2_CLK */ else if (clkdev->id == USBP1_HSIC_CLK) clksel |= (0x5 << 11); /* HSIC60M_P1_CLK + HSIC480M_P1_CLK */ else if (clkdev->id == USBP2_HSIC_CLK) clksel |= (0x5 << 12); /* HSIC60M_P2_CLK + HSIC480M_P2_CLK */ break; default: return (EINVAL); } bus_write_4(clk_mem_res, clksel_reg_off, clksel); /* Try MAX_MODULE_ENABLE_WAIT number of times to check if enabled */ for (i = 0; i < MAX_MODULE_ENABLE_WAIT; i++) { clksel = bus_read_4(clk_mem_res, clksel_reg_off); if ((clksel & CLKCTRL_IDLEST_MASK) == CLKCTRL_IDLEST_ENABLED) break; } /* Check the enabled state */ if ((clksel & CLKCTRL_IDLEST_MASK) != CLKCTRL_IDLEST_ENABLED) { printf("Error: HERE failed to enable module with clock %d\n", clkdev->id); printf("Error: 0x%08x => 0x%08x\n", clksel_reg_off, clksel); return (ETIMEDOUT); } return (0); } /** * omap4_clk_generic_deactivate - checks if a module is accessible * @clkdev: pointer to the clock device structure. * @mem_res: array of memory resources allocated by the top level PRCM driver. * * * * LOCKING: * Inherits the locks from the omap_prcm driver, no internal locking. * * RETURNS: * Returns 0 on success or a positive error code on failure. */ static int omap4_clk_hsusbhost_deactivate(struct ti_clock_dev *clkdev) { struct omap4_prcm_softc *sc; struct resource* clk_mem_res; uint32_t clksel_reg_off; uint32_t clksel; sc = omap4_prcm_get_instance_softc(CM2_INSTANCE); if (sc == NULL) return ENXIO; switch (clkdev->id) { case USBTLL_CLK: /* We need the CM_L3INIT_HSUSBTLL_CLKCTRL register in CM2 register set */ clk_mem_res = sc->sc_res; clksel_reg_off = L3INIT_CM2_OFFSET + 0x68; clksel = bus_read_4(clk_mem_res, clksel_reg_off); clksel &= ~CLKCTRL_MODULEMODE_MASK; clksel |= CLKCTRL_MODULEMODE_DISABLE; break; case USBHSHOST_CLK: case USBP1_PHY_CLK: case USBP2_PHY_CLK: case USBP1_UTMI_CLK: case USBP2_UTMI_CLK: case USBP1_HSIC_CLK: case USBP2_HSIC_CLK: /* For the USB HS HOST module we need to enable the following clocks: * - INIT_L4_ICLK (will be enabled by bootloader) * - INIT_L3_ICLK (will be enabled by bootloader) * - INIT_48MC_FCLK * - UTMI_ROOT_GFCLK (UTMI only, create a new clock for that ?) * - UTMI_P1_FCLK (UTMI only, create a new clock for that ?) * - UTMI_P2_FCLK (UTMI only, create a new clock for that ?) * - HSIC_P1_60 (HSIC only, create a new clock for that ?) * - HSIC_P1_480 (HSIC only, create a new clock for that ?) * - HSIC_P2_60 (HSIC only, create a new clock for that ?) * - HSIC_P2_480 (HSIC only, create a new clock for that ?) */ /* We need the CM_L3INIT_HSUSBHOST_CLKCTRL register in CM2 register set */ clk_mem_res = sc->sc_res; clksel_reg_off = L3INIT_CM2_OFFSET + 0x58; clksel = bus_read_4(clk_mem_res, clksel_reg_off); /* Enable the module and also enable the optional func clocks */ if (clkdev->id == USBHSHOST_CLK) { clksel &= ~CLKCTRL_MODULEMODE_MASK; clksel |= CLKCTRL_MODULEMODE_DISABLE; clksel &= ~(0x1 << 15); /* USB-HOST clock control: FUNC48MCLK */ } else if (clkdev->id == USBP1_UTMI_CLK) clksel &= ~(0x1 << 8); /* UTMI_P1_CLK */ else if (clkdev->id == USBP2_UTMI_CLK) clksel &= ~(0x1 << 9); /* UTMI_P2_CLK */ else if (clkdev->id == USBP1_HSIC_CLK) clksel &= ~(0x5 << 11); /* HSIC60M_P1_CLK + HSIC480M_P1_CLK */ else if (clkdev->id == USBP2_HSIC_CLK) clksel &= ~(0x5 << 12); /* HSIC60M_P2_CLK + HSIC480M_P2_CLK */ break; default: return (EINVAL); } bus_write_4(clk_mem_res, clksel_reg_off, clksel); return (0); } /** * omap4_clk_hsusbhost_accessible - checks if a module is accessible * @clkdev: pointer to the clock device structure. * @mem_res: array of memory resources allocated by the top level PRCM driver. * * * * LOCKING: * Inherits the locks from the omap_prcm driver, no internal locking. * * RETURNS: * Returns 0 if module is not enable, 1 if module is enabled or a negative * error code on failure. */ static int omap4_clk_hsusbhost_accessible(struct ti_clock_dev *clkdev) { struct omap4_prcm_softc *sc; struct resource* clk_mem_res; uint32_t clksel_reg_off; uint32_t clksel; sc = omap4_prcm_get_instance_softc(CM2_INSTANCE); if (sc == NULL) return ENXIO; if (clkdev->id == USBTLL_CLK) { /* We need the CM_L3INIT_HSUSBTLL_CLKCTRL register in CM2 register set */ clk_mem_res = sc->sc_res; clksel_reg_off = L3INIT_CM2_OFFSET + 0x68; } else if (clkdev->id == USBHSHOST_CLK) { /* We need the CM_L3INIT_HSUSBHOST_CLKCTRL register in CM2 register set */ clk_mem_res = sc->sc_res; clksel_reg_off = L3INIT_CM2_OFFSET + 0x58; } else { return (EINVAL); } clksel = bus_read_4(clk_mem_res, clksel_reg_off); /* Check the enabled state */ if ((clksel & CLKCTRL_IDLEST_MASK) != CLKCTRL_IDLEST_ENABLED) return (0); return (1); } /** * omap4_clk_hsusbhost_set_source - sets the source clocks * @clkdev: pointer to the clock device structure. * @clksrc: the clock source ID for the given clock. * @mem_res: array of memory resources allocated by the top level PRCM driver. * * * * LOCKING: * Inherits the locks from the omap_prcm driver, no internal locking. * * RETURNS: - * Returns 0 if sucessful otherwise a negative error code on failure. + * Returns 0 if successful otherwise a negative error code on failure. */ static int omap4_clk_hsusbhost_set_source(struct ti_clock_dev *clkdev, clk_src_t clksrc) { struct omap4_prcm_softc *sc; struct resource* clk_mem_res; uint32_t clksel_reg_off; uint32_t clksel; unsigned int bit; sc = omap4_prcm_get_instance_softc(CM2_INSTANCE); if (sc == NULL) return ENXIO; if (clkdev->id == USBP1_PHY_CLK) bit = 24; else if (clkdev->id != USBP2_PHY_CLK) bit = 25; else return (EINVAL); /* We need the CM_L3INIT_HSUSBHOST_CLKCTRL register in CM2 register set */ clk_mem_res = sc->sc_res; clksel_reg_off = L3INIT_CM2_OFFSET + 0x58; clksel = bus_read_4(clk_mem_res, clksel_reg_off); /* Set the clock source to either external or internal */ if (clksrc == EXT_CLK) clksel |= (0x1 << bit); else clksel &= ~(0x1 << bit); bus_write_4(clk_mem_res, clksel_reg_off, clksel); return (0); } #define PRM_RSTCTRL 0x1b00 #define PRM_RSTCTRL_RESET 0x2 static void omap4_prcm_reset(void) { struct omap4_prcm_softc *sc; sc = omap4_prcm_get_instance_softc(PRM_INSTANCE); if (sc == NULL) return; bus_write_4(sc->sc_res, PRM_RSTCTRL, bus_read_4(sc->sc_res, PRM_RSTCTRL) | PRM_RSTCTRL_RESET); bus_read_4(sc->sc_res, PRM_RSTCTRL); } /** * omap4_prcm_probe - probe function for the driver * @dev: prcm device handle * * Simply sets the name of the driver module. * * LOCKING: * None * * RETURNS: * Always returns 0 */ static int omap4_prcm_probe(device_t dev) { const struct ofw_compat_data *ocd; if (!ofw_bus_status_okay(dev)) return (ENXIO); ocd = ofw_bus_search_compatible(dev, compat_data); if ((int)ocd->ocd_data == 0) return (ENXIO); switch ((int)ocd->ocd_data) { case PRM_INSTANCE: device_set_desc(dev, "TI OMAP Power, Reset and Clock Management (PRM)"); break; case CM1_INSTANCE: device_set_desc(dev, "TI OMAP Power, Reset and Clock Management (C1)"); break; case CM2_INSTANCE: device_set_desc(dev, "TI OMAP Power, Reset and Clock Management (C2)"); break; default: device_printf(dev, "unknown instance type: %d\n", (int)ocd->ocd_data); return (ENXIO); } return (BUS_PROBE_DEFAULT); } /** * omap_prcm_attach - attach function for the driver * @dev: prcm device handle * * Allocates and sets up the driver context, this simply entails creating a * bus mappings for the PRCM register set. * * LOCKING: * None * * RETURNS: * Always returns 0 */ extern uint32_t platform_arm_tmr_freq; static int omap4_prcm_attach(device_t dev) { struct omap4_prcm_softc *sc; unsigned int freq; const struct ofw_compat_data *ocd; sc = device_get_softc(dev); ocd = ofw_bus_search_compatible(dev, compat_data); sc->sc_instance = (int)ocd->ocd_data; sc->sc_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->sc_rid, RF_ACTIVE); if (sc->sc_res == NULL) { device_printf(dev, "could not allocate resources\n"); return (ENXIO); } ti_cpu_reset = omap4_prcm_reset; /* * In order to determine ARM frequency we need both RPM and CM1 * instances up and running. So wait until all CRM devices are * initialized. Should be replaced with proper clock framework */ if (device_get_unit(dev) == 2) { omap4_clk_get_arm_fclk_freq(NULL, &freq); arm_tmr_change_frequency(freq / 2); } return (0); } static device_method_t omap4_prcm_methods[] = { DEVMETHOD(device_probe, omap4_prcm_probe), DEVMETHOD(device_attach, omap4_prcm_attach), {0, 0}, }; static driver_t omap4_prcm_driver = { "omap4_prcm", omap4_prcm_methods, sizeof(struct omap4_prcm_softc), }; static devclass_t omap4_prcm_devclass; EARLY_DRIVER_MODULE(omap4_prcm, simplebus, omap4_prcm_driver, omap4_prcm_devclass, 0, 0, BUS_PASS_TIMER + BUS_PASS_ORDER_EARLY); MODULE_VERSION(omap4_prcm, 1); Index: user/ngie/release-pkg-fix-tests/sys/arm/ti/ti_i2c.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/ti/ti_i2c.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/ti/ti_i2c.c (revision 299091) @@ -1,988 +1,988 @@ /*- * Copyright (c) 2011 Ben Gray . * Copyright (c) 2014 Luiz Otavio O Souza . * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /** * Driver for the I2C module on the TI SoC. * * This driver is heavily based on the TWI driver for the AT91 (at91_twi.c). * * CAUTION: The I2Ci registers are limited to 16 bit and 8 bit data accesses, * 32 bit data access is not allowed and can corrupt register content. * * This driver currently doesn't use DMA for the transfer, although I hope to * incorporate that sometime in the future. The idea being that for transaction * larger than a certain size the DMA engine is used, for anything less the * normal interrupt/fifo driven option is used. * * * WARNING: This driver uses mtx_sleep and interrupts to perform transactions, * which means you can't do a transaction during startup before the interrupts * have been enabled. Hint - the freebsd function config_intrhook_establish(). */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "iicbus_if.h" /** * I2C device driver context, a pointer to this is stored in the device * driver structure. */ struct ti_i2c_softc { device_t sc_dev; clk_ident_t clk_id; struct resource* sc_irq_res; struct resource* sc_mem_res; device_t sc_iicbus; void* sc_irq_h; struct mtx sc_mtx; struct iic_msg* sc_buffer; int sc_bus_inuse; int sc_buffer_pos; int sc_error; int sc_fifo_trsh; int sc_timeout; uint16_t sc_con_reg; uint16_t sc_rev; }; struct ti_i2c_clock_config { u_int frequency; /* Bus frequency in Hz */ uint8_t psc; /* Fast/Standard mode prescale divider */ uint8_t scll; /* Fast/Standard mode SCL low time */ uint8_t sclh; /* Fast/Standard mode SCL high time */ uint8_t hsscll; /* High Speed mode SCL low time */ uint8_t hssclh; /* High Speed mode SCL high time */ }; #if defined(SOC_OMAP4) /* * OMAP4 i2c bus clock is 96MHz / ((psc + 1) * (scll + 7 + sclh + 5)). * The prescaler values for 100KHz and 400KHz modes come from the table in the * OMAP4 TRM. The table doesn't list 1MHz; these values should give that speed. */ static struct ti_i2c_clock_config ti_omap4_i2c_clock_configs[] = { { 100000, 23, 13, 15, 0, 0}, { 400000, 9, 5, 7, 0, 0}, { 1000000, 3, 5, 7, 0, 0}, /* { 3200000, 1, 113, 115, 7, 10}, - HS mode */ { 0 /* Table terminator */ } }; #endif #if defined(SOC_TI_AM335X) /* * AM335x i2c bus clock is 48MHZ / ((psc + 1) * (scll + 7 + sclh + 5)) * In all cases we prescale the clock to 24MHz as recommended in the manual. */ static struct ti_i2c_clock_config ti_am335x_i2c_clock_configs[] = { { 100000, 1, 111, 117, 0, 0}, { 400000, 1, 23, 25, 0, 0}, { 1000000, 1, 5, 7, 0, 0}, { 0 /* Table terminator */ } }; #endif /** * Locking macros used throughout the driver */ #define TI_I2C_LOCK(_sc) mtx_lock(&(_sc)->sc_mtx) #define TI_I2C_UNLOCK(_sc) mtx_unlock(&(_sc)->sc_mtx) #define TI_I2C_LOCK_INIT(_sc) \ mtx_init(&_sc->sc_mtx, device_get_nameunit(_sc->sc_dev), \ "ti_i2c", MTX_DEF) #define TI_I2C_LOCK_DESTROY(_sc) mtx_destroy(&_sc->sc_mtx) #define TI_I2C_ASSERT_LOCKED(_sc) mtx_assert(&_sc->sc_mtx, MA_OWNED) #define TI_I2C_ASSERT_UNLOCKED(_sc) mtx_assert(&_sc->sc_mtx, MA_NOTOWNED) #ifdef DEBUG #define ti_i2c_dbg(_sc, fmt, args...) \ device_printf((_sc)->sc_dev, fmt, ##args) #else #define ti_i2c_dbg(_sc, fmt, args...) #endif /** * ti_i2c_read_2 - reads a 16-bit value from one of the I2C registers * @sc: I2C device context * @off: the byte offset within the register bank to read from. * * * LOCKING: * No locking required * * RETURNS: * 16-bit value read from the register. */ static inline uint16_t ti_i2c_read_2(struct ti_i2c_softc *sc, bus_size_t off) { return (bus_read_2(sc->sc_mem_res, off)); } /** * ti_i2c_write_2 - writes a 16-bit value to one of the I2C registers * @sc: I2C device context * @off: the byte offset within the register bank to read from. * @val: the value to write into the register * * LOCKING: * No locking required * * RETURNS: * 16-bit value read from the register. */ static inline void ti_i2c_write_2(struct ti_i2c_softc *sc, bus_size_t off, uint16_t val) { bus_write_2(sc->sc_mem_res, off, val); } static int ti_i2c_transfer_intr(struct ti_i2c_softc* sc, uint16_t status) { int amount, done, i; done = 0; amount = 0; /* Check for the error conditions. */ if (status & I2C_STAT_NACK) { /* No ACK from slave. */ ti_i2c_dbg(sc, "NACK\n"); ti_i2c_write_2(sc, I2C_REG_STATUS, I2C_STAT_NACK); sc->sc_error = ENXIO; } else if (status & I2C_STAT_AL) { /* Arbitration lost. */ ti_i2c_dbg(sc, "Arbitration lost\n"); ti_i2c_write_2(sc, I2C_REG_STATUS, I2C_STAT_AL); sc->sc_error = ENXIO; } /* Check if we have finished. */ if (status & I2C_STAT_ARDY) { /* Register access ready - transaction complete basically. */ ti_i2c_dbg(sc, "ARDY transaction complete\n"); if (sc->sc_error != 0 && sc->sc_buffer->flags & IIC_M_NOSTOP) { ti_i2c_write_2(sc, I2C_REG_CON, sc->sc_con_reg | I2C_CON_STP); } ti_i2c_write_2(sc, I2C_REG_STATUS, I2C_STAT_ARDY | I2C_STAT_RDR | I2C_STAT_RRDY | I2C_STAT_XDR | I2C_STAT_XRDY); return (1); } if (sc->sc_buffer->flags & IIC_M_RD) { /* Read some data. */ if (status & I2C_STAT_RDR) { /* * Receive draining interrupt - last data received. - * The set FIFO threshold wont be reached to trigger + * The set FIFO threshold won't be reached to trigger * RRDY. */ ti_i2c_dbg(sc, "Receive draining interrupt\n"); /* * Drain the FIFO. Read the pending data in the FIFO. */ amount = sc->sc_buffer->len - sc->sc_buffer_pos; } else if (status & I2C_STAT_RRDY) { /* * Receive data ready interrupt - FIFO has reached the * set threshold. */ ti_i2c_dbg(sc, "Receive data ready interrupt\n"); amount = min(sc->sc_fifo_trsh, sc->sc_buffer->len - sc->sc_buffer_pos); } /* Read the bytes from the fifo. */ for (i = 0; i < amount; i++) sc->sc_buffer->buf[sc->sc_buffer_pos++] = (uint8_t)(ti_i2c_read_2(sc, I2C_REG_DATA) & 0xff); if (status & I2C_STAT_RDR) ti_i2c_write_2(sc, I2C_REG_STATUS, I2C_STAT_RDR); if (status & I2C_STAT_RRDY) ti_i2c_write_2(sc, I2C_REG_STATUS, I2C_STAT_RRDY); } else { /* Write some data. */ if (status & I2C_STAT_XDR) { /* * Transmit draining interrupt - FIFO level is below * the set threshold and the amount of data still to - * be transferred wont reach the set FIFO threshold. + * be transferred won't reach the set FIFO threshold. */ ti_i2c_dbg(sc, "Transmit draining interrupt\n"); /* * Drain the TX data. Write the pending data in the * FIFO. */ amount = sc->sc_buffer->len - sc->sc_buffer_pos; } else if (status & I2C_STAT_XRDY) { /* * Transmit data ready interrupt - the FIFO level * is below the set threshold. */ ti_i2c_dbg(sc, "Transmit data ready interrupt\n"); amount = min(sc->sc_fifo_trsh, sc->sc_buffer->len - sc->sc_buffer_pos); } /* Write the bytes from the fifo. */ for (i = 0; i < amount; i++) ti_i2c_write_2(sc, I2C_REG_DATA, sc->sc_buffer->buf[sc->sc_buffer_pos++]); if (status & I2C_STAT_XDR) ti_i2c_write_2(sc, I2C_REG_STATUS, I2C_STAT_XDR); if (status & I2C_STAT_XRDY) ti_i2c_write_2(sc, I2C_REG_STATUS, I2C_STAT_XRDY); } return (done); } /** * ti_i2c_intr - interrupt handler for the I2C module * @dev: i2c device handle * * * * LOCKING: * Called from timer context * * RETURNS: * EH_HANDLED or EH_NOT_HANDLED */ static void ti_i2c_intr(void *arg) { int done; struct ti_i2c_softc *sc; uint16_t events, status; sc = (struct ti_i2c_softc *)arg; TI_I2C_LOCK(sc); status = ti_i2c_read_2(sc, I2C_REG_STATUS); if (status == 0) { TI_I2C_UNLOCK(sc); return; } /* Save enabled interrupts. */ events = ti_i2c_read_2(sc, I2C_REG_IRQENABLE_SET); /* We only care about enabled interrupts. */ status &= events; done = 0; if (sc->sc_buffer != NULL) done = ti_i2c_transfer_intr(sc, status); else { ti_i2c_dbg(sc, "Transfer interrupt without buffer\n"); sc->sc_error = EINVAL; done = 1; } if (done) /* Wakeup the process that started the transaction. */ wakeup(sc); TI_I2C_UNLOCK(sc); } /** * ti_i2c_transfer - called to perform the transfer * @dev: i2c device handle * @msgs: the messages to send/receive * @nmsgs: the number of messages in the msgs array * * * LOCKING: * Internally locked * * RETURNS: * 0 on function succeeded * EINVAL if invalid message is passed as an arg */ static int ti_i2c_transfer(device_t dev, struct iic_msg *msgs, uint32_t nmsgs) { int err, i, repstart, timeout; struct ti_i2c_softc *sc; uint16_t reg; sc = device_get_softc(dev); TI_I2C_LOCK(sc); /* If the controller is busy wait until it is available. */ while (sc->sc_bus_inuse == 1) mtx_sleep(sc, &sc->sc_mtx, 0, "i2cbuswait", 0); /* Now we have control over the I2C controller. */ sc->sc_bus_inuse = 1; err = 0; repstart = 0; for (i = 0; i < nmsgs; i++) { sc->sc_buffer = &msgs[i]; sc->sc_buffer_pos = 0; sc->sc_error = 0; /* Zero byte transfers aren't allowed. */ if (sc->sc_buffer == NULL || sc->sc_buffer->buf == NULL || sc->sc_buffer->len == 0) { err = EINVAL; break; } /* Check if the i2c bus is free. */ if (repstart == 0) { /* * On repeated start we send the START condition while * the bus _is_ busy. */ timeout = 0; while (ti_i2c_read_2(sc, I2C_REG_STATUS_RAW) & I2C_STAT_BB) { if (timeout++ > 100) { err = EBUSY; goto out; } DELAY(1000); } timeout = 0; } else repstart = 0; if (sc->sc_buffer->flags & IIC_M_NOSTOP) repstart = 1; /* Set the slave address. */ ti_i2c_write_2(sc, I2C_REG_SA, msgs[i].slave >> 1); /* Write the data length. */ ti_i2c_write_2(sc, I2C_REG_CNT, sc->sc_buffer->len); /* Clear the RX and the TX FIFO. */ reg = ti_i2c_read_2(sc, I2C_REG_BUF); reg |= I2C_BUF_RXFIFO_CLR | I2C_BUF_TXFIFO_CLR; ti_i2c_write_2(sc, I2C_REG_BUF, reg); reg = sc->sc_con_reg | I2C_CON_STT; if (repstart == 0) reg |= I2C_CON_STP; if ((sc->sc_buffer->flags & IIC_M_RD) == 0) reg |= I2C_CON_TRX; ti_i2c_write_2(sc, I2C_REG_CON, reg); /* Wait for an event. */ err = mtx_sleep(sc, &sc->sc_mtx, 0, "i2ciowait", sc->sc_timeout); if (err == 0) err = sc->sc_error; if (err) break; } out: if (timeout == 0) { while (ti_i2c_read_2(sc, I2C_REG_STATUS_RAW) & I2C_STAT_BB) { if (timeout++ > 100) break; DELAY(1000); } } /* Put the controller in master mode again. */ if ((ti_i2c_read_2(sc, I2C_REG_CON) & I2C_CON_MST) == 0) ti_i2c_write_2(sc, I2C_REG_CON, sc->sc_con_reg); sc->sc_buffer = NULL; sc->sc_bus_inuse = 0; /* Wake up the processes that are waiting for the bus. */ wakeup(sc); TI_I2C_UNLOCK(sc); return (err); } static int ti_i2c_reset(struct ti_i2c_softc *sc, u_char speed) { int timeout; struct ti_i2c_clock_config *clkcfg; u_int busfreq; uint16_t fifo_trsh, reg, scll, sclh; switch (ti_chip()) { #ifdef SOC_OMAP4 case CHIP_OMAP_4: clkcfg = ti_omap4_i2c_clock_configs; break; #endif #ifdef SOC_TI_AM335X case CHIP_AM335X: clkcfg = ti_am335x_i2c_clock_configs; break; #endif default: panic("Unknown TI SoC, unable to reset the i2c"); } /* * If we haven't attached the bus yet, just init at the default slow * speed. This lets us get the hardware initialized enough to attach * the bus which is where the real speed configuration is handled. After * the bus is attached, get the configured speed from it. Search the * configuration table for the best speed we can do that doesn't exceed * the requested speed. */ if (sc->sc_iicbus == NULL) busfreq = 100000; else busfreq = IICBUS_GET_FREQUENCY(sc->sc_iicbus, speed); for (;;) { if (clkcfg[1].frequency == 0 || clkcfg[1].frequency > busfreq) break; clkcfg++; } /* * 23.1.4.3 - HS I2C Software Reset * From OMAP4 TRM at page 4068. * * 1. Ensure that the module is disabled. */ sc->sc_con_reg = 0; ti_i2c_write_2(sc, I2C_REG_CON, sc->sc_con_reg); /* 2. Issue a softreset to the controller. */ bus_write_2(sc->sc_mem_res, I2C_REG_SYSC, I2C_REG_SYSC_SRST); /* * 3. Enable the module. * The I2Ci.I2C_SYSS[0] RDONE bit is asserted only after the module * is enabled by setting the I2Ci.I2C_CON[15] I2C_EN bit to 1. */ ti_i2c_write_2(sc, I2C_REG_CON, I2C_CON_I2C_EN); /* 4. Wait for the software reset to complete. */ timeout = 0; while ((ti_i2c_read_2(sc, I2C_REG_SYSS) & I2C_SYSS_RDONE) == 0) { if (timeout++ > 100) return (EBUSY); DELAY(100); } /* * Disable the I2C controller once again, now that the reset has * finished. */ ti_i2c_write_2(sc, I2C_REG_CON, sc->sc_con_reg); /* * The following sequence is taken from the OMAP4 TRM at page 4077. * * 1. Enable the functional and interface clocks (see Section * 23.1.5.1.1.1.1). Done at ti_i2c_activate(). * * 2. Program the prescaler to obtain an approximately 12MHz internal * sampling clock (I2Ci_INTERNAL_CLK) by programming the * corresponding value in the I2Ci.I2C_PSC[3:0] PSC field. * This value depends on the frequency of the functional clock * (I2Ci_FCLK). Because this frequency is 96MHz, the * I2Ci.I2C_PSC[7:0] PSC field value is 0x7. */ ti_i2c_write_2(sc, I2C_REG_PSC, clkcfg->psc); /* * 3. Program the I2Ci.I2C_SCLL[7:0] SCLL and I2Ci.I2C_SCLH[7:0] SCLH * bit fields to obtain a bit rate of 100 Kbps, 400 Kbps or 1Mbps. * These values depend on the internal sampling clock frequency * (see Table 23-8). */ scll = clkcfg->scll & I2C_SCLL_MASK; sclh = clkcfg->sclh & I2C_SCLH_MASK; /* * 4. (Optional) Program the I2Ci.I2C_SCLL[15:8] HSSCLL and * I2Ci.I2C_SCLH[15:8] HSSCLH fields to obtain a bit rate of * 400K bps or 3.4M bps (for the second phase of HS mode). These * values depend on the internal sampling clock frequency (see * Table 23-8). * * 5. (Optional) If a bit rate of 3.4M bps is used and the bus line * capacitance exceeds 45 pF, (see Section 18.4.8, PAD Functional * Multiplexing and Configuration). */ switch (ti_chip()) { #ifdef SOC_OMAP4 case CHIP_OMAP_4: if ((clkcfg->hsscll + clkcfg->hssclh) > 0) { scll |= clkcfg->hsscll << I2C_HSSCLL_SHIFT; sclh |= clkcfg->hssclh << I2C_HSSCLH_SHIFT; sc->sc_con_reg |= I2C_CON_OPMODE_HS; } break; #endif } /* Write the selected bit rate. */ ti_i2c_write_2(sc, I2C_REG_SCLL, scll); ti_i2c_write_2(sc, I2C_REG_SCLH, sclh); /* * 6. Configure the Own Address of the I2C controller by storing it in * the I2Ci.I2C_OA0 register. Up to four Own Addresses can be * programmed in the I2Ci.I2C_OAi registers (where i = 0, 1, 2, 3) * for each I2C controller. * * Note: For a 10-bit address, set the corresponding expand Own Address * bit in the I2Ci.I2C_CON register. * * Driver currently always in single master mode so ignore this step. */ /* * 7. Set the TX threshold (in transmitter mode) and the RX threshold * (in receiver mode) by setting the I2Ci.I2C_BUF[5:0]XTRSH field to * (TX threshold - 1) and the I2Ci.I2C_BUF[13:8]RTRSH field to (RX * threshold - 1), where the TX and RX thresholds are greater than * or equal to 1. * * The threshold is set to 5 for now. */ fifo_trsh = (sc->sc_fifo_trsh - 1) & I2C_BUF_TRSH_MASK; reg = fifo_trsh | (fifo_trsh << I2C_BUF_RXTRSH_SHIFT); ti_i2c_write_2(sc, I2C_REG_BUF, reg); /* * 8. Take the I2C controller out of reset by setting the * I2Ci.I2C_CON[15] I2C_EN bit to 1. * * 23.1.5.1.1.1.2 - Initialize the I2C Controller * * To initialize the I2C controller, perform the following steps: * * 1. Configure the I2Ci.I2C_CON register: * . For master or slave mode, set the I2Ci.I2C_CON[10] MST bit * (0: slave, 1: master). * . For transmitter or receiver mode, set the I2Ci.I2C_CON[9] TRX * bit (0: receiver, 1: transmitter). */ /* Enable the I2C controller in master mode. */ sc->sc_con_reg |= I2C_CON_I2C_EN | I2C_CON_MST; ti_i2c_write_2(sc, I2C_REG_CON, sc->sc_con_reg); /* * 2. If using an interrupt to transmit/receive data, set the * corresponding bit in the I2Ci.I2C_IE register (the I2Ci.I2C_IE[4] * XRDY_IE bit for the transmit interrupt, the I2Ci.I2C_IE[3] RRDY * bit for the receive interrupt). */ /* Set the interrupts we want to be notified. */ reg = I2C_IE_XDR | /* Transmit draining interrupt. */ I2C_IE_XRDY | /* Transmit Data Ready interrupt. */ I2C_IE_RDR | /* Receive draining interrupt. */ I2C_IE_RRDY | /* Receive Data Ready interrupt. */ I2C_IE_ARDY | /* Register Access Ready interrupt. */ I2C_IE_NACK | /* No Acknowledgment interrupt. */ I2C_IE_AL; /* Arbitration lost interrupt. */ /* Enable the interrupts. */ ti_i2c_write_2(sc, I2C_REG_IRQENABLE_SET, reg); /* * 3. If using DMA to receive/transmit data, set to 1 the corresponding * bit in the I2Ci.I2C_BUF register (the I2Ci.I2C_BUF[15] RDMA_EN * bit for the receive DMA channel, the I2Ci.I2C_BUF[7] XDMA_EN bit * for the transmit DMA channel). * * Not using DMA for now, so ignore this. */ return (0); } static int ti_i2c_iicbus_reset(device_t dev, u_char speed, u_char addr, u_char *oldaddr) { struct ti_i2c_softc *sc; int err; sc = device_get_softc(dev); TI_I2C_LOCK(sc); err = ti_i2c_reset(sc, speed); TI_I2C_UNLOCK(sc); if (err) return (err); return (IIC_ENOADDR); } static int ti_i2c_activate(device_t dev) { int err; struct ti_i2c_softc *sc; sc = (struct ti_i2c_softc*)device_get_softc(dev); /* * 1. Enable the functional and interface clocks (see Section * 23.1.5.1.1.1.1). */ err = ti_prcm_clk_enable(sc->clk_id); if (err) return (err); return (ti_i2c_reset(sc, IIC_UNKNOWN)); } /** * ti_i2c_deactivate - deactivates the controller and releases resources * @dev: i2c device handle * * * * LOCKING: * Assumed called in an atomic context. * * RETURNS: * nothing */ static void ti_i2c_deactivate(device_t dev) { struct ti_i2c_softc *sc = device_get_softc(dev); /* Disable the controller - cancel all transactions. */ ti_i2c_write_2(sc, I2C_REG_IRQENABLE_CLR, 0xffff); ti_i2c_write_2(sc, I2C_REG_STATUS, 0xffff); ti_i2c_write_2(sc, I2C_REG_CON, 0); /* Release the interrupt handler. */ if (sc->sc_irq_h != NULL) { bus_teardown_intr(dev, sc->sc_irq_res, sc->sc_irq_h); sc->sc_irq_h = NULL; } bus_generic_detach(sc->sc_dev); /* Unmap the I2C controller registers. */ if (sc->sc_mem_res != NULL) { bus_release_resource(dev, SYS_RES_MEMORY, 0, sc->sc_mem_res); sc->sc_mem_res = NULL; } /* Release the IRQ resource. */ if (sc->sc_irq_res != NULL) { bus_release_resource(dev, SYS_RES_IRQ, 0, sc->sc_irq_res); sc->sc_irq_res = NULL; } /* Finally disable the functional and interface clocks. */ ti_prcm_clk_disable(sc->clk_id); } static int ti_i2c_sysctl_clk(SYSCTL_HANDLER_ARGS) { int clk, psc, sclh, scll; struct ti_i2c_softc *sc; sc = arg1; TI_I2C_LOCK(sc); /* Get the system prescaler value. */ psc = (int)ti_i2c_read_2(sc, I2C_REG_PSC) + 1; /* Get the bitrate. */ scll = (int)ti_i2c_read_2(sc, I2C_REG_SCLL) & I2C_SCLL_MASK; sclh = (int)ti_i2c_read_2(sc, I2C_REG_SCLH) & I2C_SCLH_MASK; clk = I2C_CLK / psc / (scll + 7 + sclh + 5); TI_I2C_UNLOCK(sc); return (sysctl_handle_int(oidp, &clk, 0, req)); } static int ti_i2c_sysctl_timeout(SYSCTL_HANDLER_ARGS) { struct ti_i2c_softc *sc; unsigned int val; int err; sc = arg1; /* * MTX_DEF lock can't be held while doing uimove in * sysctl_handle_int */ TI_I2C_LOCK(sc); val = sc->sc_timeout; TI_I2C_UNLOCK(sc); err = sysctl_handle_int(oidp, &val, 0, req); /* Write request? */ if ((err == 0) && (req->newptr != NULL)) { TI_I2C_LOCK(sc); sc->sc_timeout = val; TI_I2C_UNLOCK(sc); } return (err); } static int ti_i2c_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, "ti,omap4-i2c")) return (ENXIO); device_set_desc(dev, "TI I2C Controller"); return (0); } static int ti_i2c_attach(device_t dev) { int err, rid; phandle_t node; struct ti_i2c_softc *sc; struct sysctl_ctx_list *ctx; struct sysctl_oid_list *tree; uint16_t fifosz; sc = device_get_softc(dev); sc->sc_dev = dev; /* Get the i2c device id from FDT. */ node = ofw_bus_get_node(dev); /* i2c ti,hwmods bindings is special: it start with index 1 */ sc->clk_id = ti_hwmods_get_clock(dev); if (sc->clk_id == INVALID_CLK_IDENT) { device_printf(dev, "failed to get device id using ti,hwmod\n"); return (ENXIO); } /* Get the memory resource for the register mapping. */ rid = 0; sc->sc_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->sc_mem_res == NULL) { device_printf(dev, "Cannot map registers.\n"); return (ENXIO); } /* Allocate our IRQ resource. */ rid = 0; sc->sc_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE | RF_SHAREABLE); if (sc->sc_irq_res == NULL) { bus_release_resource(dev, SYS_RES_MEMORY, 0, sc->sc_mem_res); device_printf(dev, "Cannot allocate interrupt.\n"); return (ENXIO); } TI_I2C_LOCK_INIT(sc); /* First of all, we _must_ activate the H/W. */ err = ti_i2c_activate(dev); if (err) { device_printf(dev, "ti_i2c_activate failed\n"); goto out; } /* Read the version number of the I2C module */ sc->sc_rev = ti_i2c_read_2(sc, I2C_REG_REVNB_HI) & 0xff; /* Get the fifo size. */ fifosz = ti_i2c_read_2(sc, I2C_REG_BUFSTAT); fifosz >>= I2C_BUFSTAT_FIFODEPTH_SHIFT; fifosz &= I2C_BUFSTAT_FIFODEPTH_MASK; device_printf(dev, "I2C revision %d.%d FIFO size: %d bytes\n", sc->sc_rev >> 4, sc->sc_rev & 0xf, 8 << fifosz); /* Set the FIFO threshold to 5 for now. */ sc->sc_fifo_trsh = 5; /* Set I2C bus timeout */ sc->sc_timeout = 5*hz; ctx = device_get_sysctl_ctx(dev); tree = SYSCTL_CHILDREN(device_get_sysctl_tree(dev)); SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "i2c_clock", CTLFLAG_RD | CTLTYPE_UINT | CTLFLAG_MPSAFE, sc, 0, ti_i2c_sysctl_clk, "IU", "I2C bus clock"); SYSCTL_ADD_PROC(ctx, tree, OID_AUTO, "i2c_timeout", CTLFLAG_RW | CTLTYPE_UINT | CTLFLAG_MPSAFE, sc, 0, ti_i2c_sysctl_timeout, "IU", "I2C bus timeout (in ticks)"); /* Activate the interrupt. */ err = bus_setup_intr(dev, sc->sc_irq_res, INTR_TYPE_MISC | INTR_MPSAFE, NULL, ti_i2c_intr, sc, &sc->sc_irq_h); if (err) goto out; /* Attach the iicbus. */ if ((sc->sc_iicbus = device_add_child(dev, "iicbus", -1)) == NULL) { device_printf(dev, "could not allocate iicbus instance\n"); err = ENXIO; goto out; } /* Probe and attach the iicbus */ bus_generic_attach(dev); out: if (err) { ti_i2c_deactivate(dev); TI_I2C_LOCK_DESTROY(sc); } return (err); } static int ti_i2c_detach(device_t dev) { struct ti_i2c_softc *sc; int rv; sc = device_get_softc(dev); ti_i2c_deactivate(dev); TI_I2C_LOCK_DESTROY(sc); if (sc->sc_iicbus && (rv = device_delete_child(dev, sc->sc_iicbus)) != 0) return (rv); return (0); } static phandle_t ti_i2c_get_node(device_t bus, device_t dev) { /* Share controller node with iibus device. */ return (ofw_bus_get_node(bus)); } static device_method_t ti_i2c_methods[] = { /* Device interface */ DEVMETHOD(device_probe, ti_i2c_probe), DEVMETHOD(device_attach, ti_i2c_attach), DEVMETHOD(device_detach, ti_i2c_detach), /* Bus interface */ DEVMETHOD(bus_setup_intr, bus_generic_setup_intr), DEVMETHOD(bus_teardown_intr, bus_generic_teardown_intr), DEVMETHOD(bus_alloc_resource, bus_generic_alloc_resource), DEVMETHOD(bus_release_resource, bus_generic_release_resource), DEVMETHOD(bus_activate_resource, bus_generic_activate_resource), DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource), DEVMETHOD(bus_adjust_resource, bus_generic_adjust_resource), DEVMETHOD(bus_set_resource, bus_generic_rl_set_resource), DEVMETHOD(bus_get_resource, bus_generic_rl_get_resource), /* OFW methods */ DEVMETHOD(ofw_bus_get_node, ti_i2c_get_node), /* iicbus interface */ DEVMETHOD(iicbus_callback, iicbus_null_callback), DEVMETHOD(iicbus_reset, ti_i2c_iicbus_reset), DEVMETHOD(iicbus_transfer, ti_i2c_transfer), DEVMETHOD_END }; static driver_t ti_i2c_driver = { "iichb", ti_i2c_methods, sizeof(struct ti_i2c_softc), }; static devclass_t ti_i2c_devclass; DRIVER_MODULE(ti_iic, simplebus, ti_i2c_driver, ti_i2c_devclass, 0, 0); DRIVER_MODULE(iicbus, ti_iic, iicbus_driver, iicbus_devclass, 0, 0); MODULE_DEPEND(ti_iic, ti_prcm, 1, 1, 1); MODULE_DEPEND(ti_iic, iicbus, 1, 1, 1); Index: user/ngie/release-pkg-fix-tests/sys/arm/ti/ti_pinmux.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/ti/ti_pinmux.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/ti/ti_pinmux.c (revision 299091) @@ -1,445 +1,445 @@ /* * Copyright (c) 2010 * Ben Gray . * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Ben Gray. * 4. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY BEN GRAY ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL BEN GRAY BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /** * Exposes pinmux module to pinctrl-compatible interface */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "ti_pinmux.h" struct pincfg { uint32_t reg; uint32_t conf; }; static struct resource_spec ti_pinmux_res_spec[] = { { SYS_RES_MEMORY, 0, RF_ACTIVE }, /* Control memory window */ { -1, 0 } }; static struct ti_pinmux_softc *ti_pinmux_sc; #define ti_pinmux_read_2(sc, reg) \ bus_space_read_2((sc)->sc_bst, (sc)->sc_bsh, (reg)) #define ti_pinmux_write_2(sc, reg, val) \ bus_space_write_2((sc)->sc_bst, (sc)->sc_bsh, (reg), (val)) #define ti_pinmux_read_4(sc, reg) \ bus_space_read_4((sc)->sc_bst, (sc)->sc_bsh, (reg)) #define ti_pinmux_write_4(sc, reg, val) \ bus_space_write_4((sc)->sc_bst, (sc)->sc_bsh, (reg), (val)) /** * ti_padconf_devmap - Array of pins, should be defined one per SoC * * This array is typically defined in one of the targeted *_scm_pinumx.c * files and is specific to the given SoC platform. Each entry in the array * corresponds to an individual pin. */ extern const struct ti_pinmux_device ti_pinmux_dev; /** * ti_pinmux_padconf_from_name - searches the list of pads and returns entry * with matching ball name. * @ballname: the name of the ball * * RETURNS: * A pointer to the matching padconf or NULL if the ball wasn't found. */ static const struct ti_pinmux_padconf* ti_pinmux_padconf_from_name(const char *ballname) { const struct ti_pinmux_padconf *padconf; padconf = ti_pinmux_dev.padconf; while (padconf->ballname != NULL) { if (strcmp(ballname, padconf->ballname) == 0) return(padconf); padconf++; } return (NULL); } /** * ti_pinmux_padconf_set_internal - sets the muxmode and state for a pad/pin * @padconf: pointer to the pad structure * @muxmode: the name of the mode to use for the pin, i.e. "uart1_rx" * @state: the state to put the pad/pin in, i.e. PADCONF_PIN_??? * * * LOCKING: * Internally locks it's own context. * * RETURNS: * 0 on success. * EINVAL if pin requested is outside valid range or already in use. */ static int ti_pinmux_padconf_set_internal(struct ti_pinmux_softc *sc, const struct ti_pinmux_padconf *padconf, const char *muxmode, unsigned int state) { unsigned int mode; uint16_t reg_val; /* populate the new value for the PADCONF register */ reg_val = (uint16_t)(state & ti_pinmux_dev.padconf_sate_mask); /* find the new mode requested */ for (mode = 0; mode < 8; mode++) { if ((padconf->muxmodes[mode] != NULL) && (strcmp(padconf->muxmodes[mode], muxmode) == 0)) { break; } } /* couldn't find the mux mode */ if (mode >= 8) { printf("Invalid mode \"%s\"\n", muxmode); return (EINVAL); } /* set the mux mode */ reg_val |= (uint16_t)(mode & ti_pinmux_dev.padconf_muxmode_mask); if (bootverbose) device_printf(sc->sc_dev, "setting internal %x for %s\n", reg_val, muxmode); /* write the register value (16-bit writes) */ ti_pinmux_write_2(sc, padconf->reg_off, reg_val); return (0); } /** * ti_pinmux_padconf_set - sets the muxmode and state for a pad/pin * @padname: the name of the pad, i.e. "c12" * @muxmode: the name of the mode to use for the pin, i.e. "uart1_rx" * @state: the state to put the pad/pin in, i.e. PADCONF_PIN_??? * * * LOCKING: * Internally locks it's own context. * * RETURNS: * 0 on success. * EINVAL if pin requested is outside valid range or already in use. */ int ti_pinmux_padconf_set(const char *padname, const char *muxmode, unsigned int state) { const struct ti_pinmux_padconf *padconf; if (!ti_pinmux_sc) return (ENXIO); /* find the pin in the devmap */ padconf = ti_pinmux_padconf_from_name(padname); if (padconf == NULL) return (EINVAL); return (ti_pinmux_padconf_set_internal(ti_pinmux_sc, padconf, muxmode, state)); } /** * ti_pinmux_padconf_get - gets the muxmode and state for a pad/pin * @padname: the name of the pad, i.e. "c12" * @muxmode: upon return will contain the name of the muxmode of the pin * @state: upon return will contain the state of the pad/pin * * * LOCKING: * Internally locks it's own context. * * RETURNS: * 0 on success. * EINVAL if pin requested is outside valid range or already in use. */ int ti_pinmux_padconf_get(const char *padname, const char **muxmode, unsigned int *state) { const struct ti_pinmux_padconf *padconf; uint16_t reg_val; if (!ti_pinmux_sc) return (ENXIO); /* find the pin in the devmap */ padconf = ti_pinmux_padconf_from_name(padname); if (padconf == NULL) return (EINVAL); /* read the register value (16-bit reads) */ reg_val = ti_pinmux_read_2(ti_pinmux_sc, padconf->reg_off); /* save the state */ if (state) *state = (reg_val & ti_pinmux_dev.padconf_sate_mask); /* save the mode */ if (muxmode) *muxmode = padconf->muxmodes[(reg_val & ti_pinmux_dev.padconf_muxmode_mask)]; return (0); } /** * ti_pinmux_padconf_set_gpiomode - converts a pad to GPIO mode. * @gpio: the GPIO pin number (0-195) * @state: the state to put the pad/pin in, i.e. PADCONF_PIN_??? * * * * LOCKING: * Internally locks it's own context. * * RETURNS: * 0 on success. * EINVAL if pin requested is outside valid range or already in use. */ int ti_pinmux_padconf_set_gpiomode(uint32_t gpio, unsigned int state) { const struct ti_pinmux_padconf *padconf; uint16_t reg_val; if (!ti_pinmux_sc) return (ENXIO); /* find the gpio pin in the padconf array */ padconf = ti_pinmux_dev.padconf; while (padconf->ballname != NULL) { if (padconf->gpio_pin == gpio) break; padconf++; } if (padconf->ballname == NULL) return (EINVAL); /* populate the new value for the PADCONF register */ reg_val = (uint16_t)(state & ti_pinmux_dev.padconf_sate_mask); /* set the mux mode */ reg_val |= (uint16_t)(padconf->gpio_mode & ti_pinmux_dev.padconf_muxmode_mask); /* write the register value (16-bit writes) */ ti_pinmux_write_2(ti_pinmux_sc, padconf->reg_off, reg_val); return (0); } /** * ti_pinmux_padconf_get_gpiomode - gets the current GPIO mode of the pin * @gpio: the GPIO pin number (0-195) * @state: upon return will contain the state * * * * LOCKING: * Internally locks it's own context. * * RETURNS: * 0 on success. * EINVAL if pin requested is outside valid range or not configured as GPIO. */ int ti_pinmux_padconf_get_gpiomode(uint32_t gpio, unsigned int *state) { const struct ti_pinmux_padconf *padconf; uint16_t reg_val; if (!ti_pinmux_sc) return (ENXIO); /* find the gpio pin in the padconf array */ padconf = ti_pinmux_dev.padconf; while (padconf->ballname != NULL) { if (padconf->gpio_pin == gpio) break; padconf++; } if (padconf->ballname == NULL) return (EINVAL); /* read the current register settings */ reg_val = ti_pinmux_read_2(ti_pinmux_sc, padconf->reg_off); /* check to make sure the pins is configured as GPIO in the first state */ if ((reg_val & ti_pinmux_dev.padconf_muxmode_mask) != padconf->gpio_mode) return (EINVAL); /* read and store the reset of the state, i.e. pull-up, pull-down, etc */ if (state) *state = (reg_val & ti_pinmux_dev.padconf_sate_mask); return (0); } static int ti_pinmux_configure_pins(device_t dev, phandle_t cfgxref) { struct pincfg *cfgtuples, *cfg; phandle_t cfgnode; int i, ntuples; static struct ti_pinmux_softc *sc; sc = device_get_softc(dev); cfgnode = OF_node_from_xref(cfgxref); ntuples = OF_getencprop_alloc(cfgnode, "pinctrl-single,pins", sizeof(*cfgtuples), (void **)&cfgtuples); if (ntuples < 0) return (ENOENT); if (ntuples == 0) return (0); /* Empty property is not an error. */ for (i = 0, cfg = cfgtuples; i < ntuples; i++, cfg++) { if (bootverbose) { char name[32]; OF_getprop(cfgnode, "name", &name, sizeof(name)); printf("%16s: muxreg 0x%04x muxval 0x%02x\n", name, cfg->reg, cfg->conf); } /* write the register value (16-bit writes) */ ti_pinmux_write_2(sc, cfg->reg, cfg->conf); } free(cfgtuples, M_OFWPROP); return (0); } /* * Device part of OMAP SCM driver */ static int ti_pinmux_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, "pinctrl-single")) return (ENXIO); if (ti_pinmux_sc) { printf("%s: multiple pinctrl modules in device tree data, ignoring\n", __func__); return (EEXIST); } device_set_desc(dev, "TI Pinmux Module"); return (BUS_PROBE_DEFAULT); } /** * ti_pinmux_attach - attaches the pinmux to the simplebus * @dev: new device * * RETURNS - * Zero on sucess or ENXIO if an error occuried. + * Zero on success or ENXIO if an error occuried. */ static int ti_pinmux_attach(device_t dev) { struct ti_pinmux_softc *sc = device_get_softc(dev); #if 0 if (ti_pinmux_sc) return (ENXIO); #endif sc->sc_dev = dev; if (bus_alloc_resources(dev, ti_pinmux_res_spec, sc->sc_res)) { device_printf(dev, "could not allocate resources\n"); return (ENXIO); } sc->sc_bst = rman_get_bustag(sc->sc_res[0]); sc->sc_bsh = rman_get_bushandle(sc->sc_res[0]); if (ti_pinmux_sc == NULL) ti_pinmux_sc = sc; fdt_pinctrl_register(dev, "pinctrl-single,pins"); fdt_pinctrl_configure_tree(dev); return (0); } static device_method_t ti_pinmux_methods[] = { DEVMETHOD(device_probe, ti_pinmux_probe), DEVMETHOD(device_attach, ti_pinmux_attach), /* fdt_pinctrl interface */ DEVMETHOD(fdt_pinctrl_configure, ti_pinmux_configure_pins), { 0, 0 } }; static driver_t ti_pinmux_driver = { "ti_pinmux", ti_pinmux_methods, sizeof(struct ti_pinmux_softc), }; static devclass_t ti_pinmux_devclass; DRIVER_MODULE(ti_pinmux, simplebus, ti_pinmux_driver, ti_pinmux_devclass, 0, 0); Index: user/ngie/release-pkg-fix-tests/sys/arm/ti/ti_prcm.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/ti/ti_prcm.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/ti/ti_prcm.c (revision 299091) @@ -1,357 +1,357 @@ /* * Copyright (c) 2010 * Ben Gray . * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Ben Gray. * 4. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY BEN GRAY ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL BEN GRAY BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /** - * Power, Reset and Clock Managment Module + * Power, Reset and Clock Management Module * * This is a very simple driver wrapper around the PRCM set of registers in * the OMAP3 chip. It allows you to turn on and off things like the functional * and interface clocks to the various on-chip modules. * */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /** * ti_*_clk_devmap - Array of clock devices, should be defined one per SoC * * This array is typically defined in one of the targeted *_prcm_clk.c * files and is specific to the given SoC platform. Each entry in the array * corresponds to an individual clock device. */ extern struct ti_clock_dev ti_omap4_clk_devmap[]; extern struct ti_clock_dev ti_am335x_clk_devmap[]; /** * ti_prcm_clk_dev - returns a pointer to the clock device with given id * @clk: the ID of the clock device to get * * Simply iterates through the clk_devmap global array and returns a pointer * to the clock device if found. * * LOCKING: * None * * RETURNS: * The pointer to the clock device on success, on failure NULL is returned. */ static struct ti_clock_dev * ti_prcm_clk_dev(clk_ident_t clk) { struct ti_clock_dev *clk_dev; /* Find the clock within the devmap - it's a bit inefficent having a for * loop for this, but this function should only called when a driver is * being activated so IMHO not a big issue. */ clk_dev = NULL; switch(ti_chip()) { #ifdef SOC_OMAP4 case CHIP_OMAP_4: clk_dev = &(ti_omap4_clk_devmap[0]); break; #endif #ifdef SOC_TI_AM335X case CHIP_AM335X: clk_dev = &(ti_am335x_clk_devmap[0]); break; #endif } if (clk_dev == NULL) panic("No clock devmap found"); while (clk_dev->id != INVALID_CLK_IDENT) { if (clk_dev->id == clk) { return (clk_dev); } clk_dev++; } /* Sanity check we managed to find the clock */ printf("ti_prcm: Failed to find clock device (%d)\n", clk); return (NULL); } /** * ti_prcm_clk_valid - enables a clock for a particular module * @clk: identifier for the module to enable, see ti_prcm.h for a list * of possible modules. * Example: OMAP3_MODULE_MMC1_ICLK or OMAP3_MODULE_GPTIMER10_FCLK. * * This function can enable either a functional or interface clock. * * The real work done to enable the clock is really done in the callback * function associated with the clock, this function is simply a wrapper * around that. * * LOCKING: * Internally locks the driver context. * * RETURNS: * Returns 0 on success or positive error code on failure. */ int ti_prcm_clk_valid(clk_ident_t clk) { int ret = 0; if (ti_prcm_clk_dev(clk) == NULL) ret = EINVAL; return (ret); } /** * ti_prcm_clk_enable - enables a clock for a particular module * @clk: identifier for the module to enable, see ti_prcm.h for a list * of possible modules. * Example: OMAP3_MODULE_MMC1_ICLK or OMAP3_MODULE_GPTIMER10_FCLK. * * This function can enable either a functional or interface clock. * * The real work done to enable the clock is really done in the callback * function associated with the clock, this function is simply a wrapper * around that. * * LOCKING: * Internally locks the driver context. * * RETURNS: * Returns 0 on success or positive error code on failure. */ int ti_prcm_clk_enable(clk_ident_t clk) { struct ti_clock_dev *clk_dev; int ret; /* Find the clock within the devmap - it's a bit inefficent having a for * loop for this, but this function should only called when a driver is * being activated so IMHO not a big issue. */ clk_dev = ti_prcm_clk_dev(clk); /* Sanity check we managed to find the clock */ if (clk_dev == NULL) return (EINVAL); /* Activate the clock */ if (clk_dev->clk_activate) ret = clk_dev->clk_activate(clk_dev); else ret = EINVAL; return (ret); } /** * ti_prcm_clk_disable - disables a clock for a particular module * @clk: identifier for the module to enable, see ti_prcm.h for a list * of possible modules. * Example: OMAP3_MODULE_MMC1_ICLK or OMAP3_MODULE_GPTIMER10_FCLK. * * This function can enable either a functional or interface clock. * * The real work done to enable the clock is really done in the callback * function associated with the clock, this function is simply a wrapper * around that. * * LOCKING: * Internally locks the driver context. * * RETURNS: * Returns 0 on success or positive error code on failure. */ int ti_prcm_clk_disable(clk_ident_t clk) { struct ti_clock_dev *clk_dev; int ret; /* Find the clock within the devmap - it's a bit inefficent having a for * loop for this, but this function should only called when a driver is * being activated so IMHO not a big issue. */ clk_dev = ti_prcm_clk_dev(clk); /* Sanity check we managed to find the clock */ if (clk_dev == NULL) return (EINVAL); /* Activate the clock */ if (clk_dev->clk_deactivate) ret = clk_dev->clk_deactivate(clk_dev); else ret = EINVAL; return (ret); } /** * ti_prcm_clk_set_source - sets the source * @clk: identifier for the module to enable, see ti_prcm.h for a list * of possible modules. * Example: OMAP3_MODULE_MMC1_ICLK or OMAP3_MODULE_GPTIMER10_FCLK. * * This function can enable either a functional or interface clock. * * The real work done to enable the clock is really done in the callback * function associated with the clock, this function is simply a wrapper * around that. * * LOCKING: * Internally locks the driver context. * * RETURNS: * Returns 0 on success or positive error code on failure. */ int ti_prcm_clk_set_source(clk_ident_t clk, clk_src_t clksrc) { struct ti_clock_dev *clk_dev; int ret; /* Find the clock within the devmap - it's a bit inefficent having a for * loop for this, but this function should only called when a driver is * being activated so IMHO not a big issue. */ clk_dev = ti_prcm_clk_dev(clk); /* Sanity check we managed to find the clock */ if (clk_dev == NULL) return (EINVAL); /* Activate the clock */ if (clk_dev->clk_set_source) ret = clk_dev->clk_set_source(clk_dev, clksrc); else ret = EINVAL; return (ret); } /** * ti_prcm_clk_get_source_freq - gets the source clock frequency * @clk: identifier for the module to enable, see ti_prcm.h for a list * of possible modules. * @freq: pointer to an integer that upon return will contain the src freq * * This function returns the frequency of the source clock. * * The real work done to enable the clock is really done in the callback * function associated with the clock, this function is simply a wrapper * around that. * * LOCKING: * Internally locks the driver context. * * RETURNS: * Returns 0 on success or positive error code on failure. */ int ti_prcm_clk_get_source_freq(clk_ident_t clk, unsigned int *freq) { struct ti_clock_dev *clk_dev; int ret; /* Find the clock within the devmap - it's a bit inefficent having a for * loop for this, but this function should only called when a driver is * being activated so IMHO not a big issue. */ clk_dev = ti_prcm_clk_dev(clk); /* Sanity check we managed to find the clock */ if (clk_dev == NULL) return (EINVAL); /* Get the source frequency of the clock */ if (clk_dev->clk_get_source_freq) ret = clk_dev->clk_get_source_freq(clk_dev, freq); else ret = EINVAL; return (ret); } /** * ti_prcm_clk_set_source_freq - sets the source clock frequency as close to freq as possible * @clk: identifier for the module to enable, see ti_prcm.h for a list * of possible modules. * @freq: requested freq * * LOCKING: * Internally locks the driver context. * * RETURNS: * Returns 0 on success or positive error code on failure. */ int ti_prcm_clk_set_source_freq(clk_ident_t clk, unsigned int freq) { struct ti_clock_dev *clk_dev; int ret; clk_dev = ti_prcm_clk_dev(clk); /* Sanity check we managed to find the clock */ if (clk_dev == NULL) return (EINVAL); /* Get the source frequency of the clock */ if (clk_dev->clk_set_source_freq) ret = clk_dev->clk_set_source_freq(clk_dev, freq); else ret = EINVAL; return (ret); } Index: user/ngie/release-pkg-fix-tests/sys/arm/ti/ti_scm.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/ti/ti_scm.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/ti/ti_scm.c (revision 299091) @@ -1,174 +1,174 @@ /* * Copyright (c) 2010 * Ben Gray . * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Ben Gray. * 4. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY BEN GRAY ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL BEN GRAY BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /** * SCM - System Control Module * * Hopefully in the end this module will contain a bunch of utility functions * for configuring and querying the general system control registers, but for * now it only does pin(pad) multiplexing. * * This is different from the GPIO module in that it is used to configure the * pins between modules not just GPIO input/output. * * This file contains the generic top level driver, however it relies on chip * specific settings and therefore expects an array of ti_scm_padconf structs * call ti_padconf_devmap to be located somewhere in the kernel. * */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "ti_scm.h" static struct resource_spec ti_scm_res_spec[] = { { SYS_RES_MEMORY, 0, RF_ACTIVE }, /* Control memory window */ { -1, 0 } }; static struct ti_scm_softc *ti_scm_sc; #define ti_scm_read_4(sc, reg) \ bus_space_read_4((sc)->sc_bst, (sc)->sc_bsh, (reg)) #define ti_scm_write_4(sc, reg, val) \ bus_space_write_4((sc)->sc_bst, (sc)->sc_bsh, (reg), (val)) /* * Device part of OMAP SCM driver */ static int ti_scm_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, "syscon")) return (ENXIO); if (ti_scm_sc) { return (EEXIST); } device_set_desc(dev, "TI Control Module"); return (BUS_PROBE_DEFAULT); } /** * ti_scm_attach - attaches the timer to the simplebus * @dev: new device * * Reserves memory and interrupt resources, stores the softc structure * globally and registers both the timecount and eventtimer objects. * * RETURNS - * Zero on sucess or ENXIO if an error occuried. + * Zero on success or ENXIO if an error occuried. */ static int ti_scm_attach(device_t dev) { struct ti_scm_softc *sc = device_get_softc(dev); sc->sc_dev = dev; if (bus_alloc_resources(dev, ti_scm_res_spec, sc->sc_res)) { device_printf(dev, "could not allocate resources\n"); return (ENXIO); } /* Global timer interface */ sc->sc_bst = rman_get_bustag(sc->sc_res[0]); sc->sc_bsh = rman_get_bushandle(sc->sc_res[0]); ti_scm_sc = sc; return (0); } int ti_scm_reg_read_4(uint32_t reg, uint32_t *val) { if (!ti_scm_sc) return (ENXIO); *val = ti_scm_read_4(ti_scm_sc, reg); return (0); } int ti_scm_reg_write_4(uint32_t reg, uint32_t val) { if (!ti_scm_sc) return (ENXIO); ti_scm_write_4(ti_scm_sc, reg, val); return (0); } static device_method_t ti_scm_methods[] = { DEVMETHOD(device_probe, ti_scm_probe), DEVMETHOD(device_attach, ti_scm_attach), { 0, 0 } }; static driver_t ti_scm_driver = { "ti_scm", ti_scm_methods, sizeof(struct ti_scm_softc), }; static devclass_t ti_scm_devclass; EARLY_DRIVER_MODULE(ti_scm, simplebus, ti_scm_driver, ti_scm_devclass, 0, 0, BUS_PASS_BUS + BUS_PASS_ORDER_MIDDLE); Index: user/ngie/release-pkg-fix-tests/sys/arm/ti/ti_sdma.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/ti/ti_sdma.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/ti/ti_sdma.c (revision 299091) @@ -1,1250 +1,1250 @@ /*- * Copyright (c) 2011 * Ben Gray . * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /** * Kernel functions for using the DMA controller * * * DMA TRANSFERS: * A DMA transfer block consists of a number of frames (FN). Each frame * consists of a number of elements, and each element can have a size of 8, 16, * or 32 bits. * * OMAP44xx and newer chips support linked list (aka scatter gather) transfers, * where a linked list of source/destination pairs can be placed in memory * for the H/W to process. Earlier chips only allowed you to chain multiple * channels together. However currently this linked list feature is not * supported by the driver. * */ /** * Data structure per DMA channel. * * */ struct ti_sdma_channel { /* * The configuration registers for the given channel, these are modified * by the set functions and only written to the actual registers when a * transaction is started. */ uint32_t reg_csdp; uint32_t reg_ccr; uint32_t reg_cicr; /* Set when one of the configuration registers above change */ uint32_t need_reg_write; /* Callback function used when an interrupt is tripped on the given channel */ void (*callback)(unsigned int ch, uint32_t ch_status, void *data); /* Callback data passed in the callback ... duh */ void* callback_data; }; /** * DMA driver context, allocated and stored globally, this driver is not * intetned to ever be unloaded (see ti_sdma_sc). * */ struct ti_sdma_softc { device_t sc_dev; struct resource* sc_irq_res; struct resource* sc_mem_res; /* * I guess in theory we should have a mutex per DMA channel for register * modifications. But since we know we are never going to be run on a SMP * system, we can use just the single lock for all channels. */ struct mtx sc_mtx; /* Stores the H/W revision read from the registers */ uint32_t sc_hw_rev; /* * Bits in the sc_active_channels data field indicate if the channel has * been activated. */ uint32_t sc_active_channels; struct ti_sdma_channel sc_channel[NUM_DMA_CHANNELS]; }; static struct ti_sdma_softc *ti_sdma_sc = NULL; /** * Macros for driver mutex locking */ #define TI_SDMA_LOCK(_sc) mtx_lock_spin(&(_sc)->sc_mtx) #define TI_SDMA_UNLOCK(_sc) mtx_unlock_spin(&(_sc)->sc_mtx) #define TI_SDMA_LOCK_INIT(_sc) \ mtx_init(&_sc->sc_mtx, device_get_nameunit(_sc->sc_dev), \ "ti_sdma", MTX_SPIN) #define TI_SDMA_LOCK_DESTROY(_sc) mtx_destroy(&_sc->sc_mtx); #define TI_SDMA_ASSERT_LOCKED(_sc) mtx_assert(&_sc->sc_mtx, MA_OWNED); #define TI_SDMA_ASSERT_UNLOCKED(_sc) mtx_assert(&_sc->sc_mtx, MA_NOTOWNED); /** * Function prototypes * */ static void ti_sdma_intr(void *); /** * ti_sdma_read_4 - reads a 32-bit value from one of the DMA registers * @sc: DMA device context * @off: The offset of a register from the DMA register address range * * * RETURNS: * 32-bit value read from the register. */ static inline uint32_t ti_sdma_read_4(struct ti_sdma_softc *sc, bus_size_t off) { return bus_read_4(sc->sc_mem_res, off); } /** * ti_sdma_write_4 - writes a 32-bit value to one of the DMA registers * @sc: DMA device context * @off: The offset of a register from the DMA register address range * * * RETURNS: * 32-bit value read from the register. */ static inline void ti_sdma_write_4(struct ti_sdma_softc *sc, bus_size_t off, uint32_t val) { bus_write_4(sc->sc_mem_res, off, val); } /** * ti_sdma_is_omap3_rev - returns true if H/W is from OMAP3 series * @sc: DMA device context * */ static inline int ti_sdma_is_omap3_rev(struct ti_sdma_softc *sc) { return (sc->sc_hw_rev == DMA4_OMAP3_REV); } /** * ti_sdma_is_omap4_rev - returns true if H/W is from OMAP4 series * @sc: DMA device context * */ static inline int ti_sdma_is_omap4_rev(struct ti_sdma_softc *sc) { return (sc->sc_hw_rev == DMA4_OMAP4_REV); } /** * ti_sdma_intr - interrupt handler for all 4 DMA IRQs * @arg: ignored * * Called when any of the four DMA IRQs are triggered. * * LOCKING: * DMA registers protected by internal mutex * * RETURNS: * nothing */ static void ti_sdma_intr(void *arg) { struct ti_sdma_softc *sc = ti_sdma_sc; uint32_t intr; uint32_t csr; unsigned int ch, j; struct ti_sdma_channel* channel; TI_SDMA_LOCK(sc); for (j = 0; j < NUM_DMA_IRQS; j++) { /* Get the flag interrupts (enabled) */ intr = ti_sdma_read_4(sc, DMA4_IRQSTATUS_L(j)); intr &= ti_sdma_read_4(sc, DMA4_IRQENABLE_L(j)); if (intr == 0x00000000) continue; /* Loop through checking the status bits */ for (ch = 0; ch < NUM_DMA_CHANNELS; ch++) { if (intr & (1 << ch)) { channel = &sc->sc_channel[ch]; /* Read the CSR regsiter and verify we don't have a spurious IRQ */ csr = ti_sdma_read_4(sc, DMA4_CSR(ch)); if (csr == 0) { device_printf(sc->sc_dev, "Spurious DMA IRQ for channel " "%d\n", ch); continue; } /* Sanity check this channel is active */ if ((sc->sc_active_channels & (1 << ch)) == 0) { device_printf(sc->sc_dev, "IRQ %d for a non-activated " "channel %d\n", j, ch); continue; } /* Check the status error codes */ if (csr & DMA4_CSR_DROP) device_printf(sc->sc_dev, "Synchronization event drop " "occurred during the transfer on channel %u\n", ch); if (csr & DMA4_CSR_SECURE_ERR) device_printf(sc->sc_dev, "Secure transaction error event " "on channel %u\n", ch); if (csr & DMA4_CSR_MISALIGNED_ADRS_ERR) device_printf(sc->sc_dev, "Misaligned address error event " "on channel %u\n", ch); if (csr & DMA4_CSR_TRANS_ERR) { device_printf(sc->sc_dev, "Transaction error event on " "channel %u\n", ch); /* * Apparently according to linux code, there is an errata * that says the channel is not disabled upon this error. * They explicitly disable the channel here .. since I * haven't seen the errata, I'm going to ignore for now. */ } /* Clear the status flags for the IRQ */ ti_sdma_write_4(sc, DMA4_CSR(ch), DMA4_CSR_CLEAR_MASK); ti_sdma_write_4(sc, DMA4_IRQSTATUS_L(j), (1 << ch)); /* Call the callback for the given channel */ if (channel->callback) channel->callback(ch, csr, channel->callback_data); } } } TI_SDMA_UNLOCK(sc); return; } /** * ti_sdma_activate_channel - activates a DMA channel * @ch: upon return contains the channel allocated * @callback: a callback function to associate with the channel * @data: optional data supplied when the callback is called * * Simply activates a channel be enabling and writing default values to the * channel's register set. It doesn't start a transaction, just populates the * internal data structures and sets defaults. * * Note this function doesn't enable interrupts, for that you need to call * ti_sdma_enable_channel_irq(). If not using IRQ to detect the end of the * transfer, you can use ti_sdma_status_poll() to detect a change in the * status. * * A channel must be activated before any of the other DMA functions can be * called on it. * * LOCKING: * DMA registers protected by internal mutex * * RETURNS: * 0 on success, otherwise an error code */ int ti_sdma_activate_channel(unsigned int *ch, void (*callback)(unsigned int ch, uint32_t status, void *data), void *data) { struct ti_sdma_softc *sc = ti_sdma_sc; struct ti_sdma_channel *channel = NULL; uint32_t addr; unsigned int i; /* Sanity check */ if (sc == NULL) return (ENOMEM); if (ch == NULL) return (EINVAL); TI_SDMA_LOCK(sc); /* Check to see if all channels are in use */ if (sc->sc_active_channels == 0xffffffff) { TI_SDMA_UNLOCK(sc); return (ENOMEM); } /* Find the first non-active channel */ for (i = 0; i < NUM_DMA_CHANNELS; i++) { if (!(sc->sc_active_channels & (0x1 << i))) { sc->sc_active_channels |= (0x1 << i); *ch = i; break; } } /* Get the channel struct and populate the fields */ channel = &sc->sc_channel[*ch]; channel->callback = callback; channel->callback_data = data; channel->need_reg_write = 1; /* Set the default configuration for the DMA channel */ channel->reg_csdp = DMA4_CSDP_DATA_TYPE(0x2) | DMA4_CSDP_SRC_BURST_MODE(0) | DMA4_CSDP_DST_BURST_MODE(0) | DMA4_CSDP_SRC_ENDIANISM(0) | DMA4_CSDP_DST_ENDIANISM(0) | DMA4_CSDP_WRITE_MODE(0) | DMA4_CSDP_SRC_PACKED(0) | DMA4_CSDP_DST_PACKED(0); channel->reg_ccr = DMA4_CCR_DST_ADDRESS_MODE(1) | DMA4_CCR_SRC_ADDRESS_MODE(1) | DMA4_CCR_READ_PRIORITY(0) | DMA4_CCR_WRITE_PRIORITY(0) | DMA4_CCR_SYNC_TRIGGER(0) | DMA4_CCR_FRAME_SYNC(0) | DMA4_CCR_BLOCK_SYNC(0); channel->reg_cicr = DMA4_CICR_TRANS_ERR_IE | DMA4_CICR_SECURE_ERR_IE | DMA4_CICR_SUPERVISOR_ERR_IE | DMA4_CICR_MISALIGNED_ADRS_ERR_IE; /* Clear all the channel registers, this should abort any transaction */ for (addr = DMA4_CCR(*ch); addr <= DMA4_COLOR(*ch); addr += 4) ti_sdma_write_4(sc, addr, 0x00000000); TI_SDMA_UNLOCK(sc); return 0; } /** * ti_sdma_deactivate_channel - deactivates a channel * @ch: the channel to deactivate * * * * LOCKING: * DMA registers protected by internal mutex * * RETURNS: * EH_HANDLED or EH_NOT_HANDLED */ int ti_sdma_deactivate_channel(unsigned int ch) { struct ti_sdma_softc *sc = ti_sdma_sc; unsigned int j; unsigned int addr; /* Sanity check */ if (sc == NULL) return (ENOMEM); TI_SDMA_LOCK(sc); /* First check if the channel is currently active */ if ((sc->sc_active_channels & (1 << ch)) == 0) { TI_SDMA_UNLOCK(sc); return (EBUSY); } /* Mark the channel as inactive */ sc->sc_active_channels &= ~(1 << ch); /* Disable all DMA interrupts for the channel. */ ti_sdma_write_4(sc, DMA4_CICR(ch), 0); /* Make sure the DMA transfer is stopped. */ ti_sdma_write_4(sc, DMA4_CCR(ch), 0); /* Clear the CSR register and IRQ status register */ ti_sdma_write_4(sc, DMA4_CSR(ch), DMA4_CSR_CLEAR_MASK); for (j = 0; j < NUM_DMA_IRQS; j++) { ti_sdma_write_4(sc, DMA4_IRQSTATUS_L(j), (1 << ch)); } /* Clear all the channel registers, this should abort any transaction */ for (addr = DMA4_CCR(ch); addr <= DMA4_COLOR(ch); addr += 4) ti_sdma_write_4(sc, addr, 0x00000000); TI_SDMA_UNLOCK(sc); return 0; } /** * ti_sdma_disable_channel_irq - disables IRQ's on the given channel * @ch: the channel to disable IRQ's on * - * Disable interupt generation for the given channel. + * Disable interrupt generation for the given channel. * * LOCKING: * DMA registers protected by internal mutex * * RETURNS: * EH_HANDLED or EH_NOT_HANDLED */ int ti_sdma_disable_channel_irq(unsigned int ch) { struct ti_sdma_softc *sc = ti_sdma_sc; uint32_t irq_enable; unsigned int j; /* Sanity check */ if (sc == NULL) return (ENOMEM); TI_SDMA_LOCK(sc); if ((sc->sc_active_channels & (1 << ch)) == 0) { TI_SDMA_UNLOCK(sc); return (EINVAL); } /* Disable all the individual error conditions */ sc->sc_channel[ch].reg_cicr = 0x0000; ti_sdma_write_4(sc, DMA4_CICR(ch), 0x0000); /* Disable the channel interrupt enable */ for (j = 0; j < NUM_DMA_IRQS; j++) { irq_enable = ti_sdma_read_4(sc, DMA4_IRQENABLE_L(j)); irq_enable &= ~(1 << ch); ti_sdma_write_4(sc, DMA4_IRQENABLE_L(j), irq_enable); } /* Indicate the registers need to be rewritten on the next transaction */ sc->sc_channel[ch].need_reg_write = 1; TI_SDMA_UNLOCK(sc); return (0); } /** * ti_sdma_disable_channel_irq - enables IRQ's on the given channel * @ch: the channel to enable IRQ's on * @flags: bitmask of interrupt types to enable * * Flags can be a bitmask of the following options: * DMA_IRQ_FLAG_DROP * DMA_IRQ_FLAG_HALF_FRAME_COMPL * DMA_IRQ_FLAG_FRAME_COMPL * DMA_IRQ_FLAG_START_LAST_FRAME * DMA_IRQ_FLAG_BLOCK_COMPL * DMA_IRQ_FLAG_ENDOF_PKT * DMA_IRQ_FLAG_DRAIN * * * LOCKING: * DMA registers protected by internal mutex * * RETURNS: * EH_HANDLED or EH_NOT_HANDLED */ int ti_sdma_enable_channel_irq(unsigned int ch, uint32_t flags) { struct ti_sdma_softc *sc = ti_sdma_sc; uint32_t irq_enable; /* Sanity check */ if (sc == NULL) return (ENOMEM); TI_SDMA_LOCK(sc); if ((sc->sc_active_channels & (1 << ch)) == 0) { TI_SDMA_UNLOCK(sc); return (EINVAL); } /* Always enable the error interrupts if we have interrupts enabled */ flags |= DMA4_CICR_TRANS_ERR_IE | DMA4_CICR_SECURE_ERR_IE | DMA4_CICR_SUPERVISOR_ERR_IE | DMA4_CICR_MISALIGNED_ADRS_ERR_IE; sc->sc_channel[ch].reg_cicr = flags; /* Write the values to the register */ ti_sdma_write_4(sc, DMA4_CICR(ch), flags); /* Enable the channel interrupt enable */ irq_enable = ti_sdma_read_4(sc, DMA4_IRQENABLE_L(0)); irq_enable |= (1 << ch); ti_sdma_write_4(sc, DMA4_IRQENABLE_L(0), irq_enable); /* Indicate the registers need to be rewritten on the next transaction */ sc->sc_channel[ch].need_reg_write = 1; TI_SDMA_UNLOCK(sc); return (0); } /** * ti_sdma_get_channel_status - returns the status of a given channel * @ch: the channel number to get the status of * @status: upon return will contain the status bitmask, see below for possible * values. * * DMA_STATUS_DROP * DMA_STATUS_HALF * DMA_STATUS_FRAME * DMA_STATUS_LAST * DMA_STATUS_BLOCK * DMA_STATUS_SYNC * DMA_STATUS_PKT * DMA_STATUS_TRANS_ERR * DMA_STATUS_SECURE_ERR * DMA_STATUS_SUPERVISOR_ERR * DMA_STATUS_MISALIGNED_ADRS_ERR * DMA_STATUS_DRAIN_END * * * LOCKING: * DMA registers protected by internal mutex * * RETURNS: * EH_HANDLED or EH_NOT_HANDLED */ int ti_sdma_get_channel_status(unsigned int ch, uint32_t *status) { struct ti_sdma_softc *sc = ti_sdma_sc; uint32_t csr; /* Sanity check */ if (sc == NULL) return (ENOMEM); TI_SDMA_LOCK(sc); if ((sc->sc_active_channels & (1 << ch)) == 0) { TI_SDMA_UNLOCK(sc); return (EINVAL); } TI_SDMA_UNLOCK(sc); csr = ti_sdma_read_4(sc, DMA4_CSR(ch)); if (status != NULL) *status = csr; return (0); } /** * ti_sdma_start_xfer - starts a DMA transfer - * @ch: the channel number to set the endianess of + * @ch: the channel number to set the endianness of * @src_paddr: the source phsyical address * @dst_paddr: the destination phsyical address * @frmcnt: the number of frames per block * @elmcnt: the number of elements in a frame, an element is either an 8, 16 * or 32-bit value as defined by ti_sdma_set_xfer_burst() * * * LOCKING: * DMA registers protected by internal mutex * * RETURNS: * EH_HANDLED or EH_NOT_HANDLED */ int ti_sdma_start_xfer(unsigned int ch, unsigned int src_paddr, unsigned long dst_paddr, unsigned int frmcnt, unsigned int elmcnt) { struct ti_sdma_softc *sc = ti_sdma_sc; struct ti_sdma_channel *channel; uint32_t ccr; /* Sanity check */ if (sc == NULL) return (ENOMEM); TI_SDMA_LOCK(sc); if ((sc->sc_active_channels & (1 << ch)) == 0) { TI_SDMA_UNLOCK(sc); return (EINVAL); } channel = &sc->sc_channel[ch]; /* a) Write the CSDP register */ ti_sdma_write_4(sc, DMA4_CSDP(ch), channel->reg_csdp | DMA4_CSDP_WRITE_MODE(1)); /* b) Set the number of element per frame CEN[23:0] */ ti_sdma_write_4(sc, DMA4_CEN(ch), elmcnt); /* c) Set the number of frame per block CFN[15:0] */ ti_sdma_write_4(sc, DMA4_CFN(ch), frmcnt); /* d) Set the Source/dest start address index CSSA[31:0]/CDSA[31:0] */ ti_sdma_write_4(sc, DMA4_CSSA(ch), src_paddr); ti_sdma_write_4(sc, DMA4_CDSA(ch), dst_paddr); /* e) Write the CCR register */ ti_sdma_write_4(sc, DMA4_CCR(ch), channel->reg_ccr); /* f) - Set the source element index increment CSEI[15:0] */ ti_sdma_write_4(sc, DMA4_CSE(ch), 0x0001); /* - Set the source frame index increment CSFI[15:0] */ ti_sdma_write_4(sc, DMA4_CSF(ch), 0x0001); /* - Set the destination element index increment CDEI[15:0]*/ ti_sdma_write_4(sc, DMA4_CDE(ch), 0x0001); /* - Set the destination frame index increment CDFI[31:0] */ ti_sdma_write_4(sc, DMA4_CDF(ch), 0x0001); /* Clear the status register */ ti_sdma_write_4(sc, DMA4_CSR(ch), 0x1FFE); /* Write the start-bit and away we go */ ccr = ti_sdma_read_4(sc, DMA4_CCR(ch)); ccr |= (1 << 7); ti_sdma_write_4(sc, DMA4_CCR(ch), ccr); /* Clear the reg write flag */ channel->need_reg_write = 0; TI_SDMA_UNLOCK(sc); return (0); } /** * ti_sdma_start_xfer_packet - starts a packet DMA transfer * @ch: the channel number to use for the transfer * @src_paddr: the source physical address * @dst_paddr: the destination physical address * @frmcnt: the number of frames to transfer * @elmcnt: the number of elements in a frame, an element is either an 8, 16 * or 32-bit value as defined by ti_sdma_set_xfer_burst() * @pktsize: the number of elements in each transfer packet * * The @frmcnt and @elmcnt define the overall number of bytes to transfer, * typically @frmcnt is 1 and @elmcnt contains the total number of elements. * @pktsize is the size of each individual packet, there might be multiple * packets per transfer. i.e. for the following with element size of 32-bits * * frmcnt = 1, elmcnt = 512, pktsize = 128 * * Total transfer bytes = 1 * 512 = 512 elements or 2048 bytes - * Packets transfered = 128 / 512 = 4 + * Packets transferred = 128 / 512 = 4 * * * LOCKING: * DMA registers protected by internal mutex * * RETURNS: * EH_HANDLED or EH_NOT_HANDLED */ int ti_sdma_start_xfer_packet(unsigned int ch, unsigned int src_paddr, unsigned long dst_paddr, unsigned int frmcnt, unsigned int elmcnt, unsigned int pktsize) { struct ti_sdma_softc *sc = ti_sdma_sc; struct ti_sdma_channel *channel; uint32_t ccr; /* Sanity check */ if (sc == NULL) return (ENOMEM); TI_SDMA_LOCK(sc); if ((sc->sc_active_channels & (1 << ch)) == 0) { TI_SDMA_UNLOCK(sc); return (EINVAL); } channel = &sc->sc_channel[ch]; /* a) Write the CSDP register */ if (channel->need_reg_write) ti_sdma_write_4(sc, DMA4_CSDP(ch), channel->reg_csdp | DMA4_CSDP_WRITE_MODE(1)); /* b) Set the number of elements to transfer CEN[23:0] */ ti_sdma_write_4(sc, DMA4_CEN(ch), elmcnt); /* c) Set the number of frames to transfer CFN[15:0] */ ti_sdma_write_4(sc, DMA4_CFN(ch), frmcnt); /* d) Set the Source/dest start address index CSSA[31:0]/CDSA[31:0] */ ti_sdma_write_4(sc, DMA4_CSSA(ch), src_paddr); ti_sdma_write_4(sc, DMA4_CDSA(ch), dst_paddr); /* e) Write the CCR register */ ti_sdma_write_4(sc, DMA4_CCR(ch), channel->reg_ccr | DMA4_CCR_PACKET_TRANS); /* f) - Set the source element index increment CSEI[15:0] */ ti_sdma_write_4(sc, DMA4_CSE(ch), 0x0001); /* - Set the packet size, this is dependent on the sync source */ if (channel->reg_ccr & DMA4_CCR_SEL_SRC_DST_SYNC(1)) ti_sdma_write_4(sc, DMA4_CSF(ch), pktsize); else ti_sdma_write_4(sc, DMA4_CDF(ch), pktsize); /* - Set the destination frame index increment CDFI[31:0] */ ti_sdma_write_4(sc, DMA4_CDE(ch), 0x0001); /* Clear the status register */ ti_sdma_write_4(sc, DMA4_CSR(ch), 0x1FFE); /* Write the start-bit and away we go */ ccr = ti_sdma_read_4(sc, DMA4_CCR(ch)); ccr |= (1 << 7); ti_sdma_write_4(sc, DMA4_CCR(ch), ccr); /* Clear the reg write flag */ channel->need_reg_write = 0; TI_SDMA_UNLOCK(sc); return (0); } /** * ti_sdma_stop_xfer - stops any currently active transfers - * @ch: the channel number to set the endianess of + * @ch: the channel number to set the endianness of * * This function call is effectively a NOP if no transaction is in progress. * * LOCKING: * DMA registers protected by internal mutex * * RETURNS: * EH_HANDLED or EH_NOT_HANDLED */ int ti_sdma_stop_xfer(unsigned int ch) { struct ti_sdma_softc *sc = ti_sdma_sc; unsigned int j; /* Sanity check */ if (sc == NULL) return (ENOMEM); TI_SDMA_LOCK(sc); if ((sc->sc_active_channels & (1 << ch)) == 0) { TI_SDMA_UNLOCK(sc); return (EINVAL); } /* Disable all DMA interrupts for the channel. */ ti_sdma_write_4(sc, DMA4_CICR(ch), 0); /* Make sure the DMA transfer is stopped. */ ti_sdma_write_4(sc, DMA4_CCR(ch), 0); /* Clear the CSR register and IRQ status register */ ti_sdma_write_4(sc, DMA4_CSR(ch), DMA4_CSR_CLEAR_MASK); for (j = 0; j < NUM_DMA_IRQS; j++) { ti_sdma_write_4(sc, DMA4_IRQSTATUS_L(j), (1 << ch)); } /* Configuration registers need to be re-written on the next xfer */ sc->sc_channel[ch].need_reg_write = 1; TI_SDMA_UNLOCK(sc); return (0); } /** - * ti_sdma_set_xfer_endianess - sets the endianess of subsequent transfers - * @ch: the channel number to set the endianess of - * @src: the source endianess (either DMA_ENDIAN_LITTLE or DMA_ENDIAN_BIG) - * @dst: the destination endianess (either DMA_ENDIAN_LITTLE or DMA_ENDIAN_BIG) + * ti_sdma_set_xfer_endianess - sets the endianness of subsequent transfers + * @ch: the channel number to set the endianness of + * @src: the source endianness (either DMA_ENDIAN_LITTLE or DMA_ENDIAN_BIG) + * @dst: the destination endianness (either DMA_ENDIAN_LITTLE or DMA_ENDIAN_BIG) * * * LOCKING: * DMA registers protected by internal mutex * * RETURNS: * EH_HANDLED or EH_NOT_HANDLED */ int ti_sdma_set_xfer_endianess(unsigned int ch, unsigned int src, unsigned int dst) { struct ti_sdma_softc *sc = ti_sdma_sc; /* Sanity check */ if (sc == NULL) return (ENOMEM); TI_SDMA_LOCK(sc); if ((sc->sc_active_channels & (1 << ch)) == 0) { TI_SDMA_UNLOCK(sc); return (EINVAL); } sc->sc_channel[ch].reg_csdp &= ~DMA4_CSDP_SRC_ENDIANISM(1); sc->sc_channel[ch].reg_csdp |= DMA4_CSDP_SRC_ENDIANISM(src); sc->sc_channel[ch].reg_csdp &= ~DMA4_CSDP_DST_ENDIANISM(1); sc->sc_channel[ch].reg_csdp |= DMA4_CSDP_DST_ENDIANISM(dst); sc->sc_channel[ch].need_reg_write = 1; TI_SDMA_UNLOCK(sc); return 0; } /** * ti_sdma_set_xfer_burst - sets the source and destination element size * @ch: the channel number to set the burst settings of - * @src: the source endianess (either DMA_BURST_NONE, DMA_BURST_16, DMA_BURST_32 + * @src: the source endianness (either DMA_BURST_NONE, DMA_BURST_16, DMA_BURST_32 * or DMA_BURST_64) - * @dst: the destination endianess (either DMA_BURST_NONE, DMA_BURST_16, + * @dst: the destination endianness (either DMA_BURST_NONE, DMA_BURST_16, * DMA_BURST_32 or DMA_BURST_64) * * This function sets the size of the elements for all subsequent transfers. * * LOCKING: * DMA registers protected by internal mutex * * RETURNS: * EH_HANDLED or EH_NOT_HANDLED */ int ti_sdma_set_xfer_burst(unsigned int ch, unsigned int src, unsigned int dst) { struct ti_sdma_softc *sc = ti_sdma_sc; /* Sanity check */ if (sc == NULL) return (ENOMEM); TI_SDMA_LOCK(sc); if ((sc->sc_active_channels & (1 << ch)) == 0) { TI_SDMA_UNLOCK(sc); return (EINVAL); } sc->sc_channel[ch].reg_csdp &= ~DMA4_CSDP_SRC_BURST_MODE(0x3); sc->sc_channel[ch].reg_csdp |= DMA4_CSDP_SRC_BURST_MODE(src); sc->sc_channel[ch].reg_csdp &= ~DMA4_CSDP_DST_BURST_MODE(0x3); sc->sc_channel[ch].reg_csdp |= DMA4_CSDP_DST_BURST_MODE(dst); sc->sc_channel[ch].need_reg_write = 1; TI_SDMA_UNLOCK(sc); return 0; } /** * ti_sdma_set_xfer_data_type - driver attach function - * @ch: the channel number to set the endianess of + * @ch: the channel number to set the endianness of * @type: the xfer data type (either DMA_DATA_8BITS_SCALAR, DMA_DATA_16BITS_SCALAR * or DMA_DATA_32BITS_SCALAR) * * * LOCKING: * DMA registers protected by internal mutex * * RETURNS: * EH_HANDLED or EH_NOT_HANDLED */ int ti_sdma_set_xfer_data_type(unsigned int ch, unsigned int type) { struct ti_sdma_softc *sc = ti_sdma_sc; /* Sanity check */ if (sc == NULL) return (ENOMEM); TI_SDMA_LOCK(sc); if ((sc->sc_active_channels & (1 << ch)) == 0) { TI_SDMA_UNLOCK(sc); return (EINVAL); } sc->sc_channel[ch].reg_csdp &= ~DMA4_CSDP_DATA_TYPE(0x3); sc->sc_channel[ch].reg_csdp |= DMA4_CSDP_DATA_TYPE(type); sc->sc_channel[ch].need_reg_write = 1; TI_SDMA_UNLOCK(sc); return 0; } /** * ti_sdma_set_callback - driver attach function * @dev: dma device handle * * * * LOCKING: * DMA registers protected by internal mutex * * RETURNS: * EH_HANDLED or EH_NOT_HANDLED */ int ti_sdma_set_callback(unsigned int ch, void (*callback)(unsigned int ch, uint32_t status, void *data), void *data) { struct ti_sdma_softc *sc = ti_sdma_sc; /* Sanity check */ if (sc == NULL) return (ENOMEM); TI_SDMA_LOCK(sc); if ((sc->sc_active_channels & (1 << ch)) == 0) { TI_SDMA_UNLOCK(sc); return (EINVAL); } sc->sc_channel[ch].callback = callback; sc->sc_channel[ch].callback_data = data; sc->sc_channel[ch].need_reg_write = 1; TI_SDMA_UNLOCK(sc); return 0; } /** * ti_sdma_sync_params - sets channel sync settings * @ch: the channel number to set the sync on * @trigger: the number of the sync trigger, this depends on what other H/W * module is triggering/receiving the DMA transactions * @mode: flags describing the sync mode to use, it may have one or more of * the following bits set; TI_SDMA_SYNC_FRAME, * TI_SDMA_SYNC_BLOCK, TI_SDMA_SYNC_TRIG_ON_SRC. * * * * LOCKING: * DMA registers protected by internal mutex * * RETURNS: * EH_HANDLED or EH_NOT_HANDLED */ int ti_sdma_sync_params(unsigned int ch, unsigned int trigger, unsigned int mode) { struct ti_sdma_softc *sc = ti_sdma_sc; uint32_t ccr; /* Sanity check */ if (sc == NULL) return (ENOMEM); TI_SDMA_LOCK(sc); if ((sc->sc_active_channels & (1 << ch)) == 0) { TI_SDMA_UNLOCK(sc); return (EINVAL); } ccr = sc->sc_channel[ch].reg_ccr; ccr &= ~DMA4_CCR_SYNC_TRIGGER(0x7F); ccr |= DMA4_CCR_SYNC_TRIGGER(trigger + 1); if (mode & TI_SDMA_SYNC_FRAME) ccr |= DMA4_CCR_FRAME_SYNC(1); else ccr &= ~DMA4_CCR_FRAME_SYNC(1); if (mode & TI_SDMA_SYNC_BLOCK) ccr |= DMA4_CCR_BLOCK_SYNC(1); else ccr &= ~DMA4_CCR_BLOCK_SYNC(1); if (mode & TI_SDMA_SYNC_TRIG_ON_SRC) ccr |= DMA4_CCR_SEL_SRC_DST_SYNC(1); else ccr &= ~DMA4_CCR_SEL_SRC_DST_SYNC(1); sc->sc_channel[ch].reg_ccr = ccr; sc->sc_channel[ch].need_reg_write = 1; TI_SDMA_UNLOCK(sc); return 0; } /** * ti_sdma_set_addr_mode - driver attach function - * @ch: the channel number to set the endianess of + * @ch: the channel number to set the endianness of * @rd_mode: the xfer source addressing mode (either DMA_ADDR_CONSTANT, * DMA_ADDR_POST_INCREMENT, DMA_ADDR_SINGLE_INDEX or * DMA_ADDR_DOUBLE_INDEX) * @wr_mode: the xfer destination addressing mode (either DMA_ADDR_CONSTANT, * DMA_ADDR_POST_INCREMENT, DMA_ADDR_SINGLE_INDEX or * DMA_ADDR_DOUBLE_INDEX) * * * LOCKING: * DMA registers protected by internal mutex * * RETURNS: * EH_HANDLED or EH_NOT_HANDLED */ int ti_sdma_set_addr_mode(unsigned int ch, unsigned int src_mode, unsigned int dst_mode) { struct ti_sdma_softc *sc = ti_sdma_sc; uint32_t ccr; /* Sanity check */ if (sc == NULL) return (ENOMEM); TI_SDMA_LOCK(sc); if ((sc->sc_active_channels & (1 << ch)) == 0) { TI_SDMA_UNLOCK(sc); return (EINVAL); } ccr = sc->sc_channel[ch].reg_ccr; ccr &= ~DMA4_CCR_SRC_ADDRESS_MODE(0x3); ccr |= DMA4_CCR_SRC_ADDRESS_MODE(src_mode); ccr &= ~DMA4_CCR_DST_ADDRESS_MODE(0x3); ccr |= DMA4_CCR_DST_ADDRESS_MODE(dst_mode); sc->sc_channel[ch].reg_ccr = ccr; sc->sc_channel[ch].need_reg_write = 1; TI_SDMA_UNLOCK(sc); return 0; } /** * ti_sdma_probe - driver probe function * @dev: dma device handle * * * * RETURNS: * Always returns 0. */ static int ti_sdma_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, "ti,omap4430-sdma")) return (ENXIO); device_set_desc(dev, "TI sDMA Controller"); return (0); } /** * ti_sdma_attach - driver attach function * @dev: dma device handle * * Initialises memory mapping/pointers to the DMA register set and requests * IRQs. This is effectively the setup function for the driver. * * RETURNS: * 0 on success or a negative error code failure. */ static int ti_sdma_attach(device_t dev) { struct ti_sdma_softc *sc = device_get_softc(dev); unsigned int timeout; unsigned int i; int rid; void *ihl; int err; /* Setup the basics */ sc->sc_dev = dev; /* No channels active at the moment */ sc->sc_active_channels = 0x00000000; /* Mutex to protect the shared data structures */ TI_SDMA_LOCK_INIT(sc); /* Get the memory resource for the register mapping */ rid = 0; sc->sc_mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->sc_mem_res == NULL) panic("%s: Cannot map registers", device_get_name(dev)); /* Enable the interface and functional clocks */ ti_prcm_clk_enable(SDMA_CLK); /* Read the sDMA revision register and sanity check it's known */ sc->sc_hw_rev = ti_sdma_read_4(sc, DMA4_REVISION); device_printf(dev, "sDMA revision %08x\n", sc->sc_hw_rev); if (!ti_sdma_is_omap4_rev(sc) && !ti_sdma_is_omap3_rev(sc)) { device_printf(sc->sc_dev, "error - unknown sDMA H/W revision\n"); return (EINVAL); } /* Disable all interrupts */ for (i = 0; i < NUM_DMA_IRQS; i++) { ti_sdma_write_4(sc, DMA4_IRQENABLE_L(i), 0x00000000); } /* Soft-reset is only supported on pre-OMAP44xx devices */ if (ti_sdma_is_omap3_rev(sc)) { /* Soft-reset */ ti_sdma_write_4(sc, DMA4_OCP_SYSCONFIG, 0x0002); /* Set the timeout to 100ms*/ timeout = (hz < 10) ? 1 : ((100 * hz) / 1000); /* Wait for DMA reset to complete */ while ((ti_sdma_read_4(sc, DMA4_SYSSTATUS) & 0x1) == 0x0) { /* Sleep for a tick */ pause("DMARESET", 1); if (timeout-- == 0) { device_printf(sc->sc_dev, "sDMA reset operation timed out\n"); return (EINVAL); } } } /* * Install interrupt handlers for the for possible interrupts. Any channel * can trip one of the four IRQs */ rid = 0; sc->sc_irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE | RF_SHAREABLE); if (sc->sc_irq_res == NULL) panic("Unable to setup the dma irq handler.\n"); err = bus_setup_intr(dev, sc->sc_irq_res, INTR_TYPE_MISC | INTR_MPSAFE, NULL, ti_sdma_intr, NULL, &ihl); if (err) panic("%s: Cannot register IRQ", device_get_name(dev)); /* Store the DMA structure globally ... this driver should never be unloaded */ ti_sdma_sc = sc; return (0); } static device_method_t ti_sdma_methods[] = { DEVMETHOD(device_probe, ti_sdma_probe), DEVMETHOD(device_attach, ti_sdma_attach), {0, 0}, }; static driver_t ti_sdma_driver = { "ti_sdma", ti_sdma_methods, sizeof(struct ti_sdma_softc), }; static devclass_t ti_sdma_devclass; DRIVER_MODULE(ti_sdma, simplebus, ti_sdma_driver, ti_sdma_devclass, 0, 0); MODULE_DEPEND(ti_sdma, ti_prcm, 1, 1, 1); Index: user/ngie/release-pkg-fix-tests/sys/arm/xilinx/zy7_slcr.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/xilinx/zy7_slcr.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/xilinx/zy7_slcr.c (revision 299091) @@ -1,712 +1,712 @@ /*- * Copyright (c) 2013 Thomas Skibo * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ /* * Zynq-700 SLCR driver. Provides hooks for cpu_reset and PL control stuff. * In the future, maybe MIO control, clock control, etc. could go here. * * Reference: Zynq-7000 All Programmable SoC Technical Reference Manual. * (v1.4) November 16, 2012. Xilinx doc UG585. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include struct zy7_slcr_softc { device_t dev; struct mtx sc_mtx; struct resource *mem_res; }; static struct zy7_slcr_softc *zy7_slcr_softc_p; extern void (*zynq7_cpu_reset); #define ZSLCR_LOCK(sc) mtx_lock(&(sc)->sc_mtx) #define ZSLCR_UNLOCK(sc) mtx_unlock(&(sc)->sc_mtx) #define ZSLCR_LOCK_INIT(sc) \ mtx_init(&(sc)->sc_mtx, device_get_nameunit((sc)->dev), \ "zy7_slcr", MTX_DEF) #define ZSLCR_LOCK_DESTROY(_sc) mtx_destroy(&_sc->sc_mtx); #define RD4(sc, off) (bus_read_4((sc)->mem_res, (off))) #define WR4(sc, off, val) (bus_write_4((sc)->mem_res, (off), (val))) #define ZYNQ_DEFAULT_PS_CLK_FREQUENCY 33333333 /* 33.3 Mhz */ SYSCTL_NODE(_hw, OID_AUTO, zynq, CTLFLAG_RD, 0, "Xilinx Zynq-7000"); static char zynq_bootmode[64]; SYSCTL_STRING(_hw_zynq, OID_AUTO, bootmode, CTLFLAG_RD, zynq_bootmode, 0, "Zynq boot mode"); static char zynq_pssid[100]; SYSCTL_STRING(_hw_zynq, OID_AUTO, pssid, CTLFLAG_RD, zynq_pssid, 0, "Zynq PSS IDCODE"); static uint32_t zynq_reboot_status; SYSCTL_INT(_hw_zynq, OID_AUTO, reboot_status, CTLFLAG_RD, &zynq_reboot_status, 0, "Zynq REBOOT_STATUS register"); static int ps_clk_frequency; SYSCTL_INT(_hw_zynq, OID_AUTO, ps_clk_frequency, CTLFLAG_RD, &ps_clk_frequency, 0, "Zynq PS_CLK Frequency"); static int io_pll_frequency; SYSCTL_INT(_hw_zynq, OID_AUTO, io_pll_frequency, CTLFLAG_RD, &io_pll_frequency, 0, "Zynq IO PLL Frequency"); static int arm_pll_frequency; SYSCTL_INT(_hw_zynq, OID_AUTO, arm_pll_frequency, CTLFLAG_RD, &arm_pll_frequency, 0, "Zynq ARM PLL Frequency"); static int ddr_pll_frequency; SYSCTL_INT(_hw_zynq, OID_AUTO, ddr_pll_frequency, CTLFLAG_RD, &ddr_pll_frequency, 0, "Zynq DDR PLL Frequency"); static void zy7_slcr_unlock(struct zy7_slcr_softc *sc) { /* Unlock SLCR with magic number. */ WR4(sc, ZY7_SLCR_UNLOCK, ZY7_SLCR_UNLOCK_MAGIC); } static void zy7_slcr_lock(struct zy7_slcr_softc *sc) { /* Lock SLCR with magic number. */ WR4(sc, ZY7_SLCR_LOCK, ZY7_SLCR_LOCK_MAGIC); } static void zy7_slcr_cpu_reset(void) { struct zy7_slcr_softc *sc = zy7_slcr_softc_p; /* Unlock SLCR registers. */ zy7_slcr_unlock(sc); /* This has something to do with a work-around so the fsbl will load * the bitstream after soft-reboot. It's very important. */ WR4(sc, ZY7_SLCR_REBOOT_STAT, RD4(sc, ZY7_SLCR_REBOOT_STAT) & 0xf0ffffff); /* Soft reset */ WR4(sc, ZY7_SLCR_PSS_RST_CTRL, ZY7_SLCR_PSS_RST_CTRL_SOFT_RESET); for (;;) ; } /* Assert PL resets and disable level shifters in preparation of programming * the PL (FPGA) section. Called from zy7_devcfg.c. */ void zy7_slcr_preload_pl(void) { struct zy7_slcr_softc *sc = zy7_slcr_softc_p; if (!sc) return; ZSLCR_LOCK(sc); /* Unlock SLCR registers. */ zy7_slcr_unlock(sc); /* Assert top level output resets. */ WR4(sc, ZY7_SLCR_FPGA_RST_CTRL, ZY7_SLCR_FPGA_RST_CTRL_RST_ALL); /* Disable all level shifters. */ WR4(sc, ZY7_SLCR_LVL_SHFTR_EN, 0); /* Lock SLCR registers. */ zy7_slcr_lock(sc); ZSLCR_UNLOCK(sc); } /* After PL configuration, enable level shifters and deassert top-level * PL resets. Called from zy7_devcfg.c. Optionally, the level shifters * can be left disabled but that's rare of an FPGA application. That option - * is controled by a sysctl in the devcfg driver. + * is controlled by a sysctl in the devcfg driver. */ void zy7_slcr_postload_pl(int en_level_shifters) { struct zy7_slcr_softc *sc = zy7_slcr_softc_p; if (!sc) return; ZSLCR_LOCK(sc); /* Unlock SLCR registers. */ zy7_slcr_unlock(sc); if (en_level_shifters) /* Enable level shifters. */ WR4(sc, ZY7_SLCR_LVL_SHFTR_EN, ZY7_SLCR_LVL_SHFTR_EN_ALL); /* Deassert top level output resets. */ WR4(sc, ZY7_SLCR_FPGA_RST_CTRL, 0); /* Lock SLCR registers. */ zy7_slcr_lock(sc); ZSLCR_UNLOCK(sc); } /* Override cgem_set_refclk() in gigabit ethernet driver * (sys/dev/cadence/if_cgem.c). This function is called to * request a change in the gem's reference clock speed. */ int cgem_set_ref_clk(int unit, int frequency) { struct zy7_slcr_softc *sc = zy7_slcr_softc_p; int div0, div1; if (!sc) return (-1); /* Find suitable divisor pairs. Round result to nearest khz * to test for match. */ for (div1 = 1; div1 <= ZY7_SLCR_GEM_CLK_CTRL_DIVISOR1_MAX; div1++) { div0 = (io_pll_frequency + div1 * frequency / 2) / div1 / frequency; if (div0 > 0 && div0 <= ZY7_SLCR_GEM_CLK_CTRL_DIVISOR_MAX && ((io_pll_frequency / div0 / div1) + 500) / 1000 == (frequency + 500) / 1000) break; } if (div1 > ZY7_SLCR_GEM_CLK_CTRL_DIVISOR1_MAX) return (-1); ZSLCR_LOCK(sc); /* Unlock SLCR registers. */ zy7_slcr_unlock(sc); /* Modify GEM reference clock. */ WR4(sc, unit ? ZY7_SLCR_GEM1_CLK_CTRL : ZY7_SLCR_GEM0_CLK_CTRL, (div1 << ZY7_SLCR_GEM_CLK_CTRL_DIVISOR1_SHIFT) | (div0 << ZY7_SLCR_GEM_CLK_CTRL_DIVISOR_SHIFT) | ZY7_SLCR_GEM_CLK_CTRL_SRCSEL_IO_PLL | ZY7_SLCR_GEM_CLK_CTRL_CLKACT); /* Lock SLCR registers. */ zy7_slcr_lock(sc); ZSLCR_UNLOCK(sc); return (0); } /* * PL clocks management function */ int zy7_pl_fclk_set_source(int unit, int source) { struct zy7_slcr_softc *sc = zy7_slcr_softc_p; uint32_t reg; if (!sc) return (-1); ZSLCR_LOCK(sc); /* Unlock SLCR registers. */ zy7_slcr_unlock(sc); /* Modify FPGAx source. */ reg = RD4(sc, ZY7_SLCR_FPGA_CLK_CTRL(unit)); reg &= ~(ZY7_SLCR_FPGA_CLK_CTRL_SRCSEL_MASK); reg |= (source << ZY7_SLCR_FPGA_CLK_CTRL_SRCSEL_SHIFT); WR4(sc, ZY7_SLCR_FPGA_CLK_CTRL(unit), reg); /* Lock SLCR registers. */ zy7_slcr_lock(sc); ZSLCR_UNLOCK(sc); return (0); } int zy7_pl_fclk_get_source(int unit) { struct zy7_slcr_softc *sc = zy7_slcr_softc_p; uint32_t reg; int source; if (!sc) return (-1); ZSLCR_LOCK(sc); /* Modify GEM reference clock. */ reg = RD4(sc, ZY7_SLCR_FPGA_CLK_CTRL(unit)); source = (reg & ZY7_SLCR_FPGA_CLK_CTRL_SRCSEL_MASK) >> ZY7_SLCR_FPGA_CLK_CTRL_SRCSEL_SHIFT; /* ZY7_PL_FCLK_SRC_IO is actually b0x */ if ((source & 2) == 0) source = ZY7_PL_FCLK_SRC_IO; ZSLCR_UNLOCK(sc); return (source); } int zy7_pl_fclk_set_freq(int unit, int frequency) { struct zy7_slcr_softc *sc = zy7_slcr_softc_p; int div0, div1; int base_frequency; uint32_t reg; int source; if (!sc) return (-1); source = zy7_pl_fclk_get_source(unit); switch (source) { case ZY7_PL_FCLK_SRC_IO: base_frequency = io_pll_frequency; break; case ZY7_PL_FCLK_SRC_ARM: base_frequency = arm_pll_frequency; break; case ZY7_PL_FCLK_SRC_DDR: base_frequency = ddr_pll_frequency; break; default: return (-1); } /* Find suitable divisor pairs. Round result to nearest khz * to test for match. */ for (div1 = 1; div1 <= ZY7_SLCR_FPGA_CLK_CTRL_DIVISOR_MAX; div1++) { div0 = (base_frequency + div1 * frequency / 2) / div1 / frequency; if (div0 > 0 && div0 <= ZY7_SLCR_FPGA_CLK_CTRL_DIVISOR_MAX && ((base_frequency / div0 / div1) + 500) / 1000 == (frequency + 500) / 1000) break; } if (div1 > ZY7_SLCR_FPGA_CLK_CTRL_DIVISOR_MAX) return (-1); ZSLCR_LOCK(sc); /* Unlock SLCR registers. */ zy7_slcr_unlock(sc); /* Modify FPGAx reference clock. */ reg = RD4(sc, ZY7_SLCR_FPGA_CLK_CTRL(unit)); reg &= ~(ZY7_SLCR_FPGA_CLK_CTRL_DIVISOR1_MASK | ZY7_SLCR_FPGA_CLK_CTRL_DIVISOR0_MASK); reg |= (div1 << ZY7_SLCR_FPGA_CLK_CTRL_DIVISOR1_SHIFT) | (div0 << ZY7_SLCR_FPGA_CLK_CTRL_DIVISOR0_SHIFT); WR4(sc, ZY7_SLCR_FPGA_CLK_CTRL(unit), reg); /* Lock SLCR registers. */ zy7_slcr_lock(sc); ZSLCR_UNLOCK(sc); return (base_frequency / div0 / div1); } int zy7_pl_fclk_get_freq(int unit) { struct zy7_slcr_softc *sc = zy7_slcr_softc_p; int div0, div1; int base_frequency; int frequency; uint32_t reg; int source; if (!sc) return (-1); source = zy7_pl_fclk_get_source(unit); switch (source) { case ZY7_PL_FCLK_SRC_IO: base_frequency = io_pll_frequency; break; case ZY7_PL_FCLK_SRC_ARM: base_frequency = arm_pll_frequency; break; case ZY7_PL_FCLK_SRC_DDR: base_frequency = ddr_pll_frequency; break; default: return (-1); } ZSLCR_LOCK(sc); /* Modify FPGAx reference clock. */ reg = RD4(sc, ZY7_SLCR_FPGA_CLK_CTRL(unit)); div1 = (reg & ZY7_SLCR_FPGA_CLK_CTRL_DIVISOR1_MASK) >> ZY7_SLCR_FPGA_CLK_CTRL_DIVISOR1_SHIFT; div0 = (reg & ZY7_SLCR_FPGA_CLK_CTRL_DIVISOR0_MASK) >> ZY7_SLCR_FPGA_CLK_CTRL_DIVISOR0_SHIFT; ZSLCR_UNLOCK(sc); if (div0 == 0) div0 = 1; if (div1 == 0) div1 = 1; frequency = (base_frequency / div0 / div1); /* Round to KHz */ frequency = (frequency + 500) / 1000; frequency = frequency * 1000; return (frequency); } int zy7_pl_fclk_enable(int unit) { struct zy7_slcr_softc *sc = zy7_slcr_softc_p; if (!sc) return (-1); ZSLCR_LOCK(sc); /* Unlock SLCR registers. */ zy7_slcr_unlock(sc); WR4(sc, ZY7_SLCR_FPGA_THR_CTRL(unit), 0); WR4(sc, ZY7_SLCR_FPGA_THR_CNT(unit), 0); /* Lock SLCR registers. */ zy7_slcr_lock(sc); ZSLCR_UNLOCK(sc); return (0); } int zy7_pl_fclk_disable(int unit) { struct zy7_slcr_softc *sc = zy7_slcr_softc_p; if (!sc) return (-1); ZSLCR_LOCK(sc); /* Unlock SLCR registers. */ zy7_slcr_unlock(sc); WR4(sc, ZY7_SLCR_FPGA_THR_CTRL(unit), 0); WR4(sc, ZY7_SLCR_FPGA_THR_CNT(unit), 1); /* Lock SLCR registers. */ zy7_slcr_lock(sc); ZSLCR_UNLOCK(sc); return (0); } int zy7_pl_fclk_enabled(int unit) { struct zy7_slcr_softc *sc = zy7_slcr_softc_p; uint32_t reg; if (!sc) return (-1); ZSLCR_LOCK(sc); reg = RD4(sc, ZY7_SLCR_FPGA_THR_CNT(unit)); ZSLCR_UNLOCK(sc); return !(reg & 1); } int zy7_pl_level_shifters_enabled() { struct zy7_slcr_softc *sc = zy7_slcr_softc_p; uint32_t reg; if (!sc) return (-1); ZSLCR_LOCK(sc); reg = RD4(sc, ZY7_SLCR_LVL_SHFTR_EN); ZSLCR_UNLOCK(sc); return (reg == ZY7_SLCR_LVL_SHFTR_EN_ALL); } void zy7_pl_level_shifters_enable() { struct zy7_slcr_softc *sc = zy7_slcr_softc_p; if (!sc) return; ZSLCR_LOCK(sc); zy7_slcr_unlock(sc); WR4(sc, ZY7_SLCR_LVL_SHFTR_EN, ZY7_SLCR_LVL_SHFTR_EN_ALL); zy7_slcr_lock(sc); ZSLCR_UNLOCK(sc); } void zy7_pl_level_shifters_disable() { struct zy7_slcr_softc *sc = zy7_slcr_softc_p; if (!sc) return; ZSLCR_LOCK(sc); zy7_slcr_unlock(sc); WR4(sc, ZY7_SLCR_LVL_SHFTR_EN, 0); zy7_slcr_lock(sc); ZSLCR_UNLOCK(sc); } static int zy7_slcr_probe(device_t dev) { if (!ofw_bus_status_okay(dev)) return (ENXIO); if (!ofw_bus_is_compatible(dev, "xlnx,zy7_slcr")) return (ENXIO); device_set_desc(dev, "Zynq-7000 slcr block"); return (0); } static int zy7_slcr_attach(device_t dev) { struct zy7_slcr_softc *sc = device_get_softc(dev); int rid; phandle_t node; pcell_t cell; uint32_t bootmode; uint32_t pss_idcode; uint32_t arm_pll_ctrl; uint32_t ddr_pll_ctrl; uint32_t io_pll_ctrl; static char *bootdev_names[] = { "JTAG", "Quad-SPI", "NOR", "(3?)", "NAND", "SD Card", "(6?)", "(7?)" }; /* Allow only one attach. */ if (zy7_slcr_softc_p != NULL) return (ENXIO); sc->dev = dev; ZSLCR_LOCK_INIT(sc); /* Get memory resource. */ rid = 0; sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->mem_res == NULL) { device_printf(dev, "could not allocate memory resources.\n"); return (ENOMEM); } /* Hook up cpu_reset. */ zy7_slcr_softc_p = sc; zynq7_cpu_reset = zy7_slcr_cpu_reset; /* Read info and set sysctls. */ bootmode = RD4(sc, ZY7_SLCR_BOOT_MODE); snprintf(zynq_bootmode, sizeof(zynq_bootmode), "0x%x: boot device: %s", bootmode, bootdev_names[bootmode & ZY7_SLCR_BOOT_MODE_BOOTDEV_MASK]); pss_idcode = RD4(sc, ZY7_SLCR_PSS_IDCODE); snprintf(zynq_pssid, sizeof(zynq_pssid), "0x%x: manufacturer: 0x%x device: 0x%x " "family: 0x%x sub-family: 0x%x rev: 0x%x", pss_idcode, (pss_idcode & ZY7_SLCR_PSS_IDCODE_MNFR_ID_MASK) >> ZY7_SLCR_PSS_IDCODE_MNFR_ID_SHIFT, (pss_idcode & ZY7_SLCR_PSS_IDCODE_DEVICE_MASK) >> ZY7_SLCR_PSS_IDCODE_DEVICE_SHIFT, (pss_idcode & ZY7_SLCR_PSS_IDCODE_FAMILY_MASK) >> ZY7_SLCR_PSS_IDCODE_FAMILY_SHIFT, (pss_idcode & ZY7_SLCR_PSS_IDCODE_SUB_FAMILY_MASK) >> ZY7_SLCR_PSS_IDCODE_SUB_FAMILY_SHIFT, (pss_idcode & ZY7_SLCR_PSS_IDCODE_REVISION_MASK) >> ZY7_SLCR_PSS_IDCODE_REVISION_SHIFT); zynq_reboot_status = RD4(sc, ZY7_SLCR_REBOOT_STAT); /* Derive PLL frequencies from PS_CLK. */ node = ofw_bus_get_node(dev); if (OF_getprop(node, "clock-frequency", &cell, sizeof(cell)) > 0) ps_clk_frequency = fdt32_to_cpu(cell); else ps_clk_frequency = ZYNQ_DEFAULT_PS_CLK_FREQUENCY; arm_pll_ctrl = RD4(sc, ZY7_SLCR_ARM_PLL_CTRL); ddr_pll_ctrl = RD4(sc, ZY7_SLCR_DDR_PLL_CTRL); io_pll_ctrl = RD4(sc, ZY7_SLCR_IO_PLL_CTRL); /* Determine ARM PLL frequency. */ if (((arm_pll_ctrl & ZY7_SLCR_PLL_CTRL_BYPASS_QUAL) == 0 && (arm_pll_ctrl & ZY7_SLCR_PLL_CTRL_BYPASS_FORCE) != 0) || ((arm_pll_ctrl & ZY7_SLCR_PLL_CTRL_BYPASS_QUAL) != 0 && (bootmode & ZY7_SLCR_BOOT_MODE_PLL_BYPASS) != 0)) /* PLL is bypassed. */ arm_pll_frequency = ps_clk_frequency; else arm_pll_frequency = ps_clk_frequency * ((arm_pll_ctrl & ZY7_SLCR_PLL_CTRL_FDIV_MASK) >> ZY7_SLCR_PLL_CTRL_FDIV_SHIFT); /* Determine DDR PLL frequency. */ if (((ddr_pll_ctrl & ZY7_SLCR_PLL_CTRL_BYPASS_QUAL) == 0 && (ddr_pll_ctrl & ZY7_SLCR_PLL_CTRL_BYPASS_FORCE) != 0) || ((ddr_pll_ctrl & ZY7_SLCR_PLL_CTRL_BYPASS_QUAL) != 0 && (bootmode & ZY7_SLCR_BOOT_MODE_PLL_BYPASS) != 0)) /* PLL is bypassed. */ ddr_pll_frequency = ps_clk_frequency; else ddr_pll_frequency = ps_clk_frequency * ((ddr_pll_ctrl & ZY7_SLCR_PLL_CTRL_FDIV_MASK) >> ZY7_SLCR_PLL_CTRL_FDIV_SHIFT); /* Determine IO PLL frequency. */ if (((io_pll_ctrl & ZY7_SLCR_PLL_CTRL_BYPASS_QUAL) == 0 && (io_pll_ctrl & ZY7_SLCR_PLL_CTRL_BYPASS_FORCE) != 0) || ((io_pll_ctrl & ZY7_SLCR_PLL_CTRL_BYPASS_QUAL) != 0 && (bootmode & ZY7_SLCR_BOOT_MODE_PLL_BYPASS) != 0)) /* PLL is bypassed. */ io_pll_frequency = ps_clk_frequency; else io_pll_frequency = ps_clk_frequency * ((io_pll_ctrl & ZY7_SLCR_PLL_CTRL_FDIV_MASK) >> ZY7_SLCR_PLL_CTRL_FDIV_SHIFT); /* Lock SLCR registers. */ zy7_slcr_lock(sc); return (0); } static int zy7_slcr_detach(device_t dev) { struct zy7_slcr_softc *sc = device_get_softc(dev); bus_generic_detach(dev); /* Release memory resource. */ if (sc->mem_res != NULL) bus_release_resource(dev, SYS_RES_MEMORY, rman_get_rid(sc->mem_res), sc->mem_res); zy7_slcr_softc_p = NULL; zynq7_cpu_reset = NULL; ZSLCR_LOCK_DESTROY(sc); return (0); } static device_method_t zy7_slcr_methods[] = { /* device_if */ DEVMETHOD(device_probe, zy7_slcr_probe), DEVMETHOD(device_attach, zy7_slcr_attach), DEVMETHOD(device_detach, zy7_slcr_detach), DEVMETHOD_END }; static driver_t zy7_slcr_driver = { "zy7_slcr", zy7_slcr_methods, sizeof(struct zy7_slcr_softc), }; static devclass_t zy7_slcr_devclass; DRIVER_MODULE(zy7_slcr, simplebus, zy7_slcr_driver, zy7_slcr_devclass, 0, 0); MODULE_VERSION(zy7_slcr, 1); Index: user/ngie/release-pkg-fix-tests/sys/arm/xscale/i8134x/crb_machdep.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/xscale/i8134x/crb_machdep.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/xscale/i8134x/crb_machdep.c (revision 299091) @@ -1,331 +1,331 @@ /* $NetBSD: hpc_machdep.c,v 1.70 2003/09/16 08:18:22 agc Exp $ */ /*- * Copyright (c) 1994-1998 Mark Brinicombe. * Copyright (c) 1994 Brini. * All rights reserved. * * This code is derived from software written for Brini by Mark Brinicombe * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Brini. * 4. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * RiscBSD kernel project * * machdep.c * - * Machine dependant functions for kernel setup + * Machine dependent functions for kernel setup * * This file needs a lot of work. * * Created : 17/09/94 */ #include __FBSDID("$FreeBSD$"); #include "opt_kstack_pages.h" #define _ARM32_BUS_DMA_PRIVATE #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* For i80321_calibrate_delay() */ #include #include #include #define KERNEL_PT_SYS 0 /* Page table for mapping proc0 zero page */ #define KERNEL_PT_IOPXS 1 #define KERNEL_PT_BEFOREKERN 2 #define KERNEL_PT_AFKERNEL 3 /* L2 table for mapping after kernel */ #define KERNEL_PT_AFKERNEL_NUM 9 /* this should be evenly divisable by PAGE_SIZE / L2_TABLE_SIZE_REAL (or 4) */ #define NUM_KERNEL_PTS (KERNEL_PT_AFKERNEL + KERNEL_PT_AFKERNEL_NUM) struct pv_addr kernel_pt_table[NUM_KERNEL_PTS]; /* Physical and virtual addresses for some global pages */ struct pv_addr systempage; struct pv_addr msgbufpv; struct pv_addr irqstack; struct pv_addr undstack; struct pv_addr abtstack; struct pv_addr kernelstack; /* Static device mappings. */ static const struct devmap_entry iq81342_devmap[] = { { IOP34X_VADDR, IOP34X_HWADDR, IOP34X_SIZE, }, { /* * Cheat and map a whole section, this will bring * both PCI-X and PCI-E outbound I/O */ rounddown2(IOP34X_PCIX_OIOBAR_VADDR, 0x100000), rounddown2(IOP34X_PCIX_OIOBAR, 0x100000), 0x100000, }, { IOP34X_PCE1_VADDR, IOP34X_PCE1, IOP34X_PCE1_SIZE, }, { 0, 0, 0, } }; #define SDRAM_START 0x00000000 extern vm_offset_t xscale_cache_clean_addr; void * initarm(struct arm_boot_params *abp) { struct pv_addr kernel_l1pt; struct pv_addr dpcpu; int loop, i; u_int l1pagetable; vm_offset_t freemempos; vm_offset_t freemem_pt; vm_offset_t afterkern; vm_offset_t freemem_after; vm_offset_t lastaddr; uint32_t memsize, memstart; lastaddr = parse_boot_param(abp); arm_physmem_kernaddr = abp->abp_physaddr; set_cpufuncs(); pcpu_init(pcpup, 0, sizeof(struct pcpu)); PCPU_SET(curthread, &thread0); /* Do basic tuning, hz etc */ init_param1(); freemempos = 0x00200000; /* Define a macro to simplify memory allocation */ #define valloc_pages(var, np) \ alloc_pages((var).pv_pa, (np)); \ (var).pv_va = (var).pv_pa + 0xc0000000; #define alloc_pages(var, np) \ freemempos -= (np * PAGE_SIZE); \ (var) = freemempos; \ memset((char *)(var), 0, ((np) * PAGE_SIZE)); while (((freemempos - L1_TABLE_SIZE) & (L1_TABLE_SIZE - 1)) != 0) freemempos -= PAGE_SIZE; valloc_pages(kernel_l1pt, L1_TABLE_SIZE / PAGE_SIZE); for (loop = 0; loop < NUM_KERNEL_PTS; ++loop) { if (!(loop % (PAGE_SIZE / L2_TABLE_SIZE_REAL))) { valloc_pages(kernel_pt_table[loop], L2_TABLE_SIZE / PAGE_SIZE); } else { kernel_pt_table[loop].pv_pa = freemempos + (loop % (PAGE_SIZE / L2_TABLE_SIZE_REAL)) * L2_TABLE_SIZE_REAL; kernel_pt_table[loop].pv_va = kernel_pt_table[loop].pv_pa + 0xc0000000; } } freemem_pt = freemempos; freemempos = 0x00100000; /* * Allocate a page for the system page mapped to V0x00000000 * This page will just contain the system vectors and can be * shared by all processes. */ valloc_pages(systempage, 1); /* Allocate dynamic per-cpu area. */ valloc_pages(dpcpu, DPCPU_SIZE / PAGE_SIZE); dpcpu_init((void *)dpcpu.pv_va, 0); /* Allocate stacks for all modes */ valloc_pages(irqstack, IRQ_STACK_SIZE); valloc_pages(abtstack, ABT_STACK_SIZE); valloc_pages(undstack, UND_STACK_SIZE); valloc_pages(kernelstack, kstack_pages); valloc_pages(msgbufpv, round_page(msgbufsize) / PAGE_SIZE); /* * Now we start construction of the L1 page table * We start by mapping the L2 page tables into the L1. * This means that we can replace L1 mappings later on if necessary */ l1pagetable = kernel_l1pt.pv_va; /* Map the L2 pages tables in the L1 page table */ pmap_link_l2pt(l1pagetable, rounddown2(ARM_VECTORS_HIGH, 0x00100000), &kernel_pt_table[KERNEL_PT_SYS]); pmap_map_chunk(l1pagetable, KERNBASE, SDRAM_START, 0x100000, VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); pmap_map_chunk(l1pagetable, KERNBASE + 0x100000, SDRAM_START + 0x100000, 0x100000, VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE); pmap_map_chunk(l1pagetable, KERNBASE + 0x200000, SDRAM_START + 0x200000, rounddown2(((uint32_t)(lastaddr) - KERNBASE - 0x200000) + L1_S_SIZE, L1_S_SIZE), VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); freemem_after = rounddown2((int)lastaddr + PAGE_SIZE, PAGE_SIZE); afterkern = round_page(rounddown2((vm_offset_t)lastaddr + L1_S_SIZE, L1_S_SIZE)); for (i = 0; i < KERNEL_PT_AFKERNEL_NUM; i++) { pmap_link_l2pt(l1pagetable, afterkern + i * 0x00100000, &kernel_pt_table[KERNEL_PT_AFKERNEL + i]); } /* Map the vector page. */ pmap_map_entry(l1pagetable, ARM_VECTORS_HIGH, systempage.pv_pa, VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); devmap_bootstrap(l1pagetable, iq81342_devmap); /* * Give the XScale global cache clean code an appropriately * sized chunk of unmapped VA space starting at 0xff000000 * (our device mappings end before this address). */ xscale_cache_clean_addr = 0xff000000U; cpu_domains((DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL*2)) | DOMAIN_CLIENT); cpu_setttb(kernel_l1pt.pv_pa); cpu_tlb_flushID(); cpu_domains(DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL*2)); /* * Pages were allocated during the secondary bootstrap for the * stacks for different CPU modes. * We must now set the r13 registers in the different CPU modes to * point to these stacks. * Since the ARM stacks use STMFD etc. we must set r13 to the top end * of the stack memory. */ set_stackptrs(0); /* * We must now clean the cache again.... * Cleaning may be done by reading new data to displace any * dirty data in the cache. This will have happened in cpu_setttb() * but since we are boot strapping the addresses used for the read * may have just been remapped and thus the cache could be out * of sync. A re-clean after the switch will cure this. * After booting there are no gross relocations of the kernel thus * this problem will not occur after initarm(). */ cpu_idcache_wbinv_all(); cpu_setup(); i80321_calibrate_delay(); i81342_sdram_bounds(obio_bs_tag, IOP34X_VADDR, &memstart, &memsize); physmem = memsize / PAGE_SIZE; cninit(); /* Set stack for exception handlers */ undefined_init(); init_proc0(kernelstack.pv_va); arm_vector_init(ARM_VECTORS_HIGH, ARM_VEC_ALL); pmap_curmaxkvaddr = afterkern + PAGE_SIZE; vm_max_kernel_address = 0xe0000000; pmap_bootstrap(pmap_curmaxkvaddr, &kernel_l1pt); msgbufp = (void*)msgbufpv.pv_va; msgbufinit(msgbufp, msgbufsize); mutex_init(); /* * Add the physical ram we have available. * * Exclude the kernel (and all the things we allocated which immediately * follow the kernel) from the VM allocation pool but not from crash * dumps. virtual_avail is a global variable which tracks the kva we've * "allocated" while setting up pmaps. * * Prepare the list of physical memory available to the vm subsystem. */ arm_physmem_hardware_region(SDRAM_START, memsize); arm_physmem_exclude_region(freemem_pt, abp->abp_physaddr - freemem_pt, EXFLAG_NOALLOC); arm_physmem_exclude_region(freemempos, abp->abp_physaddr - 0x100000 - freemempos, EXFLAG_NOALLOC); arm_physmem_exclude_region(abp->abp_physaddr, virtual_avail - KERNVIRTADDR, EXFLAG_NOALLOC); arm_physmem_init_kernel_globals(); init_param2(physmem); kdb_init(); return ((void *)(kernelstack.pv_va + USPACE_SVC_STACK_TOP - sizeof(struct pcb))); } Index: user/ngie/release-pkg-fix-tests/sys/arm/xscale/i8134x/i81342reg.h =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/xscale/i8134x/i81342reg.h (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/xscale/i8134x/i81342reg.h (revision 299091) @@ -1,348 +1,348 @@ /*- * Copyright (c) 2006 Olivier Houchard * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ /* $FreeBSD$ */ #ifndef I83142_REG_H_ #define I83142_REG_H_ /* Physical Memory Map */ /* * 0x000000000 - 0x07FFFFFFF SDRAM * 0x090100000 - 0x0901FFFFF ATUe Outbound IO Window * 0x0F0000000 - 0x0F1FFFFFF Flash * 0x0F2000000 - 0x0F20FFFFF PCE1 * 0x0F3000000 - 0x0FFCFFFFF Compact Flash * 0x0FFD00000 - 0x0FFDFFFFF MMR * 0x0FFFB0000 - 0x0FFFBFFFF ATU-X Outbound I/O Window * 0x0FFFD0000 - 0x0FFFDFFFF ATUe Outbound I/O Window * 0x100000000 - 0x1FFFFFFFF ATU-X outbound Memory Translation Window * 0x2FF000000 - 0x2FFFFFFFF ATUe Outbound Memory Translation Window */ #define IOP34X_VADDR 0xf0000000 #define IOP34X_HWADDR 0xffd00000 #define IOP34X_SIZE 0x100000 #define IOP34X_ADMA0_OFFSET 0x00080000 #define IOP34X_ADMA1_OFFSET 0x00080200 #define IOP34X_ADMA2_OFFSET 0x00080400 #define IOP34X_ADMA_SIZE 0x200 /* ADMA Registers */ #define IOP34X_ADMA_CCR 0x0000 /* Channel Control Register */ #define IOP34X_ADMA_CSR 0x0004 /* Channel Status Register */ #define IOP34X_ADMA_DAR 0x0008 /* Descriptor Address Register */ #define IOP34X_ADMA_IPCR 0x0018 /* Internal Interface Parity Ctrl Reg */ #define IOP34X_ADMA_NDAR 0x0024 /* Next Descriptor Register */ #define IOP34X_ADMA_DCR 0x0028 /* Descriptor Control Register */ #define IOP34X_ADMA_IE (1 << 0) /* Interrupt enable */ #define IOP34X_ADMA_TR (1 << 1) /* Transfert Direction */ /* * Source Destination * 00 Host I/O Interface Local Memory * 01 Local Memory Host I/O Interface * 10 Internal Bus Local Memory * 11 Local Memory Internal Bus */ #define IOP34X_ADMA_SS (1 << 3) /* Source selection */ /* 0000: Data Transfer / CRC / Memory Block Fill */ #define IOP34X_ADMA_ZRBCE (1 << 7) /* Zero Result Buffer Check Enable */ #define IOP34X_ADMA_MBFE (1 << 8) /* Memory Block Fill Enable */ #define IOP34X_ADMA_CGE (1 << 9) /* CRC Generation enable */ #define IOP34X_ADMA_CTD (1 << 10) /* CRC Transfer disable */ #define IOP34X_ADMA_CSFD (1 << 11) /* CRC Seed fetch disable */ #define IOP34X_ADMA_SWBE (1 << 12) /* Status write back enable */ #define IOP34X_ADMA_ESE (1 << 13) /* Endian swap enable */ #define IOP34X_ADMA_PQUTE (1 << 16) /* P+Q Update Transfer Enable */ #define IOP34X_ADMA_DXE (1 << 17) /* Dual XOR Enable */ #define IOP34X_ADMA_PQTE (1 << 18) /* P+Q Transfer Enable */ #define IOP34X_ADMA_PTD (1 << 19) /* P Transfer Disable */ #define IOP34X_ADMA_ROE (1 << 30) /* Relaxed Ordering Enable */ #define IOP34X_ADMA_NSE (1U << 31) /* No Snoop Enable */ #define IOP34X_PBBAR0 0x81588 /* PBI Base Address Register 0 */ #define IOP34X_PBBAR0_ADDRMASK 0xfffff000 #define IOP34X_PBBAR1 0x81590 #define IOP34X_PCE1 0xF2000000 #define IOP34X_PCE1_SIZE 0x00100000 #define IOP34X_PCE1_VADDR 0xF1000000 #define IOP34X_ESSTSR0 0x82188 #define IOP34X_CONTROLLER_ONLY (1 << 14) #define IOP34X_INT_SEL_PCIX (1 << 15) #define IOP34X_PFR 0x82180 /* Processor Frequency Register */ #define IOP34X_FREQ_MASK ((1 << 16) | (1 << 17) | (1 << 18)) #define IOP34X_FREQ_600 (0) #define IOP34X_FREQ_667 (1 << 16) #define IOP34X_FREQ_800 (1 << 17) #define IOP34X_FREQ_833 ((1 << 17) | (1 << 16)) #define IOP34X_FREQ_1000 (1 << 18) #define IOP34X_FREQ_1200 ((1 << 16) | (1 << 18)) #define IOP34X_UART0_VADDR IOP34X_VADDR + 0x82300 #define IOP34X_UART0_HWADDR IOP34X_HWADDR + 0x82300 #define IOP34X_UART1_VADDR IOP34X_VADDR + 0x82340 #define IOP34X_UART1_HWADDR IOP34X_HWADDR + 0x82340 #define IOP34X_PBI_HWADDR 0xffd81580 /* SDRAM Memory Controller */ #define SMC_SDBR 0x8180c /* Base Register */ #define SMC_SDBR_BASEADDR (1 << 27) #define SMC_SDBR_BASEADDR_MASK ((1 << 27) | (1 << 28) | (1 << 29) | (1 << 30) \ | (1U << 31)) #define SMC_SDUBR 0x81810 /* Upper Base Register */ #define SMC_SBSR 0x81814 /* SDRAM Bank Size Register */ #define SMC_SBSR_BANK_NB (1 << 2) /* Number of DDR Banks 0 => 2 Banks 1 => 1 Bank */ #define SMC_SBSR_BANK_SZ (1 << 27) /* SDRAM Bank Size : 0x00000 Empty 0x00001 128MB 0x00010 256MB 0x00100 512MB 0x01000 1GB */ #define SMC_SBSR_BANK_SZ_MASK ((1 << 27) | (1 << 28) | (1 << 29) | (1 << 30) \ | (1U << 31)) /* Two possible addresses for ATUe depending on configuration. */ #define IOP34X_ATUE_ADDR(esstrsr) ((((esstrsr) & (IOP34X_CONTROLLER_ONLY | \ IOP34X_INT_SEL_PCIX)) == (IOP34X_CONTROLLER_ONLY | IOP34X_INT_SEL_PCIX)) ? \ 0xffdc8000 : 0xffdcd000) /* Three possible addresses for ATU-X depending on configuration. */ #define IOP34X_ATUX_ADDR(esstrsr) (!((esstrsr) & IOP34X_CONTROLLER_ONLY) ? \ 0xffdcc000 : !((esstrsr) & IOP34X_INT_SEL_PCIX) ? 0xffdc8000 : 0xffdcd000) #define IOP34X_OIOBAR_SIZE 0x10000 #define IOP34X_PCIX_OIOBAR 0xfffb0000 #define IOP34X_PCIX_OIOBAR_VADDR 0xf01b0000 #define IOP34X_PCIX_OMBAR 0x100000000 #define IOP34X_PCIE_OIOBAR 0xfffd0000 #define IOP34X_PCIE_OIOBAR_VADDR 0xf01d0000 #define IOP34X_PCIE_OMBAR 0x200000000 /* ATU Registers */ /* Common for ATU-X and ATUe */ #define ATU_VID 0x0000 /* ATU Vendor ID */ #define ATU_DID 0x0002 /* ATU Device ID */ #define ATU_CMD 0x0004 /* ATU Command Register */ #define ATU_SR 0x0006 /* ATU Status Register */ #define ATU_RID 0x0008 /* ATU Revision ID */ #define ATU_CCR 0x0009 /* ATU Class Code */ #define ATU_CLSR 0x000c /* ATU Cacheline Size */ #define ATU_LT 0x000d /* ATU Latency Timer */ #define ATU_HTR 0x000e /* ATU Header Type */ #define ATU_BISTR 0x000f /* ATU BIST Register */ #define ATU_IABAR0 0x0010 /* Inbound ATU Base Address register 0 */ #define ATU_IAUBAR0 0x0014 /* Inbound ATU Upper Base Address Register 0 */ #define ATU_IABAR1 0x0018 /* Inbound ATU Base Address Register 1 */ #define ATU_IAUBAR1 0x001c /* Inbound ATU Upper Base Address Register 1 */ #define ATU_IABAR2 0x0020 /* Inbound ATU Base Address Register 2 */ #define ATU_IAUBAR2 0x0024 /* Inbound ATU Upper Base Address Register 2 */ #define ATU_VSIR 0x002c /* ATU Subsystem Vendor ID Register */ #define ATU_SIR 0x002e /* ATU Subsystem ID Register */ #define ATU_ERBAR 0x0030 /* Expansion ROM Base Address Register */ #define ATU_CAPPTR 0x0034 /* ATU Capabilities Pointer Register */ #define ATU_ILR 0x003c /* ATU Interrupt Line Register */ #define ATU_IPR 0x003d /* ATU Interrupt Pin Register */ #define ATU_MGNT 0x003e /* ATU Minimum Grand Register */ #define ATU_MLAT 0x003f /* ATU Maximum Latency Register */ #define ATU_IALR0 0x0040 /* Inbound ATU Limit Register 0 */ #define ATU_IATVR0 0x0044 /* Inbound ATU Translate Value Register 0 */ #define ATU_IAUTVR0 0x0048 /* Inbound ATU Upper Translate Value Register 0*/ #define ATU_IALR1 0x004c /* Inbound ATU Limit Register 1 */ #define ATU_IATVR1 0x0050 /* Inbound ATU Translate Value Register 1 */ #define ATU_IAUTVR1 0x0054 /* Inbound ATU Upper Translate Value Register 1*/ #define ATU_IALR2 0x0058 /* Inbound ATU Limit Register 2 */ #define ATU_IATVR2 0x005c /* Inbound ATU Translate Value Register 2 */ #define ATU_IAUTVR2 0x0060 /* Inbound ATU Upper Translate Value Register 2*/ #define ATU_ERLR 0x0064 /* Expansion ROM Limit Register */ -#define ATU_ERTVR 0x0068 /* Expansion ROM Translater Value Register */ +#define ATU_ERTVR 0x0068 /* Expansion ROM Translator Value Register */ #define ATU_ERUTVR 0x006c /* Expansion ROM Upper Translate Value Register*/ #define ATU_CR 0x0070 /* ATU Configuration Register */ #define ATU_CR_OUT_EN (1 << 1) #define ATU_PCSR 0x0074 /* PCI Configuration and Status Register */ #define PCIE_BUSNO(x) ((x & 0xff000000) >> 24) #define ATUX_CORE_RST ((1 << 30) | (1U << 31)) /* Core Processor Reset */ #define ATUX_P_RSTOUT (1 << 21) /* Central Resource PCI Bus Reset */ #define ATUE_CORE_RST ((1 << 9) | (1 << 8)) /* Core Processor Reset */ #define ATU_ISR 0x0078 /* ATU Interrupt Status Register */ #define ATUX_ISR_PIE (1 << 18) /* PCI Interface error */ #define ATUX_ISR_IBPR (1 << 16) /* Internal Bus Parity Error */ #define ATUX_ISR_DCE (1 << 14) /* Detected Correctable error */ #define ATUX_ISR_ISCE (1 << 13) /* Initiated Split Completion Error Msg */ #define ATUX_ISR_RSCE (1 << 12) /* Received Split Completion Error Msg */ #define ATUX_ISR_DPE (1 << 9) /* Detected Parity Error */ #define ATUX_ISR_IBMA (1 << 7) /* Internal Bus Master Abort */ #define ATUX_ISR_PMA (1 << 3) /* PCI Master Abort */ #define ATUX_ISR_PTAM (1 << 2) /* PCI Target Abort (Master) */ #define ATUX_ISR_PTAT (1 << 1) /* PCI Target Abort (Target) */ #define ATUX_ISR_PMPE (1 << 0) /* PCI Master Parity Error */ #define ATUX_ISR_ERRMSK (ATUX_ISR_PIE | ATUX_ISR_IBPR | ATUX_ISR_DCE | \ ATUX_ISR_ISCE | ATUX_ISR_RSCE | ATUX_ISR_DPE | ATUX_ISR_IBMA | ATUX_ISR_PMA\ | ATUX_ISR_PTAM | ATUX_ISR_PTAT | ATUX_ISR_PMPE) #define ATUE_ISR_HON (1 << 13) /* Halt on Error Interrupt */ #define ATUE_ISR_RSE (1 << 12) /* Root System Error Message */ #define ATUE_ISR_REM (1 << 11) /* Root Error Message */ #define ATUE_ISR_PIE (1 << 10) /* PCI Interface error */ #define ATUE_ISR_CEM (1 << 9) /* Correctable Error Message */ #define ATUE_ISR_UEM (1 << 8) /* Uncorrectable error message */ #define ATUE_ISR_CRS (1 << 7) /* Received Configuration Retry Status */ #define ATUE_ISR_IBMA (1 << 5) /* Internal Bus Master Abort */ #define ATUE_ISR_DPE (1 << 4) /* Detected Parity Error Interrupt */ #define ATUE_ISR_MAI (1 << 3) /* Received Master Abort Interrupt */ #define ATUE_ISR_STAI (1 << 2) /* Signaled Target Abort Interrupt */ #define ATUE_ISR_TAI (1 << 1) /* Received Target Abort Interrupt */ #define ATUE_ISR_MDPE (1 << 0) /* Master Data Parity Error Interrupt */ #define ATUE_ISR_ERRMSK (ATUE_ISR_HON | ATUE_ISR_RSE | ATUE_ISR_REM | \ ATUE_ISR_PIE | ATUE_ISR_CEM | ATUE_ISR_UEM | ATUE_ISR_CRS | ATUE_ISR_IBMA |\ ATUE_ISR_DPE | ATUE_ISR_MAI | ATUE_ISR_STAI | ATUE_ISR_TAI | ATUE_ISR_MDPE) #define ATU_IMR 0x007c /* ATU Interrupt Mask Register */ /* 0x0080 - 0x008f reserved */ #define ATU_VPDCID 0x0090 /* VPD Capability Identifier Register */ #define ATU_VPDNIP 0x0091 /* VPD Next Item Pointer Register */ #define ATU_VPDAR 0x0092 /* VPD Address Register */ #define ATU_VPDDR 0x0094 /* VPD Data Register */ #define ATU_PMCID 0x0098 /* PM Capability Identifier Register */ #define ATU_PMNIPR 0x0099 /* PM Next Item Pointer Register */ #define ATU_PMCR 0x009a /* ATU Power Management Capabilities Register */ #define ATU_PMCSR 0x009c /* ATU Power Management Control/Status Register*/ #define ATU_MSICIR 0x00a0 /* MSI Capability Identifier Register */ #define ATU_MSINIPR 0x00a1 /* MSI Next Item Pointer Register */ #define ATU_MCR 0x00a2 /* Message Control Register */ #define ATU_MAR 0x00a4 /* Message Address Register */ #define ATU_MUAR 0x00a8 /* Message Upper Address Register */ #define ATU_MDR 0x00ac /* Message Data Register */ #define ATU_PCIXSR 0x00d4 /* PCI-X Status Register */ #define PCIXSR_BUSNO(x) (((x) & 0xff00) >> 8) #define ATU_IABAR3 0x0200 /* Inbound ATU Base Address Register 3 */ #define ATU_IAUBAR3 0x0204 /* Inbound ATU Upper Base Address Register 3 */ #define ATU_IALR3 0x0208 /* Inbound ATU Limit Register 3 */ #define ATU_ITVR3 0x020c /* Inbound ATU Upper Translate Value Reg 3 */ #define ATU_OIOBAR 0x0300 /* Outbound I/O Base Address Register */ #define ATU_OIOWTVR 0x0304 /* Outbound I/O Window Translate Value Reg */ #define ATU_OUMBAR0 0x0308 /* Outbound Upper Memory Window base addr reg 0*/ #define ATU_OUMBAR_FUNC (28) #define ATU_OUMBAR_EN (1U << 31) #define ATU_OUMWTVR0 0x030c /* Outbound Upper 32bit Memory Window Translate Value Register 0 */ #define ATU_OUMBAR1 0x0310 /* Outbound Upper Memory Window base addr reg1*/ #define ATU_OUMWTVR1 0x0314 /* Outbound Upper 32bit Memory Window Translate Value Register 1 */ #define ATU_OUMBAR2 0x0318 /* Outbound Upper Memory Window base addr reg2*/ #define ATU_OUMWTVR2 0x031c /* Outbount Upper 32bit Memory Window Translate Value Register 2 */ #define ATU_OUMBAR3 0x0320 /* Outbound Upper Memory Window base addr reg3*/ #define ATU_OUMWTVR3 0x0324 /* Outbound Upper 32bit Memory Window Translate Value Register 3 */ /* ATU-X specific */ #define ATUX_OCCAR 0x0330 /* Outbound Configuration Cycle Address Reg */ #define ATUX_OCCDR 0x0334 /* Outbound Configuration Cycle Data Reg */ #define ATUX_OCCFN 0x0338 /* Outbound Configuration Cycle Function Number*/ /* ATUe specific */ #define ATUE_OCCAR 0x032c /* Outbound Configuration Cycle Address Reg */ #define ATUE_OCCDR 0x0330 /* Outbound Configuration Cycle Data Reg */ #define ATUE_OCCFN 0x0334 /* Outbound Configuration Cycle Function Number*/ /* Interrupts */ /* IINTRSRC0 */ #define ICU_INT_ADMA0_EOT (0) /* ADMA 0 End of transfer */ #define ICU_INT_ADMA0_EOC (1) /* ADMA 0 End of Chain */ #define ICU_INT_ADMA1_EOT (2) /* ADMA 1 End of transfer */ #define ICU_INT_ADMA1_EOC (3) /* ADMA 1 End of chain */ #define ICU_INT_ADMA2_EOT (4) /* ADMA 2 End of transfer */ #define ICU_INT_ADMA2_EOC (5) /* ADMA 2 end of chain */ #define ICU_INT_WDOG (6) /* Watchdog timer */ /* 7 Reserved */ #define ICU_INT_TIMER0 (8) /* Timer 0 */ #define ICU_INT_TIMER1 (9) /* Timer 1 */ #define ICU_INT_I2C0 (10) /* I2C bus interface 0 */ #define ICU_INT_I2C1 (11) /* I2C bus interface 1 */ #define ICU_INT_MU (12) /* Message Unit */ #define ICU_INT_MU_IPQ (13) /* Message unit inbound post queue */ #define ICU_INT_ATUE_IM (14) /* ATU-E inbound message */ #define ICU_INT_ATU_BIST (15) /* ATU/Start BIST */ #define ICU_INT_PMC (16) /* PMC */ #define ICU_INT_PMU (17) /* PMU */ #define ICU_INT_PC (18) /* Processor cache */ /* 19-23 Reserved */ #define ICU_INT_XINT0 (24) #define ICU_INT_XINT1 (25) #define ICU_INT_XINT2 (26) #define ICU_INT_XINT3 (27) #define ICU_INT_XINT4 (28) #define ICU_INT_XINT5 (29) #define ICU_INT_XINT6 (30) #define ICU_INT_XINT7 (31) /* IINTSRC1 */ #define ICU_INT_XINT8 (32) #define ICU_INT_XINT9 (33) #define ICU_INT_XINT10 (34) #define ICU_INT_XINT11 (35) #define ICU_INT_XINT12 (36) #define ICU_INT_XINT13 (37) #define ICU_INT_XINT14 (38) #define ICU_INT_XINT15 (39) /* 40-50 reserved */ #define ICU_INT_UART0 (51) /* UART 0 */ #define ICU_INT_UART1 (52) /* UART 1 */ #define ICU_INT_PBIUE (53) /* Peripheral bus interface unit error */ #define ICU_INT_ATUCRW (54) /* ATU Configuration register write */ #define ICU_INT_ATUE (55) /* ATU error */ #define ICU_INT_MCUE (56) /* Memory controller unit error */ #define ICU_INT_ADMA0E (57) /* ADMA Channel 0 error */ #define ICU_INT_ADMA1E (58) /* ADMA Channel 1 error */ #define ICU_INT_ADMA2E (59) /* ADMA Channel 2 error */ /* 60-61 reserved */ #define ICU_INT_MUE (62) /* Messaging Unit Error */ /* 63 reserved */ /* IINTSRC2 */ #define ICU_INT_IP (64) /* Inter-processor */ /* 65-93 reserved */ #define ICU_INT_SIBBE (94) /* South internal bus bridge error */ /* 95 reserved */ /* IINTSRC3 */ #define ICU_INT_I2C2 (96) /* I2C bus interface 2 */ #define ICU_INT_ATUE_BIST (97) /* ATU-E/Start BIST */ #define ICU_INT_ATUE_CRW (98) /* ATU-E Configuration register write */ #define ICU_INT_ATUEE (99) /* ATU-E Error */ #define ICU_INT_IMU (100) /* IMU */ /* 101-106 reserved */ #define ICU_INT_ATUE_MA (107) /* ATUE Interrupt message A */ #define ICU_INT_ATUE_MB (108) /* ATUE Interrupt message B */ #define ICU_INT_ATUE_MC (109) /* ATUE Interrupt message C */ #define ICU_INT_ATUE_MD (110) /* ATUE Interrupt message D */ #define ICU_INT_MU_MSIX_TW (111) /* MU MSI-X Table write */ /* 112 reserved */ #define ICU_INT_IMSI (113) /* Inbound MSI */ /* 114-126 reserved */ #define ICU_INT_HPI (127) /* HPI */ #endif /* I81342_REG_H_ */ Index: user/ngie/release-pkg-fix-tests/sys/arm/xscale/ixp425/avila_machdep.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/xscale/ixp425/avila_machdep.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/xscale/ixp425/avila_machdep.c (revision 299091) @@ -1,417 +1,417 @@ /* $NetBSD: hpc_machdep.c,v 1.70 2003/09/16 08:18:22 agc Exp $ */ /*- * Copyright (c) 1994-1998 Mark Brinicombe. * Copyright (c) 1994 Brini. * All rights reserved. * * This code is derived from software written for Brini by Mark Brinicombe * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Brini. * 4. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * RiscBSD kernel project * * machdep.c * - * Machine dependant functions for kernel setup + * Machine dependent functions for kernel setup * * This file needs a lot of work. * * Created : 17/09/94 */ #include __FBSDID("$FreeBSD$"); #include "opt_kstack_pages.h" #define _ARM32_BUS_DMA_PRIVATE #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define KERNEL_PT_SYS 0 /* Page table for mapping proc0 zero page */ #define KERNEL_PT_IO 1 #define KERNEL_PT_IO_NUM 3 #define KERNEL_PT_BEFOREKERN KERNEL_PT_IO + KERNEL_PT_IO_NUM #define KERNEL_PT_AFKERNEL KERNEL_PT_BEFOREKERN + 1 /* L2 table for mapping after kernel */ #define KERNEL_PT_AFKERNEL_NUM 9 /* this should be evenly divisable by PAGE_SIZE / L2_TABLE_SIZE_REAL (or 4) */ #define NUM_KERNEL_PTS (KERNEL_PT_AFKERNEL + KERNEL_PT_AFKERNEL_NUM) struct pv_addr kernel_pt_table[NUM_KERNEL_PTS]; /* Physical and virtual addresses for some global pages */ struct pv_addr systempage; struct pv_addr msgbufpv; struct pv_addr irqstack; struct pv_addr undstack; struct pv_addr abtstack; struct pv_addr kernelstack; struct pv_addr minidataclean; /* Static device mappings. */ static const struct devmap_entry ixp425_devmap[] = { /* Physical/Virtual address for I/O space */ { IXP425_IO_VBASE, IXP425_IO_HWBASE, IXP425_IO_SIZE, }, /* Expansion Bus */ { IXP425_EXP_VBASE, IXP425_EXP_HWBASE, IXP425_EXP_SIZE, }, /* CFI Flash on the Expansion Bus */ { IXP425_EXP_BUS_CS0_VBASE, IXP425_EXP_BUS_CS0_HWBASE, IXP425_EXP_BUS_CS0_SIZE, }, /* IXP425 PCI Configuration */ { IXP425_PCI_VBASE, IXP425_PCI_HWBASE, IXP425_PCI_SIZE, }, /* SDRAM Controller */ { IXP425_MCU_VBASE, IXP425_MCU_HWBASE, IXP425_MCU_SIZE, }, /* PCI Memory Space */ { IXP425_PCI_MEM_VBASE, IXP425_PCI_MEM_HWBASE, IXP425_PCI_MEM_SIZE, }, /* Q-Mgr Memory Space */ { IXP425_QMGR_VBASE, IXP425_QMGR_HWBASE, IXP425_QMGR_SIZE, }, { 0 }, }; /* Static device mappings. */ static const struct devmap_entry ixp435_devmap[] = { /* Physical/Virtual address for I/O space */ { IXP425_IO_VBASE, IXP425_IO_HWBASE, IXP425_IO_SIZE, }, { IXP425_EXP_VBASE, IXP425_EXP_HWBASE, IXP425_EXP_SIZE, }, /* IXP425 PCI Configuration */ { IXP425_PCI_VBASE, IXP425_PCI_HWBASE, IXP425_PCI_SIZE, }, /* DDRII Controller NB: mapped same place as IXP425 */ { IXP425_MCU_VBASE, IXP435_MCU_HWBASE, IXP425_MCU_SIZE, }, /* PCI Memory Space */ { IXP425_PCI_MEM_VBASE, IXP425_PCI_MEM_HWBASE, IXP425_PCI_MEM_SIZE, }, /* Q-Mgr Memory Space */ { IXP425_QMGR_VBASE, IXP425_QMGR_HWBASE, IXP425_QMGR_SIZE, }, /* CFI Flash on the Expansion Bus */ { IXP425_EXP_BUS_CS0_VBASE, IXP425_EXP_BUS_CS0_HWBASE, IXP425_EXP_BUS_CS0_SIZE, }, /* USB1 Memory Space */ { IXP435_USB1_VBASE, IXP435_USB1_HWBASE, IXP435_USB1_SIZE, }, /* USB2 Memory Space */ { IXP435_USB2_VBASE, IXP435_USB2_HWBASE, IXP435_USB2_SIZE, }, /* GPS Memory Space */ { CAMBRIA_GPS_VBASE, CAMBRIA_GPS_HWBASE, CAMBRIA_GPS_SIZE, }, /* RS485 Memory Space */ { CAMBRIA_RS485_VBASE, CAMBRIA_RS485_HWBASE, CAMBRIA_RS485_SIZE, }, { 0 } }; extern vm_offset_t xscale_cache_clean_addr; void * initarm(struct arm_boot_params *abp) { #define next_chunk2(a,b) (((a) + (b)) &~ ((b)-1)) #define next_page(a) next_chunk2(a,PAGE_SIZE) struct pv_addr kernel_l1pt; struct pv_addr dpcpu; int loop, i; u_int l1pagetable; vm_offset_t freemempos; vm_offset_t freemem_pt; vm_offset_t afterkern; vm_offset_t freemem_after; vm_offset_t lastaddr; uint32_t memsize; /* kernel text starts where we were loaded at boot */ #define KERNEL_TEXT_OFF (abp->abp_physaddr - PHYSADDR) #define KERNEL_TEXT_BASE (KERNBASE + KERNEL_TEXT_OFF) #define KERNEL_TEXT_PHYS (PHYSADDR + KERNEL_TEXT_OFF) lastaddr = parse_boot_param(abp); arm_physmem_kernaddr = abp->abp_physaddr; set_cpufuncs(); /* NB: sets cputype */ pcpu_init(pcpup, 0, sizeof(struct pcpu)); PCPU_SET(curthread, &thread0); init_static_kenv(NULL, 0); /* Do basic tuning, hz etc */ init_param1(); /* * We allocate memory downwards from where we were loaded * by RedBoot; first the L1 page table, then NUM_KERNEL_PTS * entries in the L2 page table. Past that we re-align the * allocation boundary so later data structures (stacks, etc) * can be mapped with different attributes (write-back vs * write-through). Note this leaves a gap for expansion * (or might be repurposed). */ freemempos = abp->abp_physaddr; /* macros to simplify initial memory allocation */ #define alloc_pages(var, np) do { \ freemempos -= (np * PAGE_SIZE); \ (var) = freemempos; \ /* NB: this works because locore maps PA=VA */ \ memset((char *)(var), 0, ((np) * PAGE_SIZE)); \ } while (0) #define valloc_pages(var, np) do { \ alloc_pages((var).pv_pa, (np)); \ (var).pv_va = (var).pv_pa + (KERNVIRTADDR - abp->abp_physaddr); \ } while (0) /* force L1 page table alignment */ while (((freemempos - L1_TABLE_SIZE) & (L1_TABLE_SIZE - 1)) != 0) freemempos -= PAGE_SIZE; /* allocate contiguous L1 page table */ valloc_pages(kernel_l1pt, L1_TABLE_SIZE / PAGE_SIZE); /* now allocate L2 page tables; they are linked to L1 below */ for (loop = 0; loop < NUM_KERNEL_PTS; ++loop) { if (!(loop % (PAGE_SIZE / L2_TABLE_SIZE_REAL))) { valloc_pages(kernel_pt_table[loop], L2_TABLE_SIZE / PAGE_SIZE); } else { kernel_pt_table[loop].pv_pa = freemempos + (loop % (PAGE_SIZE / L2_TABLE_SIZE_REAL)) * L2_TABLE_SIZE_REAL; kernel_pt_table[loop].pv_va = kernel_pt_table[loop].pv_pa + (KERNVIRTADDR - abp->abp_physaddr); } } freemem_pt = freemempos; /* base of allocated pt's */ /* * Re-align allocation boundary so we can map the area * write-back instead of write-through for the stacks and * related structures allocated below. */ freemempos = PHYSADDR + 0x100000; /* * Allocate a page for the system page mapped to V0x00000000 * This page will just contain the system vectors and can be * shared by all processes. */ valloc_pages(systempage, 1); /* Allocate dynamic per-cpu area. */ valloc_pages(dpcpu, DPCPU_SIZE / PAGE_SIZE); dpcpu_init((void *)dpcpu.pv_va, 0); /* Allocate stacks for all modes */ valloc_pages(irqstack, IRQ_STACK_SIZE); valloc_pages(abtstack, ABT_STACK_SIZE); valloc_pages(undstack, UND_STACK_SIZE); valloc_pages(kernelstack, kstack_pages); alloc_pages(minidataclean.pv_pa, 1); valloc_pages(msgbufpv, round_page(msgbufsize) / PAGE_SIZE); /* * Now construct the L1 page table. First map the L2 * page tables into the L1 so we can replace L1 mappings * later on if necessary */ l1pagetable = kernel_l1pt.pv_va; /* Map the L2 pages tables in the L1 page table */ pmap_link_l2pt(l1pagetable, rounddown2(ARM_VECTORS_HIGH, 0x00100000), &kernel_pt_table[KERNEL_PT_SYS]); pmap_link_l2pt(l1pagetable, IXP425_IO_VBASE, &kernel_pt_table[KERNEL_PT_IO]); pmap_link_l2pt(l1pagetable, IXP425_MCU_VBASE, &kernel_pt_table[KERNEL_PT_IO + 1]); pmap_link_l2pt(l1pagetable, IXP425_PCI_MEM_VBASE, &kernel_pt_table[KERNEL_PT_IO + 2]); pmap_link_l2pt(l1pagetable, KERNBASE, &kernel_pt_table[KERNEL_PT_BEFOREKERN]); pmap_map_chunk(l1pagetable, KERNBASE, PHYSADDR, 0x100000, VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); pmap_map_chunk(l1pagetable, KERNBASE + 0x100000, PHYSADDR + 0x100000, 0x100000, VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE); pmap_map_chunk(l1pagetable, KERNEL_TEXT_BASE, KERNEL_TEXT_PHYS, next_chunk2(((uint32_t)lastaddr) - KERNEL_TEXT_BASE, L1_S_SIZE), VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); freemem_after = next_page((int)lastaddr); afterkern = round_page(next_chunk2((vm_offset_t)lastaddr, L1_S_SIZE)); for (i = 0; i < KERNEL_PT_AFKERNEL_NUM; i++) { pmap_link_l2pt(l1pagetable, afterkern + i * 0x00100000, &kernel_pt_table[KERNEL_PT_AFKERNEL + i]); } pmap_map_entry(l1pagetable, afterkern, minidataclean.pv_pa, VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); /* Map the Mini-Data cache clean area. */ xscale_setup_minidata(l1pagetable, afterkern, minidataclean.pv_pa); /* Map the vector page. */ pmap_map_entry(l1pagetable, ARM_VECTORS_HIGH, systempage.pv_pa, VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); if (cpu_is_ixp43x()) devmap_bootstrap(l1pagetable, ixp435_devmap); else devmap_bootstrap(l1pagetable, ixp425_devmap); /* * Give the XScale global cache clean code an appropriately * sized chunk of unmapped VA space starting at 0xff000000 * (our device mappings end before this address). */ xscale_cache_clean_addr = 0xff000000U; cpu_domains((DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL*2)) | DOMAIN_CLIENT); cpu_setttb(kernel_l1pt.pv_pa); cpu_tlb_flushID(); cpu_domains(DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL*2)); /* * Pages were allocated during the secondary bootstrap for the * stacks for different CPU modes. * We must now set the r13 registers in the different CPU modes to * point to these stacks. * Since the ARM stacks use STMFD etc. we must set r13 to the top end * of the stack memory. */ set_stackptrs(0); /* * We must now clean the cache again.... * Cleaning may be done by reading new data to displace any * dirty data in the cache. This will have happened in cpu_setttb() * but since we are boot strapping the addresses used for the read * may have just been remapped and thus the cache could be out * of sync. A re-clean after the switch will cure this. * After booting there are no gross relocations of the kernel thus * this problem will not occur after initarm(). */ cpu_idcache_wbinv_all(); cpu_setup(); /* ready to setup the console (XXX move earlier if possible) */ cninit(); /* * Fetch the RAM size from the MCU registers. The * expansion bus was mapped above so we can now read 'em. */ if (cpu_is_ixp43x()) memsize = ixp435_ddram_size(); else memsize = ixp425_sdram_size(); undefined_init(); init_proc0(kernelstack.pv_va); arm_vector_init(ARM_VECTORS_HIGH, ARM_VEC_ALL); pmap_curmaxkvaddr = afterkern + PAGE_SIZE; vm_max_kernel_address = 0xe0000000; pmap_bootstrap(pmap_curmaxkvaddr, &kernel_l1pt); msgbufp = (void*)msgbufpv.pv_va; msgbufinit(msgbufp, msgbufsize); mutex_init(); /* * Add the physical ram we have available. * * Exclude the kernel, and all the things we allocated which immediately * follow the kernel, from the VM allocation pool but not from crash * dumps. virtual_avail is a global variable which tracks the kva we've * "allocated" while setting up pmaps. * * Prepare the list of physical memory available to the vm subsystem. */ arm_physmem_hardware_region(PHYSADDR, memsize); arm_physmem_exclude_region(freemem_pt, abp->abp_physaddr - freemem_pt, EXFLAG_NOALLOC); arm_physmem_exclude_region(freemempos, abp->abp_physaddr - 0x100000 - freemempos, EXFLAG_NOALLOC); arm_physmem_exclude_region(abp->abp_physaddr, virtual_avail - KERNVIRTADDR, EXFLAG_NOALLOC); arm_physmem_init_kernel_globals(); init_param2(physmem); kdb_init(); return ((void *)(kernelstack.pv_va + USPACE_SVC_STACK_TOP - sizeof(struct pcb))); #undef next_page #undef next_chunk2 } Index: user/ngie/release-pkg-fix-tests/sys/arm/xscale/ixp425/cambria_gpio.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/xscale/ixp425/cambria_gpio.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/xscale/ixp425/cambria_gpio.c (revision 299091) @@ -1,503 +1,503 @@ /*- * Copyright (c) 2010, Andrew Thompson * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice unmodified, this list of conditions, and the following * disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /* * GPIO driver for Gateworks Cambria * * Note: * The Cambria PLD does not set the i2c ack bit after each write, if we used the * regular iicbus interface it would abort the xfer after the address byte * times out and not write our latch. To get around this we grab the iicbus and - * then do our own bit banging. This is a comprimise to changing all the iicbb + * then do our own bit banging. This is a compromise to changing all the iicbb * device methods to allow a flag to be passed down and is similir to how Linux * does it. * */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "iicbb_if.h" #include "gpio_if.h" #define IIC_M_WR 0 /* write operation */ #define PLD_ADDR 0xac /* slave address */ #define I2C_DELAY 10 #define GPIO_CONF_CLR(sc, reg, mask) \ GPIO_CONF_WRITE_4(sc, reg, GPIO_CONF_READ_4(sc, reg) &~ (mask)) #define GPIO_CONF_SET(sc, reg, mask) \ GPIO_CONF_WRITE_4(sc, reg, GPIO_CONF_READ_4(sc, reg) | (mask)) #define GPIO_LOCK(_sc) mtx_lock(&(_sc)->sc_mtx) #define GPIO_UNLOCK(_sc) mtx_unlock(&(_sc)->sc_mtx) #define GPIO_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->sc_mtx, MA_OWNED) #define GPIO_PINS 5 struct cambria_gpio_softc { device_t sc_dev; device_t sc_busdev; bus_space_tag_t sc_iot; bus_space_handle_t sc_gpio_ioh; struct mtx sc_mtx; struct gpio_pin sc_pins[GPIO_PINS]; uint8_t sc_latch; uint8_t sc_val; }; struct cambria_gpio_pin { const char *name; int pin; int flags; }; extern struct ixp425_softc *ixp425_softc; static struct cambria_gpio_pin cambria_gpio_pins[GPIO_PINS] = { { "PLD0", 0, GPIO_PIN_OUTPUT }, { "PLD1", 1, GPIO_PIN_OUTPUT }, { "PLD2", 2, GPIO_PIN_OUTPUT }, { "PLD3", 3, GPIO_PIN_OUTPUT }, { "PLD4", 4, GPIO_PIN_OUTPUT }, }; /* * Helpers */ static int cambria_gpio_read(struct cambria_gpio_softc *, uint32_t, unsigned int *); static int cambria_gpio_write(struct cambria_gpio_softc *); /* * Driver stuff */ static int cambria_gpio_probe(device_t dev); static int cambria_gpio_attach(device_t dev); static int cambria_gpio_detach(device_t dev); /* * GPIO interface */ static device_t cambria_gpio_get_bus(device_t); static int cambria_gpio_pin_max(device_t dev, int *maxpin); static int cambria_gpio_pin_getcaps(device_t dev, uint32_t pin, uint32_t *caps); static int cambria_gpio_pin_getflags(device_t dev, uint32_t pin, uint32_t *flags); static int cambria_gpio_pin_getname(device_t dev, uint32_t pin, char *name); static int cambria_gpio_pin_setflags(device_t dev, uint32_t pin, uint32_t flags); static int cambria_gpio_pin_set(device_t dev, uint32_t pin, unsigned int value); static int cambria_gpio_pin_get(device_t dev, uint32_t pin, unsigned int *val); static int cambria_gpio_pin_toggle(device_t dev, uint32_t pin); static int i2c_getsda(struct cambria_gpio_softc *sc) { uint32_t reg; IXP4XX_GPIO_LOCK(); GPIO_CONF_SET(sc, IXP425_GPIO_GPOER, GPIO_I2C_SDA_BIT); reg = GPIO_CONF_READ_4(sc, IXP425_GPIO_GPINR); IXP4XX_GPIO_UNLOCK(); return (reg & GPIO_I2C_SDA_BIT); } static void i2c_setsda(struct cambria_gpio_softc *sc, int val) { IXP4XX_GPIO_LOCK(); GPIO_CONF_CLR(sc, IXP425_GPIO_GPOUTR, GPIO_I2C_SDA_BIT); if (val) GPIO_CONF_SET(sc, IXP425_GPIO_GPOER, GPIO_I2C_SDA_BIT); else GPIO_CONF_CLR(sc, IXP425_GPIO_GPOER, GPIO_I2C_SDA_BIT); IXP4XX_GPIO_UNLOCK(); DELAY(I2C_DELAY); } static void i2c_setscl(struct cambria_gpio_softc *sc, int val) { IXP4XX_GPIO_LOCK(); GPIO_CONF_CLR(sc, IXP425_GPIO_GPOUTR, GPIO_I2C_SCL_BIT); if (val) GPIO_CONF_SET(sc, IXP425_GPIO_GPOER, GPIO_I2C_SCL_BIT); else GPIO_CONF_CLR(sc, IXP425_GPIO_GPOER, GPIO_I2C_SCL_BIT); IXP4XX_GPIO_UNLOCK(); DELAY(I2C_DELAY); } static void i2c_sendstart(struct cambria_gpio_softc *sc) { i2c_setsda(sc, 1); i2c_setscl(sc, 1); i2c_setsda(sc, 0); i2c_setscl(sc, 0); } static void i2c_sendstop(struct cambria_gpio_softc *sc) { i2c_setscl(sc, 1); i2c_setsda(sc, 1); i2c_setscl(sc, 0); i2c_setsda(sc, 0); } static void i2c_sendbyte(struct cambria_gpio_softc *sc, u_char data) { int i; for (i=7; i>=0; i--) { i2c_setsda(sc, data & (1<=0; i--) { i2c_setscl(sc, 1); if (i2c_getsda(sc)) data |= (1<sc_dev; int error; error = iicbus_request_bus(device_get_parent(dev), dev, IIC_DONTWAIT); if (error) return (error); i2c_sendstart(sc); i2c_sendbyte(sc, PLD_ADDR | LSB); *val = (i2c_readbyte(sc) & (1 << pin)) != 0; i2c_sendstop(sc); iicbus_release_bus(device_get_parent(dev), dev); return (0); } static int cambria_gpio_write(struct cambria_gpio_softc *sc) { device_t dev = sc->sc_dev; int error; error = iicbus_request_bus(device_get_parent(dev), dev, IIC_DONTWAIT); if (error) return (error); i2c_sendstart(sc); i2c_sendbyte(sc, PLD_ADDR & ~LSB); i2c_sendbyte(sc, sc->sc_latch); i2c_sendstop(sc); iicbus_release_bus(device_get_parent(dev), dev); return (0); } static device_t cambria_gpio_get_bus(device_t dev) { struct cambria_gpio_softc *sc; sc = device_get_softc(dev); return (sc->sc_busdev); } static int cambria_gpio_pin_max(device_t dev, int *maxpin) { *maxpin = GPIO_PINS - 1; return (0); } static int cambria_gpio_pin_getcaps(device_t dev, uint32_t pin, uint32_t *caps) { struct cambria_gpio_softc *sc = device_get_softc(dev); if (pin >= GPIO_PINS) return (EINVAL); *caps = sc->sc_pins[pin].gp_caps; return (0); } static int cambria_gpio_pin_getflags(device_t dev, uint32_t pin, uint32_t *flags) { struct cambria_gpio_softc *sc = device_get_softc(dev); if (pin >= GPIO_PINS) return (EINVAL); *flags = sc->sc_pins[pin].gp_flags; return (0); } static int cambria_gpio_pin_getname(device_t dev, uint32_t pin, char *name) { struct cambria_gpio_softc *sc = device_get_softc(dev); if (pin >= GPIO_PINS) return (EINVAL); memcpy(name, sc->sc_pins[pin].gp_name, GPIOMAXNAME); return (0); } static int cambria_gpio_pin_setflags(device_t dev, uint32_t pin, uint32_t flags) { struct cambria_gpio_softc *sc = device_get_softc(dev); int error; uint8_t mask; mask = 1 << pin; if (pin >= GPIO_PINS) return (EINVAL); GPIO_LOCK(sc); sc->sc_pins[pin].gp_flags = flags; /* * Writing a logical one sets the signal high and writing a logical * zero sets the signal low. To configure a digital I/O signal as an * input, a logical one must first be written to the data bit to * three-state the associated output. */ if (flags & GPIO_PIN_INPUT || sc->sc_val & mask) sc->sc_latch |= mask; /* input or output & high */ else sc->sc_latch &= ~mask; error = cambria_gpio_write(sc); GPIO_UNLOCK(sc); return (error); } static int cambria_gpio_pin_set(device_t dev, uint32_t pin, unsigned int value) { struct cambria_gpio_softc *sc = device_get_softc(dev); int error; uint8_t mask; mask = 1 << pin; if (pin >= GPIO_PINS) return (EINVAL); GPIO_LOCK(sc); if (value) sc->sc_val |= mask; else sc->sc_val &= ~mask; if (sc->sc_pins[pin].gp_flags != GPIO_PIN_OUTPUT) { /* just save, altering the latch will disable input */ GPIO_UNLOCK(sc); return (0); } if (value) sc->sc_latch |= mask; else sc->sc_latch &= ~mask; error = cambria_gpio_write(sc); GPIO_UNLOCK(sc); return (error); } static int cambria_gpio_pin_get(device_t dev, uint32_t pin, unsigned int *val) { struct cambria_gpio_softc *sc = device_get_softc(dev); int error = 0; if (pin >= GPIO_PINS) return (EINVAL); GPIO_LOCK(sc); if (sc->sc_pins[pin].gp_flags == GPIO_PIN_OUTPUT) *val = (sc->sc_latch & (1 << pin)) ? 1 : 0; else error = cambria_gpio_read(sc, pin, val); GPIO_UNLOCK(sc); return (error); } static int cambria_gpio_pin_toggle(device_t dev, uint32_t pin) { struct cambria_gpio_softc *sc = device_get_softc(dev); int error = 0; if (pin >= GPIO_PINS) return (EINVAL); GPIO_LOCK(sc); sc->sc_val ^= (1 << pin); if (sc->sc_pins[pin].gp_flags == GPIO_PIN_OUTPUT) { sc->sc_latch ^= (1 << pin); error = cambria_gpio_write(sc); } GPIO_UNLOCK(sc); return (error); } static int cambria_gpio_probe(device_t dev) { device_set_desc(dev, "Gateworks Cambria GPIO driver"); return (0); } static int cambria_gpio_attach(device_t dev) { struct cambria_gpio_softc *sc = device_get_softc(dev); int pin; sc->sc_dev = dev; sc->sc_iot = ixp425_softc->sc_iot; sc->sc_gpio_ioh = ixp425_softc->sc_gpio_ioh; mtx_init(&sc->sc_mtx, device_get_nameunit(dev), NULL, MTX_DEF); for (pin = 0; pin < GPIO_PINS; pin++) { struct cambria_gpio_pin *p = &cambria_gpio_pins[pin]; strncpy(sc->sc_pins[pin].gp_name, p->name, GPIOMAXNAME); sc->sc_pins[pin].gp_pin = pin; sc->sc_pins[pin].gp_caps = GPIO_PIN_INPUT|GPIO_PIN_OUTPUT; sc->sc_pins[pin].gp_flags = 0; cambria_gpio_pin_setflags(dev, pin, p->flags); } sc->sc_busdev = gpiobus_attach_bus(dev); if (sc->sc_busdev == NULL) { mtx_destroy(&sc->sc_mtx); return (ENXIO); } return (0); } static int cambria_gpio_detach(device_t dev) { struct cambria_gpio_softc *sc = device_get_softc(dev); KASSERT(mtx_initialized(&sc->sc_mtx), ("gpio mutex not initialized")); gpiobus_detach_bus(dev); mtx_destroy(&sc->sc_mtx); return(0); } static device_method_t cambria_gpio_methods[] = { DEVMETHOD(device_probe, cambria_gpio_probe), DEVMETHOD(device_attach, cambria_gpio_attach), DEVMETHOD(device_detach, cambria_gpio_detach), /* GPIO protocol */ DEVMETHOD(gpio_get_bus, cambria_gpio_get_bus), DEVMETHOD(gpio_pin_max, cambria_gpio_pin_max), DEVMETHOD(gpio_pin_getname, cambria_gpio_pin_getname), DEVMETHOD(gpio_pin_getflags, cambria_gpio_pin_getflags), DEVMETHOD(gpio_pin_getcaps, cambria_gpio_pin_getcaps), DEVMETHOD(gpio_pin_setflags, cambria_gpio_pin_setflags), DEVMETHOD(gpio_pin_get, cambria_gpio_pin_get), DEVMETHOD(gpio_pin_set, cambria_gpio_pin_set), DEVMETHOD(gpio_pin_toggle, cambria_gpio_pin_toggle), {0, 0}, }; static driver_t cambria_gpio_driver = { "gpio", cambria_gpio_methods, sizeof(struct cambria_gpio_softc), }; static devclass_t cambria_gpio_devclass; DRIVER_MODULE(gpio_cambria, iicbus, cambria_gpio_driver, cambria_gpio_devclass, 0, 0); MODULE_VERSION(gpio_cambria, 1); MODULE_DEPEND(gpio_cambria, iicbus, 1, 1, 1); Index: user/ngie/release-pkg-fix-tests/sys/arm/xscale/ixp425/ixp425_npe.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/xscale/ixp425/ixp425_npe.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/xscale/ixp425/ixp425_npe.c (revision 299091) @@ -1,1574 +1,1574 @@ /*- * Copyright (c) 2006-2008 Sam Leffler, Errno Consulting * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer, * without modification. * 2. Redistributions in binary form must reproduce at minimum a disclaimer * similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any * redistribution must be conditioned upon including a substantially * similar Disclaimer requirement for further binary redistribution. * * NO WARRANTY * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGES. */ /*- * Copyright (c) 2001-2005, Intel Corporation. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the Intel Corporation nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); /* * Intel XScale Network Processing Engine (NPE) support. * * Each NPE has an ixpnpeX device associated with it that is * attached at boot. Depending on the microcode loaded into * an NPE there may be an Ethernet interface (npeX) or some * other network interface (e.g. for ATM). This file has support * for loading microcode images and the associated NPE CPU * manipulations (start, stop, reset). * * The code here basically replaces the npeDl and npeMh classes * in the Intel Access Library (IAL). * * NB: Microcode images are loaded with firmware(9). To * include microcode in a static kernel include the * ixpnpe_fw device. Otherwise the firmware will be * automatically loaded from the filesystem. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include struct ixpnpe_softc { device_t sc_dev; bus_space_tag_t sc_iot; bus_space_handle_t sc_ioh; bus_size_t sc_size; /* size of mapped register window */ struct resource *sc_irq; /* IRQ resource */ void *sc_ih; /* interrupt handler */ struct mtx sc_mtx; /* mailbox lock */ uint32_t sc_msg[2]; /* reply msg collected in ixpnpe_intr */ int sc_msgwaiting; /* sc_msg holds valid data */ int sc_npeid; int sc_nrefs; /* # of references */ int validImage; /* valid ucode image loaded */ int started; /* NPE is started */ uint8_t functionalityId;/* ucode functionality ID */ int insMemSize; /* size of instruction memory */ int dataMemSize; /* size of data memory */ uint32_t savedExecCount; uint32_t savedEcsDbgCtxtReg2; }; static struct ixpnpe_softc *npes[NPE_MAX]; #define IX_NPEDL_NPEIMAGE_FIELD_MASK 0xff /* used to read download map from version in microcode image */ #define IX_NPEDL_BLOCK_TYPE_INSTRUCTION 0x00000000 #define IX_NPEDL_BLOCK_TYPE_DATA 0x00000001 #define IX_NPEDL_BLOCK_TYPE_STATE 0x00000002 #define IX_NPEDL_END_OF_DOWNLOAD_MAP 0x0000000F /* * masks used to extract address info from State information context * register addresses as read from microcode image */ #define IX_NPEDL_MASK_STATE_ADDR_CTXT_REG 0x0000000F #define IX_NPEDL_MASK_STATE_ADDR_CTXT_NUM 0x000000F0 /* LSB offset of Context Number field in State-Info Context Address */ #define IX_NPEDL_OFFSET_STATE_ADDR_CTXT_NUM 4 /* size (in words) of single State Information entry (ctxt reg address|data) */ #define IX_NPEDL_STATE_INFO_ENTRY_SIZE 2 typedef struct { uint32_t type; uint32_t offset; } IxNpeDlNpeMgrDownloadMapBlockEntry; typedef union { IxNpeDlNpeMgrDownloadMapBlockEntry block; uint32_t eodmMarker; } IxNpeDlNpeMgrDownloadMapEntry; typedef struct { /* 1st entry in the download map (there may be more than one) */ IxNpeDlNpeMgrDownloadMapEntry entry[1]; } IxNpeDlNpeMgrDownloadMap; /* used to access an instruction or data block in a microcode image */ typedef struct { uint32_t npeMemAddress; uint32_t size; uint32_t data[1]; } IxNpeDlNpeMgrCodeBlock; /* used to access each Context Reg entry state-information block */ typedef struct { uint32_t addressInfo; uint32_t value; } IxNpeDlNpeMgrStateInfoCtxtRegEntry; /* used to access a state-information block in a microcode image */ typedef struct { uint32_t size; IxNpeDlNpeMgrStateInfoCtxtRegEntry ctxtRegEntry[1]; } IxNpeDlNpeMgrStateInfoBlock; static int npe_debug = 0; SYSCTL_INT(_debug, OID_AUTO, ixp425npe, CTLFLAG_RWTUN, &npe_debug, 0, "IXP4XX NPE debug msgs"); #define DPRINTF(dev, fmt, ...) do { \ if (npe_debug) device_printf(dev, fmt, __VA_ARGS__); \ } while (0) #define DPRINTFn(n, dev, fmt, ...) do { \ if (npe_debug >= n) printf(fmt, __VA_ARGS__); \ } while (0) static int npe_checkbits(struct ixpnpe_softc *, uint32_t reg, uint32_t); static int npe_isstopped(struct ixpnpe_softc *); static int npe_load_ins(struct ixpnpe_softc *, const IxNpeDlNpeMgrCodeBlock *bp, int verify); static int npe_load_data(struct ixpnpe_softc *, const IxNpeDlNpeMgrCodeBlock *bp, int verify); static int npe_load_stateinfo(struct ixpnpe_softc *, const IxNpeDlNpeMgrStateInfoBlock *bp, int verify); static int npe_load_image(struct ixpnpe_softc *, const uint32_t *imageCodePtr, int verify); static int npe_cpu_reset(struct ixpnpe_softc *); static int npe_cpu_start(struct ixpnpe_softc *); static int npe_cpu_stop(struct ixpnpe_softc *); static void npe_cmd_issue_write(struct ixpnpe_softc *, uint32_t cmd, uint32_t addr, uint32_t data); static uint32_t npe_cmd_issue_read(struct ixpnpe_softc *, uint32_t cmd, uint32_t addr); static int npe_ins_write(struct ixpnpe_softc *, uint32_t addr, uint32_t data, int verify); static int npe_data_write(struct ixpnpe_softc *, uint32_t addr, uint32_t data, int verify); static void npe_ecs_reg_write(struct ixpnpe_softc *, uint32_t reg, uint32_t data); static uint32_t npe_ecs_reg_read(struct ixpnpe_softc *, uint32_t reg); static void npe_issue_cmd(struct ixpnpe_softc *, uint32_t command); static void npe_cpu_step_save(struct ixpnpe_softc *); static int npe_cpu_step(struct ixpnpe_softc *, uint32_t npeInstruction, uint32_t ctxtNum, uint32_t ldur); static void npe_cpu_step_restore(struct ixpnpe_softc *); static int npe_logical_reg_read(struct ixpnpe_softc *, uint32_t regAddr, uint32_t regSize, uint32_t ctxtNum, uint32_t *regVal); static int npe_logical_reg_write(struct ixpnpe_softc *, uint32_t regAddr, uint32_t regVal, uint32_t regSize, uint32_t ctxtNum, int verify); static int npe_physical_reg_write(struct ixpnpe_softc *, uint32_t regAddr, uint32_t regValue, int verify); static int npe_ctx_reg_write(struct ixpnpe_softc *, uint32_t ctxtNum, uint32_t ctxtReg, uint32_t ctxtRegVal, int verify); static void ixpnpe_intr(void *arg); static uint32_t npe_reg_read(struct ixpnpe_softc *sc, bus_size_t off) { uint32_t v = bus_space_read_4(sc->sc_iot, sc->sc_ioh, off); DPRINTFn(9, sc->sc_dev, "%s(0x%lx) => 0x%x\n", __func__, off, v); return v; } static void npe_reg_write(struct ixpnpe_softc *sc, bus_size_t off, uint32_t val) { DPRINTFn(9, sc->sc_dev, "%s(0x%lx, 0x%x)\n", __func__, off, val); bus_space_write_4(sc->sc_iot, sc->sc_ioh, off, val); } struct ixpnpe_softc * ixpnpe_attach(device_t dev, int npeid) { struct npeconfig { uint32_t base; uint32_t size; int irq; uint32_t ins_memsize; uint32_t data_memsize; }; static const struct npeconfig npeconfigs[NPE_MAX] = { [NPE_A] = { .base = IXP425_NPE_A_HWBASE, .size = IXP425_NPE_A_SIZE, .irq = IXP425_INT_NPE_A, .ins_memsize = IX_NPEDL_INS_MEMSIZE_WORDS_NPEA, .data_memsize = IX_NPEDL_DATA_MEMSIZE_WORDS_NPEA }, [NPE_B] = { .base = IXP425_NPE_B_HWBASE, .size = IXP425_NPE_B_SIZE, .irq = IXP425_INT_NPE_B, .ins_memsize = IX_NPEDL_INS_MEMSIZE_WORDS_NPEB, .data_memsize = IX_NPEDL_DATA_MEMSIZE_WORDS_NPEB }, [NPE_C] = { .base = IXP425_NPE_C_HWBASE, .size = IXP425_NPE_C_SIZE, .irq = IXP425_INT_NPE_C, .ins_memsize = IX_NPEDL_INS_MEMSIZE_WORDS_NPEC, .data_memsize = IX_NPEDL_DATA_MEMSIZE_WORDS_NPEC }, }; struct ixp425_softc *sa = device_get_softc(device_get_parent(dev)); struct ixpnpe_softc *sc; const struct npeconfig *config; int rid; if (npeid >= NPE_MAX) { device_printf(dev, "%s: bad npeid %d\n", __func__, npeid); return NULL; } sc = npes[npeid]; if (sc != NULL) { sc->sc_nrefs++; return sc; } config = &npeconfigs[npeid]; /* XXX M_BUS */ sc = malloc(sizeof(struct ixpnpe_softc), M_TEMP, M_WAITOK | M_ZERO); sc->sc_dev = dev; sc->sc_iot = sa->sc_iot; mtx_init(&sc->sc_mtx, device_get_nameunit(dev), "npe driver", MTX_DEF); sc->sc_npeid = npeid; sc->sc_nrefs = 1; sc->sc_size = config->size; if (cpu_is_ixp42x()) { /* NB: instruction/data memory sizes are NPE-dependent */ sc->insMemSize = config->ins_memsize; sc->dataMemSize = config->data_memsize; } else { sc->insMemSize = IXP46X_NPEDL_INS_MEMSIZE_WORDS; sc->dataMemSize = IXP46X_NPEDL_DATA_MEMSIZE_WORDS; } if (bus_space_map(sc->sc_iot, config->base, sc->sc_size, 0, &sc->sc_ioh)) panic("%s: Cannot map registers", device_get_name(dev)); /* * Setup IRQ and handler for NPE message support. */ rid = 0; sc->sc_irq = bus_alloc_resource(dev, SYS_RES_IRQ, &rid, config->irq, config->irq, 1, RF_ACTIVE); if (sc->sc_irq == NULL) panic("%s: Unable to allocate irq %u", device_get_name(dev), config->irq); /* XXX could be a source of entropy */ bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET | INTR_MPSAFE, NULL, ixpnpe_intr, sc, &sc->sc_ih); /* * Enable output fifo interrupts (NB: must also set OFIFO Write Enable) */ npe_reg_write(sc, IX_NPECTL, npe_reg_read(sc, IX_NPECTL) | (IX_NPECTL_OFE | IX_NPECTL_OFWE)); npes[npeid] = sc; return sc; } void ixpnpe_detach(struct ixpnpe_softc *sc) { if (--sc->sc_nrefs == 0) { npes[sc->sc_npeid] = NULL; /* disable output fifo interrupts */ npe_reg_write(sc, IX_NPECTL, npe_reg_read(sc, IX_NPECTL) &~ (IX_NPECTL_OFE | IX_NPECTL_OFWE)); bus_teardown_intr(sc->sc_dev, sc->sc_irq, sc->sc_ih); bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_size); mtx_destroy(&sc->sc_mtx); free(sc, M_TEMP); } } int ixpnpe_stopandreset(struct ixpnpe_softc *sc) { int error; mtx_lock(&sc->sc_mtx); error = npe_cpu_stop(sc); /* stop NPE */ if (error == 0) error = npe_cpu_reset(sc); /* reset it */ if (error == 0) sc->started = 0; /* mark stopped */ mtx_unlock(&sc->sc_mtx); DPRINTF(sc->sc_dev, "%s: error %d\n", __func__, error); return error; } static int ixpnpe_start_locked(struct ixpnpe_softc *sc) { int error; if (!sc->started) { error = npe_cpu_start(sc); if (error == 0) sc->started = 1; } else error = 0; DPRINTF(sc->sc_dev, "%s: error %d\n", __func__, error); return error; } int ixpnpe_start(struct ixpnpe_softc *sc) { int ret; mtx_lock(&sc->sc_mtx); ret = ixpnpe_start_locked(sc); mtx_unlock(&sc->sc_mtx); return (ret); } int ixpnpe_stop(struct ixpnpe_softc *sc) { int error; mtx_lock(&sc->sc_mtx); error = npe_cpu_stop(sc); if (error == 0) sc->started = 0; mtx_unlock(&sc->sc_mtx); DPRINTF(sc->sc_dev, "%s: error %d\n", __func__, error); return error; } /* * Indicates the start of an NPE Image, in new NPE Image Library format. * 2 consecutive occurrences indicates the end of the NPE Image Library */ #define NPE_IMAGE_MARKER 0xfeedf00d /* * NPE Image Header definition, used in new NPE Image Library format */ typedef struct { uint32_t marker; uint32_t id; uint32_t size; } IxNpeDlImageMgrImageHeader; static int npe_findimage(struct ixpnpe_softc *sc, const uint32_t *imageLibrary, uint32_t imageId, const uint32_t **imagePtr, uint32_t *imageSize) { const IxNpeDlImageMgrImageHeader *image; uint32_t offset = 0; while (imageLibrary[offset] == NPE_IMAGE_MARKER) { image = (const IxNpeDlImageMgrImageHeader *) &imageLibrary[offset]; offset += sizeof(IxNpeDlImageMgrImageHeader)/sizeof(uint32_t); DPRINTF(sc->sc_dev, "%s: off %u mark 0x%x id 0x%x size %u\n", __func__, offset, image->marker, image->id, image->size); if (image->id == imageId) { *imagePtr = imageLibrary + offset; *imageSize = image->size; return 0; } /* 2 consecutive NPE_IMAGE_MARKER's indicates end of library */ if (image->id == NPE_IMAGE_MARKER) { DPRINTF(sc->sc_dev, "imageId 0x%08x not found in " "image library header\n", imageId); /* reached end of library, image not found */ return ESRCH; } offset += image->size; } return ESRCH; } static int ixpnpe_load_firmware(struct ixpnpe_softc *sc, const char *imageName, uint32_t imageId) { static const char *devname[4] = { "IXP425", "IXP435/IXP465", "DeviceID#2", "DeviceID#3" }; uint32_t imageSize; const uint32_t *imageCodePtr; const struct firmware *fw; int error; DPRINTF(sc->sc_dev, "load %s, imageId 0x%08x\n", imageName, imageId); #if 0 IxFeatureCtrlDeviceId devid = IX_NPEDL_DEVICEID_FROM_IMAGEID_GET(imageId); /* * Checking if image being loaded is meant for device that is running. * Image is forward compatible. i.e Image built for IXP42X should run * on IXP46X but not vice versa. */ if (devid > (ixFeatureCtrlDeviceRead() & IX_FEATURE_CTRL_DEVICE_TYPE_MASK)) return EINVAL; #endif error = ixpnpe_stopandreset(sc); /* stop and reset the NPE */ if (error != 0) return error; fw = firmware_get(imageName); if (fw == NULL) return ENOENT; /* Locate desired image in files w/ combined images */ error = npe_findimage(sc, fw->data, imageId, &imageCodePtr, &imageSize); if (error != 0) goto done; device_printf(sc->sc_dev, "load fw image %s.NPE-%c Func 0x%x Rev %u.%u\n", devname[NPEIMAGE_DEVID(imageId)], 'A' + NPEIMAGE_NPEID(imageId), NPEIMAGE_FUNCID(imageId), NPEIMAGE_MAJOR(imageId), NPEIMAGE_MINOR(imageId)); /* * If download was successful, store image Id in list of - * currently loaded images. If a critical error occured + * currently loaded images. If a critical error occurred * during download, record that the NPE has an invalid image */ mtx_lock(&sc->sc_mtx); error = npe_load_image(sc, imageCodePtr, 1 /*VERIFY*/); if (error == 0) { sc->validImage = 1; error = ixpnpe_start_locked(sc); } else { sc->validImage = 0; } sc->functionalityId = IX_NPEDL_FUNCTIONID_FROM_IMAGEID_GET(imageId); mtx_unlock(&sc->sc_mtx); done: firmware_put(fw, FIRMWARE_UNLOAD); DPRINTF(sc->sc_dev, "%s: error %d\n", __func__, error); return error; } static int override_imageid(device_t dev, const char *resname, uint32_t *val) { int unit = device_get_unit(dev); int resval; if (resource_int_value("npe", unit, resname, &resval) != 0) return 0; /* XXX validate */ if (bootverbose) device_printf(dev, "using npe.%d.%s=0x%x override\n", unit, resname, resval); *val = resval; return 1; } int ixpnpe_init(struct ixpnpe_softc *sc) { static const uint32_t npeconfig[NPE_MAX] = { [NPE_A] = IXP425_NPE_A_IMAGEID, [NPE_B] = IXP425_NPE_B_IMAGEID, [NPE_C] = IXP425_NPE_C_IMAGEID, }; uint32_t imageid, msg[2]; int error; if (sc->started) return 0; /* * Load NPE firmware and start it running. We assume * that minor version bumps remain compatible so probe * the firmware image starting with the expected version * and then bump the minor version up to the max. */ if (!override_imageid(sc->sc_dev, "imageid", &imageid)) imageid = npeconfig[sc->sc_npeid]; for (;;) { error = ixpnpe_load_firmware(sc, "npe_fw", imageid); if (error == 0) break; /* * ESRCH is returned when the requested image * is not present */ if (error != ESRCH) { device_printf(sc->sc_dev, "cannot init NPE (error %d)\n", error); return error; } /* bump the minor version up to the max possible */ if (NPEIMAGE_MINOR(imageid) == 0xff) { device_printf(sc->sc_dev, "cannot locate firmware " "(imageid 0x%08x)\n", imageid); return error; } imageid++; } /* NB: firmware should respond with a status msg */ if (ixpnpe_recvmsg_sync(sc, msg) != 0) { device_printf(sc->sc_dev, "firmware did not respond as expected\n"); return EIO; } return 0; } int ixpnpe_getfunctionality(struct ixpnpe_softc *sc) { return (sc->validImage ? sc->functionalityId : 0); } static int npe_checkbits(struct ixpnpe_softc *sc, uint32_t reg, uint32_t expectedBitsSet) { uint32_t val; val = npe_reg_read(sc, reg); DPRINTFn(5, sc->sc_dev, "%s(0x%x, 0x%x) => 0x%x (%u)\n", __func__, reg, expectedBitsSet, val, (val & expectedBitsSet) == expectedBitsSet); return ((val & expectedBitsSet) == expectedBitsSet); } static int npe_isstopped(struct ixpnpe_softc *sc) { return npe_checkbits(sc, IX_NPEDL_REG_OFFSET_EXCTL, IX_NPEDL_EXCTL_STATUS_STOP); } static int npe_load_ins(struct ixpnpe_softc *sc, const IxNpeDlNpeMgrCodeBlock *bp, int verify) { uint32_t npeMemAddress; int i, blockSize; npeMemAddress = bp->npeMemAddress; blockSize = bp->size; /* NB: instruction/data count */ if (npeMemAddress + blockSize > sc->insMemSize) { device_printf(sc->sc_dev, "Block size %u too big for NPE memory\n", blockSize); return EINVAL; /* XXX */ } for (i = 0; i < blockSize; i++, npeMemAddress++) { if (npe_ins_write(sc, npeMemAddress, bp->data[i], verify) != 0) { device_printf(sc->sc_dev, "NPE instruction write failed"); return EIO; } } return 0; } static int npe_load_data(struct ixpnpe_softc *sc, const IxNpeDlNpeMgrCodeBlock *bp, int verify) { uint32_t npeMemAddress; int i, blockSize; npeMemAddress = bp->npeMemAddress; blockSize = bp->size; /* NB: instruction/data count */ if (npeMemAddress + blockSize > sc->dataMemSize) { device_printf(sc->sc_dev, "Block size %u too big for NPE memory\n", blockSize); return EINVAL; } for (i = 0; i < blockSize; i++, npeMemAddress++) { if (npe_data_write(sc, npeMemAddress, bp->data[i], verify) != 0) { device_printf(sc->sc_dev, "NPE data write failed\n"); return EIO; } } return 0; } static int npe_load_stateinfo(struct ixpnpe_softc *sc, const IxNpeDlNpeMgrStateInfoBlock *bp, int verify) { int i, nentries, error; npe_cpu_step_save(sc); /* for each state-info context register entry in block */ nentries = bp->size / IX_NPEDL_STATE_INFO_ENTRY_SIZE; error = 0; for (i = 0; i < nentries; i++) { /* each state-info entry is 2 words (address, value) */ uint32_t regVal = bp->ctxtRegEntry[i].value; uint32_t addrInfo = bp->ctxtRegEntry[i].addressInfo; uint32_t reg = (addrInfo & IX_NPEDL_MASK_STATE_ADDR_CTXT_REG); uint32_t cNum = (addrInfo & IX_NPEDL_MASK_STATE_ADDR_CTXT_NUM) >> IX_NPEDL_OFFSET_STATE_ADDR_CTXT_NUM; /* error-check Context Register No. and Context Number values */ if (!(0 <= reg && reg < IX_NPEDL_CTXT_REG_MAX)) { device_printf(sc->sc_dev, "invalid Context Register %u\n", reg); error = EINVAL; break; } if (!(0 <= cNum && cNum < IX_NPEDL_CTXT_NUM_MAX)) { device_printf(sc->sc_dev, "invalid Context Number %u\n", cNum); error = EINVAL; break; } /* NOTE that there is no STEVT register for Context 0 */ if (cNum == 0 && reg == IX_NPEDL_CTXT_REG_STEVT) { device_printf(sc->sc_dev, "no STEVT for Context 0\n"); error = EINVAL; break; } if (npe_ctx_reg_write(sc, cNum, reg, regVal, verify) != 0) { device_printf(sc->sc_dev, "write of state-info to NPE failed\n"); error = EIO; break; } } npe_cpu_step_restore(sc); return error; } static int npe_load_image(struct ixpnpe_softc *sc, const uint32_t *imageCodePtr, int verify) { #define EOM(marker) ((marker) == IX_NPEDL_END_OF_DOWNLOAD_MAP) const IxNpeDlNpeMgrDownloadMap *downloadMap; int i, error; if (!npe_isstopped(sc)) { /* verify NPE is stopped */ device_printf(sc->sc_dev, "cannot load image, NPE not stopped\n"); return EIO; } /* * Read Download Map, checking each block type and calling * appropriate function to perform download */ error = 0; downloadMap = (const IxNpeDlNpeMgrDownloadMap *) imageCodePtr; for (i = 0; !EOM(downloadMap->entry[i].eodmMarker); i++) { /* calculate pointer to block to be downloaded */ const uint32_t *bp = imageCodePtr + downloadMap->entry[i].block.offset; switch (downloadMap->entry[i].block.type) { case IX_NPEDL_BLOCK_TYPE_INSTRUCTION: error = npe_load_ins(sc, (const IxNpeDlNpeMgrCodeBlock *) bp, verify); DPRINTF(sc->sc_dev, "%s: inst, error %d\n", __func__, error); break; case IX_NPEDL_BLOCK_TYPE_DATA: error = npe_load_data(sc, (const IxNpeDlNpeMgrCodeBlock *) bp, verify); DPRINTF(sc->sc_dev, "%s: data, error %d\n", __func__, error); break; case IX_NPEDL_BLOCK_TYPE_STATE: error = npe_load_stateinfo(sc, (const IxNpeDlNpeMgrStateInfoBlock *) bp, verify); DPRINTF(sc->sc_dev, "%s: state, error %d\n", __func__, error); break; default: device_printf(sc->sc_dev, "unknown block type 0x%x in download map\n", downloadMap->entry[i].block.type); error = EIO; /* XXX */ break; } if (error != 0) break; } return error; #undef EOM } /* contains Reset values for Context Store Registers */ static const struct { uint32_t regAddr; uint32_t regResetVal; } ixNpeDlEcsRegResetValues[] = { { IX_NPEDL_ECS_BG_CTXT_REG_0, IX_NPEDL_ECS_BG_CTXT_REG_0_RESET }, { IX_NPEDL_ECS_BG_CTXT_REG_1, IX_NPEDL_ECS_BG_CTXT_REG_1_RESET }, { IX_NPEDL_ECS_BG_CTXT_REG_2, IX_NPEDL_ECS_BG_CTXT_REG_2_RESET }, { IX_NPEDL_ECS_PRI_1_CTXT_REG_0, IX_NPEDL_ECS_PRI_1_CTXT_REG_0_RESET }, { IX_NPEDL_ECS_PRI_1_CTXT_REG_1, IX_NPEDL_ECS_PRI_1_CTXT_REG_1_RESET }, { IX_NPEDL_ECS_PRI_1_CTXT_REG_2, IX_NPEDL_ECS_PRI_1_CTXT_REG_2_RESET }, { IX_NPEDL_ECS_PRI_2_CTXT_REG_0, IX_NPEDL_ECS_PRI_2_CTXT_REG_0_RESET }, { IX_NPEDL_ECS_PRI_2_CTXT_REG_1, IX_NPEDL_ECS_PRI_2_CTXT_REG_1_RESET }, { IX_NPEDL_ECS_PRI_2_CTXT_REG_2, IX_NPEDL_ECS_PRI_2_CTXT_REG_2_RESET }, { IX_NPEDL_ECS_DBG_CTXT_REG_0, IX_NPEDL_ECS_DBG_CTXT_REG_0_RESET }, { IX_NPEDL_ECS_DBG_CTXT_REG_1, IX_NPEDL_ECS_DBG_CTXT_REG_1_RESET }, { IX_NPEDL_ECS_DBG_CTXT_REG_2, IX_NPEDL_ECS_DBG_CTXT_REG_2_RESET }, { IX_NPEDL_ECS_INSTRUCT_REG, IX_NPEDL_ECS_INSTRUCT_REG_RESET } }; /* contains Reset values for Context Store Registers */ static const uint32_t ixNpeDlCtxtRegResetValues[] = { IX_NPEDL_CTXT_REG_RESET_STEVT, IX_NPEDL_CTXT_REG_RESET_STARTPC, IX_NPEDL_CTXT_REG_RESET_REGMAP, IX_NPEDL_CTXT_REG_RESET_CINDEX, }; #define IX_NPEDL_PARITY_BIT_MASK 0x3F00FFFF #define IX_NPEDL_CONFIG_CTRL_REG_MASK 0x3F3FFFFF #if 0 /* * Reset the NPE and its coprocessor using the * fuse bits in the feature control register. */ static void npe_reset(int npeid) { uint32_t mask = EXP_FCTRL_NPEA << npeid; uint32_t v; v = ixp4xx_read_feature_bits(); ixp4xx_write_feature_bits(v &~ mask); /* un-fuse and un-reset the NPE & coprocessor */ ixp4xx_write_feature_bits(v | mask); } #endif static int npe_cpu_reset(struct ixpnpe_softc *sc) { #define N(a) (sizeof(a) / sizeof(a[0])) uint32_t ctxtReg; /* identifies Context Store reg (0-3) */ uint32_t regAddr; uint32_t regVal; uint32_t ixNpeConfigCtrlRegVal; int i, error = 0; /* pre-store the NPE Config Control Register Value */ ixNpeConfigCtrlRegVal = npe_reg_read(sc, IX_NPEDL_REG_OFFSET_CTL); ixNpeConfigCtrlRegVal |= 0x3F000000; /* disable the parity interrupt */ npe_reg_write(sc, IX_NPEDL_REG_OFFSET_CTL, (ixNpeConfigCtrlRegVal & IX_NPEDL_PARITY_BIT_MASK)); DPRINTFn(2, sc->sc_dev, "%s: dis parity int, CTL => 0x%x\n", __func__, ixNpeConfigCtrlRegVal & IX_NPEDL_PARITY_BIT_MASK); npe_cpu_step_save(sc); /* * Clear the FIFOs. */ while (npe_checkbits(sc, IX_NPEDL_REG_OFFSET_WFIFO, IX_NPEDL_MASK_WFIFO_VALID)) { /* read from the Watch-point FIFO until empty */ (void) npe_reg_read(sc, IX_NPEDL_REG_OFFSET_WFIFO); } while (npe_checkbits(sc, IX_NPEDL_REG_OFFSET_STAT, IX_NPEDL_MASK_STAT_OFNE)) { /* read from the outFIFO until empty */ (void) npe_reg_read(sc, IX_NPEDL_REG_OFFSET_FIFO); } while (npe_checkbits(sc, IX_NPEDL_REG_OFFSET_STAT, IX_NPEDL_MASK_STAT_IFNE)) { /* - * Step execution of the NPE intruction to read inFIFO using + * Step execution of the NPE instruction to read inFIFO using * the Debug Executing Context stack. */ error = npe_cpu_step(sc, IX_NPEDL_INSTR_RD_FIFO, 0, 0); if (error != 0) { DPRINTF(sc->sc_dev, "%s: cannot step (1), error %u\n", __func__, error); npe_cpu_step_restore(sc); return error; } } /* * Reset the mailbox reg */ /* ...from XScale side */ npe_reg_write(sc, IX_NPEDL_REG_OFFSET_MBST, IX_NPEDL_REG_RESET_MBST); /* ...from NPE side */ error = npe_cpu_step(sc, IX_NPEDL_INSTR_RESET_MBOX, 0, 0); if (error != 0) { DPRINTF(sc->sc_dev, "%s: cannot step (2), error %u\n", __func__, error); npe_cpu_step_restore(sc); return error; } /* * Reset the physical registers in the NPE register file: * Note: no need to save/restore REGMAP for Context 0 here * since all Context Store regs are reset in subsequent code. */ for (regAddr = 0; regAddr < IX_NPEDL_TOTAL_NUM_PHYS_REG && error == 0; regAddr++) { /* for each physical register in the NPE reg file, write 0 : */ error = npe_physical_reg_write(sc, regAddr, 0, TRUE); if (error != 0) { DPRINTF(sc->sc_dev, "%s: cannot write phy reg," "error %u\n", __func__, error); npe_cpu_step_restore(sc); return error; /* abort reset */ } } /* * Reset the context store: */ for (i = IX_NPEDL_CTXT_NUM_MIN; i <= IX_NPEDL_CTXT_NUM_MAX; i++) { /* set each context's Context Store registers to reset values */ for (ctxtReg = 0; ctxtReg < IX_NPEDL_CTXT_REG_MAX; ctxtReg++) { /* NOTE that there is no STEVT register for Context 0 */ if (i == 0 && ctxtReg == IX_NPEDL_CTXT_REG_STEVT) continue; regVal = ixNpeDlCtxtRegResetValues[ctxtReg]; error = npe_ctx_reg_write(sc, i, ctxtReg, regVal, TRUE); if (error != 0) { DPRINTF(sc->sc_dev, "%s: cannot write ctx reg," "error %u\n", __func__, error); npe_cpu_step_restore(sc); return error; /* abort reset */ } } } npe_cpu_step_restore(sc); /* write Reset values to Execution Context Stack registers */ for (i = 0; i < N(ixNpeDlEcsRegResetValues); i++) npe_ecs_reg_write(sc, ixNpeDlEcsRegResetValues[i].regAddr, ixNpeDlEcsRegResetValues[i].regResetVal); /* clear the profile counter */ npe_issue_cmd(sc, IX_NPEDL_EXCTL_CMD_CLR_PROFILE_CNT); /* clear registers EXCT, AP0, AP1, AP2 and AP3 */ for (regAddr = IX_NPEDL_REG_OFFSET_EXCT; regAddr <= IX_NPEDL_REG_OFFSET_AP3; regAddr += sizeof(uint32_t)) npe_reg_write(sc, regAddr, 0); /* Reset the Watch-count register */ npe_reg_write(sc, IX_NPEDL_REG_OFFSET_WC, 0); #if 0 /* * WR IXA00055043 - Remove IMEM Parity Introduced by NPE Reset Operation * XXX Removed because it breaks IXP435 operation; e.g. on Gateworks * XXX 2358 boards reseting NPE-A after NPE-C is running causes both * XXX npe's to stop working */ npe_reset(sc->sc_npeid); #endif /* * Call NpeMgr function to stop the NPE again after the Feature Control * has unfused and Un-Reset the NPE and its associated Coprocessors. */ error = npe_cpu_stop(sc); /* restore NPE configuration bus Control Register - Parity Settings */ npe_reg_write(sc, IX_NPEDL_REG_OFFSET_CTL, (ixNpeConfigCtrlRegVal & IX_NPEDL_CONFIG_CTRL_REG_MASK)); DPRINTFn(2, sc->sc_dev, "%s: restore CTL => 0x%x\n", __func__, npe_reg_read(sc, IX_NPEDL_REG_OFFSET_CTL)); return error; #undef N } static int npe_cpu_start(struct ixpnpe_softc *sc) { uint32_t ecsRegVal; /* * Ensure only Background Context Stack Level is Active by turning off * the Active bit in each of the other Executing Context Stack levels. */ ecsRegVal = npe_ecs_reg_read(sc, IX_NPEDL_ECS_PRI_1_CTXT_REG_0); ecsRegVal &= ~IX_NPEDL_MASK_ECS_REG_0_ACTIVE; npe_ecs_reg_write(sc, IX_NPEDL_ECS_PRI_1_CTXT_REG_0, ecsRegVal); ecsRegVal = npe_ecs_reg_read(sc, IX_NPEDL_ECS_PRI_2_CTXT_REG_0); ecsRegVal &= ~IX_NPEDL_MASK_ECS_REG_0_ACTIVE; npe_ecs_reg_write(sc, IX_NPEDL_ECS_PRI_2_CTXT_REG_0, ecsRegVal); ecsRegVal = npe_ecs_reg_read(sc, IX_NPEDL_ECS_DBG_CTXT_REG_0); ecsRegVal &= ~IX_NPEDL_MASK_ECS_REG_0_ACTIVE; npe_ecs_reg_write(sc, IX_NPEDL_ECS_DBG_CTXT_REG_0, ecsRegVal); /* clear the pipeline */ npe_issue_cmd(sc, IX_NPEDL_EXCTL_CMD_NPE_CLR_PIPE); /* start NPE execution by issuing cmd through EXCTL register on NPE */ npe_issue_cmd(sc, IX_NPEDL_EXCTL_CMD_NPE_START); /* * Check execution status of NPE to verify operation was successful. */ return npe_checkbits(sc, IX_NPEDL_REG_OFFSET_EXCTL, IX_NPEDL_EXCTL_STATUS_RUN) ? 0 : EIO; } static int npe_cpu_stop(struct ixpnpe_softc *sc) { /* stop NPE execution by issuing cmd through EXCTL register on NPE */ npe_issue_cmd(sc, IX_NPEDL_EXCTL_CMD_NPE_STOP); /* verify that NPE Stop was successful */ return npe_checkbits(sc, IX_NPEDL_REG_OFFSET_EXCTL, IX_NPEDL_EXCTL_STATUS_STOP) ? 0 : EIO; } #define IX_NPEDL_REG_SIZE_BYTE 8 #define IX_NPEDL_REG_SIZE_SHORT 16 #define IX_NPEDL_REG_SIZE_WORD 32 /* * Introduce extra read cycles after issuing read command to NPE * so that we read the register after the NPE has updated it * This is to overcome race condition between XScale and NPE */ #define IX_NPEDL_DELAY_READ_CYCLES 2 /* * To mask top three MSBs of 32bit word to download into NPE IMEM */ #define IX_NPEDL_MASK_UNUSED_IMEM_BITS 0x1FFFFFFF; static void npe_cmd_issue_write(struct ixpnpe_softc *sc, uint32_t cmd, uint32_t addr, uint32_t data) { npe_reg_write(sc, IX_NPEDL_REG_OFFSET_EXDATA, data); npe_reg_write(sc, IX_NPEDL_REG_OFFSET_EXAD, addr); npe_reg_write(sc, IX_NPEDL_REG_OFFSET_EXCTL, cmd); } static uint32_t npe_cmd_issue_read(struct ixpnpe_softc *sc, uint32_t cmd, uint32_t addr) { uint32_t data; int i; npe_reg_write(sc, IX_NPEDL_REG_OFFSET_EXAD, addr); npe_reg_write(sc, IX_NPEDL_REG_OFFSET_EXCTL, cmd); for (i = 0; i <= IX_NPEDL_DELAY_READ_CYCLES; i++) data = npe_reg_read(sc, IX_NPEDL_REG_OFFSET_EXDATA); return data; } static int npe_ins_write(struct ixpnpe_softc *sc, uint32_t addr, uint32_t data, int verify) { DPRINTFn(4, sc->sc_dev, "%s(0x%x, 0x%x)\n", __func__, addr, data); npe_cmd_issue_write(sc, IX_NPEDL_EXCTL_CMD_WR_INS_MEM, addr, data); if (verify) { uint32_t rdata; /* * Write invalid data to this reg, so we can see if we're * reading the EXDATA register too early. */ npe_reg_write(sc, IX_NPEDL_REG_OFFSET_EXDATA, ~data); /* * Disabled since top 3 MSB are not used for Azusa * hardware Refer WR:IXA00053900 */ data &= IX_NPEDL_MASK_UNUSED_IMEM_BITS; rdata = npe_cmd_issue_read(sc, IX_NPEDL_EXCTL_CMD_RD_INS_MEM, addr); rdata &= IX_NPEDL_MASK_UNUSED_IMEM_BITS; if (data != rdata) return EIO; } return 0; } static int npe_data_write(struct ixpnpe_softc *sc, uint32_t addr, uint32_t data, int verify) { DPRINTFn(4, sc->sc_dev, "%s(0x%x, 0x%x)\n", __func__, addr, data); npe_cmd_issue_write(sc, IX_NPEDL_EXCTL_CMD_WR_DATA_MEM, addr, data); if (verify) { /* * Write invalid data to this reg, so we can see if we're * reading the EXDATA register too early. */ npe_reg_write(sc, IX_NPEDL_REG_OFFSET_EXDATA, ~data); if (data != npe_cmd_issue_read(sc, IX_NPEDL_EXCTL_CMD_RD_DATA_MEM, addr)) return EIO; } return 0; } static void npe_ecs_reg_write(struct ixpnpe_softc *sc, uint32_t reg, uint32_t data) { npe_cmd_issue_write(sc, IX_NPEDL_EXCTL_CMD_WR_ECS_REG, reg, data); } static uint32_t npe_ecs_reg_read(struct ixpnpe_softc *sc, uint32_t reg) { return npe_cmd_issue_read(sc, IX_NPEDL_EXCTL_CMD_RD_ECS_REG, reg); } static void npe_issue_cmd(struct ixpnpe_softc *sc, uint32_t command) { npe_reg_write(sc, IX_NPEDL_REG_OFFSET_EXCTL, command); } static void npe_cpu_step_save(struct ixpnpe_softc *sc) { /* turn off the halt bit by clearing Execution Count register. */ /* save reg contents 1st and restore later */ sc->savedExecCount = npe_reg_read(sc, IX_NPEDL_REG_OFFSET_EXCT); npe_reg_write(sc, IX_NPEDL_REG_OFFSET_EXCT, 0); /* ensure that IF and IE are on (temporarily), so that we don't end up * stepping forever */ sc->savedEcsDbgCtxtReg2 = npe_ecs_reg_read(sc, IX_NPEDL_ECS_DBG_CTXT_REG_2); npe_ecs_reg_write(sc, IX_NPEDL_ECS_DBG_CTXT_REG_2, (sc->savedEcsDbgCtxtReg2 | IX_NPEDL_MASK_ECS_DBG_REG_2_IF | IX_NPEDL_MASK_ECS_DBG_REG_2_IE)); } static int npe_cpu_step(struct ixpnpe_softc *sc, uint32_t npeInstruction, uint32_t ctxtNum, uint32_t ldur) { #define IX_NPE_DL_MAX_NUM_OF_RETRIES 1000000 uint32_t ecsDbgRegVal; uint32_t oldWatchcount, newWatchcount; int tries; /* set the Active bit, and the LDUR, in the debug level */ ecsDbgRegVal = IX_NPEDL_MASK_ECS_REG_0_ACTIVE | (ldur << IX_NPEDL_OFFSET_ECS_REG_0_LDUR); npe_ecs_reg_write(sc, IX_NPEDL_ECS_DBG_CTXT_REG_0, ecsDbgRegVal); /* * Set CCTXT at ECS DEBUG L3 to specify in which context to execute the * instruction, and set SELCTXT at ECS DEBUG Level to specify which * context store to access. * Debug ECS Level Reg 1 has form 0x000n000n, where n = context number */ ecsDbgRegVal = (ctxtNum << IX_NPEDL_OFFSET_ECS_REG_1_CCTXT) | (ctxtNum << IX_NPEDL_OFFSET_ECS_REG_1_SELCTXT); npe_ecs_reg_write(sc, IX_NPEDL_ECS_DBG_CTXT_REG_1, ecsDbgRegVal); /* clear the pipeline */ npe_issue_cmd(sc, IX_NPEDL_EXCTL_CMD_NPE_CLR_PIPE); /* load NPE instruction into the instruction register */ npe_ecs_reg_write(sc, IX_NPEDL_ECS_INSTRUCT_REG, npeInstruction); /* need this value later to wait for completion of NPE execution step */ oldWatchcount = npe_reg_read(sc, IX_NPEDL_REG_OFFSET_WC); /* issue a Step One command via the Execution Control register */ npe_issue_cmd(sc, IX_NPEDL_EXCTL_CMD_NPE_STEP); /* * Force the XScale to wait until the NPE has finished execution step * NOTE that this delay will be very small, just long enough to allow a * single NPE instruction to complete execution; if instruction * execution is not completed before timeout retries, exit the while * loop. */ newWatchcount = npe_reg_read(sc, IX_NPEDL_REG_OFFSET_WC); for (tries = 0; tries < IX_NPE_DL_MAX_NUM_OF_RETRIES && newWatchcount == oldWatchcount; tries++) { /* Watch Count register incr's when NPE completes an inst */ newWatchcount = npe_reg_read(sc, IX_NPEDL_REG_OFFSET_WC); } return (tries < IX_NPE_DL_MAX_NUM_OF_RETRIES) ? 0 : EIO; #undef IX_NPE_DL_MAX_NUM_OF_RETRIES } static void npe_cpu_step_restore(struct ixpnpe_softc *sc) { /* clear active bit in debug level */ npe_ecs_reg_write(sc, IX_NPEDL_ECS_DBG_CTXT_REG_0, 0); /* clear the pipeline */ npe_issue_cmd(sc, IX_NPEDL_EXCTL_CMD_NPE_CLR_PIPE); /* restore Execution Count register contents. */ npe_reg_write(sc, IX_NPEDL_REG_OFFSET_EXCT, sc->savedExecCount); /* restore IF and IE bits to original values */ npe_ecs_reg_write(sc, IX_NPEDL_ECS_DBG_CTXT_REG_2, sc->savedEcsDbgCtxtReg2); } static int npe_logical_reg_read(struct ixpnpe_softc *sc, uint32_t regAddr, uint32_t regSize, uint32_t ctxtNum, uint32_t *regVal) { uint32_t npeInstruction, mask; int error; switch (regSize) { case IX_NPEDL_REG_SIZE_BYTE: npeInstruction = IX_NPEDL_INSTR_RD_REG_BYTE; mask = 0xff; break; case IX_NPEDL_REG_SIZE_SHORT: npeInstruction = IX_NPEDL_INSTR_RD_REG_SHORT; mask = 0xffff; break; case IX_NPEDL_REG_SIZE_WORD: npeInstruction = IX_NPEDL_INSTR_RD_REG_WORD; mask = 0xffffffff; break; default: return EINVAL; } /* make regAddr be the SRC and DEST operands (e.g. movX d0, d0) */ npeInstruction |= (regAddr << IX_NPEDL_OFFSET_INSTR_SRC) | (regAddr << IX_NPEDL_OFFSET_INSTR_DEST); /* step execution of NPE inst using Debug Executing Context stack */ error = npe_cpu_step(sc, npeInstruction, ctxtNum, IX_NPEDL_RD_INSTR_LDUR); if (error != 0) { DPRINTF(sc->sc_dev, "%s(0x%x, %u, %u), cannot step, error %d\n", __func__, regAddr, regSize, ctxtNum, error); return error; } /* read value of register from Execution Data register */ *regVal = npe_reg_read(sc, IX_NPEDL_REG_OFFSET_EXDATA); /* align value from left to right */ *regVal = (*regVal >> (IX_NPEDL_REG_SIZE_WORD - regSize)) & mask; return 0; } static int npe_logical_reg_write(struct ixpnpe_softc *sc, uint32_t regAddr, uint32_t regVal, uint32_t regSize, uint32_t ctxtNum, int verify) { int error; DPRINTFn(4, sc->sc_dev, "%s(0x%x, 0x%x, %u, %u)\n", __func__, regAddr, regVal, regSize, ctxtNum); if (regSize == IX_NPEDL_REG_SIZE_WORD) { /* * NPE register addressing is left-to-right: e.g. |d0|d1|d2|d3| * Write upper half-word (short) to |d0|d1| */ error = npe_logical_reg_write(sc, regAddr, regVal >> IX_NPEDL_REG_SIZE_SHORT, IX_NPEDL_REG_SIZE_SHORT, ctxtNum, verify); if (error != 0) return error; /* Write lower half-word (short) to |d2|d3| */ error = npe_logical_reg_write(sc, regAddr + sizeof(uint16_t), regVal & 0xffff, IX_NPEDL_REG_SIZE_SHORT, ctxtNum, verify); } else { uint32_t npeInstruction; switch (regSize) { case IX_NPEDL_REG_SIZE_BYTE: npeInstruction = IX_NPEDL_INSTR_WR_REG_BYTE; regVal &= 0xff; break; case IX_NPEDL_REG_SIZE_SHORT: npeInstruction = IX_NPEDL_INSTR_WR_REG_SHORT; regVal &= 0xffff; break; default: return EINVAL; } /* fill dest operand field of inst with dest reg addr */ npeInstruction |= (regAddr << IX_NPEDL_OFFSET_INSTR_DEST); /* fill src operand field of inst with least-sig 5 bits of val*/ npeInstruction |= ((regVal & IX_NPEDL_MASK_IMMED_INSTR_SRC_DATA) << IX_NPEDL_OFFSET_INSTR_SRC); /* fill coprocessor field of inst with most-sig 11 bits of val*/ npeInstruction |= ((regVal & IX_NPEDL_MASK_IMMED_INSTR_COPROC_DATA) << IX_NPEDL_DISPLACE_IMMED_INSTR_COPROC_DATA); - /* step execution of NPE intruction using Debug ECS */ + /* step execution of NPE instruction using Debug ECS */ error = npe_cpu_step(sc, npeInstruction, ctxtNum, IX_NPEDL_WR_INSTR_LDUR); } if (error != 0) { DPRINTF(sc->sc_dev, "%s(0x%x, 0x%x, %u, %u), error %u " "writing reg\n", __func__, regAddr, regVal, regSize, ctxtNum, error); return error; } if (verify) { uint32_t retRegVal; error = npe_logical_reg_read(sc, regAddr, regSize, ctxtNum, &retRegVal); if (error == 0 && regVal != retRegVal) error = EIO; /* XXX ambiguous */ } return error; } /* * There are 32 physical registers used in an NPE. These are * treated as 16 pairs of 32-bit registers. To write one of the pair, * write the pair number (0-16) to the REGMAP for Context 0. Then write * the value to register 0 or 4 in the regfile, depending on which * register of the pair is to be written */ static int npe_physical_reg_write(struct ixpnpe_softc *sc, uint32_t regAddr, uint32_t regValue, int verify) { int error; /* * Set REGMAP for context 0 to (regAddr >> 1) to choose which pair * (0-16) of physical registers to write . */ error = npe_logical_reg_write(sc, IX_NPEDL_CTXT_REG_ADDR_REGMAP, (regAddr >> IX_NPEDL_OFFSET_PHYS_REG_ADDR_REGMAP), IX_NPEDL_REG_SIZE_SHORT, 0, verify); if (error == 0) { /* regAddr = 0 or 4 */ regAddr = (regAddr & IX_NPEDL_MASK_PHYS_REG_ADDR_LOGICAL_ADDR) * sizeof(uint32_t); error = npe_logical_reg_write(sc, regAddr, regValue, IX_NPEDL_REG_SIZE_WORD, 0, verify); } return error; } static int npe_ctx_reg_write(struct ixpnpe_softc *sc, uint32_t ctxtNum, uint32_t ctxtReg, uint32_t ctxtRegVal, int verify) { DPRINTFn(4, sc->sc_dev, "%s(%u, %u, %u)\n", __func__, ctxtNum, ctxtReg, ctxtRegVal); /* * Context 0 has no STARTPC. Instead, this value is used to set * NextPC for Background ECS, to set where NPE starts executing code */ if (ctxtNum == 0 && ctxtReg == IX_NPEDL_CTXT_REG_STARTPC) { /* read BG_CTXT_REG_0, update NEXTPC bits, & write back to reg*/ uint32_t v = npe_ecs_reg_read(sc, IX_NPEDL_ECS_BG_CTXT_REG_0); v &= ~IX_NPEDL_MASK_ECS_REG_0_NEXTPC; v |= (ctxtRegVal << IX_NPEDL_OFFSET_ECS_REG_0_NEXTPC) & IX_NPEDL_MASK_ECS_REG_0_NEXTPC; npe_ecs_reg_write(sc, IX_NPEDL_ECS_BG_CTXT_REG_0, v); return 0; } else { static const struct { uint32_t regAddress; uint32_t regSize; } regAccInfo[IX_NPEDL_CTXT_REG_MAX] = { { IX_NPEDL_CTXT_REG_ADDR_STEVT, IX_NPEDL_REG_SIZE_BYTE }, { IX_NPEDL_CTXT_REG_ADDR_STARTPC, IX_NPEDL_REG_SIZE_SHORT }, { IX_NPEDL_CTXT_REG_ADDR_REGMAP, IX_NPEDL_REG_SIZE_SHORT }, { IX_NPEDL_CTXT_REG_ADDR_CINDEX, IX_NPEDL_REG_SIZE_BYTE } }; return npe_logical_reg_write(sc, regAccInfo[ctxtReg].regAddress, ctxtRegVal, regAccInfo[ctxtReg].regSize, ctxtNum, verify); } } /* * NPE Mailbox support. */ #define IX_NPEMH_MAXTRIES 100000 static int ofifo_wait(struct ixpnpe_softc *sc) { int i; for (i = 0; i < IX_NPEMH_MAXTRIES; i++) { if (npe_reg_read(sc, IX_NPESTAT) & IX_NPESTAT_OFNE) return 1; DELAY(10); } device_printf(sc->sc_dev, "%s: timeout, last status 0x%x\n", __func__, npe_reg_read(sc, IX_NPESTAT)); return 0; } static int getmsg(struct ixpnpe_softc *sc, uint32_t msg[2]) { mtx_assert(&sc->sc_mtx, MA_OWNED); if (!ofifo_wait(sc)) return EAGAIN; msg[0] = npe_reg_read(sc, IX_NPEFIFO); DPRINTF(sc->sc_dev, "%s: msg0 0x%x\n", __func__, msg[0]); if (!ofifo_wait(sc)) return EAGAIN; msg[1] = npe_reg_read(sc, IX_NPEFIFO); DPRINTF(sc->sc_dev, "%s: msg1 0x%x\n", __func__, msg[1]); return 0; } static void ixpnpe_intr(void *arg) { struct ixpnpe_softc *sc = arg; uint32_t status; mtx_lock(&sc->sc_mtx); status = npe_reg_read(sc, IX_NPESTAT); DPRINTF(sc->sc_dev, "%s: status 0x%x\n", __func__, status); if ((status & IX_NPESTAT_OFINT) == 0) { /* NB: should not happen */ device_printf(sc->sc_dev, "%s: status 0x%x\n", __func__, status); /* XXX must silence interrupt? */ mtx_unlock(&sc->sc_mtx); return; } /* * A message is waiting in the output FIFO, copy it so * the interrupt will be silenced. */ if (getmsg(sc, sc->sc_msg) == 0) sc->sc_msgwaiting = 1; mtx_unlock(&sc->sc_mtx); } static int ififo_wait(struct ixpnpe_softc *sc) { int i; for (i = 0; i < IX_NPEMH_MAXTRIES; i++) { if (npe_reg_read(sc, IX_NPESTAT) & IX_NPESTAT_IFNF) return 1; DELAY(10); } device_printf(sc->sc_dev, "%s: timeout, last status 0x%x\n", __func__, npe_reg_read(sc, IX_NPESTAT)); return 0; } static int putmsg(struct ixpnpe_softc *sc, const uint32_t msg[2]) { mtx_assert(&sc->sc_mtx, MA_OWNED); DPRINTF(sc->sc_dev, "%s: msg 0x%x:0x%x\n", __func__, msg[0], msg[1]); if (!ififo_wait(sc)) return EIO; npe_reg_write(sc, IX_NPEFIFO, msg[0]); if (!ififo_wait(sc)) return EIO; npe_reg_write(sc, IX_NPEFIFO, msg[1]); return 0; } /* * Send a msg to the NPE and wait for a reply. We spin as * we may be called early with interrupts not properly setup. */ int ixpnpe_sendandrecvmsg_sync(struct ixpnpe_softc *sc, const uint32_t send[2], uint32_t recv[2]) { int error; mtx_lock(&sc->sc_mtx); error = putmsg(sc, send); if (error == 0) error = getmsg(sc, recv); mtx_unlock(&sc->sc_mtx); return error; } /* * Send a msg to the NPE w/o waiting for a reply. */ int ixpnpe_sendmsg_async(struct ixpnpe_softc *sc, const uint32_t msg[2]) { int error; mtx_lock(&sc->sc_mtx); error = putmsg(sc, msg); mtx_unlock(&sc->sc_mtx); return error; } static int recvmsg_locked(struct ixpnpe_softc *sc, uint32_t msg[2]) { mtx_assert(&sc->sc_mtx, MA_OWNED); DPRINTF(sc->sc_dev, "%s: msgwaiting %d\n", __func__, sc->sc_msgwaiting); if (sc->sc_msgwaiting) { msg[0] = sc->sc_msg[0]; msg[1] = sc->sc_msg[1]; sc->sc_msgwaiting = 0; return 0; } return EAGAIN; } /* * Receive any msg previously received from the NPE. If nothing * is available we return EAGAIN and the caller is required to * do a synchronous receive or try again later. */ int ixpnpe_recvmsg_async(struct ixpnpe_softc *sc, uint32_t msg[2]) { int error; mtx_lock(&sc->sc_mtx); error = recvmsg_locked(sc, msg); mtx_unlock(&sc->sc_mtx); return error; } /* * Receive a msg from the NPE. If one was received asynchronously * then it's returned; otherwise we poll synchronously. */ int ixpnpe_recvmsg_sync(struct ixpnpe_softc *sc, uint32_t msg[2]) { int error; mtx_lock(&sc->sc_mtx); error = recvmsg_locked(sc, msg); if (error == EAGAIN) error = getmsg(sc, msg); mtx_unlock(&sc->sc_mtx); return error; } Index: user/ngie/release-pkg-fix-tests/sys/arm/xscale/ixp425/ixp425_npereg.h =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/xscale/ixp425/ixp425_npereg.h (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/xscale/ixp425/ixp425_npereg.h (revision 299091) @@ -1,427 +1,427 @@ /*- * Copyright (c) 2006 Sam Leffler, Errno Consulting * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer, * without modification. * 2. Redistributions in binary form must reproduce at minimum a disclaimer * similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any * redistribution must be conditioned upon including a substantially * similar Disclaimer requirement for further binary redistribution. * * NO WARRANTY * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGES. * * $FreeBSD$ */ /*- * Copyright (c) 2001-2005, Intel Corporation. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the Intel Corporation nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifndef _IXP425_NPEREG_H_ #define _IXP425_NPEREG_H_ /* signature found as 1st word in a microcode image library */ #define IX_NPEDL_IMAGEMGR_SIGNATURE 0xDEADBEEF /* marks end of header in a microcode image library */ #define IX_NPEDL_IMAGEMGR_END_OF_HEADER 0xFFFFFFFF /* * Intel (R) IXP400 Software NPE Image ID Definition * * Definition of NPE Image ID to be passed to ixNpeDlNpeInitAndStart() * as input of type uint32_t which has the following fields format: * * Field [Bit Location] * ----------------------------------- * Device ID [31 - 28] * NPE ID [27 - 24] * NPE Functionality ID [23 - 16] * Major Release Number [15 - 8] * Minor Release Number [7 - 0] */ #define IX_NPEDL_NPEID_FROM_IMAGEID_GET(imageId) \ (((imageId) >> 24) & 0xf) #define IX_NPEDL_DEVICEID_FROM_IMAGEID_GET(imageId) \ (((imageId) >> 28) & 0xf) #define IX_NPEDL_FUNCTIONID_FROM_IMAGEID_GET(imageId) \ (((imageId) >> 16) & 0xff) #define IX_NPEDL_MAJOR_FROM_IMAGEID_GET(imageId) \ (((imageId) >> 8) & 0xff) #define IX_NPEDL_MINOR_FROM_IMAGEID_GET(imageId) \ (((imageId) >> 0) & 0xff) /* * Instruction and Data Memory Size (in words) for each NPE */ #define IX_NPEDL_INS_MEMSIZE_WORDS_NPEA 4096 #define IX_NPEDL_INS_MEMSIZE_WORDS_NPEB 2048 #define IX_NPEDL_INS_MEMSIZE_WORDS_NPEC 2048 #define IX_NPEDL_DATA_MEMSIZE_WORDS_NPEA 2048 #define IX_NPEDL_DATA_MEMSIZE_WORDS_NPEB 2048 #define IX_NPEDL_DATA_MEMSIZE_WORDS_NPEC 2048 #define IXP46X_NPEDL_INS_MEMSIZE_WORDS 4096 #define IXP46X_NPEDL_DATA_MEMSIZE_WORDS 4096 /* BAR offsets */ #define IX_NPEDL_REG_OFFSET_EXAD 0x00000000 /* Execution Address */ #define IX_NPEDL_REG_OFFSET_EXDATA 0x00000004 /* Execution Data */ #define IX_NPEDL_REG_OFFSET_EXCTL 0x00000008 /* Execution Control */ #define IX_NPEDL_REG_OFFSET_EXCT 0x0000000C /* Execution Count */ #define IX_NPEDL_REG_OFFSET_AP0 0x00000010 /* Action Point 0 */ #define IX_NPEDL_REG_OFFSET_AP1 0x00000014 /* Action Point 1 */ #define IX_NPEDL_REG_OFFSET_AP2 0x00000018 /* Action Point 2 */ #define IX_NPEDL_REG_OFFSET_AP3 0x0000001C /* Action Point 3 */ #define IX_NPEDL_REG_OFFSET_WFIFO 0x00000020 /* Watchpoint FIFO */ #define IX_NPEDL_REG_OFFSET_WC 0x00000024 /* Watch Count */ #define IX_NPEDL_REG_OFFSET_PROFCT 0x00000028 /* Profile Count */ #define IX_NPEDL_REG_OFFSET_STAT 0x0000002C /* Messaging Status */ #define IX_NPEDL_REG_OFFSET_CTL 0x00000030 /* Messaging Control */ #define IX_NPEDL_REG_OFFSET_MBST 0x00000034 /* Mailbox Status */ #define IX_NPEDL_REG_OFFSET_FIFO 0x00000038 /* Message FIFO */ /* * Reset value for Mailbox (MBST) register - * NOTE that if used, it should be complemented with an NPE intruction + * NOTE that if used, it should be complemented with an NPE instruction * to clear the Mailbox at the NPE side as well */ #define IX_NPEDL_REG_RESET_MBST 0x0000F0F0 #define IX_NPEDL_MASK_WFIFO_VALID 0x80000000 /* VALID bit */ #define IX_NPEDL_MASK_STAT_OFNE 0x00010000 /* OFNE bit */ #define IX_NPEDL_MASK_STAT_IFNE 0x00080000 /* IFNE bit */ /* * EXCTL (Execution Control) Register commands */ #define IX_NPEDL_EXCTL_CMD_NPE_STEP 0x01 /* Step 1 instruction */ #define IX_NPEDL_EXCTL_CMD_NPE_START 0x02 /* Start execution */ #define IX_NPEDL_EXCTL_CMD_NPE_STOP 0x03 /* Stop execution */ #define IX_NPEDL_EXCTL_CMD_NPE_CLR_PIPE 0x04 /* Clear ins pipeline */ /* * Read/write operations use address in EXAD and data in EXDATA. */ #define IX_NPEDL_EXCTL_CMD_RD_INS_MEM 0x10 /* Read ins memory */ #define IX_NPEDL_EXCTL_CMD_WR_INS_MEM 0x11 /* Write ins memory */ #define IX_NPEDL_EXCTL_CMD_RD_DATA_MEM 0x12 /* Read data memory */ #define IX_NPEDL_EXCTL_CMD_WR_DATA_MEM 0x13 /* Write data memory */ #define IX_NPEDL_EXCTL_CMD_RD_ECS_REG 0x14 /* Read ECS register */ #define IX_NPEDL_EXCTL_CMD_WR_ECS_REG 0x15 /* Write ECS register */ #define IX_NPEDL_EXCTL_CMD_CLR_PROFILE_CNT 0x0C /* Clear Profile Count register */ /* * EXCTL (Execution Control) Register status bit masks */ #define IX_NPEDL_EXCTL_STATUS_RUN 0x80000000 #define IX_NPEDL_EXCTL_STATUS_STOP 0x40000000 #define IX_NPEDL_EXCTL_STATUS_CLEAR 0x20000000 #define IX_NPEDL_EXCTL_STATUS_ECS_K 0x00800000 /* pipeline Klean */ /* * Executing Context Stack (ECS) level registers */ #define IX_NPEDL_ECS_BG_CTXT_REG_0 0x00 /* reg 0 @ bg ctx */ #define IX_NPEDL_ECS_BG_CTXT_REG_1 0x01 /* reg 1 @ bg ctx */ #define IX_NPEDL_ECS_BG_CTXT_REG_2 0x02 /* reg 2 @ bg ctx */ #define IX_NPEDL_ECS_PRI_1_CTXT_REG_0 0x04 /* reg 0 @ pri 1 ctx */ #define IX_NPEDL_ECS_PRI_1_CTXT_REG_1 0x05 /* reg 1 @ pri 1 ctx */ #define IX_NPEDL_ECS_PRI_1_CTXT_REG_2 0x06 /* reg 2 @ pri 1 ctx */ #define IX_NPEDL_ECS_PRI_2_CTXT_REG_0 0x08 /* reg 0 @ pri 2 ctx */ #define IX_NPEDL_ECS_PRI_2_CTXT_REG_1 0x09 /* reg 1 @ pri 2 ctx */ #define IX_NPEDL_ECS_PRI_2_CTXT_REG_2 0x0A /* reg 2 @ pri 2 ctx */ #define IX_NPEDL_ECS_DBG_CTXT_REG_0 0x0C /* reg 0 @ debug ctx */ #define IX_NPEDL_ECS_DBG_CTXT_REG_1 0x0D /* reg 1 @ debug ctx */ #define IX_NPEDL_ECS_DBG_CTXT_REG_2 0x0E /* reg 2 @ debug ctx */ #define IX_NPEDL_ECS_INSTRUCT_REG 0x11 /* Instruction reg */ /* * Execution Access register reset values */ #define IX_NPEDL_ECS_BG_CTXT_REG_0_RESET 0xA0000000 #define IX_NPEDL_ECS_BG_CTXT_REG_1_RESET 0x01000000 #define IX_NPEDL_ECS_BG_CTXT_REG_2_RESET 0x00008000 #define IX_NPEDL_ECS_PRI_1_CTXT_REG_0_RESET 0x20000080 #define IX_NPEDL_ECS_PRI_1_CTXT_REG_1_RESET 0x01000000 #define IX_NPEDL_ECS_PRI_1_CTXT_REG_2_RESET 0x00008000 #define IX_NPEDL_ECS_PRI_2_CTXT_REG_0_RESET 0x20000080 #define IX_NPEDL_ECS_PRI_2_CTXT_REG_1_RESET 0x01000000 #define IX_NPEDL_ECS_PRI_2_CTXT_REG_2_RESET 0x00008000 #define IX_NPEDL_ECS_DBG_CTXT_REG_0_RESET 0x20000000 #define IX_NPEDL_ECS_DBG_CTXT_REG_1_RESET 0x00000000 #define IX_NPEDL_ECS_DBG_CTXT_REG_2_RESET 0x001E0000 #define IX_NPEDL_ECS_INSTRUCT_REG_RESET 0x1003C00F /* * Masks used to read/write particular bits in Execution Access registers */ #define IX_NPEDL_MASK_ECS_REG_0_ACTIVE 0x80000000 /* Active bit */ #define IX_NPEDL_MASK_ECS_REG_0_NEXTPC 0x1FFF0000 /* NextPC bits */ #define IX_NPEDL_MASK_ECS_REG_0_LDUR 0x00000700 /* LDUR bits */ #define IX_NPEDL_MASK_ECS_REG_1_CCTXT 0x000F0000 /* NextPC bits */ #define IX_NPEDL_MASK_ECS_REG_1_SELCTXT 0x0000000F #define IX_NPEDL_MASK_ECS_DBG_REG_2_IF 0x00100000 /* IF bit */ #define IX_NPEDL_MASK_ECS_DBG_REG_2_IE 0x00080000 /* IE bit */ /* * Bit-Offsets from LSB of particular bit-fields in Execution Access registers. */ #define IX_NPEDL_OFFSET_ECS_REG_0_NEXTPC 16 #define IX_NPEDL_OFFSET_ECS_REG_0_LDUR 8 #define IX_NPEDL_OFFSET_ECS_REG_1_CCTXT 16 #define IX_NPEDL_OFFSET_ECS_REG_1_SELCTXT 0 /* * NPE core & co-processor instruction templates to load into NPE Instruction * Register, for read/write of NPE register file registers. */ /* * Read an 8-bit NPE internal logical register * and return the value in the EXDATA register (aligned to MSB). * NPE Assembler instruction: "mov8 d0, d0 &&& DBG_WrExec" */ #define IX_NPEDL_INSTR_RD_REG_BYTE 0x0FC00000 /* * Read a 16-bit NPE internal logical register * and return the value in the EXDATA register (aligned to MSB). * NPE Assembler instruction: "mov16 d0, d0 &&& DBG_WrExec" */ #define IX_NPEDL_INSTR_RD_REG_SHORT 0x0FC08010 /* * Read a 16-bit NPE internal logical register * and return the value in the EXDATA register. * NPE Assembler instruction: "mov32 d0, d0 &&& DBG_WrExec" */ #define IX_NPEDL_INSTR_RD_REG_WORD 0x0FC08210 /* * Write an 8-bit NPE internal logical register. * NPE Assembler instruction: "mov8 d0, #0" */ #define IX_NPEDL_INSTR_WR_REG_BYTE 0x00004000 /* * Write a 16-bit NPE internal logical register. * NPE Assembler instruction: "mov16 d0, #0" */ #define IX_NPEDL_INSTR_WR_REG_SHORT 0x0000C000 /* * Write a 16-bit NPE internal logical register. * NPE Assembler instruction: "cprd32 d0 &&& DBG_RdInFIFO" */ #define IX_NPEDL_INSTR_RD_FIFO 0x0F888220 /* * Reset Mailbox (MBST) register * NPE Assembler instruction: "mov32 d0, d0 &&& DBG_ClearM" */ #define IX_NPEDL_INSTR_RESET_MBOX 0x0FAC8210 /* * Bit-offsets from LSB, of particular bit-fields in an NPE instruction */ #define IX_NPEDL_OFFSET_INSTR_SRC 4 /* src operand */ #define IX_NPEDL_OFFSET_INSTR_DEST 9 /* dest operand */ #define IX_NPEDL_OFFSET_INSTR_COPROC 18 /* coprocessor ins */ /* * Masks used to read/write particular bits of an NPE Instruction */ /** * Mask the bits of 16-bit data value (least-sig 5 bits) to be used in * SRC field of immediate-mode NPE instruction */ #define IX_NPEDL_MASK_IMMED_INSTR_SRC_DATA 0x1F /** * Mask the bits of 16-bit data value (most-sig 11 bits) to be used in * COPROC field of immediate-mode NPE instruction */ #define IX_NPEDL_MASK_IMMED_INSTR_COPROC_DATA 0xFFE0 /** * LSB offset of the bit-field of 16-bit data value (most-sig 11 bits) * to be used in COPROC field of immediate-mode NPE instruction */ #define IX_NPEDL_OFFSET_IMMED_INSTR_COPROC_DATA 5 /** * Number of left-shifts required to align most-sig 11 bits of 16-bit * data value into COPROC field of immediate-mode NPE instruction */ #define IX_NPEDL_DISPLACE_IMMED_INSTR_COPROC_DATA \ (IX_NPEDL_OFFSET_INSTR_COPROC - IX_NPEDL_OFFSET_IMMED_INSTR_COPROC_DATA) /** * LDUR value used with immediate-mode NPE Instructions by the NpeDl * for writing to NPE internal logical registers */ #define IX_NPEDL_WR_INSTR_LDUR 1 /** * LDUR value used with NON-immediate-mode NPE Instructions by the NpeDl * for reading from NPE internal logical registers */ #define IX_NPEDL_RD_INSTR_LDUR 0 /** * NPE internal Context Store registers. */ typedef enum { IX_NPEDL_CTXT_REG_STEVT = 0, /**< identifies STEVT */ IX_NPEDL_CTXT_REG_STARTPC, /**< identifies STARTPC */ IX_NPEDL_CTXT_REG_REGMAP, /**< identifies REGMAP */ IX_NPEDL_CTXT_REG_CINDEX, /**< identifies CINDEX */ IX_NPEDL_CTXT_REG_MAX /**< Total number of Context Store registers */ } IxNpeDlCtxtRegNum; /* * NPE Context Store register logical addresses */ #define IX_NPEDL_CTXT_REG_ADDR_STEVT 0x0000001B #define IX_NPEDL_CTXT_REG_ADDR_STARTPC 0x0000001C #define IX_NPEDL_CTXT_REG_ADDR_REGMAP 0x0000001E #define IX_NPEDL_CTXT_REG_ADDR_CINDEX 0x0000001F /* * NPE Context Store register reset values */ /** * Reset value of STEVT NPE internal Context Store register * (STEVT = off, 0x80) */ #define IX_NPEDL_CTXT_REG_RESET_STEVT 0x80 /** * Reset value of STARTPC NPE internal Context Store register * (STARTPC = 0x0000) */ #define IX_NPEDL_CTXT_REG_RESET_STARTPC 0x0000 /** * Reset value of REGMAP NPE internal Context Store register * (REGMAP = d0->p0, d8->p2, d16->p4) */ #define IX_NPEDL_CTXT_REG_RESET_REGMAP 0x0820 /** * Reset value of CINDEX NPE internal Context Store register * (CINDEX = 0) */ #define IX_NPEDL_CTXT_REG_RESET_CINDEX 0x00 /* * Numeric range of context levels available on an NPE */ #define IX_NPEDL_CTXT_NUM_MIN 0 #define IX_NPEDL_CTXT_NUM_MAX 15 /** * Number of Physical registers currently supported * Initial NPE implementations will have a 32-word register file. * Later implementations may have a 64-word register file. */ #define IX_NPEDL_TOTAL_NUM_PHYS_REG 32 /** * LSB-offset of Regmap number in Physical NPE register address, used * for Physical To Logical register address mapping in the NPE */ #define IX_NPEDL_OFFSET_PHYS_REG_ADDR_REGMAP 1 /** * Mask to extract a logical NPE register address from a physical * register address, used for Physical To Logical address mapping */ #define IX_NPEDL_MASK_PHYS_REG_ADDR_LOGICAL_ADDR 0x1 /* * NPE Message/Mailbox interface. */ #define IX_NPESTAT IX_NPEDL_REG_OFFSET_STAT /* status register */ #define IX_NPECTL IX_NPEDL_REG_OFFSET_CTL /* control register */ #define IX_NPEFIFO IX_NPEDL_REG_OFFSET_FIFO /* FIFO register */ /* control register */ #define IX_NPECTL_OFE 0x00010000 /* output fifo enable */ #define IX_NPECTL_IFE 0x00020000 /* input fifo enable */ #define IX_NPECTL_OFWE 0x01000000 /* output fifo write enable */ #define IX_NPECTL_IFWE 0x02000000 /* input fifo write enable */ /* status register */ #define IX_NPESTAT_OFNE 0x00010000 /* output fifo not empty */ #define IX_NPESTAT_IFNF 0x00020000 /* input fifo not full */ #define IX_NPESTAT_OFNF 0x00040000 /* output fifo not full */ #define IX_NPESTAT_IFNE 0x00080000 /* input fifo not empty */ #define IX_NPESTAT_MBINT 0x00100000 /* Mailbox interrupt */ #define IX_NPESTAT_IFINT 0x00200000 /* input fifo interrupt */ #define IX_NPESTAT_OFINT 0x00400000 /* output fifo interrupt */ #define IX_NPESTAT_WFINT 0x00800000 /* watch fifo interrupt */ #endif /* _IXP425_NPEREG_H_ */ Index: user/ngie/release-pkg-fix-tests/sys/arm/xscale/ixp425/ixp425_qmgr.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/xscale/ixp425/ixp425_qmgr.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/xscale/ixp425/ixp425_qmgr.c (revision 299091) @@ -1,1104 +1,1104 @@ /*- * Copyright (c) 2006 Sam Leffler, Errno Consulting * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer, * without modification. * 2. Redistributions in binary form must reproduce at minimum a disclaimer * similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any * redistribution must be conditioned upon including a substantially * similar Disclaimer requirement for further binary redistribution. * * NO WARRANTY * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGES. */ /*- * Copyright (c) 2001-2005, Intel Corporation. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the Intel Corporation nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); /* * Intel XScale Queue Manager support. * * Each IXP4XXX device has a hardware block that implements a priority * queue manager that is shared between the XScale cpu and the backend * devices (such as the NPE). Queues are accessed by reading/writing * special memory locations. The queue contents are mapped into a shared * SRAM region with entries managed in a circular buffer. The XScale * processor can receive interrupts based on queue contents (a condition * code determines when interrupts should be delivered). * * The code here basically replaces the qmgr class in the Intel Access * Library (IAL). */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* * State per AQM hw queue. * This structure holds q configuration and dispatch state. */ struct qmgrInfo { int qSizeInWords; /* queue size in words */ uint32_t qOflowStatBitMask; /* overflow status mask */ int qWriteCount; /* queue write count */ bus_size_t qAccRegAddr; /* access register */ bus_size_t qUOStatRegAddr; /* status register */ bus_size_t qConfigRegAddr; /* config register */ int qSizeInEntries; /* queue size in entries */ uint32_t qUflowStatBitMask; /* underflow status mask */ int qReadCount; /* queue read count */ /* XXX union */ uint32_t qStatRegAddr; uint32_t qStatBitsOffset; uint32_t qStat0BitMask; uint32_t qStat1BitMask; uint32_t intRegCheckMask; /* interrupt reg check mask */ void (*cb)(int, void *); /* callback function */ void *cbarg; /* callback argument */ int priority; /* dispatch priority */ #if 0 /* NB: needed only for A0 parts */ u_int statusWordOffset; /* status word offset */ uint32_t statusMask; /* status mask */ uint32_t statusCheckValue; /* status check value */ #endif }; struct ixpqmgr_softc { device_t sc_dev; bus_space_tag_t sc_iot; bus_space_handle_t sc_ioh; struct resource *sc_irq1; /* IRQ resource */ void *sc_ih1; /* interrupt handler */ int sc_rid1; /* resource id for irq */ struct resource *sc_irq2; void *sc_ih2; int sc_rid2; struct qmgrInfo qinfo[IX_QMGR_MAX_NUM_QUEUES]; /* * This array contains a list of queue identifiers ordered by * priority. The table is split logically between queue * identifiers 0-31 and 32-63. To optimize lookups bit masks * are kept for the first-32 and last-32 q's. When the * table needs to be rebuilt mark rebuildTable and it'll * happen after the next interrupt. */ int priorityTable[IX_QMGR_MAX_NUM_QUEUES]; uint32_t lowPriorityTableFirstHalfMask; uint32_t uppPriorityTableFirstHalfMask; int rebuildTable; /* rebuild priorityTable */ uint32_t aqmFreeSramAddress; /* SRAM free space */ }; static int qmgr_debug; SYSCTL_INT(_debug, OID_AUTO, qmgr, CTLFLAG_RWTUN, &qmgr_debug, 0, "IXP4XX Q-Manager debug msgs"); #define DPRINTF(dev, fmt, ...) do { \ if (qmgr_debug) printf(fmt, __VA_ARGS__); \ } while (0) #define DPRINTFn(n, dev, fmt, ...) do { \ if (qmgr_debug >= n) printf(fmt, __VA_ARGS__); \ } while (0) static struct ixpqmgr_softc *ixpqmgr_sc = NULL; static void ixpqmgr_rebuild(struct ixpqmgr_softc *); static void ixpqmgr_intr(void *); static void aqm_int_enable(struct ixpqmgr_softc *sc, int qId); static void aqm_int_disable(struct ixpqmgr_softc *sc, int qId); static void aqm_qcfg(struct ixpqmgr_softc *sc, int qId, u_int ne, u_int nf); static void aqm_srcsel_write(struct ixpqmgr_softc *sc, int qId, int sourceId); static void aqm_reset(struct ixpqmgr_softc *sc); static void dummyCallback(int qId, void *arg) { /* XXX complain */ } static uint32_t aqm_reg_read(struct ixpqmgr_softc *sc, bus_size_t off) { DPRINTFn(9, sc->sc_dev, "%s(0x%x)\n", __func__, (int)off); return bus_space_read_4(sc->sc_iot, sc->sc_ioh, off); } static void aqm_reg_write(struct ixpqmgr_softc *sc, bus_size_t off, uint32_t val) { DPRINTFn(9, sc->sc_dev, "%s(0x%x, 0x%x)\n", __func__, (int)off, val); bus_space_write_4(sc->sc_iot, sc->sc_ioh, off, val); } static int ixpqmgr_probe(device_t dev) { device_set_desc(dev, "IXP4XX Q-Manager"); return 0; } static int ixpqmgr_attach(device_t dev) { struct ixpqmgr_softc *sc = device_get_softc(dev); struct ixp425_softc *sa = device_get_softc(device_get_parent(dev)); int i, err; ixpqmgr_sc = sc; sc->sc_dev = dev; sc->sc_iot = sa->sc_iot; if (bus_space_map(sc->sc_iot, IXP425_QMGR_HWBASE, IXP425_QMGR_SIZE, 0, &sc->sc_ioh)) panic("%s: Cannot map registers", device_get_name(dev)); /* NB: we only use the lower 32 q's */ /* Set up QMGR interrupts */ sc->sc_rid1 = 0; sc->sc_irq1 = bus_alloc_resource(dev, SYS_RES_IRQ, &sc->sc_rid1, IXP425_INT_QUE1_32, IXP425_INT_QUE1_32, 1, RF_ACTIVE); sc->sc_rid2 = 1; sc->sc_irq2 = bus_alloc_resource(dev, SYS_RES_IRQ, &sc->sc_rid2, IXP425_INT_QUE33_64, IXP425_INT_QUE33_64, 1, RF_ACTIVE); if (sc->sc_irq1 == NULL || sc->sc_irq2 == NULL) panic("Unable to allocate the qmgr irqs.\n"); err = bus_setup_intr(dev, sc->sc_irq1, INTR_TYPE_NET | INTR_MPSAFE, NULL, ixpqmgr_intr, NULL, &sc->sc_ih1); if (err) { device_printf(dev, "failed to set up qmgr irq=%d\n", IXP425_INT_QUE1_32); return (ENXIO); } err = bus_setup_intr(dev, sc->sc_irq2, INTR_TYPE_NET | INTR_MPSAFE, NULL, ixpqmgr_intr, NULL, &sc->sc_ih2); if (err) { device_printf(dev, "failed to set up qmgr irq=%d\n", IXP425_INT_QUE33_64); return (ENXIO); } /* NB: softc is pre-zero'd */ for (i = 0; i < IX_QMGR_MAX_NUM_QUEUES; i++) { struct qmgrInfo *qi = &sc->qinfo[i]; qi->cb = dummyCallback; qi->priority = IX_QMGR_Q_PRIORITY_0; /* default priority */ /* * There are two interrupt registers, 32 bits each. One * for the lower queues(0-31) and one for the upper * queues(32-63). Therefore need to mod by 32 i.e the * min upper queue identifier. */ qi->intRegCheckMask = (1<<(i%(IX_QMGR_MIN_QUEUPP_QID))); /* * Register addresses and bit masks are calculated and * stored here to optimize QRead, QWrite and QStatusGet * functions. */ /* AQM Queue access reg addresses, per queue */ qi->qAccRegAddr = IX_QMGR_Q_ACCESS_ADDR_GET(i); qi->qAccRegAddr = IX_QMGR_Q_ACCESS_ADDR_GET(i); qi->qConfigRegAddr = IX_QMGR_Q_CONFIG_ADDR_GET(i); /* AQM Queue lower-group (0-31), only */ if (i < IX_QMGR_MIN_QUEUPP_QID) { /* AQM Q underflow/overflow status reg address, per queue */ qi->qUOStatRegAddr = IX_QMGR_QUEUOSTAT0_OFFSET + ((i / IX_QMGR_QUEUOSTAT_NUM_QUE_PER_WORD) * sizeof(uint32_t)); /* AQM Q underflow status bit masks for status reg per queue */ qi->qUflowStatBitMask = (IX_QMGR_UNDERFLOW_BIT_OFFSET + 1) << ((i & (IX_QMGR_QUEUOSTAT_NUM_QUE_PER_WORD - 1)) * (32 / IX_QMGR_QUEUOSTAT_NUM_QUE_PER_WORD)); /* AQM Q overflow status bit masks for status reg, per queue */ qi->qOflowStatBitMask = (IX_QMGR_OVERFLOW_BIT_OFFSET + 1) << ((i & (IX_QMGR_QUEUOSTAT_NUM_QUE_PER_WORD - 1)) * (32 / IX_QMGR_QUEUOSTAT_NUM_QUE_PER_WORD)); /* AQM Q lower-group (0-31) status reg addresses, per queue */ qi->qStatRegAddr = IX_QMGR_QUELOWSTAT0_OFFSET + ((i / IX_QMGR_QUELOWSTAT_NUM_QUE_PER_WORD) * sizeof(uint32_t)); /* AQM Q lower-group (0-31) status register bit offset */ qi->qStatBitsOffset = (i & (IX_QMGR_QUELOWSTAT_NUM_QUE_PER_WORD - 1)) * (32 / IX_QMGR_QUELOWSTAT_NUM_QUE_PER_WORD); } else { /* AQM Q upper-group (32-63), only */ qi->qUOStatRegAddr = 0; /* XXX */ /* AQM Q upper-group (32-63) Nearly Empty status reg bitmasks */ qi->qStat0BitMask = (1 << (i - IX_QMGR_MIN_QUEUPP_QID)); /* AQM Q upper-group (32-63) Full status register bitmasks */ qi->qStat1BitMask = (1 << (i - IX_QMGR_MIN_QUEUPP_QID)); } } sc->aqmFreeSramAddress = 0x100; /* Q buffer space starts at 0x2100 */ ixpqmgr_rebuild(sc); /* build initial priority table */ aqm_reset(sc); /* reset h/w */ return (0); } static int ixpqmgr_detach(device_t dev) { struct ixpqmgr_softc *sc = device_get_softc(dev); aqm_reset(sc); /* disable interrupts */ bus_teardown_intr(dev, sc->sc_irq1, sc->sc_ih1); bus_teardown_intr(dev, sc->sc_irq2, sc->sc_ih2); bus_release_resource(dev, SYS_RES_IRQ, sc->sc_rid1, sc->sc_irq1); bus_release_resource(dev, SYS_RES_IRQ, sc->sc_rid2, sc->sc_irq2); bus_space_unmap(sc->sc_iot, sc->sc_ioh, IXP425_QMGR_SIZE); return (0); } int ixpqmgr_qconfig(int qId, int qEntries, int ne, int nf, int srcSel, qconfig_hand_t *cb, void *cbarg) { struct ixpqmgr_softc *sc = ixpqmgr_sc; struct qmgrInfo *qi = &sc->qinfo[qId]; DPRINTF(sc->sc_dev, "%s(%u, %u, %u, %u, %u, %p, %p)\n", __func__, qId, qEntries, ne, nf, srcSel, cb, cbarg); /* NB: entry size is always 1 */ qi->qSizeInWords = qEntries; qi->qReadCount = 0; qi->qWriteCount = 0; qi->qSizeInEntries = qEntries; /* XXX kept for code clarity */ if (cb == NULL) { /* Reset to dummy callback */ qi->cb = dummyCallback; qi->cbarg = NULL; } else { qi->cb = cb; qi->cbarg = cbarg; } /* Write the config register; NB must be AFTER qinfo setup */ aqm_qcfg(sc, qId, ne, nf); /* * Account for space just allocated to queue. */ sc->aqmFreeSramAddress += (qi->qSizeInWords * sizeof(uint32_t)); /* Set the interrupt source if this queue is in the range 0-31 */ if (qId < IX_QMGR_MIN_QUEUPP_QID) aqm_srcsel_write(sc, qId, srcSel); if (cb != NULL) /* Enable the interrupt */ aqm_int_enable(sc, qId); sc->rebuildTable = TRUE; return 0; /* XXX */ } int ixpqmgr_qwrite(int qId, uint32_t entry) { struct ixpqmgr_softc *sc = ixpqmgr_sc; struct qmgrInfo *qi = &sc->qinfo[qId]; DPRINTFn(3, sc->sc_dev, "%s(%u, 0x%x) writeCount %u size %u\n", __func__, qId, entry, qi->qWriteCount, qi->qSizeInEntries); /* write the entry */ aqm_reg_write(sc, qi->qAccRegAddr, entry); /* NB: overflow is available for lower queues only */ if (qId < IX_QMGR_MIN_QUEUPP_QID) { int qSize = qi->qSizeInEntries; /* * Increment the current number of entries in the queue * and check for overflow . */ if (qi->qWriteCount++ == qSize) { /* check for overflow */ uint32_t status = aqm_reg_read(sc, qi->qUOStatRegAddr); int qPtrs; /* * Read the status twice because the status may * not be immediately ready after the write operation */ if ((status & qi->qOflowStatBitMask) || ((status = aqm_reg_read(sc, qi->qUOStatRegAddr)) & qi->qOflowStatBitMask)) { /* * The queue is full, clear the overflow status bit if set. */ aqm_reg_write(sc, qi->qUOStatRegAddr, status & ~qi->qOflowStatBitMask); qi->qWriteCount = qSize; DPRINTFn(5, sc->sc_dev, "%s(%u, 0x%x) Q full, overflow status cleared\n", __func__, qId, entry); return ENOSPC; } /* - * No overflow occured : someone is draining the queue + * No overflow occurred : someone is draining the queue * and the current counter needs to be * updated from the current number of entries in the queue */ /* calculate number of words in q */ qPtrs = aqm_reg_read(sc, qi->qConfigRegAddr); DPRINTFn(2, sc->sc_dev, "%s(%u, 0x%x) Q full, no overflow status, qConfig 0x%x\n", __func__, qId, entry, qPtrs); qPtrs = (qPtrs - (qPtrs >> 7)) & 0x7f; if (qPtrs == 0) { /* * The queue may be full at the time of the * snapshot. Next access will check * the overflow status again. */ qi->qWriteCount = qSize; } else { /* convert the number of words to a number of entries */ qi->qWriteCount = qPtrs & (qSize - 1); } } } return 0; } int ixpqmgr_qread(int qId, uint32_t *entry) { struct ixpqmgr_softc *sc = ixpqmgr_sc; struct qmgrInfo *qi = &sc->qinfo[qId]; bus_size_t off = qi->qAccRegAddr; *entry = aqm_reg_read(sc, off); /* * Reset the current read count : next access to the read function * will force a underflow status check. */ qi->qReadCount = 0; /* Check if underflow occurred on the read */ if (*entry == 0 && qId < IX_QMGR_MIN_QUEUPP_QID) { /* get the queue status */ uint32_t status = aqm_reg_read(sc, qi->qUOStatRegAddr); if (status & qi->qUflowStatBitMask) { /* clear underflow status */ aqm_reg_write(sc, qi->qUOStatRegAddr, status &~ qi->qUflowStatBitMask); return ENOSPC; } } return 0; } int ixpqmgr_qreadm(int qId, uint32_t n, uint32_t *p) { struct ixpqmgr_softc *sc = ixpqmgr_sc; struct qmgrInfo *qi = &sc->qinfo[qId]; uint32_t entry; bus_size_t off = qi->qAccRegAddr; entry = aqm_reg_read(sc, off); while (--n) { if (entry == 0) { /* if we read a NULL entry, stop. We have underflowed */ break; } *p++ = entry; /* store */ entry = aqm_reg_read(sc, off); } *p = entry; /* * Reset the current read count : next access to the read function * will force a underflow status check. */ qi->qReadCount = 0; /* Check if underflow occurred on the read */ if (entry == 0 && qId < IX_QMGR_MIN_QUEUPP_QID) { /* get the queue status */ uint32_t status = aqm_reg_read(sc, qi->qUOStatRegAddr); if (status & qi->qUflowStatBitMask) { /* clear underflow status */ aqm_reg_write(sc, qi->qUOStatRegAddr, status &~ qi->qUflowStatBitMask); return ENOSPC; } } return 0; } uint32_t ixpqmgr_getqstatus(int qId) { #define QLOWSTATMASK \ ((1 << (32 / IX_QMGR_QUELOWSTAT_NUM_QUE_PER_WORD)) - 1) struct ixpqmgr_softc *sc = ixpqmgr_sc; const struct qmgrInfo *qi = &sc->qinfo[qId]; uint32_t status; if (qId < IX_QMGR_MIN_QUEUPP_QID) { /* read the status of a queue in the range 0-31 */ status = aqm_reg_read(sc, qi->qStatRegAddr); /* mask out the status bits relevant only to this queue */ status = (status >> qi->qStatBitsOffset) & QLOWSTATMASK; } else { /* read status of a queue in the range 32-63 */ status = 0; if (aqm_reg_read(sc, IX_QMGR_QUEUPPSTAT0_OFFSET)&qi->qStat0BitMask) status |= IX_QMGR_Q_STATUS_NE_BIT_MASK; /* nearly empty */ if (aqm_reg_read(sc, IX_QMGR_QUEUPPSTAT1_OFFSET)&qi->qStat1BitMask) status |= IX_QMGR_Q_STATUS_F_BIT_MASK; /* full */ } return status; #undef QLOWSTATMASK } uint32_t ixpqmgr_getqconfig(int qId) { struct ixpqmgr_softc *sc = ixpqmgr_sc; return aqm_reg_read(sc, IX_QMGR_Q_CONFIG_ADDR_GET(qId)); } void ixpqmgr_dump(void) { struct ixpqmgr_softc *sc = ixpqmgr_sc; int i, a; /* status registers */ printf("0x%04x: %08x %08x %08x %08x\n" , 0x400 , aqm_reg_read(sc, 0x400) , aqm_reg_read(sc, 0x400+4) , aqm_reg_read(sc, 0x400+8) , aqm_reg_read(sc, 0x400+12) ); printf("0x%04x: %08x %08x %08x %08x\n" , 0x410 , aqm_reg_read(sc, 0x410) , aqm_reg_read(sc, 0x410+4) , aqm_reg_read(sc, 0x410+8) , aqm_reg_read(sc, 0x410+12) ); printf("0x%04x: %08x %08x %08x %08x\n" , 0x420 , aqm_reg_read(sc, 0x420) , aqm_reg_read(sc, 0x420+4) , aqm_reg_read(sc, 0x420+8) , aqm_reg_read(sc, 0x420+12) ); printf("0x%04x: %08x %08x %08x %08x\n" , 0x430 , aqm_reg_read(sc, 0x430) , aqm_reg_read(sc, 0x430+4) , aqm_reg_read(sc, 0x430+8) , aqm_reg_read(sc, 0x430+12) ); /* q configuration registers */ for (a = 0x2000; a < 0x20ff; a += 32) printf("0x%04x: %08x %08x %08x %08x %08x %08x %08x %08x\n" , a , aqm_reg_read(sc, a) , aqm_reg_read(sc, a+4) , aqm_reg_read(sc, a+8) , aqm_reg_read(sc, a+12) , aqm_reg_read(sc, a+16) , aqm_reg_read(sc, a+20) , aqm_reg_read(sc, a+24) , aqm_reg_read(sc, a+28) ); /* allocated SRAM */ for (i = 0x100; i < sc->aqmFreeSramAddress; i += 32) { a = 0x2000 + i; printf("0x%04x: %08x %08x %08x %08x %08x %08x %08x %08x\n" , a , aqm_reg_read(sc, a) , aqm_reg_read(sc, a+4) , aqm_reg_read(sc, a+8) , aqm_reg_read(sc, a+12) , aqm_reg_read(sc, a+16) , aqm_reg_read(sc, a+20) , aqm_reg_read(sc, a+24) , aqm_reg_read(sc, a+28) ); } for (i = 0; i < 16; i++) { printf("Q[%2d] config 0x%08x status 0x%02x " "Q[%2d] config 0x%08x status 0x%02x\n" , i, ixpqmgr_getqconfig(i), ixpqmgr_getqstatus(i) , i+16, ixpqmgr_getqconfig(i+16), ixpqmgr_getqstatus(i+16) ); } } void ixpqmgr_notify_enable(int qId, int srcSel) { struct ixpqmgr_softc *sc = ixpqmgr_sc; #if 0 /* Calculate the checkMask and checkValue for this q */ aqm_calc_statuscheck(sc, qId, srcSel); #endif /* Set the interrupt source if this queue is in the range 0-31 */ if (qId < IX_QMGR_MIN_QUEUPP_QID) aqm_srcsel_write(sc, qId, srcSel); /* Enable the interrupt */ aqm_int_enable(sc, qId); } void ixpqmgr_notify_disable(int qId) { struct ixpqmgr_softc *sc = ixpqmgr_sc; aqm_int_disable(sc, qId); } /* * Rebuild the priority table used by the dispatcher. */ static void ixpqmgr_rebuild(struct ixpqmgr_softc *sc) { int q, pri; int lowQuePriorityTableIndex, uppQuePriorityTableIndex; struct qmgrInfo *qi; sc->lowPriorityTableFirstHalfMask = 0; sc->uppPriorityTableFirstHalfMask = 0; lowQuePriorityTableIndex = 0; uppQuePriorityTableIndex = 32; for (pri = 0; pri < IX_QMGR_NUM_PRIORITY_LEVELS; pri++) { /* low priority q's */ for (q = 0; q < IX_QMGR_MIN_QUEUPP_QID; q++) { qi = &sc->qinfo[q]; if (qi->priority == pri) { /* * Build the priority table bitmask which match the * queues of the first half of the priority table. */ if (lowQuePriorityTableIndex < 16) { sc->lowPriorityTableFirstHalfMask |= qi->intRegCheckMask; } sc->priorityTable[lowQuePriorityTableIndex++] = q; } } /* high priority q's */ for (; q < IX_QMGR_MAX_NUM_QUEUES; q++) { qi = &sc->qinfo[q]; if (qi->priority == pri) { /* * Build the priority table bitmask which match the * queues of the first half of the priority table . */ if (uppQuePriorityTableIndex < 48) { sc->uppPriorityTableFirstHalfMask |= qi->intRegCheckMask; } sc->priorityTable[uppQuePriorityTableIndex++] = q; } } } sc->rebuildTable = FALSE; } /* * Count the number of leading zero bits in a word, * and return the same value than the CLZ instruction. * Note this is similar to the standard ffs function but * it counts zero's from the MSB instead of the LSB. * * word (in) return value (out) * 0x80000000 0 * 0x40000000 1 * ,,, ,,, * 0x00000002 30 * 0x00000001 31 * 0x00000000 32 * * The C version of this function is used as a replacement * for system not providing the equivalent of the CLZ * assembly language instruction. * * Note that this version is big-endian */ static unsigned int _lzcount(uint32_t word) { unsigned int lzcount = 0; if (word == 0) return 32; while ((word & 0x80000000) == 0) { word <<= 1; lzcount++; } return lzcount; } static void ixpqmgr_intr(void *arg) { struct ixpqmgr_softc *sc = ixpqmgr_sc; uint32_t intRegVal; /* Interrupt reg val */ struct qmgrInfo *qi; int priorityTableIndex; /* Priority table index */ int qIndex; /* Current queue being processed */ /* Read the interrupt register */ intRegVal = aqm_reg_read(sc, IX_QMGR_QINTREG0_OFFSET); /* Write back to clear interrupt */ aqm_reg_write(sc, IX_QMGR_QINTREG0_OFFSET, intRegVal); DPRINTFn(5, sc->sc_dev, "%s: ISR0 0x%x ISR1 0x%x\n", __func__, intRegVal, aqm_reg_read(sc, IX_QMGR_QINTREG1_OFFSET)); /* No queue has interrupt register set */ if (intRegVal != 0) { /* get the first queue Id from the interrupt register value */ qIndex = (32 - 1) - _lzcount(intRegVal); DPRINTFn(2, sc->sc_dev, "%s: ISR0 0x%x qIndex %u\n", __func__, intRegVal, qIndex); /* * Optimize for single callback case. */ qi = &sc->qinfo[qIndex]; if (intRegVal == qi->intRegCheckMask) { /* * Only 1 queue event triggered a notification. * Call the callback function for this queue */ qi->cb(qIndex, qi->cbarg); } else { /* * The event is triggered by more than 1 queue, * the queue search will start from the beginning * or the middle of the priority table. * * The search will end when all the bits of the interrupt * register are cleared. There is no need to maintain * a separate value and test it at each iteration. */ if (intRegVal & sc->lowPriorityTableFirstHalfMask) { priorityTableIndex = 0; } else { priorityTableIndex = 16; } /* * Iterate over the priority table until all the bits * of the interrupt register are cleared. */ do { qIndex = sc->priorityTable[priorityTableIndex++]; qi = &sc->qinfo[qIndex]; /* If this queue caused this interrupt to be raised */ if (intRegVal & qi->intRegCheckMask) { /* Call the callback function for this queue */ qi->cb(qIndex, qi->cbarg); /* Clear the interrupt register bit */ intRegVal &= ~qi->intRegCheckMask; } } while (intRegVal); } } /* Rebuild the priority table if needed */ if (sc->rebuildTable) ixpqmgr_rebuild(sc); } #if 0 /* * Generate the parameters used to check if a Q's status matches * the specified source select. We calculate which status word * to check (statusWordOffset), the value to check the status * against (statusCheckValue) and the mask (statusMask) to mask * out all but the bits to check in the status word. */ static void aqm_calc_statuscheck(int qId, IxQMgrSourceId srcSel) { struct qmgrInfo *qi = &qinfo[qId]; uint32_t shiftVal; if (qId < IX_QMGR_MIN_QUEUPP_QID) { switch (srcSel) { case IX_QMGR_Q_SOURCE_ID_E: qi->statusCheckValue = IX_QMGR_Q_STATUS_E_BIT_MASK; qi->statusMask = IX_QMGR_Q_STATUS_E_BIT_MASK; break; case IX_QMGR_Q_SOURCE_ID_NE: qi->statusCheckValue = IX_QMGR_Q_STATUS_NE_BIT_MASK; qi->statusMask = IX_QMGR_Q_STATUS_NE_BIT_MASK; break; case IX_QMGR_Q_SOURCE_ID_NF: qi->statusCheckValue = IX_QMGR_Q_STATUS_NF_BIT_MASK; qi->statusMask = IX_QMGR_Q_STATUS_NF_BIT_MASK; break; case IX_QMGR_Q_SOURCE_ID_F: qi->statusCheckValue = IX_QMGR_Q_STATUS_F_BIT_MASK; qi->statusMask = IX_QMGR_Q_STATUS_F_BIT_MASK; break; case IX_QMGR_Q_SOURCE_ID_NOT_E: qi->statusCheckValue = 0; qi->statusMask = IX_QMGR_Q_STATUS_E_BIT_MASK; break; case IX_QMGR_Q_SOURCE_ID_NOT_NE: qi->statusCheckValue = 0; qi->statusMask = IX_QMGR_Q_STATUS_NE_BIT_MASK; break; case IX_QMGR_Q_SOURCE_ID_NOT_NF: qi->statusCheckValue = 0; qi->statusMask = IX_QMGR_Q_STATUS_NF_BIT_MASK; break; case IX_QMGR_Q_SOURCE_ID_NOT_F: qi->statusCheckValue = 0; qi->statusMask = IX_QMGR_Q_STATUS_F_BIT_MASK; break; default: /* Should never hit */ IX_OSAL_ASSERT(0); break; } /* One nibble of status per queue so need to shift the * check value and mask out to the correct position. */ shiftVal = (qId % IX_QMGR_QUELOWSTAT_NUM_QUE_PER_WORD) * IX_QMGR_QUELOWSTAT_BITS_PER_Q; /* Calculate the which status word to check from the qId, * 8 Qs status per word */ qi->statusWordOffset = qId / IX_QMGR_QUELOWSTAT_NUM_QUE_PER_WORD; qi->statusCheckValue <<= shiftVal; qi->statusMask <<= shiftVal; } else { /* One status word */ qi->statusWordOffset = 0; /* Single bits per queue and int source bit hardwired NE, * Qs start at 32. */ qi->statusMask = 1 << (qId - IX_QMGR_MIN_QUEUPP_QID); qi->statusCheckValue = qi->statusMask; } } #endif static void aqm_int_enable(struct ixpqmgr_softc *sc, int qId) { bus_size_t reg; uint32_t v; if (qId < IX_QMGR_MIN_QUEUPP_QID) reg = IX_QMGR_QUEIEREG0_OFFSET; else reg = IX_QMGR_QUEIEREG1_OFFSET; v = aqm_reg_read(sc, reg); aqm_reg_write(sc, reg, v | (1 << (qId % IX_QMGR_MIN_QUEUPP_QID))); DPRINTF(sc->sc_dev, "%s(%u) 0x%lx: 0x%x => 0x%x\n", __func__, qId, reg, v, aqm_reg_read(sc, reg)); } static void aqm_int_disable(struct ixpqmgr_softc *sc, int qId) { bus_size_t reg; uint32_t v; if (qId < IX_QMGR_MIN_QUEUPP_QID) reg = IX_QMGR_QUEIEREG0_OFFSET; else reg = IX_QMGR_QUEIEREG1_OFFSET; v = aqm_reg_read(sc, reg); aqm_reg_write(sc, reg, v &~ (1 << (qId % IX_QMGR_MIN_QUEUPP_QID))); DPRINTF(sc->sc_dev, "%s(%u) 0x%lx: 0x%x => 0x%x\n", __func__, qId, reg, v, aqm_reg_read(sc, reg)); } static unsigned log2(unsigned n) { unsigned count; /* * N.B. this function will return 0 if supplied 0. */ for (count = 0; n/2; count++) n /= 2; return count; } static __inline unsigned toAqmEntrySize(int entrySize) { /* entrySize 1("00"),2("01"),4("10") */ return log2(entrySize); } static __inline unsigned toAqmBufferSize(unsigned bufferSizeInWords) { /* bufferSize 16("00"),32("01),64("10"),128("11") */ return log2(bufferSizeInWords / IX_QMGR_MIN_BUFFER_SIZE); } static __inline unsigned toAqmWatermark(int watermark) { /* * Watermarks 0("000"),1("001"),2("010"),4("011"), * 8("100"),16("101"),32("110"),64("111") */ return log2(2 * watermark); } static void aqm_qcfg(struct ixpqmgr_softc *sc, int qId, u_int ne, u_int nf) { const struct qmgrInfo *qi = &sc->qinfo[qId]; uint32_t qCfg; uint32_t baseAddress; /* Build config register */ qCfg = ((toAqmEntrySize(1) & IX_QMGR_ENTRY_SIZE_MASK) << IX_QMGR_Q_CONFIG_ESIZE_OFFSET) | ((toAqmBufferSize(qi->qSizeInWords) & IX_QMGR_SIZE_MASK) << IX_QMGR_Q_CONFIG_BSIZE_OFFSET); /* baseAddress, calculated relative to start address */ baseAddress = sc->aqmFreeSramAddress; /* base address must be word-aligned */ KASSERT((baseAddress % IX_QMGR_BASE_ADDR_16_WORD_ALIGN) == 0, ("address not word-aligned")); /* Now convert to a 16 word pointer as required by QUECONFIG register */ baseAddress >>= IX_QMGR_BASE_ADDR_16_WORD_SHIFT; qCfg |= baseAddress << IX_QMGR_Q_CONFIG_BADDR_OFFSET; /* set watermarks */ qCfg |= (toAqmWatermark(ne) << IX_QMGR_Q_CONFIG_NE_OFFSET) | (toAqmWatermark(nf) << IX_QMGR_Q_CONFIG_NF_OFFSET); DPRINTF(sc->sc_dev, "%s(%u, %u, %u) 0x%x => 0x%x @ 0x%x\n", __func__, qId, ne, nf, aqm_reg_read(sc, IX_QMGR_Q_CONFIG_ADDR_GET(qId)), qCfg, IX_QMGR_Q_CONFIG_ADDR_GET(qId)); aqm_reg_write(sc, IX_QMGR_Q_CONFIG_ADDR_GET(qId), qCfg); } static void aqm_srcsel_write(struct ixpqmgr_softc *sc, int qId, int sourceId) { bus_size_t off; uint32_t v; /* * Calculate the register offset; multiple queues split across registers */ off = IX_QMGR_INT0SRCSELREG0_OFFSET + ((qId / IX_QMGR_INTSRC_NUM_QUE_PER_WORD) * sizeof(uint32_t)); v = aqm_reg_read(sc, off); if (off == IX_QMGR_INT0SRCSELREG0_OFFSET && qId == 0) { /* Queue 0 at INT0SRCSELREG should not corrupt the value bit-3 */ v |= 0x7; } else { const uint32_t bpq = 32 / IX_QMGR_INTSRC_NUM_QUE_PER_WORD; uint32_t mask; int qshift; qshift = (qId & (IX_QMGR_INTSRC_NUM_QUE_PER_WORD-1)) * bpq; mask = ((1 << bpq) - 1) << qshift; /* q's status mask */ /* merge sourceId */ v = (v &~ mask) | ((sourceId << qshift) & mask); } DPRINTF(sc->sc_dev, "%s(%u, %u) 0x%x => 0x%x @ 0x%lx\n", __func__, qId, sourceId, aqm_reg_read(sc, off), v, off); aqm_reg_write(sc, off, v); } /* * Reset AQM registers to default values. */ static void aqm_reset(struct ixpqmgr_softc *sc) { int i; /* Reset queues 0..31 status registers 0..3 */ aqm_reg_write(sc, IX_QMGR_QUELOWSTAT0_OFFSET, IX_QMGR_QUELOWSTAT_RESET_VALUE); aqm_reg_write(sc, IX_QMGR_QUELOWSTAT1_OFFSET, IX_QMGR_QUELOWSTAT_RESET_VALUE); aqm_reg_write(sc, IX_QMGR_QUELOWSTAT2_OFFSET, IX_QMGR_QUELOWSTAT_RESET_VALUE); aqm_reg_write(sc, IX_QMGR_QUELOWSTAT3_OFFSET, IX_QMGR_QUELOWSTAT_RESET_VALUE); /* Reset underflow/overflow status registers 0..1 */ aqm_reg_write(sc, IX_QMGR_QUEUOSTAT0_OFFSET, IX_QMGR_QUEUOSTAT_RESET_VALUE); aqm_reg_write(sc, IX_QMGR_QUEUOSTAT1_OFFSET, IX_QMGR_QUEUOSTAT_RESET_VALUE); /* Reset queues 32..63 nearly empty status registers */ aqm_reg_write(sc, IX_QMGR_QUEUPPSTAT0_OFFSET, IX_QMGR_QUEUPPSTAT0_RESET_VALUE); /* Reset queues 32..63 full status registers */ aqm_reg_write(sc, IX_QMGR_QUEUPPSTAT1_OFFSET, IX_QMGR_QUEUPPSTAT1_RESET_VALUE); /* Reset int0 status flag source select registers 0..3 */ aqm_reg_write(sc, IX_QMGR_INT0SRCSELREG0_OFFSET, IX_QMGR_INT0SRCSELREG_RESET_VALUE); aqm_reg_write(sc, IX_QMGR_INT0SRCSELREG1_OFFSET, IX_QMGR_INT0SRCSELREG_RESET_VALUE); aqm_reg_write(sc, IX_QMGR_INT0SRCSELREG2_OFFSET, IX_QMGR_INT0SRCSELREG_RESET_VALUE); aqm_reg_write(sc, IX_QMGR_INT0SRCSELREG3_OFFSET, IX_QMGR_INT0SRCSELREG_RESET_VALUE); /* Reset queue interrupt enable register 0..1 */ aqm_reg_write(sc, IX_QMGR_QUEIEREG0_OFFSET, IX_QMGR_QUEIEREG_RESET_VALUE); aqm_reg_write(sc, IX_QMGR_QUEIEREG1_OFFSET, IX_QMGR_QUEIEREG_RESET_VALUE); /* Reset queue interrupt register 0..1 */ aqm_reg_write(sc, IX_QMGR_QINTREG0_OFFSET, IX_QMGR_QINTREG_RESET_VALUE); aqm_reg_write(sc, IX_QMGR_QINTREG1_OFFSET, IX_QMGR_QINTREG_RESET_VALUE); /* Reset queue configuration words 0..63 */ for (i = 0; i < IX_QMGR_MAX_NUM_QUEUES; i++) aqm_reg_write(sc, sc->qinfo[i].qConfigRegAddr, IX_QMGR_QUECONFIG_RESET_VALUE); /* XXX zero SRAM to simplify debugging */ for (i = IX_QMGR_QUEBUFFER_SPACE_OFFSET; i < IX_QMGR_AQM_SRAM_SIZE_IN_BYTES; i += sizeof(uint32_t)) aqm_reg_write(sc, i, 0); } static device_method_t ixpqmgr_methods[] = { DEVMETHOD(device_probe, ixpqmgr_probe), DEVMETHOD(device_attach, ixpqmgr_attach), DEVMETHOD(device_detach, ixpqmgr_detach), { 0, 0 } }; static driver_t ixpqmgr_driver = { "ixpqmgr", ixpqmgr_methods, sizeof(struct ixpqmgr_softc), }; static devclass_t ixpqmgr_devclass; DRIVER_MODULE(ixpqmgr, ixp, ixpqmgr_driver, ixpqmgr_devclass, 0, 0); Index: user/ngie/release-pkg-fix-tests/sys/arm/xscale/pxa/pxa_machdep.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/arm/xscale/pxa/pxa_machdep.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/arm/xscale/pxa/pxa_machdep.c (revision 299091) @@ -1,437 +1,437 @@ /* $NetBSD: hpc_machdep.c,v 1.70 2003/09/16 08:18:22 agc Exp $ */ /*- * Copyright (c) 1994-1998 Mark Brinicombe. * Copyright (c) 1994 Brini. * All rights reserved. * * This code is derived from software written for Brini by Mark Brinicombe * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Brini. * 4. The name of the company nor the name of the author may be used to * endorse or promote products derived from this software without specific * prior written permission. * * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * RiscBSD kernel project * * machdep.c * - * Machine dependant functions for kernel setup + * Machine dependent functions for kernel setup * * This file needs a lot of work. * * Created : 17/09/94 */ #include "opt_ddb.h" #include "opt_kstack_pages.h" #include __FBSDID("$FreeBSD$"); #define _ARM32_BUS_DMA_PRIVATE #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define KERNEL_PT_SYS 0 /* Page table for mapping proc0 zero page */ #define KERNEL_PT_IOPXS 1 #define KERNEL_PT_BEFOREKERN 2 #define KERNEL_PT_AFKERNEL 3 /* L2 table for mapping after kernel */ #define KERNEL_PT_AFKERNEL_NUM 9 /* this should be evenly divisable by PAGE_SIZE / L2_TABLE_SIZE_REAL (or 4) */ #define NUM_KERNEL_PTS (KERNEL_PT_AFKERNEL + KERNEL_PT_AFKERNEL_NUM) struct pv_addr kernel_pt_table[NUM_KERNEL_PTS]; /* Physical and virtual addresses for some global pages */ struct pv_addr systempage; struct pv_addr msgbufpv; struct pv_addr irqstack; struct pv_addr undstack; struct pv_addr abtstack; struct pv_addr kernelstack; struct pv_addr minidataclean; static void pxa_probe_sdram(bus_space_tag_t, bus_space_handle_t, uint32_t *, uint32_t *); /* Static device mappings. */ static const struct devmap_entry pxa_devmap[] = { /* * Map the on-board devices up into the KVA region so we don't muck * up user-space. */ { PXA2X0_PERIPH_START + PXA2X0_PERIPH_OFFSET, PXA2X0_PERIPH_START, PXA250_PERIPH_END - PXA2X0_PERIPH_START, }, { 0, 0, 0, } }; #define SDRAM_START 0xa0000000 extern vm_offset_t xscale_cache_clean_addr; void * initarm(struct arm_boot_params *abp) { struct pv_addr kernel_l1pt; struct pv_addr dpcpu; int loop; u_int l1pagetable; vm_offset_t freemempos; vm_offset_t freemem_pt; vm_offset_t afterkern; vm_offset_t freemem_after; vm_offset_t lastaddr; int i, j; uint32_t memsize[PXA2X0_SDRAM_BANKS], memstart[PXA2X0_SDRAM_BANKS]; lastaddr = parse_boot_param(abp); arm_physmem_kernaddr = abp->abp_physaddr; set_cpufuncs(); pcpu_init(pcpup, 0, sizeof(struct pcpu)); PCPU_SET(curthread, &thread0); /* Do basic tuning, hz etc */ init_param1(); freemempos = 0xa0200000; /* Define a macro to simplify memory allocation */ #define valloc_pages(var, np) \ alloc_pages((var).pv_pa, (np)); \ (var).pv_va = (var).pv_pa + 0x20000000; #define alloc_pages(var, np) \ freemempos -= (np * PAGE_SIZE); \ (var) = freemempos; \ memset((char *)(var), 0, ((np) * PAGE_SIZE)); while (((freemempos - L1_TABLE_SIZE) & (L1_TABLE_SIZE - 1)) != 0) freemempos -= PAGE_SIZE; valloc_pages(kernel_l1pt, L1_TABLE_SIZE / PAGE_SIZE); for (loop = 0; loop < NUM_KERNEL_PTS; ++loop) { if (!(loop % (PAGE_SIZE / L2_TABLE_SIZE_REAL))) { valloc_pages(kernel_pt_table[loop], L2_TABLE_SIZE / PAGE_SIZE); } else { kernel_pt_table[loop].pv_pa = freemempos + (loop % (PAGE_SIZE / L2_TABLE_SIZE_REAL)) * L2_TABLE_SIZE_REAL; kernel_pt_table[loop].pv_va = kernel_pt_table[loop].pv_pa + 0x20000000; } } freemem_pt = freemempos; freemempos = 0xa0100000; /* * Allocate a page for the system page mapped to V0x00000000 * This page will just contain the system vectors and can be * shared by all processes. */ valloc_pages(systempage, 1); /* Allocate dynamic per-cpu area. */ valloc_pages(dpcpu, DPCPU_SIZE / PAGE_SIZE); dpcpu_init((void *)dpcpu.pv_va, 0); /* Allocate stacks for all modes */ valloc_pages(irqstack, IRQ_STACK_SIZE); valloc_pages(abtstack, ABT_STACK_SIZE); valloc_pages(undstack, UND_STACK_SIZE); valloc_pages(kernelstack, kstack_pages); alloc_pages(minidataclean.pv_pa, 1); valloc_pages(msgbufpv, round_page(msgbufsize) / PAGE_SIZE); /* * Allocate memory for the l1 and l2 page tables. The scheme to avoid * wasting memory by allocating the l1pt on the first 16k memory was * taken from NetBSD rpc_machdep.c. NKPT should be greater than 12 for * this to work (which is supposed to be the case). */ /* * Now we start construction of the L1 page table * We start by mapping the L2 page tables into the L1. * This means that we can replace L1 mappings later on if necessary */ l1pagetable = kernel_l1pt.pv_va; /* Map the L2 pages tables in the L1 page table */ pmap_link_l2pt(l1pagetable, rounddown2(ARM_VECTORS_HIGH, 0x00100000), &kernel_pt_table[KERNEL_PT_SYS]); #if 0 /* XXXBJR: What is this? Don't know if there's an analogue. */ pmap_link_l2pt(l1pagetable, IQ80321_IOPXS_VBASE, &kernel_pt_table[KERNEL_PT_IOPXS]); #endif pmap_link_l2pt(l1pagetable, KERNBASE, &kernel_pt_table[KERNEL_PT_BEFOREKERN]); pmap_map_chunk(l1pagetable, KERNBASE, SDRAM_START, 0x100000, VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); pmap_map_chunk(l1pagetable, KERNBASE + 0x100000, SDRAM_START + 0x100000, 0x100000, VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE); pmap_map_chunk(l1pagetable, KERNBASE + 0x200000, SDRAM_START + 0x200000, rounddown2(((uint32_t)(lastaddr) - KERNBASE - 0x200000) + L1_S_SIZE, L1_S_SIZE), VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); freemem_after = rounddown2((int)lastaddr + PAGE_SIZE, PAGE_SIZE); afterkern = round_page(rounddown2((vm_offset_t)lastaddr + L1_S_SIZE, L1_S_SIZE)); for (i = 0; i < KERNEL_PT_AFKERNEL_NUM; i++) { pmap_link_l2pt(l1pagetable, afterkern + i * 0x00100000, &kernel_pt_table[KERNEL_PT_AFKERNEL + i]); } pmap_map_entry(l1pagetable, afterkern, minidataclean.pv_pa, VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); /* Map the Mini-Data cache clean area. */ xscale_setup_minidata(l1pagetable, afterkern, minidataclean.pv_pa); /* Map the vector page. */ pmap_map_entry(l1pagetable, ARM_VECTORS_HIGH, systempage.pv_pa, VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); devmap_bootstrap(l1pagetable, pxa_devmap); /* * Give the XScale global cache clean code an appropriately * sized chunk of unmapped VA space starting at 0xff000000 * (our device mappings end before this address). */ xscale_cache_clean_addr = 0xff000000U; cpu_domains((DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL*2)) | DOMAIN_CLIENT); cpu_setttb(kernel_l1pt.pv_pa); cpu_tlb_flushID(); cpu_domains(DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL*2)); /* * Pages were allocated during the secondary bootstrap for the * stacks for different CPU modes. * We must now set the r13 registers in the different CPU modes to * point to these stacks. * Since the ARM stacks use STMFD etc. we must set r13 to the top end * of the stack memory. */ set_stackptrs(0); /* * We must now clean the cache again.... * Cleaning may be done by reading new data to displace any * dirty data in the cache. This will have happened in cpu_setttb() * but since we are boot strapping the addresses used for the read * may have just been remapped and thus the cache could be out * of sync. A re-clean after the switch will cure this. * After booting there are no gross relocations of the kernel thus * this problem will not occur after initarm(). */ cpu_idcache_wbinv_all(); cpu_setup(); /* * Sort out bus_space for on-board devices. */ pxa_obio_tag_init(); /* * Fetch the SDRAM start/size from the PXA2X0 SDRAM configration * registers. */ pxa_probe_sdram(obio_tag, PXA2X0_MEMCTL_BASE, memstart, memsize); /* Fire up consoles. */ cninit(); undefined_init(); init_proc0(kernelstack.pv_va); /* Enable MMU, I-cache, D-cache, write buffer. */ arm_vector_init(ARM_VECTORS_HIGH, ARM_VEC_ALL); pmap_curmaxkvaddr = afterkern + PAGE_SIZE; vm_max_kernel_address = 0xe0000000; pmap_bootstrap(pmap_curmaxkvaddr, &kernel_l1pt); msgbufp = (void*)msgbufpv.pv_va; msgbufinit(msgbufp, msgbufsize); mutex_init(); /* * Add the physical ram we have available. * * Exclude the kernel (and all the things we allocated which immediately * follow the kernel) from the VM allocation pool but not from crash * dumps. virtual_avail is a global variable which tracks the kva we've * "allocated" while setting up pmaps. * * Prepare the list of physical memory available to the vm subsystem. */ for (j = 0; j < PXA2X0_SDRAM_BANKS; j++) { if (memsize[j] > 0) arm_physmem_hardware_region(memstart[j], memsize[j]); } arm_physmem_exclude_region(freemem_pt, abp->abp_physaddr - freemem_pt, EXFLAG_NOALLOC); arm_physmem_exclude_region(freemempos, abp->abp_physaddr - 0x100000 - freemempos, EXFLAG_NOALLOC); arm_physmem_exclude_region(abp->abp_physaddr, virtual_avail - KERNVIRTADDR, EXFLAG_NOALLOC); arm_physmem_init_kernel_globals(); init_param2(physmem); kdb_init(); return ((void *)(kernelstack.pv_va + USPACE_SVC_STACK_TOP - sizeof(struct pcb))); } static void pxa_probe_sdram(bus_space_tag_t bst, bus_space_handle_t bsh, uint32_t *memstart, uint32_t *memsize) { uint32_t mdcnfg, dwid, dcac, drac, dnb; int i; mdcnfg = bus_space_read_4(bst, bsh, MEMCTL_MDCNFG); /* * Scan all 4 SDRAM banks */ for (i = 0; i < PXA2X0_SDRAM_BANKS; i++) { memstart[i] = 0; memsize[i] = 0; switch (i) { case 0: case 1: if ((i == 0 && (mdcnfg & MDCNFG_DE0) == 0) || (i == 1 && (mdcnfg & MDCNFG_DE1) == 0)) continue; dwid = mdcnfg >> MDCNFD_DWID01_SHIFT; dcac = mdcnfg >> MDCNFD_DCAC01_SHIFT; drac = mdcnfg >> MDCNFD_DRAC01_SHIFT; dnb = mdcnfg >> MDCNFD_DNB01_SHIFT; break; case 2: case 3: if ((i == 2 && (mdcnfg & MDCNFG_DE2) == 0) || (i == 3 && (mdcnfg & MDCNFG_DE3) == 0)) continue; dwid = mdcnfg >> MDCNFD_DWID23_SHIFT; dcac = mdcnfg >> MDCNFD_DCAC23_SHIFT; drac = mdcnfg >> MDCNFD_DRAC23_SHIFT; dnb = mdcnfg >> MDCNFD_DNB23_SHIFT; break; default: panic("pxa_probe_sdram: impossible"); } dwid = 2 << (1 - (dwid & MDCNFD_DWID_MASK)); /* 16/32 width */ dcac = 1 << ((dcac & MDCNFD_DCAC_MASK) + 8); /* 8-11 columns */ drac = 1 << ((drac & MDCNFD_DRAC_MASK) + 11); /* 11-13 rows */ dnb = 2 << (dnb & MDCNFD_DNB_MASK); /* # of banks */ memsize[i] = dwid * dcac * drac * dnb; memstart[i] = PXA2X0_SDRAM0_START + (i * PXA2X0_SDRAM_BANK_SIZE); } } #define TIMER_FREQUENCY 3686400 #define UNIMPLEMENTED panic("%s: unimplemented", __func__) /* XXXBJR: Belongs with DELAY in a timer.c of some sort. */ void cpu_startprofclock(void) { UNIMPLEMENTED; } void cpu_stopprofclock(void) { UNIMPLEMENTED; } static struct arm32_dma_range pxa_range = { .dr_sysbase = 0, .dr_busbase = 0, .dr_len = ~0u, }; struct arm32_dma_range * bus_dma_get_range(void) { return (&pxa_range); } int bus_dma_get_range_nb(void) { return (1); } Index: user/ngie/release-pkg-fix-tests/sys/dev/iwn/if_iwn.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/dev/iwn/if_iwn.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/dev/iwn/if_iwn.c (revision 299091) @@ -1,9005 +1,8989 @@ /*- * Copyright (c) 2007-2009 Damien Bergamini * Copyright (c) 2008 Benjamin Close * Copyright (c) 2008 Sam Leffler, Errno Consulting * Copyright (c) 2011 Intel Corporation * Copyright (c) 2013 Cedric GROSS * Copyright (c) 2013 Adrian Chadd * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ /* * Driver for Intel WiFi Link 4965 and 1000/5000/6000 Series 802.11 network * adapters. */ #include __FBSDID("$FreeBSD$"); #include "opt_wlan.h" #include "opt_iwn.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include struct iwn_ident { uint16_t vendor; uint16_t device; const char *name; }; static const struct iwn_ident iwn_ident_table[] = { { 0x8086, IWN_DID_6x05_1, "Intel Centrino Advanced-N 6205" }, { 0x8086, IWN_DID_1000_1, "Intel Centrino Wireless-N 1000" }, { 0x8086, IWN_DID_1000_2, "Intel Centrino Wireless-N 1000" }, { 0x8086, IWN_DID_6x05_2, "Intel Centrino Advanced-N 6205" }, { 0x8086, IWN_DID_6050_1, "Intel Centrino Advanced-N + WiMAX 6250" }, { 0x8086, IWN_DID_6050_2, "Intel Centrino Advanced-N + WiMAX 6250" }, { 0x8086, IWN_DID_x030_1, "Intel Centrino Wireless-N 1030" }, { 0x8086, IWN_DID_x030_2, "Intel Centrino Wireless-N 1030" }, { 0x8086, IWN_DID_x030_3, "Intel Centrino Advanced-N 6230" }, { 0x8086, IWN_DID_x030_4, "Intel Centrino Advanced-N 6230" }, { 0x8086, IWN_DID_6150_1, "Intel Centrino Wireless-N + WiMAX 6150" }, { 0x8086, IWN_DID_6150_2, "Intel Centrino Wireless-N + WiMAX 6150" }, { 0x8086, IWN_DID_2x00_1, "Intel(R) Centrino(R) Wireless-N 2200 BGN" }, { 0x8086, IWN_DID_2x00_2, "Intel(R) Centrino(R) Wireless-N 2200 BGN" }, /* XXX 2200D is IWN_SDID_2x00_4; there's no way to express this here! */ { 0x8086, IWN_DID_2x30_1, "Intel Centrino Wireless-N 2230" }, { 0x8086, IWN_DID_2x30_2, "Intel Centrino Wireless-N 2230" }, { 0x8086, IWN_DID_130_1, "Intel Centrino Wireless-N 130" }, { 0x8086, IWN_DID_130_2, "Intel Centrino Wireless-N 130" }, { 0x8086, IWN_DID_100_1, "Intel Centrino Wireless-N 100" }, { 0x8086, IWN_DID_100_2, "Intel Centrino Wireless-N 100" }, { 0x8086, IWN_DID_105_1, "Intel Centrino Wireless-N 105" }, { 0x8086, IWN_DID_105_2, "Intel Centrino Wireless-N 105" }, { 0x8086, IWN_DID_135_1, "Intel Centrino Wireless-N 135" }, { 0x8086, IWN_DID_135_2, "Intel Centrino Wireless-N 135" }, { 0x8086, IWN_DID_4965_1, "Intel Wireless WiFi Link 4965" }, { 0x8086, IWN_DID_6x00_1, "Intel Centrino Ultimate-N 6300" }, { 0x8086, IWN_DID_6x00_2, "Intel Centrino Advanced-N 6200" }, { 0x8086, IWN_DID_4965_2, "Intel Wireless WiFi Link 4965" }, { 0x8086, IWN_DID_4965_3, "Intel Wireless WiFi Link 4965" }, { 0x8086, IWN_DID_5x00_1, "Intel WiFi Link 5100" }, { 0x8086, IWN_DID_4965_4, "Intel Wireless WiFi Link 4965" }, { 0x8086, IWN_DID_5x00_3, "Intel Ultimate N WiFi Link 5300" }, { 0x8086, IWN_DID_5x00_4, "Intel Ultimate N WiFi Link 5300" }, { 0x8086, IWN_DID_5x00_2, "Intel WiFi Link 5100" }, { 0x8086, IWN_DID_6x00_3, "Intel Centrino Ultimate-N 6300" }, { 0x8086, IWN_DID_6x00_4, "Intel Centrino Advanced-N 6200" }, { 0x8086, IWN_DID_5x50_1, "Intel WiMAX/WiFi Link 5350" }, { 0x8086, IWN_DID_5x50_2, "Intel WiMAX/WiFi Link 5350" }, { 0x8086, IWN_DID_5x50_3, "Intel WiMAX/WiFi Link 5150" }, { 0x8086, IWN_DID_5x50_4, "Intel WiMAX/WiFi Link 5150" }, { 0x8086, IWN_DID_6035_1, "Intel Centrino Advanced 6235" }, { 0x8086, IWN_DID_6035_2, "Intel Centrino Advanced 6235" }, { 0, 0, NULL } }; static int iwn_probe(device_t); static int iwn_attach(device_t); static int iwn4965_attach(struct iwn_softc *, uint16_t); static int iwn5000_attach(struct iwn_softc *, uint16_t); static int iwn_config_specific(struct iwn_softc *, uint16_t); static void iwn_radiotap_attach(struct iwn_softc *); static void iwn_sysctlattach(struct iwn_softc *); static struct ieee80211vap *iwn_vap_create(struct ieee80211com *, const char [IFNAMSIZ], int, enum ieee80211_opmode, int, const uint8_t [IEEE80211_ADDR_LEN], const uint8_t [IEEE80211_ADDR_LEN]); static void iwn_vap_delete(struct ieee80211vap *); static int iwn_detach(device_t); static int iwn_shutdown(device_t); static int iwn_suspend(device_t); static int iwn_resume(device_t); static int iwn_nic_lock(struct iwn_softc *); static int iwn_eeprom_lock(struct iwn_softc *); static int iwn_init_otprom(struct iwn_softc *); static int iwn_read_prom_data(struct iwn_softc *, uint32_t, void *, int); static void iwn_dma_map_addr(void *, bus_dma_segment_t *, int, int); static int iwn_dma_contig_alloc(struct iwn_softc *, struct iwn_dma_info *, void **, bus_size_t, bus_size_t); static void iwn_dma_contig_free(struct iwn_dma_info *); static int iwn_alloc_sched(struct iwn_softc *); static void iwn_free_sched(struct iwn_softc *); static int iwn_alloc_kw(struct iwn_softc *); static void iwn_free_kw(struct iwn_softc *); static int iwn_alloc_ict(struct iwn_softc *); static void iwn_free_ict(struct iwn_softc *); static int iwn_alloc_fwmem(struct iwn_softc *); static void iwn_free_fwmem(struct iwn_softc *); static int iwn_alloc_rx_ring(struct iwn_softc *, struct iwn_rx_ring *); static void iwn_reset_rx_ring(struct iwn_softc *, struct iwn_rx_ring *); static void iwn_free_rx_ring(struct iwn_softc *, struct iwn_rx_ring *); static int iwn_alloc_tx_ring(struct iwn_softc *, struct iwn_tx_ring *, int); static void iwn_reset_tx_ring(struct iwn_softc *, struct iwn_tx_ring *); static void iwn_free_tx_ring(struct iwn_softc *, struct iwn_tx_ring *); static void iwn5000_ict_reset(struct iwn_softc *); static int iwn_read_eeprom(struct iwn_softc *, uint8_t macaddr[IEEE80211_ADDR_LEN]); static void iwn4965_read_eeprom(struct iwn_softc *); #ifdef IWN_DEBUG static void iwn4965_print_power_group(struct iwn_softc *, int); #endif static void iwn5000_read_eeprom(struct iwn_softc *); static uint32_t iwn_eeprom_channel_flags(struct iwn_eeprom_chan *); static void iwn_read_eeprom_band(struct iwn_softc *, int, int, int *, struct ieee80211_channel[]); static void iwn_read_eeprom_ht40(struct iwn_softc *, int, int, int *, struct ieee80211_channel[]); static void iwn_read_eeprom_channels(struct iwn_softc *, int, uint32_t); static struct iwn_eeprom_chan *iwn_find_eeprom_channel(struct iwn_softc *, struct ieee80211_channel *); static void iwn_getradiocaps(struct ieee80211com *, int, int *, struct ieee80211_channel[]); static int iwn_setregdomain(struct ieee80211com *, struct ieee80211_regdomain *, int, struct ieee80211_channel[]); static void iwn_read_eeprom_enhinfo(struct iwn_softc *); static struct ieee80211_node *iwn_node_alloc(struct ieee80211vap *, const uint8_t mac[IEEE80211_ADDR_LEN]); static void iwn_newassoc(struct ieee80211_node *, int); static int iwn_media_change(struct ifnet *); static int iwn_newstate(struct ieee80211vap *, enum ieee80211_state, int); static void iwn_calib_timeout(void *); static void iwn_rx_phy(struct iwn_softc *, struct iwn_rx_desc *, struct iwn_rx_data *); static void iwn_rx_done(struct iwn_softc *, struct iwn_rx_desc *, struct iwn_rx_data *); static void iwn_rx_compressed_ba(struct iwn_softc *, struct iwn_rx_desc *, struct iwn_rx_data *); static void iwn5000_rx_calib_results(struct iwn_softc *, struct iwn_rx_desc *, struct iwn_rx_data *); static void iwn_rx_statistics(struct iwn_softc *, struct iwn_rx_desc *, struct iwn_rx_data *); static void iwn4965_tx_done(struct iwn_softc *, struct iwn_rx_desc *, struct iwn_rx_data *); static void iwn5000_tx_done(struct iwn_softc *, struct iwn_rx_desc *, struct iwn_rx_data *); static void iwn_tx_done(struct iwn_softc *, struct iwn_rx_desc *, int, uint8_t); static void iwn_ampdu_tx_done(struct iwn_softc *, int, int, int, int, void *); static void iwn_cmd_done(struct iwn_softc *, struct iwn_rx_desc *); static void iwn_notif_intr(struct iwn_softc *); static void iwn_wakeup_intr(struct iwn_softc *); static void iwn_rftoggle_intr(struct iwn_softc *); static void iwn_fatal_intr(struct iwn_softc *); static void iwn_intr(void *); static void iwn4965_update_sched(struct iwn_softc *, int, int, uint8_t, uint16_t); static void iwn5000_update_sched(struct iwn_softc *, int, int, uint8_t, uint16_t); #ifdef notyet static void iwn5000_reset_sched(struct iwn_softc *, int, int); #endif static int iwn_tx_data(struct iwn_softc *, struct mbuf *, struct ieee80211_node *); static int iwn_tx_data_raw(struct iwn_softc *, struct mbuf *, struct ieee80211_node *, const struct ieee80211_bpf_params *params); static void iwn_xmit_task(void *arg0, int pending); static int iwn_raw_xmit(struct ieee80211_node *, struct mbuf *, const struct ieee80211_bpf_params *); static int iwn_transmit(struct ieee80211com *, struct mbuf *); static void iwn_watchdog(void *); static int iwn_ioctl(struct ieee80211com *, u_long , void *); static void iwn_parent(struct ieee80211com *); static int iwn_cmd(struct iwn_softc *, int, const void *, int, int); static int iwn4965_add_node(struct iwn_softc *, struct iwn_node_info *, int); static int iwn5000_add_node(struct iwn_softc *, struct iwn_node_info *, int); static int iwn_set_link_quality(struct iwn_softc *, struct ieee80211_node *); static int iwn_add_broadcast_node(struct iwn_softc *, int); static int iwn_updateedca(struct ieee80211com *); static void iwn_update_mcast(struct ieee80211com *); static void iwn_set_led(struct iwn_softc *, uint8_t, uint8_t, uint8_t); static int iwn_set_critical_temp(struct iwn_softc *); static int iwn_set_timing(struct iwn_softc *, struct ieee80211_node *); static void iwn4965_power_calibration(struct iwn_softc *, int); static int iwn4965_set_txpower(struct iwn_softc *, struct ieee80211_channel *, int); static int iwn5000_set_txpower(struct iwn_softc *, struct ieee80211_channel *, int); static int iwn4965_get_rssi(struct iwn_softc *, struct iwn_rx_stat *); static int iwn5000_get_rssi(struct iwn_softc *, struct iwn_rx_stat *); static int iwn_get_noise(const struct iwn_rx_general_stats *); static int iwn4965_get_temperature(struct iwn_softc *); static int iwn5000_get_temperature(struct iwn_softc *); static int iwn_init_sensitivity(struct iwn_softc *); static void iwn_collect_noise(struct iwn_softc *, const struct iwn_rx_general_stats *); static int iwn4965_init_gains(struct iwn_softc *); static int iwn5000_init_gains(struct iwn_softc *); static int iwn4965_set_gains(struct iwn_softc *); static int iwn5000_set_gains(struct iwn_softc *); static void iwn_tune_sensitivity(struct iwn_softc *, const struct iwn_rx_stats *); static void iwn_save_stats_counters(struct iwn_softc *, const struct iwn_stats *); static int iwn_send_sensitivity(struct iwn_softc *); static void iwn_check_rx_recovery(struct iwn_softc *, struct iwn_stats *); static int iwn_set_pslevel(struct iwn_softc *, int, int, int); static int iwn_send_btcoex(struct iwn_softc *); static int iwn_send_advanced_btcoex(struct iwn_softc *); static int iwn5000_runtime_calib(struct iwn_softc *); static int iwn_config(struct iwn_softc *); static int iwn_scan(struct iwn_softc *, struct ieee80211vap *, struct ieee80211_scan_state *, struct ieee80211_channel *); static int iwn_auth(struct iwn_softc *, struct ieee80211vap *vap); static int iwn_run(struct iwn_softc *, struct ieee80211vap *vap); static int iwn_ampdu_rx_start(struct ieee80211_node *, struct ieee80211_rx_ampdu *, int, int, int); static void iwn_ampdu_rx_stop(struct ieee80211_node *, struct ieee80211_rx_ampdu *); static int iwn_addba_request(struct ieee80211_node *, struct ieee80211_tx_ampdu *, int, int, int); static int iwn_addba_response(struct ieee80211_node *, struct ieee80211_tx_ampdu *, int, int, int); static int iwn_ampdu_tx_start(struct ieee80211com *, struct ieee80211_node *, uint8_t); static void iwn_ampdu_tx_stop(struct ieee80211_node *, struct ieee80211_tx_ampdu *); static void iwn4965_ampdu_tx_start(struct iwn_softc *, struct ieee80211_node *, int, uint8_t, uint16_t); static void iwn4965_ampdu_tx_stop(struct iwn_softc *, int, uint8_t, uint16_t); static void iwn5000_ampdu_tx_start(struct iwn_softc *, struct ieee80211_node *, int, uint8_t, uint16_t); static void iwn5000_ampdu_tx_stop(struct iwn_softc *, int, uint8_t, uint16_t); static int iwn5000_query_calibration(struct iwn_softc *); static int iwn5000_send_calibration(struct iwn_softc *); static int iwn5000_send_wimax_coex(struct iwn_softc *); static int iwn5000_crystal_calib(struct iwn_softc *); static int iwn5000_temp_offset_calib(struct iwn_softc *); static int iwn5000_temp_offset_calibv2(struct iwn_softc *); static int iwn4965_post_alive(struct iwn_softc *); static int iwn5000_post_alive(struct iwn_softc *); static int iwn4965_load_bootcode(struct iwn_softc *, const uint8_t *, int); static int iwn4965_load_firmware(struct iwn_softc *); static int iwn5000_load_firmware_section(struct iwn_softc *, uint32_t, const uint8_t *, int); static int iwn5000_load_firmware(struct iwn_softc *); static int iwn_read_firmware_leg(struct iwn_softc *, struct iwn_fw_info *); static int iwn_read_firmware_tlv(struct iwn_softc *, struct iwn_fw_info *, uint16_t); static int iwn_read_firmware(struct iwn_softc *); static void iwn_unload_firmware(struct iwn_softc *); static int iwn_clock_wait(struct iwn_softc *); static int iwn_apm_init(struct iwn_softc *); static void iwn_apm_stop_master(struct iwn_softc *); static void iwn_apm_stop(struct iwn_softc *); static int iwn4965_nic_config(struct iwn_softc *); static int iwn5000_nic_config(struct iwn_softc *); static int iwn_hw_prepare(struct iwn_softc *); static int iwn_hw_init(struct iwn_softc *); static void iwn_hw_stop(struct iwn_softc *); static void iwn_radio_on(void *, int); static void iwn_radio_off(void *, int); static void iwn_panicked(void *, int); static void iwn_init_locked(struct iwn_softc *); static void iwn_init(struct iwn_softc *); static void iwn_stop_locked(struct iwn_softc *); static void iwn_stop(struct iwn_softc *); static void iwn_scan_start(struct ieee80211com *); static void iwn_scan_end(struct ieee80211com *); static void iwn_set_channel(struct ieee80211com *); static void iwn_scan_curchan(struct ieee80211_scan_state *, unsigned long); static void iwn_scan_mindwell(struct ieee80211_scan_state *); -static void iwn_hw_reset(void *, int); #ifdef IWN_DEBUG static char *iwn_get_csr_string(int); static void iwn_debug_register(struct iwn_softc *); #endif static device_method_t iwn_methods[] = { /* Device interface */ DEVMETHOD(device_probe, iwn_probe), DEVMETHOD(device_attach, iwn_attach), DEVMETHOD(device_detach, iwn_detach), DEVMETHOD(device_shutdown, iwn_shutdown), DEVMETHOD(device_suspend, iwn_suspend), DEVMETHOD(device_resume, iwn_resume), DEVMETHOD_END }; static driver_t iwn_driver = { "iwn", iwn_methods, sizeof(struct iwn_softc) }; static devclass_t iwn_devclass; DRIVER_MODULE(iwn, pci, iwn_driver, iwn_devclass, NULL, NULL); MODULE_VERSION(iwn, 1); MODULE_DEPEND(iwn, firmware, 1, 1, 1); MODULE_DEPEND(iwn, pci, 1, 1, 1); MODULE_DEPEND(iwn, wlan, 1, 1, 1); static d_ioctl_t iwn_cdev_ioctl; static d_open_t iwn_cdev_open; static d_close_t iwn_cdev_close; static struct cdevsw iwn_cdevsw = { .d_version = D_VERSION, .d_flags = 0, .d_open = iwn_cdev_open, .d_close = iwn_cdev_close, .d_ioctl = iwn_cdev_ioctl, .d_name = "iwn", }; static int iwn_probe(device_t dev) { const struct iwn_ident *ident; for (ident = iwn_ident_table; ident->name != NULL; ident++) { if (pci_get_vendor(dev) == ident->vendor && pci_get_device(dev) == ident->device) { device_set_desc(dev, ident->name); return (BUS_PROBE_DEFAULT); } } return ENXIO; } static int iwn_is_3stream_device(struct iwn_softc *sc) { /* XXX for now only 5300, until the 5350 can be tested */ if (sc->hw_type == IWN_HW_REV_TYPE_5300) return (1); return (0); } static int iwn_attach(device_t dev) { struct iwn_softc *sc = device_get_softc(dev); struct ieee80211com *ic; int i, error, rid; sc->sc_dev = dev; #ifdef IWN_DEBUG error = resource_int_value(device_get_name(sc->sc_dev), device_get_unit(sc->sc_dev), "debug", &(sc->sc_debug)); if (error != 0) sc->sc_debug = 0; #else sc->sc_debug = 0; #endif DPRINTF(sc, IWN_DEBUG_TRACE, "->%s: begin\n",__func__); /* * Get the offset of the PCI Express Capability Structure in PCI * Configuration Space. */ error = pci_find_cap(dev, PCIY_EXPRESS, &sc->sc_cap_off); if (error != 0) { device_printf(dev, "PCIe capability structure not found!\n"); return error; } /* Clear device-specific "PCI retry timeout" register (41h). */ pci_write_config(dev, 0x41, 0, 1); /* Enable bus-mastering. */ pci_enable_busmaster(dev); rid = PCIR_BAR(0); sc->mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); if (sc->mem == NULL) { device_printf(dev, "can't map mem space\n"); error = ENOMEM; return error; } sc->sc_st = rman_get_bustag(sc->mem); sc->sc_sh = rman_get_bushandle(sc->mem); i = 1; rid = 0; if (pci_alloc_msi(dev, &i) == 0) rid = 1; /* Install interrupt handler. */ sc->irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE | (rid != 0 ? 0 : RF_SHAREABLE)); if (sc->irq == NULL) { device_printf(dev, "can't map interrupt\n"); error = ENOMEM; goto fail; } IWN_LOCK_INIT(sc); /* Read hardware revision and attach. */ sc->hw_type = (IWN_READ(sc, IWN_HW_REV) >> IWN_HW_REV_TYPE_SHIFT) & IWN_HW_REV_TYPE_MASK; sc->subdevice_id = pci_get_subdevice(dev); /* * 4965 versus 5000 and later have different methods. * Let's set those up first. */ if (sc->hw_type == IWN_HW_REV_TYPE_4965) error = iwn4965_attach(sc, pci_get_device(dev)); else error = iwn5000_attach(sc, pci_get_device(dev)); if (error != 0) { device_printf(dev, "could not attach device, error %d\n", error); goto fail; } /* * Next, let's setup the various parameters of each NIC. */ error = iwn_config_specific(sc, pci_get_device(dev)); if (error != 0) { device_printf(dev, "could not attach device, error %d\n", error); goto fail; } if ((error = iwn_hw_prepare(sc)) != 0) { device_printf(dev, "hardware not ready, error %d\n", error); goto fail; } /* Allocate DMA memory for firmware transfers. */ if ((error = iwn_alloc_fwmem(sc)) != 0) { device_printf(dev, "could not allocate memory for firmware, error %d\n", error); goto fail; } /* Allocate "Keep Warm" page. */ if ((error = iwn_alloc_kw(sc)) != 0) { device_printf(dev, "could not allocate keep warm page, error %d\n", error); goto fail; } /* Allocate ICT table for 5000 Series. */ if (sc->hw_type != IWN_HW_REV_TYPE_4965 && (error = iwn_alloc_ict(sc)) != 0) { device_printf(dev, "could not allocate ICT table, error %d\n", error); goto fail; } /* Allocate TX scheduler "rings". */ if ((error = iwn_alloc_sched(sc)) != 0) { device_printf(dev, "could not allocate TX scheduler rings, error %d\n", error); goto fail; } /* Allocate TX rings (16 on 4965AGN, 20 on >=5000). */ for (i = 0; i < sc->ntxqs; i++) { if ((error = iwn_alloc_tx_ring(sc, &sc->txq[i], i)) != 0) { device_printf(dev, "could not allocate TX ring %d, error %d\n", i, error); goto fail; } } /* Allocate RX ring. */ if ((error = iwn_alloc_rx_ring(sc, &sc->rxq)) != 0) { device_printf(dev, "could not allocate RX ring, error %d\n", error); goto fail; } /* Clear pending interrupts. */ IWN_WRITE(sc, IWN_INT, 0xffffffff); ic = &sc->sc_ic; ic->ic_softc = sc; ic->ic_name = device_get_nameunit(dev); ic->ic_phytype = IEEE80211_T_OFDM; /* not only, but not used */ ic->ic_opmode = IEEE80211_M_STA; /* default to BSS mode */ /* Set device capabilities. */ ic->ic_caps = IEEE80211_C_STA /* station mode supported */ | IEEE80211_C_MONITOR /* monitor mode supported */ #if 0 | IEEE80211_C_BGSCAN /* background scanning */ #endif | IEEE80211_C_TXPMGT /* tx power management */ | IEEE80211_C_SHSLOT /* short slot time supported */ | IEEE80211_C_WPA | IEEE80211_C_SHPREAMBLE /* short preamble supported */ #if 0 | IEEE80211_C_IBSS /* ibss/adhoc mode */ #endif | IEEE80211_C_WME /* WME */ | IEEE80211_C_PMGT /* Station-side power mgmt */ ; /* Read MAC address, channels, etc from EEPROM. */ if ((error = iwn_read_eeprom(sc, ic->ic_macaddr)) != 0) { device_printf(dev, "could not read EEPROM, error %d\n", error); goto fail; } /* Count the number of available chains. */ sc->ntxchains = ((sc->txchainmask >> 2) & 1) + ((sc->txchainmask >> 1) & 1) + ((sc->txchainmask >> 0) & 1); sc->nrxchains = ((sc->rxchainmask >> 2) & 1) + ((sc->rxchainmask >> 1) & 1) + ((sc->rxchainmask >> 0) & 1); if (bootverbose) { device_printf(dev, "MIMO %dT%dR, %.4s, address %6D\n", sc->ntxchains, sc->nrxchains, sc->eeprom_domain, ic->ic_macaddr, ":"); } if (sc->sc_flags & IWN_FLAG_HAS_11N) { ic->ic_rxstream = sc->nrxchains; ic->ic_txstream = sc->ntxchains; /* * Some of the 3 antenna devices (ie, the 4965) only supports * 2x2 operation. So correct the number of streams if * it's not a 3-stream device. */ if (! iwn_is_3stream_device(sc)) { if (ic->ic_rxstream > 2) ic->ic_rxstream = 2; if (ic->ic_txstream > 2) ic->ic_txstream = 2; } ic->ic_htcaps = IEEE80211_HTCAP_SMPS_OFF /* SMPS mode disabled */ | IEEE80211_HTCAP_SHORTGI20 /* short GI in 20MHz */ | IEEE80211_HTCAP_CHWIDTH40 /* 40MHz channel width*/ | IEEE80211_HTCAP_SHORTGI40 /* short GI in 40MHz */ #ifdef notyet | IEEE80211_HTCAP_GREENFIELD #if IWN_RBUF_SIZE == 8192 | IEEE80211_HTCAP_MAXAMSDU_7935 /* max A-MSDU length */ #else | IEEE80211_HTCAP_MAXAMSDU_3839 /* max A-MSDU length */ #endif #endif /* s/w capabilities */ | IEEE80211_HTC_HT /* HT operation */ | IEEE80211_HTC_AMPDU /* tx A-MPDU */ #ifdef notyet | IEEE80211_HTC_AMSDU /* tx A-MSDU */ #endif ; } ieee80211_ifattach(ic); ic->ic_vap_create = iwn_vap_create; ic->ic_ioctl = iwn_ioctl; ic->ic_parent = iwn_parent; ic->ic_vap_delete = iwn_vap_delete; ic->ic_transmit = iwn_transmit; ic->ic_raw_xmit = iwn_raw_xmit; ic->ic_node_alloc = iwn_node_alloc; sc->sc_ampdu_rx_start = ic->ic_ampdu_rx_start; ic->ic_ampdu_rx_start = iwn_ampdu_rx_start; sc->sc_ampdu_rx_stop = ic->ic_ampdu_rx_stop; ic->ic_ampdu_rx_stop = iwn_ampdu_rx_stop; sc->sc_addba_request = ic->ic_addba_request; ic->ic_addba_request = iwn_addba_request; sc->sc_addba_response = ic->ic_addba_response; ic->ic_addba_response = iwn_addba_response; sc->sc_addba_stop = ic->ic_addba_stop; ic->ic_addba_stop = iwn_ampdu_tx_stop; ic->ic_newassoc = iwn_newassoc; ic->ic_wme.wme_update = iwn_updateedca; ic->ic_update_mcast = iwn_update_mcast; ic->ic_scan_start = iwn_scan_start; ic->ic_scan_end = iwn_scan_end; ic->ic_set_channel = iwn_set_channel; ic->ic_scan_curchan = iwn_scan_curchan; ic->ic_scan_mindwell = iwn_scan_mindwell; ic->ic_getradiocaps = iwn_getradiocaps; ic->ic_setregdomain = iwn_setregdomain; iwn_radiotap_attach(sc); callout_init_mtx(&sc->calib_to, &sc->sc_mtx, 0); callout_init_mtx(&sc->watchdog_to, &sc->sc_mtx, 0); - TASK_INIT(&sc->sc_reinit_task, 0, iwn_hw_reset, sc); TASK_INIT(&sc->sc_radioon_task, 0, iwn_radio_on, sc); TASK_INIT(&sc->sc_radiooff_task, 0, iwn_radio_off, sc); TASK_INIT(&sc->sc_panic_task, 0, iwn_panicked, sc); TASK_INIT(&sc->sc_xmit_task, 0, iwn_xmit_task, sc); mbufq_init(&sc->sc_xmit_queue, 1024); sc->sc_tq = taskqueue_create("iwn_taskq", M_WAITOK, taskqueue_thread_enqueue, &sc->sc_tq); error = taskqueue_start_threads(&sc->sc_tq, 1, 0, "iwn_taskq"); if (error != 0) { device_printf(dev, "can't start threads, error %d\n", error); goto fail; } iwn_sysctlattach(sc); /* * Hook our interrupt after all initialization is complete. */ error = bus_setup_intr(dev, sc->irq, INTR_TYPE_NET | INTR_MPSAFE, NULL, iwn_intr, sc, &sc->sc_ih); if (error != 0) { device_printf(dev, "can't establish interrupt, error %d\n", error); goto fail; } #if 0 device_printf(sc->sc_dev, "%s: rx_stats=%d, rx_stats_bt=%d\n", __func__, sizeof(struct iwn_stats), sizeof(struct iwn_stats_bt)); #endif if (bootverbose) ieee80211_announce(ic); DPRINTF(sc, IWN_DEBUG_TRACE, "->%s: end\n",__func__); /* Add debug ioctl right at the end */ sc->sc_cdev = make_dev(&iwn_cdevsw, device_get_unit(dev), UID_ROOT, GID_WHEEL, 0600, "%s", device_get_nameunit(dev)); if (sc->sc_cdev == NULL) { device_printf(dev, "failed to create debug character device\n"); } else { sc->sc_cdev->si_drv1 = sc; } return 0; fail: iwn_detach(dev); DPRINTF(sc, IWN_DEBUG_TRACE, "->%s: end in error\n",__func__); return error; } /* * Define specific configuration based on device id and subdevice id * pid : PCI device id */ static int iwn_config_specific(struct iwn_softc *sc, uint16_t pid) { switch (pid) { /* 4965 series */ case IWN_DID_4965_1: case IWN_DID_4965_2: case IWN_DID_4965_3: case IWN_DID_4965_4: sc->base_params = &iwn4965_base_params; sc->limits = &iwn4965_sensitivity_limits; sc->fwname = "iwn4965fw"; /* Override chains masks, ROM is known to be broken. */ sc->txchainmask = IWN_ANT_AB; sc->rxchainmask = IWN_ANT_ABC; /* Enable normal btcoex */ sc->sc_flags |= IWN_FLAG_BTCOEX; break; /* 1000 Series */ case IWN_DID_1000_1: case IWN_DID_1000_2: switch(sc->subdevice_id) { case IWN_SDID_1000_1: case IWN_SDID_1000_2: case IWN_SDID_1000_3: case IWN_SDID_1000_4: case IWN_SDID_1000_5: case IWN_SDID_1000_6: case IWN_SDID_1000_7: case IWN_SDID_1000_8: case IWN_SDID_1000_9: case IWN_SDID_1000_10: case IWN_SDID_1000_11: case IWN_SDID_1000_12: sc->limits = &iwn1000_sensitivity_limits; sc->base_params = &iwn1000_base_params; sc->fwname = "iwn1000fw"; break; default: device_printf(sc->sc_dev, "adapter type id : 0x%04x sub id :" "0x%04x rev %d not supported (subdevice)\n", pid, sc->subdevice_id,sc->hw_type); return ENOTSUP; } break; /* 6x00 Series */ case IWN_DID_6x00_2: case IWN_DID_6x00_4: case IWN_DID_6x00_1: case IWN_DID_6x00_3: sc->fwname = "iwn6000fw"; sc->limits = &iwn6000_sensitivity_limits; switch(sc->subdevice_id) { case IWN_SDID_6x00_1: case IWN_SDID_6x00_2: case IWN_SDID_6x00_8: //iwl6000_3agn_cfg sc->base_params = &iwn_6000_base_params; break; case IWN_SDID_6x00_3: case IWN_SDID_6x00_6: case IWN_SDID_6x00_9: ////iwl6000i_2agn case IWN_SDID_6x00_4: case IWN_SDID_6x00_7: case IWN_SDID_6x00_10: //iwl6000i_2abg_cfg case IWN_SDID_6x00_5: //iwl6000i_2bg_cfg sc->base_params = &iwn_6000i_base_params; sc->sc_flags |= IWN_FLAG_INTERNAL_PA; sc->txchainmask = IWN_ANT_BC; sc->rxchainmask = IWN_ANT_BC; break; default: device_printf(sc->sc_dev, "adapter type id : 0x%04x sub id :" "0x%04x rev %d not supported (subdevice)\n", pid, sc->subdevice_id,sc->hw_type); return ENOTSUP; } break; /* 6x05 Series */ case IWN_DID_6x05_1: case IWN_DID_6x05_2: switch(sc->subdevice_id) { case IWN_SDID_6x05_1: case IWN_SDID_6x05_4: case IWN_SDID_6x05_6: //iwl6005_2agn_cfg case IWN_SDID_6x05_2: case IWN_SDID_6x05_5: case IWN_SDID_6x05_7: //iwl6005_2abg_cfg case IWN_SDID_6x05_3: //iwl6005_2bg_cfg case IWN_SDID_6x05_8: case IWN_SDID_6x05_9: //iwl6005_2agn_sff_cfg case IWN_SDID_6x05_10: //iwl6005_2agn_d_cfg case IWN_SDID_6x05_11: //iwl6005_2agn_mow1_cfg case IWN_SDID_6x05_12: //iwl6005_2agn_mow2_cfg sc->fwname = "iwn6000g2afw"; sc->limits = &iwn6000_sensitivity_limits; sc->base_params = &iwn_6000g2_base_params; break; default: device_printf(sc->sc_dev, "adapter type id : 0x%04x sub id :" "0x%04x rev %d not supported (subdevice)\n", pid, sc->subdevice_id,sc->hw_type); return ENOTSUP; } break; /* 6x35 Series */ case IWN_DID_6035_1: case IWN_DID_6035_2: switch(sc->subdevice_id) { case IWN_SDID_6035_1: case IWN_SDID_6035_2: case IWN_SDID_6035_3: case IWN_SDID_6035_4: sc->fwname = "iwn6000g2bfw"; sc->limits = &iwn6235_sensitivity_limits; sc->base_params = &iwn_6235_base_params; break; default: device_printf(sc->sc_dev, "adapter type id : 0x%04x sub id :" "0x%04x rev %d not supported (subdevice)\n", pid, sc->subdevice_id,sc->hw_type); return ENOTSUP; } break; /* 6x50 WiFi/WiMax Series */ case IWN_DID_6050_1: case IWN_DID_6050_2: switch(sc->subdevice_id) { case IWN_SDID_6050_1: case IWN_SDID_6050_3: case IWN_SDID_6050_5: //iwl6050_2agn_cfg case IWN_SDID_6050_2: case IWN_SDID_6050_4: case IWN_SDID_6050_6: //iwl6050_2abg_cfg sc->fwname = "iwn6050fw"; sc->txchainmask = IWN_ANT_AB; sc->rxchainmask = IWN_ANT_AB; sc->limits = &iwn6000_sensitivity_limits; sc->base_params = &iwn_6050_base_params; break; default: device_printf(sc->sc_dev, "adapter type id : 0x%04x sub id :" "0x%04x rev %d not supported (subdevice)\n", pid, sc->subdevice_id,sc->hw_type); return ENOTSUP; } break; /* 6150 WiFi/WiMax Series */ case IWN_DID_6150_1: case IWN_DID_6150_2: switch(sc->subdevice_id) { case IWN_SDID_6150_1: case IWN_SDID_6150_3: case IWN_SDID_6150_5: // iwl6150_bgn_cfg case IWN_SDID_6150_2: case IWN_SDID_6150_4: case IWN_SDID_6150_6: //iwl6150_bg_cfg sc->fwname = "iwn6050fw"; sc->limits = &iwn6000_sensitivity_limits; sc->base_params = &iwn_6150_base_params; break; default: device_printf(sc->sc_dev, "adapter type id : 0x%04x sub id :" "0x%04x rev %d not supported (subdevice)\n", pid, sc->subdevice_id,sc->hw_type); return ENOTSUP; } break; /* 6030 Series and 1030 Series */ case IWN_DID_x030_1: case IWN_DID_x030_2: case IWN_DID_x030_3: case IWN_DID_x030_4: switch(sc->subdevice_id) { case IWN_SDID_x030_1: case IWN_SDID_x030_3: case IWN_SDID_x030_5: // iwl1030_bgn_cfg case IWN_SDID_x030_2: case IWN_SDID_x030_4: case IWN_SDID_x030_6: //iwl1030_bg_cfg case IWN_SDID_x030_7: case IWN_SDID_x030_10: case IWN_SDID_x030_14: //iwl6030_2agn_cfg case IWN_SDID_x030_8: case IWN_SDID_x030_11: case IWN_SDID_x030_15: // iwl6030_2bgn_cfg case IWN_SDID_x030_9: case IWN_SDID_x030_12: case IWN_SDID_x030_16: // iwl6030_2abg_cfg case IWN_SDID_x030_13: //iwl6030_2bg_cfg sc->fwname = "iwn6000g2bfw"; sc->limits = &iwn6000_sensitivity_limits; sc->base_params = &iwn_6000g2b_base_params; break; default: device_printf(sc->sc_dev, "adapter type id : 0x%04x sub id :" "0x%04x rev %d not supported (subdevice)\n", pid, sc->subdevice_id,sc->hw_type); return ENOTSUP; } break; /* 130 Series WiFi */ /* XXX: This series will need adjustment for rate. * see rx_with_siso_diversity in linux kernel */ case IWN_DID_130_1: case IWN_DID_130_2: switch(sc->subdevice_id) { case IWN_SDID_130_1: case IWN_SDID_130_3: case IWN_SDID_130_5: //iwl130_bgn_cfg case IWN_SDID_130_2: case IWN_SDID_130_4: case IWN_SDID_130_6: //iwl130_bg_cfg sc->fwname = "iwn6000g2bfw"; sc->limits = &iwn6000_sensitivity_limits; sc->base_params = &iwn_6000g2b_base_params; break; default: device_printf(sc->sc_dev, "adapter type id : 0x%04x sub id :" "0x%04x rev %d not supported (subdevice)\n", pid, sc->subdevice_id,sc->hw_type); return ENOTSUP; } break; /* 100 Series WiFi */ case IWN_DID_100_1: case IWN_DID_100_2: switch(sc->subdevice_id) { case IWN_SDID_100_1: case IWN_SDID_100_2: case IWN_SDID_100_3: case IWN_SDID_100_4: case IWN_SDID_100_5: case IWN_SDID_100_6: sc->limits = &iwn1000_sensitivity_limits; sc->base_params = &iwn1000_base_params; sc->fwname = "iwn100fw"; break; default: device_printf(sc->sc_dev, "adapter type id : 0x%04x sub id :" "0x%04x rev %d not supported (subdevice)\n", pid, sc->subdevice_id,sc->hw_type); return ENOTSUP; } break; /* 105 Series */ /* XXX: This series will need adjustment for rate. * see rx_with_siso_diversity in linux kernel */ case IWN_DID_105_1: case IWN_DID_105_2: switch(sc->subdevice_id) { case IWN_SDID_105_1: case IWN_SDID_105_2: case IWN_SDID_105_3: //iwl105_bgn_cfg case IWN_SDID_105_4: //iwl105_bgn_d_cfg sc->limits = &iwn2030_sensitivity_limits; sc->base_params = &iwn2000_base_params; sc->fwname = "iwn105fw"; break; default: device_printf(sc->sc_dev, "adapter type id : 0x%04x sub id :" "0x%04x rev %d not supported (subdevice)\n", pid, sc->subdevice_id,sc->hw_type); return ENOTSUP; } break; /* 135 Series */ /* XXX: This series will need adjustment for rate. * see rx_with_siso_diversity in linux kernel */ case IWN_DID_135_1: case IWN_DID_135_2: switch(sc->subdevice_id) { case IWN_SDID_135_1: case IWN_SDID_135_2: case IWN_SDID_135_3: sc->limits = &iwn2030_sensitivity_limits; sc->base_params = &iwn2030_base_params; sc->fwname = "iwn135fw"; break; default: device_printf(sc->sc_dev, "adapter type id : 0x%04x sub id :" "0x%04x rev %d not supported (subdevice)\n", pid, sc->subdevice_id,sc->hw_type); return ENOTSUP; } break; /* 2x00 Series */ case IWN_DID_2x00_1: case IWN_DID_2x00_2: switch(sc->subdevice_id) { case IWN_SDID_2x00_1: case IWN_SDID_2x00_2: case IWN_SDID_2x00_3: //iwl2000_2bgn_cfg case IWN_SDID_2x00_4: //iwl2000_2bgn_d_cfg sc->limits = &iwn2030_sensitivity_limits; sc->base_params = &iwn2000_base_params; sc->fwname = "iwn2000fw"; break; default: device_printf(sc->sc_dev, "adapter type id : 0x%04x sub id :" "0x%04x rev %d not supported (subdevice) \n", pid, sc->subdevice_id, sc->hw_type); return ENOTSUP; } break; /* 2x30 Series */ case IWN_DID_2x30_1: case IWN_DID_2x30_2: switch(sc->subdevice_id) { case IWN_SDID_2x30_1: case IWN_SDID_2x30_3: case IWN_SDID_2x30_5: //iwl100_bgn_cfg case IWN_SDID_2x30_2: case IWN_SDID_2x30_4: case IWN_SDID_2x30_6: //iwl100_bg_cfg sc->limits = &iwn2030_sensitivity_limits; sc->base_params = &iwn2030_base_params; sc->fwname = "iwn2030fw"; break; default: device_printf(sc->sc_dev, "adapter type id : 0x%04x sub id :" "0x%04x rev %d not supported (subdevice)\n", pid, sc->subdevice_id,sc->hw_type); return ENOTSUP; } break; /* 5x00 Series */ case IWN_DID_5x00_1: case IWN_DID_5x00_2: case IWN_DID_5x00_3: case IWN_DID_5x00_4: sc->limits = &iwn5000_sensitivity_limits; sc->base_params = &iwn5000_base_params; sc->fwname = "iwn5000fw"; switch(sc->subdevice_id) { case IWN_SDID_5x00_1: case IWN_SDID_5x00_2: case IWN_SDID_5x00_3: case IWN_SDID_5x00_4: case IWN_SDID_5x00_9: case IWN_SDID_5x00_10: case IWN_SDID_5x00_11: case IWN_SDID_5x00_12: case IWN_SDID_5x00_17: case IWN_SDID_5x00_18: case IWN_SDID_5x00_19: case IWN_SDID_5x00_20: //iwl5100_agn_cfg sc->txchainmask = IWN_ANT_B; sc->rxchainmask = IWN_ANT_AB; break; case IWN_SDID_5x00_5: case IWN_SDID_5x00_6: case IWN_SDID_5x00_13: case IWN_SDID_5x00_14: case IWN_SDID_5x00_21: case IWN_SDID_5x00_22: //iwl5100_bgn_cfg sc->txchainmask = IWN_ANT_B; sc->rxchainmask = IWN_ANT_AB; break; case IWN_SDID_5x00_7: case IWN_SDID_5x00_8: case IWN_SDID_5x00_15: case IWN_SDID_5x00_16: case IWN_SDID_5x00_23: case IWN_SDID_5x00_24: //iwl5100_abg_cfg sc->txchainmask = IWN_ANT_B; sc->rxchainmask = IWN_ANT_AB; break; case IWN_SDID_5x00_25: case IWN_SDID_5x00_26: case IWN_SDID_5x00_27: case IWN_SDID_5x00_28: case IWN_SDID_5x00_29: case IWN_SDID_5x00_30: case IWN_SDID_5x00_31: case IWN_SDID_5x00_32: case IWN_SDID_5x00_33: case IWN_SDID_5x00_34: case IWN_SDID_5x00_35: case IWN_SDID_5x00_36: //iwl5300_agn_cfg sc->txchainmask = IWN_ANT_ABC; sc->rxchainmask = IWN_ANT_ABC; break; default: device_printf(sc->sc_dev, "adapter type id : 0x%04x sub id :" "0x%04x rev %d not supported (subdevice)\n", pid, sc->subdevice_id,sc->hw_type); return ENOTSUP; } break; /* 5x50 Series */ case IWN_DID_5x50_1: case IWN_DID_5x50_2: case IWN_DID_5x50_3: case IWN_DID_5x50_4: sc->limits = &iwn5000_sensitivity_limits; sc->base_params = &iwn5000_base_params; sc->fwname = "iwn5000fw"; switch(sc->subdevice_id) { case IWN_SDID_5x50_1: case IWN_SDID_5x50_2: case IWN_SDID_5x50_3: //iwl5350_agn_cfg sc->limits = &iwn5000_sensitivity_limits; sc->base_params = &iwn5000_base_params; sc->fwname = "iwn5000fw"; break; case IWN_SDID_5x50_4: case IWN_SDID_5x50_5: case IWN_SDID_5x50_8: case IWN_SDID_5x50_9: case IWN_SDID_5x50_10: case IWN_SDID_5x50_11: //iwl5150_agn_cfg case IWN_SDID_5x50_6: case IWN_SDID_5x50_7: case IWN_SDID_5x50_12: case IWN_SDID_5x50_13: //iwl5150_abg_cfg sc->limits = &iwn5000_sensitivity_limits; sc->fwname = "iwn5150fw"; sc->base_params = &iwn_5x50_base_params; break; default: device_printf(sc->sc_dev, "adapter type id : 0x%04x sub id :" "0x%04x rev %d not supported (subdevice)\n", pid, sc->subdevice_id,sc->hw_type); return ENOTSUP; } break; default: device_printf(sc->sc_dev, "adapter type id : 0x%04x sub id : 0x%04x" "rev 0x%08x not supported (device)\n", pid, sc->subdevice_id, sc->hw_type); return ENOTSUP; } return 0; } static int iwn4965_attach(struct iwn_softc *sc, uint16_t pid) { struct iwn_ops *ops = &sc->ops; DPRINTF(sc, IWN_DEBUG_TRACE, "->%s begin\n", __func__); ops->load_firmware = iwn4965_load_firmware; ops->read_eeprom = iwn4965_read_eeprom; ops->post_alive = iwn4965_post_alive; ops->nic_config = iwn4965_nic_config; ops->update_sched = iwn4965_update_sched; ops->get_temperature = iwn4965_get_temperature; ops->get_rssi = iwn4965_get_rssi; ops->set_txpower = iwn4965_set_txpower; ops->init_gains = iwn4965_init_gains; ops->set_gains = iwn4965_set_gains; ops->add_node = iwn4965_add_node; ops->tx_done = iwn4965_tx_done; ops->ampdu_tx_start = iwn4965_ampdu_tx_start; ops->ampdu_tx_stop = iwn4965_ampdu_tx_stop; sc->ntxqs = IWN4965_NTXQUEUES; sc->firstaggqueue = IWN4965_FIRSTAGGQUEUE; sc->ndmachnls = IWN4965_NDMACHNLS; sc->broadcast_id = IWN4965_ID_BROADCAST; sc->rxonsz = IWN4965_RXONSZ; sc->schedsz = IWN4965_SCHEDSZ; sc->fw_text_maxsz = IWN4965_FW_TEXT_MAXSZ; sc->fw_data_maxsz = IWN4965_FW_DATA_MAXSZ; sc->fwsz = IWN4965_FWSZ; sc->sched_txfact_addr = IWN4965_SCHED_TXFACT; sc->limits = &iwn4965_sensitivity_limits; sc->fwname = "iwn4965fw"; /* Override chains masks, ROM is known to be broken. */ sc->txchainmask = IWN_ANT_AB; sc->rxchainmask = IWN_ANT_ABC; /* Enable normal btcoex */ sc->sc_flags |= IWN_FLAG_BTCOEX; DPRINTF(sc, IWN_DEBUG_TRACE, "%s: end\n",__func__); return 0; } static int iwn5000_attach(struct iwn_softc *sc, uint16_t pid) { struct iwn_ops *ops = &sc->ops; DPRINTF(sc, IWN_DEBUG_TRACE, "->%s begin\n", __func__); ops->load_firmware = iwn5000_load_firmware; ops->read_eeprom = iwn5000_read_eeprom; ops->post_alive = iwn5000_post_alive; ops->nic_config = iwn5000_nic_config; ops->update_sched = iwn5000_update_sched; ops->get_temperature = iwn5000_get_temperature; ops->get_rssi = iwn5000_get_rssi; ops->set_txpower = iwn5000_set_txpower; ops->init_gains = iwn5000_init_gains; ops->set_gains = iwn5000_set_gains; ops->add_node = iwn5000_add_node; ops->tx_done = iwn5000_tx_done; ops->ampdu_tx_start = iwn5000_ampdu_tx_start; ops->ampdu_tx_stop = iwn5000_ampdu_tx_stop; sc->ntxqs = IWN5000_NTXQUEUES; sc->firstaggqueue = IWN5000_FIRSTAGGQUEUE; sc->ndmachnls = IWN5000_NDMACHNLS; sc->broadcast_id = IWN5000_ID_BROADCAST; sc->rxonsz = IWN5000_RXONSZ; sc->schedsz = IWN5000_SCHEDSZ; sc->fw_text_maxsz = IWN5000_FW_TEXT_MAXSZ; sc->fw_data_maxsz = IWN5000_FW_DATA_MAXSZ; sc->fwsz = IWN5000_FWSZ; sc->sched_txfact_addr = IWN5000_SCHED_TXFACT; sc->reset_noise_gain = IWN5000_PHY_CALIB_RESET_NOISE_GAIN; sc->noise_gain = IWN5000_PHY_CALIB_NOISE_GAIN; return 0; } /* * Attach the interface to 802.11 radiotap. */ static void iwn_radiotap_attach(struct iwn_softc *sc) { DPRINTF(sc, IWN_DEBUG_TRACE, "->%s begin\n", __func__); ieee80211_radiotap_attach(&sc->sc_ic, &sc->sc_txtap.wt_ihdr, sizeof(sc->sc_txtap), IWN_TX_RADIOTAP_PRESENT, &sc->sc_rxtap.wr_ihdr, sizeof(sc->sc_rxtap), IWN_RX_RADIOTAP_PRESENT); DPRINTF(sc, IWN_DEBUG_TRACE, "->%s end\n", __func__); } static void iwn_sysctlattach(struct iwn_softc *sc) { #ifdef IWN_DEBUG struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->sc_dev); struct sysctl_oid *tree = device_get_sysctl_tree(sc->sc_dev); SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "debug", CTLFLAG_RW, &sc->sc_debug, sc->sc_debug, "control debugging printfs"); #endif } static struct ieee80211vap * iwn_vap_create(struct ieee80211com *ic, const char name[IFNAMSIZ], int unit, enum ieee80211_opmode opmode, int flags, const uint8_t bssid[IEEE80211_ADDR_LEN], const uint8_t mac[IEEE80211_ADDR_LEN]) { struct iwn_softc *sc = ic->ic_softc; struct iwn_vap *ivp; struct ieee80211vap *vap; if (!TAILQ_EMPTY(&ic->ic_vaps)) /* only one at a time */ return NULL; ivp = malloc(sizeof(struct iwn_vap), M_80211_VAP, M_WAITOK | M_ZERO); vap = &ivp->iv_vap; ieee80211_vap_setup(ic, vap, name, unit, opmode, flags, bssid); ivp->ctx = IWN_RXON_BSS_CTX; vap->iv_bmissthreshold = 10; /* override default */ /* Override with driver methods. */ ivp->iv_newstate = vap->iv_newstate; vap->iv_newstate = iwn_newstate; sc->ivap[IWN_RXON_BSS_CTX] = vap; ieee80211_ratectl_init(vap); /* Complete setup. */ ieee80211_vap_attach(vap, iwn_media_change, ieee80211_media_status, mac); ic->ic_opmode = opmode; return vap; } static void iwn_vap_delete(struct ieee80211vap *vap) { struct iwn_vap *ivp = IWN_VAP(vap); ieee80211_ratectl_deinit(vap); ieee80211_vap_detach(vap); free(ivp, M_80211_VAP); } static void iwn_xmit_queue_drain(struct iwn_softc *sc) { struct mbuf *m; struct ieee80211_node *ni; IWN_LOCK_ASSERT(sc); while ((m = mbufq_dequeue(&sc->sc_xmit_queue)) != NULL) { ni = (struct ieee80211_node *)m->m_pkthdr.rcvif; ieee80211_free_node(ni); m_freem(m); } } static int iwn_xmit_queue_enqueue(struct iwn_softc *sc, struct mbuf *m) { IWN_LOCK_ASSERT(sc); return (mbufq_enqueue(&sc->sc_xmit_queue, m)); } static int iwn_detach(device_t dev) { struct iwn_softc *sc = device_get_softc(dev); int qid; DPRINTF(sc, IWN_DEBUG_TRACE, "->%s begin\n", __func__); if (sc->sc_ic.ic_softc != NULL) { /* Free the mbuf queue and node references */ IWN_LOCK(sc); iwn_xmit_queue_drain(sc); IWN_UNLOCK(sc); - ieee80211_draintask(&sc->sc_ic, &sc->sc_reinit_task); ieee80211_draintask(&sc->sc_ic, &sc->sc_radioon_task); ieee80211_draintask(&sc->sc_ic, &sc->sc_radiooff_task); iwn_stop(sc); taskqueue_drain_all(sc->sc_tq); taskqueue_free(sc->sc_tq); callout_drain(&sc->watchdog_to); callout_drain(&sc->calib_to); ieee80211_ifdetach(&sc->sc_ic); } /* Uninstall interrupt handler. */ if (sc->irq != NULL) { bus_teardown_intr(dev, sc->irq, sc->sc_ih); bus_release_resource(dev, SYS_RES_IRQ, rman_get_rid(sc->irq), sc->irq); pci_release_msi(dev); } /* Free DMA resources. */ iwn_free_rx_ring(sc, &sc->rxq); for (qid = 0; qid < sc->ntxqs; qid++) iwn_free_tx_ring(sc, &sc->txq[qid]); iwn_free_sched(sc); iwn_free_kw(sc); if (sc->ict != NULL) iwn_free_ict(sc); iwn_free_fwmem(sc); if (sc->mem != NULL) bus_release_resource(dev, SYS_RES_MEMORY, rman_get_rid(sc->mem), sc->mem); if (sc->sc_cdev) { destroy_dev(sc->sc_cdev); sc->sc_cdev = NULL; } DPRINTF(sc, IWN_DEBUG_TRACE, "->%s: end\n", __func__); IWN_LOCK_DESTROY(sc); return 0; } static int iwn_shutdown(device_t dev) { struct iwn_softc *sc = device_get_softc(dev); iwn_stop(sc); return 0; } static int iwn_suspend(device_t dev) { struct iwn_softc *sc = device_get_softc(dev); ieee80211_suspend_all(&sc->sc_ic); return 0; } static int iwn_resume(device_t dev) { struct iwn_softc *sc = device_get_softc(dev); /* Clear device-specific "PCI retry timeout" register (41h). */ pci_write_config(dev, 0x41, 0, 1); ieee80211_resume_all(&sc->sc_ic); return 0; } static int iwn_nic_lock(struct iwn_softc *sc) { int ntries; /* Request exclusive access to NIC. */ IWN_SETBITS(sc, IWN_GP_CNTRL, IWN_GP_CNTRL_MAC_ACCESS_REQ); /* Spin until we actually get the lock. */ for (ntries = 0; ntries < 1000; ntries++) { if ((IWN_READ(sc, IWN_GP_CNTRL) & (IWN_GP_CNTRL_MAC_ACCESS_ENA | IWN_GP_CNTRL_SLEEP)) == IWN_GP_CNTRL_MAC_ACCESS_ENA) return 0; DELAY(10); } return ETIMEDOUT; } static __inline void iwn_nic_unlock(struct iwn_softc *sc) { IWN_CLRBITS(sc, IWN_GP_CNTRL, IWN_GP_CNTRL_MAC_ACCESS_REQ); } static __inline uint32_t iwn_prph_read(struct iwn_softc *sc, uint32_t addr) { IWN_WRITE(sc, IWN_PRPH_RADDR, IWN_PRPH_DWORD | addr); IWN_BARRIER_READ_WRITE(sc); return IWN_READ(sc, IWN_PRPH_RDATA); } static __inline void iwn_prph_write(struct iwn_softc *sc, uint32_t addr, uint32_t data) { IWN_WRITE(sc, IWN_PRPH_WADDR, IWN_PRPH_DWORD | addr); IWN_BARRIER_WRITE(sc); IWN_WRITE(sc, IWN_PRPH_WDATA, data); } static __inline void iwn_prph_setbits(struct iwn_softc *sc, uint32_t addr, uint32_t mask) { iwn_prph_write(sc, addr, iwn_prph_read(sc, addr) | mask); } static __inline void iwn_prph_clrbits(struct iwn_softc *sc, uint32_t addr, uint32_t mask) { iwn_prph_write(sc, addr, iwn_prph_read(sc, addr) & ~mask); } static __inline void iwn_prph_write_region_4(struct iwn_softc *sc, uint32_t addr, const uint32_t *data, int count) { for (; count > 0; count--, data++, addr += 4) iwn_prph_write(sc, addr, *data); } static __inline uint32_t iwn_mem_read(struct iwn_softc *sc, uint32_t addr) { IWN_WRITE(sc, IWN_MEM_RADDR, addr); IWN_BARRIER_READ_WRITE(sc); return IWN_READ(sc, IWN_MEM_RDATA); } static __inline void iwn_mem_write(struct iwn_softc *sc, uint32_t addr, uint32_t data) { IWN_WRITE(sc, IWN_MEM_WADDR, addr); IWN_BARRIER_WRITE(sc); IWN_WRITE(sc, IWN_MEM_WDATA, data); } static __inline void iwn_mem_write_2(struct iwn_softc *sc, uint32_t addr, uint16_t data) { uint32_t tmp; tmp = iwn_mem_read(sc, addr & ~3); if (addr & 3) tmp = (tmp & 0x0000ffff) | data << 16; else tmp = (tmp & 0xffff0000) | data; iwn_mem_write(sc, addr & ~3, tmp); } static __inline void iwn_mem_read_region_4(struct iwn_softc *sc, uint32_t addr, uint32_t *data, int count) { for (; count > 0; count--, addr += 4) *data++ = iwn_mem_read(sc, addr); } static __inline void iwn_mem_set_region_4(struct iwn_softc *sc, uint32_t addr, uint32_t val, int count) { for (; count > 0; count--, addr += 4) iwn_mem_write(sc, addr, val); } static int iwn_eeprom_lock(struct iwn_softc *sc) { int i, ntries; for (i = 0; i < 100; i++) { /* Request exclusive access to EEPROM. */ IWN_SETBITS(sc, IWN_HW_IF_CONFIG, IWN_HW_IF_CONFIG_EEPROM_LOCKED); /* Spin until we actually get the lock. */ for (ntries = 0; ntries < 100; ntries++) { if (IWN_READ(sc, IWN_HW_IF_CONFIG) & IWN_HW_IF_CONFIG_EEPROM_LOCKED) return 0; DELAY(10); } } DPRINTF(sc, IWN_DEBUG_TRACE, "->%s end timeout\n", __func__); return ETIMEDOUT; } static __inline void iwn_eeprom_unlock(struct iwn_softc *sc) { IWN_CLRBITS(sc, IWN_HW_IF_CONFIG, IWN_HW_IF_CONFIG_EEPROM_LOCKED); } /* * Initialize access by host to One Time Programmable ROM. * NB: This kind of ROM can be found on 1000 or 6000 Series only. */ static int iwn_init_otprom(struct iwn_softc *sc) { uint16_t prev, base, next; int count, error; DPRINTF(sc, IWN_DEBUG_TRACE, "->%s begin\n", __func__); /* Wait for clock stabilization before accessing prph. */ if ((error = iwn_clock_wait(sc)) != 0) return error; if ((error = iwn_nic_lock(sc)) != 0) return error; iwn_prph_setbits(sc, IWN_APMG_PS, IWN_APMG_PS_RESET_REQ); DELAY(5); iwn_prph_clrbits(sc, IWN_APMG_PS, IWN_APMG_PS_RESET_REQ); iwn_nic_unlock(sc); /* Set auto clock gate disable bit for HW with OTP shadow RAM. */ if (sc->base_params->shadow_ram_support) { IWN_SETBITS(sc, IWN_DBG_LINK_PWR_MGMT, IWN_RESET_LINK_PWR_MGMT_DIS); } IWN_CLRBITS(sc, IWN_EEPROM_GP, IWN_EEPROM_GP_IF_OWNER); /* Clear ECC status. */ IWN_SETBITS(sc, IWN_OTP_GP, IWN_OTP_GP_ECC_CORR_STTS | IWN_OTP_GP_ECC_UNCORR_STTS); /* * Find the block before last block (contains the EEPROM image) * for HW without OTP shadow RAM. */ if (! sc->base_params->shadow_ram_support) { /* Switch to absolute addressing mode. */ IWN_CLRBITS(sc, IWN_OTP_GP, IWN_OTP_GP_RELATIVE_ACCESS); base = prev = 0; for (count = 0; count < sc->base_params->max_ll_items; count++) { error = iwn_read_prom_data(sc, base, &next, 2); if (error != 0) return error; if (next == 0) /* End of linked-list. */ break; prev = base; base = le16toh(next); } if (count == 0 || count == sc->base_params->max_ll_items) return EIO; /* Skip "next" word. */ sc->prom_base = prev + 1; } DPRINTF(sc, IWN_DEBUG_TRACE, "->%s end\n", __func__); return 0; } static int iwn_read_prom_data(struct iwn_softc *sc, uint32_t addr, void *data, int count) { uint8_t *out = data; uint32_t val, tmp; int ntries; DPRINTF(sc, IWN_DEBUG_TRACE, "->%s begin\n", __func__); addr += sc->prom_base; for (; count > 0; count -= 2, addr++) { IWN_WRITE(sc, IWN_EEPROM, addr << 2); for (ntries = 0; ntries < 10; ntries++) { val = IWN_READ(sc, IWN_EEPROM); if (val & IWN_EEPROM_READ_VALID) break; DELAY(5); } if (ntries == 10) { device_printf(sc->sc_dev, "timeout reading ROM at 0x%x\n", addr); return ETIMEDOUT; } if (sc->sc_flags & IWN_FLAG_HAS_OTPROM) { /* OTPROM, check for ECC errors. */ tmp = IWN_READ(sc, IWN_OTP_GP); if (tmp & IWN_OTP_GP_ECC_UNCORR_STTS) { device_printf(sc->sc_dev, "OTPROM ECC error at 0x%x\n", addr); return EIO; } if (tmp & IWN_OTP_GP_ECC_CORR_STTS) { /* Correctable ECC error, clear bit. */ IWN_SETBITS(sc, IWN_OTP_GP, IWN_OTP_GP_ECC_CORR_STTS); } } *out++ = val >> 16; if (count > 1) *out++ = val >> 24; } DPRINTF(sc, IWN_DEBUG_TRACE, "->%s end\n", __func__); return 0; } static void iwn_dma_map_addr(void *arg, bus_dma_segment_t *segs, int nsegs, int error) { if (error != 0) return; KASSERT(nsegs == 1, ("too many DMA segments, %d should be 1", nsegs)); *(bus_addr_t *)arg = segs[0].ds_addr; } static int iwn_dma_contig_alloc(struct iwn_softc *sc, struct iwn_dma_info *dma, void **kvap, bus_size_t size, bus_size_t alignment) { int error; dma->tag = NULL; dma->size = size; error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), alignment, 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, size, 1, size, 0, NULL, NULL, &dma->tag); if (error != 0) goto fail; error = bus_dmamem_alloc(dma->tag, (void **)&dma->vaddr, BUS_DMA_NOWAIT | BUS_DMA_ZERO | BUS_DMA_COHERENT, &dma->map); if (error != 0) goto fail; error = bus_dmamap_load(dma->tag, dma->map, dma->vaddr, size, iwn_dma_map_addr, &dma->paddr, BUS_DMA_NOWAIT); if (error != 0) goto fail; bus_dmamap_sync(dma->tag, dma->map, BUS_DMASYNC_PREWRITE); if (kvap != NULL) *kvap = dma->vaddr; return 0; fail: iwn_dma_contig_free(dma); return error; } static void iwn_dma_contig_free(struct iwn_dma_info *dma) { if (dma->vaddr != NULL) { bus_dmamap_sync(dma->tag, dma->map, BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(dma->tag, dma->map); bus_dmamem_free(dma->tag, dma->vaddr, dma->map); dma->vaddr = NULL; } if (dma->tag != NULL) { bus_dma_tag_destroy(dma->tag); dma->tag = NULL; } } static int iwn_alloc_sched(struct iwn_softc *sc) { /* TX scheduler rings must be aligned on a 1KB boundary. */ return iwn_dma_contig_alloc(sc, &sc->sched_dma, (void **)&sc->sched, sc->schedsz, 1024); } static void iwn_free_sched(struct iwn_softc *sc) { iwn_dma_contig_free(&sc->sched_dma); } static int iwn_alloc_kw(struct iwn_softc *sc) { /* "Keep Warm" page must be aligned on a 4KB boundary. */ return iwn_dma_contig_alloc(sc, &sc->kw_dma, NULL, 4096, 4096); } static void iwn_free_kw(struct iwn_softc *sc) { iwn_dma_contig_free(&sc->kw_dma); } static int iwn_alloc_ict(struct iwn_softc *sc) { /* ICT table must be aligned on a 4KB boundary. */ return iwn_dma_contig_alloc(sc, &sc->ict_dma, (void **)&sc->ict, IWN_ICT_SIZE, 4096); } static void iwn_free_ict(struct iwn_softc *sc) { iwn_dma_contig_free(&sc->ict_dma); } static int iwn_alloc_fwmem(struct iwn_softc *sc) { /* Must be aligned on a 16-byte boundary. */ return iwn_dma_contig_alloc(sc, &sc->fw_dma, NULL, sc->fwsz, 16); } static void iwn_free_fwmem(struct iwn_softc *sc) { iwn_dma_contig_free(&sc->fw_dma); } static int iwn_alloc_rx_ring(struct iwn_softc *sc, struct iwn_rx_ring *ring) { bus_size_t size; int i, error; ring->cur = 0; DPRINTF(sc, IWN_DEBUG_TRACE, "->%s begin\n", __func__); /* Allocate RX descriptors (256-byte aligned). */ size = IWN_RX_RING_COUNT * sizeof (uint32_t); error = iwn_dma_contig_alloc(sc, &ring->desc_dma, (void **)&ring->desc, size, 256); if (error != 0) { device_printf(sc->sc_dev, "%s: could not allocate RX ring DMA memory, error %d\n", __func__, error); goto fail; } /* Allocate RX status area (16-byte aligned). */ error = iwn_dma_contig_alloc(sc, &ring->stat_dma, (void **)&ring->stat, sizeof (struct iwn_rx_status), 16); if (error != 0) { device_printf(sc->sc_dev, "%s: could not allocate RX status DMA memory, error %d\n", __func__, error); goto fail; } /* Create RX buffer DMA tag. */ error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), 1, 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, IWN_RBUF_SIZE, 1, IWN_RBUF_SIZE, 0, NULL, NULL, &ring->data_dmat); if (error != 0) { device_printf(sc->sc_dev, "%s: could not create RX buf DMA tag, error %d\n", __func__, error); goto fail; } /* * Allocate and map RX buffers. */ for (i = 0; i < IWN_RX_RING_COUNT; i++) { struct iwn_rx_data *data = &ring->data[i]; bus_addr_t paddr; error = bus_dmamap_create(ring->data_dmat, 0, &data->map); if (error != 0) { device_printf(sc->sc_dev, "%s: could not create RX buf DMA map, error %d\n", __func__, error); goto fail; } data->m = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, IWN_RBUF_SIZE); if (data->m == NULL) { device_printf(sc->sc_dev, "%s: could not allocate RX mbuf\n", __func__); error = ENOBUFS; goto fail; } error = bus_dmamap_load(ring->data_dmat, data->map, mtod(data->m, void *), IWN_RBUF_SIZE, iwn_dma_map_addr, &paddr, BUS_DMA_NOWAIT); if (error != 0 && error != EFBIG) { device_printf(sc->sc_dev, "%s: can't map mbuf, error %d\n", __func__, error); goto fail; } /* Set physical address of RX buffer (256-byte aligned). */ ring->desc[i] = htole32(paddr >> 8); } bus_dmamap_sync(ring->desc_dma.tag, ring->desc_dma.map, BUS_DMASYNC_PREWRITE); DPRINTF(sc, IWN_DEBUG_TRACE, "->%s: end\n",__func__); return 0; fail: iwn_free_rx_ring(sc, ring); DPRINTF(sc, IWN_DEBUG_TRACE, "->%s: end in error\n",__func__); return error; } static void iwn_reset_rx_ring(struct iwn_softc *sc, struct iwn_rx_ring *ring) { int ntries; DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); if (iwn_nic_lock(sc) == 0) { IWN_WRITE(sc, IWN_FH_RX_CONFIG, 0); for (ntries = 0; ntries < 1000; ntries++) { if (IWN_READ(sc, IWN_FH_RX_STATUS) & IWN_FH_RX_STATUS_IDLE) break; DELAY(10); } iwn_nic_unlock(sc); } ring->cur = 0; sc->last_rx_valid = 0; } static void iwn_free_rx_ring(struct iwn_softc *sc, struct iwn_rx_ring *ring) { int i; DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s \n", __func__); iwn_dma_contig_free(&ring->desc_dma); iwn_dma_contig_free(&ring->stat_dma); for (i = 0; i < IWN_RX_RING_COUNT; i++) { struct iwn_rx_data *data = &ring->data[i]; if (data->m != NULL) { bus_dmamap_sync(ring->data_dmat, data->map, BUS_DMASYNC_POSTREAD); bus_dmamap_unload(ring->data_dmat, data->map); m_freem(data->m); data->m = NULL; } if (data->map != NULL) bus_dmamap_destroy(ring->data_dmat, data->map); } if (ring->data_dmat != NULL) { bus_dma_tag_destroy(ring->data_dmat); ring->data_dmat = NULL; } } static int iwn_alloc_tx_ring(struct iwn_softc *sc, struct iwn_tx_ring *ring, int qid) { bus_addr_t paddr; bus_size_t size; int i, error; ring->qid = qid; ring->queued = 0; ring->cur = 0; DPRINTF(sc, IWN_DEBUG_TRACE, "->%s begin\n", __func__); /* Allocate TX descriptors (256-byte aligned). */ size = IWN_TX_RING_COUNT * sizeof (struct iwn_tx_desc); error = iwn_dma_contig_alloc(sc, &ring->desc_dma, (void **)&ring->desc, size, 256); if (error != 0) { device_printf(sc->sc_dev, "%s: could not allocate TX ring DMA memory, error %d\n", __func__, error); goto fail; } size = IWN_TX_RING_COUNT * sizeof (struct iwn_tx_cmd); error = iwn_dma_contig_alloc(sc, &ring->cmd_dma, (void **)&ring->cmd, size, 4); if (error != 0) { device_printf(sc->sc_dev, "%s: could not allocate TX cmd DMA memory, error %d\n", __func__, error); goto fail; } error = bus_dma_tag_create(bus_get_dma_tag(sc->sc_dev), 1, 0, BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, MCLBYTES, IWN_MAX_SCATTER - 1, MCLBYTES, 0, NULL, NULL, &ring->data_dmat); if (error != 0) { device_printf(sc->sc_dev, "%s: could not create TX buf DMA tag, error %d\n", __func__, error); goto fail; } paddr = ring->cmd_dma.paddr; for (i = 0; i < IWN_TX_RING_COUNT; i++) { struct iwn_tx_data *data = &ring->data[i]; data->cmd_paddr = paddr; data->scratch_paddr = paddr + 12; paddr += sizeof (struct iwn_tx_cmd); error = bus_dmamap_create(ring->data_dmat, 0, &data->map); if (error != 0) { device_printf(sc->sc_dev, "%s: could not create TX buf DMA map, error %d\n", __func__, error); goto fail; } } DPRINTF(sc, IWN_DEBUG_TRACE, "->%s end\n", __func__); return 0; fail: iwn_free_tx_ring(sc, ring); DPRINTF(sc, IWN_DEBUG_TRACE, "->%s end in error\n", __func__); return error; } static void iwn_reset_tx_ring(struct iwn_softc *sc, struct iwn_tx_ring *ring) { int i; DPRINTF(sc, IWN_DEBUG_TRACE, "->doing %s \n", __func__); for (i = 0; i < IWN_TX_RING_COUNT; i++) { struct iwn_tx_data *data = &ring->data[i]; if (data->m != NULL) { bus_dmamap_sync(ring->data_dmat, data->map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(ring->data_dmat, data->map); m_freem(data->m); data->m = NULL; } if (data->ni != NULL) { ieee80211_free_node(data->ni); data->ni = NULL; } } /* Clear TX descriptors. */ memset(ring->desc, 0, ring->desc_dma.size); bus_dmamap_sync(ring->desc_dma.tag, ring->desc_dma.map, BUS_DMASYNC_PREWRITE); sc->qfullmsk &= ~(1 << ring->qid); ring->queued = 0; ring->cur = 0; } static void iwn_free_tx_ring(struct iwn_softc *sc, struct iwn_tx_ring *ring) { int i; DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s \n", __func__); iwn_dma_contig_free(&ring->desc_dma); iwn_dma_contig_free(&ring->cmd_dma); for (i = 0; i < IWN_TX_RING_COUNT; i++) { struct iwn_tx_data *data = &ring->data[i]; if (data->m != NULL) { bus_dmamap_sync(ring->data_dmat, data->map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(ring->data_dmat, data->map); m_freem(data->m); } if (data->map != NULL) bus_dmamap_destroy(ring->data_dmat, data->map); } if (ring->data_dmat != NULL) { bus_dma_tag_destroy(ring->data_dmat); ring->data_dmat = NULL; } } static void iwn5000_ict_reset(struct iwn_softc *sc) { /* Disable interrupts. */ IWN_WRITE(sc, IWN_INT_MASK, 0); /* Reset ICT table. */ memset(sc->ict, 0, IWN_ICT_SIZE); sc->ict_cur = 0; /* Set physical address of ICT table (4KB aligned). */ DPRINTF(sc, IWN_DEBUG_RESET, "%s: enabling ICT\n", __func__); IWN_WRITE(sc, IWN_DRAM_INT_TBL, IWN_DRAM_INT_TBL_ENABLE | IWN_DRAM_INT_TBL_WRAP_CHECK | sc->ict_dma.paddr >> 12); /* Enable periodic RX interrupt. */ sc->int_mask |= IWN_INT_RX_PERIODIC; /* Switch to ICT interrupt mode in driver. */ sc->sc_flags |= IWN_FLAG_USE_ICT; /* Re-enable interrupts. */ IWN_WRITE(sc, IWN_INT, 0xffffffff); IWN_WRITE(sc, IWN_INT_MASK, sc->int_mask); } static int iwn_read_eeprom(struct iwn_softc *sc, uint8_t macaddr[IEEE80211_ADDR_LEN]) { struct iwn_ops *ops = &sc->ops; uint16_t val; int error; DPRINTF(sc, IWN_DEBUG_TRACE, "->%s begin\n", __func__); /* Check whether adapter has an EEPROM or an OTPROM. */ if (sc->hw_type >= IWN_HW_REV_TYPE_1000 && (IWN_READ(sc, IWN_OTP_GP) & IWN_OTP_GP_DEV_SEL_OTP)) sc->sc_flags |= IWN_FLAG_HAS_OTPROM; DPRINTF(sc, IWN_DEBUG_RESET, "%s found\n", (sc->sc_flags & IWN_FLAG_HAS_OTPROM) ? "OTPROM" : "EEPROM"); /* Adapter has to be powered on for EEPROM access to work. */ if ((error = iwn_apm_init(sc)) != 0) { device_printf(sc->sc_dev, "%s: could not power ON adapter, error %d\n", __func__, error); return error; } if ((IWN_READ(sc, IWN_EEPROM_GP) & 0x7) == 0) { device_printf(sc->sc_dev, "%s: bad ROM signature\n", __func__); return EIO; } if ((error = iwn_eeprom_lock(sc)) != 0) { device_printf(sc->sc_dev, "%s: could not lock ROM, error %d\n", __func__, error); return error; } if (sc->sc_flags & IWN_FLAG_HAS_OTPROM) { if ((error = iwn_init_otprom(sc)) != 0) { device_printf(sc->sc_dev, "%s: could not initialize OTPROM, error %d\n", __func__, error); return error; } } iwn_read_prom_data(sc, IWN_EEPROM_SKU_CAP, &val, 2); DPRINTF(sc, IWN_DEBUG_RESET, "SKU capabilities=0x%04x\n", le16toh(val)); /* Check if HT support is bonded out. */ if (val & htole16(IWN_EEPROM_SKU_CAP_11N)) sc->sc_flags |= IWN_FLAG_HAS_11N; iwn_read_prom_data(sc, IWN_EEPROM_RFCFG, &val, 2); sc->rfcfg = le16toh(val); DPRINTF(sc, IWN_DEBUG_RESET, "radio config=0x%04x\n", sc->rfcfg); /* Read Tx/Rx chains from ROM unless it's known to be broken. */ if (sc->txchainmask == 0) sc->txchainmask = IWN_RFCFG_TXANTMSK(sc->rfcfg); if (sc->rxchainmask == 0) sc->rxchainmask = IWN_RFCFG_RXANTMSK(sc->rfcfg); /* Read MAC address. */ iwn_read_prom_data(sc, IWN_EEPROM_MAC, macaddr, 6); /* Read adapter-specific information from EEPROM. */ ops->read_eeprom(sc); iwn_apm_stop(sc); /* Power OFF adapter. */ iwn_eeprom_unlock(sc); DPRINTF(sc, IWN_DEBUG_TRACE, "->%s end\n", __func__); return 0; } static void iwn4965_read_eeprom(struct iwn_softc *sc) { uint32_t addr; uint16_t val; int i; DPRINTF(sc, IWN_DEBUG_TRACE, "->%s begin\n", __func__); /* Read regulatory domain (4 ASCII characters). */ iwn_read_prom_data(sc, IWN4965_EEPROM_DOMAIN, sc->eeprom_domain, 4); /* Read the list of authorized channels (20MHz ones only). */ for (i = 0; i < IWN_NBANDS - 1; i++) { addr = iwn4965_regulatory_bands[i]; iwn_read_eeprom_channels(sc, i, addr); } /* Read maximum allowed TX power for 2GHz and 5GHz bands. */ iwn_read_prom_data(sc, IWN4965_EEPROM_MAXPOW, &val, 2); sc->maxpwr2GHz = val & 0xff; sc->maxpwr5GHz = val >> 8; /* Check that EEPROM values are within valid range. */ if (sc->maxpwr5GHz < 20 || sc->maxpwr5GHz > 50) sc->maxpwr5GHz = 38; if (sc->maxpwr2GHz < 20 || sc->maxpwr2GHz > 50) sc->maxpwr2GHz = 38; DPRINTF(sc, IWN_DEBUG_RESET, "maxpwr 2GHz=%d 5GHz=%d\n", sc->maxpwr2GHz, sc->maxpwr5GHz); /* Read samples for each TX power group. */ iwn_read_prom_data(sc, IWN4965_EEPROM_BANDS, sc->bands, sizeof sc->bands); /* Read voltage at which samples were taken. */ iwn_read_prom_data(sc, IWN4965_EEPROM_VOLTAGE, &val, 2); sc->eeprom_voltage = (int16_t)le16toh(val); DPRINTF(sc, IWN_DEBUG_RESET, "voltage=%d (in 0.3V)\n", sc->eeprom_voltage); #ifdef IWN_DEBUG /* Print samples. */ if (sc->sc_debug & IWN_DEBUG_ANY) { for (i = 0; i < IWN_NBANDS - 1; i++) iwn4965_print_power_group(sc, i); } #endif DPRINTF(sc, IWN_DEBUG_TRACE, "->%s end\n", __func__); } #ifdef IWN_DEBUG static void iwn4965_print_power_group(struct iwn_softc *sc, int i) { struct iwn4965_eeprom_band *band = &sc->bands[i]; struct iwn4965_eeprom_chan_samples *chans = band->chans; int j, c; printf("===band %d===\n", i); printf("chan lo=%d, chan hi=%d\n", band->lo, band->hi); printf("chan1 num=%d\n", chans[0].num); for (c = 0; c < 2; c++) { for (j = 0; j < IWN_NSAMPLES; j++) { printf("chain %d, sample %d: temp=%d gain=%d " "power=%d pa_det=%d\n", c, j, chans[0].samples[c][j].temp, chans[0].samples[c][j].gain, chans[0].samples[c][j].power, chans[0].samples[c][j].pa_det); } } printf("chan2 num=%d\n", chans[1].num); for (c = 0; c < 2; c++) { for (j = 0; j < IWN_NSAMPLES; j++) { printf("chain %d, sample %d: temp=%d gain=%d " "power=%d pa_det=%d\n", c, j, chans[1].samples[c][j].temp, chans[1].samples[c][j].gain, chans[1].samples[c][j].power, chans[1].samples[c][j].pa_det); } } } #endif static void iwn5000_read_eeprom(struct iwn_softc *sc) { struct iwn5000_eeprom_calib_hdr hdr; int32_t volt; uint32_t base, addr; uint16_t val; int i; DPRINTF(sc, IWN_DEBUG_TRACE, "->%s begin\n", __func__); /* Read regulatory domain (4 ASCII characters). */ iwn_read_prom_data(sc, IWN5000_EEPROM_REG, &val, 2); base = le16toh(val); iwn_read_prom_data(sc, base + IWN5000_EEPROM_DOMAIN, sc->eeprom_domain, 4); /* Read the list of authorized channels (20MHz ones only). */ for (i = 0; i < IWN_NBANDS - 1; i++) { addr = base + sc->base_params->regulatory_bands[i]; iwn_read_eeprom_channels(sc, i, addr); } /* Read enhanced TX power information for 6000 Series. */ if (sc->base_params->enhanced_TX_power) iwn_read_eeprom_enhinfo(sc); iwn_read_prom_data(sc, IWN5000_EEPROM_CAL, &val, 2); base = le16toh(val); iwn_read_prom_data(sc, base, &hdr, sizeof hdr); DPRINTF(sc, IWN_DEBUG_CALIBRATE, "%s: calib version=%u pa type=%u voltage=%u\n", __func__, hdr.version, hdr.pa_type, le16toh(hdr.volt)); sc->calib_ver = hdr.version; if (sc->base_params->calib_need & IWN_FLG_NEED_PHY_CALIB_TEMP_OFFSETv2) { sc->eeprom_voltage = le16toh(hdr.volt); iwn_read_prom_data(sc, base + IWN5000_EEPROM_TEMP, &val, 2); sc->eeprom_temp_high=le16toh(val); iwn_read_prom_data(sc, base + IWN5000_EEPROM_VOLT, &val, 2); sc->eeprom_temp = le16toh(val); } if (sc->hw_type == IWN_HW_REV_TYPE_5150) { /* Compute temperature offset. */ iwn_read_prom_data(sc, base + IWN5000_EEPROM_TEMP, &val, 2); sc->eeprom_temp = le16toh(val); iwn_read_prom_data(sc, base + IWN5000_EEPROM_VOLT, &val, 2); volt = le16toh(val); sc->temp_off = sc->eeprom_temp - (volt / -5); DPRINTF(sc, IWN_DEBUG_CALIBRATE, "temp=%d volt=%d offset=%dK\n", sc->eeprom_temp, volt, sc->temp_off); } else { /* Read crystal calibration. */ iwn_read_prom_data(sc, base + IWN5000_EEPROM_CRYSTAL, &sc->eeprom_crystal, sizeof (uint32_t)); DPRINTF(sc, IWN_DEBUG_CALIBRATE, "crystal calibration 0x%08x\n", le32toh(sc->eeprom_crystal)); } DPRINTF(sc, IWN_DEBUG_TRACE, "->%s end\n", __func__); } /* * Translate EEPROM flags to net80211. */ static uint32_t iwn_eeprom_channel_flags(struct iwn_eeprom_chan *channel) { uint32_t nflags; nflags = 0; if ((channel->flags & IWN_EEPROM_CHAN_ACTIVE) == 0) nflags |= IEEE80211_CHAN_PASSIVE; if ((channel->flags & IWN_EEPROM_CHAN_IBSS) == 0) nflags |= IEEE80211_CHAN_NOADHOC; if (channel->flags & IWN_EEPROM_CHAN_RADAR) { nflags |= IEEE80211_CHAN_DFS; /* XXX apparently IBSS may still be marked */ nflags |= IEEE80211_CHAN_NOADHOC; } return nflags; } static void iwn_read_eeprom_band(struct iwn_softc *sc, int n, int maxchans, int *nchans, struct ieee80211_channel chans[]) { struct iwn_eeprom_chan *channels = sc->eeprom_channels[n]; const struct iwn_chan_band *band = &iwn_bands[n]; uint8_t bands[IEEE80211_MODE_BYTES]; uint8_t chan; int i, error, nflags; DPRINTF(sc, IWN_DEBUG_TRACE, "->%s begin\n", __func__); memset(bands, 0, sizeof(bands)); if (n == 0) { setbit(bands, IEEE80211_MODE_11B); setbit(bands, IEEE80211_MODE_11G); if (sc->sc_flags & IWN_FLAG_HAS_11N) setbit(bands, IEEE80211_MODE_11NG); } else { setbit(bands, IEEE80211_MODE_11A); if (sc->sc_flags & IWN_FLAG_HAS_11N) setbit(bands, IEEE80211_MODE_11NA); } for (i = 0; i < band->nchan; i++) { if (!(channels[i].flags & IWN_EEPROM_CHAN_VALID)) { DPRINTF(sc, IWN_DEBUG_RESET, "skip chan %d flags 0x%x maxpwr %d\n", band->chan[i], channels[i].flags, channels[i].maxpwr); continue; } chan = band->chan[i]; nflags = iwn_eeprom_channel_flags(&channels[i]); error = ieee80211_add_channel(chans, maxchans, nchans, chan, 0, channels[i].maxpwr, nflags, bands); if (error != 0) break; /* Save maximum allowed TX power for this channel. */ /* XXX wrong */ sc->maxpwr[chan] = channels[i].maxpwr; DPRINTF(sc, IWN_DEBUG_RESET, "add chan %d flags 0x%x maxpwr %d\n", chan, channels[i].flags, channels[i].maxpwr); } DPRINTF(sc, IWN_DEBUG_TRACE, "->%s end\n", __func__); } static void iwn_read_eeprom_ht40(struct iwn_softc *sc, int n, int maxchans, int *nchans, struct ieee80211_channel chans[]) { struct iwn_eeprom_chan *channels = sc->eeprom_channels[n]; const struct iwn_chan_band *band = &iwn_bands[n]; uint8_t chan; int i, error, nflags; DPRINTF(sc, IWN_DEBUG_TRACE, "->%s start\n", __func__); if (!(sc->sc_flags & IWN_FLAG_HAS_11N)) { DPRINTF(sc, IWN_DEBUG_TRACE, "->%s end no 11n\n", __func__); return; } for (i = 0; i < band->nchan; i++) { if (!(channels[i].flags & IWN_EEPROM_CHAN_VALID)) { DPRINTF(sc, IWN_DEBUG_RESET, "skip chan %d flags 0x%x maxpwr %d\n", band->chan[i], channels[i].flags, channels[i].maxpwr); continue; } chan = band->chan[i]; nflags = iwn_eeprom_channel_flags(&channels[i]); nflags |= (n == 5 ? IEEE80211_CHAN_G : IEEE80211_CHAN_A); error = ieee80211_add_channel_ht40(chans, maxchans, nchans, chan, channels[i].maxpwr, nflags); switch (error) { case EINVAL: device_printf(sc->sc_dev, "%s: no entry for channel %d\n", __func__, chan); continue; case ENOENT: DPRINTF(sc, IWN_DEBUG_RESET, "%s: skip chan %d, extension channel not found\n", __func__, chan); continue; case ENOBUFS: device_printf(sc->sc_dev, "%s: channel table is full!\n", __func__); break; case 0: DPRINTF(sc, IWN_DEBUG_RESET, "add ht40 chan %d flags 0x%x maxpwr %d\n", chan, channels[i].flags, channels[i].maxpwr); /* FALLTHROUGH */ default: break; } } DPRINTF(sc, IWN_DEBUG_TRACE, "->%s end\n", __func__); } static void iwn_read_eeprom_channels(struct iwn_softc *sc, int n, uint32_t addr) { struct ieee80211com *ic = &sc->sc_ic; iwn_read_prom_data(sc, addr, &sc->eeprom_channels[n], iwn_bands[n].nchan * sizeof (struct iwn_eeprom_chan)); if (n < 5) { iwn_read_eeprom_band(sc, n, IEEE80211_CHAN_MAX, &ic->ic_nchans, ic->ic_channels); } else { iwn_read_eeprom_ht40(sc, n, IEEE80211_CHAN_MAX, &ic->ic_nchans, ic->ic_channels); } ieee80211_sort_channels(ic->ic_channels, ic->ic_nchans); } static struct iwn_eeprom_chan * iwn_find_eeprom_channel(struct iwn_softc *sc, struct ieee80211_channel *c) { int band, chan, i, j; if (IEEE80211_IS_CHAN_HT40(c)) { band = IEEE80211_IS_CHAN_5GHZ(c) ? 6 : 5; if (IEEE80211_IS_CHAN_HT40D(c)) chan = c->ic_extieee; else chan = c->ic_ieee; for (i = 0; i < iwn_bands[band].nchan; i++) { if (iwn_bands[band].chan[i] == chan) return &sc->eeprom_channels[band][i]; } } else { for (j = 0; j < 5; j++) { for (i = 0; i < iwn_bands[j].nchan; i++) { if (iwn_bands[j].chan[i] == c->ic_ieee && ((j == 0) ^ IEEE80211_IS_CHAN_A(c)) == 1) return &sc->eeprom_channels[j][i]; } } } return NULL; } static void iwn_getradiocaps(struct ieee80211com *ic, int maxchans, int *nchans, struct ieee80211_channel chans[]) { struct iwn_softc *sc = ic->ic_softc; int i; /* Parse the list of authorized channels. */ for (i = 0; i < 5 && *nchans < maxchans; i++) iwn_read_eeprom_band(sc, i, maxchans, nchans, chans); for (i = 5; i < IWN_NBANDS - 1 && *nchans < maxchans; i++) iwn_read_eeprom_ht40(sc, i, maxchans, nchans, chans); } /* * Enforce flags read from EEPROM. */ static int iwn_setregdomain(struct ieee80211com *ic, struct ieee80211_regdomain *rd, int nchan, struct ieee80211_channel chans[]) { struct iwn_softc *sc = ic->ic_softc; int i; for (i = 0; i < nchan; i++) { struct ieee80211_channel *c = &chans[i]; struct iwn_eeprom_chan *channel; channel = iwn_find_eeprom_channel(sc, c); if (channel == NULL) { ic_printf(ic, "%s: invalid channel %u freq %u/0x%x\n", __func__, c->ic_ieee, c->ic_freq, c->ic_flags); return EINVAL; } c->ic_flags |= iwn_eeprom_channel_flags(channel); } return 0; } static void iwn_read_eeprom_enhinfo(struct iwn_softc *sc) { struct iwn_eeprom_enhinfo enhinfo[35]; struct ieee80211com *ic = &sc->sc_ic; struct ieee80211_channel *c; uint16_t val, base; int8_t maxpwr; uint8_t flags; int i, j; DPRINTF(sc, IWN_DEBUG_TRACE, "->%s begin\n", __func__); iwn_read_prom_data(sc, IWN5000_EEPROM_REG, &val, 2); base = le16toh(val); iwn_read_prom_data(sc, base + IWN6000_EEPROM_ENHINFO, enhinfo, sizeof enhinfo); for (i = 0; i < nitems(enhinfo); i++) { flags = enhinfo[i].flags; if (!(flags & IWN_ENHINFO_VALID)) continue; /* Skip invalid entries. */ maxpwr = 0; if (sc->txchainmask & IWN_ANT_A) maxpwr = MAX(maxpwr, enhinfo[i].chain[0]); if (sc->txchainmask & IWN_ANT_B) maxpwr = MAX(maxpwr, enhinfo[i].chain[1]); if (sc->txchainmask & IWN_ANT_C) maxpwr = MAX(maxpwr, enhinfo[i].chain[2]); if (sc->ntxchains == 2) maxpwr = MAX(maxpwr, enhinfo[i].mimo2); else if (sc->ntxchains == 3) maxpwr = MAX(maxpwr, enhinfo[i].mimo3); for (j = 0; j < ic->ic_nchans; j++) { c = &ic->ic_channels[j]; if ((flags & IWN_ENHINFO_5GHZ)) { if (!IEEE80211_IS_CHAN_A(c)) continue; } else if ((flags & IWN_ENHINFO_OFDM)) { if (!IEEE80211_IS_CHAN_G(c)) continue; } else if (!IEEE80211_IS_CHAN_B(c)) continue; if ((flags & IWN_ENHINFO_HT40)) { if (!IEEE80211_IS_CHAN_HT40(c)) continue; } else { if (IEEE80211_IS_CHAN_HT40(c)) continue; } if (enhinfo[i].chan != 0 && enhinfo[i].chan != c->ic_ieee) continue; DPRINTF(sc, IWN_DEBUG_RESET, "channel %d(%x), maxpwr %d\n", c->ic_ieee, c->ic_flags, maxpwr / 2); c->ic_maxregpower = maxpwr / 2; c->ic_maxpower = maxpwr; } } DPRINTF(sc, IWN_DEBUG_TRACE, "->%s end\n", __func__); } static struct ieee80211_node * iwn_node_alloc(struct ieee80211vap *vap, const uint8_t mac[IEEE80211_ADDR_LEN]) { return malloc(sizeof (struct iwn_node), M_80211_NODE,M_NOWAIT | M_ZERO); } static __inline int rate2plcp(int rate) { switch (rate & 0xff) { case 12: return 0xd; case 18: return 0xf; case 24: return 0x5; case 36: return 0x7; case 48: return 0x9; case 72: return 0xb; case 96: return 0x1; case 108: return 0x3; case 2: return 10; case 4: return 20; case 11: return 55; case 22: return 110; } return 0; } static int iwn_get_1stream_tx_antmask(struct iwn_softc *sc) { return IWN_LSB(sc->txchainmask); } static int iwn_get_2stream_tx_antmask(struct iwn_softc *sc) { int tx; /* * The '2 stream' setup is a bit .. odd. * * For NICs that support only 1 antenna, default to IWN_ANT_AB or * the firmware panics (eg Intel 5100.) * * For NICs that support two antennas, we use ANT_AB. * * For NICs that support three antennas, we use the two that * wasn't the default one. * * XXX TODO: if bluetooth (full concurrent) is enabled, restrict * this to only one antenna. */ /* Default - transmit on the other antennas */ tx = (sc->txchainmask & ~IWN_LSB(sc->txchainmask)); /* Now, if it's zero, set it to IWN_ANT_AB, so to not panic firmware */ if (tx == 0) tx = IWN_ANT_AB; /* * If the NIC is a two-stream TX NIC, configure the TX mask to * the default chainmask */ else if (sc->ntxchains == 2) tx = sc->txchainmask; return (tx); } /* * Calculate the required PLCP value from the given rate, * to the given node. * * This will take the node configuration (eg 11n, rate table * setup, etc) into consideration. */ static uint32_t iwn_rate_to_plcp(struct iwn_softc *sc, struct ieee80211_node *ni, uint8_t rate) { struct ieee80211com *ic = ni->ni_ic; uint32_t plcp = 0; int ridx; /* * If it's an MCS rate, let's set the plcp correctly * and set the relevant flags based on the node config. */ if (rate & IEEE80211_RATE_MCS) { /* * Set the initial PLCP value to be between 0->31 for * MCS 0 -> MCS 31, then set the "I'm an MCS rate!" * flag. */ plcp = IEEE80211_RV(rate) | IWN_RFLAG_MCS; /* * XXX the following should only occur if both * the local configuration _and_ the remote node * advertise these capabilities. Thus this code * may need fixing! */ /* * Set the channel width and guard interval. */ if (IEEE80211_IS_CHAN_HT40(ni->ni_chan)) { plcp |= IWN_RFLAG_HT40; if (ni->ni_htcap & IEEE80211_HTCAP_SHORTGI40) plcp |= IWN_RFLAG_SGI; } else if (ni->ni_htcap & IEEE80211_HTCAP_SHORTGI20) { plcp |= IWN_RFLAG_SGI; } /* * Ensure the selected rate matches the link quality * table entries being used. */ if (rate > 0x8f) plcp |= IWN_RFLAG_ANT(sc->txchainmask); else if (rate > 0x87) plcp |= IWN_RFLAG_ANT(iwn_get_2stream_tx_antmask(sc)); else plcp |= IWN_RFLAG_ANT(iwn_get_1stream_tx_antmask(sc)); } else { /* * Set the initial PLCP - fine for both * OFDM and CCK rates. */ plcp = rate2plcp(rate); /* Set CCK flag if it's CCK */ /* XXX It would be nice to have a method * to map the ridx -> phy table entry * so we could just query that, rather than * this hack to check against IWN_RIDX_OFDM6. */ ridx = ieee80211_legacy_rate_lookup(ic->ic_rt, rate & IEEE80211_RATE_VAL); if (ridx < IWN_RIDX_OFDM6 && IEEE80211_IS_CHAN_2GHZ(ni->ni_chan)) plcp |= IWN_RFLAG_CCK; /* Set antenna configuration */ /* XXX TODO: is this the right antenna to use for legacy? */ plcp |= IWN_RFLAG_ANT(iwn_get_1stream_tx_antmask(sc)); } DPRINTF(sc, IWN_DEBUG_TXRATE, "%s: rate=0x%02x, plcp=0x%08x\n", __func__, rate, plcp); return (htole32(plcp)); } static void iwn_newassoc(struct ieee80211_node *ni, int isnew) { /* Doesn't do anything at the moment */ } static int iwn_media_change(struct ifnet *ifp) { int error; error = ieee80211_media_change(ifp); /* NB: only the fixed rate can change and that doesn't need a reset */ return (error == ENETRESET ? 0 : error); } static int iwn_newstate(struct ieee80211vap *vap, enum ieee80211_state nstate, int arg) { struct iwn_vap *ivp = IWN_VAP(vap); struct ieee80211com *ic = vap->iv_ic; struct iwn_softc *sc = ic->ic_softc; int error = 0; DPRINTF(sc, IWN_DEBUG_TRACE, "->%s begin\n", __func__); DPRINTF(sc, IWN_DEBUG_STATE, "%s: %s -> %s\n", __func__, ieee80211_state_name[vap->iv_state], ieee80211_state_name[nstate]); IEEE80211_UNLOCK(ic); IWN_LOCK(sc); callout_stop(&sc->calib_to); sc->rxon = &sc->rx_on[IWN_RXON_BSS_CTX]; switch (nstate) { case IEEE80211_S_ASSOC: if (vap->iv_state != IEEE80211_S_RUN) break; /* FALLTHROUGH */ case IEEE80211_S_AUTH: if (vap->iv_state == IEEE80211_S_AUTH) break; /* * !AUTH -> AUTH transition requires state reset to handle * reassociations correctly. */ sc->rxon->associd = 0; sc->rxon->filter &= ~htole32(IWN_FILTER_BSS); sc->calib.state = IWN_CALIB_STATE_INIT; /* Wait until we hear a beacon before we transmit */ if (IEEE80211_IS_CHAN_PASSIVE(ic->ic_curchan)) sc->sc_beacon_wait = 1; if ((error = iwn_auth(sc, vap)) != 0) { device_printf(sc->sc_dev, "%s: could not move to auth state\n", __func__); } break; case IEEE80211_S_RUN: /* * RUN -> RUN transition; Just restart the timers. */ if (vap->iv_state == IEEE80211_S_RUN) { sc->calib_cnt = 0; break; } /* Wait until we hear a beacon before we transmit */ if (IEEE80211_IS_CHAN_PASSIVE(ic->ic_curchan)) sc->sc_beacon_wait = 1; /* * !RUN -> RUN requires setting the association id * which is done with a firmware cmd. We also defer * starting the timers until that work is done. */ if ((error = iwn_run(sc, vap)) != 0) { device_printf(sc->sc_dev, "%s: could not move to run state\n", __func__); } break; case IEEE80211_S_INIT: sc->calib.state = IWN_CALIB_STATE_INIT; /* * Purge the xmit queue so we don't have old frames * during a new association attempt. */ sc->sc_beacon_wait = 0; iwn_xmit_queue_drain(sc); break; default: break; } IWN_UNLOCK(sc); IEEE80211_LOCK(ic); if (error != 0){ DPRINTF(sc, IWN_DEBUG_TRACE, "->%s end in error\n", __func__); return error; } DPRINTF(sc, IWN_DEBUG_TRACE, "->%s: end\n",__func__); return ivp->iv_newstate(vap, nstate, arg); } static void iwn_calib_timeout(void *arg) { struct iwn_softc *sc = arg; IWN_LOCK_ASSERT(sc); /* Force automatic TX power calibration every 60 secs. */ if (++sc->calib_cnt >= 120) { uint32_t flags = 0; DPRINTF(sc, IWN_DEBUG_CALIBRATE, "%s\n", "sending request for statistics"); (void)iwn_cmd(sc, IWN_CMD_GET_STATISTICS, &flags, sizeof flags, 1); sc->calib_cnt = 0; } callout_reset(&sc->calib_to, msecs_to_ticks(500), iwn_calib_timeout, sc); } /* * Process an RX_PHY firmware notification. This is usually immediately * followed by an MPDU_RX_DONE notification. */ static void iwn_rx_phy(struct iwn_softc *sc, struct iwn_rx_desc *desc, struct iwn_rx_data *data) { struct iwn_rx_stat *stat = (struct iwn_rx_stat *)(desc + 1); DPRINTF(sc, IWN_DEBUG_CALIBRATE, "%s: received PHY stats\n", __func__); bus_dmamap_sync(sc->rxq.data_dmat, data->map, BUS_DMASYNC_POSTREAD); /* Save RX statistics, they will be used on MPDU_RX_DONE. */ memcpy(&sc->last_rx_stat, stat, sizeof (*stat)); sc->last_rx_valid = 1; } /* * Process an RX_DONE (4965AGN only) or MPDU_RX_DONE firmware notification. * Each MPDU_RX_DONE notification must be preceded by an RX_PHY one. */ static void iwn_rx_done(struct iwn_softc *sc, struct iwn_rx_desc *desc, struct iwn_rx_data *data) { struct iwn_ops *ops = &sc->ops; struct ieee80211com *ic = &sc->sc_ic; struct iwn_rx_ring *ring = &sc->rxq; struct ieee80211_frame *wh; struct ieee80211_node *ni; struct mbuf *m, *m1; struct iwn_rx_stat *stat; caddr_t head; bus_addr_t paddr; uint32_t flags; int error, len, rssi, nf; DPRINTF(sc, IWN_DEBUG_TRACE, "->%s begin\n", __func__); if (desc->type == IWN_MPDU_RX_DONE) { /* Check for prior RX_PHY notification. */ if (!sc->last_rx_valid) { DPRINTF(sc, IWN_DEBUG_ANY, "%s: missing RX_PHY\n", __func__); return; } stat = &sc->last_rx_stat; } else stat = (struct iwn_rx_stat *)(desc + 1); bus_dmamap_sync(ring->data_dmat, data->map, BUS_DMASYNC_POSTREAD); if (stat->cfg_phy_len > IWN_STAT_MAXLEN) { device_printf(sc->sc_dev, "%s: invalid RX statistic header, len %d\n", __func__, stat->cfg_phy_len); return; } if (desc->type == IWN_MPDU_RX_DONE) { struct iwn_rx_mpdu *mpdu = (struct iwn_rx_mpdu *)(desc + 1); head = (caddr_t)(mpdu + 1); len = le16toh(mpdu->len); } else { head = (caddr_t)(stat + 1) + stat->cfg_phy_len; len = le16toh(stat->len); } flags = le32toh(*(uint32_t *)(head + len)); /* Discard frames with a bad FCS early. */ if ((flags & IWN_RX_NOERROR) != IWN_RX_NOERROR) { DPRINTF(sc, IWN_DEBUG_RECV, "%s: RX flags error %x\n", __func__, flags); counter_u64_add(ic->ic_ierrors, 1); return; } /* Discard frames that are too short. */ if (len < sizeof (struct ieee80211_frame_ack)) { DPRINTF(sc, IWN_DEBUG_RECV, "%s: frame too short: %d\n", __func__, len); counter_u64_add(ic->ic_ierrors, 1); return; } m1 = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, IWN_RBUF_SIZE); if (m1 == NULL) { DPRINTF(sc, IWN_DEBUG_ANY, "%s: no mbuf to restock ring\n", __func__); counter_u64_add(ic->ic_ierrors, 1); return; } bus_dmamap_unload(ring->data_dmat, data->map); error = bus_dmamap_load(ring->data_dmat, data->map, mtod(m1, void *), IWN_RBUF_SIZE, iwn_dma_map_addr, &paddr, BUS_DMA_NOWAIT); if (error != 0 && error != EFBIG) { device_printf(sc->sc_dev, "%s: bus_dmamap_load failed, error %d\n", __func__, error); m_freem(m1); /* Try to reload the old mbuf. */ error = bus_dmamap_load(ring->data_dmat, data->map, mtod(data->m, void *), IWN_RBUF_SIZE, iwn_dma_map_addr, &paddr, BUS_DMA_NOWAIT); if (error != 0 && error != EFBIG) { panic("%s: could not load old RX mbuf", __func__); } /* Physical address may have changed. */ ring->desc[ring->cur] = htole32(paddr >> 8); bus_dmamap_sync(ring->data_dmat, ring->desc_dma.map, BUS_DMASYNC_PREWRITE); counter_u64_add(ic->ic_ierrors, 1); return; } m = data->m; data->m = m1; /* Update RX descriptor. */ ring->desc[ring->cur] = htole32(paddr >> 8); bus_dmamap_sync(ring->desc_dma.tag, ring->desc_dma.map, BUS_DMASYNC_PREWRITE); /* Finalize mbuf. */ m->m_data = head; m->m_pkthdr.len = m->m_len = len; /* Grab a reference to the source node. */ wh = mtod(m, struct ieee80211_frame *); if (len >= sizeof(struct ieee80211_frame_min)) ni = ieee80211_find_rxnode(ic, (struct ieee80211_frame_min *)wh); else ni = NULL; nf = (ni != NULL && ni->ni_vap->iv_state == IEEE80211_S_RUN && (ic->ic_flags & IEEE80211_F_SCAN) == 0) ? sc->noise : -95; rssi = ops->get_rssi(sc, stat); if (ieee80211_radiotap_active(ic)) { struct iwn_rx_radiotap_header *tap = &sc->sc_rxtap; tap->wr_flags = 0; if (stat->flags & htole16(IWN_STAT_FLAG_SHPREAMBLE)) tap->wr_flags |= IEEE80211_RADIOTAP_F_SHORTPRE; tap->wr_dbm_antsignal = (int8_t)rssi; tap->wr_dbm_antnoise = (int8_t)nf; tap->wr_tsft = stat->tstamp; switch (stat->rate) { /* CCK rates. */ case 10: tap->wr_rate = 2; break; case 20: tap->wr_rate = 4; break; case 55: tap->wr_rate = 11; break; case 110: tap->wr_rate = 22; break; /* OFDM rates. */ case 0xd: tap->wr_rate = 12; break; case 0xf: tap->wr_rate = 18; break; case 0x5: tap->wr_rate = 24; break; case 0x7: tap->wr_rate = 36; break; case 0x9: tap->wr_rate = 48; break; case 0xb: tap->wr_rate = 72; break; case 0x1: tap->wr_rate = 96; break; case 0x3: tap->wr_rate = 108; break; /* Unknown rate: should not happen. */ default: tap->wr_rate = 0; } } /* * If it's a beacon and we're waiting, then do the * wakeup. This should unblock raw_xmit/start. */ if (sc->sc_beacon_wait) { uint8_t type, subtype; /* NB: Re-assign wh */ wh = mtod(m, struct ieee80211_frame *); type = wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK; subtype = wh->i_fc[0] & IEEE80211_FC0_SUBTYPE_MASK; /* * This assumes at this point we've received our own * beacon. */ DPRINTF(sc, IWN_DEBUG_TRACE, "%s: beacon_wait, type=%d, subtype=%d\n", __func__, type, subtype); if (type == IEEE80211_FC0_TYPE_MGT && subtype == IEEE80211_FC0_SUBTYPE_BEACON) { DPRINTF(sc, IWN_DEBUG_TRACE | IWN_DEBUG_XMIT, "%s: waking things up\n", __func__); /* queue taskqueue to transmit! */ taskqueue_enqueue(sc->sc_tq, &sc->sc_xmit_task); } } IWN_UNLOCK(sc); /* Send the frame to the 802.11 layer. */ if (ni != NULL) { if (ni->ni_flags & IEEE80211_NODE_HT) m->m_flags |= M_AMPDU; (void)ieee80211_input(ni, m, rssi - nf, nf); /* Node is no longer needed. */ ieee80211_free_node(ni); } else (void)ieee80211_input_all(ic, m, rssi - nf, nf); IWN_LOCK(sc); DPRINTF(sc, IWN_DEBUG_TRACE, "->%s: end\n",__func__); } /* Process an incoming Compressed BlockAck. */ static void iwn_rx_compressed_ba(struct iwn_softc *sc, struct iwn_rx_desc *desc, struct iwn_rx_data *data) { struct iwn_ops *ops = &sc->ops; struct iwn_node *wn; struct ieee80211_node *ni; struct iwn_compressed_ba *ba = (struct iwn_compressed_ba *)(desc + 1); struct iwn_tx_ring *txq; struct iwn_tx_data *txdata; struct ieee80211_tx_ampdu *tap; struct mbuf *m; uint64_t bitmap; uint16_t ssn; uint8_t tid; int ackfailcnt = 0, i, lastidx, qid, *res, shift; int tx_ok = 0, tx_err = 0; DPRINTF(sc, IWN_DEBUG_TRACE | IWN_DEBUG_XMIT, "->%s begin\n", __func__); bus_dmamap_sync(sc->rxq.data_dmat, data->map, BUS_DMASYNC_POSTREAD); qid = le16toh(ba->qid); txq = &sc->txq[ba->qid]; tap = sc->qid2tap[ba->qid]; tid = tap->txa_tid; wn = (void *)tap->txa_ni; res = NULL; ssn = 0; if (!IEEE80211_AMPDU_RUNNING(tap)) { res = tap->txa_private; ssn = tap->txa_start & 0xfff; } for (lastidx = le16toh(ba->ssn) & 0xff; txq->read != lastidx;) { txdata = &txq->data[txq->read]; /* Unmap and free mbuf. */ bus_dmamap_sync(txq->data_dmat, txdata->map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(txq->data_dmat, txdata->map); m = txdata->m, txdata->m = NULL; ni = txdata->ni, txdata->ni = NULL; KASSERT(ni != NULL, ("no node")); KASSERT(m != NULL, ("no mbuf")); DPRINTF(sc, IWN_DEBUG_XMIT, "%s: freeing m=%p\n", __func__, m); ieee80211_tx_complete(ni, m, 1); txq->queued--; txq->read = (txq->read + 1) % IWN_TX_RING_COUNT; } if (txq->queued == 0 && res != NULL) { iwn_nic_lock(sc); ops->ampdu_tx_stop(sc, qid, tid, ssn); iwn_nic_unlock(sc); sc->qid2tap[qid] = NULL; free(res, M_DEVBUF); return; } if (wn->agg[tid].bitmap == 0) return; shift = wn->agg[tid].startidx - ((le16toh(ba->seq) >> 4) & 0xff); if (shift < 0) shift += 0x100; if (wn->agg[tid].nframes > (64 - shift)) return; /* * Walk the bitmap and calculate how many successful and failed * attempts are made. * * Yes, the rate control code doesn't know these are A-MPDU * subframes and that it's okay to fail some of these. */ ni = tap->txa_ni; bitmap = (le64toh(ba->bitmap) >> shift) & wn->agg[tid].bitmap; for (i = 0; bitmap; i++) { if ((bitmap & 1) == 0) { tx_err ++; ieee80211_ratectl_tx_complete(ni->ni_vap, ni, IEEE80211_RATECTL_TX_FAILURE, &ackfailcnt, NULL); } else { tx_ok ++; ieee80211_ratectl_tx_complete(ni->ni_vap, ni, IEEE80211_RATECTL_TX_SUCCESS, &ackfailcnt, NULL); } bitmap >>= 1; } DPRINTF(sc, IWN_DEBUG_TRACE | IWN_DEBUG_XMIT, "->%s: end; %d ok; %d err\n",__func__, tx_ok, tx_err); } /* * Process a CALIBRATION_RESULT notification sent by the initialization * firmware on response to a CMD_CALIB_CONFIG command (5000 only). */ static void iwn5000_rx_calib_results(struct iwn_softc *sc, struct iwn_rx_desc *desc, struct iwn_rx_data *data) { struct iwn_phy_calib *calib = (struct iwn_phy_calib *)(desc + 1); int len, idx = -1; DPRINTF(sc, IWN_DEBUG_TRACE, "->%s begin\n", __func__); /* Runtime firmware should not send such a notification. */ if (sc->sc_flags & IWN_FLAG_CALIB_DONE){ DPRINTF(sc, IWN_DEBUG_TRACE, "->%s received after clib done\n", __func__); return; } len = (le32toh(desc->len) & 0x3fff) - 4; bus_dmamap_sync(sc->rxq.data_dmat, data->map, BUS_DMASYNC_POSTREAD); switch (calib->code) { case IWN5000_PHY_CALIB_DC: if (sc->base_params->calib_need & IWN_FLG_NEED_PHY_CALIB_DC) idx = 0; break; case IWN5000_PHY_CALIB_LO: if (sc->base_params->calib_need & IWN_FLG_NEED_PHY_CALIB_LO) idx = 1; break; case IWN5000_PHY_CALIB_TX_IQ: if (sc->base_params->calib_need & IWN_FLG_NEED_PHY_CALIB_TX_IQ) idx = 2; break; case IWN5000_PHY_CALIB_TX_IQ_PERIODIC: if (sc->base_params->calib_need & IWN_FLG_NEED_PHY_CALIB_TX_IQ_PERIODIC) idx = 3; break; case IWN5000_PHY_CALIB_BASE_BAND: if (sc->base_params->calib_need & IWN_FLG_NEED_PHY_CALIB_BASE_BAND) idx = 4; break; } if (idx == -1) /* Ignore other results. */ return; /* Save calibration result. */ if (sc->calibcmd[idx].buf != NULL) free(sc->calibcmd[idx].buf, M_DEVBUF); sc->calibcmd[idx].buf = malloc(len, M_DEVBUF, M_NOWAIT); if (sc->calibcmd[idx].buf == NULL) { DPRINTF(sc, IWN_DEBUG_CALIBRATE, "not enough memory for calibration result %d\n", calib->code); return; } DPRINTF(sc, IWN_DEBUG_CALIBRATE, "saving calibration result idx=%d, code=%d len=%d\n", idx, calib->code, len); sc->calibcmd[idx].len = len; memcpy(sc->calibcmd[idx].buf, calib, len); } static void iwn_stats_update(struct iwn_softc *sc, struct iwn_calib_state *calib, struct iwn_stats *stats, int len) { struct iwn_stats_bt *stats_bt; struct iwn_stats *lstats; /* * First - check whether the length is the bluetooth or normal. * * If it's normal - just copy it and bump out. * Otherwise we have to convert things. */ if (len == sizeof(struct iwn_stats) + 4) { memcpy(&sc->last_stat, stats, sizeof(struct iwn_stats)); sc->last_stat_valid = 1; return; } /* * If it's not the bluetooth size - log, then just copy. */ if (len != sizeof(struct iwn_stats_bt) + 4) { DPRINTF(sc, IWN_DEBUG_STATS, "%s: size of rx statistics (%d) not an expected size!\n", __func__, len); memcpy(&sc->last_stat, stats, sizeof(struct iwn_stats)); sc->last_stat_valid = 1; return; } /* * Ok. Time to copy. */ stats_bt = (struct iwn_stats_bt *) stats; lstats = &sc->last_stat; /* flags */ lstats->flags = stats_bt->flags; /* rx_bt */ memcpy(&lstats->rx.ofdm, &stats_bt->rx_bt.ofdm, sizeof(struct iwn_rx_phy_stats)); memcpy(&lstats->rx.cck, &stats_bt->rx_bt.cck, sizeof(struct iwn_rx_phy_stats)); memcpy(&lstats->rx.general, &stats_bt->rx_bt.general_bt.common, sizeof(struct iwn_rx_general_stats)); memcpy(&lstats->rx.ht, &stats_bt->rx_bt.ht, sizeof(struct iwn_rx_ht_phy_stats)); /* tx */ memcpy(&lstats->tx, &stats_bt->tx, sizeof(struct iwn_tx_stats)); /* general */ memcpy(&lstats->general, &stats_bt->general, sizeof(struct iwn_general_stats)); /* XXX TODO: Squirrel away the extra bluetooth stats somewhere */ sc->last_stat_valid = 1; } /* * Process an RX_STATISTICS or BEACON_STATISTICS firmware notification. * The latter is sent by the firmware after each received beacon. */ static void iwn_rx_statistics(struct iwn_softc *sc, struct iwn_rx_desc *desc, struct iwn_rx_data *data) { struct iwn_ops *ops = &sc->ops; struct ieee80211com *ic = &sc->sc_ic; struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps); struct iwn_calib_state *calib = &sc->calib; struct iwn_stats *stats = (struct iwn_stats *)(desc + 1); struct iwn_stats *lstats; int temp; DPRINTF(sc, IWN_DEBUG_TRACE, "->%s begin\n", __func__); /* Ignore statistics received during a scan. */ if (vap->iv_state != IEEE80211_S_RUN || (ic->ic_flags & IEEE80211_F_SCAN)){ DPRINTF(sc, IWN_DEBUG_TRACE, "->%s received during calib\n", __func__); return; } bus_dmamap_sync(sc->rxq.data_dmat, data->map, BUS_DMASYNC_POSTREAD); DPRINTF(sc, IWN_DEBUG_CALIBRATE | IWN_DEBUG_STATS, "%s: received statistics, cmd %d, len %d\n", __func__, desc->type, le16toh(desc->len)); sc->calib_cnt = 0; /* Reset TX power calibration timeout. */ /* * Collect/track general statistics for reporting. * * This takes care of ensuring that the bluetooth sized message * will be correctly converted to the legacy sized message. */ iwn_stats_update(sc, calib, stats, le16toh(desc->len)); /* * And now, let's take a reference of it to use! */ lstats = &sc->last_stat; /* Test if temperature has changed. */ if (lstats->general.temp != sc->rawtemp) { /* Convert "raw" temperature to degC. */ sc->rawtemp = stats->general.temp; temp = ops->get_temperature(sc); DPRINTF(sc, IWN_DEBUG_CALIBRATE, "%s: temperature %d\n", __func__, temp); /* Update TX power if need be (4965AGN only). */ if (sc->hw_type == IWN_HW_REV_TYPE_4965) iwn4965_power_calibration(sc, temp); } if (desc->type != IWN_BEACON_STATISTICS) return; /* Reply to a statistics request. */ sc->noise = iwn_get_noise(&lstats->rx.general); DPRINTF(sc, IWN_DEBUG_CALIBRATE, "%s: noise %d\n", __func__, sc->noise); /* Test that RSSI and noise are present in stats report. */ if (le32toh(lstats->rx.general.flags) != 1) { DPRINTF(sc, IWN_DEBUG_ANY, "%s\n", "received statistics without RSSI"); return; } if (calib->state == IWN_CALIB_STATE_ASSOC) iwn_collect_noise(sc, &lstats->rx.general); else if (calib->state == IWN_CALIB_STATE_RUN) { iwn_tune_sensitivity(sc, &lstats->rx); /* * XXX TODO: Only run the RX recovery if we're associated! */ iwn_check_rx_recovery(sc, lstats); iwn_save_stats_counters(sc, lstats); } DPRINTF(sc, IWN_DEBUG_TRACE, "->%s: end\n",__func__); } /* * Save the relevant statistic counters for the next calibration * pass. */ static void iwn_save_stats_counters(struct iwn_softc *sc, const struct iwn_stats *rs) { struct iwn_calib_state *calib = &sc->calib; /* Save counters values for next call. */ calib->bad_plcp_cck = le32toh(rs->rx.cck.bad_plcp); calib->fa_cck = le32toh(rs->rx.cck.fa); calib->bad_plcp_ht = le32toh(rs->rx.ht.bad_plcp); calib->bad_plcp_ofdm = le32toh(rs->rx.ofdm.bad_plcp); calib->fa_ofdm = le32toh(rs->rx.ofdm.fa); /* Last time we received these tick values */ sc->last_calib_ticks = ticks; } /* * Process a TX_DONE firmware notification. Unfortunately, the 4965AGN * and 5000 adapters have different incompatible TX status formats. */ static void iwn4965_tx_done(struct iwn_softc *sc, struct iwn_rx_desc *desc, struct iwn_rx_data *data) { struct iwn4965_tx_stat *stat = (struct iwn4965_tx_stat *)(desc + 1); struct iwn_tx_ring *ring; int qid; qid = desc->qid & 0xf; ring = &sc->txq[qid]; DPRINTF(sc, IWN_DEBUG_XMIT, "%s: " "qid %d idx %d RTS retries %d ACK retries %d nkill %d rate %x duration %d status %x\n", __func__, desc->qid, desc->idx, stat->rtsfailcnt, stat->ackfailcnt, stat->btkillcnt, stat->rate, le16toh(stat->duration), le32toh(stat->status)); bus_dmamap_sync(ring->data_dmat, data->map, BUS_DMASYNC_POSTREAD); if (qid >= sc->firstaggqueue) { iwn_ampdu_tx_done(sc, qid, desc->idx, stat->nframes, stat->ackfailcnt, &stat->status); } else { iwn_tx_done(sc, desc, stat->ackfailcnt, le32toh(stat->status) & 0xff); } } static void iwn5000_tx_done(struct iwn_softc *sc, struct iwn_rx_desc *desc, struct iwn_rx_data *data) { struct iwn5000_tx_stat *stat = (struct iwn5000_tx_stat *)(desc + 1); struct iwn_tx_ring *ring; int qid; qid = desc->qid & 0xf; ring = &sc->txq[qid]; DPRINTF(sc, IWN_DEBUG_XMIT, "%s: " "qid %d idx %d RTS retries %d ACK retries %d nkill %d rate %x duration %d status %x\n", __func__, desc->qid, desc->idx, stat->rtsfailcnt, stat->ackfailcnt, stat->btkillcnt, stat->rate, le16toh(stat->duration), le32toh(stat->status)); #ifdef notyet /* Reset TX scheduler slot. */ iwn5000_reset_sched(sc, desc->qid & 0xf, desc->idx); #endif bus_dmamap_sync(ring->data_dmat, data->map, BUS_DMASYNC_POSTREAD); if (qid >= sc->firstaggqueue) { iwn_ampdu_tx_done(sc, qid, desc->idx, stat->nframes, stat->ackfailcnt, &stat->status); } else { iwn_tx_done(sc, desc, stat->ackfailcnt, le16toh(stat->status) & 0xff); } } /* * Adapter-independent backend for TX_DONE firmware notifications. */ static void iwn_tx_done(struct iwn_softc *sc, struct iwn_rx_desc *desc, int ackfailcnt, uint8_t status) { struct iwn_tx_ring *ring = &sc->txq[desc->qid & 0xf]; struct iwn_tx_data *data = &ring->data[desc->idx]; struct mbuf *m; struct ieee80211_node *ni; struct ieee80211vap *vap; KASSERT(data->ni != NULL, ("no node")); DPRINTF(sc, IWN_DEBUG_TRACE, "->%s begin\n", __func__); /* Unmap and free mbuf. */ bus_dmamap_sync(ring->data_dmat, data->map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(ring->data_dmat, data->map); m = data->m, data->m = NULL; ni = data->ni, data->ni = NULL; vap = ni->ni_vap; /* * Update rate control statistics for the node. */ if (status & IWN_TX_FAIL) ieee80211_ratectl_tx_complete(vap, ni, IEEE80211_RATECTL_TX_FAILURE, &ackfailcnt, NULL); else ieee80211_ratectl_tx_complete(vap, ni, IEEE80211_RATECTL_TX_SUCCESS, &ackfailcnt, NULL); /* * Channels marked for "radar" require traffic to be received * to unlock before we can transmit. Until traffic is seen * any attempt to transmit is returned immediately with status * set to IWN_TX_FAIL_TX_LOCKED. Unfortunately this can easily * happen on first authenticate after scanning. To workaround * this we ignore a failure of this sort in AUTH state so the * 802.11 layer will fall back to using a timeout to wait for * the AUTH reply. This allows the firmware time to see * traffic so a subsequent retry of AUTH succeeds. It's * unclear why the firmware does not maintain state for * channels recently visited as this would allow immediate * use of the channel after a scan (where we see traffic). */ if (status == IWN_TX_FAIL_TX_LOCKED && ni->ni_vap->iv_state == IEEE80211_S_AUTH) ieee80211_tx_complete(ni, m, 0); else ieee80211_tx_complete(ni, m, (status & IWN_TX_FAIL) != 0); sc->sc_tx_timer = 0; if (--ring->queued < IWN_TX_RING_LOMARK) sc->qfullmsk &= ~(1 << ring->qid); DPRINTF(sc, IWN_DEBUG_TRACE, "->%s: end\n",__func__); } /* * Process a "command done" firmware notification. This is where we wakeup * processes waiting for a synchronous command completion. */ static void iwn_cmd_done(struct iwn_softc *sc, struct iwn_rx_desc *desc) { struct iwn_tx_ring *ring; struct iwn_tx_data *data; int cmd_queue_num; if (sc->sc_flags & IWN_FLAG_PAN_SUPPORT) cmd_queue_num = IWN_PAN_CMD_QUEUE; else cmd_queue_num = IWN_CMD_QUEUE_NUM; if ((desc->qid & IWN_RX_DESC_QID_MSK) != cmd_queue_num) return; /* Not a command ack. */ ring = &sc->txq[cmd_queue_num]; data = &ring->data[desc->idx]; /* If the command was mapped in an mbuf, free it. */ if (data->m != NULL) { bus_dmamap_sync(ring->data_dmat, data->map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(ring->data_dmat, data->map); m_freem(data->m); data->m = NULL; } wakeup(&ring->desc[desc->idx]); } static void iwn_ampdu_tx_done(struct iwn_softc *sc, int qid, int idx, int nframes, int ackfailcnt, void *stat) { struct iwn_ops *ops = &sc->ops; struct iwn_tx_ring *ring = &sc->txq[qid]; struct iwn_tx_data *data; struct mbuf *m; struct iwn_node *wn; struct ieee80211_node *ni; struct ieee80211_tx_ampdu *tap; uint64_t bitmap; uint32_t *status = stat; uint16_t *aggstatus = stat; uint16_t ssn; uint8_t tid; int bit, i, lastidx, *res, seqno, shift, start; /* XXX TODO: status is le16 field! Grr */ DPRINTF(sc, IWN_DEBUG_TRACE, "->%s begin\n", __func__); DPRINTF(sc, IWN_DEBUG_XMIT, "%s: nframes=%d, status=0x%08x\n", __func__, nframes, *status); tap = sc->qid2tap[qid]; tid = tap->txa_tid; wn = (void *)tap->txa_ni; ni = tap->txa_ni; /* * XXX TODO: ACK and RTS failures would be nice here! */ /* * A-MPDU single frame status - if we failed to transmit it * in A-MPDU, then it may be a permanent failure. * * XXX TODO: check what the Linux iwlwifi driver does here; * there's some permanent and temporary failures that may be * handled differently. */ if (nframes == 1) { if ((*status & 0xff) != 1 && (*status & 0xff) != 2) { #ifdef NOT_YET printf("ieee80211_send_bar()\n"); #endif /* * If we completely fail a transmit, make sure a * notification is pushed up to the rate control * layer. */ ieee80211_ratectl_tx_complete(ni->ni_vap, ni, IEEE80211_RATECTL_TX_FAILURE, &ackfailcnt, NULL); } else { /* * If nframes=1, then we won't be getting a BA for * this frame. Ensure that we correctly update the * rate control code with how many retries were * needed to send it. */ ieee80211_ratectl_tx_complete(ni->ni_vap, ni, IEEE80211_RATECTL_TX_SUCCESS, &ackfailcnt, NULL); } } bitmap = 0; start = idx; for (i = 0; i < nframes; i++) { if (le16toh(aggstatus[i * 2]) & 0xc) continue; idx = le16toh(aggstatus[2*i + 1]) & 0xff; bit = idx - start; shift = 0; if (bit >= 64) { shift = 0x100 - idx + start; bit = 0; start = idx; } else if (bit <= -64) bit = 0x100 - start + idx; else if (bit < 0) { shift = start - idx; start = idx; bit = 0; } bitmap = bitmap << shift; bitmap |= 1ULL << bit; } tap = sc->qid2tap[qid]; tid = tap->txa_tid; wn = (void *)tap->txa_ni; wn->agg[tid].bitmap = bitmap; wn->agg[tid].startidx = start; wn->agg[tid].nframes = nframes; res = NULL; ssn = 0; if (!IEEE80211_AMPDU_RUNNING(tap)) { res = tap->txa_private; ssn = tap->txa_start & 0xfff; } /* This is going nframes DWORDS into the descriptor? */ seqno = le32toh(*(status + nframes)) & 0xfff; for (lastidx = (seqno & 0xff); ring->read != lastidx;) { data = &ring->data[ring->read]; /* Unmap and free mbuf. */ bus_dmamap_sync(ring->data_dmat, data->map, BUS_DMASYNC_POSTWRITE); bus_dmamap_unload(ring->data_dmat, data->map); m = data->m, data->m = NULL; ni = data->ni, data->ni = NULL; KASSERT(ni != NULL, ("no node")); KASSERT(m != NULL, ("no mbuf")); DPRINTF(sc, IWN_DEBUG_XMIT, "%s: freeing m=%p\n", __func__, m); ieee80211_tx_complete(ni, m, 1); ring->queued--; ring->read = (ring->read + 1) % IWN_TX_RING_COUNT; } if (ring->queued == 0 && res != NULL) { iwn_nic_lock(sc); ops->ampdu_tx_stop(sc, qid, tid, ssn); iwn_nic_unlock(sc); sc->qid2tap[qid] = NULL; free(res, M_DEVBUF); return; } sc->sc_tx_timer = 0; if (ring->queued < IWN_TX_RING_LOMARK) sc->qfullmsk &= ~(1 << ring->qid); DPRINTF(sc, IWN_DEBUG_TRACE, "->%s: end\n",__func__); } /* * Process an INT_FH_RX or INT_SW_RX interrupt. */ static void iwn_notif_intr(struct iwn_softc *sc) { struct iwn_ops *ops = &sc->ops; struct ieee80211com *ic = &sc->sc_ic; struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps); uint16_t hw; bus_dmamap_sync(sc->rxq.stat_dma.tag, sc->rxq.stat_dma.map, BUS_DMASYNC_POSTREAD); hw = le16toh(sc->rxq.stat->closed_count) & 0xfff; while (sc->rxq.cur != hw) { struct iwn_rx_data *data = &sc->rxq.data[sc->rxq.cur]; struct iwn_rx_desc *desc; bus_dmamap_sync(sc->rxq.data_dmat, data->map, BUS_DMASYNC_POSTREAD); desc = mtod(data->m, struct iwn_rx_desc *); DPRINTF(sc, IWN_DEBUG_RECV, "%s: cur=%d; qid %x idx %d flags %x type %d(%s) len %d\n", __func__, sc->rxq.cur, desc->qid & 0xf, desc->idx, desc->flags, desc->type, iwn_intr_str(desc->type), le16toh(desc->len)); if (!(desc->qid & IWN_UNSOLICITED_RX_NOTIF)) /* Reply to a command. */ iwn_cmd_done(sc, desc); switch (desc->type) { case IWN_RX_PHY: iwn_rx_phy(sc, desc, data); break; case IWN_RX_DONE: /* 4965AGN only. */ case IWN_MPDU_RX_DONE: /* An 802.11 frame has been received. */ iwn_rx_done(sc, desc, data); break; case IWN_RX_COMPRESSED_BA: /* A Compressed BlockAck has been received. */ iwn_rx_compressed_ba(sc, desc, data); break; case IWN_TX_DONE: /* An 802.11 frame has been transmitted. */ ops->tx_done(sc, desc, data); break; case IWN_RX_STATISTICS: case IWN_BEACON_STATISTICS: iwn_rx_statistics(sc, desc, data); break; case IWN_BEACON_MISSED: { struct iwn_beacon_missed *miss = (struct iwn_beacon_missed *)(desc + 1); int misses; bus_dmamap_sync(sc->rxq.data_dmat, data->map, BUS_DMASYNC_POSTREAD); misses = le32toh(miss->consecutive); DPRINTF(sc, IWN_DEBUG_STATE, "%s: beacons missed %d/%d\n", __func__, misses, le32toh(miss->total)); /* * If more than 5 consecutive beacons are missed, * reinitialize the sensitivity state machine. */ if (vap->iv_state == IEEE80211_S_RUN && (ic->ic_flags & IEEE80211_F_SCAN) == 0) { if (misses > 5) (void)iwn_init_sensitivity(sc); if (misses >= vap->iv_bmissthreshold) { IWN_UNLOCK(sc); ieee80211_beacon_miss(ic); IWN_LOCK(sc); } } break; } case IWN_UC_READY: { struct iwn_ucode_info *uc = (struct iwn_ucode_info *)(desc + 1); /* The microcontroller is ready. */ bus_dmamap_sync(sc->rxq.data_dmat, data->map, BUS_DMASYNC_POSTREAD); DPRINTF(sc, IWN_DEBUG_RESET, "microcode alive notification version=%d.%d " "subtype=%x alive=%x\n", uc->major, uc->minor, uc->subtype, le32toh(uc->valid)); if (le32toh(uc->valid) != 1) { device_printf(sc->sc_dev, "microcontroller initialization failed"); break; } if (uc->subtype == IWN_UCODE_INIT) { /* Save microcontroller report. */ memcpy(&sc->ucode_info, uc, sizeof (*uc)); } /* Save the address of the error log in SRAM. */ sc->errptr = le32toh(uc->errptr); break; } case IWN_STATE_CHANGED: { /* * State change allows hardware switch change to be * noted. However, we handle this in iwn_intr as we * get both the enable/disble intr. */ bus_dmamap_sync(sc->rxq.data_dmat, data->map, BUS_DMASYNC_POSTREAD); #ifdef IWN_DEBUG uint32_t *status = (uint32_t *)(desc + 1); DPRINTF(sc, IWN_DEBUG_INTR | IWN_DEBUG_STATE, "state changed to %x\n", le32toh(*status)); #endif break; } case IWN_START_SCAN: { bus_dmamap_sync(sc->rxq.data_dmat, data->map, BUS_DMASYNC_POSTREAD); #ifdef IWN_DEBUG struct iwn_start_scan *scan = (struct iwn_start_scan *)(desc + 1); DPRINTF(sc, IWN_DEBUG_ANY, "%s: scanning channel %d status %x\n", __func__, scan->chan, le32toh(scan->status)); #endif break; } case IWN_STOP_SCAN: { bus_dmamap_sync(sc->rxq.data_dmat, data->map, BUS_DMASYNC_POSTREAD); #ifdef IWN_DEBUG struct iwn_stop_scan *scan = (struct iwn_stop_scan *)(desc + 1); DPRINTF(sc, IWN_DEBUG_STATE | IWN_DEBUG_SCAN, "scan finished nchan=%d status=%d chan=%d\n", scan->nchan, scan->status, scan->chan); #endif sc->sc_is_scanning = 0; IWN_UNLOCK(sc); ieee80211_scan_next(vap); IWN_LOCK(sc); break; } case IWN5000_CALIBRATION_RESULT: iwn5000_rx_calib_results(sc, desc, data); break; case IWN5000_CALIBRATION_DONE: sc->sc_flags |= IWN_FLAG_CALIB_DONE; wakeup(sc); break; } sc->rxq.cur = (sc->rxq.cur + 1) % IWN_RX_RING_COUNT; } /* Tell the firmware what we have processed. */ hw = (hw == 0) ? IWN_RX_RING_COUNT - 1 : hw - 1; IWN_WRITE(sc, IWN_FH_RX_WPTR, hw & ~7); } /* * Process an INT_WAKEUP interrupt raised when the microcontroller wakes up * from power-down sleep mode. */ static void iwn_wakeup_intr(struct iwn_softc *sc) { int qid; DPRINTF(sc, IWN_DEBUG_RESET, "%s: ucode wakeup from power-down sleep\n", __func__); /* Wakeup RX and TX rings. */ IWN_WRITE(sc, IWN_FH_RX_WPTR, sc->rxq.cur & ~7); for (qid = 0; qid < sc->ntxqs; qid++) { struct iwn_tx_ring *ring = &sc->txq[qid]; IWN_WRITE(sc, IWN_HBUS_TARG_WRPTR, qid << 8 | ring->cur); } } static void iwn_rftoggle_intr(struct iwn_softc *sc) { struct ieee80211com *ic = &sc->sc_ic; uint32_t tmp = IWN_READ(sc, IWN_GP_CNTRL); IWN_LOCK_ASSERT(sc); device_printf(sc->sc_dev, "RF switch: radio %s\n", (tmp & IWN_GP_CNTRL_RFKILL) ? "enabled" : "disabled"); if (tmp & IWN_GP_CNTRL_RFKILL) ieee80211_runtask(ic, &sc->sc_radioon_task); else ieee80211_runtask(ic, &sc->sc_radiooff_task); } /* * Dump the error log of the firmware when a firmware panic occurs. Although * we can't debug the firmware because it is neither open source nor free, it * can help us to identify certain classes of problems. */ static void iwn_fatal_intr(struct iwn_softc *sc) { struct iwn_fw_dump dump; int i; IWN_LOCK_ASSERT(sc); /* Force a complete recalibration on next init. */ sc->sc_flags &= ~IWN_FLAG_CALIB_DONE; /* Check that the error log address is valid. */ if (sc->errptr < IWN_FW_DATA_BASE || sc->errptr + sizeof (dump) > IWN_FW_DATA_BASE + sc->fw_data_maxsz) { printf("%s: bad firmware error log address 0x%08x\n", __func__, sc->errptr); return; } if (iwn_nic_lock(sc) != 0) { printf("%s: could not read firmware error log\n", __func__); return; } /* Read firmware error log from SRAM. */ iwn_mem_read_region_4(sc, sc->errptr, (uint32_t *)&dump, sizeof (dump) / sizeof (uint32_t)); iwn_nic_unlock(sc); if (dump.valid == 0) { printf("%s: firmware error log is empty\n", __func__); return; } printf("firmware error log:\n"); printf(" error type = \"%s\" (0x%08X)\n", (dump.id < nitems(iwn_fw_errmsg)) ? iwn_fw_errmsg[dump.id] : "UNKNOWN", dump.id); printf(" program counter = 0x%08X\n", dump.pc); printf(" source line = 0x%08X\n", dump.src_line); printf(" error data = 0x%08X%08X\n", dump.error_data[0], dump.error_data[1]); printf(" branch link = 0x%08X%08X\n", dump.branch_link[0], dump.branch_link[1]); printf(" interrupt link = 0x%08X%08X\n", dump.interrupt_link[0], dump.interrupt_link[1]); printf(" time = %u\n", dump.time[0]); /* Dump driver status (TX and RX rings) while we're here. */ printf("driver status:\n"); for (i = 0; i < sc->ntxqs; i++) { struct iwn_tx_ring *ring = &sc->txq[i]; printf(" tx ring %2d: qid=%-2d cur=%-3d queued=%-3d\n", i, ring->qid, ring->cur, ring->queued); } printf(" rx ring: cur=%d\n", sc->rxq.cur); } static void iwn_intr(void *arg) { struct iwn_softc *sc = arg; uint32_t r1, r2, tmp; IWN_LOCK(sc); /* Disable interrupts. */ IWN_WRITE(sc, IWN_INT_MASK, 0); /* Read interrupts from ICT (fast) or from registers (slow). */ if (sc->sc_flags & IWN_FLAG_USE_ICT) { tmp = 0; while (sc->ict[sc->ict_cur] != 0) { tmp |= sc->ict[sc->ict_cur]; sc->ict[sc->ict_cur] = 0; /* Acknowledge. */ sc->ict_cur = (sc->ict_cur + 1) % IWN_ICT_COUNT; } tmp = le32toh(tmp); if (tmp == 0xffffffff) /* Shouldn't happen. */ tmp = 0; else if (tmp & 0xc0000) /* Workaround a HW bug. */ tmp |= 0x8000; r1 = (tmp & 0xff00) << 16 | (tmp & 0xff); r2 = 0; /* Unused. */ } else { r1 = IWN_READ(sc, IWN_INT); if (r1 == 0xffffffff || (r1 & 0xfffffff0) == 0xa5a5a5a0) { IWN_UNLOCK(sc); return; /* Hardware gone! */ } r2 = IWN_READ(sc, IWN_FH_INT); } DPRINTF(sc, IWN_DEBUG_INTR, "interrupt reg1=0x%08x reg2=0x%08x\n" , r1, r2); if (r1 == 0 && r2 == 0) goto done; /* Interrupt not for us. */ /* Acknowledge interrupts. */ IWN_WRITE(sc, IWN_INT, r1); if (!(sc->sc_flags & IWN_FLAG_USE_ICT)) IWN_WRITE(sc, IWN_FH_INT, r2); if (r1 & IWN_INT_RF_TOGGLED) { iwn_rftoggle_intr(sc); goto done; } if (r1 & IWN_INT_CT_REACHED) { device_printf(sc->sc_dev, "%s: critical temperature reached!\n", __func__); } if (r1 & (IWN_INT_SW_ERR | IWN_INT_HW_ERR)) { device_printf(sc->sc_dev, "%s: fatal firmware error\n", __func__); #ifdef IWN_DEBUG iwn_debug_register(sc); #endif /* Dump firmware error log and stop. */ iwn_fatal_intr(sc); taskqueue_enqueue(sc->sc_tq, &sc->sc_panic_task); goto done; } if ((r1 & (IWN_INT_FH_RX | IWN_INT_SW_RX | IWN_INT_RX_PERIODIC)) || (r2 & IWN_FH_INT_RX)) { if (sc->sc_flags & IWN_FLAG_USE_ICT) { if (r1 & (IWN_INT_FH_RX | IWN_INT_SW_RX)) IWN_WRITE(sc, IWN_FH_INT, IWN_FH_INT_RX); IWN_WRITE_1(sc, IWN_INT_PERIODIC, IWN_INT_PERIODIC_DIS); iwn_notif_intr(sc); if (r1 & (IWN_INT_FH_RX | IWN_INT_SW_RX)) { IWN_WRITE_1(sc, IWN_INT_PERIODIC, IWN_INT_PERIODIC_ENA); } } else iwn_notif_intr(sc); } if ((r1 & IWN_INT_FH_TX) || (r2 & IWN_FH_INT_TX)) { if (sc->sc_flags & IWN_FLAG_USE_ICT) IWN_WRITE(sc, IWN_FH_INT, IWN_FH_INT_TX); wakeup(sc); /* FH DMA transfer completed. */ } if (r1 & IWN_INT_ALIVE) wakeup(sc); /* Firmware is alive. */ if (r1 & IWN_INT_WAKEUP) iwn_wakeup_intr(sc); done: /* Re-enable interrupts. */ if (sc->sc_flags & IWN_FLAG_RUNNING) IWN_WRITE(sc, IWN_INT_MASK, sc->int_mask); IWN_UNLOCK(sc); } /* * Update TX scheduler ring when transmitting an 802.11 frame (4965AGN and * 5000 adapters use a slightly different format). */ static void iwn4965_update_sched(struct iwn_softc *sc, int qid, int idx, uint8_t id, uint16_t len) { uint16_t *w = &sc->sched[qid * IWN4965_SCHED_COUNT + idx]; DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); *w = htole16(len + 8); bus_dmamap_sync(sc->sched_dma.tag, sc->sched_dma.map, BUS_DMASYNC_PREWRITE); if (idx < IWN_SCHED_WINSZ) { *(w + IWN_TX_RING_COUNT) = *w; bus_dmamap_sync(sc->sched_dma.tag, sc->sched_dma.map, BUS_DMASYNC_PREWRITE); } } static void iwn5000_update_sched(struct iwn_softc *sc, int qid, int idx, uint8_t id, uint16_t len) { uint16_t *w = &sc->sched[qid * IWN5000_SCHED_COUNT + idx]; DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); *w = htole16(id << 12 | (len + 8)); bus_dmamap_sync(sc->sched_dma.tag, sc->sched_dma.map, BUS_DMASYNC_PREWRITE); if (idx < IWN_SCHED_WINSZ) { *(w + IWN_TX_RING_COUNT) = *w; bus_dmamap_sync(sc->sched_dma.tag, sc->sched_dma.map, BUS_DMASYNC_PREWRITE); } } #ifdef notyet static void iwn5000_reset_sched(struct iwn_softc *sc, int qid, int idx) { uint16_t *w = &sc->sched[qid * IWN5000_SCHED_COUNT + idx]; DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); *w = (*w & htole16(0xf000)) | htole16(1); bus_dmamap_sync(sc->sched_dma.tag, sc->sched_dma.map, BUS_DMASYNC_PREWRITE); if (idx < IWN_SCHED_WINSZ) { *(w + IWN_TX_RING_COUNT) = *w; bus_dmamap_sync(sc->sched_dma.tag, sc->sched_dma.map, BUS_DMASYNC_PREWRITE); } } #endif /* * Check whether OFDM 11g protection will be enabled for the given rate. * * The original driver code only enabled protection for OFDM rates. * It didn't check to see whether it was operating in 11a or 11bg mode. */ static int iwn_check_rate_needs_protection(struct iwn_softc *sc, struct ieee80211vap *vap, uint8_t rate) { struct ieee80211com *ic = vap->iv_ic; /* * Not in 2GHz mode? Then there's no need to enable OFDM * 11bg protection. */ if (! IEEE80211_IS_CHAN_2GHZ(ic->ic_curchan)) { return (0); } /* * 11bg protection not enabled? Then don't use it. */ if ((ic->ic_flags & IEEE80211_F_USEPROT) == 0) return (0); /* * If it's an 11n rate - no protection. * We'll do it via a specific 11n check. */ if (rate & IEEE80211_RATE_MCS) { return (0); } /* * Do a rate table lookup. If the PHY is CCK, * don't do protection. */ if (ieee80211_rate2phytype(ic->ic_rt, rate) == IEEE80211_T_CCK) return (0); /* * Yup, enable protection. */ return (1); } /* * return a value between 0 and IWN_MAX_TX_RETRIES-1 as an index into * the link quality table that reflects this particular entry. */ static int iwn_tx_rate_to_linkq_offset(struct iwn_softc *sc, struct ieee80211_node *ni, uint8_t rate) { struct ieee80211_rateset *rs; int is_11n; int nr; int i; uint8_t cmp_rate; /* * Figure out if we're using 11n or not here. */ if (IEEE80211_IS_CHAN_HT(ni->ni_chan) && ni->ni_htrates.rs_nrates > 0) is_11n = 1; else is_11n = 0; /* * Use the correct rate table. */ if (is_11n) { rs = (struct ieee80211_rateset *) &ni->ni_htrates; nr = ni->ni_htrates.rs_nrates; } else { rs = &ni->ni_rates; nr = rs->rs_nrates; } /* * Find the relevant link quality entry in the table. */ for (i = 0; i < nr && i < IWN_MAX_TX_RETRIES - 1 ; i++) { /* * The link quality table index starts at 0 == highest * rate, so we walk the rate table backwards. */ cmp_rate = rs->rs_rates[(nr - 1) - i]; if (rate & IEEE80211_RATE_MCS) cmp_rate |= IEEE80211_RATE_MCS; #if 0 DPRINTF(sc, IWN_DEBUG_XMIT, "%s: idx %d: nr=%d, rate=0x%02x, rateentry=0x%02x\n", __func__, i, nr, rate, cmp_rate); #endif if (cmp_rate == rate) return (i); } /* Failed? Start at the end */ return (IWN_MAX_TX_RETRIES - 1); } static int iwn_tx_data(struct iwn_softc *sc, struct mbuf *m, struct ieee80211_node *ni) { struct iwn_ops *ops = &sc->ops; const struct ieee80211_txparam *tp; struct ieee80211vap *vap = ni->ni_vap; struct ieee80211com *ic = ni->ni_ic; struct iwn_node *wn = (void *)ni; struct iwn_tx_ring *ring; struct iwn_tx_desc *desc; struct iwn_tx_data *data; struct iwn_tx_cmd *cmd; struct iwn_cmd_data *tx; struct ieee80211_frame *wh; struct ieee80211_key *k = NULL; struct mbuf *m1; uint32_t flags; uint16_t qos; u_int hdrlen; bus_dma_segment_t *seg, segs[IWN_MAX_SCATTER]; uint8_t tid, type; int ac, i, totlen, error, pad, nsegs = 0, rate; DPRINTF(sc, IWN_DEBUG_TRACE, "->%s begin\n", __func__); IWN_LOCK_ASSERT(sc); wh = mtod(m, struct ieee80211_frame *); hdrlen = ieee80211_anyhdrsize(wh); type = wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK; /* Select EDCA Access Category and TX ring for this frame. */ if (IEEE80211_QOS_HAS_SEQ(wh)) { qos = ((const struct ieee80211_qosframe *)wh)->i_qos[0]; tid = qos & IEEE80211_QOS_TID; } else { qos = 0; tid = 0; } ac = M_WME_GETAC(m); if (m->m_flags & M_AMPDU_MPDU) { uint16_t seqno; struct ieee80211_tx_ampdu *tap = &ni->ni_tx_ampdu[ac]; if (!IEEE80211_AMPDU_RUNNING(tap)) { return EINVAL; } /* * Queue this frame to the hardware ring that we've * negotiated AMPDU TX on. * * Note that the sequence number must match the TX slot * being used! */ ac = *(int *)tap->txa_private; seqno = ni->ni_txseqs[tid]; *(uint16_t *)wh->i_seq = htole16(seqno << IEEE80211_SEQ_SEQ_SHIFT); ring = &sc->txq[ac]; if ((seqno % 256) != ring->cur) { device_printf(sc->sc_dev, "%s: m=%p: seqno (%d) (%d) != ring index (%d) !\n", __func__, m, seqno, seqno % 256, ring->cur); } ni->ni_txseqs[tid]++; } ring = &sc->txq[ac]; desc = &ring->desc[ring->cur]; data = &ring->data[ring->cur]; /* Choose a TX rate index. */ tp = &vap->iv_txparms[ieee80211_chan2mode(ni->ni_chan)]; if (type == IEEE80211_FC0_TYPE_MGT) rate = tp->mgmtrate; else if (IEEE80211_IS_MULTICAST(wh->i_addr1)) rate = tp->mcastrate; else if (tp->ucastrate != IEEE80211_FIXED_RATE_NONE) rate = tp->ucastrate; else if (m->m_flags & M_EAPOL) rate = tp->mgmtrate; else { /* XXX pass pktlen */ (void) ieee80211_ratectl_rate(ni, NULL, 0); rate = ni->ni_txrate; } /* Encrypt the frame if need be. */ if (wh->i_fc[1] & IEEE80211_FC1_PROTECTED) { /* Retrieve key for TX. */ k = ieee80211_crypto_encap(ni, m); if (k == NULL) { return ENOBUFS; } /* 802.11 header may have moved. */ wh = mtod(m, struct ieee80211_frame *); } totlen = m->m_pkthdr.len; if (ieee80211_radiotap_active_vap(vap)) { struct iwn_tx_radiotap_header *tap = &sc->sc_txtap; tap->wt_flags = 0; tap->wt_rate = rate; if (k != NULL) tap->wt_flags |= IEEE80211_RADIOTAP_F_WEP; ieee80211_radiotap_tx(vap, m); } /* Prepare TX firmware command. */ cmd = &ring->cmd[ring->cur]; cmd->code = IWN_CMD_TX_DATA; cmd->flags = 0; cmd->qid = ring->qid; cmd->idx = ring->cur; tx = (struct iwn_cmd_data *)cmd->data; /* NB: No need to clear tx, all fields are reinitialized here. */ tx->scratch = 0; /* clear "scratch" area */ flags = 0; if (!IEEE80211_IS_MULTICAST(wh->i_addr1)) { /* Unicast frame, check if an ACK is expected. */ if (!qos || (qos & IEEE80211_QOS_ACKPOLICY) != IEEE80211_QOS_ACKPOLICY_NOACK) flags |= IWN_TX_NEED_ACK; } if ((wh->i_fc[0] & (IEEE80211_FC0_TYPE_MASK | IEEE80211_FC0_SUBTYPE_MASK)) == (IEEE80211_FC0_TYPE_CTL | IEEE80211_FC0_SUBTYPE_BAR)) flags |= IWN_TX_IMM_BA; /* Cannot happen yet. */ if (wh->i_fc[1] & IEEE80211_FC1_MORE_FRAG) flags |= IWN_TX_MORE_FRAG; /* Cannot happen yet. */ /* Check if frame must be protected using RTS/CTS or CTS-to-self. */ if (!IEEE80211_IS_MULTICAST(wh->i_addr1)) { /* NB: Group frames are sent using CCK in 802.11b/g. */ if (totlen + IEEE80211_CRC_LEN > vap->iv_rtsthreshold) { flags |= IWN_TX_NEED_RTS; } else if (iwn_check_rate_needs_protection(sc, vap, rate)) { if (ic->ic_protmode == IEEE80211_PROT_CTSONLY) flags |= IWN_TX_NEED_CTS; else if (ic->ic_protmode == IEEE80211_PROT_RTSCTS) flags |= IWN_TX_NEED_RTS; } else if ((rate & IEEE80211_RATE_MCS) && (ic->ic_htprotmode == IEEE80211_PROT_RTSCTS)) { flags |= IWN_TX_NEED_RTS; } /* XXX HT protection? */ if (flags & (IWN_TX_NEED_RTS | IWN_TX_NEED_CTS)) { if (sc->hw_type != IWN_HW_REV_TYPE_4965) { /* 5000 autoselects RTS/CTS or CTS-to-self. */ flags &= ~(IWN_TX_NEED_RTS | IWN_TX_NEED_CTS); flags |= IWN_TX_NEED_PROTECTION; } else flags |= IWN_TX_FULL_TXOP; } } if (IEEE80211_IS_MULTICAST(wh->i_addr1) || type != IEEE80211_FC0_TYPE_DATA) tx->id = sc->broadcast_id; else tx->id = wn->id; if (type == IEEE80211_FC0_TYPE_MGT) { uint8_t subtype = wh->i_fc[0] & IEEE80211_FC0_SUBTYPE_MASK; /* Tell HW to set timestamp in probe responses. */ if (subtype == IEEE80211_FC0_SUBTYPE_PROBE_RESP) flags |= IWN_TX_INSERT_TSTAMP; if (subtype == IEEE80211_FC0_SUBTYPE_ASSOC_REQ || subtype == IEEE80211_FC0_SUBTYPE_REASSOC_REQ) tx->timeout = htole16(3); else tx->timeout = htole16(2); } else tx->timeout = htole16(0); if (hdrlen & 3) { /* First segment length must be a multiple of 4. */ flags |= IWN_TX_NEED_PADDING; pad = 4 - (hdrlen & 3); } else pad = 0; tx->len = htole16(totlen); tx->tid = tid; tx->rts_ntries = 60; tx->data_ntries = 15; tx->lifetime = htole32(IWN_LIFETIME_INFINITE); tx->rate = iwn_rate_to_plcp(sc, ni, rate); if (tx->id == sc->broadcast_id) { /* Group or management frame. */ tx->linkq = 0; } else { tx->linkq = iwn_tx_rate_to_linkq_offset(sc, ni, rate); flags |= IWN_TX_LINKQ; /* enable MRR */ } /* Set physical address of "scratch area". */ tx->loaddr = htole32(IWN_LOADDR(data->scratch_paddr)); tx->hiaddr = IWN_HIADDR(data->scratch_paddr); /* Copy 802.11 header in TX command. */ memcpy((uint8_t *)(tx + 1), wh, hdrlen); /* Trim 802.11 header. */ m_adj(m, hdrlen); tx->security = 0; tx->flags = htole32(flags); error = bus_dmamap_load_mbuf_sg(ring->data_dmat, data->map, m, segs, &nsegs, BUS_DMA_NOWAIT); if (error != 0) { if (error != EFBIG) { device_printf(sc->sc_dev, "%s: can't map mbuf (error %d)\n", __func__, error); return error; } /* Too many DMA segments, linearize mbuf. */ m1 = m_collapse(m, M_NOWAIT, IWN_MAX_SCATTER - 1); if (m1 == NULL) { device_printf(sc->sc_dev, "%s: could not defrag mbuf\n", __func__); return ENOBUFS; } m = m1; error = bus_dmamap_load_mbuf_sg(ring->data_dmat, data->map, m, segs, &nsegs, BUS_DMA_NOWAIT); if (error != 0) { device_printf(sc->sc_dev, "%s: can't map mbuf (error %d)\n", __func__, error); return error; } } data->m = m; data->ni = ni; DPRINTF(sc, IWN_DEBUG_XMIT, "%s: qid %d idx %d len %d nsegs %d flags 0x%08x rate 0x%04x plcp 0x%08x\n", __func__, ring->qid, ring->cur, m->m_pkthdr.len, nsegs, flags, rate, tx->rate); /* Fill TX descriptor. */ desc->nsegs = 1; if (m->m_len != 0) desc->nsegs += nsegs; /* First DMA segment is used by the TX command. */ desc->segs[0].addr = htole32(IWN_LOADDR(data->cmd_paddr)); desc->segs[0].len = htole16(IWN_HIADDR(data->cmd_paddr) | (4 + sizeof (*tx) + hdrlen + pad) << 4); /* Other DMA segments are for data payload. */ seg = &segs[0]; for (i = 1; i <= nsegs; i++) { desc->segs[i].addr = htole32(IWN_LOADDR(seg->ds_addr)); desc->segs[i].len = htole16(IWN_HIADDR(seg->ds_addr) | seg->ds_len << 4); seg++; } bus_dmamap_sync(ring->data_dmat, data->map, BUS_DMASYNC_PREWRITE); bus_dmamap_sync(ring->data_dmat, ring->cmd_dma.map, BUS_DMASYNC_PREWRITE); bus_dmamap_sync(ring->desc_dma.tag, ring->desc_dma.map, BUS_DMASYNC_PREWRITE); /* Update TX scheduler. */ if (ring->qid >= sc->firstaggqueue) ops->update_sched(sc, ring->qid, ring->cur, tx->id, totlen); /* Kick TX ring. */ ring->cur = (ring->cur + 1) % IWN_TX_RING_COUNT; IWN_WRITE(sc, IWN_HBUS_TARG_WRPTR, ring->qid << 8 | ring->cur); /* Mark TX ring as full if we reach a certain threshold. */ if (++ring->queued > IWN_TX_RING_HIMARK) sc->qfullmsk |= 1 << ring->qid; DPRINTF(sc, IWN_DEBUG_TRACE, "->%s: end\n",__func__); return 0; } static int iwn_tx_data_raw(struct iwn_softc *sc, struct mbuf *m, struct ieee80211_node *ni, const struct ieee80211_bpf_params *params) { struct iwn_ops *ops = &sc->ops; struct ieee80211vap *vap = ni->ni_vap; struct iwn_tx_cmd *cmd; struct iwn_cmd_data *tx; struct ieee80211_frame *wh; struct iwn_tx_ring *ring; struct iwn_tx_desc *desc; struct iwn_tx_data *data; struct mbuf *m1; bus_dma_segment_t *seg, segs[IWN_MAX_SCATTER]; uint32_t flags; u_int hdrlen; int ac, totlen, error, pad, nsegs = 0, i, rate; uint8_t type; DPRINTF(sc, IWN_DEBUG_TRACE, "->%s begin\n", __func__); IWN_LOCK_ASSERT(sc); wh = mtod(m, struct ieee80211_frame *); hdrlen = ieee80211_anyhdrsize(wh); type = wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK; ac = params->ibp_pri & 3; ring = &sc->txq[ac]; desc = &ring->desc[ring->cur]; data = &ring->data[ring->cur]; /* Choose a TX rate. */ rate = params->ibp_rate0; totlen = m->m_pkthdr.len; /* Prepare TX firmware command. */ cmd = &ring->cmd[ring->cur]; cmd->code = IWN_CMD_TX_DATA; cmd->flags = 0; cmd->qid = ring->qid; cmd->idx = ring->cur; tx = (struct iwn_cmd_data *)cmd->data; /* NB: No need to clear tx, all fields are reinitialized here. */ tx->scratch = 0; /* clear "scratch" area */ flags = 0; if ((params->ibp_flags & IEEE80211_BPF_NOACK) == 0) flags |= IWN_TX_NEED_ACK; if (params->ibp_flags & IEEE80211_BPF_RTS) { if (sc->hw_type != IWN_HW_REV_TYPE_4965) { /* 5000 autoselects RTS/CTS or CTS-to-self. */ flags &= ~IWN_TX_NEED_RTS; flags |= IWN_TX_NEED_PROTECTION; } else flags |= IWN_TX_NEED_RTS | IWN_TX_FULL_TXOP; } if (params->ibp_flags & IEEE80211_BPF_CTS) { if (sc->hw_type != IWN_HW_REV_TYPE_4965) { /* 5000 autoselects RTS/CTS or CTS-to-self. */ flags &= ~IWN_TX_NEED_CTS; flags |= IWN_TX_NEED_PROTECTION; } else flags |= IWN_TX_NEED_CTS | IWN_TX_FULL_TXOP; } if (type == IEEE80211_FC0_TYPE_MGT) { uint8_t subtype = wh->i_fc[0] & IEEE80211_FC0_SUBTYPE_MASK; /* Tell HW to set timestamp in probe responses. */ if (subtype == IEEE80211_FC0_SUBTYPE_PROBE_RESP) flags |= IWN_TX_INSERT_TSTAMP; if (subtype == IEEE80211_FC0_SUBTYPE_ASSOC_REQ || subtype == IEEE80211_FC0_SUBTYPE_REASSOC_REQ) tx->timeout = htole16(3); else tx->timeout = htole16(2); } else tx->timeout = htole16(0); if (hdrlen & 3) { /* First segment length must be a multiple of 4. */ flags |= IWN_TX_NEED_PADDING; pad = 4 - (hdrlen & 3); } else pad = 0; if (ieee80211_radiotap_active_vap(vap)) { struct iwn_tx_radiotap_header *tap = &sc->sc_txtap; tap->wt_flags = 0; tap->wt_rate = rate; ieee80211_radiotap_tx(vap, m); } tx->len = htole16(totlen); tx->tid = 0; tx->id = sc->broadcast_id; tx->rts_ntries = params->ibp_try1; tx->data_ntries = params->ibp_try0; tx->lifetime = htole32(IWN_LIFETIME_INFINITE); tx->rate = iwn_rate_to_plcp(sc, ni, rate); /* Group or management frame. */ tx->linkq = 0; /* Set physical address of "scratch area". */ tx->loaddr = htole32(IWN_LOADDR(data->scratch_paddr)); tx->hiaddr = IWN_HIADDR(data->scratch_paddr); /* Copy 802.11 header in TX command. */ memcpy((uint8_t *)(tx + 1), wh, hdrlen); /* Trim 802.11 header. */ m_adj(m, hdrlen); tx->security = 0; tx->flags = htole32(flags); error = bus_dmamap_load_mbuf_sg(ring->data_dmat, data->map, m, segs, &nsegs, BUS_DMA_NOWAIT); if (error != 0) { if (error != EFBIG) { device_printf(sc->sc_dev, "%s: can't map mbuf (error %d)\n", __func__, error); return error; } /* Too many DMA segments, linearize mbuf. */ m1 = m_collapse(m, M_NOWAIT, IWN_MAX_SCATTER - 1); if (m1 == NULL) { device_printf(sc->sc_dev, "%s: could not defrag mbuf\n", __func__); return ENOBUFS; } m = m1; error = bus_dmamap_load_mbuf_sg(ring->data_dmat, data->map, m, segs, &nsegs, BUS_DMA_NOWAIT); if (error != 0) { device_printf(sc->sc_dev, "%s: can't map mbuf (error %d)\n", __func__, error); return error; } } data->m = m; data->ni = ni; DPRINTF(sc, IWN_DEBUG_XMIT, "%s: qid %d idx %d len %d nsegs %d\n", __func__, ring->qid, ring->cur, m->m_pkthdr.len, nsegs); /* Fill TX descriptor. */ desc->nsegs = 1; if (m->m_len != 0) desc->nsegs += nsegs; /* First DMA segment is used by the TX command. */ desc->segs[0].addr = htole32(IWN_LOADDR(data->cmd_paddr)); desc->segs[0].len = htole16(IWN_HIADDR(data->cmd_paddr) | (4 + sizeof (*tx) + hdrlen + pad) << 4); /* Other DMA segments are for data payload. */ seg = &segs[0]; for (i = 1; i <= nsegs; i++) { desc->segs[i].addr = htole32(IWN_LOADDR(seg->ds_addr)); desc->segs[i].len = htole16(IWN_HIADDR(seg->ds_addr) | seg->ds_len << 4); seg++; } bus_dmamap_sync(ring->data_dmat, data->map, BUS_DMASYNC_PREWRITE); bus_dmamap_sync(ring->data_dmat, ring->cmd_dma.map, BUS_DMASYNC_PREWRITE); bus_dmamap_sync(ring->desc_dma.tag, ring->desc_dma.map, BUS_DMASYNC_PREWRITE); /* Update TX scheduler. */ if (ring->qid >= sc->firstaggqueue) ops->update_sched(sc, ring->qid, ring->cur, tx->id, totlen); /* Kick TX ring. */ ring->cur = (ring->cur + 1) % IWN_TX_RING_COUNT; IWN_WRITE(sc, IWN_HBUS_TARG_WRPTR, ring->qid << 8 | ring->cur); /* Mark TX ring as full if we reach a certain threshold. */ if (++ring->queued > IWN_TX_RING_HIMARK) sc->qfullmsk |= 1 << ring->qid; DPRINTF(sc, IWN_DEBUG_TRACE, "->%s: end\n",__func__); return 0; } static void iwn_xmit_task(void *arg0, int pending) { struct iwn_softc *sc = arg0; struct ieee80211_node *ni; struct mbuf *m; int error; struct ieee80211_bpf_params p; int have_p; DPRINTF(sc, IWN_DEBUG_XMIT, "%s: called\n", __func__); IWN_LOCK(sc); /* * Dequeue frames, attempt to transmit, * then disable beaconwait when we're done. */ while ((m = mbufq_dequeue(&sc->sc_xmit_queue)) != NULL) { have_p = 0; ni = (struct ieee80211_node *)m->m_pkthdr.rcvif; /* Get xmit params if appropriate */ if (ieee80211_get_xmit_params(m, &p) == 0) have_p = 1; DPRINTF(sc, IWN_DEBUG_XMIT, "%s: m=%p, have_p=%d\n", __func__, m, have_p); /* If we have xmit params, use them */ if (have_p) error = iwn_tx_data_raw(sc, m, ni, &p); else error = iwn_tx_data(sc, m, ni); if (error != 0) { if_inc_counter(ni->ni_vap->iv_ifp, IFCOUNTER_OERRORS, 1); ieee80211_free_node(ni); m_freem(m); } } sc->sc_beacon_wait = 0; IWN_UNLOCK(sc); } /* * raw frame xmit - free node/reference if failed. */ static int iwn_raw_xmit(struct ieee80211_node *ni, struct mbuf *m, const struct ieee80211_bpf_params *params) { struct ieee80211com *ic = ni->ni_ic; struct iwn_softc *sc = ic->ic_softc; int error = 0; DPRINTF(sc, IWN_DEBUG_XMIT | IWN_DEBUG_TRACE, "->%s begin\n", __func__); IWN_LOCK(sc); if ((sc->sc_flags & IWN_FLAG_RUNNING) == 0) { m_freem(m); IWN_UNLOCK(sc); return (ENETDOWN); } /* queue frame if we have to */ if (sc->sc_beacon_wait) { if (iwn_xmit_queue_enqueue(sc, m) != 0) { m_freem(m); IWN_UNLOCK(sc); return (ENOBUFS); } /* Queued, so just return OK */ IWN_UNLOCK(sc); return (0); } if (params == NULL) { /* * Legacy path; interpret frame contents to decide * precisely how to send the frame. */ error = iwn_tx_data(sc, m, ni); } else { /* * Caller supplied explicit parameters to use in * sending the frame. */ error = iwn_tx_data_raw(sc, m, ni, params); } if (error == 0) sc->sc_tx_timer = 5; else m_freem(m); IWN_UNLOCK(sc); DPRINTF(sc, IWN_DEBUG_TRACE | IWN_DEBUG_XMIT, "->%s: end\n",__func__); return (error); } /* * transmit - don't free mbuf if failed; don't free node ref if failed. */ static int iwn_transmit(struct ieee80211com *ic, struct mbuf *m) { struct iwn_softc *sc = ic->ic_softc; struct ieee80211_node *ni; int error; ni = (struct ieee80211_node *)m->m_pkthdr.rcvif; IWN_LOCK(sc); if ((sc->sc_flags & IWN_FLAG_RUNNING) == 0 || sc->sc_beacon_wait) { IWN_UNLOCK(sc); return (ENXIO); } if (sc->qfullmsk) { IWN_UNLOCK(sc); return (ENOBUFS); } error = iwn_tx_data(sc, m, ni); if (!error) sc->sc_tx_timer = 5; IWN_UNLOCK(sc); return (error); } static void iwn_watchdog(void *arg) { struct iwn_softc *sc = arg; struct ieee80211com *ic = &sc->sc_ic; IWN_LOCK_ASSERT(sc); KASSERT(sc->sc_flags & IWN_FLAG_RUNNING, ("not running")); DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); if (sc->sc_tx_timer > 0) { if (--sc->sc_tx_timer == 0) { ic_printf(ic, "device timeout\n"); - ieee80211_runtask(ic, &sc->sc_reinit_task); + ieee80211_restart_all(ic); return; } } callout_reset(&sc->watchdog_to, hz, iwn_watchdog, sc); } static int iwn_cdev_open(struct cdev *dev, int flags, int type, struct thread *td) { return (0); } static int iwn_cdev_close(struct cdev *dev, int flags, int type, struct thread *td) { return (0); } static int iwn_cdev_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag, struct thread *td) { int rc; struct iwn_softc *sc = dev->si_drv1; struct iwn_ioctl_data *d; rc = priv_check(td, PRIV_DRIVER); if (rc != 0) return (0); switch (cmd) { case SIOCGIWNSTATS: d = (struct iwn_ioctl_data *) data; IWN_LOCK(sc); /* XXX validate permissions/memory/etc? */ rc = copyout(&sc->last_stat, d->dst_addr, sizeof(struct iwn_stats)); IWN_UNLOCK(sc); break; case SIOCZIWNSTATS: IWN_LOCK(sc); memset(&sc->last_stat, 0, sizeof(struct iwn_stats)); IWN_UNLOCK(sc); break; default: rc = EINVAL; break; } return (rc); } static int iwn_ioctl(struct ieee80211com *ic, u_long cmd, void *data) { return (ENOTTY); } static void iwn_parent(struct ieee80211com *ic) { struct iwn_softc *sc = ic->ic_softc; struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps); int startall = 0, stop = 0; IWN_LOCK(sc); if (ic->ic_nrunning > 0) { if (!(sc->sc_flags & IWN_FLAG_RUNNING)) { iwn_init_locked(sc); if (IWN_READ(sc, IWN_GP_CNTRL) & IWN_GP_CNTRL_RFKILL) startall = 1; else stop = 1; } } else if (sc->sc_flags & IWN_FLAG_RUNNING) iwn_stop_locked(sc); IWN_UNLOCK(sc); if (startall) ieee80211_start_all(ic); else if (vap != NULL && stop) ieee80211_stop(vap); } /* * Send a command to the firmware. */ static int iwn_cmd(struct iwn_softc *sc, int code, const void *buf, int size, int async) { struct iwn_tx_ring *ring; struct iwn_tx_desc *desc; struct iwn_tx_data *data; struct iwn_tx_cmd *cmd; struct mbuf *m; bus_addr_t paddr; int totlen, error; int cmd_queue_num; DPRINTF(sc, IWN_DEBUG_TRACE, "->%s begin\n", __func__); if (async == 0) IWN_LOCK_ASSERT(sc); if (sc->sc_flags & IWN_FLAG_PAN_SUPPORT) cmd_queue_num = IWN_PAN_CMD_QUEUE; else cmd_queue_num = IWN_CMD_QUEUE_NUM; ring = &sc->txq[cmd_queue_num]; desc = &ring->desc[ring->cur]; data = &ring->data[ring->cur]; totlen = 4 + size; if (size > sizeof cmd->data) { /* Command is too large to fit in a descriptor. */ if (totlen > MCLBYTES) return EINVAL; m = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, MJUMPAGESIZE); if (m == NULL) return ENOMEM; cmd = mtod(m, struct iwn_tx_cmd *); error = bus_dmamap_load(ring->data_dmat, data->map, cmd, totlen, iwn_dma_map_addr, &paddr, BUS_DMA_NOWAIT); if (error != 0) { m_freem(m); return error; } data->m = m; } else { cmd = &ring->cmd[ring->cur]; paddr = data->cmd_paddr; } cmd->code = code; cmd->flags = 0; cmd->qid = ring->qid; cmd->idx = ring->cur; memcpy(cmd->data, buf, size); desc->nsegs = 1; desc->segs[0].addr = htole32(IWN_LOADDR(paddr)); desc->segs[0].len = htole16(IWN_HIADDR(paddr) | totlen << 4); DPRINTF(sc, IWN_DEBUG_CMD, "%s: %s (0x%x) flags %d qid %d idx %d\n", __func__, iwn_intr_str(cmd->code), cmd->code, cmd->flags, cmd->qid, cmd->idx); if (size > sizeof cmd->data) { bus_dmamap_sync(ring->data_dmat, data->map, BUS_DMASYNC_PREWRITE); } else { bus_dmamap_sync(ring->data_dmat, ring->cmd_dma.map, BUS_DMASYNC_PREWRITE); } bus_dmamap_sync(ring->desc_dma.tag, ring->desc_dma.map, BUS_DMASYNC_PREWRITE); /* Kick command ring. */ ring->cur = (ring->cur + 1) % IWN_TX_RING_COUNT; IWN_WRITE(sc, IWN_HBUS_TARG_WRPTR, ring->qid << 8 | ring->cur); DPRINTF(sc, IWN_DEBUG_TRACE, "->%s: end\n",__func__); return async ? 0 : msleep(desc, &sc->sc_mtx, PCATCH, "iwncmd", hz); } static int iwn4965_add_node(struct iwn_softc *sc, struct iwn_node_info *node, int async) { struct iwn4965_node_info hnode; caddr_t src, dst; DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); /* * We use the node structure for 5000 Series internally (it is * a superset of the one for 4965AGN). We thus copy the common * fields before sending the command. */ src = (caddr_t)node; dst = (caddr_t)&hnode; memcpy(dst, src, 48); /* Skip TSC, RX MIC and TX MIC fields from ``src''. */ memcpy(dst + 48, src + 72, 20); return iwn_cmd(sc, IWN_CMD_ADD_NODE, &hnode, sizeof hnode, async); } static int iwn5000_add_node(struct iwn_softc *sc, struct iwn_node_info *node, int async) { DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); /* Direct mapping. */ return iwn_cmd(sc, IWN_CMD_ADD_NODE, node, sizeof (*node), async); } static int iwn_set_link_quality(struct iwn_softc *sc, struct ieee80211_node *ni) { struct iwn_node *wn = (void *)ni; struct ieee80211_rateset *rs; struct iwn_cmd_link_quality linkq; int i, rate, txrate; int is_11n; DPRINTF(sc, IWN_DEBUG_TRACE, "->%s begin\n", __func__); memset(&linkq, 0, sizeof linkq); linkq.id = wn->id; linkq.antmsk_1stream = iwn_get_1stream_tx_antmask(sc); linkq.antmsk_2stream = iwn_get_2stream_tx_antmask(sc); linkq.ampdu_max = 32; /* XXX negotiated? */ linkq.ampdu_threshold = 3; linkq.ampdu_limit = htole16(4000); /* 4ms */ DPRINTF(sc, IWN_DEBUG_XMIT, "%s: 1stream antenna=0x%02x, 2stream antenna=0x%02x, ntxstreams=%d\n", __func__, linkq.antmsk_1stream, linkq.antmsk_2stream, sc->ntxchains); /* * Are we using 11n rates? Ensure the channel is * 11n _and_ we have some 11n rates, or don't * try. */ if (IEEE80211_IS_CHAN_HT(ni->ni_chan) && ni->ni_htrates.rs_nrates > 0) { rs = (struct ieee80211_rateset *) &ni->ni_htrates; is_11n = 1; } else { rs = &ni->ni_rates; is_11n = 0; } /* Start at highest available bit-rate. */ /* * XXX this is all very dirty! */ if (is_11n) txrate = ni->ni_htrates.rs_nrates - 1; else txrate = rs->rs_nrates - 1; for (i = 0; i < IWN_MAX_TX_RETRIES; i++) { uint32_t plcp; /* * XXX TODO: ensure the last two slots are the two lowest * rate entries, just for now. */ if (i == 14 || i == 15) txrate = 0; if (is_11n) rate = IEEE80211_RATE_MCS | rs->rs_rates[txrate]; else rate = IEEE80211_RV(rs->rs_rates[txrate]); /* Do rate -> PLCP config mapping */ plcp = iwn_rate_to_plcp(sc, ni, rate); linkq.retry[i] = plcp; DPRINTF(sc, IWN_DEBUG_XMIT, "%s: i=%d, txrate=%d, rate=0x%02x, plcp=0x%08x\n", __func__, i, txrate, rate, le32toh(plcp)); /* * The mimo field is an index into the table which * indicates the first index where it and subsequent entries * will not be using MIMO. * * Since we're filling linkq from 0..15 and we're filling * from the highest MCS rates to the lowest rates, if we * _are_ doing a dual-stream rate, set mimo to idx+1 (ie, * the next entry.) That way if the next entry is a non-MIMO * entry, we're already pointing at it. */ if ((le32toh(plcp) & IWN_RFLAG_MCS) && IEEE80211_RV(le32toh(plcp)) > 7) linkq.mimo = i + 1; /* Next retry at immediate lower bit-rate. */ if (txrate > 0) txrate--; } /* * If we reached the end of the list and indeed we hit * all MIMO rates (eg 5300 doing MCS23-15) then yes, * set mimo to 15. Setting it to 16 panics the firmware. */ if (linkq.mimo > 15) linkq.mimo = 15; DPRINTF(sc, IWN_DEBUG_XMIT, "%s: mimo = %d\n", __func__, linkq.mimo); DPRINTF(sc, IWN_DEBUG_TRACE, "->%s: end\n",__func__); return iwn_cmd(sc, IWN_CMD_LINK_QUALITY, &linkq, sizeof linkq, 1); } /* * Broadcast node is used to send group-addressed and management frames. */ static int iwn_add_broadcast_node(struct iwn_softc *sc, int async) { struct iwn_ops *ops = &sc->ops; struct ieee80211com *ic = &sc->sc_ic; struct iwn_node_info node; struct iwn_cmd_link_quality linkq; uint8_t txant; int i, error; DPRINTF(sc, IWN_DEBUG_TRACE, "->%s begin\n", __func__); sc->rxon = &sc->rx_on[IWN_RXON_BSS_CTX]; memset(&node, 0, sizeof node); IEEE80211_ADDR_COPY(node.macaddr, ieee80211broadcastaddr); node.id = sc->broadcast_id; DPRINTF(sc, IWN_DEBUG_RESET, "%s: adding broadcast node\n", __func__); if ((error = ops->add_node(sc, &node, async)) != 0) return error; /* Use the first valid TX antenna. */ txant = IWN_LSB(sc->txchainmask); memset(&linkq, 0, sizeof linkq); linkq.id = sc->broadcast_id; linkq.antmsk_1stream = iwn_get_1stream_tx_antmask(sc); linkq.antmsk_2stream = iwn_get_2stream_tx_antmask(sc); linkq.ampdu_max = 64; linkq.ampdu_threshold = 3; linkq.ampdu_limit = htole16(4000); /* 4ms */ /* Use lowest mandatory bit-rate. */ /* XXX rate table lookup? */ if (IEEE80211_IS_CHAN_5GHZ(ic->ic_curchan)) linkq.retry[0] = htole32(0xd); else linkq.retry[0] = htole32(10 | IWN_RFLAG_CCK); linkq.retry[0] |= htole32(IWN_RFLAG_ANT(txant)); /* Use same bit-rate for all TX retries. */ for (i = 1; i < IWN_MAX_TX_RETRIES; i++) { linkq.retry[i] = linkq.retry[0]; } DPRINTF(sc, IWN_DEBUG_TRACE, "->%s: end\n",__func__); return iwn_cmd(sc, IWN_CMD_LINK_QUALITY, &linkq, sizeof linkq, async); } static int iwn_updateedca(struct ieee80211com *ic) { #define IWN_EXP2(x) ((1 << (x)) - 1) /* CWmin = 2^ECWmin - 1 */ struct iwn_softc *sc = ic->ic_softc; struct iwn_edca_params cmd; int aci; DPRINTF(sc, IWN_DEBUG_TRACE, "->%s begin\n", __func__); memset(&cmd, 0, sizeof cmd); cmd.flags = htole32(IWN_EDCA_UPDATE); IEEE80211_LOCK(ic); for (aci = 0; aci < WME_NUM_AC; aci++) { const struct wmeParams *ac = &ic->ic_wme.wme_chanParams.cap_wmeParams[aci]; cmd.ac[aci].aifsn = ac->wmep_aifsn; cmd.ac[aci].cwmin = htole16(IWN_EXP2(ac->wmep_logcwmin)); cmd.ac[aci].cwmax = htole16(IWN_EXP2(ac->wmep_logcwmax)); cmd.ac[aci].txoplimit = htole16(IEEE80211_TXOP_TO_US(ac->wmep_txopLimit)); } IEEE80211_UNLOCK(ic); IWN_LOCK(sc); (void)iwn_cmd(sc, IWN_CMD_EDCA_PARAMS, &cmd, sizeof cmd, 1); IWN_UNLOCK(sc); DPRINTF(sc, IWN_DEBUG_TRACE, "->%s: end\n",__func__); return 0; #undef IWN_EXP2 } static void iwn_update_mcast(struct ieee80211com *ic) { /* Ignore */ } static void iwn_set_led(struct iwn_softc *sc, uint8_t which, uint8_t off, uint8_t on) { struct iwn_cmd_led led; DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); #if 0 /* XXX don't set LEDs during scan? */ if (sc->sc_is_scanning) return; #endif /* Clear microcode LED ownership. */ IWN_CLRBITS(sc, IWN_LED, IWN_LED_BSM_CTRL); led.which = which; led.unit = htole32(10000); /* on/off in unit of 100ms */ led.off = off; led.on = on; (void)iwn_cmd(sc, IWN_CMD_SET_LED, &led, sizeof led, 1); } /* * Set the critical temperature at which the firmware will stop the radio * and notify us. */ static int iwn_set_critical_temp(struct iwn_softc *sc) { struct iwn_critical_temp crit; int32_t temp; DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); IWN_WRITE(sc, IWN_UCODE_GP1_CLR, IWN_UCODE_GP1_CTEMP_STOP_RF); if (sc->hw_type == IWN_HW_REV_TYPE_5150) temp = (IWN_CTOK(110) - sc->temp_off) * -5; else if (sc->hw_type == IWN_HW_REV_TYPE_4965) temp = IWN_CTOK(110); else temp = 110; memset(&crit, 0, sizeof crit); crit.tempR = htole32(temp); DPRINTF(sc, IWN_DEBUG_RESET, "setting critical temp to %d\n", temp); return iwn_cmd(sc, IWN_CMD_SET_CRITICAL_TEMP, &crit, sizeof crit, 0); } static int iwn_set_timing(struct iwn_softc *sc, struct ieee80211_node *ni) { struct iwn_cmd_timing cmd; uint64_t val, mod; DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); memset(&cmd, 0, sizeof cmd); memcpy(&cmd.tstamp, ni->ni_tstamp.data, sizeof (uint64_t)); cmd.bintval = htole16(ni->ni_intval); cmd.lintval = htole16(10); /* Compute remaining time until next beacon. */ val = (uint64_t)ni->ni_intval * IEEE80211_DUR_TU; mod = le64toh(cmd.tstamp) % val; cmd.binitval = htole32((uint32_t)(val - mod)); DPRINTF(sc, IWN_DEBUG_RESET, "timing bintval=%u tstamp=%ju, init=%u\n", ni->ni_intval, le64toh(cmd.tstamp), (uint32_t)(val - mod)); return iwn_cmd(sc, IWN_CMD_TIMING, &cmd, sizeof cmd, 1); } static void iwn4965_power_calibration(struct iwn_softc *sc, int temp) { struct ieee80211com *ic = &sc->sc_ic; DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); /* Adjust TX power if need be (delta >= 3 degC). */ DPRINTF(sc, IWN_DEBUG_CALIBRATE, "%s: temperature %d->%d\n", __func__, sc->temp, temp); if (abs(temp - sc->temp) >= 3) { /* Record temperature of last calibration. */ sc->temp = temp; (void)iwn4965_set_txpower(sc, ic->ic_bsschan, 1); } } /* * Set TX power for current channel (each rate has its own power settings). * This function takes into account the regulatory information from EEPROM, * the current temperature and the current voltage. */ static int iwn4965_set_txpower(struct iwn_softc *sc, struct ieee80211_channel *ch, int async) { /* Fixed-point arithmetic division using a n-bit fractional part. */ #define fdivround(a, b, n) \ ((((1 << n) * (a)) / (b) + (1 << n) / 2) / (1 << n)) /* Linear interpolation. */ #define interpolate(x, x1, y1, x2, y2, n) \ ((y1) + fdivround(((int)(x) - (x1)) * ((y2) - (y1)), (x2) - (x1), n)) static const int tdiv[IWN_NATTEN_GROUPS] = { 9, 8, 8, 8, 6 }; struct iwn_ucode_info *uc = &sc->ucode_info; struct iwn4965_cmd_txpower cmd; struct iwn4965_eeprom_chan_samples *chans; const uint8_t *rf_gain, *dsp_gain; int32_t vdiff, tdiff; int i, c, grp, maxpwr; uint8_t chan; sc->rxon = &sc->rx_on[IWN_RXON_BSS_CTX]; /* Retrieve current channel from last RXON. */ chan = sc->rxon->chan; DPRINTF(sc, IWN_DEBUG_RESET, "setting TX power for channel %d\n", chan); memset(&cmd, 0, sizeof cmd); cmd.band = IEEE80211_IS_CHAN_5GHZ(ch) ? 0 : 1; cmd.chan = chan; if (IEEE80211_IS_CHAN_5GHZ(ch)) { maxpwr = sc->maxpwr5GHz; rf_gain = iwn4965_rf_gain_5ghz; dsp_gain = iwn4965_dsp_gain_5ghz; } else { maxpwr = sc->maxpwr2GHz; rf_gain = iwn4965_rf_gain_2ghz; dsp_gain = iwn4965_dsp_gain_2ghz; } /* Compute voltage compensation. */ vdiff = ((int32_t)le32toh(uc->volt) - sc->eeprom_voltage) / 7; if (vdiff > 0) vdiff *= 2; if (abs(vdiff) > 2) vdiff = 0; DPRINTF(sc, IWN_DEBUG_CALIBRATE | IWN_DEBUG_TXPOW, "%s: voltage compensation=%d (UCODE=%d, EEPROM=%d)\n", __func__, vdiff, le32toh(uc->volt), sc->eeprom_voltage); /* Get channel attenuation group. */ if (chan <= 20) /* 1-20 */ grp = 4; else if (chan <= 43) /* 34-43 */ grp = 0; else if (chan <= 70) /* 44-70 */ grp = 1; else if (chan <= 124) /* 71-124 */ grp = 2; else /* 125-200 */ grp = 3; DPRINTF(sc, IWN_DEBUG_CALIBRATE | IWN_DEBUG_TXPOW, "%s: chan %d, attenuation group=%d\n", __func__, chan, grp); /* Get channel sub-band. */ for (i = 0; i < IWN_NBANDS; i++) if (sc->bands[i].lo != 0 && sc->bands[i].lo <= chan && chan <= sc->bands[i].hi) break; if (i == IWN_NBANDS) /* Can't happen in real-life. */ return EINVAL; chans = sc->bands[i].chans; DPRINTF(sc, IWN_DEBUG_CALIBRATE | IWN_DEBUG_TXPOW, "%s: chan %d sub-band=%d\n", __func__, chan, i); for (c = 0; c < 2; c++) { uint8_t power, gain, temp; int maxchpwr, pwr, ridx, idx; power = interpolate(chan, chans[0].num, chans[0].samples[c][1].power, chans[1].num, chans[1].samples[c][1].power, 1); gain = interpolate(chan, chans[0].num, chans[0].samples[c][1].gain, chans[1].num, chans[1].samples[c][1].gain, 1); temp = interpolate(chan, chans[0].num, chans[0].samples[c][1].temp, chans[1].num, chans[1].samples[c][1].temp, 1); DPRINTF(sc, IWN_DEBUG_CALIBRATE | IWN_DEBUG_TXPOW, "%s: Tx chain %d: power=%d gain=%d temp=%d\n", __func__, c, power, gain, temp); /* Compute temperature compensation. */ tdiff = ((sc->temp - temp) * 2) / tdiv[grp]; DPRINTF(sc, IWN_DEBUG_CALIBRATE | IWN_DEBUG_TXPOW, "%s: temperature compensation=%d (current=%d, EEPROM=%d)\n", __func__, tdiff, sc->temp, temp); for (ridx = 0; ridx <= IWN_RIDX_MAX; ridx++) { /* Convert dBm to half-dBm. */ maxchpwr = sc->maxpwr[chan] * 2; if ((ridx / 8) & 1) maxchpwr -= 6; /* MIMO 2T: -3dB */ pwr = maxpwr; /* Adjust TX power based on rate. */ if ((ridx % 8) == 5) pwr -= 15; /* OFDM48: -7.5dB */ else if ((ridx % 8) == 6) pwr -= 17; /* OFDM54: -8.5dB */ else if ((ridx % 8) == 7) pwr -= 20; /* OFDM60: -10dB */ else pwr -= 10; /* Others: -5dB */ /* Do not exceed channel max TX power. */ if (pwr > maxchpwr) pwr = maxchpwr; idx = gain - (pwr - power) - tdiff - vdiff; if ((ridx / 8) & 1) /* MIMO */ idx += (int32_t)le32toh(uc->atten[grp][c]); if (cmd.band == 0) idx += 9; /* 5GHz */ if (ridx == IWN_RIDX_MAX) idx += 5; /* CCK */ /* Make sure idx stays in a valid range. */ if (idx < 0) idx = 0; else if (idx > IWN4965_MAX_PWR_INDEX) idx = IWN4965_MAX_PWR_INDEX; DPRINTF(sc, IWN_DEBUG_CALIBRATE | IWN_DEBUG_TXPOW, "%s: Tx chain %d, rate idx %d: power=%d\n", __func__, c, ridx, idx); cmd.power[ridx].rf_gain[c] = rf_gain[idx]; cmd.power[ridx].dsp_gain[c] = dsp_gain[idx]; } } DPRINTF(sc, IWN_DEBUG_CALIBRATE | IWN_DEBUG_TXPOW, "%s: set tx power for chan %d\n", __func__, chan); return iwn_cmd(sc, IWN_CMD_TXPOWER, &cmd, sizeof cmd, async); #undef interpolate #undef fdivround } static int iwn5000_set_txpower(struct iwn_softc *sc, struct ieee80211_channel *ch, int async) { struct iwn5000_cmd_txpower cmd; int cmdid; DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); /* * TX power calibration is handled automatically by the firmware * for 5000 Series. */ memset(&cmd, 0, sizeof cmd); cmd.global_limit = 2 * IWN5000_TXPOWER_MAX_DBM; /* 16 dBm */ cmd.flags = IWN5000_TXPOWER_NO_CLOSED; cmd.srv_limit = IWN5000_TXPOWER_AUTO; DPRINTF(sc, IWN_DEBUG_CALIBRATE | IWN_DEBUG_XMIT, "%s: setting TX power; rev=%d\n", __func__, IWN_UCODE_API(sc->ucode_rev)); if (IWN_UCODE_API(sc->ucode_rev) == 1) cmdid = IWN_CMD_TXPOWER_DBM_V1; else cmdid = IWN_CMD_TXPOWER_DBM; return iwn_cmd(sc, cmdid, &cmd, sizeof cmd, async); } /* * Retrieve the maximum RSSI (in dBm) among receivers. */ static int iwn4965_get_rssi(struct iwn_softc *sc, struct iwn_rx_stat *stat) { struct iwn4965_rx_phystat *phy = (void *)stat->phybuf; uint8_t mask, agc; int rssi; DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); mask = (le16toh(phy->antenna) >> 4) & IWN_ANT_ABC; agc = (le16toh(phy->agc) >> 7) & 0x7f; rssi = 0; if (mask & IWN_ANT_A) rssi = MAX(rssi, phy->rssi[0]); if (mask & IWN_ANT_B) rssi = MAX(rssi, phy->rssi[2]); if (mask & IWN_ANT_C) rssi = MAX(rssi, phy->rssi[4]); DPRINTF(sc, IWN_DEBUG_RECV, "%s: agc %d mask 0x%x rssi %d %d %d result %d\n", __func__, agc, mask, phy->rssi[0], phy->rssi[2], phy->rssi[4], rssi - agc - IWN_RSSI_TO_DBM); return rssi - agc - IWN_RSSI_TO_DBM; } static int iwn5000_get_rssi(struct iwn_softc *sc, struct iwn_rx_stat *stat) { struct iwn5000_rx_phystat *phy = (void *)stat->phybuf; uint8_t agc; int rssi; DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); agc = (le32toh(phy->agc) >> 9) & 0x7f; rssi = MAX(le16toh(phy->rssi[0]) & 0xff, le16toh(phy->rssi[1]) & 0xff); rssi = MAX(le16toh(phy->rssi[2]) & 0xff, rssi); DPRINTF(sc, IWN_DEBUG_RECV, "%s: agc %d rssi %d %d %d result %d\n", __func__, agc, phy->rssi[0], phy->rssi[1], phy->rssi[2], rssi - agc - IWN_RSSI_TO_DBM); return rssi - agc - IWN_RSSI_TO_DBM; } /* * Retrieve the average noise (in dBm) among receivers. */ static int iwn_get_noise(const struct iwn_rx_general_stats *stats) { int i, total, nbant, noise; total = nbant = 0; for (i = 0; i < 3; i++) { if ((noise = le32toh(stats->noise[i]) & 0xff) == 0) continue; total += noise; nbant++; } /* There should be at least one antenna but check anyway. */ return (nbant == 0) ? -127 : (total / nbant) - 107; } /* * Compute temperature (in degC) from last received statistics. */ static int iwn4965_get_temperature(struct iwn_softc *sc) { struct iwn_ucode_info *uc = &sc->ucode_info; int32_t r1, r2, r3, r4, temp; DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); r1 = le32toh(uc->temp[0].chan20MHz); r2 = le32toh(uc->temp[1].chan20MHz); r3 = le32toh(uc->temp[2].chan20MHz); r4 = le32toh(sc->rawtemp); if (r1 == r3) /* Prevents division by 0 (should not happen). */ return 0; /* Sign-extend 23-bit R4 value to 32-bit. */ r4 = ((r4 & 0xffffff) ^ 0x800000) - 0x800000; /* Compute temperature in Kelvin. */ temp = (259 * (r4 - r2)) / (r3 - r1); temp = (temp * 97) / 100 + 8; DPRINTF(sc, IWN_DEBUG_ANY, "temperature %dK/%dC\n", temp, IWN_KTOC(temp)); return IWN_KTOC(temp); } static int iwn5000_get_temperature(struct iwn_softc *sc) { int32_t temp; DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); /* * Temperature is not used by the driver for 5000 Series because * TX power calibration is handled by firmware. */ temp = le32toh(sc->rawtemp); if (sc->hw_type == IWN_HW_REV_TYPE_5150) { temp = (temp / -5) + sc->temp_off; temp = IWN_KTOC(temp); } return temp; } /* * Initialize sensitivity calibration state machine. */ static int iwn_init_sensitivity(struct iwn_softc *sc) { struct iwn_ops *ops = &sc->ops; struct iwn_calib_state *calib = &sc->calib; uint32_t flags; int error; DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); /* Reset calibration state machine. */ memset(calib, 0, sizeof (*calib)); calib->state = IWN_CALIB_STATE_INIT; calib->cck_state = IWN_CCK_STATE_HIFA; /* Set initial correlation values. */ calib->ofdm_x1 = sc->limits->min_ofdm_x1; calib->ofdm_mrc_x1 = sc->limits->min_ofdm_mrc_x1; calib->ofdm_x4 = sc->limits->min_ofdm_x4; calib->ofdm_mrc_x4 = sc->limits->min_ofdm_mrc_x4; calib->cck_x4 = 125; calib->cck_mrc_x4 = sc->limits->min_cck_mrc_x4; calib->energy_cck = sc->limits->energy_cck; /* Write initial sensitivity. */ if ((error = iwn_send_sensitivity(sc)) != 0) return error; /* Write initial gains. */ if ((error = ops->init_gains(sc)) != 0) return error; /* Request statistics at each beacon interval. */ flags = 0; DPRINTF(sc, IWN_DEBUG_CALIBRATE, "%s: sending request for statistics\n", __func__); return iwn_cmd(sc, IWN_CMD_GET_STATISTICS, &flags, sizeof flags, 1); } /* * Collect noise and RSSI statistics for the first 20 beacons received * after association and use them to determine connected antennas and * to set differential gains. */ static void iwn_collect_noise(struct iwn_softc *sc, const struct iwn_rx_general_stats *stats) { struct iwn_ops *ops = &sc->ops; struct iwn_calib_state *calib = &sc->calib; struct ieee80211com *ic = &sc->sc_ic; uint32_t val; int i; DPRINTF(sc, IWN_DEBUG_TRACE, "->%s begin\n", __func__); /* Accumulate RSSI and noise for all 3 antennas. */ for (i = 0; i < 3; i++) { calib->rssi[i] += le32toh(stats->rssi[i]) & 0xff; calib->noise[i] += le32toh(stats->noise[i]) & 0xff; } /* NB: We update differential gains only once after 20 beacons. */ if (++calib->nbeacons < 20) return; /* Determine highest average RSSI. */ val = MAX(calib->rssi[0], calib->rssi[1]); val = MAX(calib->rssi[2], val); /* Determine which antennas are connected. */ sc->chainmask = sc->rxchainmask; for (i = 0; i < 3; i++) if (val - calib->rssi[i] > 15 * 20) sc->chainmask &= ~(1 << i); DPRINTF(sc, IWN_DEBUG_CALIBRATE | IWN_DEBUG_XMIT, "%s: RX chains mask: theoretical=0x%x, actual=0x%x\n", __func__, sc->rxchainmask, sc->chainmask); /* If none of the TX antennas are connected, keep at least one. */ if ((sc->chainmask & sc->txchainmask) == 0) sc->chainmask |= IWN_LSB(sc->txchainmask); (void)ops->set_gains(sc); calib->state = IWN_CALIB_STATE_RUN; #ifdef notyet /* XXX Disable RX chains with no antennas connected. */ sc->rxon->rxchain = htole16(IWN_RXCHAIN_SEL(sc->chainmask)); if (sc->sc_is_scanning) device_printf(sc->sc_dev, "%s: is_scanning set, before RXON\n", __func__); (void)iwn_cmd(sc, IWN_CMD_RXON, sc->rxon, sc->rxonsz, 1); #endif /* Enable power-saving mode if requested by user. */ if (ic->ic_flags & IEEE80211_F_PMGTON) (void)iwn_set_pslevel(sc, 0, 3, 1); DPRINTF(sc, IWN_DEBUG_TRACE, "->%s: end\n",__func__); } static int iwn4965_init_gains(struct iwn_softc *sc) { struct iwn_phy_calib_gain cmd; DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); memset(&cmd, 0, sizeof cmd); cmd.code = IWN4965_PHY_CALIB_DIFF_GAIN; /* Differential gains initially set to 0 for all 3 antennas. */ DPRINTF(sc, IWN_DEBUG_CALIBRATE, "%s: setting initial differential gains\n", __func__); return iwn_cmd(sc, IWN_CMD_PHY_CALIB, &cmd, sizeof cmd, 1); } static int iwn5000_init_gains(struct iwn_softc *sc) { struct iwn_phy_calib cmd; DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); memset(&cmd, 0, sizeof cmd); cmd.code = sc->reset_noise_gain; cmd.ngroups = 1; cmd.isvalid = 1; DPRINTF(sc, IWN_DEBUG_CALIBRATE, "%s: setting initial differential gains\n", __func__); return iwn_cmd(sc, IWN_CMD_PHY_CALIB, &cmd, sizeof cmd, 1); } static int iwn4965_set_gains(struct iwn_softc *sc) { struct iwn_calib_state *calib = &sc->calib; struct iwn_phy_calib_gain cmd; int i, delta, noise; DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); /* Get minimal noise among connected antennas. */ noise = INT_MAX; /* NB: There's at least one antenna. */ for (i = 0; i < 3; i++) if (sc->chainmask & (1 << i)) noise = MIN(calib->noise[i], noise); memset(&cmd, 0, sizeof cmd); cmd.code = IWN4965_PHY_CALIB_DIFF_GAIN; /* Set differential gains for connected antennas. */ for (i = 0; i < 3; i++) { if (sc->chainmask & (1 << i)) { /* Compute attenuation (in unit of 1.5dB). */ delta = (noise - (int32_t)calib->noise[i]) / 30; /* NB: delta <= 0 */ /* Limit to [-4.5dB,0]. */ cmd.gain[i] = MIN(abs(delta), 3); if (delta < 0) cmd.gain[i] |= 1 << 2; /* sign bit */ } } DPRINTF(sc, IWN_DEBUG_CALIBRATE, "setting differential gains Ant A/B/C: %x/%x/%x (%x)\n", cmd.gain[0], cmd.gain[1], cmd.gain[2], sc->chainmask); return iwn_cmd(sc, IWN_CMD_PHY_CALIB, &cmd, sizeof cmd, 1); } static int iwn5000_set_gains(struct iwn_softc *sc) { struct iwn_calib_state *calib = &sc->calib; struct iwn_phy_calib_gain cmd; int i, ant, div, delta; DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); /* We collected 20 beacons and !=6050 need a 1.5 factor. */ div = (sc->hw_type == IWN_HW_REV_TYPE_6050) ? 20 : 30; memset(&cmd, 0, sizeof cmd); cmd.code = sc->noise_gain; cmd.ngroups = 1; cmd.isvalid = 1; /* Get first available RX antenna as referential. */ ant = IWN_LSB(sc->rxchainmask); /* Set differential gains for other antennas. */ for (i = ant + 1; i < 3; i++) { if (sc->chainmask & (1 << i)) { /* The delta is relative to antenna "ant". */ delta = ((int32_t)calib->noise[ant] - (int32_t)calib->noise[i]) / div; /* Limit to [-4.5dB,+4.5dB]. */ cmd.gain[i - 1] = MIN(abs(delta), 3); if (delta < 0) cmd.gain[i - 1] |= 1 << 2; /* sign bit */ } } DPRINTF(sc, IWN_DEBUG_CALIBRATE | IWN_DEBUG_XMIT, "setting differential gains Ant B/C: %x/%x (%x)\n", cmd.gain[0], cmd.gain[1], sc->chainmask); return iwn_cmd(sc, IWN_CMD_PHY_CALIB, &cmd, sizeof cmd, 1); } /* * Tune RF RX sensitivity based on the number of false alarms detected * during the last beacon period. */ static void iwn_tune_sensitivity(struct iwn_softc *sc, const struct iwn_rx_stats *stats) { #define inc(val, inc, max) \ if ((val) < (max)) { \ if ((val) < (max) - (inc)) \ (val) += (inc); \ else \ (val) = (max); \ needs_update = 1; \ } #define dec(val, dec, min) \ if ((val) > (min)) { \ if ((val) > (min) + (dec)) \ (val) -= (dec); \ else \ (val) = (min); \ needs_update = 1; \ } const struct iwn_sensitivity_limits *limits = sc->limits; struct iwn_calib_state *calib = &sc->calib; uint32_t val, rxena, fa; uint32_t energy[3], energy_min; uint8_t noise[3], noise_ref; int i, needs_update = 0; DPRINTF(sc, IWN_DEBUG_TRACE, "->%s begin\n", __func__); /* Check that we've been enabled long enough. */ if ((rxena = le32toh(stats->general.load)) == 0){ DPRINTF(sc, IWN_DEBUG_TRACE, "->%s end not so long\n", __func__); return; } /* Compute number of false alarms since last call for OFDM. */ fa = le32toh(stats->ofdm.bad_plcp) - calib->bad_plcp_ofdm; fa += le32toh(stats->ofdm.fa) - calib->fa_ofdm; fa *= 200 * IEEE80211_DUR_TU; /* 200TU */ if (fa > 50 * rxena) { /* High false alarm count, decrease sensitivity. */ DPRINTF(sc, IWN_DEBUG_CALIBRATE, "%s: OFDM high false alarm count: %u\n", __func__, fa); inc(calib->ofdm_x1, 1, limits->max_ofdm_x1); inc(calib->ofdm_mrc_x1, 1, limits->max_ofdm_mrc_x1); inc(calib->ofdm_x4, 1, limits->max_ofdm_x4); inc(calib->ofdm_mrc_x4, 1, limits->max_ofdm_mrc_x4); } else if (fa < 5 * rxena) { /* Low false alarm count, increase sensitivity. */ DPRINTF(sc, IWN_DEBUG_CALIBRATE, "%s: OFDM low false alarm count: %u\n", __func__, fa); dec(calib->ofdm_x1, 1, limits->min_ofdm_x1); dec(calib->ofdm_mrc_x1, 1, limits->min_ofdm_mrc_x1); dec(calib->ofdm_x4, 1, limits->min_ofdm_x4); dec(calib->ofdm_mrc_x4, 1, limits->min_ofdm_mrc_x4); } /* Compute maximum noise among 3 receivers. */ for (i = 0; i < 3; i++) noise[i] = (le32toh(stats->general.noise[i]) >> 8) & 0xff; val = MAX(noise[0], noise[1]); val = MAX(noise[2], val); /* Insert it into our samples table. */ calib->noise_samples[calib->cur_noise_sample] = val; calib->cur_noise_sample = (calib->cur_noise_sample + 1) % 20; /* Compute maximum noise among last 20 samples. */ noise_ref = calib->noise_samples[0]; for (i = 1; i < 20; i++) noise_ref = MAX(noise_ref, calib->noise_samples[i]); /* Compute maximum energy among 3 receivers. */ for (i = 0; i < 3; i++) energy[i] = le32toh(stats->general.energy[i]); val = MIN(energy[0], energy[1]); val = MIN(energy[2], val); /* Insert it into our samples table. */ calib->energy_samples[calib->cur_energy_sample] = val; calib->cur_energy_sample = (calib->cur_energy_sample + 1) % 10; /* Compute minimum energy among last 10 samples. */ energy_min = calib->energy_samples[0]; for (i = 1; i < 10; i++) energy_min = MAX(energy_min, calib->energy_samples[i]); energy_min += 6; /* Compute number of false alarms since last call for CCK. */ fa = le32toh(stats->cck.bad_plcp) - calib->bad_plcp_cck; fa += le32toh(stats->cck.fa) - calib->fa_cck; fa *= 200 * IEEE80211_DUR_TU; /* 200TU */ if (fa > 50 * rxena) { /* High false alarm count, decrease sensitivity. */ DPRINTF(sc, IWN_DEBUG_CALIBRATE, "%s: CCK high false alarm count: %u\n", __func__, fa); calib->cck_state = IWN_CCK_STATE_HIFA; calib->low_fa = 0; if (calib->cck_x4 > 160) { calib->noise_ref = noise_ref; if (calib->energy_cck > 2) dec(calib->energy_cck, 2, energy_min); } if (calib->cck_x4 < 160) { calib->cck_x4 = 161; needs_update = 1; } else inc(calib->cck_x4, 3, limits->max_cck_x4); inc(calib->cck_mrc_x4, 3, limits->max_cck_mrc_x4); } else if (fa < 5 * rxena) { /* Low false alarm count, increase sensitivity. */ DPRINTF(sc, IWN_DEBUG_CALIBRATE, "%s: CCK low false alarm count: %u\n", __func__, fa); calib->cck_state = IWN_CCK_STATE_LOFA; calib->low_fa++; if (calib->cck_state != IWN_CCK_STATE_INIT && (((int32_t)calib->noise_ref - (int32_t)noise_ref) > 2 || calib->low_fa > 100)) { inc(calib->energy_cck, 2, limits->min_energy_cck); dec(calib->cck_x4, 3, limits->min_cck_x4); dec(calib->cck_mrc_x4, 3, limits->min_cck_mrc_x4); } } else { /* Not worth to increase or decrease sensitivity. */ DPRINTF(sc, IWN_DEBUG_CALIBRATE, "%s: CCK normal false alarm count: %u\n", __func__, fa); calib->low_fa = 0; calib->noise_ref = noise_ref; if (calib->cck_state == IWN_CCK_STATE_HIFA) { /* Previous interval had many false alarms. */ dec(calib->energy_cck, 8, energy_min); } calib->cck_state = IWN_CCK_STATE_INIT; } if (needs_update) (void)iwn_send_sensitivity(sc); DPRINTF(sc, IWN_DEBUG_TRACE, "->%s: end\n",__func__); #undef dec #undef inc } static int iwn_send_sensitivity(struct iwn_softc *sc) { struct iwn_calib_state *calib = &sc->calib; struct iwn_enhanced_sensitivity_cmd cmd; int len; memset(&cmd, 0, sizeof cmd); len = sizeof (struct iwn_sensitivity_cmd); cmd.which = IWN_SENSITIVITY_WORKTBL; /* OFDM modulation. */ cmd.corr_ofdm_x1 = htole16(calib->ofdm_x1); cmd.corr_ofdm_mrc_x1 = htole16(calib->ofdm_mrc_x1); cmd.corr_ofdm_x4 = htole16(calib->ofdm_x4); cmd.corr_ofdm_mrc_x4 = htole16(calib->ofdm_mrc_x4); cmd.energy_ofdm = htole16(sc->limits->energy_ofdm); cmd.energy_ofdm_th = htole16(62); /* CCK modulation. */ cmd.corr_cck_x4 = htole16(calib->cck_x4); cmd.corr_cck_mrc_x4 = htole16(calib->cck_mrc_x4); cmd.energy_cck = htole16(calib->energy_cck); /* Barker modulation: use default values. */ cmd.corr_barker = htole16(190); cmd.corr_barker_mrc = htole16(sc->limits->barker_mrc); DPRINTF(sc, IWN_DEBUG_CALIBRATE, "%s: set sensitivity %d/%d/%d/%d/%d/%d/%d\n", __func__, calib->ofdm_x1, calib->ofdm_mrc_x1, calib->ofdm_x4, calib->ofdm_mrc_x4, calib->cck_x4, calib->cck_mrc_x4, calib->energy_cck); if (!(sc->sc_flags & IWN_FLAG_ENH_SENS)) goto send; /* Enhanced sensitivity settings. */ len = sizeof (struct iwn_enhanced_sensitivity_cmd); cmd.ofdm_det_slope_mrc = htole16(668); cmd.ofdm_det_icept_mrc = htole16(4); cmd.ofdm_det_slope = htole16(486); cmd.ofdm_det_icept = htole16(37); cmd.cck_det_slope_mrc = htole16(853); cmd.cck_det_icept_mrc = htole16(4); cmd.cck_det_slope = htole16(476); cmd.cck_det_icept = htole16(99); send: return iwn_cmd(sc, IWN_CMD_SET_SENSITIVITY, &cmd, len, 1); } /* * Look at the increase of PLCP errors over time; if it exceeds * a programmed threshold then trigger an RF retune. */ static void iwn_check_rx_recovery(struct iwn_softc *sc, struct iwn_stats *rs) { int32_t delta_ofdm, delta_ht, delta_cck; struct iwn_calib_state *calib = &sc->calib; int delta_ticks, cur_ticks; int delta_msec; int thresh; /* * Calculate the difference between the current and * previous statistics. */ delta_cck = le32toh(rs->rx.cck.bad_plcp) - calib->bad_plcp_cck; delta_ofdm = le32toh(rs->rx.ofdm.bad_plcp) - calib->bad_plcp_ofdm; delta_ht = le32toh(rs->rx.ht.bad_plcp) - calib->bad_plcp_ht; /* * Calculate the delta in time between successive statistics * messages. Yes, it can roll over; so we make sure that * this doesn't happen. * * XXX go figure out what to do about rollover * XXX go figure out what to do if ticks rolls over to -ve instead! * XXX go stab signed integer overflow undefined-ness in the face. */ cur_ticks = ticks; delta_ticks = cur_ticks - sc->last_calib_ticks; /* * If any are negative, then the firmware likely reset; so just * bail. We'll pick this up next time. */ if (delta_cck < 0 || delta_ofdm < 0 || delta_ht < 0 || delta_ticks < 0) return; /* * delta_ticks is in ticks; we need to convert it up to milliseconds * so we can do some useful math with it. */ delta_msec = ticks_to_msecs(delta_ticks); /* * Calculate what our threshold is given the current delta_msec. */ thresh = sc->base_params->plcp_err_threshold * delta_msec; DPRINTF(sc, IWN_DEBUG_STATE, "%s: time delta: %d; cck=%d, ofdm=%d, ht=%d, total=%d, thresh=%d\n", __func__, delta_msec, delta_cck, delta_ofdm, delta_ht, (delta_msec + delta_cck + delta_ofdm + delta_ht), thresh); /* * If we need a retune, then schedule a single channel scan * to a channel that isn't the currently active one! * * The math from linux iwlwifi: * * if ((delta * 100 / msecs) > threshold) */ if (thresh > 0 && (delta_cck + delta_ofdm + delta_ht) * 100 > thresh) { DPRINTF(sc, IWN_DEBUG_ANY, "%s: PLCP error threshold raw (%d) comparison (%d) " "over limit (%d); retune!\n", __func__, (delta_cck + delta_ofdm + delta_ht), (delta_cck + delta_ofdm + delta_ht) * 100, thresh); } } /* * Set STA mode power saving level (between 0 and 5). * Level 0 is CAM (Continuously Aware Mode), 5 is for maximum power saving. */ static int iwn_set_pslevel(struct iwn_softc *sc, int dtim, int level, int async) { struct iwn_pmgt_cmd cmd; const struct iwn_pmgt *pmgt; uint32_t max, skip_dtim; uint32_t reg; int i; DPRINTF(sc, IWN_DEBUG_PWRSAVE, "%s: dtim=%d, level=%d, async=%d\n", __func__, dtim, level, async); /* Select which PS parameters to use. */ if (dtim <= 2) pmgt = &iwn_pmgt[0][level]; else if (dtim <= 10) pmgt = &iwn_pmgt[1][level]; else pmgt = &iwn_pmgt[2][level]; memset(&cmd, 0, sizeof cmd); if (level != 0) /* not CAM */ cmd.flags |= htole16(IWN_PS_ALLOW_SLEEP); if (level == 5) cmd.flags |= htole16(IWN_PS_FAST_PD); /* Retrieve PCIe Active State Power Management (ASPM). */ reg = pci_read_config(sc->sc_dev, sc->sc_cap_off + PCIER_LINK_CTL, 4); if (!(reg & PCIEM_LINK_CTL_ASPMC_L0S)) /* L0s Entry disabled. */ cmd.flags |= htole16(IWN_PS_PCI_PMGT); cmd.rxtimeout = htole32(pmgt->rxtimeout * 1024); cmd.txtimeout = htole32(pmgt->txtimeout * 1024); if (dtim == 0) { dtim = 1; skip_dtim = 0; } else skip_dtim = pmgt->skip_dtim; if (skip_dtim != 0) { cmd.flags |= htole16(IWN_PS_SLEEP_OVER_DTIM); max = pmgt->intval[4]; if (max == (uint32_t)-1) max = dtim * (skip_dtim + 1); else if (max > dtim) max = rounddown(max, dtim); } else max = dtim; for (i = 0; i < 5; i++) cmd.intval[i] = htole32(MIN(max, pmgt->intval[i])); DPRINTF(sc, IWN_DEBUG_RESET, "setting power saving level to %d\n", level); return iwn_cmd(sc, IWN_CMD_SET_POWER_MODE, &cmd, sizeof cmd, async); } static int iwn_send_btcoex(struct iwn_softc *sc) { struct iwn_bluetooth cmd; memset(&cmd, 0, sizeof cmd); cmd.flags = IWN_BT_COEX_CHAN_ANN | IWN_BT_COEX_BT_PRIO; cmd.lead_time = IWN_BT_LEAD_TIME_DEF; cmd.max_kill = IWN_BT_MAX_KILL_DEF; DPRINTF(sc, IWN_DEBUG_RESET, "%s: configuring bluetooth coexistence\n", __func__); return iwn_cmd(sc, IWN_CMD_BT_COEX, &cmd, sizeof(cmd), 0); } static int iwn_send_advanced_btcoex(struct iwn_softc *sc) { static const uint32_t btcoex_3wire[12] = { 0xaaaaaaaa, 0xaaaaaaaa, 0xaeaaaaaa, 0xaaaaaaaa, 0xcc00ff28, 0x0000aaaa, 0xcc00aaaa, 0x0000aaaa, 0xc0004000, 0x00004000, 0xf0005000, 0xf0005000, }; struct iwn6000_btcoex_config btconfig; struct iwn2000_btcoex_config btconfig2k; struct iwn_btcoex_priotable btprio; struct iwn_btcoex_prot btprot; int error, i; uint8_t flags; memset(&btconfig, 0, sizeof btconfig); memset(&btconfig2k, 0, sizeof btconfig2k); flags = IWN_BT_FLAG_COEX6000_MODE_3W << IWN_BT_FLAG_COEX6000_MODE_SHIFT; // Done as is in linux kernel 3.2 if (sc->base_params->bt_sco_disable) flags &= ~IWN_BT_FLAG_SYNC_2_BT_DISABLE; else flags |= IWN_BT_FLAG_SYNC_2_BT_DISABLE; flags |= IWN_BT_FLAG_COEX6000_CHAN_INHIBITION; /* Default flags result is 145 as old value */ /* * Flags value has to be review. Values must change if we * which to disable it */ if (sc->base_params->bt_session_2) { btconfig2k.flags = flags; btconfig2k.max_kill = 5; btconfig2k.bt3_t7_timer = 1; btconfig2k.kill_ack = htole32(0xffff0000); btconfig2k.kill_cts = htole32(0xffff0000); btconfig2k.sample_time = 2; btconfig2k.bt3_t2_timer = 0xc; for (i = 0; i < 12; i++) btconfig2k.lookup_table[i] = htole32(btcoex_3wire[i]); btconfig2k.valid = htole16(0xff); btconfig2k.prio_boost = htole32(0xf0); DPRINTF(sc, IWN_DEBUG_RESET, "%s: configuring advanced bluetooth coexistence" " session 2, flags : 0x%x\n", __func__, flags); error = iwn_cmd(sc, IWN_CMD_BT_COEX, &btconfig2k, sizeof(btconfig2k), 1); } else { btconfig.flags = flags; btconfig.max_kill = 5; btconfig.bt3_t7_timer = 1; btconfig.kill_ack = htole32(0xffff0000); btconfig.kill_cts = htole32(0xffff0000); btconfig.sample_time = 2; btconfig.bt3_t2_timer = 0xc; for (i = 0; i < 12; i++) btconfig.lookup_table[i] = htole32(btcoex_3wire[i]); btconfig.valid = htole16(0xff); btconfig.prio_boost = 0xf0; DPRINTF(sc, IWN_DEBUG_RESET, "%s: configuring advanced bluetooth coexistence," " flags : 0x%x\n", __func__, flags); error = iwn_cmd(sc, IWN_CMD_BT_COEX, &btconfig, sizeof(btconfig), 1); } if (error != 0) return error; memset(&btprio, 0, sizeof btprio); btprio.calib_init1 = 0x6; btprio.calib_init2 = 0x7; btprio.calib_periodic_low1 = 0x2; btprio.calib_periodic_low2 = 0x3; btprio.calib_periodic_high1 = 0x4; btprio.calib_periodic_high2 = 0x5; btprio.dtim = 0x6; btprio.scan52 = 0x8; btprio.scan24 = 0xa; error = iwn_cmd(sc, IWN_CMD_BT_COEX_PRIOTABLE, &btprio, sizeof(btprio), 1); if (error != 0) return error; /* Force BT state machine change. */ memset(&btprot, 0, sizeof btprot); btprot.open = 1; btprot.type = 1; error = iwn_cmd(sc, IWN_CMD_BT_COEX_PROT, &btprot, sizeof(btprot), 1); if (error != 0) return error; btprot.open = 0; return iwn_cmd(sc, IWN_CMD_BT_COEX_PROT, &btprot, sizeof(btprot), 1); } static int iwn5000_runtime_calib(struct iwn_softc *sc) { struct iwn5000_calib_config cmd; memset(&cmd, 0, sizeof cmd); cmd.ucode.once.enable = 0xffffffff; cmd.ucode.once.start = IWN5000_CALIB_DC; DPRINTF(sc, IWN_DEBUG_CALIBRATE, "%s: configuring runtime calibration\n", __func__); return iwn_cmd(sc, IWN5000_CMD_CALIB_CONFIG, &cmd, sizeof(cmd), 0); } static uint32_t iwn_get_rxon_ht_flags(struct iwn_softc *sc, struct ieee80211_channel *c) { struct ieee80211com *ic = &sc->sc_ic; uint32_t htflags = 0; if (! IEEE80211_IS_CHAN_HT(c)) return (0); htflags |= IWN_RXON_HT_PROTMODE(ic->ic_curhtprotmode); if (IEEE80211_IS_CHAN_HT40(c)) { switch (ic->ic_curhtprotmode) { case IEEE80211_HTINFO_OPMODE_HT20PR: htflags |= IWN_RXON_HT_MODEPURE40; break; default: htflags |= IWN_RXON_HT_MODEMIXED; break; } } if (IEEE80211_IS_CHAN_HT40D(c)) htflags |= IWN_RXON_HT_HT40MINUS; return (htflags); } static int iwn_config(struct iwn_softc *sc) { struct iwn_ops *ops = &sc->ops; struct ieee80211com *ic = &sc->sc_ic; struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps); const uint8_t *macaddr; uint32_t txmask; uint16_t rxchain; int error; DPRINTF(sc, IWN_DEBUG_TRACE, "->%s begin\n", __func__); if ((sc->base_params->calib_need & IWN_FLG_NEED_PHY_CALIB_TEMP_OFFSET) && (sc->base_params->calib_need & IWN_FLG_NEED_PHY_CALIB_TEMP_OFFSETv2)) { device_printf(sc->sc_dev,"%s: temp_offset and temp_offsetv2 are" " exclusive each together. Review NIC config file. Conf" " : 0x%08x Flags : 0x%08x \n", __func__, sc->base_params->calib_need, (IWN_FLG_NEED_PHY_CALIB_TEMP_OFFSET | IWN_FLG_NEED_PHY_CALIB_TEMP_OFFSETv2)); return (EINVAL); } /* Compute temperature calib if needed. Will be send by send calib */ if (sc->base_params->calib_need & IWN_FLG_NEED_PHY_CALIB_TEMP_OFFSET) { error = iwn5000_temp_offset_calib(sc); if (error != 0) { device_printf(sc->sc_dev, "%s: could not set temperature offset\n", __func__); return (error); } } else if (sc->base_params->calib_need & IWN_FLG_NEED_PHY_CALIB_TEMP_OFFSETv2) { error = iwn5000_temp_offset_calibv2(sc); if (error != 0) { device_printf(sc->sc_dev, "%s: could not compute temperature offset v2\n", __func__); return (error); } } if (sc->hw_type == IWN_HW_REV_TYPE_6050) { /* Configure runtime DC calibration. */ error = iwn5000_runtime_calib(sc); if (error != 0) { device_printf(sc->sc_dev, "%s: could not configure runtime calibration\n", __func__); return error; } } /* Configure valid TX chains for >=5000 Series. */ if (sc->hw_type != IWN_HW_REV_TYPE_4965 && IWN_UCODE_API(sc->ucode_rev) > 1) { txmask = htole32(sc->txchainmask); DPRINTF(sc, IWN_DEBUG_RESET | IWN_DEBUG_XMIT, "%s: configuring valid TX chains 0x%x\n", __func__, txmask); error = iwn_cmd(sc, IWN5000_CMD_TX_ANT_CONFIG, &txmask, sizeof txmask, 0); if (error != 0) { device_printf(sc->sc_dev, "%s: could not configure valid TX chains, " "error %d\n", __func__, error); return error; } } /* Configure bluetooth coexistence. */ error = 0; /* Configure bluetooth coexistence if needed. */ if (sc->base_params->bt_mode == IWN_BT_ADVANCED) error = iwn_send_advanced_btcoex(sc); if (sc->base_params->bt_mode == IWN_BT_SIMPLE) error = iwn_send_btcoex(sc); if (error != 0) { device_printf(sc->sc_dev, "%s: could not configure bluetooth coexistence, error %d\n", __func__, error); return error; } /* Set mode, channel, RX filter and enable RX. */ sc->rxon = &sc->rx_on[IWN_RXON_BSS_CTX]; memset(sc->rxon, 0, sizeof (struct iwn_rxon)); macaddr = vap ? vap->iv_myaddr : ic->ic_macaddr; IEEE80211_ADDR_COPY(sc->rxon->myaddr, macaddr); IEEE80211_ADDR_COPY(sc->rxon->wlap, macaddr); sc->rxon->chan = ieee80211_chan2ieee(ic, ic->ic_curchan); sc->rxon->flags = htole32(IWN_RXON_TSF | IWN_RXON_CTS_TO_SELF); if (IEEE80211_IS_CHAN_2GHZ(ic->ic_curchan)) sc->rxon->flags |= htole32(IWN_RXON_AUTO | IWN_RXON_24GHZ); switch (ic->ic_opmode) { case IEEE80211_M_STA: sc->rxon->mode = IWN_MODE_STA; sc->rxon->filter = htole32(IWN_FILTER_MULTICAST); break; case IEEE80211_M_MONITOR: sc->rxon->mode = IWN_MODE_MONITOR; sc->rxon->filter = htole32(IWN_FILTER_MULTICAST | IWN_FILTER_CTL | IWN_FILTER_PROMISC); break; default: /* Should not get there. */ break; } sc->rxon->cck_mask = 0x0f; /* not yet negotiated */ sc->rxon->ofdm_mask = 0xff; /* not yet negotiated */ sc->rxon->ht_single_mask = 0xff; sc->rxon->ht_dual_mask = 0xff; sc->rxon->ht_triple_mask = 0xff; /* * In active association mode, ensure that * all the receive chains are enabled. * * Since we're not yet doing SMPS, don't allow the * number of idle RX chains to be less than the active * number. */ rxchain = IWN_RXCHAIN_VALID(sc->rxchainmask) | IWN_RXCHAIN_MIMO_COUNT(sc->nrxchains) | IWN_RXCHAIN_IDLE_COUNT(sc->nrxchains); sc->rxon->rxchain = htole16(rxchain); DPRINTF(sc, IWN_DEBUG_RESET | IWN_DEBUG_XMIT, "%s: rxchainmask=0x%x, nrxchains=%d\n", __func__, sc->rxchainmask, sc->nrxchains); sc->rxon->flags |= htole32(iwn_get_rxon_ht_flags(sc, ic->ic_curchan)); DPRINTF(sc, IWN_DEBUG_RESET, "%s: setting configuration; flags=0x%08x\n", __func__, le32toh(sc->rxon->flags)); if (sc->sc_is_scanning) device_printf(sc->sc_dev, "%s: is_scanning set, before RXON\n", __func__); error = iwn_cmd(sc, IWN_CMD_RXON, sc->rxon, sc->rxonsz, 0); if (error != 0) { device_printf(sc->sc_dev, "%s: RXON command failed\n", __func__); return error; } if ((error = iwn_add_broadcast_node(sc, 0)) != 0) { device_printf(sc->sc_dev, "%s: could not add broadcast node\n", __func__); return error; } /* Configuration has changed, set TX power accordingly. */ if ((error = ops->set_txpower(sc, ic->ic_curchan, 0)) != 0) { device_printf(sc->sc_dev, "%s: could not set TX power\n", __func__); return error; } if ((error = iwn_set_critical_temp(sc)) != 0) { device_printf(sc->sc_dev, "%s: could not set critical temperature\n", __func__); return error; } /* Set power saving level to CAM during initialization. */ if ((error = iwn_set_pslevel(sc, 0, 0, 0)) != 0) { device_printf(sc->sc_dev, "%s: could not set power saving level\n", __func__); return error; } DPRINTF(sc, IWN_DEBUG_TRACE, "->%s: end\n",__func__); return 0; } static uint16_t iwn_get_active_dwell_time(struct iwn_softc *sc, struct ieee80211_channel *c, uint8_t n_probes) { /* No channel? Default to 2GHz settings */ if (c == NULL || IEEE80211_IS_CHAN_2GHZ(c)) { return (IWN_ACTIVE_DWELL_TIME_2GHZ + IWN_ACTIVE_DWELL_FACTOR_2GHZ * (n_probes + 1)); } /* 5GHz dwell time */ return (IWN_ACTIVE_DWELL_TIME_5GHZ + IWN_ACTIVE_DWELL_FACTOR_5GHZ * (n_probes + 1)); } /* * Limit the total dwell time to 85% of the beacon interval. * * Returns the dwell time in milliseconds. */ static uint16_t iwn_limit_dwell(struct iwn_softc *sc, uint16_t dwell_time) { struct ieee80211com *ic = &sc->sc_ic; struct ieee80211vap *vap = NULL; int bintval = 0; /* bintval is in TU (1.024mS) */ if (! TAILQ_EMPTY(&ic->ic_vaps)) { vap = TAILQ_FIRST(&ic->ic_vaps); bintval = vap->iv_bss->ni_intval; } /* * If it's non-zero, we should calculate the minimum of * it and the DWELL_BASE. * * XXX Yes, the math should take into account that bintval * is 1.024mS, not 1mS.. */ if (bintval > 0) { DPRINTF(sc, IWN_DEBUG_SCAN, "%s: bintval=%d\n", __func__, bintval); return (MIN(IWN_PASSIVE_DWELL_BASE, ((bintval * 85) / 100))); } /* No association context? Default */ return (IWN_PASSIVE_DWELL_BASE); } static uint16_t iwn_get_passive_dwell_time(struct iwn_softc *sc, struct ieee80211_channel *c) { uint16_t passive; if (c == NULL || IEEE80211_IS_CHAN_2GHZ(c)) { passive = IWN_PASSIVE_DWELL_BASE + IWN_PASSIVE_DWELL_TIME_2GHZ; } else { passive = IWN_PASSIVE_DWELL_BASE + IWN_PASSIVE_DWELL_TIME_5GHZ; } /* Clamp to the beacon interval if we're associated */ return (iwn_limit_dwell(sc, passive)); } static int iwn_scan(struct iwn_softc *sc, struct ieee80211vap *vap, struct ieee80211_scan_state *ss, struct ieee80211_channel *c) { struct ieee80211com *ic = &sc->sc_ic; struct ieee80211_node *ni = vap->iv_bss; struct iwn_scan_hdr *hdr; struct iwn_cmd_data *tx; struct iwn_scan_essid *essid; struct iwn_scan_chan *chan; struct ieee80211_frame *wh; struct ieee80211_rateset *rs; uint8_t *buf, *frm; uint16_t rxchain; uint8_t txant; int buflen, error; int is_active; uint16_t dwell_active, dwell_passive; uint32_t extra, scan_service_time; DPRINTF(sc, IWN_DEBUG_TRACE, "->%s begin\n", __func__); /* * We are absolutely not allowed to send a scan command when another * scan command is pending. */ if (sc->sc_is_scanning) { device_printf(sc->sc_dev, "%s: called whilst scanning!\n", __func__); return (EAGAIN); } /* Assign the scan channel */ c = ic->ic_curchan; sc->rxon = &sc->rx_on[IWN_RXON_BSS_CTX]; buf = malloc(IWN_SCAN_MAXSZ, M_DEVBUF, M_NOWAIT | M_ZERO); if (buf == NULL) { device_printf(sc->sc_dev, "%s: could not allocate buffer for scan command\n", __func__); return ENOMEM; } hdr = (struct iwn_scan_hdr *)buf; /* * Move to the next channel if no frames are received within 10ms * after sending the probe request. */ hdr->quiet_time = htole16(10); /* timeout in milliseconds */ hdr->quiet_threshold = htole16(1); /* min # of packets */ /* * Max needs to be greater than active and passive and quiet! * It's also in microseconds! */ hdr->max_svc = htole32(250 * 1024); /* * Reset scan: interval=100 * Normal scan: interval=becaon interval * suspend_time: 100 (TU) * */ extra = (100 /* suspend_time */ / 100 /* beacon interval */) << 22; //scan_service_time = extra | ((100 /* susp */ % 100 /* int */) * 1024); scan_service_time = (4 << 22) | (100 * 1024); /* Hardcode for now! */ hdr->pause_svc = htole32(scan_service_time); /* Select antennas for scanning. */ rxchain = IWN_RXCHAIN_VALID(sc->rxchainmask) | IWN_RXCHAIN_FORCE_MIMO_SEL(sc->rxchainmask) | IWN_RXCHAIN_DRIVER_FORCE; if (IEEE80211_IS_CHAN_A(c) && sc->hw_type == IWN_HW_REV_TYPE_4965) { /* Ant A must be avoided in 5GHz because of an HW bug. */ rxchain |= IWN_RXCHAIN_FORCE_SEL(IWN_ANT_B); } else /* Use all available RX antennas. */ rxchain |= IWN_RXCHAIN_FORCE_SEL(sc->rxchainmask); hdr->rxchain = htole16(rxchain); hdr->filter = htole32(IWN_FILTER_MULTICAST | IWN_FILTER_BEACON); tx = (struct iwn_cmd_data *)(hdr + 1); tx->flags = htole32(IWN_TX_AUTO_SEQ); tx->id = sc->broadcast_id; tx->lifetime = htole32(IWN_LIFETIME_INFINITE); if (IEEE80211_IS_CHAN_5GHZ(c)) { /* Send probe requests at 6Mbps. */ tx->rate = htole32(0xd); rs = &ic->ic_sup_rates[IEEE80211_MODE_11A]; } else { hdr->flags = htole32(IWN_RXON_24GHZ | IWN_RXON_AUTO); if (sc->hw_type == IWN_HW_REV_TYPE_4965 && sc->rxon->associd && sc->rxon->chan > 14) tx->rate = htole32(0xd); else { /* Send probe requests at 1Mbps. */ tx->rate = htole32(10 | IWN_RFLAG_CCK); } rs = &ic->ic_sup_rates[IEEE80211_MODE_11G]; } /* Use the first valid TX antenna. */ txant = IWN_LSB(sc->txchainmask); tx->rate |= htole32(IWN_RFLAG_ANT(txant)); /* * Only do active scanning if we're announcing a probe request * for a given SSID (or more, if we ever add it to the driver.) */ is_active = 0; /* * If we're scanning for a specific SSID, add it to the command. * * XXX maybe look at adding support for scanning multiple SSIDs? */ essid = (struct iwn_scan_essid *)(tx + 1); if (ss != NULL) { if (ss->ss_ssid[0].len != 0) { essid[0].id = IEEE80211_ELEMID_SSID; essid[0].len = ss->ss_ssid[0].len; memcpy(essid[0].data, ss->ss_ssid[0].ssid, ss->ss_ssid[0].len); } DPRINTF(sc, IWN_DEBUG_SCAN, "%s: ssid_len=%d, ssid=%*s\n", __func__, ss->ss_ssid[0].len, ss->ss_ssid[0].len, ss->ss_ssid[0].ssid); if (ss->ss_nssid > 0) is_active = 1; } /* * Build a probe request frame. Most of the following code is a * copy & paste of what is done in net80211. */ wh = (struct ieee80211_frame *)(essid + 20); wh->i_fc[0] = IEEE80211_FC0_VERSION_0 | IEEE80211_FC0_TYPE_MGT | IEEE80211_FC0_SUBTYPE_PROBE_REQ; wh->i_fc[1] = IEEE80211_FC1_DIR_NODS; IEEE80211_ADDR_COPY(wh->i_addr1, vap->iv_ifp->if_broadcastaddr); IEEE80211_ADDR_COPY(wh->i_addr2, IF_LLADDR(vap->iv_ifp)); IEEE80211_ADDR_COPY(wh->i_addr3, vap->iv_ifp->if_broadcastaddr); *(uint16_t *)&wh->i_dur[0] = 0; /* filled by HW */ *(uint16_t *)&wh->i_seq[0] = 0; /* filled by HW */ frm = (uint8_t *)(wh + 1); frm = ieee80211_add_ssid(frm, NULL, 0); frm = ieee80211_add_rates(frm, rs); if (rs->rs_nrates > IEEE80211_RATE_SIZE) frm = ieee80211_add_xrates(frm, rs); if (ic->ic_htcaps & IEEE80211_HTC_HT) frm = ieee80211_add_htcap(frm, ni); /* Set length of probe request. */ tx->len = htole16(frm - (uint8_t *)wh); /* * If active scanning is requested but a certain channel is * marked passive, we can do active scanning if we detect * transmissions. * * There is an issue with some firmware versions that triggers * a sysassert on a "good CRC threshold" of zero (== disabled), * on a radar channel even though this means that we should NOT * send probes. * * The "good CRC threshold" is the number of frames that we * need to receive during our dwell time on a channel before * sending out probes -- setting this to a huge value will * mean we never reach it, but at the same time work around * the aforementioned issue. Thus use IWL_GOOD_CRC_TH_NEVER * here instead of IWL_GOOD_CRC_TH_DISABLED. * * This was fixed in later versions along with some other * scan changes, and the threshold behaves as a flag in those * versions. */ /* * If we're doing active scanning, set the crc_threshold * to a suitable value. This is different to active veruss * passive scanning depending upon the channel flags; the * firmware will obey that particular check for us. */ if (sc->tlv_feature_flags & IWN_UCODE_TLV_FLAGS_NEWSCAN) hdr->crc_threshold = is_active ? IWN_GOOD_CRC_TH_DEFAULT : IWN_GOOD_CRC_TH_DISABLED; else hdr->crc_threshold = is_active ? IWN_GOOD_CRC_TH_DEFAULT : IWN_GOOD_CRC_TH_NEVER; chan = (struct iwn_scan_chan *)frm; chan->chan = htole16(ieee80211_chan2ieee(ic, c)); chan->flags = 0; if (ss->ss_nssid > 0) chan->flags |= htole32(IWN_CHAN_NPBREQS(1)); chan->dsp_gain = 0x6e; /* * Set the passive/active flag depending upon the channel mode. * XXX TODO: take the is_active flag into account as well? */ if (c->ic_flags & IEEE80211_CHAN_PASSIVE) chan->flags |= htole32(IWN_CHAN_PASSIVE); else chan->flags |= htole32(IWN_CHAN_ACTIVE); /* * Calculate the active/passive dwell times. */ dwell_active = iwn_get_active_dwell_time(sc, c, ss->ss_nssid); dwell_passive = iwn_get_passive_dwell_time(sc, c); /* Make sure they're valid */ if (dwell_passive <= dwell_active) dwell_passive = dwell_active + 1; chan->active = htole16(dwell_active); chan->passive = htole16(dwell_passive); if (IEEE80211_IS_CHAN_5GHZ(c)) chan->rf_gain = 0x3b; else chan->rf_gain = 0x28; DPRINTF(sc, IWN_DEBUG_STATE, "%s: chan %u flags 0x%x rf_gain 0x%x " "dsp_gain 0x%x active %d passive %d scan_svc_time %d crc 0x%x " "isactive=%d numssid=%d\n", __func__, chan->chan, chan->flags, chan->rf_gain, chan->dsp_gain, dwell_active, dwell_passive, scan_service_time, hdr->crc_threshold, is_active, ss->ss_nssid); hdr->nchan++; chan++; buflen = (uint8_t *)chan - buf; hdr->len = htole16(buflen); if (sc->sc_is_scanning) { device_printf(sc->sc_dev, "%s: called with is_scanning set!\n", __func__); } sc->sc_is_scanning = 1; DPRINTF(sc, IWN_DEBUG_STATE, "sending scan command nchan=%d\n", hdr->nchan); error = iwn_cmd(sc, IWN_CMD_SCAN, buf, buflen, 1); free(buf, M_DEVBUF); DPRINTF(sc, IWN_DEBUG_TRACE, "->%s: end\n",__func__); return error; } static int iwn_auth(struct iwn_softc *sc, struct ieee80211vap *vap) { struct iwn_ops *ops = &sc->ops; struct ieee80211com *ic = &sc->sc_ic; struct ieee80211_node *ni = vap->iv_bss; int error; DPRINTF(sc, IWN_DEBUG_TRACE, "->%s begin\n", __func__); sc->rxon = &sc->rx_on[IWN_RXON_BSS_CTX]; /* Update adapter configuration. */ IEEE80211_ADDR_COPY(sc->rxon->bssid, ni->ni_bssid); sc->rxon->chan = ieee80211_chan2ieee(ic, ni->ni_chan); sc->rxon->flags = htole32(IWN_RXON_TSF | IWN_RXON_CTS_TO_SELF); if (IEEE80211_IS_CHAN_2GHZ(ni->ni_chan)) sc->rxon->flags |= htole32(IWN_RXON_AUTO | IWN_RXON_24GHZ); if (ic->ic_flags & IEEE80211_F_SHSLOT) sc->rxon->flags |= htole32(IWN_RXON_SHSLOT); if (ic->ic_flags & IEEE80211_F_SHPREAMBLE) sc->rxon->flags |= htole32(IWN_RXON_SHPREAMBLE); if (IEEE80211_IS_CHAN_A(ni->ni_chan)) { sc->rxon->cck_mask = 0; sc->rxon->ofdm_mask = 0x15; } else if (IEEE80211_IS_CHAN_B(ni->ni_chan)) { sc->rxon->cck_mask = 0x03; sc->rxon->ofdm_mask = 0; } else { /* Assume 802.11b/g. */ sc->rxon->cck_mask = 0x03; sc->rxon->ofdm_mask = 0x15; } /* try HT */ sc->rxon->flags |= htole32(iwn_get_rxon_ht_flags(sc, ic->ic_curchan)); DPRINTF(sc, IWN_DEBUG_STATE, "rxon chan %d flags %x cck %x ofdm %x\n", sc->rxon->chan, sc->rxon->flags, sc->rxon->cck_mask, sc->rxon->ofdm_mask); if (sc->sc_is_scanning) device_printf(sc->sc_dev, "%s: is_scanning set, before RXON\n", __func__); error = iwn_cmd(sc, IWN_CMD_RXON, sc->rxon, sc->rxonsz, 1); if (error != 0) { device_printf(sc->sc_dev, "%s: RXON command failed, error %d\n", __func__, error); return error; } /* Configuration has changed, set TX power accordingly. */ if ((error = ops->set_txpower(sc, ni->ni_chan, 1)) != 0) { device_printf(sc->sc_dev, "%s: could not set TX power, error %d\n", __func__, error); return error; } /* * Reconfiguring RXON clears the firmware nodes table so we must * add the broadcast node again. */ if ((error = iwn_add_broadcast_node(sc, 1)) != 0) { device_printf(sc->sc_dev, "%s: could not add broadcast node, error %d\n", __func__, error); return error; } DPRINTF(sc, IWN_DEBUG_TRACE, "->%s: end\n",__func__); return 0; } static int iwn_run(struct iwn_softc *sc, struct ieee80211vap *vap) { struct iwn_ops *ops = &sc->ops; struct ieee80211com *ic = &sc->sc_ic; struct ieee80211_node *ni = vap->iv_bss; struct iwn_node_info node; int error; DPRINTF(sc, IWN_DEBUG_TRACE, "->%s begin\n", __func__); sc->rxon = &sc->rx_on[IWN_RXON_BSS_CTX]; if (ic->ic_opmode == IEEE80211_M_MONITOR) { /* Link LED blinks while monitoring. */ iwn_set_led(sc, IWN_LED_LINK, 5, 5); return 0; } if ((error = iwn_set_timing(sc, ni)) != 0) { device_printf(sc->sc_dev, "%s: could not set timing, error %d\n", __func__, error); return error; } /* Update adapter configuration. */ IEEE80211_ADDR_COPY(sc->rxon->bssid, ni->ni_bssid); sc->rxon->associd = htole16(IEEE80211_AID(ni->ni_associd)); sc->rxon->chan = ieee80211_chan2ieee(ic, ni->ni_chan); sc->rxon->flags = htole32(IWN_RXON_TSF | IWN_RXON_CTS_TO_SELF); if (IEEE80211_IS_CHAN_2GHZ(ni->ni_chan)) sc->rxon->flags |= htole32(IWN_RXON_AUTO | IWN_RXON_24GHZ); if (ic->ic_flags & IEEE80211_F_SHSLOT) sc->rxon->flags |= htole32(IWN_RXON_SHSLOT); if (ic->ic_flags & IEEE80211_F_SHPREAMBLE) sc->rxon->flags |= htole32(IWN_RXON_SHPREAMBLE); if (IEEE80211_IS_CHAN_A(ni->ni_chan)) { sc->rxon->cck_mask = 0; sc->rxon->ofdm_mask = 0x15; } else if (IEEE80211_IS_CHAN_B(ni->ni_chan)) { sc->rxon->cck_mask = 0x03; sc->rxon->ofdm_mask = 0; } else { /* Assume 802.11b/g. */ sc->rxon->cck_mask = 0x0f; sc->rxon->ofdm_mask = 0x15; } /* try HT */ sc->rxon->flags |= htole32(iwn_get_rxon_ht_flags(sc, ni->ni_chan)); sc->rxon->filter |= htole32(IWN_FILTER_BSS); DPRINTF(sc, IWN_DEBUG_STATE, "rxon chan %d flags %x, curhtprotmode=%d\n", sc->rxon->chan, le32toh(sc->rxon->flags), ic->ic_curhtprotmode); if (sc->sc_is_scanning) device_printf(sc->sc_dev, "%s: is_scanning set, before RXON\n", __func__); error = iwn_cmd(sc, IWN_CMD_RXON, sc->rxon, sc->rxonsz, 1); if (error != 0) { device_printf(sc->sc_dev, "%s: could not update configuration, error %d\n", __func__, error); return error; } /* Configuration has changed, set TX power accordingly. */ if ((error = ops->set_txpower(sc, ni->ni_chan, 1)) != 0) { device_printf(sc->sc_dev, "%s: could not set TX power, error %d\n", __func__, error); return error; } /* Fake a join to initialize the TX rate. */ ((struct iwn_node *)ni)->id = IWN_ID_BSS; iwn_newassoc(ni, 1); /* Add BSS node. */ memset(&node, 0, sizeof node); IEEE80211_ADDR_COPY(node.macaddr, ni->ni_macaddr); node.id = IWN_ID_BSS; if (IEEE80211_IS_CHAN_HT(ni->ni_chan)) { switch (ni->ni_htcap & IEEE80211_HTCAP_SMPS) { case IEEE80211_HTCAP_SMPS_ENA: node.htflags |= htole32(IWN_SMPS_MIMO_DIS); break; case IEEE80211_HTCAP_SMPS_DYNAMIC: node.htflags |= htole32(IWN_SMPS_MIMO_PROT); break; } node.htflags |= htole32(IWN_AMDPU_SIZE_FACTOR(3) | IWN_AMDPU_DENSITY(5)); /* 4us */ if (IEEE80211_IS_CHAN_HT40(ni->ni_chan)) node.htflags |= htole32(IWN_NODE_HT40); } DPRINTF(sc, IWN_DEBUG_STATE, "%s: adding BSS node\n", __func__); error = ops->add_node(sc, &node, 1); if (error != 0) { device_printf(sc->sc_dev, "%s: could not add BSS node, error %d\n", __func__, error); return error; } DPRINTF(sc, IWN_DEBUG_STATE, "%s: setting link quality for node %d\n", __func__, node.id); if ((error = iwn_set_link_quality(sc, ni)) != 0) { device_printf(sc->sc_dev, "%s: could not setup link quality for node %d, error %d\n", __func__, node.id, error); return error; } if ((error = iwn_init_sensitivity(sc)) != 0) { device_printf(sc->sc_dev, "%s: could not set sensitivity, error %d\n", __func__, error); return error; } /* Start periodic calibration timer. */ sc->calib.state = IWN_CALIB_STATE_ASSOC; sc->calib_cnt = 0; callout_reset(&sc->calib_to, msecs_to_ticks(500), iwn_calib_timeout, sc); /* Link LED always on while associated. */ iwn_set_led(sc, IWN_LED_LINK, 0, 1); DPRINTF(sc, IWN_DEBUG_TRACE, "->%s: end\n",__func__); return 0; } /* * This function is called by upper layer when an ADDBA request is received * from another STA and before the ADDBA response is sent. */ static int iwn_ampdu_rx_start(struct ieee80211_node *ni, struct ieee80211_rx_ampdu *rap, int baparamset, int batimeout, int baseqctl) { #define MS(_v, _f) (((_v) & _f) >> _f##_S) struct iwn_softc *sc = ni->ni_ic->ic_softc; struct iwn_ops *ops = &sc->ops; struct iwn_node *wn = (void *)ni; struct iwn_node_info node; uint16_t ssn; uint8_t tid; int error; DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); tid = MS(le16toh(baparamset), IEEE80211_BAPS_TID); ssn = MS(le16toh(baseqctl), IEEE80211_BASEQ_START); memset(&node, 0, sizeof node); node.id = wn->id; node.control = IWN_NODE_UPDATE; node.flags = IWN_FLAG_SET_ADDBA; node.addba_tid = tid; node.addba_ssn = htole16(ssn); DPRINTF(sc, IWN_DEBUG_RECV, "ADDBA RA=%d TID=%d SSN=%d\n", wn->id, tid, ssn); error = ops->add_node(sc, &node, 1); if (error != 0) return error; return sc->sc_ampdu_rx_start(ni, rap, baparamset, batimeout, baseqctl); #undef MS } /* * This function is called by upper layer on teardown of an HT-immediate * Block Ack agreement (eg. uppon receipt of a DELBA frame). */ static void iwn_ampdu_rx_stop(struct ieee80211_node *ni, struct ieee80211_rx_ampdu *rap) { struct ieee80211com *ic = ni->ni_ic; struct iwn_softc *sc = ic->ic_softc; struct iwn_ops *ops = &sc->ops; struct iwn_node *wn = (void *)ni; struct iwn_node_info node; uint8_t tid; DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); /* XXX: tid as an argument */ for (tid = 0; tid < WME_NUM_TID; tid++) { if (&ni->ni_rx_ampdu[tid] == rap) break; } memset(&node, 0, sizeof node); node.id = wn->id; node.control = IWN_NODE_UPDATE; node.flags = IWN_FLAG_SET_DELBA; node.delba_tid = tid; DPRINTF(sc, IWN_DEBUG_RECV, "DELBA RA=%d TID=%d\n", wn->id, tid); (void)ops->add_node(sc, &node, 1); sc->sc_ampdu_rx_stop(ni, rap); } static int iwn_addba_request(struct ieee80211_node *ni, struct ieee80211_tx_ampdu *tap, int dialogtoken, int baparamset, int batimeout) { struct iwn_softc *sc = ni->ni_ic->ic_softc; int qid; DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); for (qid = sc->firstaggqueue; qid < sc->ntxqs; qid++) { if (sc->qid2tap[qid] == NULL) break; } if (qid == sc->ntxqs) { DPRINTF(sc, IWN_DEBUG_XMIT, "%s: not free aggregation queue\n", __func__); return 0; } tap->txa_private = malloc(sizeof(int), M_DEVBUF, M_NOWAIT); if (tap->txa_private == NULL) { device_printf(sc->sc_dev, "%s: failed to alloc TX aggregation structure\n", __func__); return 0; } sc->qid2tap[qid] = tap; *(int *)tap->txa_private = qid; return sc->sc_addba_request(ni, tap, dialogtoken, baparamset, batimeout); } static int iwn_addba_response(struct ieee80211_node *ni, struct ieee80211_tx_ampdu *tap, int code, int baparamset, int batimeout) { struct iwn_softc *sc = ni->ni_ic->ic_softc; int qid = *(int *)tap->txa_private; uint8_t tid = tap->txa_tid; int ret; DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); if (code == IEEE80211_STATUS_SUCCESS) { ni->ni_txseqs[tid] = tap->txa_start & 0xfff; ret = iwn_ampdu_tx_start(ni->ni_ic, ni, tid); if (ret != 1) return ret; } else { sc->qid2tap[qid] = NULL; free(tap->txa_private, M_DEVBUF); tap->txa_private = NULL; } return sc->sc_addba_response(ni, tap, code, baparamset, batimeout); } /* * This function is called by upper layer when an ADDBA response is received * from another STA. */ static int iwn_ampdu_tx_start(struct ieee80211com *ic, struct ieee80211_node *ni, uint8_t tid) { struct ieee80211_tx_ampdu *tap = &ni->ni_tx_ampdu[tid]; struct iwn_softc *sc = ni->ni_ic->ic_softc; struct iwn_ops *ops = &sc->ops; struct iwn_node *wn = (void *)ni; struct iwn_node_info node; int error, qid; DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); /* Enable TX for the specified RA/TID. */ wn->disable_tid &= ~(1 << tid); memset(&node, 0, sizeof node); node.id = wn->id; node.control = IWN_NODE_UPDATE; node.flags = IWN_FLAG_SET_DISABLE_TID; node.disable_tid = htole16(wn->disable_tid); error = ops->add_node(sc, &node, 1); if (error != 0) return 0; if ((error = iwn_nic_lock(sc)) != 0) return 0; qid = *(int *)tap->txa_private; DPRINTF(sc, IWN_DEBUG_XMIT, "%s: ra=%d tid=%d ssn=%d qid=%d\n", __func__, wn->id, tid, tap->txa_start, qid); ops->ampdu_tx_start(sc, ni, qid, tid, tap->txa_start & 0xfff); iwn_nic_unlock(sc); iwn_set_link_quality(sc, ni); return 1; } static void iwn_ampdu_tx_stop(struct ieee80211_node *ni, struct ieee80211_tx_ampdu *tap) { struct iwn_softc *sc = ni->ni_ic->ic_softc; struct iwn_ops *ops = &sc->ops; uint8_t tid = tap->txa_tid; int qid; DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); sc->sc_addba_stop(ni, tap); if (tap->txa_private == NULL) return; qid = *(int *)tap->txa_private; if (sc->txq[qid].queued != 0) return; if (iwn_nic_lock(sc) != 0) return; ops->ampdu_tx_stop(sc, qid, tid, tap->txa_start & 0xfff); iwn_nic_unlock(sc); sc->qid2tap[qid] = NULL; free(tap->txa_private, M_DEVBUF); tap->txa_private = NULL; } static void iwn4965_ampdu_tx_start(struct iwn_softc *sc, struct ieee80211_node *ni, int qid, uint8_t tid, uint16_t ssn) { struct iwn_node *wn = (void *)ni; DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); /* Stop TX scheduler while we're changing its configuration. */ iwn_prph_write(sc, IWN4965_SCHED_QUEUE_STATUS(qid), IWN4965_TXQ_STATUS_CHGACT); /* Assign RA/TID translation to the queue. */ iwn_mem_write_2(sc, sc->sched_base + IWN4965_SCHED_TRANS_TBL(qid), wn->id << 4 | tid); /* Enable chain-building mode for the queue. */ iwn_prph_setbits(sc, IWN4965_SCHED_QCHAIN_SEL, 1 << qid); /* Set starting sequence number from the ADDBA request. */ sc->txq[qid].cur = sc->txq[qid].read = (ssn & 0xff); IWN_WRITE(sc, IWN_HBUS_TARG_WRPTR, qid << 8 | (ssn & 0xff)); iwn_prph_write(sc, IWN4965_SCHED_QUEUE_RDPTR(qid), ssn); /* Set scheduler window size. */ iwn_mem_write(sc, sc->sched_base + IWN4965_SCHED_QUEUE_OFFSET(qid), IWN_SCHED_WINSZ); /* Set scheduler frame limit. */ iwn_mem_write(sc, sc->sched_base + IWN4965_SCHED_QUEUE_OFFSET(qid) + 4, IWN_SCHED_LIMIT << 16); /* Enable interrupts for the queue. */ iwn_prph_setbits(sc, IWN4965_SCHED_INTR_MASK, 1 << qid); /* Mark the queue as active. */ iwn_prph_write(sc, IWN4965_SCHED_QUEUE_STATUS(qid), IWN4965_TXQ_STATUS_ACTIVE | IWN4965_TXQ_STATUS_AGGR_ENA | iwn_tid2fifo[tid] << 1); } static void iwn4965_ampdu_tx_stop(struct iwn_softc *sc, int qid, uint8_t tid, uint16_t ssn) { DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); /* Stop TX scheduler while we're changing its configuration. */ iwn_prph_write(sc, IWN4965_SCHED_QUEUE_STATUS(qid), IWN4965_TXQ_STATUS_CHGACT); /* Set starting sequence number from the ADDBA request. */ IWN_WRITE(sc, IWN_HBUS_TARG_WRPTR, qid << 8 | (ssn & 0xff)); iwn_prph_write(sc, IWN4965_SCHED_QUEUE_RDPTR(qid), ssn); /* Disable interrupts for the queue. */ iwn_prph_clrbits(sc, IWN4965_SCHED_INTR_MASK, 1 << qid); /* Mark the queue as inactive. */ iwn_prph_write(sc, IWN4965_SCHED_QUEUE_STATUS(qid), IWN4965_TXQ_STATUS_INACTIVE | iwn_tid2fifo[tid] << 1); } static void iwn5000_ampdu_tx_start(struct iwn_softc *sc, struct ieee80211_node *ni, int qid, uint8_t tid, uint16_t ssn) { DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); struct iwn_node *wn = (void *)ni; /* Stop TX scheduler while we're changing its configuration. */ iwn_prph_write(sc, IWN5000_SCHED_QUEUE_STATUS(qid), IWN5000_TXQ_STATUS_CHGACT); /* Assign RA/TID translation to the queue. */ iwn_mem_write_2(sc, sc->sched_base + IWN5000_SCHED_TRANS_TBL(qid), wn->id << 4 | tid); /* Enable chain-building mode for the queue. */ iwn_prph_setbits(sc, IWN5000_SCHED_QCHAIN_SEL, 1 << qid); /* Enable aggregation for the queue. */ iwn_prph_setbits(sc, IWN5000_SCHED_AGGR_SEL, 1 << qid); /* Set starting sequence number from the ADDBA request. */ sc->txq[qid].cur = sc->txq[qid].read = (ssn & 0xff); IWN_WRITE(sc, IWN_HBUS_TARG_WRPTR, qid << 8 | (ssn & 0xff)); iwn_prph_write(sc, IWN5000_SCHED_QUEUE_RDPTR(qid), ssn); /* Set scheduler window size and frame limit. */ iwn_mem_write(sc, sc->sched_base + IWN5000_SCHED_QUEUE_OFFSET(qid) + 4, IWN_SCHED_LIMIT << 16 | IWN_SCHED_WINSZ); /* Enable interrupts for the queue. */ iwn_prph_setbits(sc, IWN5000_SCHED_INTR_MASK, 1 << qid); /* Mark the queue as active. */ iwn_prph_write(sc, IWN5000_SCHED_QUEUE_STATUS(qid), IWN5000_TXQ_STATUS_ACTIVE | iwn_tid2fifo[tid]); } static void iwn5000_ampdu_tx_stop(struct iwn_softc *sc, int qid, uint8_t tid, uint16_t ssn) { DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); /* Stop TX scheduler while we're changing its configuration. */ iwn_prph_write(sc, IWN5000_SCHED_QUEUE_STATUS(qid), IWN5000_TXQ_STATUS_CHGACT); /* Disable aggregation for the queue. */ iwn_prph_clrbits(sc, IWN5000_SCHED_AGGR_SEL, 1 << qid); /* Set starting sequence number from the ADDBA request. */ IWN_WRITE(sc, IWN_HBUS_TARG_WRPTR, qid << 8 | (ssn & 0xff)); iwn_prph_write(sc, IWN5000_SCHED_QUEUE_RDPTR(qid), ssn); /* Disable interrupts for the queue. */ iwn_prph_clrbits(sc, IWN5000_SCHED_INTR_MASK, 1 << qid); /* Mark the queue as inactive. */ iwn_prph_write(sc, IWN5000_SCHED_QUEUE_STATUS(qid), IWN5000_TXQ_STATUS_INACTIVE | iwn_tid2fifo[tid]); } /* * Query calibration tables from the initialization firmware. We do this * only once at first boot. Called from a process context. */ static int iwn5000_query_calibration(struct iwn_softc *sc) { struct iwn5000_calib_config cmd; int error; memset(&cmd, 0, sizeof cmd); cmd.ucode.once.enable = htole32(0xffffffff); cmd.ucode.once.start = htole32(0xffffffff); cmd.ucode.once.send = htole32(0xffffffff); cmd.ucode.flags = htole32(0xffffffff); DPRINTF(sc, IWN_DEBUG_CALIBRATE, "%s: sending calibration query\n", __func__); error = iwn_cmd(sc, IWN5000_CMD_CALIB_CONFIG, &cmd, sizeof cmd, 0); if (error != 0) return error; /* Wait at most two seconds for calibration to complete. */ if (!(sc->sc_flags & IWN_FLAG_CALIB_DONE)) error = msleep(sc, &sc->sc_mtx, PCATCH, "iwncal", 2 * hz); return error; } /* * Send calibration results to the runtime firmware. These results were * obtained on first boot from the initialization firmware. */ static int iwn5000_send_calibration(struct iwn_softc *sc) { int idx, error; for (idx = 0; idx < IWN5000_PHY_CALIB_MAX_RESULT; idx++) { if (!(sc->base_params->calib_need & (1<calibcmd[idx].buf == NULL) { DPRINTF(sc, IWN_DEBUG_CALIBRATE, "Need calib idx : %d but no available data\n", idx); continue; } DPRINTF(sc, IWN_DEBUG_CALIBRATE, "send calibration result idx=%d len=%d\n", idx, sc->calibcmd[idx].len); error = iwn_cmd(sc, IWN_CMD_PHY_CALIB, sc->calibcmd[idx].buf, sc->calibcmd[idx].len, 0); if (error != 0) { device_printf(sc->sc_dev, "%s: could not send calibration result, error %d\n", __func__, error); return error; } } return 0; } static int iwn5000_send_wimax_coex(struct iwn_softc *sc) { struct iwn5000_wimax_coex wimax; #if 0 if (sc->hw_type == IWN_HW_REV_TYPE_6050) { /* Enable WiMAX coexistence for combo adapters. */ wimax.flags = IWN_WIMAX_COEX_ASSOC_WA_UNMASK | IWN_WIMAX_COEX_UNASSOC_WA_UNMASK | IWN_WIMAX_COEX_STA_TABLE_VALID | IWN_WIMAX_COEX_ENABLE; memcpy(wimax.events, iwn6050_wimax_events, sizeof iwn6050_wimax_events); } else #endif { /* Disable WiMAX coexistence. */ wimax.flags = 0; memset(wimax.events, 0, sizeof wimax.events); } DPRINTF(sc, IWN_DEBUG_RESET, "%s: Configuring WiMAX coexistence\n", __func__); return iwn_cmd(sc, IWN5000_CMD_WIMAX_COEX, &wimax, sizeof wimax, 0); } static int iwn5000_crystal_calib(struct iwn_softc *sc) { struct iwn5000_phy_calib_crystal cmd; memset(&cmd, 0, sizeof cmd); cmd.code = IWN5000_PHY_CALIB_CRYSTAL; cmd.ngroups = 1; cmd.isvalid = 1; cmd.cap_pin[0] = le32toh(sc->eeprom_crystal) & 0xff; cmd.cap_pin[1] = (le32toh(sc->eeprom_crystal) >> 16) & 0xff; DPRINTF(sc, IWN_DEBUG_CALIBRATE, "sending crystal calibration %d, %d\n", cmd.cap_pin[0], cmd.cap_pin[1]); return iwn_cmd(sc, IWN_CMD_PHY_CALIB, &cmd, sizeof cmd, 0); } static int iwn5000_temp_offset_calib(struct iwn_softc *sc) { struct iwn5000_phy_calib_temp_offset cmd; memset(&cmd, 0, sizeof cmd); cmd.code = IWN5000_PHY_CALIB_TEMP_OFFSET; cmd.ngroups = 1; cmd.isvalid = 1; if (sc->eeprom_temp != 0) cmd.offset = htole16(sc->eeprom_temp); else cmd.offset = htole16(IWN_DEFAULT_TEMP_OFFSET); DPRINTF(sc, IWN_DEBUG_CALIBRATE, "setting radio sensor offset to %d\n", le16toh(cmd.offset)); return iwn_cmd(sc, IWN_CMD_PHY_CALIB, &cmd, sizeof cmd, 0); } static int iwn5000_temp_offset_calibv2(struct iwn_softc *sc) { struct iwn5000_phy_calib_temp_offsetv2 cmd; memset(&cmd, 0, sizeof cmd); cmd.code = IWN5000_PHY_CALIB_TEMP_OFFSET; cmd.ngroups = 1; cmd.isvalid = 1; if (sc->eeprom_temp != 0) { cmd.offset_low = htole16(sc->eeprom_temp); cmd.offset_high = htole16(sc->eeprom_temp_high); } else { cmd.offset_low = htole16(IWN_DEFAULT_TEMP_OFFSET); cmd.offset_high = htole16(IWN_DEFAULT_TEMP_OFFSET); } cmd.burnt_voltage_ref = htole16(sc->eeprom_voltage); DPRINTF(sc, IWN_DEBUG_CALIBRATE, "setting radio sensor low offset to %d, high offset to %d, voltage to %d\n", le16toh(cmd.offset_low), le16toh(cmd.offset_high), le16toh(cmd.burnt_voltage_ref)); return iwn_cmd(sc, IWN_CMD_PHY_CALIB, &cmd, sizeof cmd, 0); } /* * This function is called after the runtime firmware notifies us of its * readiness (called in a process context). */ static int iwn4965_post_alive(struct iwn_softc *sc) { int error, qid; if ((error = iwn_nic_lock(sc)) != 0) return error; DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); /* Clear TX scheduler state in SRAM. */ sc->sched_base = iwn_prph_read(sc, IWN_SCHED_SRAM_ADDR); iwn_mem_set_region_4(sc, sc->sched_base + IWN4965_SCHED_CTX_OFF, 0, IWN4965_SCHED_CTX_LEN / sizeof (uint32_t)); /* Set physical address of TX scheduler rings (1KB aligned). */ iwn_prph_write(sc, IWN4965_SCHED_DRAM_ADDR, sc->sched_dma.paddr >> 10); IWN_SETBITS(sc, IWN_FH_TX_CHICKEN, IWN_FH_TX_CHICKEN_SCHED_RETRY); /* Disable chain mode for all our 16 queues. */ iwn_prph_write(sc, IWN4965_SCHED_QCHAIN_SEL, 0); for (qid = 0; qid < IWN4965_NTXQUEUES; qid++) { iwn_prph_write(sc, IWN4965_SCHED_QUEUE_RDPTR(qid), 0); IWN_WRITE(sc, IWN_HBUS_TARG_WRPTR, qid << 8 | 0); /* Set scheduler window size. */ iwn_mem_write(sc, sc->sched_base + IWN4965_SCHED_QUEUE_OFFSET(qid), IWN_SCHED_WINSZ); /* Set scheduler frame limit. */ iwn_mem_write(sc, sc->sched_base + IWN4965_SCHED_QUEUE_OFFSET(qid) + 4, IWN_SCHED_LIMIT << 16); } /* Enable interrupts for all our 16 queues. */ iwn_prph_write(sc, IWN4965_SCHED_INTR_MASK, 0xffff); /* Identify TX FIFO rings (0-7). */ iwn_prph_write(sc, IWN4965_SCHED_TXFACT, 0xff); /* Mark TX rings (4 EDCA + cmd + 2 HCCA) as active. */ for (qid = 0; qid < 7; qid++) { static uint8_t qid2fifo[] = { 3, 2, 1, 0, 4, 5, 6 }; iwn_prph_write(sc, IWN4965_SCHED_QUEUE_STATUS(qid), IWN4965_TXQ_STATUS_ACTIVE | qid2fifo[qid] << 1); } iwn_nic_unlock(sc); return 0; } /* * This function is called after the initialization or runtime firmware * notifies us of its readiness (called in a process context). */ static int iwn5000_post_alive(struct iwn_softc *sc) { int error, qid; DPRINTF(sc, IWN_DEBUG_TRACE, "->%s begin\n", __func__); /* Switch to using ICT interrupt mode. */ iwn5000_ict_reset(sc); if ((error = iwn_nic_lock(sc)) != 0){ DPRINTF(sc, IWN_DEBUG_TRACE, "->%s end in error\n", __func__); return error; } /* Clear TX scheduler state in SRAM. */ sc->sched_base = iwn_prph_read(sc, IWN_SCHED_SRAM_ADDR); iwn_mem_set_region_4(sc, sc->sched_base + IWN5000_SCHED_CTX_OFF, 0, IWN5000_SCHED_CTX_LEN / sizeof (uint32_t)); /* Set physical address of TX scheduler rings (1KB aligned). */ iwn_prph_write(sc, IWN5000_SCHED_DRAM_ADDR, sc->sched_dma.paddr >> 10); IWN_SETBITS(sc, IWN_FH_TX_CHICKEN, IWN_FH_TX_CHICKEN_SCHED_RETRY); /* Enable chain mode for all queues, except command queue. */ if (sc->sc_flags & IWN_FLAG_PAN_SUPPORT) iwn_prph_write(sc, IWN5000_SCHED_QCHAIN_SEL, 0xfffdf); else iwn_prph_write(sc, IWN5000_SCHED_QCHAIN_SEL, 0xfffef); iwn_prph_write(sc, IWN5000_SCHED_AGGR_SEL, 0); for (qid = 0; qid < IWN5000_NTXQUEUES; qid++) { iwn_prph_write(sc, IWN5000_SCHED_QUEUE_RDPTR(qid), 0); IWN_WRITE(sc, IWN_HBUS_TARG_WRPTR, qid << 8 | 0); iwn_mem_write(sc, sc->sched_base + IWN5000_SCHED_QUEUE_OFFSET(qid), 0); /* Set scheduler window size and frame limit. */ iwn_mem_write(sc, sc->sched_base + IWN5000_SCHED_QUEUE_OFFSET(qid) + 4, IWN_SCHED_LIMIT << 16 | IWN_SCHED_WINSZ); } /* Enable interrupts for all our 20 queues. */ iwn_prph_write(sc, IWN5000_SCHED_INTR_MASK, 0xfffff); /* Identify TX FIFO rings (0-7). */ iwn_prph_write(sc, IWN5000_SCHED_TXFACT, 0xff); /* Mark TX rings (4 EDCA + cmd + 2 HCCA) as active. */ if (sc->sc_flags & IWN_FLAG_PAN_SUPPORT) { /* Mark TX rings as active. */ for (qid = 0; qid < 11; qid++) { static uint8_t qid2fifo[] = { 3, 2, 1, 0, 0, 4, 2, 5, 4, 7, 5 }; iwn_prph_write(sc, IWN5000_SCHED_QUEUE_STATUS(qid), IWN5000_TXQ_STATUS_ACTIVE | qid2fifo[qid]); } } else { /* Mark TX rings (4 EDCA + cmd + 2 HCCA) as active. */ for (qid = 0; qid < 7; qid++) { static uint8_t qid2fifo[] = { 3, 2, 1, 0, 7, 5, 6 }; iwn_prph_write(sc, IWN5000_SCHED_QUEUE_STATUS(qid), IWN5000_TXQ_STATUS_ACTIVE | qid2fifo[qid]); } } iwn_nic_unlock(sc); /* Configure WiMAX coexistence for combo adapters. */ error = iwn5000_send_wimax_coex(sc); if (error != 0) { device_printf(sc->sc_dev, "%s: could not configure WiMAX coexistence, error %d\n", __func__, error); return error; } if (sc->hw_type != IWN_HW_REV_TYPE_5150) { /* Perform crystal calibration. */ error = iwn5000_crystal_calib(sc); if (error != 0) { device_printf(sc->sc_dev, "%s: crystal calibration failed, error %d\n", __func__, error); return error; } } if (!(sc->sc_flags & IWN_FLAG_CALIB_DONE)) { /* Query calibration from the initialization firmware. */ if ((error = iwn5000_query_calibration(sc)) != 0) { device_printf(sc->sc_dev, "%s: could not query calibration, error %d\n", __func__, error); return error; } /* * We have the calibration results now, reboot with the * runtime firmware (call ourselves recursively!) */ iwn_hw_stop(sc); error = iwn_hw_init(sc); } else { /* Send calibration results to runtime firmware. */ error = iwn5000_send_calibration(sc); } DPRINTF(sc, IWN_DEBUG_TRACE, "->%s: end\n",__func__); return error; } /* * The firmware boot code is small and is intended to be copied directly into * the NIC internal memory (no DMA transfer). */ static int iwn4965_load_bootcode(struct iwn_softc *sc, const uint8_t *ucode, int size) { int error, ntries; size /= sizeof (uint32_t); if ((error = iwn_nic_lock(sc)) != 0) return error; /* Copy microcode image into NIC memory. */ iwn_prph_write_region_4(sc, IWN_BSM_SRAM_BASE, (const uint32_t *)ucode, size); iwn_prph_write(sc, IWN_BSM_WR_MEM_SRC, 0); iwn_prph_write(sc, IWN_BSM_WR_MEM_DST, IWN_FW_TEXT_BASE); iwn_prph_write(sc, IWN_BSM_WR_DWCOUNT, size); /* Start boot load now. */ iwn_prph_write(sc, IWN_BSM_WR_CTRL, IWN_BSM_WR_CTRL_START); /* Wait for transfer to complete. */ for (ntries = 0; ntries < 1000; ntries++) { if (!(iwn_prph_read(sc, IWN_BSM_WR_CTRL) & IWN_BSM_WR_CTRL_START)) break; DELAY(10); } if (ntries == 1000) { device_printf(sc->sc_dev, "%s: could not load boot firmware\n", __func__); iwn_nic_unlock(sc); return ETIMEDOUT; } /* Enable boot after power up. */ iwn_prph_write(sc, IWN_BSM_WR_CTRL, IWN_BSM_WR_CTRL_START_EN); iwn_nic_unlock(sc); return 0; } static int iwn4965_load_firmware(struct iwn_softc *sc) { struct iwn_fw_info *fw = &sc->fw; struct iwn_dma_info *dma = &sc->fw_dma; int error; /* Copy initialization sections into pre-allocated DMA-safe memory. */ memcpy(dma->vaddr, fw->init.data, fw->init.datasz); bus_dmamap_sync(dma->tag, dma->map, BUS_DMASYNC_PREWRITE); memcpy(dma->vaddr + IWN4965_FW_DATA_MAXSZ, fw->init.text, fw->init.textsz); bus_dmamap_sync(dma->tag, dma->map, BUS_DMASYNC_PREWRITE); /* Tell adapter where to find initialization sections. */ if ((error = iwn_nic_lock(sc)) != 0) return error; iwn_prph_write(sc, IWN_BSM_DRAM_DATA_ADDR, dma->paddr >> 4); iwn_prph_write(sc, IWN_BSM_DRAM_DATA_SIZE, fw->init.datasz); iwn_prph_write(sc, IWN_BSM_DRAM_TEXT_ADDR, (dma->paddr + IWN4965_FW_DATA_MAXSZ) >> 4); iwn_prph_write(sc, IWN_BSM_DRAM_TEXT_SIZE, fw->init.textsz); iwn_nic_unlock(sc); /* Load firmware boot code. */ error = iwn4965_load_bootcode(sc, fw->boot.text, fw->boot.textsz); if (error != 0) { device_printf(sc->sc_dev, "%s: could not load boot firmware\n", __func__); return error; } /* Now press "execute". */ IWN_WRITE(sc, IWN_RESET, 0); /* Wait at most one second for first alive notification. */ if ((error = msleep(sc, &sc->sc_mtx, PCATCH, "iwninit", hz)) != 0) { device_printf(sc->sc_dev, "%s: timeout waiting for adapter to initialize, error %d\n", __func__, error); return error; } /* Retrieve current temperature for initial TX power calibration. */ sc->rawtemp = sc->ucode_info.temp[3].chan20MHz; sc->temp = iwn4965_get_temperature(sc); /* Copy runtime sections into pre-allocated DMA-safe memory. */ memcpy(dma->vaddr, fw->main.data, fw->main.datasz); bus_dmamap_sync(dma->tag, dma->map, BUS_DMASYNC_PREWRITE); memcpy(dma->vaddr + IWN4965_FW_DATA_MAXSZ, fw->main.text, fw->main.textsz); bus_dmamap_sync(dma->tag, dma->map, BUS_DMASYNC_PREWRITE); /* Tell adapter where to find runtime sections. */ if ((error = iwn_nic_lock(sc)) != 0) return error; iwn_prph_write(sc, IWN_BSM_DRAM_DATA_ADDR, dma->paddr >> 4); iwn_prph_write(sc, IWN_BSM_DRAM_DATA_SIZE, fw->main.datasz); iwn_prph_write(sc, IWN_BSM_DRAM_TEXT_ADDR, (dma->paddr + IWN4965_FW_DATA_MAXSZ) >> 4); iwn_prph_write(sc, IWN_BSM_DRAM_TEXT_SIZE, IWN_FW_UPDATED | fw->main.textsz); iwn_nic_unlock(sc); return 0; } static int iwn5000_load_firmware_section(struct iwn_softc *sc, uint32_t dst, const uint8_t *section, int size) { struct iwn_dma_info *dma = &sc->fw_dma; int error; DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); /* Copy firmware section into pre-allocated DMA-safe memory. */ memcpy(dma->vaddr, section, size); bus_dmamap_sync(dma->tag, dma->map, BUS_DMASYNC_PREWRITE); if ((error = iwn_nic_lock(sc)) != 0) return error; IWN_WRITE(sc, IWN_FH_TX_CONFIG(IWN_SRVC_DMACHNL), IWN_FH_TX_CONFIG_DMA_PAUSE); IWN_WRITE(sc, IWN_FH_SRAM_ADDR(IWN_SRVC_DMACHNL), dst); IWN_WRITE(sc, IWN_FH_TFBD_CTRL0(IWN_SRVC_DMACHNL), IWN_LOADDR(dma->paddr)); IWN_WRITE(sc, IWN_FH_TFBD_CTRL1(IWN_SRVC_DMACHNL), IWN_HIADDR(dma->paddr) << 28 | size); IWN_WRITE(sc, IWN_FH_TXBUF_STATUS(IWN_SRVC_DMACHNL), IWN_FH_TXBUF_STATUS_TBNUM(1) | IWN_FH_TXBUF_STATUS_TBIDX(1) | IWN_FH_TXBUF_STATUS_TFBD_VALID); /* Kick Flow Handler to start DMA transfer. */ IWN_WRITE(sc, IWN_FH_TX_CONFIG(IWN_SRVC_DMACHNL), IWN_FH_TX_CONFIG_DMA_ENA | IWN_FH_TX_CONFIG_CIRQ_HOST_ENDTFD); iwn_nic_unlock(sc); /* Wait at most five seconds for FH DMA transfer to complete. */ return msleep(sc, &sc->sc_mtx, PCATCH, "iwninit", 5 * hz); } static int iwn5000_load_firmware(struct iwn_softc *sc) { struct iwn_fw_part *fw; int error; DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); /* Load the initialization firmware on first boot only. */ fw = (sc->sc_flags & IWN_FLAG_CALIB_DONE) ? &sc->fw.main : &sc->fw.init; error = iwn5000_load_firmware_section(sc, IWN_FW_TEXT_BASE, fw->text, fw->textsz); if (error != 0) { device_printf(sc->sc_dev, "%s: could not load firmware %s section, error %d\n", __func__, ".text", error); return error; } error = iwn5000_load_firmware_section(sc, IWN_FW_DATA_BASE, fw->data, fw->datasz); if (error != 0) { device_printf(sc->sc_dev, "%s: could not load firmware %s section, error %d\n", __func__, ".data", error); return error; } /* Now press "execute". */ IWN_WRITE(sc, IWN_RESET, 0); return 0; } /* * Extract text and data sections from a legacy firmware image. */ static int iwn_read_firmware_leg(struct iwn_softc *sc, struct iwn_fw_info *fw) { const uint32_t *ptr; size_t hdrlen = 24; uint32_t rev; ptr = (const uint32_t *)fw->data; rev = le32toh(*ptr++); sc->ucode_rev = rev; /* Check firmware API version. */ if (IWN_FW_API(rev) <= 1) { device_printf(sc->sc_dev, "%s: bad firmware, need API version >=2\n", __func__); return EINVAL; } if (IWN_FW_API(rev) >= 3) { /* Skip build number (version 2 header). */ hdrlen += 4; ptr++; } if (fw->size < hdrlen) { device_printf(sc->sc_dev, "%s: firmware too short: %zu bytes\n", __func__, fw->size); return EINVAL; } fw->main.textsz = le32toh(*ptr++); fw->main.datasz = le32toh(*ptr++); fw->init.textsz = le32toh(*ptr++); fw->init.datasz = le32toh(*ptr++); fw->boot.textsz = le32toh(*ptr++); /* Check that all firmware sections fit. */ if (fw->size < hdrlen + fw->main.textsz + fw->main.datasz + fw->init.textsz + fw->init.datasz + fw->boot.textsz) { device_printf(sc->sc_dev, "%s: firmware too short: %zu bytes\n", __func__, fw->size); return EINVAL; } /* Get pointers to firmware sections. */ fw->main.text = (const uint8_t *)ptr; fw->main.data = fw->main.text + fw->main.textsz; fw->init.text = fw->main.data + fw->main.datasz; fw->init.data = fw->init.text + fw->init.textsz; fw->boot.text = fw->init.data + fw->init.datasz; return 0; } /* * Extract text and data sections from a TLV firmware image. */ static int iwn_read_firmware_tlv(struct iwn_softc *sc, struct iwn_fw_info *fw, uint16_t alt) { const struct iwn_fw_tlv_hdr *hdr; const struct iwn_fw_tlv *tlv; const uint8_t *ptr, *end; uint64_t altmask; uint32_t len, tmp; if (fw->size < sizeof (*hdr)) { device_printf(sc->sc_dev, "%s: firmware too short: %zu bytes\n", __func__, fw->size); return EINVAL; } hdr = (const struct iwn_fw_tlv_hdr *)fw->data; if (hdr->signature != htole32(IWN_FW_SIGNATURE)) { device_printf(sc->sc_dev, "%s: bad firmware signature 0x%08x\n", __func__, le32toh(hdr->signature)); return EINVAL; } DPRINTF(sc, IWN_DEBUG_RESET, "FW: \"%.64s\", build 0x%x\n", hdr->descr, le32toh(hdr->build)); sc->ucode_rev = le32toh(hdr->rev); /* * Select the closest supported alternative that is less than * or equal to the specified one. */ altmask = le64toh(hdr->altmask); while (alt > 0 && !(altmask & (1ULL << alt))) alt--; /* Downgrade. */ DPRINTF(sc, IWN_DEBUG_RESET, "using alternative %d\n", alt); ptr = (const uint8_t *)(hdr + 1); end = (const uint8_t *)(fw->data + fw->size); /* Parse type-length-value fields. */ while (ptr + sizeof (*tlv) <= end) { tlv = (const struct iwn_fw_tlv *)ptr; len = le32toh(tlv->len); ptr += sizeof (*tlv); if (ptr + len > end) { device_printf(sc->sc_dev, "%s: firmware too short: %zu bytes\n", __func__, fw->size); return EINVAL; } /* Skip other alternatives. */ if (tlv->alt != 0 && tlv->alt != htole16(alt)) goto next; switch (le16toh(tlv->type)) { case IWN_FW_TLV_MAIN_TEXT: fw->main.text = ptr; fw->main.textsz = len; break; case IWN_FW_TLV_MAIN_DATA: fw->main.data = ptr; fw->main.datasz = len; break; case IWN_FW_TLV_INIT_TEXT: fw->init.text = ptr; fw->init.textsz = len; break; case IWN_FW_TLV_INIT_DATA: fw->init.data = ptr; fw->init.datasz = len; break; case IWN_FW_TLV_BOOT_TEXT: fw->boot.text = ptr; fw->boot.textsz = len; break; case IWN_FW_TLV_ENH_SENS: if (!len) sc->sc_flags |= IWN_FLAG_ENH_SENS; break; case IWN_FW_TLV_PHY_CALIB: tmp = le32toh(*ptr); if (tmp < 253) { sc->reset_noise_gain = tmp; sc->noise_gain = tmp + 1; } break; case IWN_FW_TLV_PAN: sc->sc_flags |= IWN_FLAG_PAN_SUPPORT; DPRINTF(sc, IWN_DEBUG_RESET, "PAN Support found: %d\n", 1); break; case IWN_FW_TLV_FLAGS: if (len < sizeof(uint32_t)) break; if (len % sizeof(uint32_t)) break; sc->tlv_feature_flags = le32toh(*ptr); DPRINTF(sc, IWN_DEBUG_RESET, "%s: feature: 0x%08x\n", __func__, sc->tlv_feature_flags); break; case IWN_FW_TLV_PBREQ_MAXLEN: case IWN_FW_TLV_RUNT_EVTLOG_PTR: case IWN_FW_TLV_RUNT_EVTLOG_SIZE: case IWN_FW_TLV_RUNT_ERRLOG_PTR: case IWN_FW_TLV_INIT_EVTLOG_PTR: case IWN_FW_TLV_INIT_EVTLOG_SIZE: case IWN_FW_TLV_INIT_ERRLOG_PTR: case IWN_FW_TLV_WOWLAN_INST: case IWN_FW_TLV_WOWLAN_DATA: DPRINTF(sc, IWN_DEBUG_RESET, "TLV type %d recognized but not handled\n", le16toh(tlv->type)); break; default: DPRINTF(sc, IWN_DEBUG_RESET, "TLV type %d not handled\n", le16toh(tlv->type)); break; } next: /* TLV fields are 32-bit aligned. */ ptr += (len + 3) & ~3; } return 0; } static int iwn_read_firmware(struct iwn_softc *sc) { struct iwn_fw_info *fw = &sc->fw; int error; DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); IWN_UNLOCK(sc); memset(fw, 0, sizeof (*fw)); /* Read firmware image from filesystem. */ sc->fw_fp = firmware_get(sc->fwname); if (sc->fw_fp == NULL) { device_printf(sc->sc_dev, "%s: could not read firmware %s\n", __func__, sc->fwname); IWN_LOCK(sc); return EINVAL; } IWN_LOCK(sc); fw->size = sc->fw_fp->datasize; fw->data = (const uint8_t *)sc->fw_fp->data; if (fw->size < sizeof (uint32_t)) { device_printf(sc->sc_dev, "%s: firmware too short: %zu bytes\n", __func__, fw->size); error = EINVAL; goto fail; } /* Retrieve text and data sections. */ if (*(const uint32_t *)fw->data != 0) /* Legacy image. */ error = iwn_read_firmware_leg(sc, fw); else error = iwn_read_firmware_tlv(sc, fw, 1); if (error != 0) { device_printf(sc->sc_dev, "%s: could not read firmware sections, error %d\n", __func__, error); goto fail; } device_printf(sc->sc_dev, "%s: ucode rev=0x%08x\n", __func__, sc->ucode_rev); /* Make sure text and data sections fit in hardware memory. */ if (fw->main.textsz > sc->fw_text_maxsz || fw->main.datasz > sc->fw_data_maxsz || fw->init.textsz > sc->fw_text_maxsz || fw->init.datasz > sc->fw_data_maxsz || fw->boot.textsz > IWN_FW_BOOT_TEXT_MAXSZ || (fw->boot.textsz & 3) != 0) { device_printf(sc->sc_dev, "%s: firmware sections too large\n", __func__); error = EINVAL; goto fail; } /* We can proceed with loading the firmware. */ return 0; fail: iwn_unload_firmware(sc); return error; } static void iwn_unload_firmware(struct iwn_softc *sc) { firmware_put(sc->fw_fp, FIRMWARE_UNLOAD); sc->fw_fp = NULL; } static int iwn_clock_wait(struct iwn_softc *sc) { int ntries; /* Set "initialization complete" bit. */ IWN_SETBITS(sc, IWN_GP_CNTRL, IWN_GP_CNTRL_INIT_DONE); /* Wait for clock stabilization. */ for (ntries = 0; ntries < 2500; ntries++) { if (IWN_READ(sc, IWN_GP_CNTRL) & IWN_GP_CNTRL_MAC_CLOCK_READY) return 0; DELAY(10); } device_printf(sc->sc_dev, "%s: timeout waiting for clock stabilization\n", __func__); return ETIMEDOUT; } static int iwn_apm_init(struct iwn_softc *sc) { uint32_t reg; int error; DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); /* Disable L0s exit timer (NMI bug workaround). */ IWN_SETBITS(sc, IWN_GIO_CHICKEN, IWN_GIO_CHICKEN_DIS_L0S_TIMER); /* Don't wait for ICH L0s (ICH bug workaround). */ IWN_SETBITS(sc, IWN_GIO_CHICKEN, IWN_GIO_CHICKEN_L1A_NO_L0S_RX); /* Set FH wait threshold to max (HW bug under stress workaround). */ IWN_SETBITS(sc, IWN_DBG_HPET_MEM, 0xffff0000); /* Enable HAP INTA to move adapter from L1a to L0s. */ IWN_SETBITS(sc, IWN_HW_IF_CONFIG, IWN_HW_IF_CONFIG_HAP_WAKE_L1A); /* Retrieve PCIe Active State Power Management (ASPM). */ reg = pci_read_config(sc->sc_dev, sc->sc_cap_off + PCIER_LINK_CTL, 4); /* Workaround for HW instability in PCIe L0->L0s->L1 transition. */ if (reg & PCIEM_LINK_CTL_ASPMC_L1) /* L1 Entry enabled. */ IWN_SETBITS(sc, IWN_GIO, IWN_GIO_L0S_ENA); else IWN_CLRBITS(sc, IWN_GIO, IWN_GIO_L0S_ENA); if (sc->base_params->pll_cfg_val) IWN_SETBITS(sc, IWN_ANA_PLL, sc->base_params->pll_cfg_val); /* Wait for clock stabilization before accessing prph. */ if ((error = iwn_clock_wait(sc)) != 0) return error; if ((error = iwn_nic_lock(sc)) != 0) return error; if (sc->hw_type == IWN_HW_REV_TYPE_4965) { /* Enable DMA and BSM (Bootstrap State Machine). */ iwn_prph_write(sc, IWN_APMG_CLK_EN, IWN_APMG_CLK_CTRL_DMA_CLK_RQT | IWN_APMG_CLK_CTRL_BSM_CLK_RQT); } else { /* Enable DMA. */ iwn_prph_write(sc, IWN_APMG_CLK_EN, IWN_APMG_CLK_CTRL_DMA_CLK_RQT); } DELAY(20); /* Disable L1-Active. */ iwn_prph_setbits(sc, IWN_APMG_PCI_STT, IWN_APMG_PCI_STT_L1A_DIS); iwn_nic_unlock(sc); return 0; } static void iwn_apm_stop_master(struct iwn_softc *sc) { int ntries; /* Stop busmaster DMA activity. */ IWN_SETBITS(sc, IWN_RESET, IWN_RESET_STOP_MASTER); for (ntries = 0; ntries < 100; ntries++) { if (IWN_READ(sc, IWN_RESET) & IWN_RESET_MASTER_DISABLED) return; DELAY(10); } device_printf(sc->sc_dev, "%s: timeout waiting for master\n", __func__); } static void iwn_apm_stop(struct iwn_softc *sc) { iwn_apm_stop_master(sc); /* Reset the entire device. */ IWN_SETBITS(sc, IWN_RESET, IWN_RESET_SW); DELAY(10); /* Clear "initialization complete" bit. */ IWN_CLRBITS(sc, IWN_GP_CNTRL, IWN_GP_CNTRL_INIT_DONE); } static int iwn4965_nic_config(struct iwn_softc *sc) { DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); if (IWN_RFCFG_TYPE(sc->rfcfg) == 1) { /* * I don't believe this to be correct but this is what the * vendor driver is doing. Probably the bits should not be * shifted in IWN_RFCFG_*. */ IWN_SETBITS(sc, IWN_HW_IF_CONFIG, IWN_RFCFG_TYPE(sc->rfcfg) | IWN_RFCFG_STEP(sc->rfcfg) | IWN_RFCFG_DASH(sc->rfcfg)); } IWN_SETBITS(sc, IWN_HW_IF_CONFIG, IWN_HW_IF_CONFIG_RADIO_SI | IWN_HW_IF_CONFIG_MAC_SI); return 0; } static int iwn5000_nic_config(struct iwn_softc *sc) { uint32_t tmp; int error; DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); if (IWN_RFCFG_TYPE(sc->rfcfg) < 3) { IWN_SETBITS(sc, IWN_HW_IF_CONFIG, IWN_RFCFG_TYPE(sc->rfcfg) | IWN_RFCFG_STEP(sc->rfcfg) | IWN_RFCFG_DASH(sc->rfcfg)); } IWN_SETBITS(sc, IWN_HW_IF_CONFIG, IWN_HW_IF_CONFIG_RADIO_SI | IWN_HW_IF_CONFIG_MAC_SI); if ((error = iwn_nic_lock(sc)) != 0) return error; iwn_prph_setbits(sc, IWN_APMG_PS, IWN_APMG_PS_EARLY_PWROFF_DIS); if (sc->hw_type == IWN_HW_REV_TYPE_1000) { /* * Select first Switching Voltage Regulator (1.32V) to * solve a stability issue related to noisy DC2DC line * in the silicon of 1000 Series. */ tmp = iwn_prph_read(sc, IWN_APMG_DIGITAL_SVR); tmp &= ~IWN_APMG_DIGITAL_SVR_VOLTAGE_MASK; tmp |= IWN_APMG_DIGITAL_SVR_VOLTAGE_1_32; iwn_prph_write(sc, IWN_APMG_DIGITAL_SVR, tmp); } iwn_nic_unlock(sc); if (sc->sc_flags & IWN_FLAG_INTERNAL_PA) { /* Use internal power amplifier only. */ IWN_WRITE(sc, IWN_GP_DRIVER, IWN_GP_DRIVER_RADIO_2X2_IPA); } if (sc->base_params->additional_nic_config && sc->calib_ver >= 6) { /* Indicate that ROM calibration version is >=6. */ IWN_SETBITS(sc, IWN_GP_DRIVER, IWN_GP_DRIVER_CALIB_VER6); } if (sc->base_params->additional_gp_drv_bit) IWN_SETBITS(sc, IWN_GP_DRIVER, sc->base_params->additional_gp_drv_bit); return 0; } /* * Take NIC ownership over Intel Active Management Technology (AMT). */ static int iwn_hw_prepare(struct iwn_softc *sc) { int ntries; DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); /* Check if hardware is ready. */ IWN_SETBITS(sc, IWN_HW_IF_CONFIG, IWN_HW_IF_CONFIG_NIC_READY); for (ntries = 0; ntries < 5; ntries++) { if (IWN_READ(sc, IWN_HW_IF_CONFIG) & IWN_HW_IF_CONFIG_NIC_READY) return 0; DELAY(10); } /* Hardware not ready, force into ready state. */ IWN_SETBITS(sc, IWN_HW_IF_CONFIG, IWN_HW_IF_CONFIG_PREPARE); for (ntries = 0; ntries < 15000; ntries++) { if (!(IWN_READ(sc, IWN_HW_IF_CONFIG) & IWN_HW_IF_CONFIG_PREPARE_DONE)) break; DELAY(10); } if (ntries == 15000) return ETIMEDOUT; /* Hardware should be ready now. */ IWN_SETBITS(sc, IWN_HW_IF_CONFIG, IWN_HW_IF_CONFIG_NIC_READY); for (ntries = 0; ntries < 5; ntries++) { if (IWN_READ(sc, IWN_HW_IF_CONFIG) & IWN_HW_IF_CONFIG_NIC_READY) return 0; DELAY(10); } return ETIMEDOUT; } static int iwn_hw_init(struct iwn_softc *sc) { struct iwn_ops *ops = &sc->ops; int error, chnl, qid; DPRINTF(sc, IWN_DEBUG_TRACE, "->%s begin\n", __func__); /* Clear pending interrupts. */ IWN_WRITE(sc, IWN_INT, 0xffffffff); if ((error = iwn_apm_init(sc)) != 0) { device_printf(sc->sc_dev, "%s: could not power ON adapter, error %d\n", __func__, error); return error; } /* Select VMAIN power source. */ if ((error = iwn_nic_lock(sc)) != 0) return error; iwn_prph_clrbits(sc, IWN_APMG_PS, IWN_APMG_PS_PWR_SRC_MASK); iwn_nic_unlock(sc); /* Perform adapter-specific initialization. */ if ((error = ops->nic_config(sc)) != 0) return error; /* Initialize RX ring. */ if ((error = iwn_nic_lock(sc)) != 0) return error; IWN_WRITE(sc, IWN_FH_RX_CONFIG, 0); IWN_WRITE(sc, IWN_FH_RX_WPTR, 0); /* Set physical address of RX ring (256-byte aligned). */ IWN_WRITE(sc, IWN_FH_RX_BASE, sc->rxq.desc_dma.paddr >> 8); /* Set physical address of RX status (16-byte aligned). */ IWN_WRITE(sc, IWN_FH_STATUS_WPTR, sc->rxq.stat_dma.paddr >> 4); /* Enable RX. */ IWN_WRITE(sc, IWN_FH_RX_CONFIG, IWN_FH_RX_CONFIG_ENA | IWN_FH_RX_CONFIG_IGN_RXF_EMPTY | /* HW bug workaround */ IWN_FH_RX_CONFIG_IRQ_DST_HOST | IWN_FH_RX_CONFIG_SINGLE_FRAME | IWN_FH_RX_CONFIG_RB_TIMEOUT(0) | IWN_FH_RX_CONFIG_NRBD(IWN_RX_RING_COUNT_LOG)); iwn_nic_unlock(sc); IWN_WRITE(sc, IWN_FH_RX_WPTR, (IWN_RX_RING_COUNT - 1) & ~7); if ((error = iwn_nic_lock(sc)) != 0) return error; /* Initialize TX scheduler. */ iwn_prph_write(sc, sc->sched_txfact_addr, 0); /* Set physical address of "keep warm" page (16-byte aligned). */ IWN_WRITE(sc, IWN_FH_KW_ADDR, sc->kw_dma.paddr >> 4); /* Initialize TX rings. */ for (qid = 0; qid < sc->ntxqs; qid++) { struct iwn_tx_ring *txq = &sc->txq[qid]; /* Set physical address of TX ring (256-byte aligned). */ IWN_WRITE(sc, IWN_FH_CBBC_QUEUE(qid), txq->desc_dma.paddr >> 8); } iwn_nic_unlock(sc); /* Enable DMA channels. */ for (chnl = 0; chnl < sc->ndmachnls; chnl++) { IWN_WRITE(sc, IWN_FH_TX_CONFIG(chnl), IWN_FH_TX_CONFIG_DMA_ENA | IWN_FH_TX_CONFIG_DMA_CREDIT_ENA); } /* Clear "radio off" and "commands blocked" bits. */ IWN_WRITE(sc, IWN_UCODE_GP1_CLR, IWN_UCODE_GP1_RFKILL); IWN_WRITE(sc, IWN_UCODE_GP1_CLR, IWN_UCODE_GP1_CMD_BLOCKED); /* Clear pending interrupts. */ IWN_WRITE(sc, IWN_INT, 0xffffffff); /* Enable interrupt coalescing. */ IWN_WRITE(sc, IWN_INT_COALESCING, 512 / 8); /* Enable interrupts. */ IWN_WRITE(sc, IWN_INT_MASK, sc->int_mask); /* _Really_ make sure "radio off" bit is cleared! */ IWN_WRITE(sc, IWN_UCODE_GP1_CLR, IWN_UCODE_GP1_RFKILL); IWN_WRITE(sc, IWN_UCODE_GP1_CLR, IWN_UCODE_GP1_RFKILL); /* Enable shadow registers. */ if (sc->base_params->shadow_reg_enable) IWN_SETBITS(sc, IWN_SHADOW_REG_CTRL, 0x800fffff); if ((error = ops->load_firmware(sc)) != 0) { device_printf(sc->sc_dev, "%s: could not load firmware, error %d\n", __func__, error); return error; } /* Wait at most one second for firmware alive notification. */ if ((error = msleep(sc, &sc->sc_mtx, PCATCH, "iwninit", hz)) != 0) { device_printf(sc->sc_dev, "%s: timeout waiting for adapter to initialize, error %d\n", __func__, error); return error; } /* Do post-firmware initialization. */ DPRINTF(sc, IWN_DEBUG_TRACE, "->%s: end\n",__func__); return ops->post_alive(sc); } static void iwn_hw_stop(struct iwn_softc *sc) { int chnl, qid, ntries; DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); IWN_WRITE(sc, IWN_RESET, IWN_RESET_NEVO); /* Disable interrupts. */ IWN_WRITE(sc, IWN_INT_MASK, 0); IWN_WRITE(sc, IWN_INT, 0xffffffff); IWN_WRITE(sc, IWN_FH_INT, 0xffffffff); sc->sc_flags &= ~IWN_FLAG_USE_ICT; /* Make sure we no longer hold the NIC lock. */ iwn_nic_unlock(sc); /* Stop TX scheduler. */ iwn_prph_write(sc, sc->sched_txfact_addr, 0); /* Stop all DMA channels. */ if (iwn_nic_lock(sc) == 0) { for (chnl = 0; chnl < sc->ndmachnls; chnl++) { IWN_WRITE(sc, IWN_FH_TX_CONFIG(chnl), 0); for (ntries = 0; ntries < 200; ntries++) { if (IWN_READ(sc, IWN_FH_TX_STATUS) & IWN_FH_TX_STATUS_IDLE(chnl)) break; DELAY(10); } } iwn_nic_unlock(sc); } /* Stop RX ring. */ iwn_reset_rx_ring(sc, &sc->rxq); /* Reset all TX rings. */ for (qid = 0; qid < sc->ntxqs; qid++) iwn_reset_tx_ring(sc, &sc->txq[qid]); if (iwn_nic_lock(sc) == 0) { iwn_prph_write(sc, IWN_APMG_CLK_DIS, IWN_APMG_CLK_CTRL_DMA_CLK_RQT); iwn_nic_unlock(sc); } DELAY(5); /* Power OFF adapter. */ iwn_apm_stop(sc); } static void iwn_radio_on(void *arg0, int pending) { struct iwn_softc *sc = arg0; struct ieee80211com *ic = &sc->sc_ic; struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps); DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); if (vap != NULL) { iwn_init(sc); ieee80211_init(vap); } } static void iwn_radio_off(void *arg0, int pending) { struct iwn_softc *sc = arg0; struct ieee80211com *ic = &sc->sc_ic; struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps); DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); iwn_stop(sc); if (vap != NULL) ieee80211_stop(vap); /* Enable interrupts to get RF toggle notification. */ IWN_LOCK(sc); IWN_WRITE(sc, IWN_INT, 0xffffffff); IWN_WRITE(sc, IWN_INT_MASK, sc->int_mask); IWN_UNLOCK(sc); } static void iwn_panicked(void *arg0, int pending) { struct iwn_softc *sc = arg0; struct ieee80211com *ic = &sc->sc_ic; struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps); #if 0 int error; #endif if (vap == NULL) { printf("%s: null vap\n", __func__); return; } device_printf(sc->sc_dev, "%s: controller panicked, iv_state = %d; " "restarting\n", __func__, vap->iv_state); /* * This is not enough work. We need to also reinitialise * the correct transmit state for aggregation enabled queues, * which has a very specific requirement of * ring index = 802.11 seqno % 256. If we don't do this (which * we definitely don't!) then the firmware will just panic again. */ #if 1 ieee80211_restart_all(ic); #else IWN_LOCK(sc); iwn_stop_locked(sc); iwn_init_locked(sc); if (vap->iv_state >= IEEE80211_S_AUTH && (error = iwn_auth(sc, vap)) != 0) { device_printf(sc->sc_dev, "%s: could not move to auth state\n", __func__); } if (vap->iv_state >= IEEE80211_S_RUN && (error = iwn_run(sc, vap)) != 0) { device_printf(sc->sc_dev, "%s: could not move to run state\n", __func__); } IWN_UNLOCK(sc); #endif } static void iwn_init_locked(struct iwn_softc *sc) { int error; DPRINTF(sc, IWN_DEBUG_TRACE, "->%s begin\n", __func__); IWN_LOCK_ASSERT(sc); sc->sc_flags |= IWN_FLAG_RUNNING; if ((error = iwn_hw_prepare(sc)) != 0) { device_printf(sc->sc_dev, "%s: hardware not ready, error %d\n", __func__, error); goto fail; } /* Initialize interrupt mask to default value. */ sc->int_mask = IWN_INT_MASK_DEF; sc->sc_flags &= ~IWN_FLAG_USE_ICT; /* Check that the radio is not disabled by hardware switch. */ if (!(IWN_READ(sc, IWN_GP_CNTRL) & IWN_GP_CNTRL_RFKILL)) { device_printf(sc->sc_dev, "radio is disabled by hardware switch\n"); /* Enable interrupts to get RF toggle notifications. */ IWN_WRITE(sc, IWN_INT, 0xffffffff); IWN_WRITE(sc, IWN_INT_MASK, sc->int_mask); return; } /* Read firmware images from the filesystem. */ if ((error = iwn_read_firmware(sc)) != 0) { device_printf(sc->sc_dev, "%s: could not read firmware, error %d\n", __func__, error); goto fail; } /* Initialize hardware and upload firmware. */ error = iwn_hw_init(sc); iwn_unload_firmware(sc); if (error != 0) { device_printf(sc->sc_dev, "%s: could not initialize hardware, error %d\n", __func__, error); goto fail; } /* Configure adapter now that it is ready. */ if ((error = iwn_config(sc)) != 0) { device_printf(sc->sc_dev, "%s: could not configure device, error %d\n", __func__, error); goto fail; } callout_reset(&sc->watchdog_to, hz, iwn_watchdog, sc); DPRINTF(sc, IWN_DEBUG_TRACE, "->%s: end\n",__func__); return; fail: sc->sc_flags &= ~IWN_FLAG_RUNNING; iwn_stop_locked(sc); DPRINTF(sc, IWN_DEBUG_TRACE, "->%s: end in error\n",__func__); } static void iwn_init(struct iwn_softc *sc) { IWN_LOCK(sc); iwn_init_locked(sc); IWN_UNLOCK(sc); if (sc->sc_flags & IWN_FLAG_RUNNING) ieee80211_start_all(&sc->sc_ic); } static void iwn_stop_locked(struct iwn_softc *sc) { IWN_LOCK_ASSERT(sc); sc->sc_is_scanning = 0; sc->sc_tx_timer = 0; callout_stop(&sc->watchdog_to); callout_stop(&sc->calib_to); sc->sc_flags &= ~IWN_FLAG_RUNNING; /* Power OFF hardware. */ iwn_hw_stop(sc); } static void iwn_stop(struct iwn_softc *sc) { IWN_LOCK(sc); iwn_stop_locked(sc); IWN_UNLOCK(sc); } /* * Callback from net80211 to start a scan. */ static void iwn_scan_start(struct ieee80211com *ic) { struct iwn_softc *sc = ic->ic_softc; IWN_LOCK(sc); /* make the link LED blink while we're scanning */ iwn_set_led(sc, IWN_LED_LINK, 20, 2); IWN_UNLOCK(sc); } /* * Callback from net80211 to terminate a scan. */ static void iwn_scan_end(struct ieee80211com *ic) { struct iwn_softc *sc = ic->ic_softc; struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps); IWN_LOCK(sc); if (vap->iv_state == IEEE80211_S_RUN) { /* Set link LED to ON status if we are associated */ iwn_set_led(sc, IWN_LED_LINK, 0, 1); } IWN_UNLOCK(sc); } /* * Callback from net80211 to force a channel change. */ static void iwn_set_channel(struct ieee80211com *ic) { const struct ieee80211_channel *c = ic->ic_curchan; struct iwn_softc *sc = ic->ic_softc; int error; DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); IWN_LOCK(sc); sc->sc_rxtap.wr_chan_freq = htole16(c->ic_freq); sc->sc_rxtap.wr_chan_flags = htole16(c->ic_flags); sc->sc_txtap.wt_chan_freq = htole16(c->ic_freq); sc->sc_txtap.wt_chan_flags = htole16(c->ic_flags); /* * Only need to set the channel in Monitor mode. AP scanning and auth * are already taken care of by their respective firmware commands. */ if (ic->ic_opmode == IEEE80211_M_MONITOR) { error = iwn_config(sc); if (error != 0) device_printf(sc->sc_dev, "%s: error %d settting channel\n", __func__, error); } IWN_UNLOCK(sc); } /* * Callback from net80211 to start scanning of the current channel. */ static void iwn_scan_curchan(struct ieee80211_scan_state *ss, unsigned long maxdwell) { struct ieee80211vap *vap = ss->ss_vap; struct ieee80211com *ic = vap->iv_ic; struct iwn_softc *sc = ic->ic_softc; int error; IWN_LOCK(sc); error = iwn_scan(sc, vap, ss, ic->ic_curchan); IWN_UNLOCK(sc); if (error != 0) ieee80211_cancel_scan(vap); } /* * Callback from net80211 to handle the minimum dwell time being met. * The intent is to terminate the scan but we just let the firmware * notify us when it's finished as we have no safe way to abort it. */ static void iwn_scan_mindwell(struct ieee80211_scan_state *ss) { /* NB: don't try to abort scan; wait for firmware to finish */ -} - -static void -iwn_hw_reset(void *arg0, int pending) -{ - struct iwn_softc *sc = arg0; - struct ieee80211com *ic = &sc->sc_ic; - - DPRINTF(sc, IWN_DEBUG_TRACE, "->Doing %s\n", __func__); - - iwn_stop(sc); - iwn_init(sc); - ieee80211_notify_radio(ic, 1); } #ifdef IWN_DEBUG #define IWN_DESC(x) case x: return #x /* * Translate CSR code to string */ static char *iwn_get_csr_string(int csr) { switch (csr) { IWN_DESC(IWN_HW_IF_CONFIG); IWN_DESC(IWN_INT_COALESCING); IWN_DESC(IWN_INT); IWN_DESC(IWN_INT_MASK); IWN_DESC(IWN_FH_INT); IWN_DESC(IWN_GPIO_IN); IWN_DESC(IWN_RESET); IWN_DESC(IWN_GP_CNTRL); IWN_DESC(IWN_HW_REV); IWN_DESC(IWN_EEPROM); IWN_DESC(IWN_EEPROM_GP); IWN_DESC(IWN_OTP_GP); IWN_DESC(IWN_GIO); IWN_DESC(IWN_GP_UCODE); IWN_DESC(IWN_GP_DRIVER); IWN_DESC(IWN_UCODE_GP1); IWN_DESC(IWN_UCODE_GP2); IWN_DESC(IWN_LED); IWN_DESC(IWN_DRAM_INT_TBL); IWN_DESC(IWN_GIO_CHICKEN); IWN_DESC(IWN_ANA_PLL); IWN_DESC(IWN_HW_REV_WA); IWN_DESC(IWN_DBG_HPET_MEM); default: return "UNKNOWN CSR"; } } /* * This function print firmware register */ static void iwn_debug_register(struct iwn_softc *sc) { int i; static const uint32_t csr_tbl[] = { IWN_HW_IF_CONFIG, IWN_INT_COALESCING, IWN_INT, IWN_INT_MASK, IWN_FH_INT, IWN_GPIO_IN, IWN_RESET, IWN_GP_CNTRL, IWN_HW_REV, IWN_EEPROM, IWN_EEPROM_GP, IWN_OTP_GP, IWN_GIO, IWN_GP_UCODE, IWN_GP_DRIVER, IWN_UCODE_GP1, IWN_UCODE_GP2, IWN_LED, IWN_DRAM_INT_TBL, IWN_GIO_CHICKEN, IWN_ANA_PLL, IWN_HW_REV_WA, IWN_DBG_HPET_MEM, }; DPRINTF(sc, IWN_DEBUG_REGISTER, "CSR values: (2nd byte of IWN_INT_COALESCING is IWN_INT_PERIODIC)%s", "\n"); for (i = 0; i < nitems(csr_tbl); i++){ DPRINTF(sc, IWN_DEBUG_REGISTER," %10s: 0x%08x ", iwn_get_csr_string(csr_tbl[i]), IWN_READ(sc, csr_tbl[i])); if ((i+1) % 3 == 0) DPRINTF(sc, IWN_DEBUG_REGISTER,"%s","\n"); } DPRINTF(sc, IWN_DEBUG_REGISTER,"%s","\n"); } #endif Index: user/ngie/release-pkg-fix-tests/sys/dev/iwn/if_iwnvar.h =================================================================== --- user/ngie/release-pkg-fix-tests/sys/dev/iwn/if_iwnvar.h (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/dev/iwn/if_iwnvar.h (revision 299091) @@ -1,437 +1,436 @@ /* $FreeBSD$ */ /* $OpenBSD: if_iwnvar.h,v 1.18 2010/04/30 16:06:46 damien Exp $ */ /*- * Copyright (c) 2013 Cedric GROSS * Copyright (c) 2011 Intel Corporation * Copyright (c) 2007, 2008 * Damien Bergamini * Copyright (c) 2008 Sam Leffler, Errno Consulting * * Permission to use, copy, modify, and distribute this software for any * purpose with or without fee is hereby granted, provided that the above * copyright notice and this permission notice appear in all copies. * * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ enum iwn_rxon_ctx_id { IWN_RXON_BSS_CTX, IWN_RXON_PAN_CTX, IWN_NUM_RXON_CTX }; struct iwn_pan_slot { uint16_t time; uint8_t type; uint8_t reserved; } __packed; struct iwn_pan_params_cmd { uint16_t flags; #define IWN_PAN_PARAMS_FLG_SLOTTED_MODE (1 << 3) uint8_t reserved; uint8_t num_slots; struct iwn_pan_slot slots[10]; } __packed; struct iwn_led_mode { uint8_t led_cur_mode; uint64_t led_cur_bt; uint64_t led_last_bt; uint64_t led_cur_tpt; uint64_t led_last_tpt; uint64_t led_bt_diff; int led_cur_time; int led_last_time; }; struct iwn_rx_radiotap_header { struct ieee80211_radiotap_header wr_ihdr; uint64_t wr_tsft; uint8_t wr_flags; uint8_t wr_rate; uint16_t wr_chan_freq; uint16_t wr_chan_flags; int8_t wr_dbm_antsignal; int8_t wr_dbm_antnoise; } __packed; #define IWN_RX_RADIOTAP_PRESENT \ ((1 << IEEE80211_RADIOTAP_TSFT) | \ (1 << IEEE80211_RADIOTAP_FLAGS) | \ (1 << IEEE80211_RADIOTAP_RATE) | \ (1 << IEEE80211_RADIOTAP_CHANNEL) | \ (1 << IEEE80211_RADIOTAP_DBM_ANTSIGNAL) | \ (1 << IEEE80211_RADIOTAP_DBM_ANTNOISE)) struct iwn_tx_radiotap_header { struct ieee80211_radiotap_header wt_ihdr; uint8_t wt_flags; uint8_t wt_rate; uint16_t wt_chan_freq; uint16_t wt_chan_flags; } __packed; #define IWN_TX_RADIOTAP_PRESENT \ ((1 << IEEE80211_RADIOTAP_FLAGS) | \ (1 << IEEE80211_RADIOTAP_RATE) | \ (1 << IEEE80211_RADIOTAP_CHANNEL)) struct iwn_dma_info { bus_dma_tag_t tag; bus_dmamap_t map; bus_dma_segment_t seg; bus_addr_t paddr; caddr_t vaddr; bus_size_t size; }; struct iwn_tx_data { bus_dmamap_t map; bus_addr_t cmd_paddr; bus_addr_t scratch_paddr; struct mbuf *m; struct ieee80211_node *ni; }; struct iwn_tx_ring { struct iwn_dma_info desc_dma; struct iwn_dma_info cmd_dma; struct iwn_tx_desc *desc; struct iwn_tx_cmd *cmd; struct iwn_tx_data data[IWN_TX_RING_COUNT]; bus_dma_tag_t data_dmat; int qid; int queued; int cur; int read; }; struct iwn_softc; struct iwn_rx_data { struct mbuf *m; bus_dmamap_t map; }; struct iwn_rx_ring { struct iwn_dma_info desc_dma; struct iwn_dma_info stat_dma; uint32_t *desc; struct iwn_rx_status *stat; struct iwn_rx_data data[IWN_RX_RING_COUNT]; bus_dma_tag_t data_dmat; int cur; }; struct iwn_node { struct ieee80211_node ni; /* must be the first */ uint16_t disable_tid; uint8_t id; struct { uint64_t bitmap; int startidx; int nframes; } agg[IEEE80211_TID_SIZE]; }; struct iwn_calib_state { uint8_t state; #define IWN_CALIB_STATE_INIT 0 #define IWN_CALIB_STATE_ASSOC 1 #define IWN_CALIB_STATE_RUN 2 u_int nbeacons; uint32_t noise[3]; uint32_t rssi[3]; uint32_t ofdm_x1; uint32_t ofdm_mrc_x1; uint32_t ofdm_x4; uint32_t ofdm_mrc_x4; uint32_t cck_x4; uint32_t cck_mrc_x4; uint32_t bad_plcp_ofdm; uint32_t fa_ofdm; uint32_t bad_plcp_cck; uint32_t fa_cck; uint32_t low_fa; uint32_t bad_plcp_ht; uint8_t cck_state; #define IWN_CCK_STATE_INIT 0 #define IWN_CCK_STATE_LOFA 1 #define IWN_CCK_STATE_HIFA 2 uint8_t noise_samples[20]; u_int cur_noise_sample; uint8_t noise_ref; uint32_t energy_samples[10]; u_int cur_energy_sample; uint32_t energy_cck; }; struct iwn_calib_info { uint8_t *buf; u_int len; }; struct iwn_fw_part { const uint8_t *text; uint32_t textsz; const uint8_t *data; uint32_t datasz; }; struct iwn_fw_info { const uint8_t *data; size_t size; struct iwn_fw_part init; struct iwn_fw_part main; struct iwn_fw_part boot; }; struct iwn_ops { int (*load_firmware)(struct iwn_softc *); void (*read_eeprom)(struct iwn_softc *); int (*post_alive)(struct iwn_softc *); int (*nic_config)(struct iwn_softc *); void (*update_sched)(struct iwn_softc *, int, int, uint8_t, uint16_t); int (*get_temperature)(struct iwn_softc *); int (*get_rssi)(struct iwn_softc *, struct iwn_rx_stat *); int (*set_txpower)(struct iwn_softc *, struct ieee80211_channel *, int); int (*init_gains)(struct iwn_softc *); int (*set_gains)(struct iwn_softc *); int (*add_node)(struct iwn_softc *, struct iwn_node_info *, int); void (*tx_done)(struct iwn_softc *, struct iwn_rx_desc *, struct iwn_rx_data *); void (*ampdu_tx_start)(struct iwn_softc *, struct ieee80211_node *, int, uint8_t, uint16_t); void (*ampdu_tx_stop)(struct iwn_softc *, int, uint8_t, uint16_t); }; struct iwn_vap { struct ieee80211vap iv_vap; uint8_t iv_ridx; int (*iv_newstate)(struct ieee80211vap *, enum ieee80211_state, int); int ctx; int beacon_int; }; #define IWN_VAP(_vap) ((struct iwn_vap *)(_vap)) struct iwn_softc { device_t sc_dev; int sc_debug; struct cdev *sc_cdev; struct mtx sc_mtx; struct ieee80211com sc_ic; u_int sc_flags; #define IWN_FLAG_HAS_OTPROM (1 << 1) #define IWN_FLAG_CALIB_DONE (1 << 2) #define IWN_FLAG_USE_ICT (1 << 3) #define IWN_FLAG_INTERNAL_PA (1 << 4) #define IWN_FLAG_HAS_11N (1 << 6) #define IWN_FLAG_ENH_SENS (1 << 7) #define IWN_FLAG_ADV_BTCOEX (1 << 8) #define IWN_FLAG_PAN_SUPPORT (1 << 9) #define IWN_FLAG_BTCOEX (1 << 10) #define IWN_FLAG_RUNNING (1 << 11) uint8_t hw_type; /* subdevice_id used to adjust configuration */ uint16_t subdevice_id; struct iwn_ops ops; const char *fwname; const struct iwn_sensitivity_limits *limits; int ntxqs; int firstaggqueue; int ndmachnls; uint8_t broadcast_id; int rxonsz; int schedsz; uint32_t fw_text_maxsz; uint32_t fw_data_maxsz; uint32_t fwsz; bus_size_t sched_txfact_addr; uint32_t reset_noise_gain; uint32_t noise_gain; /* TX scheduler rings. */ struct iwn_dma_info sched_dma; uint16_t *sched; uint32_t sched_base; /* "Keep Warm" page. */ struct iwn_dma_info kw_dma; /* Firmware image. */ const struct firmware *fw_fp; /* Firmware DMA transfer. */ struct iwn_dma_info fw_dma; /* ICT table. */ struct iwn_dma_info ict_dma; uint32_t *ict; int ict_cur; /* TX/RX rings. */ struct iwn_tx_ring txq[IWN5000_NTXQUEUES]; struct iwn_rx_ring rxq; struct resource *mem; bus_space_tag_t sc_st; bus_space_handle_t sc_sh; struct resource *irq; void *sc_ih; bus_size_t sc_sz; int sc_cap_off; /* PCIe Capabilities. */ /* Tasks used by the driver */ - struct task sc_reinit_task; struct task sc_radioon_task; struct task sc_radiooff_task; struct task sc_panic_task; struct task sc_xmit_task; /* Taskqueue */ struct taskqueue *sc_tq; /* Calibration information */ struct callout calib_to; int calib_cnt; struct iwn_calib_state calib; int last_calib_ticks; struct callout watchdog_to; struct iwn_fw_info fw; struct iwn_calib_info calibcmd[IWN5000_PHY_CALIB_MAX_RESULT]; uint32_t errptr; struct iwn_rx_stat last_rx_stat; int last_rx_valid; struct iwn_ucode_info ucode_info; struct iwn_rxon rx_on[IWN_NUM_RXON_CTX]; struct iwn_rxon *rxon; int ctx; struct ieee80211vap *ivap[IWN_NUM_RXON_CTX]; /* General statistics */ /* * The statistics are reset after each channel * change. So it may be zeroed after things like * a background scan. * * So for now, this is just a cheap hack to * expose the last received statistics dump * via an ioctl(). Later versions of this * could expose the last 'n' messages, or just * provide a pipeline for the firmware responses * via something like BPF. */ struct iwn_stats last_stat; int last_stat_valid; uint8_t uc_scan_progress; uint32_t rawtemp; int temp; int noise; uint32_t qfullmsk; uint32_t prom_base; struct iwn4965_eeprom_band bands[IWN_NBANDS]; struct iwn_eeprom_chan eeprom_channels[IWN_NBANDS][IWN_MAX_CHAN_PER_BAND]; uint16_t rfcfg; uint8_t calib_ver; char eeprom_domain[4]; uint32_t eeprom_crystal; int16_t eeprom_temp; int16_t eeprom_temp_high; int16_t eeprom_voltage; int8_t maxpwr2GHz; int8_t maxpwr5GHz; int8_t maxpwr[IEEE80211_CHAN_MAX]; uint32_t tlv_feature_flags; int32_t temp_off; uint32_t int_mask; uint8_t ntxchains; uint8_t nrxchains; uint8_t txchainmask; uint8_t rxchainmask; uint8_t chainmask; int sc_tx_timer; int sc_scan_timer; /* Are we doing a scan? */ int sc_is_scanning; /* Are we waiting for a beacon before xmit? */ int sc_beacon_wait; struct ieee80211_tx_ampdu *qid2tap[IWN5000_NTXQUEUES]; int (*sc_ampdu_rx_start)(struct ieee80211_node *, struct ieee80211_rx_ampdu *, int, int, int); void (*sc_ampdu_rx_stop)(struct ieee80211_node *, struct ieee80211_rx_ampdu *); int (*sc_addba_request)(struct ieee80211_node *, struct ieee80211_tx_ampdu *, int, int, int); int (*sc_addba_response)(struct ieee80211_node *, struct ieee80211_tx_ampdu *, int, int, int); void (*sc_addba_stop)(struct ieee80211_node *, struct ieee80211_tx_ampdu *); struct iwn_led_mode sc_led; struct iwn_rx_radiotap_header sc_rxtap; struct iwn_tx_radiotap_header sc_txtap; /* The power save level originally configured by user */ int desired_pwrsave_level; /* * The current power save level, this may differ from the * configured value due to thermal throttling etc. */ int current_pwrsave_level; /* For specific params */ const struct iwn_base_params *base_params; #define IWN_UCODE_API(ver) (((ver) & 0x0000FF00) >> 8) uint32_t ucode_rev; /* * Global queue for queuing xmit frames * when we can't yet transmit (eg raw * frames whilst waiting for beacons.) */ struct mbufq sc_xmit_queue; }; #define IWN_LOCK_INIT(_sc) \ mtx_init(&(_sc)->sc_mtx, device_get_nameunit((_sc)->sc_dev), \ MTX_NETWORK_LOCK, MTX_DEF) #define IWN_LOCK(_sc) mtx_lock(&(_sc)->sc_mtx) #define IWN_LOCK_ASSERT(_sc) mtx_assert(&(_sc)->sc_mtx, MA_OWNED) #define IWN_UNLOCK(_sc) mtx_unlock(&(_sc)->sc_mtx) #define IWN_LOCK_DESTROY(_sc) mtx_destroy(&(_sc)->sc_mtx) Index: user/ngie/release-pkg-fix-tests/sys/dev/pms/freebsd/driver/common/lxcommon.h =================================================================== --- user/ngie/release-pkg-fix-tests/sys/dev/pms/freebsd/driver/common/lxcommon.h (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/dev/pms/freebsd/driver/common/lxcommon.h (revision 299091) @@ -1,886 +1,886 @@ /******************************************************************************* *Copyright (c) 2014 PMC-Sierra, Inc. All rights reserved. * *Redistribution and use in source and binary forms, with or without modification, are permitted provided *that the following conditions are met: *1. Redistributions of source code must retain the above copyright notice, this list of conditions and the *following disclaimer. *2. Redistributions in binary form must reproduce the above copyright notice, *this list of conditions and the following disclaimer in the documentation and/or other materials provided *with the distribution. * *THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED *WARRANTIES,INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS *FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE *FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT *NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR *BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT *LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS *SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE * * $FreeBSD$ * *******************************************************************************/ /****************************************************************************** Module Name: lxcommon.h Abstract: TISA Initiator/target driver module constant define header file Environment: Kernel or loadable module ******************************************************************************/ #include #define LINUX_DMA_MEM_MAX 0x1ffe0 /* 128k - 32, real 128k - 24 */ #define DEK_MAX_TABLE_ITEMS DEK_MAX_TABLE_ENTRIES // from tisa/api/titypes.h /* ** IP address length based on character. */ #ifdef AGTIAPI_IP6_SUPPORT # define IP_ADDR_CHAR_LEN 64 #else # define IP_ADDR_CHAR_LEN 16 #endif #define MSEC_PER_TICK (1000/hz) /* milisecond per tick */ #define USEC_PER_TICK (1000000/hz) /* microsecond per tick */ #define AGTIAPI_64BIT_ALIGN 8 /* 64 bit environment alignment */ /* ** Max device supported */ #define AGTIAPI_MAX_CARDS 4 /* card supported up to system limit */ #define AGTIAPI_TOO_MANY_CARDS -1 /* beyond defined max support */ #define AGTIAPI_MAX_PORTALS 16 /* max portal per card */ /* max device per portal */ /* ** Adjustable Parameter Options */ #define AGTIAPI_OPTION_ON 1 /* adjustable parameter available */ #define AGTIAPI_KEY_MAX 64 /* max number of keys */ #define AGTIAPI_STRING_MAX 512 /* max length for string */ #define AGTIAPI_PARAM_MAX 256 /* max number of parameters */ #ifdef TARGET_DRIVER #define AGTIAPI_DMA_MEM_LIST_MAX 4096 /* max number of DMA memory list */ #define AGTIAPI_CACHE_MEM_LIST_MAX 24 /* max number of CACHE memory list */ #else /* INITIATOR_DRIVER */ #define AGTIAPI_DMA_MEM_LIST_MAX 1024 /* max number of DMA memory list */ #define AGTIAPI_CACHE_MEM_LIST_MAX 1024 /* max number of CACHE memory list */ #endif #ifndef AGTIAPI_DYNAMIC_MAX #define AGTIAPI_DYNAMIC_MAX 4096 /* max unreleased dynamic memory */ #endif #define AGTIAPI_LOOP_MAX 4 /* max loop for init process */ #define AGTIAPI_MAX_NAME 70 // Max string name length #define AGTIAPI_MIN_NAME 10 // minimum space for SAS name string #define AGTIAPI_MAX_ID 8 // Max string id length /* ** Card-port status definitions */ #define AGTIAPI_INIT_TIME 0x00000001 #define AGTIAPI_SOFT_RESET 0x00000002 #define AGTIAPI_HAD_RESET 0x00000004 // ### #define AGTIAPI_DISC_DONE 0x00000008 #define AGTIAPI_INSTALLED 0x00000010 #define AGTIAPI_RESET 0x00000020 #define AGTIAPI_FLAG_UP 0x00000040 #define AGTIAPI_CB_DONE 0x00000080 #define AGTIAPI_DISC_COMPLETE 0x00000100 #define AGTIAPI_IOREGION_REQUESTED 0x00000200 #define AGTIAPI_IRQ_REQUESTED 0x00000400 #define AGTIAPI_SCSI_REGISTERED 0x00000800 #define AGTIAPI_NAME_SERVER_UP 0x00001000 #define AGTIAPI_PORT_INITIALIZED 0x00002000 #define AGTIAPI_PORT_LINK_UP 0x00004000 #define AGTIAPI_LGN_LINK_UP 0x00008000 #define AGTIAPI_PORT_PANIC 0x00010000 #define AGTIAPI_RESET_SUCCESS 0x00020000 #define AGTIAPI_PORT_START 0x00040000 #define AGTIAPI_PORT_STOPPED 0x00080000 #define AGTIAPI_PORT_SHUTDOWN 0x00100000 #define AGTIAPI_IN_USE 0x00200000 #define AGTIAPI_SYS_INTR_ON 0x00400000 #define AGTIAPI_PORT_DISC_READY 0x00800000 #define AGTIAPI_SIG_DOWN 0x01000000 #define AGTIAPI_SIG_UP 0x02000000 #define AGTIAPI_TASK 0x04000000 #define AGTIAPI_INITIATOR 0x08000000 #define AGTIAPI_TARGET 0x10000000 #define AGTIAPI_TIMER_ON 0x20000000 #define AGTIAPI_SHUT_DOWN 0x40000000 /* reserved for ccb flag TASK_MANAGEMENT #define AGTIAPI_RESERVED 0x80000000 */ #define AGTIAPI_RESET_ALL 0xFFFFFFFF /* ** PCI defines */ #ifndef PCI_VENDOR_ID_HP #define PCI_VENDOR_ID_HP 0x103c #endif #ifndef PCI_VENDOR_ID_PMC_SIERRA #define PCI_VENDOR_ID_PMC_SIERRA 0x11F8 #endif #ifndef PCI_VENDOR_ID_AGILENT #define PCI_VENDOR_ID_AGILENT 0x15bc #endif #ifndef PCI_VENDOR_ID_CYCLONE #define PCI_VENDOR_ID_CYCLONE 0x113C #endif #ifndef PCI_VENDOR_ID_SPCV_FPGA #define PCI_VENDOR_ID_SPCV_FPGA 0x1855 #endif #ifndef PCI_VENDOR_ID_HIALEAH #define PCI_VENDOR_ID_HIALEAH 0x9005 #endif #define PCI_DEVICE_ID_HP_TS 0x102a #define PCI_DEVICE_ID_HP_TL 0x1028 #define PCI_DEVICE_ID_HP_XL2 0x1029 #define PCI_DEVICE_ID_AG_DX2 0x0100 #define PCI_DEVICE_ID_AG_DX2PLUS 0x0101 #define PCI_DEVICE_ID_AG_QX2 0x0102 #define PCI_DEVICE_ID_AG_QX4 0x0103 #define PCI_DEVICE_ID_AG_QE4 0x1200 #define PCI_DEVICE_ID_AG_DE4 0x1203 #define PCI_DEVICE_ID_AG_XL10 0x0104 #define PCI_DEVICE_ID_AG_DX4PLUS 0x0105 #define PCI_DEVICE_ID_AG_DIXL 0x0110 #define PCI_DEVICE_ID_AG_IDX1 0x050A #define PCI_DEVICE_ID_PMC_SIERRA_SPC 0x8001 #define PCI_DEVICE_ID_PMC_SIERRA_SPCV 0x8008 #define PCI_DEVICE_ID_PMC_SIERRA_SPCVE 0x8009 #define PCI_DEVICE_ID_PMC_SIERRA_SPCVPLUS 0x8018 #define PCI_DEVICE_ID_PMC_SIERRA_SPCVE_16 0x8019 #define PCI_DEVICE_ID_SPCV_FPGA 0xabcd #define PCI_DEVICE_ID_PMC_SIERRA_SPCV12G 0x8070 #define PCI_DEVICE_ID_PMC_SIERRA_SPCVE12G 0x8071 #define PCI_DEVICE_ID_PMC_SIERRA_SPCV12G_16 0x8072 #define PCI_DEVICE_ID_PMC_SIERRA_SPCVE12G_16 0x8073 #define PCI_DEVICE_ID_HIALEAH_HBA_SPC 0x8081 #define PCI_DEVICE_ID_HIALEAH_RAID_SPC 0x8091 #define PCI_DEVICE_ID_HIALEAH_HBA_SPCV 0x8088 #define PCI_DEVICE_ID_HIALEAH_RAID_SPCV 0x8098 #define PCI_DEVICE_ID_HIALEAH_HBA_SPCVE 0x8089 #define PCI_DEVICE_ID_HIALEAH_RAID_SPCVE 0x8099 #define PCI_DEVICE_ID_DELRAY_HBA_8PORTS_SPCV 0x8074 #define PCI_DEVICE_ID_DELRAY_HBA_8PORTS_SPCVE 0x8075 #define PCI_DEVICE_ID_DELRAY_HBA_16PORTS_SPCV 0x8076 #define PCI_DEVICE_ID_DELRAY_HBA_16PORTS_SPCVE 0x8077 #define PCI_DEVICE_ID_DELRAY_HBA_16PORTS_SPCV_SATA 0x8006 #define PCI_SUB_VENDOR_ID_HP PCI_VENDOR_ID_HP #define PCI_SUB_VENDOR_ID_AG PCI_VENDOR_ID_AGILENT #define PCI_SUB_VENDOR_ID_MASK 0xFFFF #define PCI_SUB_SYSTEM_ID_AG 0x0001 #define PCI_BASE_MEM_MASK (~0x0F) #define PCI_DEVICE_ID_CYCLONE 0xB555 #define PCI_ENABLE_VALUE 0x0157 #ifdef PMC_SPC #define PCI_NUMBER_BARS 6 #endif #define IOCTL_MN_GET_CARD_INFO 0x11 /* ** Constant defines */ #define _08B 8 #define _16B 16 #define _24B 24 #define _32B 32 #define _64B 64 #define _128B 128 #define _256B 256 #define _512B 512 #define _1K 1024 #define _2K 2048 #define _4K 4096 #define _128K (128*(_1K)) // Card property related info. typedef struct _ag_card_id { U16 vendorId; /* pci vendor id */ U16 deviceId; /* pci device id */ S32 cardNameIndex; /* structure index */ U16 membar; /* pci memory bar offset */ U16 iobar1; /* pci io bar 1 offset */ U16 iobar2; /* pci io bar 2 offest */ U16 reg; /* pci memory bar number */ } ag_card_id_t; #define PCI_BASE_ADDRESS_0 PCIR_BAR(0) #define PCI_BASE_ADDRESS_1 PCIR_BAR(1) #define PCI_BASE_ADDRESS_2 PCIR_BAR(2) #define PCI_BASE_ADDRESS_3 PCIR_BAR(3) #define PCI_BASE_ADDRESS_4 PCIR_BAR(4) ag_card_id_t ag_card_type[] = { #ifdef AGTIAPI_ISCSI {PCI_VENDOR_ID_AGILENTj, PCI_DEVICE_ID_AG_DIXL, 1, PCI_BASE_ADDRESS_0, PCI_BASE_ADDRESS_1, PCI_BASE_ADDRESS_0, 0}, {PCI_VENDOR_ID_AGILENT, PCI_DEVICE_ID_AG_IDX1, 2, PCI_BASE_ADDRESS_0, PCI_BASE_ADDRESS_1, PCI_BASE_ADDRESS_0, 0}, #endif #ifdef AGTIAPI_FC {PCI_VENDOR_ID_HP, PCI_DEVICE_ID_HP_TS, 3, PCI_BASE_ADDRESS_3, PCI_BASE_ADDRESS_1, PCI_BASE_ADDRESS_2, 3}, {PCI_VENDOR_ID_HP, PCI_DEVICE_ID_HP_TL, 4, PCI_BASE_ADDRESS_3, PCI_BASE_ADDRESS_1, PCI_BASE_ADDRESS_2, 3}, {PCI_VENDOR_ID_HP, PCI_DEVICE_ID_HP_XL2, 5, PCI_BASE_ADDRESS_3, PCI_BASE_ADDRESS_1, PCI_BASE_ADDRESS_2, 3}, {PCI_VENDOR_ID_AGILENT, PCI_DEVICE_ID_AG_DX2, 6, PCI_BASE_ADDRESS_4, PCI_BASE_ADDRESS_2, PCI_BASE_ADDRESS_3, 4}, {PCI_VENDOR_ID_AGILENT, PCI_DEVICE_ID_AG_DX2PLUS, 7, PCI_BASE_ADDRESS_4, PCI_BASE_ADDRESS_2, PCI_BASE_ADDRESS_3, 4}, {PCI_VENDOR_ID_AGILENT, PCI_DEVICE_ID_AG_DX4PLUS, 8, PCI_BASE_ADDRESS_4, PCI_BASE_ADDRESS_2, PCI_BASE_ADDRESS_3, 4}, {PCI_VENDOR_ID_AGILENT, PCI_DEVICE_ID_AG_QX2, 9, PCI_BASE_ADDRESS_4, PCI_BASE_ADDRESS_2, PCI_BASE_ADDRESS_3, 4}, {PCI_VENDOR_ID_AGILENT, PCI_DEVICE_ID_AG_QX4, 10, PCI_BASE_ADDRESS_4, PCI_BASE_ADDRESS_2, PCI_BASE_ADDRESS_3, 4}, {PCI_VENDOR_ID_AGILENT, PCI_DEVICE_ID_AG_DE4, 11, PCI_BASE_ADDRESS_4, PCI_BASE_ADDRESS_2, PCI_BASE_ADDRESS_3, 4}, {PCI_VENDOR_ID_AGILENT, PCI_DEVICE_ID_AG_QE4, 12, PCI_BASE_ADDRESS_4, PCI_BASE_ADDRESS_2, PCI_BASE_ADDRESS_3, 4}, {PCI_VENDOR_ID_AGILENT, PCI_DEVICE_ID_AG_XL10, 13, PCI_BASE_ADDRESS_4, PCI_BASE_ADDRESS_2, PCI_BASE_ADDRESS_3, 4}, #endif #ifdef AGTIAPI_SA #ifdef PMC_SPC {PCI_VENDOR_ID_PMC_SIERRA, PCI_DEVICE_ID_PMC_SIERRA_SPC, 14, PCI_BASE_ADDRESS_0, PCI_BASE_ADDRESS_2, PCI_BASE_ADDRESS_3, 0}, {PCI_VENDOR_ID_PMC_SIERRA, PCI_DEVICE_ID_PMC_SIERRA_SPCV, 15, PCI_BASE_ADDRESS_0, PCI_BASE_ADDRESS_2, PCI_BASE_ADDRESS_3, 0}, {PCI_VENDOR_ID_PMC_SIERRA, PCI_DEVICE_ID_PMC_SIERRA_SPCVE, 16, PCI_BASE_ADDRESS_0, PCI_BASE_ADDRESS_2, PCI_BASE_ADDRESS_3, 0}, {PCI_VENDOR_ID_PMC_SIERRA, PCI_DEVICE_ID_PMC_SIERRA_SPCVPLUS, 17, PCI_BASE_ADDRESS_0, PCI_BASE_ADDRESS_2, PCI_BASE_ADDRESS_3, 0}, {PCI_VENDOR_ID_PMC_SIERRA, PCI_DEVICE_ID_PMC_SIERRA_SPCVE_16, 18, PCI_BASE_ADDRESS_0, PCI_BASE_ADDRESS_2, PCI_BASE_ADDRESS_3, 0}, {PCI_VENDOR_ID_SPCV_FPGA, PCI_DEVICE_ID_SPCV_FPGA, 19, PCI_BASE_ADDRESS_0, PCI_BASE_ADDRESS_2, PCI_BASE_ADDRESS_3, 0}, {PCI_VENDOR_ID_PMC_SIERRA, PCI_DEVICE_ID_PMC_SIERRA_SPCV12G, 20, PCI_BASE_ADDRESS_0, PCI_BASE_ADDRESS_2, PCI_BASE_ADDRESS_3, 0}, {PCI_VENDOR_ID_PMC_SIERRA, PCI_DEVICE_ID_PMC_SIERRA_SPCVE12G, 21, PCI_BASE_ADDRESS_0, PCI_BASE_ADDRESS_2, PCI_BASE_ADDRESS_3, 0}, {PCI_VENDOR_ID_PMC_SIERRA, PCI_DEVICE_ID_PMC_SIERRA_SPCV12G_16, 22, PCI_BASE_ADDRESS_0, PCI_BASE_ADDRESS_2, PCI_BASE_ADDRESS_3, 0}, {PCI_VENDOR_ID_PMC_SIERRA, PCI_DEVICE_ID_PMC_SIERRA_SPCVE12G_16, 23, PCI_BASE_ADDRESS_0, PCI_BASE_ADDRESS_2, PCI_BASE_ADDRESS_3, 0}, {PCI_VENDOR_ID_HIALEAH, PCI_DEVICE_ID_HIALEAH_HBA_SPC, 24, PCI_BASE_ADDRESS_0, PCI_BASE_ADDRESS_2, PCI_BASE_ADDRESS_3, 0}, {PCI_VENDOR_ID_HIALEAH, PCI_DEVICE_ID_HIALEAH_RAID_SPC, 25, PCI_BASE_ADDRESS_0, PCI_BASE_ADDRESS_2, PCI_BASE_ADDRESS_3, 0}, {PCI_VENDOR_ID_HIALEAH, PCI_DEVICE_ID_HIALEAH_HBA_SPCV, 26, PCI_BASE_ADDRESS_0, PCI_BASE_ADDRESS_2, PCI_BASE_ADDRESS_3, 0}, {PCI_VENDOR_ID_HIALEAH, PCI_DEVICE_ID_HIALEAH_RAID_SPCV, 27, PCI_BASE_ADDRESS_0, PCI_BASE_ADDRESS_2, PCI_BASE_ADDRESS_3, 0}, {PCI_VENDOR_ID_HIALEAH, PCI_DEVICE_ID_HIALEAH_HBA_SPCVE, 28, PCI_BASE_ADDRESS_0, PCI_BASE_ADDRESS_2, PCI_BASE_ADDRESS_3, 0}, {PCI_VENDOR_ID_HIALEAH, PCI_DEVICE_ID_HIALEAH_RAID_SPCVE, 29, PCI_BASE_ADDRESS_0, PCI_BASE_ADDRESS_2, PCI_BASE_ADDRESS_3, 0}, {PCI_VENDOR_ID_HIALEAH, PCI_DEVICE_ID_DELRAY_HBA_8PORTS_SPCV, 30, PCI_BASE_ADDRESS_0, PCI_BASE_ADDRESS_2, PCI_BASE_ADDRESS_3, 0}, {PCI_VENDOR_ID_HIALEAH, PCI_DEVICE_ID_DELRAY_HBA_8PORTS_SPCVE, 31, PCI_BASE_ADDRESS_0, PCI_BASE_ADDRESS_2, PCI_BASE_ADDRESS_3, 0}, {PCI_VENDOR_ID_HIALEAH, PCI_DEVICE_ID_DELRAY_HBA_16PORTS_SPCV, 32, PCI_BASE_ADDRESS_0, PCI_BASE_ADDRESS_2, PCI_BASE_ADDRESS_3, 0}, {PCI_VENDOR_ID_HIALEAH, PCI_DEVICE_ID_DELRAY_HBA_16PORTS_SPCVE, 33, PCI_BASE_ADDRESS_0, PCI_BASE_ADDRESS_2, PCI_BASE_ADDRESS_3, 0}, {PCI_VENDOR_ID_HIALEAH, PCI_DEVICE_ID_DELRAY_HBA_16PORTS_SPCV_SATA, 34, PCI_BASE_ADDRESS_0, PCI_BASE_ADDRESS_2, PCI_BASE_ADDRESS_3, 0}, #endif #endif //AGTIAPI_SA }; static const char *ag_card_names[] = { "Unknown", "iSCSI DiXL Card", "iSCSI iDX1 Card", "Tachyon TS Fibre Channel Card", "Tachyon TL Fibre Channel Card", "Tachyon XL2 Fibre Channel Card", "Tachyon DX2 Fibre Channel Card", "Tachyon DX2+ Fibre Channel Card", "Tachyon DX4+ Fibre Channel Card", "Tachyon QX2 Fibre Channel Card", "Tachyon QX4 Fibre Channel Card", "Tachyon DE4 Fibre Channel Card", "Tachyon QE4 Fibre Channel Card", "Tachyon XL10 Fibre Channel Card", #ifdef AGTIAPI_SA #ifdef PMC_SPC "PMC Sierra SPC SAS-SATA Card", "PMC Sierra SPC-V SAS-SATA Card", "PMC Sierra SPC-VE SAS-SATA Card", "PMC Sierra SPC-V 16 Port SAS-SATA Card", "PMC Sierra SPC-VE 16 Port SAS-SATA Card", "PMC Sierra FPGA", "PMC Sierra SPC-V SAS-SATA Card 12Gig", "PMC Sierra SPC-VE SAS-SATA Card 12Gig", "PMC Sierra SPC-V 16 Port SAS-SATA Card 12Gig", "PMC Sierra SPC-VE 16 Port SAS-SATA Card 12Gig", "Adaptec Hialeah 4/8 Port SAS-SATA HBA Card 6Gig", "Adaptec Hialeah 4/8 Port SAS-SATA RAID Card 6Gig", "Adaptec Hialeah 8/16 Port SAS-SATA HBA Card 6Gig", "Adaptec Hialeah 8/16 Port SAS-SATA RAID Card 6Gig", "Adaptec Hialeah 8/16 Port SAS-SATA HBA Encryption Card 6Gig", "Adaptec Hialeah 8/16 Port SAS-SATA RAID Encryption Card 6Gig", "Adaptec Delray 8 Port SAS-SATA HBA Card 12Gig", "Adaptec Delray 8 Port SAS-SATA HBA Encryption Card 12Gig", "Adaptec Delray 16 Port SAS-SATA HBA Card 12Gig", "Adaptec Delray 16 Port SAS-SATA HBA Encryption Card 12Gig", "Adaptec SATA Adapter", #endif #endif }; /* ** Resource Info Structure */ typedef struct _ag_resource_info { tiLoLevelResource_t tiLoLevelResource; // Low level resource required tiInitiatorResource_t tiInitiatorResource; // Initiator resource required tiTargetResource_t tiTargetResource; // Target resource required tiTdSharedMem_t tiSharedMem; // Shared memory by ti and td } ag_resource_info_t; // DMA memory address pair typedef struct _ag_dma_addr { void *dmaVirtAddr; vm_paddr_t dmaPhysAddr; U32 memSize; bit32 type; bus_addr_t nocache_busaddr; void *nocache_mem; } ag_dma_addr_t; typedef struct _CardInfo { U32 pciIOAddrLow; /* PCI IOBASE lower */ U32 pciIOAddrUp; /* PCI IOBASE Upper */ U32_64 pciMemBase; /* PCI MEMBASE, physical */ U32_64 pciMemBaseSpc[PCI_NUMBER_BARS]; // PCI MEMBASE, physical U16 deviceId; // PCI device id U16 vendorId; // PCI Vendor id U32 busNum; U32 deviceNum; }CardInfo_t; // Card info. for all cards and drivers typedef struct _ag_card_info { struct mtx pmIOLock; device_t pPCIDev; // PCI device pointer void *pCard; // pointer to per card data structure S32 cardNameIndex; U32 cardID; // card system ID U32 cardIdIndex; U32 pciIOAddrLow; // PCI IOBASE lower U32 pciIOAddrUp; // PCI IOBASE Upper U32_64 pciMemBase; // PCI MEMBASE, physical caddr_t pciMemVirtAddr; // PCI MEMBASE, virtual ptr U32 pciMemSize; // PCI MEMBASE memory size #ifdef AGTIAPI_SA #ifdef FPGA_CARD U32_64 pciMemBase0; // PCI MEMBASE, physical caddr_t pciMemVirtAddr0; // PCI MEMBASE, virtual ptr U32 pciMemSize0; // PCI MEMBASE memory size #endif #ifdef PMC_SPC struct resource *pciMemBaseRscSpc[PCI_NUMBER_BARS]; int pciMemBaseRIDSpc[PCI_NUMBER_BARS]; U32_64 pciMemBaseSpc[PCI_NUMBER_BARS]; // PCI MEMBASE, physical caddr_t pciMemVirtAddrSpc[PCI_NUMBER_BARS];//PCI MEMBASE, virt ptr U32 pciMemSizeSpc[PCI_NUMBER_BARS]; // PCI MEMBASE memory size #endif #endif U16 memBar; U16 memReg; U32 cacheIndex; U32 dmaIndex; ag_dma_addr_t tiDmaMem[AGTIAPI_DMA_MEM_LIST_MAX]; // dma addr list // all (free and allocated) mem slots ag_dma_addr_t dynamicMem[AGTIAPI_DYNAMIC_MAX]; // ptr to free mem slots ag_dma_addr_t *freeDynamicMem[AGTIAPI_DYNAMIC_MAX]; U16 topOfFreeDynamicMem; // idx to the first free slot ptr void *tiCachedMem[AGTIAPI_CACHE_MEM_LIST_MAX];// cached mem list ag_resource_info_t tiRscInfo; /* low level resource requirement */ U08 WWN[AGTIAPI_MAX_NAME]; /* WWN for this card */ U08 WWNLen; // #define MAX_MSIX_NUM_VECTOR 64 ## #define MAX_MSIX_NUM_VECTOR 16 // 1 then 16 just for testing; #define MAX_MSIX_NUM_DPC 64 // 16 #define MAX_MSIX_NUM_ISR 64 // 16 #ifdef SPC_MSIX_INTR // ## use as a map instead of presirq struct resource *msix_entries[MAX_MSIX_NUM_VECTOR]; #endif U32 maxInterruptVectors; } ag_card_info_t; /* ** Optional Adjustable Parameters Structures. -** Not using pointer stucture for easy read and access tree structure. +** Not using pointer structure for easy read and access tree structure. ** In the future if more layer of key tree involved, it might be a good ** idea to change the structure and program. */ typedef struct _ag_param_value{ char valueName[AGTIAPI_MAX_NAME]; char valueString[AGTIAPI_STRING_MAX]; struct _ag_param_value *next; } ag_value_t; typedef struct _ag_param_key{ char keyName[AGTIAPI_MAX_NAME]; ag_value_t *pValueHead; ag_value_t *pValueTail; struct _ag_param_key *pSubkeyHead; struct _ag_param_key *pSubkeyTail; struct _ag_param_key *next; } ag_key_t; /* ** Portal info data structure */ typedef struct _ag_portal_info { U32 portID; U32 portStatus; U32 devTotal; U32 devPrev; tiPortInfo_t tiPortInfo; tiPortalContext_t tiPortalContext; #ifdef INITIATOR_DRIVER void *pDevList[AGTIAPI_HW_LIMIT_DEVICE]; #endif } ag_portal_info_t; #define MAP_TABLE_ENTRY(pC, c, d, l) (pC->encrypt_map + \ (c * pC->devDiscover * AGTIAPI_MAX_LUN) + \ (d * AGTIAPI_MAX_LUN) + \ (l)) #ifdef CHAR_DEVICE /************************************************************************* Purpose: Payload Wraper for ioctl commands ***********************************************************************/ typedef struct datatosendt{ bit32 datasize; //buffer size bit8 *data; //buffer }datatosend; /***********************************************************************/ #define AGTIAPI_IOCTL_BASE 'x' #define AGTIAPI_IOCTL _IOWR(AGTIAPI_IOCTL_BASE, 0,datatosend ) //receiving payload here// #define AGTIAPI_IOCTL_MAX 1 #endif #ifdef AGTIAPI_FLOW_DEBUG #define AGTIAPI_FLOW(format, a...) printf(format, ## a) #else #define AGTIAPI_FLOW(format, a...) #endif #ifdef AGTIAPI_DEBUG #define AGTIAPI_PRINTK(format, a...) printf(format, ## a) #else #define AGTIAPI_PRINTK(format, a...) #endif #ifdef AGTIAPI_INIT_DEBUG #define AGTIAPI_INIT(format, a...) printf(format, ## a) /* to avoid losing the logs */ #define AGTIAPI_INIT_MDELAY(dly) mdelay(dly) #else #define AGTIAPI_INIT(format, a...) #define AGTIAPI_INIT_MDELAY(dly) #endif #ifdef AGTIAPI_INIT2_DEBUG #define AGTIAPI_INIT2(format, a...) printf(format, ## a) #else #define AGTIAPI_INIT2(format, a...) #endif #ifdef AGTIAPI_INIT_MEM_DEBUG #define AGTIAPI_INITMEM(format, a...) printf(format, ## a) #else #define AGTIAPI_INITMEM(format, a...) #endif #ifdef AGTIAPI_IO_DEBUG #define AGTIAPI_IO(format, a...) printf(format, ## a) #else #define AGTIAPI_IO(format, a...) #endif #ifdef AGTIAPI_LOAD_DELAY #define AGTIAPI_INIT_DELAY(delay_time) \ { \ agtiapi_DelayMSec(delay_time); \ } #else #define AGTIAPI_INIT_DELAY(delay_time) #endif /* * AGTIAPI_KDB() will be used to drop into kernel debugger * from driver code if kdb is involved. */ #ifdef AGTIAPI_KDB_ENABLE #define AGTIAPI_KDB() KDB_ENTER() #else #define AGTIAPI_KDB() #endif #if (BITS_PER_LONG == 64) //#if 1 #define LOW_32_BITS(addr) (U32)(addr & 0xffffffff) #define HIGH_32_BITS(addr) (U32)((addr >> 32) & 0xffffffff) #else #define LOW_32_BITS(addr) (U32)addr #define HIGH_32_BITS(addr) 0 #endif #define AG_SWAP16(data) (((data<<8) & 0xFF00) | (data>>8)) #define AG_SWAP24(data) (((data<<16) & 0xFF0000) | \ ((data>>16) & 0xFF) | (data & 0xFF00)) #define AG_SWAP32(data) ((data<<24) | ((data<<8) & 0xFF0000) | \ ((data>>8) & 0xFF00) | (data>>24)) #define AG_PCI_DEV_INFO(pdev) ( \ AGTIAPI_PRINTK("vendor id 0x%x device id 0x%x, slot %d, function %d\n", \ pdev->vendor, pdev->device, PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn)) \ ) #define COUNT(arr) (sizeof(arr) / sizeof(arr[0])) #define PORTAL_CONTEXT_TO_PORTALDATA(pPortalContext) \ ((ag_portal_data_t *)(((tiPortalContext_t *)pPortalContext)->osData)) #define PORTAL_STATUS(pPortalData) (pPortalData->portalInfo.portStatus) #if (defined(DEFINE_OSTI_PORT_EVENT_IN_IBE)) || \ (defined(DEFINE_OSTI_PORT_EVENT_IN_TFE)) #define TIROOT_TO_CARD(ptiRoot) \ ((ag_card_t *)(((appRoot_t *)(ptiRoot->osData))->oscData)) #define TIROOT_TO_CARDINFO(ptiRoot) (TIROOT_TO_CARD(ptiRoot)->pCardInfo) #define TIROOT_TO_PCIDEV(ptiRoot) (TIROOT_TO_CARDINFO(ptiRoot)->pPCIDev) #else #define TIROOT_TO_CARD(ptiRoot) ((struct agtiapi_softc *)(ptiRoot->osData)) #define TIROOT_TO_CARDINFO(ptiRoot) (TIROOT_TO_CARD(ptiRoot)->pCardInfo) #define TIROOT_TO_PCIDEV(ptiRoot) (TIROOT_TO_CARD(ptiRoot)->my_dev) #endif #define Is_ADP7H(pmsc) ((0x90058088 == (pmsc->VidDid))?1:\ (0x90058089 == (pmsc->VidDid))?1:0) #define Is_ADP8H(pmsc) ((0x90058074 == (pmsc->VidDid))?1:\ (0x90058076 == (pmsc->VidDid))?1:0) #define __cacheline_aligned __attribute__((__aligned__(CACHE_LINE_SIZE))) /* ** link data, need to be included at the start (offset 0) ** of any strutures that are to be stored in the link list */ typedef struct _LINK_NODE { struct _LINK_NODE *pNext; struct _LINK_NODE *pPrev; /* ** for assertion purpose only */ struct _LINK_NODE * pHead; // track the link list the link is a member of void * pad; } LINK_NODE, * PLINK_NODE __cacheline_aligned; /* ** link list basic pointers */ typedef struct _LINK_LIST { PLINK_NODE pHead; bit32 Count; - LINK_NODE Head __cacheline_aligned; // allways one link to speed up insert&rm + LINK_NODE Head __cacheline_aligned; // always one link to speed up insert&rm } LINK_LIST, * PLINK_LIST __cacheline_aligned; /******************************************************************** ** MACROS ********************************************************************/ /******************************************************************************* ** ** MODULE NAME: comListInitialize ** ** PURPOSE: Initialize a link list. ** ** PARAMETERS: PLINK_LIST OUT - Link list definition. ** ** SIDE EFFECTS & CAVEATS: ** ** ALGORITHM: ** *******************************************************************************/ #define comListInitialize(pList) {(pList)->pHead = &((pList)->Head); \ (pList)->pHead->pNext = (pList)->pHead; \ (pList)->pHead->pPrev = (pList)->pHead; \ (pList)->Count = 0; \ } /******************************************************************************* ** ** MODULE NAME: comLinkInitialize ** ** PURPOSE: Initialize a link. ** This function should be used to initialize a new link before it ** is used in the linked list. This will initialize the link so ** the assertion will work ** ** PARAMETERS: PLINK_NODE IN - Link to be initialized. ** ** SIDE EFFECTS & CAVEATS: ** ** ALGORITHM: ** *******************************************************************************/ #define comLinkInitialize(pLink) { (pLink)->pHead = NULL; \ (pLink)->pNext = NULL; \ (pLink)->pPrev = NULL; \ } /******************************************************************************* ** ** MODULE NAME: comListAdd ** ** PURPOSE: add a link at the tail of the list ** ** PARAMETERS: PLINK_LIST OUT - Link list definition. ** PLINK_NODE IN - Link to be inserted. ** ** SIDE EFFECTS & CAVEATS: ** !!! assumes that fcllistInitialize has been called on the linklist ** !!! if not, this function behavior is un-predictable ** ** ALGORITHM: ** *******************************************************************************/ #define comListAdd(pList, pLink) { \ (pLink)->pNext = (pList)->pHead; \ (pLink)->pPrev = (pList)->pHead->pPrev; \ (pLink)->pPrev->pNext = (pLink); \ (pList)->pHead->pPrev = (pLink); \ (pList)->Count ++; \ (pLink)->pHead = (pList)->pHead; \ } /******************************************************************************* ** ** MODULE NAME: comListInsert ** ** PURPOSE: insert a link preceding the given one ** ** PARAMETERS: PLINK_LIST OUT - Link list definition. ** PLINK_NODE IN - Link to be inserted after. ** PLINK_NODE IN - Link to be inserted. ** ** SIDE EFFECTS & CAVEATS: ** !!! assumes that fcllistInitialize has been called on the linklist ** !!! if not, this function behavior is un-predictable ** ** ALGORITHM: ** *******************************************************************************/ #define comListInsert(pList, pLink, pNew) { \ (pNew)->pNext = (pLink); \ (pNew)->pPrev = (pLink)->pPrev; \ (pNew)->pPrev->pNext = (pNew); \ (pLink)->pPrev = (pNew); \ (pList)->Count ++; \ (pNew)->pHead = (pList)->pHead; \ } /******************************************************************************* ** ** MODULE NAME: comListRemove ** ** PURPOSE: remove the link from the list. ** ** PARAMETERS: PLINK_LIST OUT - Link list definition. ** PLINK_NODE IN - Link to delet from list ** ** SIDE EFFECTS & CAVEATS: ** !!! assumes that fcllistInitialize has been called on the linklist ** !!! if not, this function behavior is un-predictable ** ** !!! No validation is made on the list or the validity of the link ** !!! the caller must make sure that the link is in the list ** ** ** ALGORITHM: ** *******************************************************************************/ #define comListRemove(pList, pLink) { \ (pLink)->pPrev->pNext = (pLink)->pNext; \ (pLink)->pNext->pPrev = (pLink)->pPrev; \ (pLink)->pHead = NULL; \ (pList)->Count --; \ } /******************************************************************************* ** ** MODULE NAME: comListGetHead ** ** PURPOSE: get the link following the head link. ** ** PARAMETERS: PLINK_LIST OUT - Link list definition. ** RETURNS - PLINK_NODE the link following the head ** NULL if the following link is the head ** ** SIDE EFFECTS & CAVEATS: ** !!! assumes that fcllistInitialize has been called on the linklist ** !!! if not, this function behavior is un-predictable ** ** ALGORITHM: ** *******************************************************************************/ #define comListGetHead(pList) comListGetNext(pList,(pList)->pHead) /******************************************************************************* ** ** MODULE NAME: comListGetTail ** ** PURPOSE: get the link preceding the tail link. ** ** PARAMETERS: PLINK_LIST OUT - Link list definition. ** RETURNS - PLINK_NODE the link preceding the head ** NULL if the preceding link is the head ** ** SIDE EFFECTS & CAVEATS: ** ** ALGORITHM: ** *******************************************************************************/ #define comListGetTail(pList) comListGetPrev((pList), (pList)->pHead) /******************************************************************************* ** ** MODULE NAME: comListGetCount ** ** PURPOSE: get the number of links in the list excluding head and tail. ** ** PARAMETERS: LINK_LIST OUT - Link list definition. ** ** SIDE EFFECTS & CAVEATS: ** !!! assumes that fcllistInitialize has been called on the linklist ** !!! if not, this function behavior is un-predictable ** ** ALGORITHM: ** *******************************************************************************/ #define comListGetCount(pList) ((pList)->Count) /******************************************************************************* ** ** MODULE NAME: comListGetNext ** ** PURPOSE: get the next link in the list. (one toward tail) ** ** PARAMETERS: PLINK_LIST OUT - Link list definition. ** PLINK_NODE IN - Link to get next to ** ** return PLINK - points to next link ** NULL if next link is head ** ** SIDE EFFECTS & CAVEATS: ** !!! assumes that fcllistInitialize has been called on the linklist ** !!! if not, this function behavior is un-predictable ** ** !!! No validation is made on the list or the validity of the link ** !!! the caller must make sure that the link is in the list ** ** ALGORITHM: ** *******************************************************************************/ #define comListGetNext(pList, pLink) (((pLink)->pNext == (pList)->pHead) ? \ NULL : (pLink)->pNext) /******************************************************************************* ** ** MODULE NAME: comListGetPrev ** ** PURPOSE: get the previous link in the list. (one toward head) ** ** PARAMETERS: PLINK_LIST OUT - Link list definition. ** PLINK_NODE IN - Link to get prev to ** ** return PLINK - points to previous link ** NULL if previous link is head ** ** SIDE EFFECTS & CAVEATS: ** !!! assumes that fcllistInitialize has been called on the linklist ** !!! if not, this function behavior is un-predictable ** ** !!! No validation is made on the list or the validity of the link ** !!! the caller must make sure that the link is in the list ** ** ALGORITHM: ** *******************************************************************************/ /*lint -emacro(613,fiLlistGetPrev) */ #define comListGetPrev(pList, pLink) (((pLink)->pPrev == (pList)->pHead) ? \ NULL : (pLink)->pPrev) #define AGT_INTERRUPT IRQF_DISABLED #define AGT_SAMPLE_RANDOM IRQF_SAMPLE_RANDOM #define AGT_SHIRQ IRQF_SHARED #define AGT_PROBEIRQ IRQF_PROBE_SHARED #define AGT_PERCPU IRQF_PERCPU #include "lxproto.h" Index: user/ngie/release-pkg-fix-tests/sys/dev/pms/freebsd/driver/common/osenv.h =================================================================== --- user/ngie/release-pkg-fix-tests/sys/dev/pms/freebsd/driver/common/osenv.h (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/dev/pms/freebsd/driver/common/osenv.h (revision 299091) @@ -1,151 +1,151 @@ /******************************************************************************* *Copyright (c) 2014 PMC-Sierra, Inc. All rights reserved. * *Redistribution and use in source and binary forms, with or without modification, are permitted provided *that the following conditions are met: *1. Redistributions of source code must retain the above copyright notice, this list of conditions and the *following disclaimer. *2. Redistributions in binary form must reproduce the above copyright notice, *this list of conditions and the following disclaimer in the documentation and/or other materials provided *with the distribution. * *THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED *WARRANTIES,INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS *FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE *FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT *NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR *BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT *LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS *SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE * * $FreeBSD$ * *******************************************************************************/ /*************************************************************************** Version Control Information: $RCSfile: osenv.h,v $ $Revision: 114125 $ -Note: This file defines the working enviornment of the system. All +Note: This file defines the working environment of the system. All defines listed in this file could also be compiler flags. I am listing all the defines (even if used as a compiler flag) so that they can be seen and documented. ***************************************************************************/ #ifndef __OSENV_H__ #define __OSENV_H__ #include /* ** Define the protocols to compile with. Currently, these defines are ** only for this header file and are used further down to define the protocol -** specific enviornment: +** specific environment: ** ** #define AG_PROTOCOL_ISCSI ** #define AG_PROTOCOL_FC */ /* ** Define the application types: ** ** #define INITIATOR_DRIVER ** #define TARGET_DRIVER */ /* ** Define the endian-ness of the host CPU using one of the following: ** ** #define AG_CPU_LITTLE_ENDIAN ** #define AG_CPU_BIG_ENDIAN */ /* ** Define the host CPU word size ** ** #define AG_CPU_32_BIT ** #define AG_CPU_64_BIT ** */ #ifdef CONFIG_IA64 #define AG_CPU_64_BIT #else #define AG_CPU_32_BIT #endif /* ** The following allow the code to use defines for word alignment and adding ** to allow for 32bit and 64bit system differences. */ #ifdef AG_CPU_32_BIT #define AG_WORD_ALIGN_ADD 3 #define AG_WORD_ALIGN_MASK 0xfffffffc #else #define AG_WORD_ALIGN_ADD 7 #define AG_WORD_ALIGN_MASK 0xfffffff8 #endif /*************************************************************************** -iSCSI enviornment - The following is used for compiling the iSCSI +iSCSI environment - The following is used for compiling the iSCSI protocol. **************************************************************************/ /* -** Define the existance of an external bus swapper using on of the +** Define the existence of an external bus swapper using on of the ** following: ** ** #define AG_SWAPPING_BUS ** #define AG_NON_SWAPPING_BUS ** */ /* ** Define the use of cache memory for message system: ** ** #define AG_CACHED_MSG_SYSTEM ** */ /* #define AG_CACHED_MSG_SYSTEM */ /*************************************************************************** -FC enviornment - The following is used for compiling the FC protocol. +FC environment - The following is used for compiling the FC protocol. **************************************************************************/ /* ** Define if an PMC-Sierra card is being used: ** ** #define CCFLAGS_PMC_SIERRA_BOARD ** */ /* ** Define if the TSDK is being used: ** ** #define FCLayer_Tsdk ** */ /* ** The following defines are not changed directly, but use either previous ** defines, or compiler directives. ** */ #ifdef AG_CPU_LITTLE_ENDIAN #define FC_DMA_LITTLE_ENDIAN #define FC_CPU_LITTLE_ENDIAN #define SA_DMA_LITTLE_ENDIAN #define SA_CPU_LITTLE_ENDIAN #endif #ifdef AG_CPU_BIG_ENDIAN #define FC_DMA_BIG_ENDIAN #define FC_CPU_BIG_ENDIAN #define SA_DMA_BIG_ENDIAN #define SA_CPU_BIG_ENDIAN #endif /* warning: leave this next line as-is. it is used for FC-Layer testing */ #undef FC_CHECKMACRO #endif /* __OSENV_H__ */ Index: user/ngie/release-pkg-fix-tests/sys/dev/pms/freebsd/driver/ini/src/agtiapi.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/dev/pms/freebsd/driver/ini/src/agtiapi.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/dev/pms/freebsd/driver/ini/src/agtiapi.c (revision 299091) @@ -1,6655 +1,6655 @@ /******************************************************************************* ** *Copyright (c) 2014 PMC-Sierra, Inc. All rights reserved. * *Redistribution and use in source and binary forms, with or without modification, are permitted provided *that the following conditions are met: *1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. *2. Redistributions in binary form must reproduce the above copyright notice, *this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * *THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, * *INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE *ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, *SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS *OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, *WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF *THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE ** *******************************************************************************/ #include __FBSDID("$FreeBSD$"); #include #define MAJOR_REVISION 1 #define MINOR_REVISION 3 #define BUILD_REVISION 10800 #include // defines used in kernel.h #include #include #include #include #include // types used in module initialization #include // cdevsw struct #include // uio struct #include #include #include // structs, prototypes for pci bus stuff #include #include #include #include // 1. for vtophys #include // 2. for vtophys #include // For pci_get macros #include #include #include #include #include #include #include #include #include #include #include #include #include // #include #include #include #include #include #include #include #include #include #include #include #include MALLOC_DEFINE( M_PMC_MCCB, "CCB List", "CCB List for PMCS driver" ); MALLOC_DEFINE( M_PMC_MSTL, "STLock malloc", "allocated in agtiapi_attach as memory for lock use" ); MALLOC_DEFINE( M_PMC_MDVT, "ag_device_t malloc", "allocated in agtiapi_attach as mem for ag_device_t pDevList" ); MALLOC_DEFINE( M_PMC_MPRT, "ag_portal_data_t malloc", "allocated in agtiapi_attach as mem for *pPortalData" ); MALLOC_DEFINE( M_PMC_MDEV, "tiDeviceHandle_t * malloc", "allocated in agtiapi_GetDevHandle as local mem for **agDev" ); MALLOC_DEFINE( M_PMC_MFLG, "lDevFlags * malloc", "allocated in agtiapi_GetDevHandle as local mem for * flags" ); #ifdef LINUX_PERBI_SUPPORT MALLOC_DEFINE( M_PMC_MSLR, "ag_slr_map_t malloc", "mem allocated in agtiapi_attach for pSLRList" ); MALLOC_DEFINE( M_PMC_MTGT, "ag_tgt_map_t malloc", "mem allocated in agtiapi_attach for pWWNList" ); #endif MALLOC_DEFINE(TEMP,"tempbuff","buffer for payload"); MALLOC_DEFINE(TEMP2, "tempbuff", "buffer for agtiapi_getdevlist"); STATIC U32 agtiapi_intx_mode = 0; STATIC U08 ag_Perbi = 0; STATIC U32 agtiapi_polling_mode = 0; STATIC U32 ag_card_good = 0; // * total card initialized STATIC U32 ag_option_flag = 0; // * adjustable parameter flag STATIC U32 agtiapi_1st_time = 1; STATIC U32 ag_timeout_secs = 10; //Made timeout equivalent to linux U32 gTiDebugLevel = 1; S32 ag_encryption_enable = 0; atomic_t outstanding_encrypted_io_count; #define cache_line_size() CACHE_LINE_SIZE #define PMCoffsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER) #define CPU_TO_LE32(dst, src) \ dst.lower = htole32(LOW_32_BITS(src)); \ dst.upper = htole32(HIGH_32_BITS(src)) #define CMND_TO_CHANNEL( ccb ) ( ccb->ccb_h.path_id ) #define CMND_TO_TARGET( ccb ) ( ccb->ccb_h.target_id ) #define CMND_TO_LUN( ccb ) ( ccb->ccb_h.target_lun ) STATIC U08 agtiapi_AddrModes[AGTIAPI_MAX_CHANNEL_NUM + 1] = { AGTIAPI_PERIPHERAL }; #ifdef LINUX_PERBI_SUPPORT // Holding area for target-WWN mapping assignments on the boot line static ag_mapping_t *agMappingList = NULL; // modified by agtiapi_Setup() #endif // * For Debugging Purpose #ifdef AGTIAPI_DEBUG #define AGTIAPI_WWN(name, len) wwnprintk(name, len) #else #define AGTIAPI_WWN(name, len) #endif #define AGTIAPI_WWNPRINTK(name, len, format, a...) \ AGTIAPI_PRINTK(format "name ", a); \ AGTIAPI_WWN((unsigned char*)name, len); #define AGTIAPI_ERR_WWNPRINTK(name, len, format, a...) \ printk(KERN_DEBUG format "name ", ## a); \ wwnprintk((unsigned char*)name, len); #define AGTIAPI_CPY_DEV_INFO(root, dev, pDev) \ tiINIGetDeviceInfo(root, dev, &pDev->devInfo); \ wwncpy(pDev); #ifdef AGTIAPI_LOCAL_LOCK #define AG_CARD_LOCAL_LOCK(lock) ,(lock) #define AG_SPIN_LOCK_IRQ(lock, flags) #define AG_SPIN_UNLOCK_IRQ(lock, flags) #define AG_SPIN_LOCK(lock) #define AG_SPIN_UNLOCK(lock) #define AG_GLOBAL_ARG(arg) #define AG_PERF_SPINLOCK(lock) #define AG_PERF_SPINLOCK_IRQ(lock, flags) #define AG_LOCAL_LOCK(lock) if (lock) \ mtx_lock(lock) #define AG_LOCAL_UNLOCK(lock) if (lock) \ mtx_unlock(lock) #define AG_LOCAL_FLAGS(_flags) unsigned long _flags = 0 #endif #define AG_GET_DONE_PCCB(pccb, pmcsc) \ { \ AG_LOCAL_LOCK(&pmcsc->doneLock); \ pccb = pmcsc->ccbDoneHead; \ if (pccb != NULL) \ { \ pmcsc->ccbDoneHead = NULL; \ pmcsc->ccbDoneTail = NULL; \ AG_LOCAL_UNLOCK(&pmcsc->doneLock); \ agtiapi_Done(pmcsc, pccb); \ } \ else \ AG_LOCAL_UNLOCK(&pmcsc->doneLock); \ } #define AG_GET_DONE_SMP_PCCB(pccb, pmcsc) \ { \ AG_LOCAL_LOCK(&pmcsc->doneSMPLock); \ pccb = pmcsc->smpDoneHead; \ if (pccb != NULL) \ { \ pmcsc->smpDoneHead = NULL; \ pmcsc->smpDoneTail = NULL; \ AG_LOCAL_UNLOCK(&pmcsc->doneSMPLock); \ agtiapi_SMPDone(pmcsc, pccb); \ } \ else \ AG_LOCAL_UNLOCK(&pmcsc->doneSMPLock); \ } #ifdef AGTIAPI_DUMP_IO_DEBUG #define AG_IO_DUMPCCB(pccb) agtiapi_DumpCCB(pccb) #else #define AG_IO_DUMPCCB(pccb) #endif #define SCHED_DELAY_JIFFIES 4 /* in seconds */ #ifdef HOTPLUG_SUPPORT #define AG_HOTPLUG_LOCK_INIT(lock) mxt_init(lock) #define AG_LIST_LOCK(lock) mtx_lock(lock) #define AG_LIST_UNLOCK(lock) mtx_unlock(lock) #else #define AG_HOTPLUG_LOCK_INIT(lock) #define AG_LIST_LOCK(lock) #define AG_LIST_UNLOCK(lock) #endif STATIC void agtiapi_CheckIOTimeout(void *data); static ag_card_info_t agCardInfoList[ AGTIAPI_MAX_CARDS ]; // card info list static void agtiapi_cam_action( struct cam_sim *, union ccb * ); static void agtiapi_cam_poll( struct cam_sim * ); // Function prototypes static d_open_t agtiapi_open; static d_close_t agtiapi_close; static d_read_t agtiapi_read; static d_write_t agtiapi_write; static d_ioctl_t agtiapi_CharIoctl; static void agtiapi_async(void *callback_arg, u_int32_t code, struct cam_path *path, void *arg); void agtiapi_adjust_queue_depth(struct cam_path *path, bit32 QueueDepth); // Character device entry points static struct cdevsw agtiapi_cdevsw = { .d_version = D_VERSION, .d_open = agtiapi_open, .d_close = agtiapi_close, .d_read = agtiapi_read, .d_write = agtiapi_write, .d_ioctl = agtiapi_CharIoctl, .d_name = "pmspcv", }; U32 maxTargets = 0; U32 ag_portal_count = 0; // In the cdevsw routines, we find our softc by using the si_drv1 member // of struct cdev. We set this variable to point to our softc in our // attach routine when we create the /dev entry. int agtiapi_open( struct cdev *dev, int oflags, int devtype, struct thread *td ) { struct agtiapi_softc *sc; /* Look up our softc. */ sc = dev->si_drv1; AGTIAPI_PRINTK("agtiapi_open\n"); AGTIAPI_PRINTK("Opened successfully. sc->my_dev %p\n", sc->my_dev); return( 0 ); } int agtiapi_close( struct cdev *dev, int fflag, int devtype, struct thread *td ) { struct agtiapi_softc *sc; // Look up our softc sc = dev->si_drv1; AGTIAPI_PRINTK("agtiapi_close\n"); AGTIAPI_PRINTK("Closed. sc->my_dev %p\n", sc->my_dev); return( 0 ); } int agtiapi_read( struct cdev *dev, struct uio *uio, int ioflag ) { struct agtiapi_softc *sc; // Look up our softc sc = dev->si_drv1; AGTIAPI_PRINTK( "agtiapi_read\n" ); AGTIAPI_PRINTK( "Asked to read %lu bytes. sc->my_dev %p\n", uio->uio_resid, sc->my_dev ); return( 0 ); } int agtiapi_write( struct cdev *dev, struct uio *uio, int ioflag ) { struct agtiapi_softc *sc; // Look up our softc sc = dev->si_drv1; AGTIAPI_PRINTK( "agtiapi_write\n" ); AGTIAPI_PRINTK( "Asked to write %lu bytes. sc->my_dev %p\n", uio->uio_resid, sc->my_dev ); return( 0 ); } int agtiapi_getdevlist( struct agtiapi_softc *pCard, tiIOCTLPayload_t *agIOCTLPayload ) { tdDeviceListPayload_t *pIoctlPayload = (tdDeviceListPayload_t *) agIOCTLPayload->FunctionSpecificArea; tdDeviceInfoIOCTL_t *pDeviceInfo = NULL; bit8 *pDeviceInfoOrg; tdsaDeviceData_t *pDeviceData = NULL; tiDeviceHandle_t **devList = NULL; tiDeviceHandle_t **devHandleArray = NULL; tiDeviceHandle_t *pDeviceHandle = NULL; bit32 x, memNeeded1; bit32 count, total; bit32 MaxDeviceCount; bit32 ret_val=IOCTL_CALL_INVALID_CODE; ag_portal_data_t *pPortalData; bit8 *pDeviceHandleList = NULL; AGTIAPI_PRINTK( "agtiapi_getdevlist: Enter\n" ); pDeviceInfoOrg = pIoctlPayload -> pDeviceInfo; MaxDeviceCount = pCard->devDiscover; if (MaxDeviceCount > pIoctlPayload->deviceLength ) { AGTIAPI_PRINTK( "agtiapi_getdevlist: MaxDeviceCount: %d > Requested device length: %d\n", MaxDeviceCount, pIoctlPayload->deviceLength ); MaxDeviceCount = pIoctlPayload->deviceLength; ret_val = IOCTL_CALL_FAIL; } AGTIAPI_PRINTK( "agtiapi_getdevlist: MaxDeviceCount: %d > Requested device length: %d\n", MaxDeviceCount, pIoctlPayload->deviceLength ); memNeeded1 = AG_ALIGNSIZE( MaxDeviceCount * sizeof(tiDeviceHandle_t *), sizeof(void *) ); AGTIAPI_PRINTK("agtiapi_getdevlist: portCount %d\n", pCard->portCount); devList = malloc(memNeeded1, TEMP2, M_WAITOK); if (devList == NULL) { AGTIAPI_PRINTK("agtiapi_getdevlist: failed to allocate memory\n"); ret_val = IOCTL_CALL_FAIL; agIOCTLPayload->Status = IOCTL_ERR_STATUS_INTERNAL_ERROR; return ret_val; } osti_memset(devList, 0, memNeeded1); pPortalData = &pCard->pPortalData[0]; pDeviceHandleList = (bit8*)devList; for (total = x = 0; x < pCard->portCount; x++, pPortalData++) { count = tiINIGetDeviceHandlesForWinIOCTL(&pCard->tiRoot, &pPortalData->portalInfo.tiPortalContext, ( tiDeviceHandle_t **)pDeviceHandleList ,MaxDeviceCount ); if (count == DISCOVERY_IN_PROGRESS) { AGTIAPI_PRINTK( "agtiapi_getdevlist: DISCOVERY_IN_PROGRESS on " "portal %d\n", x ); free(devList, TEMP2); ret_val = IOCTL_CALL_FAIL; agIOCTLPayload->Status = IOCTL_ERR_STATUS_INTERNAL_ERROR; return ret_val; } total += count; pDeviceHandleList+= count*sizeof(tiDeviceHandle_t *); MaxDeviceCount-= count; } if (total > pIoctlPayload->deviceLength) { total = pIoctlPayload->deviceLength; } // dump device information from device handle list count = 0; devHandleArray = devList; for (x = 0; x < pCard->devDiscover; x++) { pDeviceHandle = (tiDeviceHandle_t*)devHandleArray[x]; if (devList[x] != agNULL) { pDeviceData = devList [x]->tdData; pDeviceInfo = (tdDeviceInfoIOCTL_t*)(pDeviceInfoOrg + sizeof(tdDeviceInfoIOCTL_t) * count); if (pDeviceData != agNULL && pDeviceInfo != agNULL) { osti_memcpy( &pDeviceInfo->sasAddressHi, pDeviceData->agDeviceInfo.sasAddressHi, sizeof(bit32) ); osti_memcpy( &pDeviceInfo->sasAddressLo, pDeviceData->agDeviceInfo.sasAddressLo, sizeof(bit32) ); #if 0 pDeviceInfo->sasAddressHi = DMA_BEBIT32_TO_BIT32( pDeviceInfo->sasAddressHi ); pDeviceInfo->sasAddressLo = DMA_BEBIT32_TO_BIT32( pDeviceInfo->sasAddressLo ); #endif pDeviceInfo->deviceType = ( pDeviceData->agDeviceInfo.devType_S_Rate & 0x30 ) >> 4; pDeviceInfo->linkRate = pDeviceData->agDeviceInfo.devType_S_Rate & 0x0F; pDeviceInfo->phyId = pDeviceData->phyID; pDeviceInfo->ishost = pDeviceData->target_ssp_stp_smp; pDeviceInfo->DeviceHandle= (unsigned long)pDeviceHandle; if(pDeviceInfo->deviceType == 0x02) { bit8 *sasAddressHi; bit8 *sasAddressLo; tiIniGetDirectSataSasAddr(&pCard->tiRoot, pDeviceData->phyID, &sasAddressHi, &sasAddressLo); pDeviceInfo->sasAddressHi = DMA_BEBIT32_TO_BIT32(*(bit32*)sasAddressHi); pDeviceInfo->sasAddressLo = DMA_BEBIT32_TO_BIT32(*(bit32*)sasAddressLo) + pDeviceData->phyID + 16; } else { pDeviceInfo->sasAddressHi = DMA_BEBIT32_TO_BIT32( pDeviceInfo->sasAddressHi ); pDeviceInfo->sasAddressLo = DMA_BEBIT32_TO_BIT32( pDeviceInfo->sasAddressLo ); } AGTIAPI_PRINTK( "agtiapi_getdevlist: devicetype %x\n", pDeviceInfo->deviceType ); AGTIAPI_PRINTK( "agtiapi_getdevlist: linkrate %x\n", pDeviceInfo->linkRate ); AGTIAPI_PRINTK( "agtiapi_getdevlist: phyID %x\n", pDeviceInfo->phyId ); AGTIAPI_PRINTK( "agtiapi_getdevlist: addresshi %x\n", pDeviceInfo->sasAddressHi ); AGTIAPI_PRINTK( "agtiapi_getdevlist: addresslo %x\n", pDeviceInfo->sasAddressHi ); } else { AGTIAPI_PRINTK( "agtiapi_getdevlist: pDeviceData %p or pDeviceInfo " "%p is NULL %d\n", pDeviceData, pDeviceInfo, x ); } count++; } } pIoctlPayload->realDeviceCount = count; AGTIAPI_PRINTK( "agtiapi_getdevlist: Exit RealDeviceCount = %d\n", count ); if (devList) { free(devList, TEMP2); } if(ret_val != IOCTL_CALL_FAIL) { ret_val = IOCTL_CALL_SUCCESS; } agIOCTLPayload->Status = IOCTL_ERR_STATUS_OK; return ret_val; } /****************************************************************************** agtiapi_getCardInfo() Purpose: This function retrives the Card information Parameters: Return: A number - error 0 - HBA has been detected Note: ******************************************************************************/ int agtiapi_getCardInfo ( struct agtiapi_softc *pCard, U32_64 size, void *buffer ) { CardInfo_t *pCardInfo; pCardInfo = (CardInfo_t *)buffer; pCardInfo->deviceId = pci_get_device(pCard->my_dev); pCardInfo->vendorId =pci_get_vendor(pCard->my_dev) ; memcpy( pCardInfo->pciMemBaseSpc, pCard->pCardInfo->pciMemBaseSpc, ((sizeof(U32_64))*PCI_NUMBER_BARS) ); pCardInfo->deviceNum = pci_get_slot(pCard->my_dev); pCardInfo->pciMemBase = pCard->pCardInfo->pciMemBase; pCardInfo->pciIOAddrLow = pCard->pCardInfo->pciIOAddrLow; pCardInfo->pciIOAddrUp = pCard->pCardInfo->pciIOAddrUp; pCardInfo->busNum =pci_get_bus(pCard->my_dev); return 0; } void agtiapi_adjust_queue_depth(struct cam_path *path, bit32 QueueDepth) { struct ccb_relsim crs; xpt_setup_ccb(&crs.ccb_h, path, 5); crs.ccb_h.func_code = XPT_REL_SIMQ; crs.ccb_h.flags = CAM_DEV_QFREEZE; crs.release_flags = RELSIM_ADJUST_OPENINGS; crs.openings = QueueDepth; xpt_action((union ccb *)&crs); if(crs.ccb_h.status != CAM_REQ_CMP) { printf("XPT_REL_SIMQ failed\n"); } } static void agtiapi_async(void *callback_arg, u_int32_t code, struct cam_path *path, void *arg) { struct agtiapi_softc *pmsc; U32 TID; ag_device_t *targ; pmsc = (struct agtiapi_softc*)callback_arg; switch (code) { case AC_FOUND_DEVICE: { struct ccb_getdev *cgd; cgd = (struct ccb_getdev *)arg; if (cgd == NULL) { break; } TID = cgd->ccb_h.target_id; if (TID >= 0 && TID < maxTargets){ if (pmsc != NULL){ TID = INDEX(pmsc, TID); targ = &pmsc->pDevList[TID]; agtiapi_adjust_queue_depth(path, targ->qdepth); } } break; } default: break; } } /****************************************************************************** agtiapi_CharIoctl() Purpose: This function handles the ioctl from application layer Parameters: Return: A number - error 0 - HBA has been detected Note: ******************************************************************************/ static int agtiapi_CharIoctl( struct cdev *dev, u_long cmd, caddr_t data, int fflag, struct thread *td ) { struct sema mx; datatosend *load; // structure defined in lxcommon.h tiIOCTLPayload_t *pIoctlPayload; struct agtiapi_softc *pCard; pCard=dev->si_drv1; void *param1 = NULL; void *param2 = NULL; void *param3 = NULL; U32 status = 0; U32 retValue; int err = 0; int error = 0; tdDeviceListPayload_t *pDeviceList = NULL; unsigned long flags; switch (cmd) { case AGTIAPI_IOCTL: load=(datatosend*)data; pIoctlPayload = malloc(load->datasize,TEMP,M_WAITOK); AGTIAPI_PRINTK( "agtiapi_CharIoctl: old load->datasize = %d\n", load->datasize ); //Copy payload to kernel buffer, on success it returns 0 err = copyin(load->data,pIoctlPayload,load->datasize); if (err) { status = IOCTL_CALL_FAIL; return status; } sema_init(&mx,0,"sem"); pCard->pIoctlSem =&mx; pCard->up_count = pCard->down_count = 0; if ( pIoctlPayload->MajorFunction == IOCTL_MJ_GET_DEVICE_LIST ) { retValue = agtiapi_getdevlist(pCard, pIoctlPayload); if (retValue == 0) { pIoctlPayload->Status = IOCTL_CALL_SUCCESS; status = IOCTL_CALL_SUCCESS; } else { pIoctlPayload->Status = IOCTL_CALL_FAIL; status = IOCTL_CALL_FAIL; } //update new device length pDeviceList = (tdDeviceListPayload_t*)pIoctlPayload->FunctionSpecificArea; load->datasize =load->datasize - sizeof(tdDeviceInfoIOCTL_t) * (pDeviceList->deviceLength - pDeviceList->realDeviceCount); AGTIAPI_PRINTK( "agtiapi_CharIoctl: new load->datasize = %d\n", load->datasize ); } else if (pIoctlPayload->MajorFunction == IOCTL_MN_GET_CARD_INFO) { retValue = agtiapi_getCardInfo( pCard, pIoctlPayload->Length, (pIoctlPayload->FunctionSpecificArea) ); if (retValue == 0) { pIoctlPayload->Status = IOCTL_CALL_SUCCESS; status = IOCTL_CALL_SUCCESS; } else { pIoctlPayload->Status = IOCTL_CALL_FAIL; status = IOCTL_CALL_FAIL; } } else if ( pIoctlPayload->MajorFunction == IOCTL_MJ_CHECK_DPMC_EVENT ) { if ( pCard->flags & AGTIAPI_PORT_PANIC ) { strcpy ( pIoctlPayload->FunctionSpecificArea, "DPMC LEAN\n" ); } else { strcpy ( pIoctlPayload->FunctionSpecificArea, "do not dpmc lean\n" ); } pIoctlPayload->Status = IOCTL_CALL_SUCCESS; status = IOCTL_CALL_SUCCESS; } else if (pIoctlPayload->MajorFunction == IOCTL_MJ_CHECK_FATAL_ERROR ) { AGTIAPI_PRINTK("agtiapi_CharIoctl: IOCTL_MJ_CHECK_FATAL_ERROR call received for card %d\n", pCard->cardNo); //read port status to see if there is a fatal event if(pCard->flags & AGTIAPI_PORT_PANIC) { printf("agtiapi_CharIoctl: Port Panic Status For Card %d is True\n",pCard->cardNo); pIoctlPayload->Status = IOCTL_MJ_FATAL_ERR_CHK_SEND_TRUE; } else { AGTIAPI_PRINTK("agtiapi_CharIoctl: Port Panic Status For Card %d is False\n",pCard->cardNo); pIoctlPayload->Status = IOCTL_MJ_FATAL_ERR_CHK_SEND_FALSE; } status = IOCTL_CALL_SUCCESS; } else if (pIoctlPayload->MajorFunction == IOCTL_MJ_FATAL_ERROR_DUMP_COMPLETE) { AGTIAPI_PRINTK("agtiapi_CharIoctl: IOCTL_MJ_FATAL_ERROR_DUMP_COMPLETE call received for card %d\n", pCard->cardNo); //set flags bit status to be a soft reset pCard->flags |= AGTIAPI_SOFT_RESET; //trigger soft reset for the card retValue = agtiapi_ResetCard (pCard, &flags); if(retValue == AGTIAPI_SUCCESS) { //clear port panic status pCard->flags &= ~AGTIAPI_PORT_PANIC; pIoctlPayload->Status = IOCTL_MJ_FATAL_ERROR_SOFT_RESET_TRIG; status = IOCTL_CALL_SUCCESS; } else { pIoctlPayload->Status = IOCTL_CALL_FAIL; status = IOCTL_CALL_FAIL; } } else { status = tiCOMMgntIOCTL( &pCard->tiRoot, pIoctlPayload, pCard, param2, param3 ); if (status == IOCTL_CALL_PENDING) { ostiIOCTLWaitForSignal(&pCard->tiRoot,NULL, NULL, NULL); status = IOCTL_CALL_SUCCESS; } } pCard->pIoctlSem = NULL; err = 0; //copy kernel buffer to userland buffer err=copyout(pIoctlPayload,load->data,load->datasize); if (err) { status = IOCTL_CALL_FAIL; return status; } free(pIoctlPayload,TEMP); pIoctlPayload=NULL; break; default: error = ENOTTY; break; } return(status); } /****************************************************************************** agtiapi_probe() Purpose: This function initialize and registere all detected HBAs. The first function being called in driver after agtiapi_probe() Parameters: device_t dev (IN) - device pointer Return: A number - error 0 - HBA has been detected Note: ******************************************************************************/ static int agtiapi_probe( device_t dev ) { int retVal; int thisCard; ag_card_info_t *thisCardInst; thisCard = device_get_unit( dev ); if ( thisCard >= AGTIAPI_MAX_CARDS ) { device_printf( dev, "Too many PMC-Sierra cards detected ERROR!\n" ); return (ENXIO); // maybe change to different return value? } thisCardInst = &agCardInfoList[ thisCard ]; retVal = agtiapi_ProbeCard( dev, thisCardInst, thisCard ); if ( retVal ) return (ENXIO); // maybe change to different return value? return( BUS_PROBE_DEFAULT ); // successful probe } /****************************************************************************** agtiapi_attach() Purpose: This function initialize and registere all detected HBAs. The first function being called in driver after agtiapi_probe() Parameters: device_t dev (IN) - device pointer Return: A number - error 0 - HBA has been detected Note: ******************************************************************************/ static int agtiapi_attach( device_t devx ) { // keeping get_unit call to once int thisCard = device_get_unit( devx ); struct agtiapi_softc *pmsc; ag_card_info_t *thisCardInst = &agCardInfoList[ thisCard ]; ag_resource_info_t *pRscInfo; int idx; int lenRecv; char buffer [256], *pLastUsedChar; union ccb *ccb; int bus, tid, lun; struct ccb_setasync csa; AGTIAPI_PRINTK("agtiapi_attach: start dev %p thisCard %d\n", devx, thisCard); // AGTIAPI_PRINTK( "agtiapi_attach: entry pointer values A %p / %p\n", // thisCardInst->pPCIDev, thisCardInst ); AGTIAPI_PRINTK( "agtiapi_attach: deviceID: 0x%x\n", pci_get_devid( devx ) ); TUNABLE_INT_FETCH( "DPMC_TIMEOUT_SECS", &ag_timeout_secs ); TUNABLE_INT_FETCH( "DPMC_TIDEBUG_LEVEL", &gTiDebugLevel ); // printf( "agtiapi_attach: debugLevel %d, timeout %d\n", // gTiDebugLevel, ag_timeout_secs ); if ( ag_timeout_secs < 1 ) { ag_timeout_secs = 1; // set minimum timeout value of 1 second } ag_timeout_secs = (ag_timeout_secs * 1000); // convert to millisecond notation // Look up our softc and initialize its fields. pmsc = device_get_softc( devx ); pmsc->my_dev = devx; /* Get NumberOfPortals */ if ((ostiGetTransportParam( &pmsc->tiRoot, "Global", "CardDefault", agNULL, agNULL, agNULL, agNULL, "NumberOfPortals", buffer, 255, &lenRecv ) == tiSuccess) && (lenRecv != 0)) { if (osti_strncmp(buffer, "0x", 2) == 0) { ag_portal_count = osti_strtoul (buffer, &pLastUsedChar, 0); } else { ag_portal_count = osti_strtoul (buffer, &pLastUsedChar, 10); } if (ag_portal_count > AGTIAPI_MAX_PORTALS) ag_portal_count = AGTIAPI_MAX_PORTALS; } else { ag_portal_count = AGTIAPI_MAX_PORTALS; } AGTIAPI_PRINTK( "agtiapi_attach: ag_portal_count=%d\n", ag_portal_count ); // initialize hostdata structure pmsc->flags |= AGTIAPI_INIT_TIME | AGTIAPI_SCSI_REGISTERED | AGTIAPI_INITIATOR; pmsc->cardNo = thisCard; pmsc->ccbTotal = 0; pmsc->portCount = ag_portal_count; pmsc->pCardInfo = thisCardInst; pmsc->tiRoot.osData = pmsc; pmsc->pCardInfo->pCard = (void *)pmsc; pmsc->VidDid = ( pci_get_vendor(devx) << 16 ) | pci_get_device( devx ); pmsc->SimQFrozen = agFALSE; pmsc->devq_flag = agFALSE; pRscInfo = &thisCardInst->tiRscInfo; osti_memset(buffer, 0, 256); lenRecv = 0; /* Get MaxTargets */ if ((ostiGetTransportParam( &pmsc->tiRoot, "Global", "InitiatorParms", agNULL, agNULL, agNULL, agNULL, "MaxTargets", buffer, sizeof(buffer), &lenRecv ) == tiSuccess) && (lenRecv != 0)) { if (osti_strncmp(buffer, "0x", 2) == 0) { maxTargets = osti_strtoul (buffer, &pLastUsedChar, 0); AGTIAPI_PRINTK( "agtiapi_attach: maxTargets = osti_strtoul 0 \n" ); } else { maxTargets = osti_strtoul (buffer, &pLastUsedChar, 10); AGTIAPI_PRINTK( "agtiapi_attach: maxTargets = osti_strtoul 10\n" ); } } else { if(Is_ADP8H(pmsc)) maxTargets = AGTIAPI_MAX_DEVICE_8H; else if(Is_ADP7H(pmsc)) maxTargets = AGTIAPI_MAX_DEVICE_7H; else maxTargets = AGTIAPI_MAX_DEVICE; } if (maxTargets > AGTIAPI_HW_LIMIT_DEVICE) { AGTIAPI_PRINTK( "agtiapi_attach: maxTargets: %d > AGTIAPI_HW_LIMIT_DEVICE: %d\n", maxTargets, AGTIAPI_HW_LIMIT_DEVICE ); AGTIAPI_PRINTK( "agtiapi_attach: change maxTargets = AGTIAPI_HW_LIMIT_DEVICE\n" ); maxTargets = AGTIAPI_HW_LIMIT_DEVICE; } pmsc->devDiscover = maxTargets ; #ifdef HIALEAH_ENCRYPTION ag_encryption_enable = 1; if(ag_encryption_enable && pci_get_device(pmsc->pCardInfo->pPCIDev) == PCI_DEVICE_ID_HIALEAH_HBA_SPCVE) { pmsc->encrypt = 1; pRscInfo->tiLoLevelResource.loLevelOption.encryption = agTRUE; printf("agtiapi_attach: Encryption Enabled\n" ); } #endif // ## for now, skip calls to ostiGetTransportParam(...) // ## for now, skip references to DIF & EDC // Create a /dev entry for this device. The kernel will assign us // a major number automatically. We use the unit number of this // device as the minor number and name the character device // "agtiapi". pmsc->my_cdev = make_dev( &agtiapi_cdevsw, thisCard, UID_ROOT, GID_WHEEL, 0600, "spcv%u", thisCard ); pmsc->my_cdev->si_drv1 = pmsc; mtx_init( &thisCardInst->pmIOLock, "pmc SAS I/O lock", NULL, MTX_DEF|MTX_RECURSE ); struct cam_devq *devq; /* set the maximum number of pending IOs */ devq = cam_simq_alloc( AGTIAPI_MAX_CAM_Q_DEPTH ); if (devq == NULL) { AGTIAPI_PRINTK("agtiapi_attach: cam_simq_alloc is NULL\n" ); return( EIO ); } struct cam_sim *lsim; lsim = cam_sim_alloc( agtiapi_cam_action, agtiapi_cam_poll, "pmspcbsd", pmsc, thisCard, &thisCardInst->pmIOLock, 1, // queued per target AGTIAPI_MAX_CAM_Q_DEPTH, // max tag depth devq ); if ( lsim == NULL ) { cam_simq_free( devq ); AGTIAPI_PRINTK("agtiapi_attach: cam_sim_alloc is NULL\n" ); return( EIO ); } pmsc->dev_scan = agFALSE; //one cam sim per scsi bus mtx_lock( &thisCardInst->pmIOLock ); if ( xpt_bus_register( lsim, devx, 0 ) != CAM_SUCCESS ) { // bus 0 cam_sim_free( lsim, TRUE ); mtx_unlock( &thisCardInst->pmIOLock ); AGTIAPI_PRINTK("agtiapi_attach: xpt_bus_register fails\n" ); return( EIO ); } pmsc->sim = lsim; bus = cam_sim_path(pmsc->sim); tid = CAM_TARGET_WILDCARD; lun = CAM_LUN_WILDCARD; ccb = xpt_alloc_ccb_nowait(); if (ccb == agNULL) { mtx_unlock( &thisCardInst->pmIOLock ); cam_sim_free( lsim, TRUE ); cam_simq_free( devq ); return ( EIO ); } if (xpt_create_path(&ccb->ccb_h.path, agNULL, bus, tid, CAM_LUN_WILDCARD) != CAM_REQ_CMP) { mtx_unlock( &thisCardInst->pmIOLock ); cam_sim_free( lsim, TRUE ); cam_simq_free( devq ); xpt_free_ccb(ccb); return( EIO ); } pmsc->path = ccb->ccb_h.path; xpt_setup_ccb(&csa.ccb_h, pmsc->path, 5); csa.ccb_h.func_code = XPT_SASYNC_CB; csa.event_enable = AC_FOUND_DEVICE; csa.callback = agtiapi_async; csa.callback_arg = pmsc; xpt_action((union ccb *)&csa); if (csa.ccb_h.status != CAM_REQ_CMP) { AGTIAPI_PRINTK("agtiapi_attach: Unable to register AC_FOUND_DEVICE\n" ); } lsim->devq = devq; mtx_unlock( &thisCardInst->pmIOLock ); // get TD and lower layer memory requirements tiCOMGetResource( &pmsc->tiRoot, &pRscInfo->tiLoLevelResource, &pRscInfo->tiInitiatorResource, NULL, &pRscInfo->tiSharedMem ); agtiapi_ScopeDMARes( thisCardInst ); AGTIAPI_PRINTK( "agtiapi_attach: size from the call agtiapi_ScopeDMARes" " 0x%x \n", pmsc->typhn ); // initialize card information and get resource ready if( agtiapi_InitResource( thisCardInst ) == AGTIAPI_FAIL ) { AGTIAPI_PRINTK( "agtiapi_attach: Card %d initialize resource ERROR\n", thisCard ); } // begin: allocate and initialize card portal info resource ag_portal_data_t *pPortalData; if (pmsc->portCount == 0) { pmsc->pPortalData = NULL; } else { pmsc->pPortalData = (ag_portal_data_t *) malloc( sizeof(ag_portal_data_t) * pmsc->portCount, M_PMC_MPRT, M_ZERO | M_WAITOK ); if (pmsc->pPortalData == NULL) { AGTIAPI_PRINTK( "agtiapi_attach: Portal memory allocation ERROR\n" ); } } pPortalData = pmsc->pPortalData; for( idx = 0; idx < pmsc->portCount; idx++ ) { pPortalData->pCard = pmsc; pPortalData->portalInfo.portID = idx; pPortalData->portalInfo.tiPortalContext.osData = (void *)pPortalData; pPortalData++; } // end: allocate and initialize card portal info resource // begin: enable msix // setup msix // map to interrupt handler int error = 0; int mesgs = MAX_MSIX_NUM_VECTOR; int i, cnt; void (*intrHandler[MAX_MSIX_NUM_ISR])(void *arg) = { agtiapi_IntrHandler0, agtiapi_IntrHandler1, agtiapi_IntrHandler2, agtiapi_IntrHandler3, agtiapi_IntrHandler4, agtiapi_IntrHandler5, agtiapi_IntrHandler6, agtiapi_IntrHandler7, agtiapi_IntrHandler8, agtiapi_IntrHandler9, agtiapi_IntrHandler10, agtiapi_IntrHandler11, agtiapi_IntrHandler12, agtiapi_IntrHandler13, agtiapi_IntrHandler14, agtiapi_IntrHandler15 }; cnt = pci_msix_count(devx); AGTIAPI_PRINTK("supported MSIX %d\n", cnt); //this should be 64 mesgs = MIN(mesgs, cnt); error = pci_alloc_msix(devx, &mesgs); if (error != 0) { printf( "pci_alloc_msix error %d\n", error ); AGTIAPI_PRINTK("error %d\n", error); return( EIO ); } for(i=0; i < mesgs; i++) { pmsc->rscID[i] = i + 1; pmsc->irq[i] = bus_alloc_resource_any( devx, SYS_RES_IRQ, &pmsc->rscID[i], RF_ACTIVE ); if( pmsc->irq[i] == NULL ) { printf( "RES_IRQ went terribly bad at %d\n", i ); return( EIO ); } if ( (error = bus_setup_intr( devx, pmsc->irq[i], INTR_TYPE_CAM | INTR_MPSAFE, NULL, intrHandler[i], pmsc, &pmsc->intrcookie[i] ) ) != 0 ) { device_printf( devx, "Failed to register handler" ); return( EIO ); } } pmsc->flags |= AGTIAPI_IRQ_REQUESTED; pmsc->pCardInfo->maxInterruptVectors = MAX_MSIX_NUM_VECTOR; // end: enable msix int ret = 0; ret = agtiapi_InitCardSW(pmsc); if (ret == AGTIAPI_FAIL || ret == AGTIAPI_UNKNOWN) { AGTIAPI_PRINTK( "agtiapi_attach: agtiapi_InitCardSW failure %d\n", ret ); return( EIO ); } pmsc->ccbFreeList = NULL; pmsc->ccbChainList = NULL; pmsc->ccbAllocList = NULL; pmsc->flags |= ( AGTIAPI_INSTALLED ); ret = agtiapi_alloc_requests( pmsc ); if( ret != 0 ) { AGTIAPI_PRINTK( "agtiapi_attach: agtiapi_alloc_requests failure %d\n", ret ); return( EIO ); } ret = agtiapi_alloc_ostimem( pmsc ); if (ret != AGTIAPI_SUCCESS) { AGTIAPI_PRINTK( "agtiapi_attach: agtiapi_alloc_ostimem failure %d\n", ret ); return( EIO ); } ret = agtiapi_InitCardHW( pmsc ); if (ret != 0) { AGTIAPI_PRINTK( "agtiapi_attach: agtiapi_InitCardHW failure %d\n", ret ); return( EIO ); } #ifdef HIALEAH_ENCRYPTION if(pmsc->encrypt) { if((agtiapi_SetupEncryption(pmsc)) < 0) AGTIAPI_PRINTK("SetupEncryption returned less than 0\n"); } #endif pmsc->flags &= ~AGTIAPI_INIT_TIME; return( 0 ); } /****************************************************************************** agtiapi_InitCardSW() Purpose: Host Bus Adapter Initialization Parameters: struct agtiapi_softc *pmsc (IN) Pointer to the HBA data structure Return: AGTIAPI_SUCCESS - success AGTIAPI_FAIL - fail Note: TBD, need chip register information ******************************************************************************/ STATIC agBOOLEAN agtiapi_InitCardSW( struct agtiapi_softc *pmsc ) { ag_card_info_t *thisCardInst = pmsc->pCardInfo; ag_resource_info_t *pRscInfo = &thisCardInst->tiRscInfo; int initSWIdx; // begin: agtiapi_InitCardSW() // now init some essential locks n agtiapi_InitCardSW mtx_init( &pmsc->sendLock, "local q send lock", NULL, MTX_DEF ); mtx_init( &pmsc->doneLock, "local q done lock", NULL, MTX_DEF ); mtx_init( &pmsc->sendSMPLock, "local q send lock", NULL, MTX_DEF ); mtx_init( &pmsc->doneSMPLock, "local q done lock", NULL, MTX_DEF ); mtx_init( &pmsc->ccbLock, "ccb list lock", NULL, MTX_DEF ); mtx_init( &pmsc->devListLock, "hotP devListLock", NULL, MTX_DEF ); mtx_init( &pmsc->memLock, "dynamic memory lock", NULL, MTX_DEF ); mtx_init( &pmsc->freezeLock, "sim freeze lock", NULL, MTX_DEF | MTX_RECURSE); // initialize lower layer resources //## if (pCard->flags & AGTIAPI_INIT_TIME) { #ifdef HIALEAH_ENCRYPTION /* Enable encryption if chip supports it */ if (pci_get_device(pmsc->pCardInfo->pPCIDev) == PCI_DEVICE_ID_HIALEAH_HBA_SPCVE) pmsc->encrypt = 1; if (pmsc->encrypt) pRscInfo->tiLoLevelResource.loLevelOption.encryption = agTRUE; #endif pmsc->flags &= ~(AGTIAPI_PORT_INITIALIZED | AGTIAPI_SYS_INTR_ON); // For now, up to 16 MSIX vectors are supported thisCardInst->tiRscInfo.tiLoLevelResource.loLevelOption. maxInterruptVectors = pmsc->pCardInfo->maxInterruptVectors; AGTIAPI_PRINTK( "agtiapi_InitCardSW: maxInterruptVectors set to %d", pmsc->pCardInfo->maxInterruptVectors ); thisCardInst->tiRscInfo.tiLoLevelResource.loLevelOption.max_MSI_InterruptVectors = 0; thisCardInst->tiRscInfo.tiLoLevelResource.loLevelOption.flag = 0; pRscInfo->tiLoLevelResource.loLevelOption.maxNumOSLocks = 0; AGTIAPI_PRINTK( "agtiapi_InitCardSW: tiCOMInit root %p, dev %p, pmsc %p\n", &pmsc->tiRoot, pmsc->my_dev, pmsc ); if( tiCOMInit( &pmsc->tiRoot, &thisCardInst->tiRscInfo.tiLoLevelResource, &thisCardInst->tiRscInfo.tiInitiatorResource, NULL, &thisCardInst->tiRscInfo.tiSharedMem ) != tiSuccess ) { AGTIAPI_PRINTK( "agtiapi_InitCardSW: tiCOMInit ERROR\n" ); return AGTIAPI_FAIL; } int maxLocks; maxLocks = pRscInfo->tiLoLevelResource.loLevelOption.numOfQueuesPerPort; pmsc->STLock = malloc( ( maxLocks * sizeof(struct mtx) ), M_PMC_MSTL, M_ZERO | M_WAITOK ); for( initSWIdx = 0; initSWIdx < maxLocks; initSWIdx++ ) { // init all indexes mtx_init( &pmsc->STLock[initSWIdx], "LL & TD lock", NULL, MTX_DEF ); } if( tiCOMPortInit( &pmsc->tiRoot, agFALSE ) != tiSuccess ) { printf( "agtiapi_InitCardSW: tiCOMPortInit ERROR -- AGTIAPI_FAIL\n" ); return AGTIAPI_FAIL; } AGTIAPI_PRINTK( "agtiapi_InitCardSW: tiCOMPortInit" " root %p, dev %p, pmsc %p\n", &pmsc->tiRoot, pmsc->my_dev, pmsc ); pmsc->flags |= AGTIAPI_PORT_INITIALIZED; pmsc->freezeSim = agFALSE; #ifdef HIALEAH_ENCRYPTION atomic_set(&outstanding_encrypted_io_count, 0); /*fix below*/ /*if(pmsc->encrypt && (pmsc->flags & AGTIAPI_INIT_TIME)) if((agtiapi_SetupEncryptionPools(pmsc)) != 0) printf("SetupEncryptionPools failed\n"); */ #endif return AGTIAPI_SUCCESS; // end: agtiapi_InitCardSW() } /****************************************************************************** agtiapi_InitCardHW() Purpose: Host Bus Adapter Initialization Parameters: struct agtiapi_softc *pmsc (IN) Pointer to the HBA data structure Return: AGTIAPI_SUCCESS - success AGTIAPI_FAIL - fail Note: TBD, need chip register information ******************************************************************************/ STATIC agBOOLEAN agtiapi_InitCardHW( struct agtiapi_softc *pmsc ) { U32 numVal; U32 count; U32 loop; // begin: agtiapi_InitCardHW() ag_portal_info_t *pPortalInfo = NULL; ag_portal_data_t *pPortalData; // ISR is registered, enable chip interrupt. tiCOMSystemInterruptsActive( &pmsc->tiRoot, agTRUE ); pmsc->flags |= AGTIAPI_SYS_INTR_ON; numVal = sizeof(ag_device_t) * pmsc->devDiscover; pmsc->pDevList = (ag_device_t *)malloc( numVal, M_PMC_MDVT, M_ZERO | M_WAITOK ); if( !pmsc->pDevList ) { AGTIAPI_PRINTK( "agtiapi_InitCardHW: kmalloc %d DevList ERROR\n", numVal ); panic( "agtiapi_InitCardHW\n" ); return AGTIAPI_FAIL; } #ifdef LINUX_PERBI_SUPPORT numVal = sizeof(ag_slr_map_t) * pmsc->devDiscover; pmsc->pSLRList = (ag_slr_map_t *)malloc( numVal, M_PMC_MSLR, M_ZERO | M_WAITOK ); if( !pmsc->pSLRList ) { AGTIAPI_PRINTK( "agtiapi_InitCardHW: kmalloc %d SLRList ERROR\n", numVal ); panic( "agtiapi_InitCardHW SLRL\n" ); return AGTIAPI_FAIL; } numVal = sizeof(ag_tgt_map_t) * pmsc->devDiscover; pmsc->pWWNList = (ag_tgt_map_t *)malloc( numVal, M_PMC_MTGT, M_ZERO | M_WAITOK ); if( !pmsc->pWWNList ) { AGTIAPI_PRINTK( "agtiapi_InitCardHW: kmalloc %d WWNList ERROR\n", numVal ); panic( "agtiapi_InitCardHW WWNL\n" ); return AGTIAPI_FAIL; } // Get the WWN_to_target_ID mappings from the // holding area which contains the input of the // system configuration file. if( ag_Perbi ) agtiapi_GetWWNMappings( pmsc, agMappingList ); else { agtiapi_GetWWNMappings( pmsc, 0 ); if( agMappingList ) printf( "agtiapi_InitCardHW: WWN PERBI disabled WARN\n" ); } #endif //agtiapi_DelaySec(5); DELAY( 500000 ); pmsc->tgtCount = 0; pmsc->flags &= ~AGTIAPI_CB_DONE; pPortalData = pmsc->pPortalData; //start port for (count = 0; count < pmsc->portCount; count++) { AG_SPIN_LOCK_IRQ( agtiapi_host_lock, flags ); pPortalInfo = &pPortalData->portalInfo; pPortalInfo->portStatus &= ~( AGTIAPI_PORT_START | AGTIAPI_PORT_DISC_READY | AGTIAPI_DISC_DONE | AGTIAPI_DISC_COMPLETE ); for (loop = 0; loop < AGTIAPI_LOOP_MAX; loop++) { AGTIAPI_PRINTK( "tiCOMPortStart entry data %p / %d / %p\n", &pmsc->tiRoot, pPortalInfo->portID, &pPortalInfo->tiPortalContext ); if( tiCOMPortStart( &pmsc->tiRoot, pPortalInfo->portID, &pPortalInfo->tiPortalContext, 0 ) != tiSuccess ) { AG_SPIN_UNLOCK_IRQ( agtiapi_host_lock, flags ); agtiapi_DelayMSec( AGTIAPI_EXTRA_DELAY ); AG_SPIN_LOCK_IRQ(agtiapi_host_lock, flags); AGTIAPI_PRINTK( "tiCOMPortStart failed -- no loop, portalData %p\n", pPortalData ); } else { AGTIAPI_PRINTK( "tiCOMPortStart success no loop, portalData %p\n", pPortalData ); break; } } // end of for loop /* release lock */ AG_SPIN_UNLOCK_IRQ( agtiapi_host_lock, flags ); if( loop >= AGTIAPI_LOOP_MAX ) { return AGTIAPI_FAIL; } tiCOMGetPortInfo( &pmsc->tiRoot, &pPortalInfo->tiPortalContext, &pPortalInfo->tiPortInfo ); pPortalData++; } /* discover target device */ #ifndef HOTPLUG_SUPPORT agtiapi_DiscoverTgt( pCard ); #endif pmsc->flags |= AGTIAPI_INSTALLED; if( pmsc->flags & AGTIAPI_INIT_TIME ) { agtiapi_TITimer( (void *)pmsc ); pmsc->flags |= AGTIAPI_TIMER_ON; } return 0; } /****************************************************************************** agtiapi_IntrHandlerx_() Purpose: Interrupt service routine. Parameters: void arg (IN) Pointer to the HBA data structure bit32 idx (IN) Vector index ******************************************************************************/ void agtiapi_IntrHandlerx_( void *arg, int index ) { struct agtiapi_softc *pCard; int rv; pCard = (struct agtiapi_softc *)arg; #ifndef AGTIAPI_DPC ccb_t *pccb; #endif AG_LOCAL_LOCK(&(pCard->pCardInfo->pmIOLock)); AG_PERF_SPINLOCK(agtiapi_host_lock); if (pCard->flags & AGTIAPI_SHUT_DOWN) goto ext; rv = tiCOMInterruptHandler(&pCard->tiRoot, index); if (rv == agFALSE) { /* not our irq */ AG_SPIN_UNLOCK(agtiapi_host_lock); AG_LOCAL_UNLOCK(&(pCard->pCardInfo->pmIOLock)); return; } #ifdef AGTIAPI_DPC tasklet_hi_schedule(&pCard->tasklet_dpc[idx]); #else /* consume all completed entries, 100 is random number to be big enough */ tiCOMDelayedInterruptHandler(&pCard->tiRoot, index, 100, tiInterruptContext); AG_GET_DONE_PCCB(pccb, pCard); AG_GET_DONE_SMP_PCCB(pccb, pCard); #endif ext: AG_SPIN_UNLOCK(agtiapi_host_lock); AG_LOCAL_UNLOCK(&(pCard->pCardInfo->pmIOLock)); return; } /****************************************************************************** agtiapi_IntrHandler0() Purpose: Interrupt service routine for interrupt vector index 0. Parameters: void arg (IN) Pointer to the HBA data structure ******************************************************************************/ void agtiapi_IntrHandler0( void *arg ) { agtiapi_IntrHandlerx_( arg, 0 ); return; } /****************************************************************************** agtiapi_IntrHandler1() Purpose: Interrupt service routine for interrupt vector index 1. Parameters: void arg (IN) Pointer to the HBA data structure ******************************************************************************/ void agtiapi_IntrHandler1( void *arg ) { agtiapi_IntrHandlerx_( arg, 1 ); return; } /****************************************************************************** agtiapi_IntrHandler2() Purpose: Interrupt service routine for interrupt vector index 2. Parameters: void arg (IN) Pointer to the HBA data structure ******************************************************************************/ void agtiapi_IntrHandler2( void *arg ) { agtiapi_IntrHandlerx_( arg, 2 ); return; } /****************************************************************************** agtiapi_IntrHandler3() Purpose: Interrupt service routine for interrupt vector index 3. Parameters: void arg (IN) Pointer to the HBA data structure ******************************************************************************/ void agtiapi_IntrHandler3( void *arg ) { agtiapi_IntrHandlerx_( arg, 3 ); return; } /****************************************************************************** agtiapi_IntrHandler4() Purpose: Interrupt service routine for interrupt vector index 4. Parameters: void arg (IN) Pointer to the HBA data structure ******************************************************************************/ void agtiapi_IntrHandler4( void *arg ) { agtiapi_IntrHandlerx_( arg, 4 ); return; } /****************************************************************************** agtiapi_IntrHandler5() Purpose: Interrupt service routine for interrupt vector index 5. Parameters: void arg (IN) Pointer to the HBA data structure ******************************************************************************/ void agtiapi_IntrHandler5( void *arg ) { agtiapi_IntrHandlerx_( arg, 5 ); return; } /****************************************************************************** agtiapi_IntrHandler6() Purpose: Interrupt service routine for interrupt vector index 6. Parameters: void arg (IN) Pointer to the HBA data structure ******************************************************************************/ void agtiapi_IntrHandler6( void *arg ) { agtiapi_IntrHandlerx_( arg, 6 ); return; } /****************************************************************************** agtiapi_IntrHandler7() Purpose: Interrupt service routine for interrupt vector index 7. Parameters: void arg (IN) Pointer to the HBA data structure ******************************************************************************/ void agtiapi_IntrHandler7( void *arg ) { agtiapi_IntrHandlerx_( arg, 7 ); return; } /****************************************************************************** agtiapi_IntrHandler8() Purpose: Interrupt service routine for interrupt vector index 8. Parameters: void arg (IN) Pointer to the HBA data structure ******************************************************************************/ void agtiapi_IntrHandler8( void *arg ) { agtiapi_IntrHandlerx_( arg, 8 ); return; } /****************************************************************************** agtiapi_IntrHandler9() Purpose: Interrupt service routine for interrupt vector index 9. Parameters: void arg (IN) Pointer to the HBA data structure ******************************************************************************/ void agtiapi_IntrHandler9( void *arg ) { agtiapi_IntrHandlerx_( arg, 9 ); return; } /****************************************************************************** agtiapi_IntrHandler10() Purpose: Interrupt service routine for interrupt vector index 10. Parameters: void arg (IN) Pointer to the HBA data structure ******************************************************************************/ void agtiapi_IntrHandler10( void *arg ) { agtiapi_IntrHandlerx_( arg, 10 ); return; } /****************************************************************************** agtiapi_IntrHandler11() Purpose: Interrupt service routine for interrupt vector index 11. Parameters: void arg (IN) Pointer to the HBA data structure ******************************************************************************/ void agtiapi_IntrHandler11( void *arg ) { agtiapi_IntrHandlerx_( arg, 11 ); return; } /****************************************************************************** agtiapi_IntrHandler12() Purpose: Interrupt service routine for interrupt vector index 12. Parameters: void arg (IN) Pointer to the HBA data structure ******************************************************************************/ void agtiapi_IntrHandler12( void *arg ) { agtiapi_IntrHandlerx_( arg, 12 ); return; } /****************************************************************************** agtiapi_IntrHandler13() Purpose: Interrupt service routine for interrupt vector index 13. Parameters: void arg (IN) Pointer to the HBA data structure ******************************************************************************/ void agtiapi_IntrHandler13( void *arg ) { agtiapi_IntrHandlerx_( arg, 13 ); return; } /****************************************************************************** agtiapi_IntrHandler14() Purpose: Interrupt service routine for interrupt vector index 14. Parameters: void arg (IN) Pointer to the HBA data structure ******************************************************************************/ void agtiapi_IntrHandler14( void *arg ) { agtiapi_IntrHandlerx_( arg, 14 ); return; } /****************************************************************************** agtiapi_IntrHandler15() Purpose: Interrupt service routine for interrupt vector index 15. Parameters: void arg (IN) Pointer to the HBA data structure ******************************************************************************/ void agtiapi_IntrHandler15( void *arg ) { agtiapi_IntrHandlerx_( arg, 15 ); return; } static void agtiapi_SglMemoryCB( void *arg, bus_dma_segment_t *dm_segs, int nseg, int error ) { bus_addr_t *addr; AGTIAPI_PRINTK("agtiapi_SglMemoryCB: start\n"); if (error != 0) { AGTIAPI_PRINTK("agtiapi_SglMemoryCB: error %d\n", error); panic("agtiapi_SglMemoryCB: error %d\n", error); return; } addr = arg; *addr = dm_segs[0].ds_addr; return; } static void agtiapi_MemoryCB( void *arg, bus_dma_segment_t *dm_segs, int nseg, int error ) { bus_addr_t *addr; AGTIAPI_PRINTK("agtiapi_MemoryCB: start\n"); if (error != 0) { AGTIAPI_PRINTK("agtiapi_MemoryCB: error %d\n", error); panic("agtiapi_MemoryCB: error %d\n", error); return; } addr = arg; *addr = dm_segs[0].ds_addr; return; } /****************************************************************************** agtiapi_alloc_requests() Purpose: Allocates resources such as dma tag and timer Parameters: struct agtiapi_softc *pmsc (IN) Pointer to the HBA data structure Return: AGTIAPI_SUCCESS - success AGTIAPI_FAIL - fail Note: ******************************************************************************/ int agtiapi_alloc_requests( struct agtiapi_softc *pmcsc ) { int rsize, nsegs; U32 next_tick; nsegs = AGTIAPI_NSEGS; rsize = AGTIAPI_MAX_DMA_SEGS; // 128 AGTIAPI_PRINTK( "agtiapi_alloc_requests: MAXPHYS 0x%x PAGE_SIZE 0x%x \n", MAXPHYS, PAGE_SIZE ); AGTIAPI_PRINTK( "agtiapi_alloc_requests: nsegs %d rsize %d \n", nsegs, rsize ); // 32, 128 // This is for csio->data_ptr if( bus_dma_tag_create( agNULL, // parent 1, // alignment 0, // boundary BUS_SPACE_MAXADDR, // lowaddr BUS_SPACE_MAXADDR, // highaddr NULL, // filter NULL, // filterarg BUS_SPACE_MAXSIZE_32BIT, // maxsize nsegs, // nsegments BUS_SPACE_MAXSIZE_32BIT, // maxsegsize BUS_DMA_ALLOCNOW, // flags busdma_lock_mutex, // lockfunc &pmcsc->pCardInfo->pmIOLock, // lockarg &pmcsc->buffer_dmat ) ) { AGTIAPI_PRINTK( "agtiapi_alloc_requests: Cannot alloc request DMA tag\n" ); return( ENOMEM ); } // This is for tiSgl_t of pccb in agtiapi_PrepCCBs() rsize = (sizeof(tiSgl_t) * AGTIAPI_NSEGS) * AGTIAPI_CCB_PER_DEVICE * maxTargets; AGTIAPI_PRINTK( "agtiapi_alloc_requests: rsize %d \n", rsize ); // 32, 128 if( bus_dma_tag_create( agNULL, // parent 32, // alignment 0, // boundary BUS_SPACE_MAXADDR_32BIT, // lowaddr BUS_SPACE_MAXADDR, // highaddr NULL, // filter NULL, // filterarg rsize, // maxsize 1, // nsegments rsize, // maxsegsize BUS_DMA_ALLOCNOW, // flags NULL, // lockfunc NULL, // lockarg &pmcsc->tisgl_dmat ) ) { AGTIAPI_PRINTK( "agtiapi_alloc_requests: Cannot alloc request DMA tag\n" ); return( ENOMEM ); } if( bus_dmamem_alloc( pmcsc->tisgl_dmat, (void **)&pmcsc->tisgl_mem, BUS_DMA_NOWAIT, &pmcsc->tisgl_map ) ) { AGTIAPI_PRINTK( "agtiapi_alloc_requests: Cannot allocate SGL memory\n" ); return( ENOMEM ); } bzero( pmcsc->tisgl_mem, rsize ); bus_dmamap_load( pmcsc->tisgl_dmat, pmcsc->tisgl_map, pmcsc->tisgl_mem, rsize, agtiapi_SglMemoryCB, &pmcsc->tisgl_busaddr, BUS_DMA_NOWAIT /* 0 */ ); mtx_init( &pmcsc->OS_timer_lock, "OS timer lock", NULL, MTX_DEF ); mtx_init( &pmcsc->IO_timer_lock, "IO timer lock", NULL, MTX_DEF ); mtx_init( &pmcsc->devRmTimerLock, "targ rm timer lock", NULL, MTX_DEF ); callout_init_mtx( &pmcsc->OS_timer, &pmcsc->OS_timer_lock, 0 ); callout_init_mtx( &pmcsc->IO_timer, &pmcsc->IO_timer_lock, 0 ); callout_init_mtx( &pmcsc->devRmTimer, &pmcsc->devRmTimerLock, 0); next_tick = pmcsc->pCardInfo->tiRscInfo.tiLoLevelResource. loLevelOption.usecsPerTick / USEC_PER_TICK; AGTIAPI_PRINTK( "agtiapi_alloc_requests: before callout_reset, " "next_tick 0x%x\n", next_tick ); callout_reset( &pmcsc->OS_timer, next_tick, agtiapi_TITimer, pmcsc ); return 0; } /****************************************************************************** agtiapi_alloc_ostimem() Purpose: Allocates memory used later in ostiAllocMemory Parameters: struct agtiapi_softc *pmcsc (IN) Pointer to the HBA data structure Return: AGTIAPI_SUCCESS - success AGTIAPI_FAIL - fail Note: This is a pre-allocation for ostiAllocMemory() "non-cacheable" function calls ******************************************************************************/ int agtiapi_alloc_ostimem( struct agtiapi_softc *pmcsc ) { int rsize, nomsize; nomsize = 4096; rsize = AGTIAPI_DYNAMIC_MAX * nomsize; // 8M AGTIAPI_PRINTK("agtiapi_alloc_ostimem: rsize %d \n", rsize); if( bus_dma_tag_create( agNULL, // parent 32, // alignment 0, // boundary BUS_SPACE_MAXADDR, // lowaddr BUS_SPACE_MAXADDR, // highaddr NULL, // filter NULL, // filterarg rsize, // maxsize (size) 1, // number of segments rsize, // maxsegsize 0, // flags NULL, // lockfunc NULL, // lockarg &pmcsc->osti_dmat ) ) { AGTIAPI_PRINTK( "agtiapi_alloc_ostimem: Can't create no-cache mem tag\n" ); return AGTIAPI_FAIL; } if( bus_dmamem_alloc( pmcsc->osti_dmat, &pmcsc->osti_mem, BUS_DMA_WAITOK | BUS_DMA_ZERO | BUS_DMA_NOCACHE, &pmcsc->osti_mapp ) ) { AGTIAPI_PRINTK( "agtiapi_alloc_ostimem: Cannot allocate cache mem %d\n", rsize ); return AGTIAPI_FAIL; } bus_dmamap_load( pmcsc->osti_dmat, pmcsc->osti_mapp, pmcsc->osti_mem, rsize, agtiapi_MemoryCB, // try reuse of CB for same goal &pmcsc->osti_busaddr, BUS_DMA_NOWAIT ); // populate all the ag_dma_addr_t osti_busaddr/mem fields with addresses for // handy reference when driver is in motion int idx; ag_card_info_t *pCardInfo = pmcsc->pCardInfo; ag_dma_addr_t *pMem; for( idx = 0; idx < AGTIAPI_DYNAMIC_MAX; idx++ ) { pMem = &pCardInfo->dynamicMem[idx]; pMem->nocache_busaddr = pmcsc->osti_busaddr + ( idx * nomsize ); pMem->nocache_mem = (void*)((U64)pmcsc->osti_mem + ( idx * nomsize )); pCardInfo->freeDynamicMem[idx] = &pCardInfo->dynamicMem[idx]; } pCardInfo->topOfFreeDynamicMem = AGTIAPI_DYNAMIC_MAX; return AGTIAPI_SUCCESS; } /****************************************************************************** agtiapi_cam_action() Purpose: Parses CAM frames and triggers a corresponding action Parameters: struct cam_sim *sim (IN) Pointer to SIM data structure union ccb * ccb (IN) Pointer to CAM ccb data structure Return: Note: ******************************************************************************/ static void agtiapi_cam_action( struct cam_sim *sim, union ccb * ccb ) { struct agtiapi_softc *pmcsc; tiDeviceHandle_t *pDevHandle = NULL; // acts as flag as well tiDeviceInfo_t devInfo; int pathID, targetID, lunID; int lRetVal; U32 TID; U32 speed = 150000; pmcsc = cam_sim_softc( sim ); AGTIAPI_IO( "agtiapi_cam_action: start pmcs %p\n", pmcsc ); if (pmcsc == agNULL) { AGTIAPI_PRINTK( "agtiapi_cam_action: start pmcs is NULL\n" ); return; } mtx_assert( &(pmcsc->pCardInfo->pmIOLock), MA_OWNED ); AGTIAPI_IO( "agtiapi_cam_action: cardNO %d func_code 0x%x\n", pmcsc->cardNo, ccb->ccb_h.func_code ); pathID = xpt_path_path_id( ccb->ccb_h.path ); targetID = xpt_path_target_id( ccb->ccb_h.path ); lunID = xpt_path_lun_id( ccb->ccb_h.path ); AGTIAPI_IO( "agtiapi_cam_action: P 0x%x T 0x%x L 0x%x\n", pathID, targetID, lunID ); switch (ccb->ccb_h.func_code) { case XPT_PATH_INQ: { struct ccb_pathinq *cpi; /* See architecure book p180*/ cpi = &ccb->cpi; cpi->version_num = 1; cpi->hba_inquiry = PI_SDTR_ABLE | PI_TAG_ABLE | PI_WIDE_16; cpi->target_sprt = 0; cpi->hba_misc = PIM_NOBUSRESET | PIM_SEQSCAN; cpi->hba_eng_cnt = 0; cpi->max_target = maxTargets - 1; cpi->max_lun = AGTIAPI_MAX_LUN; cpi->maxio = 1024 *1024; /* Max supported I/O size, in bytes. */ cpi->initiator_id = 255; strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN); strncpy(cpi->hba_vid, "PMC", HBA_IDLEN); strncpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN); cpi->unit_number = cam_sim_unit(sim); cpi->bus_id = cam_sim_bus(sim); // rate is set when XPT_GET_TRAN_SETTINGS is processed cpi->base_transfer_speed = 150000; cpi->transport = XPORT_SAS; cpi->transport_version = 0; cpi->protocol = PROTO_SCSI; cpi->protocol_version = SCSI_REV_SPC3; cpi->ccb_h.status = CAM_REQ_CMP; break; } case XPT_GET_TRAN_SETTINGS: { struct ccb_trans_settings *cts; struct ccb_trans_settings_sas *sas; struct ccb_trans_settings_scsi *scsi; if ( pmcsc->flags & AGTIAPI_SHUT_DOWN ) { return; } cts = &ccb->cts; sas = &ccb->cts.xport_specific.sas; scsi = &cts->proto_specific.scsi; cts->protocol = PROTO_SCSI; cts->protocol_version = SCSI_REV_SPC3; cts->transport = XPORT_SAS; cts->transport_version = 0; sas->valid = CTS_SAS_VALID_SPEED; /* this sets the "MB/s transfers" */ if (pmcsc != NULL && targetID >= 0 && targetID < maxTargets) { if (pmcsc->pWWNList != NULL) { TID = INDEX(pmcsc, targetID); if (TID < maxTargets) { pDevHandle = pmcsc->pDevList[TID].pDevHandle; } } } if (pDevHandle) { tiINIGetDeviceInfo( &pmcsc->tiRoot, pDevHandle, &devInfo ); switch (devInfo.info.devType_S_Rate & 0xF) { case 0x8: speed = 150000; break; case 0x9: speed = 300000; break; case 0xA: speed = 600000; break; case 0xB: speed = 1200000; break; default: speed = 150000; break; } } sas->bitrate = speed; scsi->valid = CTS_SCSI_VALID_TQ; scsi->flags = CTS_SCSI_FLAGS_TAG_ENB; ccb->ccb_h.status = CAM_REQ_CMP; break; } case XPT_RESET_BUS: { lRetVal = agtiapi_eh_HostReset( pmcsc, ccb ); // usually works first time if ( SUCCESS == lRetVal ) { AGTIAPI_PRINTK( "agtiapi_cam_action: bus reset success.\n" ); } else { AGTIAPI_PRINTK( "agtiapi_cam_action: bus reset failed.\n" ); } ccb->ccb_h.status = CAM_REQ_CMP; break; } case XPT_RESET_DEV: { ccb->ccb_h.status = CAM_REQ_CMP; break; } case XPT_ABORT: { ccb->ccb_h.status = CAM_REQ_CMP; break; } #if __FreeBSD_version >= 900026 case XPT_SMP_IO: { agtiapi_QueueSMP( pmcsc, ccb ); return; } #endif /* __FreeBSD_version >= 900026 */ case XPT_SCSI_IO: { if(pmcsc->dev_scan == agFALSE) { ccb->ccb_h.status = CAM_SEL_TIMEOUT; break; } if (pmcsc->flags & AGTIAPI_SHUT_DOWN) { AGTIAPI_PRINTK( "agtiapi_cam_action: shutdown, XPT_SCSI_IO 0x%x\n", XPT_SCSI_IO ); ccb->ccb_h.status = CAM_FUNC_NOTAVAIL; break; } else { AGTIAPI_IO( "agtiapi_cam_action: Zero XPT_SCSI_IO 0x%x, doing IOs\n", XPT_SCSI_IO ); agtiapi_QueueCmnd_( pmcsc, ccb ); return; } } case XPT_CALC_GEOMETRY: { cam_calc_geometry(&ccb->ccg, 1); ccb->ccb_h.status = CAM_REQ_CMP; break; } default: { /* XPT_SET_TRAN_SETTINGS */ AGTIAPI_IO( "agtiapi_cam_action: default function code 0x%x\n", ccb->ccb_h.func_code ); ccb->ccb_h.status = CAM_FUNC_NOTAVAIL; break; } } /* switch */ xpt_done(ccb); } /****************************************************************************** agtiapi_GetCCB() Purpose: Get a ccb from free list or allocate a new one Parameters: struct agtiapi_softc *pmcsc (IN) Pointer to HBA structure Return: Pointer to a ccb structure, or NULL if not available Note: ******************************************************************************/ STATIC pccb_t agtiapi_GetCCB( struct agtiapi_softc *pmcsc ) { pccb_t pccb; AGTIAPI_IO( "agtiapi_GetCCB: start\n" ); AG_LOCAL_LOCK( &pmcsc->ccbLock ); /* get the ccb from the head of the free list */ if ((pccb = (pccb_t)pmcsc->ccbFreeList) != NULL) { pmcsc->ccbFreeList = (caddr_t *)pccb->pccbNext; pccb->pccbNext = NULL; pccb->flags = ACTIVE; pccb->startTime = 0; pmcsc->activeCCB++; AGTIAPI_IO( "agtiapi_GetCCB: re-allocated ccb %p\n", pccb ); } else { AGTIAPI_PRINTK( "agtiapi_GetCCB: kmalloc ERROR - no ccb allocated\n" ); } AG_LOCAL_UNLOCK( &pmcsc->ccbLock ); return pccb; } /****************************************************************************** agtiapi_QueueCmnd_() Purpose: Calls for sending CCB and excuting on HBA. Parameters: struct agtiapi_softc *pmsc (IN) Pointer to the HBA data structure union ccb * ccb (IN) Pointer to CAM ccb data structure Return: 0 - Command is pending to execute 1 - Command returned without further process Note: ******************************************************************************/ int agtiapi_QueueCmnd_(struct agtiapi_softc *pmcsc, union ccb * ccb) { struct ccb_scsiio *csio = &ccb->csio; pccb_t pccb = agNULL; // call dequeue int status = tiSuccess; U32 Channel = CMND_TO_CHANNEL(ccb); U32 TID = CMND_TO_TARGET(ccb); U32 LUN = CMND_TO_LUN(ccb); AGTIAPI_IO( "agtiapi_QueueCmnd_: start\n" ); /* no support for CBD > 16 */ if (csio->cdb_len > 16) { AGTIAPI_PRINTK( "agtiapi_QueueCmnd_: unsupported CDB length %d\n", csio->cdb_len ); ccb->ccb_h.status &= ~CAM_SIM_QUEUED; ccb->ccb_h.status &= ~CAM_STATUS_MASK; ccb->ccb_h.status |= CAM_REQ_INVALID;//CAM_REQ_CMP; xpt_done(ccb); return tiError; } if (TID < 0 || TID >= maxTargets) { AGTIAPI_PRINTK("agtiapi_QueueCmnd_: INVALID TID ERROR\n"); ccb->ccb_h.status &= ~CAM_SIM_QUEUED; ccb->ccb_h.status &= ~CAM_STATUS_MASK; ccb->ccb_h.status |= CAM_DEV_NOT_THERE;//CAM_REQ_CMP; xpt_done(ccb); return tiError; } /* get a ccb */ if ((pccb = agtiapi_GetCCB(pmcsc)) == NULL) { ag_device_t *targ; AGTIAPI_PRINTK("agtiapi_QueueCmnd_: GetCCB ERROR\n"); if (pmcsc != NULL) { TID = INDEX(pmcsc, TID); targ = &pmcsc->pDevList[TID]; } if (targ != NULL) { agtiapi_adjust_queue_depth(ccb->ccb_h.path,targ->qdepth); } ccb->ccb_h.status &= ~CAM_SIM_QUEUED; ccb->ccb_h.status &= ~CAM_STATUS_MASK; ccb->ccb_h.status |= CAM_REQUEUE_REQ; xpt_done(ccb); return tiBusy; } pccb->pmcsc = pmcsc; /* initialize Command Control Block (CCB) */ pccb->targetId = TID; pccb->lun = LUN; pccb->channel = Channel; pccb->ccb = ccb; /* for struct scsi_cmnd */ pccb->senseLen = csio->sense_len; pccb->startTime = ticks; pccb->pSenseData = (caddr_t) &csio->sense_data; pccb->tiSuperScsiRequest.flags = 0; /* each channel is reserved for different addr modes */ pccb->addrMode = agtiapi_AddrModes[Channel]; status = agtiapi_PrepareSGList(pmcsc, pccb); if (status != tiSuccess) { AGTIAPI_PRINTK("agtiapi_QueueCmnd_: agtiapi_PrepareSGList failure\n"); agtiapi_FreeCCB(pmcsc, pccb); if (status == tiReject) { ccb->ccb_h.status = CAM_REQ_INVALID; } else { ccb->ccb_h.status = CAM_REQ_CMP; } xpt_done( ccb ); return tiError; } return status; } /****************************************************************************** agtiapi_DumpCDB() Purpose: Prints out CDB Parameters: const char *ptitle (IN) A string to be printed ccb_t *pccb (IN) A pointer to the driver's own CCB, not CAM's CCB Return: Note: ******************************************************************************/ STATIC void agtiapi_DumpCDB(const char *ptitle, ccb_t *pccb) { union ccb *ccb; struct ccb_scsiio *csio; bit8 cdb[64]; int len; if (pccb == NULL) { printf( "agtiapi_DumpCDB: no pccb here \n" ); panic("agtiapi_DumpCDB: pccb is NULL. called from %s\n", ptitle); return; } ccb = pccb->ccb; if (ccb == NULL) { printf( "agtiapi_DumpCDB: no ccb here \n" ); panic( "agtiapi_DumpCDB: pccb %p ccb %p flags %d ccb NULL! " "called from %s\n", pccb, pccb->ccb, pccb->flags, ptitle ); return; } csio = &ccb->csio; if (csio == NULL) { printf( "agtiapi_DumpCDB: no csio here \n" ); panic( "agtiapi_DumpCDB: pccb%p ccb%p flags%d csio NULL! called from %s\n", pccb, pccb->ccb, pccb->flags, ptitle ); return; } len = MIN(64, csio->cdb_len); if (csio->ccb_h.flags & CAM_CDB_POINTER) { bcopy(csio->cdb_io.cdb_ptr, &cdb[0], len); } else { bcopy(csio->cdb_io.cdb_bytes, &cdb[0], len); } AGTIAPI_IO( "agtiapi_DumpCDB: pccb%p CDB0x%x csio->cdb_len %d" " len %d from %s\n", pccb, cdb[0], csio->cdb_len, len, ptitle ); return; } /****************************************************************************** agtiapi_DoSoftReset() Purpose: Do card reset Parameters: *data (IN) point to pmcsc (struct agtiapi_softc *) Return: Note: ******************************************************************************/ int agtiapi_DoSoftReset (struct agtiapi_softc *pmcsc) { int ret; unsigned long flags; pmcsc->flags |= AGTIAPI_SOFT_RESET; AG_SPIN_LOCK_IRQ( agtiapi_host_lock, flags ); ret = agtiapi_ResetCard( pmcsc, &flags ); AG_SPIN_UNLOCK_IRQ( agtiapi_host_lock, flags ); if( ret != AGTIAPI_SUCCESS ) return tiError; return SUCCESS; } /****************************************************************************** agtiapi_CheckIOTimeout() Purpose: Timeout function for SCSI IO or TM Parameters: *data (IN) point to pCard (ag_card_t *) Return: Note: ******************************************************************************/ STATIC void agtiapi_CheckIOTimeout(void *data) { U32 status = AGTIAPI_SUCCESS; ccb_t *pccb; struct agtiapi_softc *pmcsc; pccb_t pccb_curr; pccb_t pccb_next; pmcsc = (struct agtiapi_softc *)data; //AGTIAPI_PRINTK("agtiapi_CheckIOTimeout: Enter\n"); //AGTIAPI_PRINTK("agtiapi_CheckIOTimeout: Active CCB %d\n", pmcsc->activeCCB); pccb = (pccb_t)pmcsc->ccbChainList; /* if link is down, do nothing */ if ((pccb == NULL) || (pmcsc->activeCCB == 0)) { //AGTIAPI_PRINTK("agtiapi_CheckIOTimeout: goto restart_timer\n"); goto restart_timer; } AG_SPIN_LOCK_IRQ(agtiapi_host_lock, flags); if (pmcsc->flags & AGTIAPI_SHUT_DOWN) goto ext; pccb_curr = pccb; /* Walk thorugh the IO Chain linked list to find the pending io */ /* Set the TM flag based on the pccb type, i.e SCSI IO or TM cmd */ while (pccb_curr != NULL) { /* start from 1st ccb in the chain */ pccb_next = pccb_curr->pccbChainNext; if( (pccb_curr->flags == 0) || (pccb_curr->tiIORequest.tdData == NULL) || (pccb_curr->startTime == 0) /* && (pccb->startTime == 0) */) { //AGTIAPI_PRINTK("agtiapi_CheckIOTimeout: move to next element\n"); } else if ( ( (ticks-pccb_curr->startTime) >= ag_timeout_secs ) && !(pccb_curr->flags & TIMEDOUT) ) { AGTIAPI_PRINTK( "agtiapi_CheckIOTimeout: pccb %p timed out, call TM " "function -- flags=%x startTime=%ld tdData = %p\n", pccb_curr, pccb_curr->flags, pccb->startTime, pccb_curr->tiIORequest.tdData ); pccb_curr->flags |= TIMEDOUT; status = agtiapi_StartTM(pmcsc, pccb_curr); if (status == AGTIAPI_SUCCESS) { AGTIAPI_PRINTK( "agtiapi_CheckIOTimeout: TM Request sent with " "success\n" ); goto restart_timer; } else { #ifdef AGTIAPI_LOCAL_RESET /* abort request did not go through */ AGTIAPI_PRINTK("agtiapi_CheckIOTimeout: Abort request failed\n"); /* TODO: call Soft reset here */ AGTIAPI_PRINTK( "agtiapi_CheckIOTimeout:in agtiapi_CheckIOTimeout() " "abort request did not go thru ==> soft reset#7, then " "restart timer\n" ); agtiapi_DoSoftReset (pmcsc); goto restart_timer; #endif } } pccb_curr = pccb_next; } restart_timer: callout_reset(&pmcsc->IO_timer, 1*hz, agtiapi_CheckIOTimeout, pmcsc); ext: AG_SPIN_UNLOCK_IRQ(agtiapi_host_lock, flags); return; } /****************************************************************************** agtiapi_StartTM() Purpose: DDI calls for aborting outstanding IO command Parameters: struct scsi_cmnd *pccb (IN) Pointer to the command to be aborted unsigned long flags (IN/out) spinlock flags used in locking from calling layers Return: AGTIAPI_SUCCESS - success AGTIAPI_FAIL - fail ******************************************************************************/ int agtiapi_StartTM(struct agtiapi_softc *pCard, ccb_t *pccb) { ccb_t *pTMccb = NULL; U32 status = AGTIAPI_SUCCESS; ag_device_t *pDevice = NULL; U32 TMstatus = tiSuccess; AGTIAPI_PRINTK( "agtiapi_StartTM: pccb %p, pccb->flags %x\n", pccb, pccb->flags ); if (pccb == NULL) { AGTIAPI_PRINTK("agtiapi_StartTM: %p not found\n",pccb); status = AGTIAPI_SUCCESS; goto ext; } if (!pccb->tiIORequest.tdData) { /* should not be the case */ AGTIAPI_PRINTK("agtiapi_StartTM: ccb %p flag 0x%x tid %d no tdData " "ERROR\n", pccb, pccb->flags, pccb->targetId); status = AGTIAPI_FAIL; } else { /* If timedout CCB is TM_ABORT_TASK command, issue LocalAbort first to clear pending TM_ABORT_TASK */ /* Else Device State will not be put back to Operational, (refer FW) */ if (pccb->flags & TASK_MANAGEMENT) { if (tiINIIOAbort(&pCard->tiRoot, &pccb->tiIORequest) != tiSuccess) { AGTIAPI_PRINTK( "agtiapi_StartTM: LocalAbort Request for Abort_TASK " "TM failed\n" ); /* TODO: call Soft reset here */ AGTIAPI_PRINTK( "agtiapi_StartTM: in agtiapi_StartTM() abort " "tiINIIOAbort() failed ==> soft reset#8\n" ); agtiapi_DoSoftReset( pCard ); } else { AGTIAPI_PRINTK( "agtiapi_StartTM: LocalAbort for Abort_TASK TM " "Request sent\n" ); status = AGTIAPI_SUCCESS; } } else { /* get a ccb */ if ((pTMccb = agtiapi_GetCCB(pCard)) == NULL) { AGTIAPI_PRINTK("agtiapi_StartTM: TM resource unavailable!\n"); status = AGTIAPI_FAIL; goto ext; } pTMccb->pmcsc = pCard; pTMccb->targetId = pccb->targetId; pTMccb->devHandle = pccb->devHandle; if (pTMccb->targetId >= pCard->devDiscover) { AGTIAPI_PRINTK("agtiapi_StartTM: Incorrect dev Id in TM!\n"); status = AGTIAPI_FAIL; goto ext; } if (pTMccb->targetId < 0 || pTMccb->targetId >= maxTargets) { return AGTIAPI_FAIL; } if (INDEX(pCard, pTMccb->targetId) >= maxTargets) { return AGTIAPI_FAIL; } pDevice = &pCard->pDevList[INDEX(pCard, pTMccb->targetId)]; if ((pDevice == NULL) || !(pDevice->flags & ACTIVE)) { return AGTIAPI_FAIL; } /* save pending io to issue local abort at Task mgmt CB */ pTMccb->pccbIO = pccb; AGTIAPI_PRINTK( "agtiapi_StartTM: pTMccb %p flag %x tid %d via TM " "request !\n", pTMccb, pTMccb->flags, pTMccb->targetId ); pTMccb->flags &= ~(TASK_SUCCESS | ACTIVE); pTMccb->flags |= TASK_MANAGEMENT; TMstatus = tiINITaskManagement(&pCard->tiRoot, pccb->devHandle, AG_ABORT_TASK, &pccb->tiSuperScsiRequest.scsiCmnd.lun, &pccb->tiIORequest, &pTMccb->tiIORequest); if (TMstatus == tiSuccess) { AGTIAPI_PRINTK( "agtiapi_StartTM: TM_ABORT_TASK request success ccb " "%p, pTMccb %p\n", pccb, pTMccb ); pTMccb->startTime = ticks; status = AGTIAPI_SUCCESS; } else if (TMstatus == tiIONoDevice) { AGTIAPI_PRINTK( "agtiapi_StartTM: TM_ABORT_TASK request tiIONoDevice ccb " "%p, pTMccb %p\n", pccb, pTMccb ); status = AGTIAPI_SUCCESS; } else { AGTIAPI_PRINTK( "agtiapi_StartTM: TM_ABORT_TASK request failed ccb %p, " "pTMccb %p\n", pccb, pTMccb ); status = AGTIAPI_FAIL; agtiapi_FreeTMCCB(pCard, pTMccb); /* TODO */ /* call TM_TARGET_RESET */ } } } ext: AGTIAPI_PRINTK("agtiapi_StartTM: return %d flgs %x\n", status, (pccb) ? pccb->flags : -1); return status; } /* agtiapi_StartTM */ #if __FreeBSD_version > 901000 /****************************************************************************** agtiapi_PrepareSGList() Purpose: This function prepares scatter-gather list for the given ccb Parameters: struct agtiapi_softc *pmsc (IN) Pointer to the HBA data structure ccb_t *pccb (IN) A pointer to the driver's own CCB, not CAM's CCB Return: 0 - success 1 - failure Note: ******************************************************************************/ static int agtiapi_PrepareSGList(struct agtiapi_softc *pmcsc, ccb_t *pccb) { union ccb *ccb = pccb->ccb; struct ccb_scsiio *csio = &ccb->csio; struct ccb_hdr *ccbh = &ccb->ccb_h; AGTIAPI_IO( "agtiapi_PrepareSGList: start\n" ); // agtiapi_DumpCDB("agtiapi_PrepareSGList", pccb); AGTIAPI_IO( "agtiapi_PrepareSGList: dxfer_len %d\n", csio->dxfer_len ); if ((ccbh->flags & CAM_DIR_MASK) != CAM_DIR_NONE) { switch((ccbh->flags & CAM_DATA_MASK)) { int error; struct bus_dma_segment seg; case CAM_DATA_VADDR: /* Virtual address that needs to translated into one or more physical address ranges. */ // int error; // AG_LOCAL_LOCK(&(pmcsc->pCardInfo->pmIOLock)); AGTIAPI_IO( "agtiapi_PrepareSGList: virtual address\n" ); error = bus_dmamap_load( pmcsc->buffer_dmat, pccb->CCB_dmamap, csio->data_ptr, csio->dxfer_len, agtiapi_PrepareSGListCB, pccb, BUS_DMA_NOWAIT/* 0 */ ); // AG_LOCAL_UNLOCK( &(pmcsc->pCardInfo->pmIOLock) ); if (error == EINPROGRESS) { /* So as to maintain ordering, freeze the controller queue until our mapping is returned. */ AGTIAPI_PRINTK("agtiapi_PrepareSGList: EINPROGRESS\n"); xpt_freeze_simq(pmcsc->sim, 1); pmcsc->SimQFrozen = agTRUE; ccbh->status |= CAM_RELEASE_SIMQ; } break; case CAM_DATA_PADDR: /* We have been given a pointer to single physical buffer. */ /* pccb->tiSuperScsiRequest.sglVirtualAddr = seg.ds_addr; */ //struct bus_dma_segment seg; AGTIAPI_PRINTK("agtiapi_PrepareSGList: physical address\n"); seg.ds_addr = (bus_addr_t)(vm_offset_t)csio->data_ptr; seg.ds_len = csio->dxfer_len; // * 0xFF to be defined agtiapi_PrepareSGListCB(pccb, &seg, 1, 0xAABBCCDD); break; default: AGTIAPI_PRINTK("agtiapi_PrepareSGList: unexpected case\n"); return tiReject; } } else { agtiapi_PrepareSGListCB(pccb, NULL, 0, 0xAAAAAAAA); } return tiSuccess; } #else /****************************************************************************** agtiapi_PrepareSGList() Purpose: This function prepares scatter-gather list for the given ccb Parameters: struct agtiapi_softc *pmsc (IN) Pointer to the HBA data structure ccb_t *pccb (IN) A pointer to the driver's own CCB, not CAM's CCB Return: 0 - success 1 - failure Note: ******************************************************************************/ static int agtiapi_PrepareSGList(struct agtiapi_softc *pmcsc, ccb_t *pccb) { union ccb *ccb = pccb->ccb; struct ccb_scsiio *csio = &ccb->csio; struct ccb_hdr *ccbh = &ccb->ccb_h; AGTIAPI_IO( "agtiapi_PrepareSGList: start\n" ); // agtiapi_DumpCDB("agtiapi_PrepareSGList", pccb); AGTIAPI_IO( "agtiapi_PrepareSGList: dxfer_len %d\n", csio->dxfer_len ); if ((ccbh->flags & CAM_DIR_MASK) != CAM_DIR_NONE) { if ((ccbh->flags & CAM_SCATTER_VALID) == 0) { /* We've been given a pointer to a single buffer. */ if ((ccbh->flags & CAM_DATA_PHYS) == 0) { /* Virtual address that needs to translated into one or more physical address ranges. */ int error; // AG_LOCAL_LOCK(&(pmcsc->pCardInfo->pmIOLock)); AGTIAPI_IO( "agtiapi_PrepareSGList: virtual address\n" ); error = bus_dmamap_load( pmcsc->buffer_dmat, pccb->CCB_dmamap, csio->data_ptr, csio->dxfer_len, agtiapi_PrepareSGListCB, pccb, BUS_DMA_NOWAIT/* 0 */ ); // AG_LOCAL_UNLOCK( &(pmcsc->pCardInfo->pmIOLock) ); if (error == EINPROGRESS) { /* So as to maintain ordering, freeze the controller queue until our mapping is returned. */ AGTIAPI_PRINTK("agtiapi_PrepareSGList: EINPROGRESS\n"); xpt_freeze_simq(pmcsc->sim, 1); pmcsc->SimQFrozen = agTRUE; ccbh->status |= CAM_RELEASE_SIMQ; } } else { /* We have been given a pointer to single physical buffer. */ /* pccb->tiSuperScsiRequest.sglVirtualAddr = seg.ds_addr; */ struct bus_dma_segment seg; AGTIAPI_PRINTK("agtiapi_PrepareSGList: physical address\n"); seg.ds_addr = (bus_addr_t)(vm_offset_t)csio->data_ptr; seg.ds_len = csio->dxfer_len; // * 0xFF to be defined agtiapi_PrepareSGListCB(pccb, &seg, 1, 0xAABBCCDD); } } else { AGTIAPI_PRINTK("agtiapi_PrepareSGList: unexpected case\n"); return tiReject; } } else { agtiapi_PrepareSGListCB(pccb, NULL, 0, 0xAAAAAAAA); } return tiSuccess; } #endif /****************************************************************************** agtiapi_PrepareSGListCB() Purpose: Callback function for bus_dmamap_load() This fuctions sends IO to LL layer. Parameters: void *arg (IN) Pointer to the HBA data structure bus_dma_segment_t *segs (IN) Pointer to dma segment int nsegs (IN) number of dma segment int error (IN) error Return: Note: ******************************************************************************/ static void agtiapi_PrepareSGListCB( void *arg, bus_dma_segment_t *segs, int nsegs, int error ) { pccb_t pccb = arg; union ccb *ccb = pccb->ccb; struct ccb_scsiio *csio = &ccb->csio; struct agtiapi_softc *pmcsc; tiIniScsiCmnd_t *pScsiCmnd; bit32 i; bus_dmasync_op_t op; U32_64 phys_addr; U08 *CDB; int io_is_encryptable = 0; unsigned long long start_lba = 0; ag_device_t *pDev; U32 TID = CMND_TO_TARGET(ccb); AGTIAPI_IO( "agtiapi_PrepareSGListCB: start, nsegs %d error 0x%x\n", nsegs, error ); pmcsc = pccb->pmcsc; if (error != tiSuccess) { if (error == 0xAABBCCDD || error == 0xAAAAAAAA) { // do nothing } else { AGTIAPI_PRINTK("agtiapi_PrepareSGListCB: error status 0x%x\n", error); bus_dmamap_unload(pmcsc->buffer_dmat, pccb->CCB_dmamap); bus_dmamap_destroy(pmcsc->buffer_dmat, pccb->CCB_dmamap); agtiapi_FreeCCB(pmcsc, pccb); ccb->ccb_h.status = CAM_REQ_CMP; xpt_done(ccb); return; } } if (nsegs > AGTIAPI_MAX_DMA_SEGS) { AGTIAPI_PRINTK( "agtiapi_PrepareSGListCB: over the limit. nsegs %d" " AGTIAPI_MAX_DMA_SEGS %d\n", nsegs, AGTIAPI_MAX_DMA_SEGS ); bus_dmamap_unload(pmcsc->buffer_dmat, pccb->CCB_dmamap); bus_dmamap_destroy(pmcsc->buffer_dmat, pccb->CCB_dmamap); agtiapi_FreeCCB(pmcsc, pccb); ccb->ccb_h.status = CAM_REQ_CMP; xpt_done(ccb); return; } /* fill in IO information */ pccb->dataLen = csio->dxfer_len; /* start fill in sgl structure */ if (nsegs == 1 && error == 0xAABBCCDD) { /* to be tested */ /* A single physical buffer */ AGTIAPI_PRINTK("agtiapi_PrepareSGListCB: nsegs is 1\n"); CPU_TO_LE32(pccb->tiSuperScsiRequest.agSgl1, segs[0].ds_addr); pccb->tiSuperScsiRequest.agSgl1.len = htole32(pccb->dataLen); pccb->tiSuperScsiRequest.agSgl1.type = htole32(tiSgl); pccb->tiSuperScsiRequest.sglVirtualAddr = (void *)segs->ds_addr; pccb->numSgElements = 1; } else if (nsegs == 0 && error == 0xAAAAAAAA) { /* no data transfer */ AGTIAPI_IO( "agtiapi_PrepareSGListCB: no data transfer\n" ); pccb->tiSuperScsiRequest.agSgl1.len = 0; pccb->dataLen = 0; pccb->numSgElements = 0; } else { /* virtual/logical buffer */ if (nsegs == 1) { pccb->dataLen = segs[0].ds_len; CPU_TO_LE32(pccb->tiSuperScsiRequest.agSgl1, segs[0].ds_addr); pccb->tiSuperScsiRequest.agSgl1.type = htole32(tiSgl); pccb->tiSuperScsiRequest.agSgl1.len = htole32(segs[0].ds_len); pccb->tiSuperScsiRequest.sglVirtualAddr = (void *)csio->data_ptr; pccb->numSgElements = nsegs; } else { pccb->dataLen = 0; /* loop */ for (i = 0; i < nsegs; i++) { pccb->sgList[i].len = htole32(segs[i].ds_len); CPU_TO_LE32(pccb->sgList[i], segs[i].ds_addr); pccb->sgList[i].type = htole32(tiSgl); pccb->dataLen += segs[i].ds_len; } /* for */ pccb->numSgElements = nsegs; /* set up sgl buffer address */ CPU_TO_LE32(pccb->tiSuperScsiRequest.agSgl1, pccb->tisgl_busaddr); pccb->tiSuperScsiRequest.agSgl1.type = htole32(tiSglList); pccb->tiSuperScsiRequest.agSgl1.len = htole32(pccb->dataLen); pccb->tiSuperScsiRequest.sglVirtualAddr = (void *)csio->data_ptr; pccb->numSgElements = nsegs; } /* else */ } /* set data transfer direction */ if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_OUT) { op = BUS_DMASYNC_PREWRITE; pccb->tiSuperScsiRequest.dataDirection = tiDirectionOut; } else { op = BUS_DMASYNC_PREREAD; pccb->tiSuperScsiRequest.dataDirection = tiDirectionIn; } pScsiCmnd = &pccb->tiSuperScsiRequest.scsiCmnd; pScsiCmnd->expDataLength = pccb->dataLen; if (csio->ccb_h.flags & CAM_CDB_POINTER) { bcopy(csio->cdb_io.cdb_ptr, &pScsiCmnd->cdb[0], csio->cdb_len); } else { bcopy(csio->cdb_io.cdb_bytes, &pScsiCmnd->cdb[0],csio->cdb_len); } CDB = &pScsiCmnd->cdb[0]; switch (CDB[0]) { case REQUEST_SENSE: /* requires different buffer */ /* This code should not be excercised because SAS support auto sense For the completeness, vtophys() is still used here. */ AGTIAPI_PRINTK("agtiapi_PrepareSGListCB: QueueCmnd - REQUEST SENSE new\n"); pccb->tiSuperScsiRequest.agSgl1.len = htole32(pccb->senseLen); phys_addr = vtophys(&csio->sense_data); CPU_TO_LE32(pccb->tiSuperScsiRequest.agSgl1, phys_addr); pccb->tiSuperScsiRequest.agSgl1.type = htole32(tiSgl); pccb->dataLen = pccb->senseLen; pccb->numSgElements = 1; break; case INQUIRY: /* only using lun 0 for device type detection */ pccb->flags |= AGTIAPI_INQUIRY; break; case TEST_UNIT_READY: case RESERVE: case RELEASE: case START_STOP: pccb->tiSuperScsiRequest.agSgl1.len = 0; pccb->dataLen = 0; break; case READ_6: case WRITE_6: /* Extract LBA */ start_lba = ((CDB[1] & 0x1f) << 16) | (CDB[2] << 8) | (CDB[3]); #ifdef HIALEAH_ENCRYPTION io_is_encryptable = 1; #endif break; case READ_10: case WRITE_10: case READ_12: case WRITE_12: /* Extract LBA */ start_lba = (CDB[2] << 24) | (CDB[3] << 16) | (CDB[4] << 8) | (CDB[5]); #ifdef HIALEAH_ENCRYPTION io_is_encryptable = 1; #endif break; case READ_16: case WRITE_16: /* Extract LBA */ start_lba = (CDB[2] << 24) | (CDB[3] << 16) | (CDB[4] << 8) | (CDB[5]); start_lba <<= 32; start_lba |= ((CDB[6] << 24) | (CDB[7] << 16) | (CDB[8] << 8) | (CDB[9])); #ifdef HIALEAH_ENCRYPTION io_is_encryptable = 1; #endif break; default: break; } /* fill device lun based one address mode */ agtiapi_SetLunField(pccb); if (pccb->targetId < 0 || pccb->targetId >= maxTargets) { pccb->ccbStatus = tiIOFailed; pccb->scsiStatus = tiDetailNoLogin; agtiapi_FreeCCB(pmcsc, pccb); ccb->ccb_h.status = CAM_DEV_NOT_THERE; // ## v. CAM_FUNC_NOTAVAIL xpt_done(ccb); pccb->ccb = NULL; return; } if (INDEX(pmcsc, pccb->targetId) >= maxTargets) { pccb->ccbStatus = tiIOFailed; pccb->scsiStatus = tiDetailNoLogin; agtiapi_FreeCCB(pmcsc, pccb); ccb->ccb_h.status = CAM_DEV_NOT_THERE; // ## v. CAM_FUNC_NOTAVAIL xpt_done(ccb); pccb->ccb = NULL; return; } pDev = &pmcsc->pDevList[INDEX(pmcsc, pccb->targetId)]; #if 1 if ((pmcsc->flags & EDC_DATA) && (pDev->flags & EDC_DATA)) { /* * EDC support: * * Possible command supported - * READ_6, READ_10, READ_12, READ_16, READ_LONG, READ_BUFFER, * READ_DEFECT_DATA, etc. * WRITE_6, WRITE_10, WRITE_12, WRITE_16, WRITE_LONG, WRITE_LONG2, * WRITE_BUFFER, WRITE_VERIFY, WRITE_VERIFY_12, etc. * * Do some data length adjustment and set chip operation instruction. */ switch (CDB[0]) { case READ_6: case READ_10: case READ_12: case READ_16: // BUG_ON(pccb->tiSuperScsiRequest.flags & TI_SCSI_INITIATOR_ENCRYPT); #ifdef AGTIAPI_TEST_DIF pccb->tiSuperScsiRequest.flags |= TI_SCSI_INITIATOR_DIF; #endif pccb->flags |= EDC_DATA; #ifdef TEST_VERIFY_AND_FORWARD pccb->tiSuperScsiRequest.Dif.flags = DIF_VERIFY_FORWARD | DIF_UDT_REF_BLOCK_COUNT; if(pDev->sector_size == 520) { pScsiCmnd->expDataLength += (pccb->dataLen / 512) * 8; } else if(pDev->sector_size == 4104) { pScsiCmnd->expDataLength += (pccb->dataLen / 4096) * 8; } #else #ifdef AGTIAPI_TEST_DIF pccb->tiSuperScsiRequest.Dif.flags = DIF_VERIFY_DELETE | DIF_UDT_REF_BLOCK_COUNT; #endif #endif #ifdef AGTIAPI_TEST_DIF switch(pDev->sector_size) { case 528: pccb->tiSuperScsiRequest.Dif.flags |= ( DIF_BLOCK_SIZE_520 << 16 ); break; case 4104: pccb->tiSuperScsiRequest.Dif.flags |= ( DIF_BLOCK_SIZE_4096 << 16 ); break; case 4168: pccb->tiSuperScsiRequest.Dif.flags |= ( DIF_BLOCK_SIZE_4160 << 16 ); break; } if(pCard->flags & EDC_DATA_CRC) pccb->tiSuperScsiRequest.Dif.flags |= DIF_CRC_VERIFICATION; /* Turn on upper 4 bits of UVM */ pccb->tiSuperScsiRequest.Dif.flags |= 0x03c00000; #endif #ifdef AGTIAPI_TEST_DPL if(agtiapi_SetupDifPerLA(pCard, pccb, start_lba) < 0) { printk(KERN_ERR "SetupDifPerLA Failed.\n"); cmnd->result = SCSI_HOST(DID_ERROR); goto err; } pccb->tiSuperScsiRequest.Dif.enableDIFPerLA = TRUE; #endif #ifdef AGTIAPI_TEST_DIF /* Set App Tag */ pccb->tiSuperScsiRequest.Dif.udtArray[0] = 0xaa; pccb->tiSuperScsiRequest.Dif.udtArray[1] = 0xbb; /* Set LBA in UDT array */ if(CDB[0] == READ_6) { pccb->tiSuperScsiRequest.Dif.udtArray[2] = CDB[3]; pccb->tiSuperScsiRequest.Dif.udtArray[3] = CDB[2]; pccb->tiSuperScsiRequest.Dif.udtArray[4] = CDB[1] & 0x1f; pccb->tiSuperScsiRequest.Dif.udtArray[5] = 0; } else if(CDB[0] == READ_10 || CDB[0] == READ_12) { pccb->tiSuperScsiRequest.Dif.udtArray[2] = CDB[5]; pccb->tiSuperScsiRequest.Dif.udtArray[3] = CDB[4]; pccb->tiSuperScsiRequest.Dif.udtArray[4] = CDB[3]; pccb->tiSuperScsiRequest.Dif.udtArray[5] = CDB[2]; } else if(CDB[0] == READ_16) { pccb->tiSuperScsiRequest.Dif.udtArray[2] = CDB[9]; pccb->tiSuperScsiRequest.Dif.udtArray[3] = CDB[8]; pccb->tiSuperScsiRequest.Dif.udtArray[4] = CDB[7]; pccb->tiSuperScsiRequest.Dif.udtArray[5] = CDB[6]; /* Note: 32 bits lost */ } #endif break; case WRITE_6: case WRITE_10: case WRITE_12: case WRITE_16: // BUG_ON(pccb->tiSuperScsiRequest.flags & TI_SCSI_INITIATOR_ENCRYPT); pccb->flags |= EDC_DATA; #ifdef AGTIAPI_TEST_DIF pccb->tiSuperScsiRequest.flags |= TI_SCSI_INITIATOR_DIF; pccb->tiSuperScsiRequest.Dif.flags = DIF_INSERT | DIF_UDT_REF_BLOCK_COUNT; switch(pDev->sector_size) { case 528: pccb->tiSuperScsiRequest.Dif.flags |= (DIF_BLOCK_SIZE_520 << 16); break; case 4104: pccb->tiSuperScsiRequest.Dif.flags |= ( DIF_BLOCK_SIZE_4096 << 16 ); break; case 4168: pccb->tiSuperScsiRequest.Dif.flags |= ( DIF_BLOCK_SIZE_4160 << 16 ); break; } /* Turn on upper 4 bits of UUM */ pccb->tiSuperScsiRequest.Dif.flags |= 0xf0000000; #endif #ifdef AGTIAPI_TEST_DPL if(agtiapi_SetupDifPerLA(pCard, pccb, start_lba) < 0) { printk(KERN_ERR "SetupDifPerLA Failed.\n"); cmnd->result = SCSI_HOST(DID_ERROR); goto err; } pccb->tiSuperScsiRequest.Dif.enableDIFPerLA = TRUE; #endif #ifdef AGTIAPI_TEST_DIF /* Set App Tag */ pccb->tiSuperScsiRequest.Dif.udtArray[0] = 0xaa; pccb->tiSuperScsiRequest.Dif.udtArray[1] = 0xbb; /* Set LBA in UDT array */ if(CDB[0] == WRITE_6) { pccb->tiSuperScsiRequest.Dif.udtArray[2] = CDB[3]; pccb->tiSuperScsiRequest.Dif.udtArray[3] = CDB[2]; pccb->tiSuperScsiRequest.Dif.udtArray[4] = CDB[1] & 0x1f; } else if(CDB[0] == WRITE_10 || CDB[0] == WRITE_12) { pccb->tiSuperScsiRequest.Dif.udtArray[2] = CDB[5]; pccb->tiSuperScsiRequest.Dif.udtArray[3] = CDB[4]; pccb->tiSuperScsiRequest.Dif.udtArray[4] = CDB[3]; pccb->tiSuperScsiRequest.Dif.udtArray[5] = CDB[2]; } else if(CDB[0] == WRITE_16) { pccb->tiSuperScsiRequest.Dif.udtArray[2] = CDB[5]; pccb->tiSuperScsiRequest.Dif.udtArray[3] = CDB[4]; pccb->tiSuperScsiRequest.Dif.udtArray[4] = CDB[3]; pccb->tiSuperScsiRequest.Dif.udtArray[5] = CDB[2]; /* Note: 32 bits lost */ } #endif break; } } #endif /* end of DIF */ if ((ccb->ccb_h.flags & CAM_TAG_ACTION_VALID) != 0) { switch(csio->tag_action) { case MSG_HEAD_OF_Q_TAG: pScsiCmnd->taskAttribute = TASK_HEAD_OF_QUEUE; break; case MSG_ACA_TASK: pScsiCmnd->taskAttribute = TASK_ACA; break; case MSG_ORDERED_Q_TAG: pScsiCmnd->taskAttribute = TASK_ORDERED; break; case MSG_SIMPLE_Q_TAG: /* fall through */ default: pScsiCmnd->taskAttribute = TASK_SIMPLE; break; } } if (pccb->tiSuperScsiRequest.agSgl1.len != 0 && pccb->dataLen != 0) { /* should be just before start IO */ bus_dmamap_sync(pmcsc->buffer_dmat, pccb->CCB_dmamap, op); } /* * If assigned pDevHandle is not available * then there is no need to send it to StartIO() */ if (pccb->targetId < 0 || pccb->targetId >= maxTargets) { pccb->ccbStatus = tiIOFailed; pccb->scsiStatus = tiDetailNoLogin; agtiapi_FreeCCB(pmcsc, pccb); ccb->ccb_h.status = CAM_DEV_NOT_THERE; // ## v. CAM_FUNC_NOTAVAIL xpt_done(ccb); pccb->ccb = NULL; return; } TID = INDEX(pmcsc, pccb->targetId); if ((TID >= pmcsc->devDiscover) || !(pccb->devHandle = pmcsc->pDevList[TID].pDevHandle)) { /* AGTIAPI_PRINTK( "agtiapi_PrepareSGListCB: not sending ccb devH %p," " target %d tid %d/%d card %p ERROR pccb %p\n", pccb->devHandle, pccb->targetId, TID, pmcsc->devDiscover, pmcsc, pccb ); */ pccb->ccbStatus = tiIOFailed; pccb->scsiStatus = tiDetailNoLogin; agtiapi_FreeCCB(pmcsc, pccb); ccb->ccb_h.status = CAM_DEV_NOT_THERE; // ## v. CAM_FUNC_NOTAVAIL xpt_done(ccb); pccb->ccb = NULL; return; } AGTIAPI_IO( "agtiapi_PrepareSGListCB: send ccb pccb->devHandle %p, " "pccb->targetId %d TID %d pmcsc->devDiscover %d card %p\n", pccb->devHandle, pccb->targetId, TID, pmcsc->devDiscover, pmcsc ); #ifdef HIALEAH_ENCRYPTION if(pmcsc->encrypt && io_is_encryptable) { agtiapi_SetupEncryptedIO(pmcsc, pccb, start_lba); } else{ io_is_encryptable = 0; pccb->tiSuperScsiRequest.flags = 0; } #endif // put the request in send queue agtiapi_QueueCCB( pmcsc, &pmcsc->ccbSendHead, &pmcsc->ccbSendTail AG_CARD_LOCAL_LOCK(&pmcsc->sendLock), pccb ); agtiapi_StartIO(pmcsc); return; } /****************************************************************************** agtiapi_StartIO() Purpose: Send IO request down for processing. Parameters: (struct agtiapi_softc *pmcsc (IN) Pointer to HBA data structure Return: Note: ******************************************************************************/ STATIC void agtiapi_StartIO( struct agtiapi_softc *pmcsc ) { ccb_t *pccb; int TID; ag_device_t *targ; struct ccb_relsim crs; AGTIAPI_IO( "agtiapi_StartIO: start\n" ); AG_LOCAL_LOCK( &pmcsc->sendLock ); pccb = pmcsc->ccbSendHead; /* if link is down, do nothing */ if ((pccb == NULL) || pmcsc->flags & AGTIAPI_RESET) { AG_LOCAL_UNLOCK( &pmcsc->sendLock ); AGTIAPI_PRINTK( "agtiapi_StartIO: goto ext\n" ); goto ext; } if (pmcsc != NULL && pccb->targetId >= 0 && pccb->targetId < maxTargets) { TID = INDEX(pmcsc, pccb->targetId); targ = &pmcsc->pDevList[TID]; } /* clear send queue */ pmcsc->ccbSendHead = NULL; pmcsc->ccbSendTail = NULL; AG_LOCAL_UNLOCK( &pmcsc->sendLock ); /* send all ccbs down */ while (pccb) { pccb_t pccb_next; U32 status; pccb_next = pccb->pccbNext; pccb->pccbNext = NULL; if (!pccb->ccb) { AGTIAPI_PRINTK( "agtiapi_StartIO: pccb->ccb is NULL ERROR!\n" ); pccb = pccb_next; continue; } AG_IO_DUMPCCB( pccb ); if (!pccb->devHandle) { agtiapi_DumpCCB( pccb ); AGTIAPI_PRINTK( "agtiapi_StartIO: ccb NULL device ERROR!\n" ); pccb = pccb_next; continue; } AGTIAPI_IO( "agtiapi_StartIO: ccb %p retry %d\n", pccb, pccb->retryCount ); #ifndef ABORT_TEST if( !pccb->devHandle || !pccb->devHandle->osData || /* in rmmod case */ !(((ag_device_t *)(pccb->devHandle->osData))->flags & ACTIVE)) { AGTIAPI_PRINTK( "agtiapi_StartIO: device %p not active! ERROR\n", pccb->devHandle ); if( pccb->devHandle ) { AGTIAPI_PRINTK( "agtiapi_StartIO: device not active detail" " -- osData:%p\n", pccb->devHandle->osData ); if( pccb->devHandle->osData ) { AGTIAPI_PRINTK( "agtiapi_StartIO: more device not active detail" " -- active flag:%d\n", ( (ag_device_t *) (pccb->devHandle->osData))->flags & ACTIVE ); } } pccb->ccbStatus = tiIOFailed; pccb->scsiStatus = tiDetailNoLogin; agtiapi_Done( pmcsc, pccb ); pccb = pccb_next; continue; } #endif #ifdef FAST_IO_TEST status = agtiapi_FastIOTest( pmcsc, pccb ); #else status = tiINISuperIOStart( &pmcsc->tiRoot, &pccb->tiIORequest, pccb->devHandle, &pccb->tiSuperScsiRequest, (void *)&pccb->tdIOReqBody, tiInterruptContext ); #endif switch( status ) { case tiSuccess: /* static int squelchCount = 0; if ( 200000 == squelchCount++ ) // squelch prints { AGTIAPI_PRINTK( "agtiapi_StartIO: tiINIIOStart stat tiSuccess %p\n", pccb ); squelchCount = 0; // reset count } */ break; case tiDeviceBusy: AGTIAPI_PRINTK( "agtiapi_StartIO: tiINIIOStart status tiDeviceBusy %p\n", pccb->ccb ); #ifdef LOGEVENT agtiapi_LogEvent( pmcsc, IOCTL_EVT_SEV_INFORMATIONAL, 0, agNULL, 0, "tiINIIOStart tiDeviceBusy " ); #endif pccb->ccbStatus = tiIOFailed; pccb->scsiStatus = tiDeviceBusy; agtiapi_Done(pmcsc, pccb); break; case tiBusy: AGTIAPI_PRINTK( "agtiapi_StartIO: tiINIIOStart status tiBusy %p\n", pccb->ccb ); #ifdef LOGEVENT agtiapi_LogEvent( pmcsc, IOCTL_EVT_SEV_INFORMATIONAL, 0, agNULL, 0, "tiINIIOStart tiBusy " ); #endif pccb->ccbStatus = tiIOFailed; pccb->scsiStatus = tiBusy; agtiapi_Done(pmcsc, pccb); break; case tiIONoDevice: AGTIAPI_PRINTK( "agtiapi_StartIO: tiINIIOStart status tiNoDevice %p " "ERROR\n", pccb->ccb ); #ifdef LOGEVENT agtiapi_LogEvent( pmcsc, IOCTL_EVT_SEV_INFORMATIONAL, 0, agNULL, 0, "tiINIIOStart invalid device handle " ); #endif #ifndef ABORT_TEST /* return command back to OS due to no device available */ ((ag_device_t *)(pccb->devHandle->osData))->flags &= ~ACTIVE; pccb->ccbStatus = tiIOFailed; pccb->scsiStatus = tiDetailNoLogin; agtiapi_Done(pmcsc, pccb); #else /* for short cable pull, we want IO retried - 3-18-2005 */ agtiapi_QueueCCB(pmcsc, &pmcsc->ccbSendHead, &pmcsc->ccbSendTail AG_CARD_LOCAL_LOCK(&pmcsc->sendLock), pccb); #endif break; case tiError: AGTIAPI_PRINTK("agtiapi_StartIO: tiINIIOStart status tiError %p\n", pccb->ccb); #ifdef LOGEVENT agtiapi_LogEvent(pmcsc, IOCTL_EVT_SEV_INFORMATIONAL, 0, agNULL, 0, "tiINIIOStart tiError "); #endif pccb->ccbStatus = tiIOFailed; pccb->scsiStatus = tiDetailOtherError; agtiapi_Done(pmcsc, pccb); break; default: AGTIAPI_PRINTK("agtiapi_StartIO: tiINIIOStart status default %x %p\n", status, pccb->ccb); #ifdef LOGEVENT agtiapi_LogEvent(pmcsc, IOCTL_EVT_SEV_ERROR, 0, agNULL, 0, "tiINIIOStart unexpected status "); #endif pccb->ccbStatus = tiIOFailed; pccb->scsiStatus = tiDetailOtherError; agtiapi_Done(pmcsc, pccb); } pccb = pccb_next; } ext: /* some IO requests might have been completed */ AG_GET_DONE_PCCB(pccb, pmcsc); return; } /****************************************************************************** agtiapi_StartSMP() Purpose: Send SMP request down for processing. Parameters: (struct agtiapi_softc *pmcsc (IN) Pointer to HBA data structure Return: Note: ******************************************************************************/ STATIC void agtiapi_StartSMP(struct agtiapi_softc *pmcsc) { ccb_t *pccb; AGTIAPI_PRINTK("agtiapi_StartSMP: start\n"); AG_LOCAL_LOCK(&pmcsc->sendSMPLock); pccb = pmcsc->smpSendHead; /* if link is down, do nothing */ if ((pccb == NULL) || pmcsc->flags & AGTIAPI_RESET) { AG_LOCAL_UNLOCK(&pmcsc->sendSMPLock); AGTIAPI_PRINTK("agtiapi_StartSMP: goto ext\n"); goto ext; } /* clear send queue */ pmcsc->smpSendHead = NULL; pmcsc->smpSendTail = NULL; AG_LOCAL_UNLOCK(&pmcsc->sendSMPLock); /* send all ccbs down */ while (pccb) { pccb_t pccb_next; U32 status; pccb_next = pccb->pccbNext; pccb->pccbNext = NULL; if (!pccb->ccb) { AGTIAPI_PRINTK("agtiapi_StartSMP: pccb->ccb is NULL ERROR!\n"); pccb = pccb_next; continue; } if (!pccb->devHandle) { AGTIAPI_PRINTK("agtiapi_StartSMP: ccb NULL device ERROR!\n"); pccb = pccb_next; continue; } pccb->flags |= TAG_SMP; // mark as SMP for later tracking AGTIAPI_PRINTK( "agtiapi_StartSMP: ccb %p retry %d\n", pccb, pccb->retryCount ); status = tiINISMPStart( &pmcsc->tiRoot, &pccb->tiIORequest, pccb->devHandle, &pccb->tiSMPFrame, (void *)&pccb->tdIOReqBody, tiInterruptContext); switch (status) { case tiSuccess: break; case tiBusy: AGTIAPI_PRINTK("agtiapi_StartSMP: tiINISMPStart status tiBusy %p\n", pccb->ccb); /* pending ccb back to send queue */ agtiapi_QueueCCB(pmcsc, &pmcsc->smpSendHead, &pmcsc->smpSendTail AG_CARD_LOCAL_LOCK(&pmcsc->sendSMPLock), pccb); break; case tiError: AGTIAPI_PRINTK("agtiapi_StartIO: tiINIIOStart status tiError %p\n", pccb->ccb); pccb->ccbStatus = tiSMPFailed; agtiapi_SMPDone(pmcsc, pccb); break; default: AGTIAPI_PRINTK("agtiapi_StartIO: tiINIIOStart status default %x %p\n", status, pccb->ccb); pccb->ccbStatus = tiSMPFailed; agtiapi_SMPDone(pmcsc, pccb); } pccb = pccb_next; } ext: /* some SMP requests might have been completed */ AG_GET_DONE_SMP_PCCB(pccb, pmcsc); return; } #if __FreeBSD_version > 901000 /****************************************************************************** agtiapi_PrepareSMPSGList() Purpose: This function prepares scatter-gather list for the given ccb Parameters: struct agtiapi_softc *pmsc (IN) Pointer to the HBA data structure ccb_t *pccb (IN) A pointer to the driver's own CCB, not CAM's CCB Return: 0 - success 1 - failure Note: ******************************************************************************/ static int agtiapi_PrepareSMPSGList( struct agtiapi_softc *pmcsc, ccb_t *pccb ) { /* Pointer to CAM's ccb */ union ccb *ccb = pccb->ccb; struct ccb_smpio *csmpio = &ccb->smpio; struct ccb_hdr *ccbh = &ccb->ccb_h; AGTIAPI_PRINTK("agtiapi_PrepareSMPSGList: start\n"); switch((ccbh->flags & CAM_DATA_MASK)) { case CAM_DATA_PADDR: case CAM_DATA_SG_PADDR: AGTIAPI_PRINTK( "agtiapi_PrepareSMPSGList: Physical Address not supported\n"); ccb->ccb_h.status = CAM_REQ_INVALID; xpt_done(ccb); return tiReject; case CAM_DATA_SG: /* * Currently we do not support Multiple SG list * return error for now */ if ( (csmpio->smp_request_sglist_cnt > 1) || (csmpio->smp_response_sglist_cnt > 1) ) { AGTIAPI_PRINTK( "agtiapi_PrepareSMPSGList: Multiple SG list not supported\n"); ccb->ccb_h.status = CAM_REQ_INVALID; xpt_done(ccb); return tiReject; } } if ( csmpio->smp_request_sglist_cnt != 0 ) { /* * Virtual address that needs to translated into * one or more physical address ranges. */ int error; //AG_LOCAL_LOCK(&(pmcsc->pCardInfo->pmIOLock)); AGTIAPI_PRINTK("agtiapi_PrepareSGList: virtual address\n"); error = bus_dmamap_load( pmcsc->buffer_dmat, pccb->CCB_dmamap, csmpio->smp_request, csmpio->smp_request_len, agtiapi_PrepareSMPSGListCB, pccb, BUS_DMA_NOWAIT /* 0 */ ); //AG_LOCAL_UNLOCK(&(pmcsc->pCardInfo->pmIOLock)); if (error == EINPROGRESS) { /* * So as to maintain ordering, * freeze the controller queue * until our mapping is * returned. */ AGTIAPI_PRINTK( "agtiapi_PrepareSGList: EINPROGRESS\n" ); xpt_freeze_simq( pmcsc->sim, 1 ); pmcsc->SimQFrozen = agTRUE; ccbh->status |= CAM_RELEASE_SIMQ; } } if( csmpio->smp_response_sglist_cnt != 0 ) { /* * Virtual address that needs to translated into * one or more physical address ranges. */ int error; //AG_LOCAL_LOCK( &(pmcsc->pCardInfo->pmIOLock) ); AGTIAPI_PRINTK( "agtiapi_PrepareSGList: virtual address\n" ); error = bus_dmamap_load( pmcsc->buffer_dmat, pccb->CCB_dmamap, csmpio->smp_response, csmpio->smp_response_len, agtiapi_PrepareSMPSGListCB, pccb, BUS_DMA_NOWAIT /* 0 */ ); //AG_LOCAL_UNLOCK( &(pmcsc->pCardInfo->pmIOLock) ); if ( error == EINPROGRESS ) { /* * So as to maintain ordering, * freeze the controller queue * until our mapping is * returned. */ AGTIAPI_PRINTK( "agtiapi_PrepareSGList: EINPROGRESS\n" ); xpt_freeze_simq( pmcsc->sim, 1 ); pmcsc->SimQFrozen = agTRUE; ccbh->status |= CAM_RELEASE_SIMQ; } } else { if ( (csmpio->smp_request_sglist_cnt == 0) && (csmpio->smp_response_sglist_cnt == 0) ) { AGTIAPI_PRINTK( "agtiapi_PrepareSMPSGList: physical address\n" ); pccb->tiSMPFrame.outFrameBuf = (void *)csmpio->smp_request; pccb->tiSMPFrame.outFrameLen = csmpio->smp_request_len; pccb->tiSMPFrame.expectedRespLen = csmpio->smp_response_len; // 0xFF to be defined agtiapi_PrepareSMPSGListCB( pccb, NULL, 0, 0xAABBCCDD ); } pccb->tiSMPFrame.flag = 0; } return tiSuccess; } #else /****************************************************************************** agtiapi_PrepareSMPSGList() Purpose: This function prepares scatter-gather list for the given ccb Parameters: struct agtiapi_softc *pmsc (IN) Pointer to the HBA data structure ccb_t *pccb (IN) A pointer to the driver's own CCB, not CAM's CCB Return: 0 - success 1 - failure Note: ******************************************************************************/ static int agtiapi_PrepareSMPSGList( struct agtiapi_softc *pmcsc, ccb_t *pccb ) { /* Pointer to CAM's ccb */ union ccb *ccb = pccb->ccb; struct ccb_smpio *csmpio = &ccb->smpio; struct ccb_hdr *ccbh = &ccb->ccb_h; AGTIAPI_PRINTK("agtiapi_PrepareSMPSGList: start\n"); if (ccbh->flags & (CAM_DATA_PHYS|CAM_SG_LIST_PHYS)) { AGTIAPI_PRINTK( "agtiapi_PrepareSMPSGList: Physical Address " "not supported\n" ); ccb->ccb_h.status = CAM_REQ_INVALID; xpt_done(ccb); return tiReject;; } if (ccbh->flags & CAM_SCATTER_VALID) { /* * Currently we do not support Multiple SG list * return error for now */ if ( (csmpio->smp_request_sglist_cnt > 1) || (csmpio->smp_response_sglist_cnt > 1) ) { AGTIAPI_PRINTK( "agtiapi_PrepareSMPSGList: Multiple SG list " "not supported\n" ); ccb->ccb_h.status = CAM_REQ_INVALID; xpt_done(ccb); return tiReject;; } if ( csmpio->smp_request_sglist_cnt != 0 ) { /* * Virtual address that needs to translated into * one or more physical address ranges. */ int error; //AG_LOCAL_LOCK(&(pmcsc->pCardInfo->pmIOLock)); AGTIAPI_PRINTK("agtiapi_PrepareSGList: virtual address\n"); error = bus_dmamap_load( pmcsc->buffer_dmat, pccb->CCB_dmamap, csmpio->smp_request, csmpio->smp_request_len, agtiapi_PrepareSMPSGListCB, pccb, BUS_DMA_NOWAIT /* 0 */ ); //AG_LOCAL_UNLOCK(&(pmcsc->pCardInfo->pmIOLock)); if (error == EINPROGRESS) { /* * So as to maintain ordering, * freeze the controller queue * until our mapping is * returned. */ AGTIAPI_PRINTK( "agtiapi_PrepareSGList: EINPROGRESS\n" ); xpt_freeze_simq( pmcsc->sim, 1 ); pmcsc->SimQFrozen = agTRUE; ccbh->status |= CAM_RELEASE_SIMQ; } } if( csmpio->smp_response_sglist_cnt != 0 ) { /* * Virtual address that needs to translated into * one or more physical address ranges. */ int error; //AG_LOCAL_LOCK( &(pmcsc->pCardInfo->pmIOLock) ); AGTIAPI_PRINTK( "agtiapi_PrepareSGList: virtual address\n" ); error = bus_dmamap_load( pmcsc->buffer_dmat, pccb->CCB_dmamap, csmpio->smp_response, csmpio->smp_response_len, agtiapi_PrepareSMPSGListCB, pccb, BUS_DMA_NOWAIT /* 0 */ ); //AG_LOCAL_UNLOCK( &(pmcsc->pCardInfo->pmIOLock) ); if ( error == EINPROGRESS ) { /* * So as to maintain ordering, * freeze the controller queue * until our mapping is * returned. */ AGTIAPI_PRINTK( "agtiapi_PrepareSGList: EINPROGRESS\n" ); xpt_freeze_simq( pmcsc->sim, 1 ); pmcsc->SimQFrozen = agTRUE; ccbh->status |= CAM_RELEASE_SIMQ; } } } else { if ( (csmpio->smp_request_sglist_cnt == 0) && (csmpio->smp_response_sglist_cnt == 0) ) { AGTIAPI_PRINTK( "agtiapi_PrepareSMPSGList: physical address\n" ); pccb->tiSMPFrame.outFrameBuf = (void *)csmpio->smp_request; pccb->tiSMPFrame.outFrameLen = csmpio->smp_request_len; pccb->tiSMPFrame.expectedRespLen = csmpio->smp_response_len; // 0xFF to be defined agtiapi_PrepareSMPSGListCB( pccb, NULL, 0, 0xAABBCCDD ); } pccb->tiSMPFrame.flag = 0; } return tiSuccess; } #endif /****************************************************************************** agtiapi_PrepareSMPSGListCB() Purpose: Callback function for bus_dmamap_load() This fuctions sends IO to LL layer. Parameters: void *arg (IN) Pointer to the HBA data structure bus_dma_segment_t *segs (IN) Pointer to dma segment int nsegs (IN) number of dma segment int error (IN) error Return: Note: ******************************************************************************/ static void agtiapi_PrepareSMPSGListCB( void *arg, bus_dma_segment_t *segs, int nsegs, int error ) { pccb_t pccb = arg; union ccb *ccb = pccb->ccb; struct agtiapi_softc *pmcsc; U32 TID = CMND_TO_TARGET(ccb); int status; tiDeviceHandle_t *tiExpDevHandle; tiPortalContext_t *tiExpPortalContext; ag_portal_info_t *tiExpPortalInfo; AGTIAPI_PRINTK( "agtiapi_PrepareSMPSGListCB: start, nsegs %d error 0x%x\n", nsegs, error ); pmcsc = pccb->pmcsc; if ( error != tiSuccess ) { if (error == 0xAABBCCDD) { // do nothing } else { AGTIAPI_PRINTK( "agtiapi_PrepareSMPSGListCB: error status 0x%x\n", error ); bus_dmamap_unload( pmcsc->buffer_dmat, pccb->CCB_dmamap ); bus_dmamap_destroy( pmcsc->buffer_dmat, pccb->CCB_dmamap ); agtiapi_FreeCCB( pmcsc, pccb ); ccb->ccb_h.status = CAM_REQ_CMP; xpt_done( ccb ); return; } } if ( nsegs > AGTIAPI_MAX_DMA_SEGS ) { AGTIAPI_PRINTK( "agtiapi_PrepareSMPSGListCB: over the limit. nsegs %d " "AGTIAPI_MAX_DMA_SEGS %d\n", nsegs, AGTIAPI_MAX_DMA_SEGS ); bus_dmamap_unload( pmcsc->buffer_dmat, pccb->CCB_dmamap ); bus_dmamap_destroy( pmcsc->buffer_dmat, pccb->CCB_dmamap ); agtiapi_FreeCCB( pmcsc, pccb ); ccb->ccb_h.status = CAM_REQ_CMP; xpt_done( ccb ); return; } /* * If assigned pDevHandle is not available * then there is no need to send it to StartIO() */ /* TODO: Add check for deviceType */ if ( pccb->targetId < 0 || pccb->targetId >= maxTargets ) { agtiapi_FreeCCB( pmcsc, pccb ); ccb->ccb_h.status = CAM_FUNC_NOTAVAIL; xpt_done(ccb); pccb->ccb = NULL; return; } TID = INDEX( pmcsc, pccb->targetId ); if ( (TID >= pmcsc->devDiscover) || !(pccb->devHandle = pmcsc->pDevList[TID].pDevHandle) ) { AGTIAPI_PRINTK( "agtiapi_PrepareSMPSGListCB: not sending ccb devH %p, " "target %d tid %d/%d " "card %p ERROR pccb %p\n", pccb->devHandle, pccb->targetId, TID, pmcsc->devDiscover, pmcsc, pccb ); agtiapi_FreeCCB( pmcsc, pccb ); ccb->ccb_h.status = CAM_FUNC_NOTAVAIL; xpt_done( ccb ); pccb->ccb = NULL; return; } /* TODO: add indirect handling */ - /* set the flag correctly based on Indiret SMP request and responce */ + /* set the flag correctly based on Indiret SMP request and response */ AGTIAPI_PRINTK( "agtiapi_PrepareSMPSGListCB: send ccb pccb->devHandle %p, " "pccb->targetId %d TID %d pmcsc->devDiscover %d card %p\n", pccb->devHandle, pccb->targetId, TID, pmcsc->devDiscover, pmcsc ); tiExpDevHandle = pccb->devHandle; tiExpPortalInfo = pmcsc->pDevList[TID].pPortalInfo; tiExpPortalContext = &tiExpPortalInfo->tiPortalContext; /* Look for the expander associated with the ses device */ status = tiINIGetExpander( &pmcsc->tiRoot, tiExpPortalContext, pccb->devHandle, &tiExpDevHandle ); if ( status != tiSuccess ) { AGTIAPI_PRINTK( "agtiapi_PrepareSMPSGListCB: Error getting Expander " "device\n" ); agtiapi_FreeCCB( pmcsc, pccb ); ccb->ccb_h.status = CAM_FUNC_NOTAVAIL; xpt_done( ccb ); pccb->ccb = NULL; return; } /* this is expander device */ pccb->devHandle = tiExpDevHandle; /* put the request in send queue */ agtiapi_QueueCCB( pmcsc, &pmcsc->smpSendHead, &pmcsc->smpSendTail AG_CARD_LOCAL_LOCK(&pmcsc->sendSMPLock), pccb ); agtiapi_StartSMP( pmcsc ); return; } /****************************************************************************** agtiapi_Done() Purpose: Processing completed ccbs Parameters: struct agtiapi_softc *pmcsc (IN) Pointer to HBA data structure ccb_t *pccb (IN) A pointer to the driver's own CCB, not CAM's CCB Return: Note: ******************************************************************************/ STATIC void agtiapi_Done(struct agtiapi_softc *pmcsc, ccb_t *pccb) { pccb_t pccb_curr = pccb; pccb_t pccb_next; tiIniScsiCmnd_t *cmnd; union ccb * ccb; AGTIAPI_IO("agtiapi_Done: start\n"); while (pccb_curr) { /* start from 1st ccb in the chain */ pccb_next = pccb_curr->pccbNext; if (agtiapi_CheckError(pmcsc, pccb_curr) != 0) { /* send command back and release the ccb */ cmnd = &pccb_curr->tiSuperScsiRequest.scsiCmnd; if (cmnd->cdb[0] == RECEIVE_DIAGNOSTIC) { AGTIAPI_PRINTK("agtiapi_Done: RECEIVE_DIAG pg %d id %d cmnd %p pccb " "%p\n", cmnd->cdb[2], pccb_curr->targetId, cmnd, pccb_curr); } CMND_DMA_UNMAP(pmcsc, ccb); /* send the request back to the CAM */ ccb = pccb_curr->ccb; agtiapi_FreeCCB(pmcsc, pccb_curr); xpt_done(ccb); } pccb_curr = pccb_next; } return; } /****************************************************************************** agtiapi_SMPDone() Purpose: Processing completed ccbs Parameters: struct agtiapi_softc *pmcsc (IN) Ponter to HBA data structure ccb_t *pccb (IN) A pointer to the driver's own CCB, not CAM's CCB Return: Note: ******************************************************************************/ STATIC void agtiapi_SMPDone(struct agtiapi_softc *pmcsc, ccb_t *pccb) { pccb_t pccb_curr = pccb; pccb_t pccb_next; union ccb * ccb; AGTIAPI_PRINTK("agtiapi_SMPDone: start\n"); while (pccb_curr) { /* start from 1st ccb in the chain */ pccb_next = pccb_curr->pccbNext; if (agtiapi_CheckSMPError(pmcsc, pccb_curr) != 0) { CMND_DMA_UNMAP(pmcsc, ccb); /* send the request back to the CAM */ ccb = pccb_curr->ccb; agtiapi_FreeSMPCCB(pmcsc, pccb_curr); xpt_done(ccb); } pccb_curr = pccb_next; } AGTIAPI_PRINTK("agtiapi_SMPDone: Done\n"); return; } /****************************************************************************** agtiapi_hexdump() Purpose: Utility function for dumping in hex Parameters: const char *ptitle (IN) A string to be printed bit8 *pbuf (IN) A pointer to a buffer to be printed. int len (IN) The lengther of the buffer Return: Note: ******************************************************************************/ void agtiapi_hexdump(const char *ptitle, bit8 *pbuf, int len) { int i; AGTIAPI_PRINTK("%s - hexdump(len=%d):\n", ptitle, (int)len); if (!pbuf) { AGTIAPI_PRINTK("pbuf is NULL\n"); return; } for (i = 0; i < len; ) { if (len - i > 4) { AGTIAPI_PRINTK( " 0x%02x, 0x%02x, 0x%02x, 0x%02x,\n", pbuf[i], pbuf[i+1], pbuf[i+2], pbuf[i+3] ); i += 4; } else { AGTIAPI_PRINTK(" 0x%02x,", pbuf[i]); i++; } } AGTIAPI_PRINTK("\n"); } /****************************************************************************** agtiapi_CheckError() Purpose: Processes status pertaining to the ccb -- whether it was completed successfully, aborted, or error encountered. Parameters: ag_card_t *pCard (IN) Pointer to HBA data structure ccb_t *pccd (IN) A pointer to the driver's own CCB, not CAM's CCB Return: 0 - the command retry is required 1 - the command process is completed Note: ******************************************************************************/ STATIC U32 agtiapi_CheckError(struct agtiapi_softc *pmcsc, ccb_t *pccb) { ag_device_t *pDevice; // union ccb * ccb = pccb->ccb; union ccb * ccb; int is_error, TID; if (pccb == NULL) { return 0; } ccb = pccb->ccb; AGTIAPI_IO("agtiapi_CheckError: start\n"); if (ccb == NULL) { /* shouldn't be here but just in case we do */ AGTIAPI_PRINTK("agtiapi_CheckError: CCB orphan = %p ERROR\n", pccb); agtiapi_FreeCCB(pmcsc, pccb); return 0; } is_error = 1; pDevice = NULL; if (pmcsc != NULL && pccb->targetId >= 0 && pccb->targetId < maxTargets) { if (pmcsc->pWWNList != NULL) { TID = INDEX(pmcsc, pccb->targetId); if (TID < maxTargets) { pDevice = &pmcsc->pDevList[TID]; if (pDevice != NULL) { is_error = 0; } } } } if (is_error) { AGTIAPI_PRINTK("agtiapi_CheckError: pDevice == NULL\n"); agtiapi_FreeCCB(pmcsc, pccb); return 0; } /* SCSI status */ ccb->csio.scsi_status = pccb->scsiStatus; if(pDevice->CCBCount > 0){ atomic_subtract_int(&pDevice->CCBCount,1); } AG_LOCAL_LOCK(&pmcsc->freezeLock); if(pmcsc->freezeSim == agTRUE) { pmcsc->freezeSim = agFALSE; xpt_release_simq(pmcsc->sim, 1); } AG_LOCAL_UNLOCK(&pmcsc->freezeLock); switch (pccb->ccbStatus) { case tiIOSuccess: AGTIAPI_IO("agtiapi_CheckError: tiIOSuccess pccb %p\n", pccb); /* CAM status */ if (pccb->scsiStatus == SCSI_STATUS_OK) { ccb->ccb_h.status = CAM_REQ_CMP; } else if (pccb->scsiStatus == SCSI_TASK_ABORTED) { ccb->ccb_h.status = CAM_REQ_ABORTED; } else { ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR; } if (ccb->csio.scsi_status == SCSI_CHECK_CONDITION) { ccb->ccb_h.status |= CAM_AUTOSNS_VALID; } break; case tiIOOverRun: AGTIAPI_PRINTK("agtiapi_CheckError: tiIOOverRun pccb %p\n", pccb); /* resid is ignored for this condition */ ccb->csio.resid = 0; ccb->ccb_h.status = CAM_DATA_RUN_ERR; break; case tiIOUnderRun: AGTIAPI_PRINTK("agtiapi_CheckError: tiIOUnderRun pccb %p\n", pccb); ccb->csio.resid = pccb->scsiStatus; ccb->ccb_h.status = CAM_REQ_CMP; ccb->csio.scsi_status = SCSI_STATUS_OK; break; case tiIOFailed: AGTIAPI_PRINTK( "agtiapi_CheckError: pccb %p tiIOFailed %d id %d ERROR\n", pccb, pccb->scsiStatus, pccb->targetId ); if (pccb->scsiStatus == tiDeviceBusy) { AGTIAPI_IO( "agtiapi_CheckError: pccb %p tiIOFailed - tiDetailBusy\n", pccb ); ccb->ccb_h.status &= ~CAM_STATUS_MASK; ccb->ccb_h.status |= CAM_REQUEUE_REQ; if ((ccb->ccb_h.status & CAM_DEV_QFRZN) == 0) { ccb->ccb_h.status |= CAM_DEV_QFRZN; xpt_freeze_devq(ccb->ccb_h.path, /*count*/1); } } else if(pccb->scsiStatus == tiBusy) { AG_LOCAL_LOCK(&pmcsc->freezeLock); if(pmcsc->freezeSim == agFALSE) { pmcsc->freezeSim = agTRUE; xpt_freeze_simq(pmcsc->sim, 1); } AG_LOCAL_UNLOCK(&pmcsc->freezeLock); ccb->ccb_h.status &= ~CAM_SIM_QUEUED; ccb->ccb_h.status |= CAM_REQUEUE_REQ; } else if (pccb->scsiStatus == tiDetailNoLogin) { AGTIAPI_PRINTK( "agtiapi_CheckError: pccb %p tiIOFailed - " "tiDetailNoLogin ERROR\n", pccb ); ccb->ccb_h.status = CAM_DEV_NOT_THERE; } else if (pccb->scsiStatus == tiDetailNotValid) { AGTIAPI_PRINTK( "agtiapi_CheckError: pccb %p tiIOFailed - " "tiDetailNotValid ERROR\n", pccb ); ccb->ccb_h.status = CAM_REQ_INVALID; } else if (pccb->scsiStatus == tiDetailAbortLogin) { AGTIAPI_PRINTK( "agtiapi_CheckError: pccb %p tiIOFailed - " "tiDetailAbortLogin ERROR\n", pccb ); ccb->ccb_h.status = CAM_REQ_ABORTED; } else if (pccb->scsiStatus == tiDetailAbortReset) { AGTIAPI_PRINTK( "agtiapi_CheckError: pccb %p tiIOFailed - " "tiDetailAbortReset ERROR\n", pccb ); ccb->ccb_h.status = CAM_REQ_ABORTED; } else if (pccb->scsiStatus == tiDetailAborted) { AGTIAPI_PRINTK( "agtiapi_CheckError: pccb %p tiIOFailed - " "tiDetailAborted ERROR\n", pccb ); ccb->ccb_h.status = CAM_REQ_ABORTED; } else if (pccb->scsiStatus == tiDetailOtherError) { AGTIAPI_PRINTK( "agtiapi_CheckError: pccb %p tiIOFailed - " "tiDetailOtherError ERROR\n", pccb ); ccb->ccb_h.status = CAM_REQ_ABORTED; } break; case tiIODifError: AGTIAPI_PRINTK( "agtiapi_CheckError: pccb %p tiIOFailed %d id %d ERROR\n", pccb, pccb->scsiStatus, pccb->targetId ); if (pccb->scsiStatus == tiDetailDifAppTagMismatch) { AGTIAPI_IO( "agtiapi_CheckError: pccb %p tiIOFailed - " "tiDetailDifAppTagMismatch\n", pccb ); ccb->ccb_h.status = CAM_REQ_CMP_ERR; } else if (pccb->scsiStatus == tiDetailDifRefTagMismatch) { AGTIAPI_PRINTK( "agtiapi_CheckError: pccb %p tiIOFailed - " "tiDetailDifRefTagMismatch\n", pccb ); ccb->ccb_h.status = CAM_REQ_CMP_ERR; } else if (pccb->scsiStatus == tiDetailDifCrcMismatch) { AGTIAPI_PRINTK( "agtiapi_CheckError: pccb %p tiIOFailed - " "tiDetailDifCrcMismatch\n", pccb ); ccb->ccb_h.status = CAM_REQ_CMP_ERR; } break; #ifdef HIALEAH_ENCRYPTION case tiIOEncryptError: AGTIAPI_PRINTK( "agtiapi_CheckError: pccb %p tiIOFailed %d id %d ERROR\n", pccb, pccb->scsiStatus, pccb->targetId ); if (pccb->scsiStatus == tiDetailDekKeyCacheMiss) { AGTIAPI_PRINTK( "agtiapi_CheckError: %s: pccb %p tiIOFailed - " "tiDetailDekKeyCacheMiss ERROR\n", __FUNCTION__, pccb ); ccb->ccb_h.status = CAM_REQ_ABORTED; agtiapi_HandleEncryptedIOFailure(pDevice, pccb); } else if (pccb->scsiStatus == tiDetailDekIVMismatch) { AGTIAPI_PRINTK( "agtiapi_CheckError: %s: pccb %p tiIOFailed - " "tiDetailDekIVMismatch ERROR\n", __FUNCTION__, pccb ); ccb->ccb_h.status = CAM_REQ_ABORTED; agtiapi_HandleEncryptedIOFailure(pDevice, pccb); } break; #endif default: AGTIAPI_PRINTK( "agtiapi_CheckError: pccb %p tiIOdefault %d id %d ERROR\n", pccb, pccb->ccbStatus, pccb->targetId ); ccb->ccb_h.status = CAM_REQ_CMP_ERR; break; } return 1; } /****************************************************************************** agtiapi_SMPCheckError() Purpose: Processes status pertaining to the ccb -- whether it was completed successfully, aborted, or error encountered. Parameters: ag_card_t *pCard (IN) Pointer to HBA data structure ccb_t *pccd (IN) A pointer to the driver's own CCB, not CAM's CCB Return: 0 - the command retry is required 1 - the command process is completed Note: ******************************************************************************/ STATIC U32 agtiapi_CheckSMPError( struct agtiapi_softc *pmcsc, ccb_t *pccb ) { union ccb * ccb = pccb->ccb; AGTIAPI_PRINTK("agtiapi_CheckSMPError: start\n"); if (!ccb) { /* shouldn't be here but just in case we do */ AGTIAPI_PRINTK( "agtiapi_CheckSMPError: CCB orphan = %p ERROR\n", pccb ); agtiapi_FreeSMPCCB(pmcsc, pccb); return 0; } switch (pccb->ccbStatus) { case tiSMPSuccess: AGTIAPI_PRINTK( "agtiapi_CheckSMPError: tiSMPSuccess pccb %p\n", pccb ); /* CAM status */ ccb->ccb_h.status = CAM_REQ_CMP; break; case tiSMPFailed: AGTIAPI_PRINTK( "agtiapi_CheckSMPError: tiSMPFailed pccb %p\n", pccb ); /* CAM status */ ccb->ccb_h.status = CAM_REQ_CMP_ERR; break; default: AGTIAPI_PRINTK( "agtiapi_CheckSMPError: pccb %p tiSMPdefault %d " "id %d ERROR\n", pccb, pccb->ccbStatus, pccb->targetId ); ccb->ccb_h.status = CAM_REQ_CMP_ERR; break; } return 1; } /****************************************************************************** agtiapi_HandleEncryptedIOFailure(): Purpose: Parameters: Return: Note: Currently not used. ******************************************************************************/ void agtiapi_HandleEncryptedIOFailure(ag_device_t *pDev, ccb_t *pccb) { AGTIAPI_PRINTK("agtiapi_HandleEncryptedIOFailure: start\n"); return; } /****************************************************************************** agtiapi_Retry() Purpose: Retry a ccb. Parameters: struct agtiapi_softc *pmcsc (IN) Pointer to the HBA structure ccb_t *pccb (IN) A pointer to the driver's own CCB, not CAM's CCB Return: Note: Currently not used. ******************************************************************************/ STATIC void agtiapi_Retry(struct agtiapi_softc *pmcsc, ccb_t *pccb) { pccb->retryCount++; pccb->flags = ACTIVE | AGTIAPI_RETRY; pccb->ccbStatus = 0; pccb->scsiStatus = 0; pccb->startTime = ticks; AGTIAPI_PRINTK( "agtiapi_Retry: start\n" ); AGTIAPI_PRINTK( "agtiapi_Retry: ccb %p retry %d flgs x%x\n", pccb, pccb->retryCount, pccb->flags ); agtiapi_QueueCCB(pmcsc, &pmcsc->ccbSendHead, &pmcsc->ccbSendTail AG_CARD_LOCAL_LOCK(&pmcsc->sendLock), pccb); return; } /****************************************************************************** agtiapi_DumpCCB() Purpose: Dump CCB for debuging Parameters: ccb_t *pccb (IN) A pointer to the driver's own CCB, not CAM's CCB Return: Note: ******************************************************************************/ STATIC void agtiapi_DumpCCB(ccb_t *pccb) { AGTIAPI_PRINTK("agtiapi_DumpCCB: pccb %p, devHandle %p, tid %d, lun %d\n", pccb, pccb->devHandle, pccb->targetId, pccb->lun); AGTIAPI_PRINTK("flag 0x%x, add_mode 0x%x, ccbStatus 0x%x, scsiStatus 0x%x\n", pccb->flags, pccb->addrMode, pccb->ccbStatus, pccb->scsiStatus); AGTIAPI_PRINTK("scsi comand = 0x%x, numSgElements = %d\n", pccb->tiSuperScsiRequest.scsiCmnd.cdb[0], pccb->numSgElements); AGTIAPI_PRINTK("dataLen = 0x%x, sens_len = 0x%x\n", pccb->dataLen, pccb->senseLen); AGTIAPI_PRINTK("tiSuperScsiRequest:\n"); AGTIAPI_PRINTK("scsiCmnd: expDataLength 0x%x, taskAttribute 0x%x\n", pccb->tiSuperScsiRequest.scsiCmnd.expDataLength, pccb->tiSuperScsiRequest.scsiCmnd.taskAttribute); AGTIAPI_PRINTK("cdb[0] = 0x%x, cdb[1] = 0x%x, cdb[2] = 0x%x, cdb[3] = 0x%x\n", pccb->tiSuperScsiRequest.scsiCmnd.cdb[0], pccb->tiSuperScsiRequest.scsiCmnd.cdb[1], pccb->tiSuperScsiRequest.scsiCmnd.cdb[2], pccb->tiSuperScsiRequest.scsiCmnd.cdb[3]); AGTIAPI_PRINTK("cdb[4] = 0x%x, cdb[5] = 0x%x, cdb[6] = 0x%x, cdb[7] = 0x%x\n", pccb->tiSuperScsiRequest.scsiCmnd.cdb[4], pccb->tiSuperScsiRequest.scsiCmnd.cdb[5], pccb->tiSuperScsiRequest.scsiCmnd.cdb[6], pccb->tiSuperScsiRequest.scsiCmnd.cdb[7]); AGTIAPI_PRINTK( "cdb[8] = 0x%x, cdb[9] = 0x%x, cdb[10] = 0x%x, " "cdb[11] = 0x%x\n", pccb->tiSuperScsiRequest.scsiCmnd.cdb[8], pccb->tiSuperScsiRequest.scsiCmnd.cdb[9], pccb->tiSuperScsiRequest.scsiCmnd.cdb[10], pccb->tiSuperScsiRequest.scsiCmnd.cdb[11] ); AGTIAPI_PRINTK("agSgl1: upper 0x%x, lower 0x%x, len 0x%x, type %d\n", pccb->tiSuperScsiRequest.agSgl1.upper, pccb->tiSuperScsiRequest.agSgl1.lower, pccb->tiSuperScsiRequest.agSgl1.len, pccb->tiSuperScsiRequest.agSgl1.type); } /****************************************************************************** agtiapi_eh_HostReset() Purpose: A new error handler of Host Reset command. Parameters: scsi_cmnd *cmnd (IN) Pointer to a command to the HBA to be reset Return: SUCCESS - success FAILED - fail Note: ******************************************************************************/ int agtiapi_eh_HostReset( struct agtiapi_softc *pmcsc, union ccb *cmnd ) { AGTIAPI_PRINTK( "agtiapi_eh_HostReset: ccb pointer %p\n", cmnd ); if( cmnd == NULL ) { printf( "agtiapi_eh_HostReset: null command, skipping reset.\n" ); return tiInvalidHandle; } #ifdef LOGEVENT agtiapi_LogEvent( pmcsc, IOCTL_EVT_SEV_INFORMATIONAL, 0, agNULL, 0, "agtiapi_eh_HostReset! " ); #endif return agtiapi_DoSoftReset( pmcsc ); } int agtiapi_eh_DeviceReset( struct agtiapi_softc *pmcsc, union ccb *cmnd ) { AGTIAPI_PRINTK( "agtiapi_eh_HostReset: ccb pointer %p\n", cmnd ); if( cmnd == NULL ) { printf( "agtiapi_eh_HostReset: null command, skipping reset.\n" ); return tiInvalidHandle; } return agtiapi_DoSoftReset( pmcsc ); } /****************************************************************************** agtiapi_QueueCCB() Purpose: Put ccb in ccb queue at the tail Parameters: struct agtiapi_softc *pmcsc (IN) Pointer to HBA data structure pccb_t *phead (IN) Double pointer to ccb queue head pccb_t *ptail (IN) Double pointer to ccb queue tail ccb_t *pccb (IN) Poiner to a ccb to be queued Return: Note: Put the ccb to the tail of queue ******************************************************************************/ STATIC void agtiapi_QueueCCB( struct agtiapi_softc *pmcsc, pccb_t *phead, pccb_t *ptail, #ifdef AGTIAPI_LOCAL_LOCK struct mtx *mutex, #endif ccb_t *pccb ) { AGTIAPI_IO( "agtiapi_QueueCCB: start\n" ); AGTIAPI_IO( "agtiapi_QueueCCB: %p to %p\n", pccb, phead ); if (phead == NULL || ptail == NULL) { panic( "agtiapi_QueueCCB: phead %p ptail %p", phead, ptail ); } pccb->pccbNext = NULL; AG_LOCAL_LOCK( mutex ); if (*phead == NULL) { //WARN_ON(*ptail != NULL); /* critical, just get more logs */ *phead = pccb; } else { //WARN_ON(*ptail == NULL); /* critical, just get more logs */ if (*ptail) (*ptail)->pccbNext = pccb; } *ptail = pccb; AG_LOCAL_UNLOCK( mutex ); return; } /****************************************************************************** agtiapi_QueueCCB() Purpose: Parameters: Return: Note: ******************************************************************************/ static int agtiapi_QueueSMP(struct agtiapi_softc *pmcsc, union ccb * ccb) { pccb_t pccb = agNULL; /* call dequeue */ int status = tiSuccess; int targetID = xpt_path_target_id(ccb->ccb_h.path); AGTIAPI_PRINTK("agtiapi_QueueSMP: start\n"); /* get a ccb */ if ((pccb = agtiapi_GetCCB(pmcsc)) == NULL) { AGTIAPI_PRINTK("agtiapi_QueueSMP: GetCCB ERROR\n"); ccb->ccb_h.status = CAM_REQ_CMP; xpt_done(ccb); return tiBusy; } pccb->pmcsc = pmcsc; /* initialize Command Control Block (CCB) */ pccb->targetId = targetID; pccb->ccb = ccb; /* for struct scsi_cmnd */ status = agtiapi_PrepareSMPSGList(pmcsc, pccb); if (status != tiSuccess) { AGTIAPI_PRINTK("agtiapi_QueueSMP: agtiapi_PrepareSMPSGList failure\n"); agtiapi_FreeCCB(pmcsc, pccb); if (status == tiReject) { ccb->ccb_h.status = CAM_REQ_INVALID; } else { ccb->ccb_h.status = CAM_REQ_CMP; } xpt_done(ccb); return tiError; } return status; } /****************************************************************************** agtiapi_SetLunField() Purpose: Set LUN field based on different address mode Parameters: ccb_t *pccb (IN) A pointer to the driver's own CCB, not CAM's CCB Return: Note: ******************************************************************************/ void agtiapi_SetLunField(ccb_t *pccb) { U08 *pchar; pchar = (U08 *)&pccb->tiSuperScsiRequest.scsiCmnd.lun; // AGTIAPI_PRINTK("agtiapi_SetLunField: start\n"); switch (pccb->addrMode) { case AGTIAPI_PERIPHERAL: *pchar++ = 0; *pchar = (U08)pccb->lun; break; case AGTIAPI_VOLUME_SET: *pchar++ = (AGTIAPI_VOLUME_SET << AGTIAPI_ADDRMODE_SHIFT) | (U08)((pccb->lun >> 8) & 0x3F); *pchar = (U08)pccb->lun; break; case AGTIAPI_LUN_ADDR: *pchar++ = (AGTIAPI_LUN_ADDR << AGTIAPI_ADDRMODE_SHIFT) | pccb->targetId; *pchar = (U08)pccb->lun; break; } } /***************************************************************************** agtiapi_FreeCCB() Purpose: Free a ccb and put it back to ccbFreeList. Parameters: struct agtiapi_softc *pmcsc (IN) Pointer to HBA data structure pccb_t pccb (IN) A pointer to the driver's own CCB, not CAM's CCB Returns: Note: *****************************************************************************/ STATIC void agtiapi_FreeCCB(struct agtiapi_softc *pmcsc, pccb_t pccb) { union ccb *ccb = pccb->ccb; bus_dmasync_op_t op; AG_LOCAL_LOCK(&pmcsc->ccbLock); AGTIAPI_IO( "agtiapi_FreeCCB: start %p\n", pccb ); #ifdef AGTIAPI_TEST_EPL tiEncrypt_t *encrypt; #endif agtiapi_DumpCDB( "agtiapi_FreeCCB", pccb ); if (pccb->sgList != agNULL) { AGTIAPI_IO( "agtiapi_FreeCCB: pccb->sgList is NOT null\n" ); } else { AGTIAPI_PRINTK( "agtiapi_FreeCCB: pccb->sgList is null\n" ); } /* set data transfer direction */ if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_OUT) { op = BUS_DMASYNC_POSTWRITE; } else { op = BUS_DMASYNC_POSTREAD; } if (pccb->numSgElements == 0) { // do nothing AGTIAPI_IO( "agtiapi_FreeCCB: numSgElements zero\n" ); } else if (pccb->numSgElements == 1) { AGTIAPI_IO( "agtiapi_FreeCCB: numSgElements is one\n" ); //op is either BUS_DMASYNC_POSTWRITE or BUS_DMASYNC_POSTREAD bus_dmamap_sync(pmcsc->buffer_dmat, pccb->CCB_dmamap, op); bus_dmamap_unload(pmcsc->buffer_dmat, pccb->CCB_dmamap); } else { AGTIAPI_PRINTK( "agtiapi_FreeCCB: numSgElements 2 or higher \n" ); //op is either BUS_DMASYNC_POSTWRITE or BUS_DMASYNC_POSTREAD bus_dmamap_sync(pmcsc->buffer_dmat, pccb->CCB_dmamap, op); bus_dmamap_unload(pmcsc->buffer_dmat, pccb->CCB_dmamap); } #ifdef AGTIAPI_TEST_DPL if (pccb->tiSuperScsiRequest.Dif.enableDIFPerLA == TRUE) { if(pccb->dplPtr) memset( (char *) pccb->dplPtr, 0, MAX_DPL_REGIONS * sizeof(dplaRegion_t) ); pccb->tiSuperScsiRequest.Dif.enableDIFPerLA = FALSE; pccb->tiSuperScsiRequest.Dif.DIFPerLAAddrLo = 0; pccb->tiSuperScsiRequest.Dif.DIFPerLAAddrHi = 0; } #endif #ifdef AGTIAPI_TEST_EPL encrypt = &pccb->tiSuperScsiRequest.Encrypt; if (encrypt->enableEncryptionPerLA == TRUE) { encrypt->enableEncryptionPerLA = FALSE; encrypt->EncryptionPerLAAddrLo = 0; encrypt->EncryptionPerLAAddrHi = 0; } #endif #ifdef ENABLE_SATA_DIF if (pccb->holePtr && pccb->dmaHandleHole) pci_free_consistent( pmcsc->pCardInfo->pPCIDev, 512, pccb->holePtr, pccb->dmaHandleHole ); pccb->holePtr = 0; pccb->dmaHandleHole = 0; #endif pccb->dataLen = 0; pccb->retryCount = 0; pccb->ccbStatus = 0; pccb->scsiStatus = 0; pccb->startTime = 0; pccb->dmaHandle = 0; pccb->numSgElements = 0; pccb->tiIORequest.tdData = 0; memset((void *)&pccb->tiSuperScsiRequest, 0, AGSCSI_INIT_XCHG_LEN); #ifdef HIALEAH_ENCRYPTION if (pmcsc->encrypt) agtiapi_CleanupEncryptedIO(pmcsc, pccb); #endif pccb->flags = 0; pccb->ccb = NULL; pccb->pccbIO = NULL; pccb->pccbNext = (pccb_t)pmcsc->ccbFreeList; pmcsc->ccbFreeList = (caddr_t *)pccb; pmcsc->activeCCB--; AG_LOCAL_UNLOCK(&pmcsc->ccbLock); return; } /****************************************************************************** agtiapi_FlushCCBs() Purpose: Flush all in processed ccbs. Parameters: ag_card_t *pCard (IN) Pointer to HBA data structure U32 flag (IN) Flag to call back Return: Note: ******************************************************************************/ STATIC void agtiapi_FlushCCBs( struct agtiapi_softc *pCard, U32 flag ) { union ccb *ccb; ccb_t *pccb; AGTIAPI_PRINTK( "agtiapi_FlushCCBs: enter \n" ); for( pccb = (pccb_t)pCard->ccbChainList; pccb != NULL; pccb = pccb->pccbChainNext ) { if( pccb->flags == 0 ) { // printf( "agtiapi_FlushCCBs: nothing, continue \n" ); continue; } ccb = pccb->ccb; if ( pccb->flags & ( TASK_MANAGEMENT | DEV_RESET ) ) { AGTIAPI_PRINTK( "agtiapi_FlushCCBs: agtiapi_FreeTMCCB \n" ); agtiapi_FreeTMCCB( pCard, pccb ); } else { if ( pccb->flags & TAG_SMP ) { AGTIAPI_PRINTK( "agtiapi_FlushCCBs: agtiapi_FreeSMPCCB \n" ); agtiapi_FreeSMPCCB( pCard, pccb ); } else { AGTIAPI_PRINTK( "agtiapi_FlushCCBs: agtiapi_FreeCCB \n" ); agtiapi_FreeCCB( pCard, pccb ); } if( ccb ) { CMND_DMA_UNMAP( pCard, ccb ); if( flag == AGTIAPI_CALLBACK ) { ccb->ccb_h.status = CAM_SCSI_BUS_RESET; xpt_done( ccb ); } } } } } /***************************************************************************** agtiapi_FreeSMPCCB() Purpose: Free a ccb and put it back to ccbFreeList. Parameters: struct agtiapi_softc *pmcsc (IN) Pointer to HBA data structure pccb_t pccb (IN) A pointer to the driver's own CCB, not CAM's CCB Returns: Note: *****************************************************************************/ STATIC void agtiapi_FreeSMPCCB(struct agtiapi_softc *pmcsc, pccb_t pccb) { union ccb *ccb = pccb->ccb; bus_dmasync_op_t op; AG_LOCAL_LOCK(&pmcsc->ccbLock); AGTIAPI_PRINTK("agtiapi_FreeSMPCCB: start %p\n", pccb); /* set data transfer direction */ if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_OUT) { op = BUS_DMASYNC_POSTWRITE; } else { op = BUS_DMASYNC_POSTREAD; } if (pccb->numSgElements == 0) { // do nothing AGTIAPI_PRINTK("agtiapi_FreeSMPCCB: numSgElements 0\n"); } else if (pccb->numSgElements == 1) { AGTIAPI_PRINTK("agtiapi_FreeSMPCCB: numSgElements 1\n"); //op is either BUS_DMASYNC_POSTWRITE or BUS_DMASYNC_POSTREAD bus_dmamap_sync(pmcsc->buffer_dmat, pccb->CCB_dmamap, op); bus_dmamap_unload(pmcsc->buffer_dmat, pccb->CCB_dmamap); } else { AGTIAPI_PRINTK("agtiapi_FreeSMPCCB: numSgElements 2 or higher \n"); //op is either BUS_DMASYNC_POSTWRITE or BUS_DMASYNC_POSTREAD bus_dmamap_sync(pmcsc->buffer_dmat, pccb->CCB_dmamap, op); bus_dmamap_unload(pmcsc->buffer_dmat, pccb->CCB_dmamap); } /*dma api cleanning*/ pccb->dataLen = 0; pccb->retryCount = 0; pccb->ccbStatus = 0; pccb->startTime = 0; pccb->dmaHandle = 0; pccb->numSgElements = 0; pccb->tiIORequest.tdData = 0; memset((void *)&pccb->tiSMPFrame, 0, AGSMP_INIT_XCHG_LEN); pccb->flags = 0; pccb->ccb = NULL; pccb->pccbNext = (pccb_t)pmcsc->ccbFreeList; pmcsc->ccbFreeList = (caddr_t *)pccb; pmcsc->activeCCB--; AG_LOCAL_UNLOCK(&pmcsc->ccbLock); return; } /***************************************************************************** agtiapi_FreeTMCCB() Purpose: Free a ccb and put it back to ccbFreeList. Parameters: struct agtiapi_softc *pmcsc (IN) Pointer to HBA data structure pccb_t pccb (IN) A pointer to the driver's own CCB, not CAM's CCB Returns: Note: *****************************************************************************/ STATIC void agtiapi_FreeTMCCB(struct agtiapi_softc *pmcsc, pccb_t pccb) { AG_LOCAL_LOCK(&pmcsc->ccbLock); AGTIAPI_PRINTK("agtiapi_FreeTMCCB: start %p\n", pccb); pccb->dataLen = 0; pccb->retryCount = 0; pccb->ccbStatus = 0; pccb->scsiStatus = 0; pccb->startTime = 0; pccb->dmaHandle = 0; pccb->numSgElements = 0; pccb->tiIORequest.tdData = 0; memset((void *)&pccb->tiSuperScsiRequest, 0, AGSCSI_INIT_XCHG_LEN); pccb->flags = 0; pccb->ccb = NULL; pccb->pccbIO = NULL; pccb->pccbNext = (pccb_t)pmcsc->ccbFreeList; pmcsc->ccbFreeList = (caddr_t *)pccb; pmcsc->activeCCB--; AG_LOCAL_UNLOCK(&pmcsc->ccbLock); return; } /****************************************************************************** agtiapi_CheckAllVectors(): Purpose: Parameters: Return: Note: Currently, not used. ******************************************************************************/ void agtiapi_CheckAllVectors( struct agtiapi_softc *pCard, bit32 context ) { #ifdef SPC_MSIX_INTR if (!agtiapi_intx_mode) { int i; for (i = 0; i < pCard->pCardInfo->maxInterruptVectors; i++) if (tiCOMInterruptHandler(&pCard->tiRoot, i) == agTRUE) tiCOMDelayedInterruptHandler(&pCard->tiRoot, i, 100, context); } else if (tiCOMInterruptHandler(&pCard->tiRoot, 0) == agTRUE) tiCOMDelayedInterruptHandler(&pCard->tiRoot, 0, 100, context); #else if (tiCOMInterruptHandler(&pCard->tiRoot, 0) == agTRUE) tiCOMDelayedInterruptHandler(&pCard->tiRoot, 0, 100, context); #endif } /****************************************************************************** agtiapi_CheckCB() Purpose: Check call back function returned event for process completion Parameters: struct agtiapi_softc *pCard Pointer to card data structure U32 milisec (IN) Waiting time for expected event U32 flag (IN) Flag of the event to check U32 *pStatus (IN) Pointer to status of the card or port to check Return: AGTIAPI_SUCCESS - event comes as expected AGTIAPI_FAIL - event not coming Note: Currently, not used ******************************************************************************/ agBOOLEAN agtiapi_CheckCB( struct agtiapi_softc *pCard, U32 milisec, U32 flag, volatile U32 *pStatus ) { U32 msecsPerTick = pCard->pCardInfo->tiRscInfo.tiInitiatorResource. initiatorOption.usecsPerTick / 1000; S32 i = milisec/msecsPerTick; AG_GLOBAL_ARG( _flags ); AGTIAPI_PRINTK( "agtiapi_CheckCB: start\n" ); AGTIAPI_FLOW( "agtiapi_CheckCB: start\n" ); if( i <= 0 ) i = 1; while (i > 0) { if (*pStatus & TASK_MANAGEMENT) { if (*pStatus & AGTIAPI_CB_DONE) { if( flag == 0 || *pStatus & flag ) return AGTIAPI_SUCCESS; else return AGTIAPI_FAIL; } } else if (pCard->flags & AGTIAPI_CB_DONE) { if( flag == 0 || *pStatus & flag ) return AGTIAPI_SUCCESS; else return AGTIAPI_FAIL; } agtiapi_DelayMSec( msecsPerTick ); AG_SPIN_LOCK_IRQ( agtiapi_host_lock, _flags ); tiCOMTimerTick( &pCard->tiRoot ); agtiapi_CheckAllVectors( pCard, tiNonInterruptContext ); AG_SPIN_UNLOCK_IRQ( agtiapi_host_lock, _flags ); i--; } if( *pStatus & TASK_MANAGEMENT ) *pStatus |= TASK_TIMEOUT; return AGTIAPI_FAIL; } /****************************************************************************** agtiapi_DiscoverTgt() Purpose: Discover available devices Parameters: struct agtiapi_softc *pCard (IN) Pointer to the HBA data structure Return: Note: ******************************************************************************/ STATIC void agtiapi_DiscoverTgt(struct agtiapi_softc *pCard) { ag_portal_data_t *pPortalData; U32 count; AGTIAPI_PRINTK("agtiapi_DiscoverTgt: start\n"); AGTIAPI_FLOW("agtiapi_DiscoverTgt\n"); AGTIAPI_INIT("agtiapi_DiscoverTgt\n"); pPortalData = pCard->pPortalData; for (count = 0; count < pCard->portCount; count++, pPortalData++) { pCard->flags &= ~AGTIAPI_CB_DONE; if (!(PORTAL_STATUS(pPortalData) & AGTIAPI_PORT_DISC_READY)) { if (pCard->flags & AGTIAPI_INIT_TIME) { if (agtiapi_CheckCB(pCard, 5000, AGTIAPI_PORT_DISC_READY, &PORTAL_STATUS(pPortalData)) == AGTIAPI_FAIL) { AGTIAPI_PRINTK( "agtiapi_DiscoverTgt: Port %p / %d not ready for " "discovery\n", pPortalData, count ); /* * There is no need to spend time on discovering device * if port is not ready to do so. */ continue; } } else continue; } AGTIAPI_FLOW( "agtiapi_DiscoverTgt: Portal %p DiscoverTargets starts\n", pPortalData ); AGTIAPI_INIT_DELAY(1000); pCard->flags &= ~AGTIAPI_CB_DONE; if (tiINIDiscoverTargets(&pCard->tiRoot, &pPortalData->portalInfo.tiPortalContext, FORCE_PERSISTENT_ASSIGN_MASK) != tiSuccess) AGTIAPI_PRINTK("agtiapi_DiscoverTgt: tiINIDiscoverTargets ERROR\n"); /* * Should wait till discovery completion to start * next portal. However, lower layer have issue on * multi-portal case under Linux. */ } pPortalData = pCard->pPortalData; for (count = 0; count < pCard->portCount; count++, pPortalData++) { if ((PORTAL_STATUS(pPortalData) & AGTIAPI_PORT_DISC_READY)) { if (agtiapi_CheckCB(pCard, 20000, AGTIAPI_DISC_COMPLETE, &PORTAL_STATUS(pPortalData)) == AGTIAPI_FAIL) { if ((PORTAL_STATUS(pPortalData) & AGTIAPI_DISC_COMPLETE)) AGTIAPI_PRINTK( "agtiapi_DiscoverTgt: Portal %p discover complete, " "status 0x%x\n", pPortalData, PORTAL_STATUS(pPortalData) ); else AGTIAPI_PRINTK( "agtiapi_DiscoverTgt: Portal %p discover is not " "completed, status 0x%x\n", pPortalData, PORTAL_STATUS(pPortalData) ); continue; } AGTIAPI_PRINTK( "agtiapi_DiscoverTgt: Portal %d discover target " "success\n", count ); } } /* * Calling to get device handle should be done per portal based * and better right after discovery is done. However, lower iscsi * layer may not returns discovery complete in correct sequence or we * ran out time. We get device handle for all portals together * after discovery is done or timed out. */ pPortalData = pCard->pPortalData; for (count = 0; count < pCard->portCount; count++, pPortalData++) { /* * We try to get device handle no matter * if discovery is completed or not. */ if (PORTAL_STATUS(pPortalData) & AGTIAPI_PORT_DISC_READY) { U32 i; for (i = 0; i < AGTIAPI_GET_DEV_MAX; i++) { if (agtiapi_GetDevHandle(pCard, &pPortalData->portalInfo, 0, 0) != 0) break; agtiapi_DelayMSec(AGTIAPI_EXTRA_DELAY); } if ((PORTAL_STATUS(pPortalData) & AGTIAPI_DISC_COMPLETE) || (pCard->tgtCount > 0)) PORTAL_STATUS(pPortalData) |= ( AGTIAPI_DISC_DONE | AGTIAPI_PORT_LINK_UP ); } } return; } /****************************************************************************** agtiapi_PrepCCBs() Purpose: Prepares CCB including DMA map. Parameters: struct agtiapi_softc *pCard (IN) Pointer to the HBA data structure ccb_hdr_t *hdr (IN) Pointer to the CCB header U32 size (IN) size U32 max_ccb (IN) count Return: Note: ******************************************************************************/ STATIC void agtiapi_PrepCCBs( struct agtiapi_softc *pCard, ccb_hdr_t *hdr, U32 size, U32 max_ccb, int tid ) { int i; U32 hdr_sz, ccb_sz; ccb_t *pccb = 0; int offset = 0; int nsegs = 0; int sgl_sz = 0; AGTIAPI_PRINTK("agtiapi_PrepCCBs: start\n"); offset = tid * AGTIAPI_CCB_PER_DEVICE; nsegs = AGTIAPI_NSEGS; sgl_sz = sizeof(tiSgl_t) * nsegs; AGTIAPI_PRINTK( "agtiapi_PrepCCBs: tid %d offset %d nsegs %d sizeof(tiSgl_t) " "%lu, max_ccb %d\n", tid, offset, nsegs, sizeof(tiSgl_t), max_ccb ); ccb_sz = roundup2(AGTIAPI_CCB_SIZE, cache_line_size()); hdr_sz = roundup2(sizeof(*hdr), cache_line_size()); AGTIAPI_PRINTK("agtiapi_PrepCCBs: after cache line\n"); memset((void *)hdr, 0, size); hdr->next = pCard->ccbAllocList; pCard->ccbAllocList = hdr; AGTIAPI_PRINTK("agtiapi_PrepCCBs: after memset\n"); pccb = (ccb_t*) ((char*)hdr + hdr_sz); for (i = 0; i < max_ccb; i++, pccb = (ccb_t*)((char*)pccb + ccb_sz)) { pccb->tiIORequest.osData = (void *)pccb; /* * Initially put all the ccbs on the free list * in addition to chainlist. * ccbChainList is a list of all available ccbs * (free/active everything) */ pccb->pccbChainNext = (pccb_t)pCard->ccbChainList; pccb->pccbNext = (pccb_t)pCard->ccbFreeList; pCard->ccbChainList = (caddr_t *)pccb; pCard->ccbFreeList = (caddr_t *)pccb; pCard->ccbTotal++; #ifdef AGTIAPI_ALIGN_CHECK if (&pccb & 0x63) AGTIAPI_PRINTK("pccb = %p\n", pccb); if (pccb->devHandle & 0x63) AGTIAPI_PRINTK("devHandle addr = %p\n", &pccb->devHandle); if (&pccb->lun & 0x63) AGTIAPI_PRINTK("lun addr = %p\n", &pccb->lun); if (&pccb->targetId & 0x63) AGTIAPI_PRINTK("tig addr = %p\n", &pccb->targetId); if (&pccb->ccbStatus & 0x63) AGTIAPI_PRINTK("ccbStatus addr = %p\n", &pccb->ccbStatus); if (&pccb->scsiStatus & 0x63) AGTIAPI_PRINTK("scsiStatus addr = %p\n", &pccb->scsiStatus); if (&pccb->dataLen & 0x63) AGTIAPI_PRINTK("dataLen addr = %p\n", &pccb->dataLen); if (&pccb->senseLen & 0x63) AGTIAPI_PRINTK("senseLen addr = %p\n", &pccb->senseLen); if (&pccb->numSgElements & 0x63) AGTIAPI_PRINTK("numSgElements addr = %p\n", &pccb->numSgElements); if (&pccb->retryCount & 0x63) AGTIAPI_PRINTK("retry cnt addr = %p\n", &pccb->retryCount); if (&pccb->flags & 0x63) AGTIAPI_PRINTK("flag addr = %p\n", &pccb->flags); if (&pccb->pSenseData & 0x63) AGTIAPI_PRINTK("senseData addr = %p\n", &pccb->pSenseData); if (&pccb->sgList[0] & 0x63) AGTIAPI_PRINTK("SgList 0 = %p\n", &pccb->sgList[0]); if (&pccb->pccbNext & 0x63) AGTIAPI_PRINTK("ccb next = %p\n", &pccb->pccbNext); if (&pccb->pccbChainNext & 0x63) AGTIAPI_PRINTK("ccbChainNext = %p\n", &pccb->pccbChainNext); if (&pccb->cmd & 0x63) AGTIAPI_PRINTK("command = %p\n", &pccb->cmd); if( &pccb->startTime & 0x63 ) AGTIAPI_PRINTK( "startTime = %p\n", &pccb->startTime ); if (&pccb->tiIORequest & 0x63) AGTIAPI_PRINTK("tiIOReq addr = %p\n", &pccb->tiIORequest); if (&pccb->tdIOReqBody & 0x63) AGTIAPI_PRINTK("tdIORequestBody addr = %p\n", &pccb->tdIOReqBody); if (&pccb->tiSuperScsiRequest & 0x63) AGTIAPI_PRINTK( "InitiatorExchange addr = %p\n", &pccb->tiSuperScsiRequest ); #endif if ( bus_dmamap_create( pCard->buffer_dmat, 0, &pccb->CCB_dmamap ) != tiSuccess) { AGTIAPI_PRINTK("agtiapi_PrepCCBs: can't create dma\n"); return; } /* assigns tiSgl_t memory to pccb */ pccb->sgList = (void*)((U64)pCard->tisgl_mem + ((i + offset) * sgl_sz)); pccb->tisgl_busaddr = pCard->tisgl_busaddr + ((i + offset) * sgl_sz); pccb->ccb = NULL; pccb->pccbIO = NULL; pccb->startTime = 0; } #ifdef AGTIAPI_ALIGN_CHECK AGTIAPI_PRINTK("ccb size = %d / %d\n", sizeof(ccb_t), ccb_sz); #endif return; } /****************************************************************************** agtiapi_InitCCBs() Purpose: Create and initialize per card based CCB pool. Parameters: struct agtiapi_softc *pCard (IN) Pointer to the HBA data structure int tgtCount (IN) Count Return: Total number of ccb allocated Note: ******************************************************************************/ STATIC U32 agtiapi_InitCCBs(struct agtiapi_softc *pCard, int tgtCount, int tid) { U32 max_ccb, size, ccb_sz, hdr_sz; int no_allocs = 0, i; ccb_hdr_t *hdr = 0; AGTIAPI_PRINTK("agtiapi_InitCCBs: start\n"); AGTIAPI_PRINTK("agtiapi_InitCCBs: tgtCount %d tid %d\n", tgtCount, tid); AGTIAPI_FLOW("agtiapi_InitCCBs: tgtCount %d tid %d\n", tgtCount, tid); #ifndef HOTPLUG_SUPPORT if (pCard->tgtCount > AGSA_MAX_INBOUND_Q) return 1; #else if (tgtCount > AGSA_MAX_INBOUND_Q) tgtCount = AGSA_MAX_INBOUND_Q; #endif max_ccb = tgtCount * AGTIAPI_CCB_PER_DEVICE;// / 4; // TBR ccb_sz = roundup2(AGTIAPI_CCB_SIZE, cache_line_size()); hdr_sz = roundup2(sizeof(*hdr), cache_line_size()); size = ccb_sz * max_ccb + hdr_sz; for (i = 0; i < (1 << no_allocs); i++) { hdr = (ccb_hdr_t*)malloc( size, M_PMC_MCCB, M_NOWAIT ); if( !hdr ) { panic( "agtiapi_InitCCBs: bug!!!\n" ); } else { agtiapi_PrepCCBs( pCard, hdr, size, max_ccb, tid ); } } return 1; } #ifdef LINUX_PERBI_SUPPORT /****************************************************************************** agtiapi_GetWWNMappings() Purpose: Get the mappings from target IDs to WWNs, if any. Store them in the WWN_list array, indexed by target ID. Leave the devListIndex field blank; this will be filled-in later. Parameters: ag_card_t *pCard (IN) Pointer to HBA data structure ag_mapping_t *pMapList (IN) Pointer to mapped device list Return: Note: The boot command line parameters are used to load the mapping information, which is contained in the system configuration file. ******************************************************************************/ STATIC void agtiapi_GetWWNMappings( struct agtiapi_softc *pCard, ag_mapping_t *pMapList ) { int devDisc; int lIdx = 0; ag_tgt_map_t *pWWNList; ag_slr_map_t *pSLRList; ag_device_t *pDevList; if( !pCard ) panic( "agtiapi_GetWWNMappings: no pCard \n" ); AGTIAPI_PRINTK( "agtiapi_GetWWNMappings: start\n" ); pWWNList = pCard->pWWNList; pSLRList = pCard->pSLRList; pDevList = pCard->pDevList; pCard->numTgtHardMapped = 0; devDisc = pCard->devDiscover; pWWNList[devDisc-1].devListIndex = maxTargets; pSLRList[devDisc-1].localeNameLen = -2; pSLRList[devDisc-1].remoteNameLen = -2; pDevList[devDisc-1].targetId = maxTargets; /* * Get the mappings from holding area which contains * the input of the system file and store them * in the WWN_list array, indexed by target ID. */ for ( lIdx = 0; lIdx < devDisc - 1; lIdx++) { pWWNList[lIdx].flags = 0; pWWNList[lIdx].devListIndex = maxTargets; pSLRList[lIdx].localeNameLen = -1; pSLRList[lIdx].remoteNameLen = -1; } // this is where we would propagate values fed to pMapList } /* agtiapi_GetWWNMappings */ #endif /****************************************************************************** agtiapi_FindWWNListNext() Purpose: finds first available new (unused) wwn list entry Parameters: ag_tgt_map_t *pWWNList Pointer to head of wwn list int lstMax Number of entries in WWNList Return: index into WWNList indicating available entry space; if available entry space is not found, return negative value ******************************************************************************/ STATIC int agtiapi_FindWWNListNext( ag_tgt_map_t *pWWNList, int lstMax ) { int lLstIdx; for ( lLstIdx = 0; lLstIdx < lstMax; lLstIdx++ ) { if ( pWWNList[lLstIdx].devListIndex == lstMax && pWWNList[lLstIdx].targetLen == 0 ) { AGTIAPI_PRINTK( "agtiapi_FindWWNListNext: %d %d %d %d v. %d\n", lLstIdx, pWWNList[lLstIdx].devListIndex, pWWNList[lLstIdx].targetLen, pWWNList[lLstIdx].portId, lstMax ); return lLstIdx; } } return -1; } /****************************************************************************** agtiapi_GetDevHandle() Purpose: Get device handle. Handles will be placed in the devlist array with same order as TargetList provided and will be mapped to a scsi target id and registered to OS later. Parameters: struct agtiapi_softc *pCard (IN) Pointer to the HBA data structure ag_portal_info_t *pPortalInfo (IN) Pointer to the portal data structure U32 eType (IN) Port event U32 eStatus (IN) Port event status Return: Number of device handle slot present Note: The sequence of device handle will match the sequence of taregt list ******************************************************************************/ STATIC U32 agtiapi_GetDevHandle( struct agtiapi_softc *pCard, ag_portal_info_t *pPortalInfo, U32 eType, U32 eStatus ) { ag_device_t *pDevice; // tiDeviceHandle_t *agDev[pCard->devDiscover]; tiDeviceHandle_t **agDev; int devIdx, szdv, devTotal, cmpsetRtn; int lDevIndex = 0, lRunScanFlag = FALSE; int *lDevFlags; tiPortInfo_t portInfT; ag_device_t lTmpDevice; ag_tgt_map_t *pWWNList; ag_slr_map_t *pSLRList; bit32 lReadRm; bit16 lReadCt; AGTIAPI_PRINTK( "agtiapi_GetDevHandle: start\n" ); AGTIAPI_PRINTK( "agtiapi_GetDevHandle: pCard->devDiscover %d / tgtCt %d\n", pCard->devDiscover, pCard->tgtCount ); AGTIAPI_FLOW( "agtiapi_GetDevHandle: portalInfo %p\n", pPortalInfo ); AGTIAPI_INIT_DELAY( 1000 ); agDev = (tiDeviceHandle_t **) malloc( sizeof(tiDeviceHandle_t *) * pCard->devDiscover, M_PMC_MDEV, M_ZERO | M_NOWAIT); if (agDev == NULL) { AGTIAPI_PRINTK( "agtiapi_GetDevHandle: failed to alloc agDev[]\n" ); return 0; } lDevFlags = (int *) malloc( sizeof(int) * pCard->devDiscover, M_PMC_MFLG, M_ZERO | M_NOWAIT ); if (lDevFlags == NULL) { free((caddr_t)agDev, M_PMC_MDEV); AGTIAPI_PRINTK( "agtiapi_GetDevHandle: failed to alloc lDevFlags[]\n" ); return 0; } pWWNList = pCard->pWWNList; pSLRList = pCard->pSLRList; memset( (void *)agDev, 0, sizeof(void *) * pCard->devDiscover ); memset( lDevFlags, 0, sizeof(int) * pCard->devDiscover ); // get device handles devTotal = tiINIGetDeviceHandles( &pCard->tiRoot, &pPortalInfo->tiPortalContext, (tiDeviceHandle_t **)agDev, pCard->devDiscover ); AGTIAPI_PRINTK( "agtiapi_GetDevHandle: portalInfo %p port id %d event %u " "status %u card %p pCard->devDiscover %d devTotal %d " "pPortalInfo->devTotal %d pPortalInfo->devPrev %d " "AGTIAPI_INIT_TIME %x\n", pPortalInfo, pPortalInfo->portID, eType, eStatus, pCard, pCard->devDiscover, devTotal, pPortalInfo->devTotal, pPortalInfo->devPrev, pCard->flags & AGTIAPI_INIT_TIME ); // reset devTotal from any previous runs of this pPortalInfo->devPrev = devTotal; pPortalInfo->devTotal = devTotal; AG_LIST_LOCK( &pCard->devListLock ); if ( tiCOMGetPortInfo( &pCard->tiRoot, &pPortalInfo->tiPortalContext, &portInfT ) != tiSuccess) { AGTIAPI_PRINTK( "agtiapi_GetDevHandle: tiCOMGetPortInfo did not succeed. \n" ); } szdv = sizeof( pPortalInfo->pDevList ) / sizeof( pPortalInfo->pDevList[0] ); if (szdv > pCard->devDiscover) { szdv = pCard->devDiscover; } // reconstructing dev list via comparison of wwn for ( devIdx = 0; devIdx < pCard->devDiscover; devIdx++ ) { if ( agDev[devIdx] != 0 ) { // AGTIAPI_PRINTK( "agtiapi_GetDevHandle: agDev %d not NULL %p\n", // devIdx, agDev[devIdx] ); // pack temp device structure for tiINIGetDeviceInfo call pDevice = &lTmpDevice; pDevice->devType = DIRECT_DEVICE; pDevice->pCard = (void *)pCard; pDevice->flags = ACTIVE; pDevice->pPortalInfo = pPortalInfo; pDevice->pDevHandle = agDev[devIdx]; pDevice->qbusy = agFALSE; //AGTIAPI_PRINTK( "agtiapi_GetDevHandle: idx %d / %d : %p \n", // devIdx, pCard->devDiscover, agDev[devIdx] ); tiINIGetDeviceInfo( &pCard->tiRoot, agDev[devIdx], &pDevice->devInfo ); //AGTIAPI_PRINTK( "agtiapi_GetDevHandle: wwn sizes %ld %d/%d ", // sizeof(pDevice->targetName), // pDevice->devInfo.osAddress1, // pDevice->devInfo.osAddress2 ); wwncpy( pDevice ); wwnprintk( (unsigned char*)pDevice->targetName, pDevice->targetLen ); for ( lDevIndex = 0; lDevIndex < szdv; lDevIndex++ ) // match w/ wwn list { if ( (pCard->pDevList[lDevIndex].portalId == pPortalInfo->portID) && pDevice->targetLen > 0 && portInfT.localNameLen > 0 && portInfT.remoteNameLen > 0 && pSLRList[pWWNList[lDevIndex].sasLrIdx].localeNameLen > 0 && pSLRList[pWWNList[lDevIndex].sasLrIdx].remoteNameLen > 0 && ( portInfT.localNameLen == pSLRList[pWWNList[lDevIndex].sasLrIdx].localeNameLen ) && ( portInfT.remoteNameLen == pSLRList[pWWNList[lDevIndex].sasLrIdx].remoteNameLen ) && memcmp( pWWNList[lDevIndex].targetName, pDevice->targetName, pDevice->targetLen ) == 0 && memcmp( pSLRList[pWWNList[lDevIndex].sasLrIdx].localeName, portInfT.localName, portInfT.localNameLen ) == 0 && memcmp( pSLRList[pWWNList[lDevIndex].sasLrIdx].remoteName, portInfT.remoteName, portInfT.remoteNameLen ) == 0 ) { AGTIAPI_PRINTK( " pWWNList match @ %d/%d/%d \n", lDevIndex, devIdx, pPortalInfo->portID ); if ( (pCard->pDevList[lDevIndex].targetId == lDevIndex) && ( pPortalInfo->pDevList[lDevIndex] == &pCard->pDevList[lDevIndex] ) ) // active { AGTIAPI_PRINTK( "agtiapi_GetDevHandle: dev in use %d of %d/%d\n", lDevIndex, devTotal, pPortalInfo->portID ); lDevFlags[devIdx] |= DPMC_LEANFLAG_AGDEVUSED; // agDev handle lDevFlags[lDevIndex] |= DPMC_LEANFLAG_PDEVSUSED; // pDevice used lReadRm = atomic_readandclear_32( &pWWNList[lDevIndex].devRemoved ); if ( lReadRm ) // cleared timeout, now remove count for timer { AGTIAPI_PRINTK( "agtiapi_GetDevHandle: clear timer count for" " %d of %d\n", lDevIndex, pPortalInfo->portID ); atomic_subtract_16( &pCard->rmChkCt, 1 ); lReadCt = atomic_load_acq_16( &pCard->rmChkCt ); if ( 0 == lReadCt ) { callout_stop( &pCard->devRmTimer ); } } break; } AGTIAPI_PRINTK( "agtiapi_GetDevHandle: goin fresh on %d of %d/%d\n", lDevIndex, // reactivate now devTotal, pPortalInfo->portID ); // pDevice going fresh lRunScanFlag = TRUE; // scan and clear outstanding removals // pCard->tgtCount++; ## pDevice->targetId = lDevIndex; pDevice->portalId = pPortalInfo->portID; memcpy ( &pCard->pDevList[lDevIndex], pDevice, sizeof(lTmpDevice) ); agDev[devIdx]->osData = (void *)&pCard->pDevList[lDevIndex]; if ( agtiapi_InitCCBs( pCard, 1, pDevice->targetId ) == 0 ) { AGTIAPI_PRINTK( "agtiapi_GetDevHandle: InitCCB " "tgtCnt %d ERROR!\n", pCard->tgtCount ); AG_LIST_UNLOCK( &pCard->devListLock ); free((caddr_t)lDevFlags, M_PMC_MFLG); free((caddr_t)agDev, M_PMC_MDEV); return 0; } pPortalInfo->pDevList[lDevIndex] = &pCard->pDevList[lDevIndex]; // (ag_device_t *) if ( 0 == lDevFlags[devIdx] ) { pPortalInfo->devTotal++; lDevFlags[devIdx] |= DPMC_LEANFLAG_AGDEVUSED; // agDev used lDevFlags[lDevIndex] |= DPMC_LEANFLAG_PDEVSUSED; // pDevice used } else { AGTIAPI_PRINTK( "agtiapi_GetDevHandle: odd dev handle " "status inspect %d %d %d\n", lDevFlags[devIdx], devIdx, lDevIndex ); pPortalInfo->devTotal++; lDevFlags[devIdx] |= DPMC_LEANFLAG_AGDEVUSED; // agDev used lDevFlags[lDevIndex] |= DPMC_LEANFLAG_PDEVSUSED; // pDevice used } break; } } // end: match this wwn with previous wwn list // we have an agDev entry, but no pWWNList target for it if ( !(lDevFlags[devIdx] & DPMC_LEANFLAG_AGDEVUSED) ) { // flag dev handle not accounted for yet lDevFlags[devIdx] |= DPMC_LEANFLAG_NOWWNLIST; // later, get an empty pDevice and map this agDev. // AGTIAPI_PRINTK( "agtiapi_GetDevHandle: devIdx %d flags 0x%x, %d\n", // devIdx, lDevFlags[devIdx], (lDevFlags[devIdx] & 8) ); } } else { lDevFlags[devIdx] |= DPMC_LEANFLAG_NOAGDEVYT; // known empty agDev handle } } // AGTIAPI_PRINTK( "agtiapi_GetDevHandle: all WWN all the time, " // "devLstIdx/flags/(WWNL)portId ... \n" ); // review device list for further action needed for ( devIdx = 0; devIdx < pCard->devDiscover; devIdx++ ) { if ( lDevFlags[devIdx] & DPMC_LEANFLAG_NOWWNLIST ) // new target, register { int lNextDyad; // find next available dyad entry AGTIAPI_PRINTK( "agtiapi_GetDevHandle: register new target, " "devIdx %d -- %d \n", devIdx, pCard->devDiscover ); lRunScanFlag = TRUE; // scan and clear outstanding removals for ( lNextDyad = 0; lNextDyad < pCard->devDiscover; lNextDyad++ ) { if ( pSLRList[lNextDyad].localeNameLen < 0 && pSLRList[lNextDyad].remoteNameLen < 0 ) break; } if ( lNextDyad == pCard->devDiscover ) { printf( "agtiapi_GetDevHandle: failed to find available SAS LR\n" ); AG_LIST_UNLOCK( &pCard->devListLock ); free( (caddr_t)lDevFlags, M_PMC_MFLG ); free( (caddr_t)agDev, M_PMC_MDEV ); return 0; } // index of new entry lDevIndex = agtiapi_FindWWNListNext( pWWNList, pCard->devDiscover ); AGTIAPI_PRINTK( "agtiapi_GetDevHandle: listIdx new target %d of %d/%d\n", lDevIndex, devTotal, pPortalInfo->portID ); if ( 0 > lDevIndex ) { printf( "agtiapi_GetDevHandle: WARNING -- WWNList exhausted.\n" ); continue; } pDevice = &pCard->pDevList[lDevIndex]; tiINIGetDeviceInfo( &pCard->tiRoot, agDev[devIdx], &pDevice->devInfo ); wwncpy( pDevice ); agtiapi_InitCCBs( pCard, 1, lDevIndex ); pDevice->pCard = (void *)pCard; pDevice->devType = DIRECT_DEVICE; // begin to populate new WWNList entry memcpy( pWWNList[lDevIndex].targetName, pDevice->targetName, pDevice->targetLen ); pWWNList[lDevIndex].targetLen = pDevice->targetLen; pWWNList[lDevIndex].flags = SOFT_MAPPED; pWWNList[lDevIndex].portId = pPortalInfo->portID; pWWNList[lDevIndex].devListIndex = lDevIndex; pWWNList[lDevIndex].sasLrIdx = lNextDyad; pSLRList[lNextDyad].localeNameLen = portInfT.localNameLen; pSLRList[lNextDyad].remoteNameLen = portInfT.remoteNameLen; memcpy( pSLRList[lNextDyad].localeName, portInfT.localName, portInfT.localNameLen ); memcpy( pSLRList[lNextDyad].remoteName, portInfT.remoteName, portInfT.remoteNameLen ); // end of populating new WWNList entry pDevice->targetId = lDevIndex; pDevice->flags = ACTIVE; pDevice->CCBCount = 0; pDevice->pDevHandle = agDev[devIdx]; agDev[devIdx]->osData = (void*)pDevice; pDevice->pPortalInfo = pPortalInfo; pDevice->portalId = pPortalInfo->portID; pPortalInfo->pDevList[lDevIndex] = (void*)pDevice; lDevFlags[lDevIndex] |= DPMC_LEANFLAG_PDEVSUSED; // mark pDevice slot used } if ( (pCard->pDevList[devIdx].portalId == pPortalInfo->portID) && !(lDevFlags[devIdx] & DPMC_LEANFLAG_PDEVSUSED) ) // pDevice not used { pDevice = &pCard->pDevList[devIdx]; //pDevice->flags &= ~ACTIVE; if ( ( pDevice->pDevHandle != NULL || pPortalInfo->pDevList[devIdx] != NULL ) ) { atomic_add_16( &pCard->rmChkCt, 1 ); // show count of lost device if (FALSE == lRunScanFlag) { AGTIAPI_PRINTK( "agtiapi_GetDevHandle: targ dropped out %d of %d/%d\n", devIdx, devTotal, pPortalInfo->portID ); // if ( 0 == pWWNList[devIdx].devRemoved ) '.devRemoved = 5; cmpsetRtn = atomic_cmpset_32( &pWWNList[devIdx].devRemoved, 0, 5 ); if ( 0 == cmpsetRtn ) { AGTIAPI_PRINTK( "agtiapi_GetDevHandle: target %d timer already set\n", devIdx ); } else { callout_reset( &pCard->devRmTimer, 1 * hz, agtiapi_devRmCheck, pCard ); } } // else ... scan coming soon enough anyway, ignore timer for dropout } } } // end of for ( devIdx = 0; ... AG_LIST_UNLOCK( &pCard->devListLock ); free((caddr_t)lDevFlags, M_PMC_MFLG); free((caddr_t)agDev, M_PMC_MDEV); if ( TRUE == lRunScanFlag ) agtiapi_clrRmScan( pCard ); return devTotal; } // end agtiapi_GetDevHandle /****************************************************************************** agtiapi_scan() Purpose: Triggers CAM's scan Parameters: struct agtiapi_softc *pCard (IN) Pointer to the HBA data structure Return: Note: ******************************************************************************/ static void agtiapi_scan(struct agtiapi_softc *pmcsc) { union ccb *ccb; int bus, tid, lun, card_no; static int num=0; AGTIAPI_PRINTK("agtiapi_scan: start cardNO %d \n", pmcsc->cardNo); bus = cam_sim_path(pmcsc->sim); tid = CAM_TARGET_WILDCARD; lun = CAM_LUN_WILDCARD; mtx_lock(&(pmcsc->pCardInfo->pmIOLock)); ccb = xpt_alloc_ccb_nowait(); if (ccb == agNULL) { mtx_unlock(&(pmcsc->pCardInfo->pmIOLock)); return; } if (xpt_create_path(&ccb->ccb_h.path, agNULL, bus, tid, CAM_LUN_WILDCARD) != CAM_REQ_CMP) { mtx_unlock(&(pmcsc->pCardInfo->pmIOLock)); xpt_free_ccb(ccb); return; } mtx_unlock(&(pmcsc->pCardInfo->pmIOLock)); pmcsc->dev_scan = agTRUE; xpt_rescan(ccb); return; } /****************************************************************************** agtiapi_DeQueueCCB() Purpose: Remove a ccb from a queue Parameters: struct agtiapi_softc *pCard (IN) Pointer to the card structure pccb_t *phead (IN) Pointer to a head of ccb queue ccb_t *pccd (IN) Pointer to the ccb to be processed Return: AGTIAPI_SUCCESS - the ccb is removed from queue AGTIAPI_FAIL - the ccb is not found from queue Note: ******************************************************************************/ STATIC agBOOLEAN agtiapi_DeQueueCCB(struct agtiapi_softc *pCard, pccb_t *phead, pccb_t *ptail, #ifdef AGTIAPI_LOCAL_LOCK struct mtx *lock, #endif ccb_t *pccb) { ccb_t *pccb_curr; U32 status = AGTIAPI_FAIL; AGTIAPI_PRINTK("agtiapi_DeQueueCCB: %p from %p\n", pccb, phead); if (pccb == NULL || *phead == NULL) { return AGTIAPI_FAIL; } AGTIAPI_PRINTK("agtiapi_DeQueueCCB: %p from %p\n", pccb, phead); AG_LOCAL_LOCK(lock); if (pccb == *phead) { *phead = (*phead)->pccbNext; if (pccb == *ptail) { *ptail = NULL; } else pccb->pccbNext = NULL; status = AGTIAPI_SUCCESS; } else { pccb_curr = *phead; while (pccb_curr->pccbNext != NULL) { if (pccb_curr->pccbNext == pccb) { pccb_curr->pccbNext = pccb->pccbNext; pccb->pccbNext = NULL; if (pccb == *ptail) { *ptail = pccb_curr; } else pccb->pccbNext = NULL; status = AGTIAPI_SUCCESS; break; } pccb_curr = pccb_curr->pccbNext; } } AG_LOCAL_UNLOCK(lock); return status; } STATIC void wwnprintk( unsigned char *name, int len ) { int i; for (i = 0; i < len; i++, name++) AGTIAPI_PRINTK("%02x", *name); AGTIAPI_PRINTK("\n"); } /* * SAS and SATA behind expander has 8 byte long unique address. * However, direct connect SATA device use 512 byte unique device id. * SPC uses remoteName to indicate length of ID and remoteAddress for the * address of memory that holding ID. */ STATIC int wwncpy( ag_device_t *pDevice ) { int rc = 0; if (sizeof(pDevice->targetName) >= pDevice->devInfo.osAddress1 + pDevice->devInfo.osAddress2) { memcpy(pDevice->targetName, pDevice->devInfo.remoteName, pDevice->devInfo.osAddress1); memcpy(pDevice->targetName + pDevice->devInfo.osAddress1, pDevice->devInfo.remoteAddress, pDevice->devInfo.osAddress2); pDevice->targetLen = pDevice->devInfo.osAddress1 + pDevice->devInfo.osAddress2; rc = pDevice->targetLen; } else { AGTIAPI_PRINTK("WWN wrong size: %d + %d ERROR\n", pDevice->devInfo.osAddress1, pDevice->devInfo.osAddress2); rc = -1; } return rc; } /****************************************************************************** agtiapi_ReleaseCCBs() Purpose: Free all allocated CCB memories for the Host Adapter. Parameters: - struct agtiapi_softc *pCard (IN) Pointer to HBA data stucture + struct agtiapi_softc *pCard (IN) Pointer to HBA data structure Return: Note: ******************************************************************************/ STATIC void agtiapi_ReleaseCCBs( struct agtiapi_softc *pCard ) { ccb_hdr_t *hdr; U32 hdr_sz; ccb_t *pccb = 0; AGTIAPI_PRINTK( "agtiapi_ReleaseCCBs: start\n" ); #if ( defined AGTIAPI_TEST_DPL || defined AGTIAPI_TEST_EPL ) ccb_t *pccb; #endif #ifdef AGTIAPI_TEST_DPL for (pccb = (pccb_t)pCard->ccbChainList; pccb != NULL; pccb = pccb->pccbChainNext) { if(pccb->dplPtr && pccb->dplDma) pci_pool_free(pCard->dpl_ctx_pool, pccb->dplPtr, pccb->dplDma); } #endif #ifdef AGTIAPI_TEST_EPL for (pccb = (pccb_t)pCard->ccbChainList; pccb != NULL; pccb = pccb->pccbChainNext) { if(pccb->epl_ptr && pccb->epl_dma_ptr) pci_pool_free( pCard->epl_ctx_pool, pccb->epl_ptr, pccb->epl_dma_ptr ); } #endif while ((hdr = pCard->ccbAllocList) != NULL) { pCard->ccbAllocList = hdr->next; hdr_sz = roundup2(sizeof(*hdr), cache_line_size()); pccb = (ccb_t*) ((char*)hdr + hdr_sz); if (pCard->buffer_dmat != NULL && pccb->CCB_dmamap != NULL) { bus_dmamap_destroy(pCard->buffer_dmat, pccb->CCB_dmamap); } free(hdr, M_PMC_MCCB); } pCard->ccbAllocList = NULL; return; } /****************************************************************************** agtiapi_TITimer() Purpose: Timer tick for tisa common layer Parameters: void *data (IN) Pointer to the HBA data structure Return: Note: ******************************************************************************/ STATIC void agtiapi_TITimer( void *data ) { U32 next_tick; struct agtiapi_softc *pCard; pCard = (struct agtiapi_softc *)data; // AGTIAPI_PRINTK("agtiapi_TITimer: start\n"); AG_GLOBAL_ARG( flags ); next_tick = pCard->pCardInfo->tiRscInfo.tiLoLevelResource. loLevelOption.usecsPerTick / USEC_PER_TICK; if( next_tick == 0 ) /* no timer required */ return; AG_SPIN_LOCK_IRQ( agtiapi_host_lock, flags ); if( pCard->flags & AGTIAPI_SHUT_DOWN ) goto ext; tiCOMTimerTick( &pCard->tiRoot ); /* tisa common layer timer tick */ //add for polling mode #ifdef PMC_SPC if( agtiapi_polling_mode ) agtiapi_CheckAllVectors( pCard, tiNonInterruptContext ); #endif callout_reset( &pCard->OS_timer, next_tick, agtiapi_TITimer, pCard ); ext: AG_SPIN_UNLOCK_IRQ( agtiapi_host_lock, flags ); return; } /****************************************************************************** agtiapi_clrRmScan() Purpose: Clears device list entries scheduled for timeout and calls scan Parameters: struct agtiapi_softc *pCard (IN) Pointer to HBA data structure ******************************************************************************/ STATIC void agtiapi_clrRmScan( struct agtiapi_softc *pCard ) { ag_tgt_map_t *pWWNList; ag_portal_info_t *pPortalInfo; ag_portal_data_t *pPortalData; int lIdx; bit32 lReadRm; bit16 lReadCt; pWWNList = pCard->pWWNList; AGTIAPI_PRINTK( "agtiapi_clrRmScan: start\n" ); AG_LIST_LOCK( &pCard->devListLock ); for ( lIdx = 0; lIdx < pCard->devDiscover; lIdx++ ) { lReadCt = atomic_load_acq_16( &pCard->rmChkCt ); if ( 0 == lReadCt ) { break; // trim to who cares } lReadRm = atomic_readandclear_32( &pWWNList[lIdx].devRemoved ); if ( lReadRm > 0 ) { pCard->pDevList[lIdx].flags &= ~ACTIVE; pCard->pDevList[lIdx].pDevHandle = NULL; pPortalData = &pCard->pPortalData[pWWNList[lIdx].portId]; pPortalInfo = &pPortalData->portalInfo; pPortalInfo->pDevList[lIdx] = NULL; AGTIAPI_PRINTK( "agtiapi_clrRmScan: cleared dev %d at port %d\n", lIdx, pWWNList[lIdx].portId ); atomic_subtract_16( &pCard->rmChkCt, 1 ); } } AG_LIST_UNLOCK( &pCard->devListLock ); agtiapi_scan( pCard ); } /****************************************************************************** agtiapi_devRmCheck() Purpose: Timer tick to check for timeout on missing targets Removes device list entry when timeout is reached Parameters: void *data (IN) Pointer to the HBA data structure ******************************************************************************/ STATIC void agtiapi_devRmCheck( void *data ) { struct agtiapi_softc *pCard; ag_tgt_map_t *pWWNList; int lIdx, cmpsetRtn, lRunScanFlag = FALSE; bit16 lReadCt; bit32 lReadRm; pCard = ( struct agtiapi_softc * )data; // routine overhead if ( callout_pending( &pCard->devRmTimer ) ) // callout was reset { return; } if ( !callout_active( &pCard->devRmTimer ) ) // callout was stopped { return; } callout_deactivate( &pCard->devRmTimer ); if( pCard->flags & AGTIAPI_SHUT_DOWN ) { return; // implicit timer clear } pWWNList = pCard->pWWNList; AG_LIST_LOCK( &pCard->devListLock ); lReadCt = atomic_load_acq_16( &pCard->rmChkCt ); if ( lReadCt ) { if ( callout_pending(&pCard->devRmTimer) == FALSE ) { callout_reset( &pCard->devRmTimer, 1 * hz, agtiapi_devRmCheck, pCard ); } else { AG_LIST_UNLOCK( &pCard->devListLock ); return; } for ( lIdx = 0; lIdx < pCard->devDiscover; lIdx++ ) { lReadCt = atomic_load_acq_16( &pCard->rmChkCt ); if ( 0 == lReadCt ) { break; // if handled somewhere else, get out } lReadRm = atomic_load_acq_32( &pWWNList[lIdx].devRemoved ); if ( lReadRm > 0 ) { if ( 1 == lReadRm ) // timed out { // no decrement of devRemoved as way to leave a clrRmScan marker lRunScanFlag = TRUE; // other devRemoved values are about to get wiped break; // ... so bail out } else { AGTIAPI_PRINTK( "agtiapi_devRmCheck: counting down dev %d @ %d; %d\n", lIdx, lReadRm, lReadCt ); cmpsetRtn = atomic_cmpset_32( &pWWNList[lIdx].devRemoved, lReadRm, lReadRm-1 ); if ( 0 == cmpsetRtn ) { printf( "agtiapi_devRmCheck: %d decrement already handled\n", lIdx ); } } } } AG_LIST_UNLOCK( &pCard->devListLock ); if ( TRUE == lRunScanFlag ) agtiapi_clrRmScan( pCard ); } else { AG_LIST_UNLOCK( &pCard->devListLock ); } return; } static void agtiapi_cam_poll( struct cam_sim *asim ) { return; } /***************************************************************************** agtiapi_ResetCard() Purpose: Hard or soft reset on the controller and resend any outstanding requests if needed. Parameters: struct agtiapi_softc *pCard (IN) Pointer to HBA data structure unsigned lomg flags (IN/OUT) Flags used in locking done from calling layers Return: AGTIAPI_SUCCESS - reset successful AGTIAPI_FAIL - reset failed Note: *****************************************************************************/ U32 agtiapi_ResetCard( struct agtiapi_softc *pCard, unsigned long *flags ) { ag_device_t *pDevice; U32 lIdx = 0; U32 lFlagVal; agBOOLEAN ret; ag_portal_info_t *pPortalInfo; ag_portal_data_t *pPortalData; U32 count, loop; int szdv; if( pCard->flags & AGTIAPI_RESET ) { AGTIAPI_PRINTK( "agtiapi_ResetCard: reset card already in progress!\n" ); return AGTIAPI_FAIL; } AGTIAPI_PRINTK( "agtiapi_ResetCard: Enter cnt %d\n", pCard->resetCount ); #ifdef LOGEVENT agtiapi_LogEvent( pCard, IOCTL_EVT_SEV_INFORMATIONAL, 0, agNULL, 0, "Reset initiator time = %d!", pCard->resetCount + 1 ); #endif pCard->flags |= AGTIAPI_RESET; pCard->flags &= ~(AGTIAPI_CB_DONE | AGTIAPI_RESET_SUCCESS); tiCOMSystemInterruptsActive( &pCard->tiRoot, FALSE ); pCard->flags &= ~AGTIAPI_SYS_INTR_ON; agtiapi_FlushCCBs( pCard, AGTIAPI_CALLBACK ); for ( lIdx = 1; 3 >= lIdx; lIdx++ ) // we try reset up to 3 times { if( pCard->flags & AGTIAPI_SOFT_RESET ) { AGTIAPI_PRINTK( "agtiapi_ResetCard: soft variant\n" ); tiCOMReset( &pCard->tiRoot, tiSoftReset ); } else { AGTIAPI_PRINTK( "agtiapi_ResetCard: no flag, no reset!\n" ); } lFlagVal = AGTIAPI_RESET_SUCCESS; AG_SPIN_UNLOCK_IRQ( agtiapi_host_lock, *flags ); ret = agtiapi_CheckCB( pCard, 50000, lFlagVal, &pCard->flags ); AG_SPIN_LOCK_IRQ( agtiapi_host_lock, *flags ); if( ret == AGTIAPI_FAIL ) { AGTIAPI_PRINTK( "agtiapi_ResetCard: CheckCB indicates failed reset call, " "try again?\n" ); } else { break; } } if ( 1 < lIdx ) { if ( AGTIAPI_FAIL == ret ) { AGTIAPI_PRINTK( "agtiapi_ResetCard: soft reset failed after try %d\n", lIdx ); } else { AGTIAPI_PRINTK( "agtiapi_ResetCard: soft reset success at try %d\n", lIdx ); } } if( AGTIAPI_FAIL == ret ) { printf( "agtiapi_ResetCard: reset ERROR\n" ); pCard->flags &= ~AGTIAPI_INSTALLED; return AGTIAPI_FAIL; } pCard->flags &= ~AGTIAPI_SOFT_RESET; // disable all devices pDevice = pCard->pDevList; for( lIdx = 0; lIdx < maxTargets; lIdx++, pDevice++ ) { /* if ( pDevice->flags & ACTIVE ) { printf( "agtiapi_ResetCard: before ... active device %d\n", lIdx ); } */ pDevice->flags &= ~ACTIVE; } AG_SPIN_UNLOCK_IRQ( agtiapi_host_lock, *flags ); if( tiCOMPortInit( &pCard->tiRoot, agFALSE ) != tiSuccess ) printf( "agtiapi_ResetCard: tiCOMPortInit FAILED \n" ); else AGTIAPI_PRINTK( "agtiapi_ResetCard: tiCOMPortInit success\n" ); if( !pCard->pDevList ) { // try to get a little sanity here AGTIAPI_PRINTK( "agtiapi_ResetCard: no pDevList ERROR %p\n", pCard->pDevList ); return AGTIAPI_FAIL; } AGTIAPI_PRINTK( "agtiapi_ResetCard: pre target-count %d port-count %d\n", pCard->tgtCount, pCard->portCount ); pCard->tgtCount = 0; DELAY( 500000 ); pCard->flags &= ~AGTIAPI_CB_DONE; pPortalData = pCard->pPortalData; for( count = 0; count < pCard->portCount; count++ ) { AG_SPIN_LOCK_IRQ( agtiapi_host_lock, flags ); pPortalInfo = &pPortalData->portalInfo; pPortalInfo->portStatus = 0; pPortalInfo->portStatus &= ~( AGTIAPI_PORT_START | AGTIAPI_PORT_DISC_READY | AGTIAPI_DISC_DONE | AGTIAPI_DISC_COMPLETE ); szdv = sizeof( pPortalInfo->pDevList ) / sizeof( pPortalInfo->pDevList[0] ); if (szdv > pCard->devDiscover) { szdv = pCard->devDiscover; } for( lIdx = 0, loop = 0; lIdx < szdv && loop < pPortalInfo->devTotal; lIdx++ ) { pDevice = (ag_device_t*)pPortalInfo->pDevList[lIdx]; if( pDevice ) { loop++; pDevice->pDevHandle = 0; // mark for availability in pCard->pDevList[] // don't erase more as the device is scheduled for removal on DPC } AGTIAPI_PRINTK( "agtiapi_ResetCard: reset pDev %p pDevList %p idx %d\n", pDevice, pPortalInfo->pDevList, lIdx ); pPortalInfo->devTotal = pPortalInfo->devPrev = 0; } for( lIdx = 0; lIdx < maxTargets; lIdx++ ) { // we reconstruct dev list later in get dev handle pPortalInfo->pDevList[lIdx] = NULL; } for( loop = 0; loop < AGTIAPI_LOOP_MAX; loop++ ) { AGTIAPI_PRINTK( "agtiapi_ResetCard: tiCOMPortStart entry data " "%p / %d / %p\n", &pCard->tiRoot, pPortalInfo->portID, &pPortalInfo->tiPortalContext ); if( tiCOMPortStart( &pCard->tiRoot, pPortalInfo->portID, &pPortalInfo->tiPortalContext, 0 ) != tiSuccess ) { printf( "agtiapi_ResetCard: tiCOMPortStart %d FAILED\n", pPortalInfo->portID ); } else { AGTIAPI_PRINTK( "agtiapi_ResetCard: tiCOMPortStart %d success\n", pPortalInfo->portID ); break; } } AG_SPIN_UNLOCK_IRQ( agtiapi_host_lock, flags ); tiCOMGetPortInfo( &pCard->tiRoot, &pPortalInfo->tiPortalContext, &pPortalInfo->tiPortInfo ); pPortalData++; } // ## fail case: pCard->flags &= ~AGTIAPI_INSTALLED; AG_SPIN_LOCK_IRQ(agtiapi_host_lock, *flags); if( !(pCard->flags & AGTIAPI_INSTALLED) ) // driver not installed ! { printf( "agtiapi_ResetCard: error, driver not intstalled? " "!AGTIAPI_INSTALLED \n" ); return AGTIAPI_FAIL; } AGTIAPI_PRINTK( "agtiapi_ResetCard: total device %d\n", pCard->tgtCount ); #ifdef LOGEVENT agtiapi_LogEvent( pCard, IOCTL_EVT_SEV_INFORMATIONAL, 0, agNULL, 0, "Reset initiator total device = %d!", pCard->tgtCount ); #endif pCard->resetCount++; AGTIAPI_PRINTK( "agtiapi_ResetCard: clear send and done queues\n" ); // clear send & done queue AG_LOCAL_LOCK( &pCard->sendLock ); pCard->ccbSendHead = NULL; pCard->ccbSendTail = NULL; AG_LOCAL_UNLOCK( &pCard->sendLock ); AG_LOCAL_LOCK( &pCard->doneLock ); pCard->ccbDoneHead = NULL; pCard->ccbDoneTail = NULL; AG_LOCAL_UNLOCK( &pCard->doneLock ); // clear smp queues also AG_LOCAL_LOCK( &pCard->sendSMPLock ); pCard->smpSendHead = NULL; pCard->smpSendTail = NULL; AG_LOCAL_UNLOCK( &pCard->sendSMPLock ); AG_LOCAL_LOCK( &pCard->doneSMPLock ); pCard->smpDoneHead = NULL; pCard->smpDoneTail = NULL; AG_LOCAL_UNLOCK( &pCard->doneSMPLock ); // finished with all reset stuff, now start things back up tiCOMSystemInterruptsActive( &pCard->tiRoot, TRUE ); pCard->flags |= AGTIAPI_SYS_INTR_ON; pCard->flags |= AGTIAPI_HAD_RESET; pCard->flags &= ~AGTIAPI_RESET; // ## agtiapi_StartIO( pCard ); AGTIAPI_PRINTK( "agtiapi_ResetCard: local return success\n" ); return AGTIAPI_SUCCESS; } // agtiapi_ResetCard /****************************************************************************** agtiapi_ReleaseHBA() Purpose: Releases all resources previously acquired to support a specific Host Adapter, including the I/O Address range, and unregisters the agtiapi Host Adapter. Parameters: device_t dev (IN) - device pointer Return: always return 0 - success Note: ******************************************************************************/ int agtiapi_ReleaseHBA( device_t dev ) { int thisCard = device_get_unit( dev ); // keeping get_unit call to once int i; ag_card_info_t *thisCardInst = &agCardInfoList[ thisCard ]; struct ccb_setasync csa; struct agtiapi_softc *pCard; pCard = device_get_softc( dev ); ag_card_info_t *pCardInfo = pCard->pCardInfo; ag_resource_info_t *pRscInfo = &thisCardInst->tiRscInfo; AG_GLOBAL_ARG(flags); AGTIAPI_PRINTK( "agtiapi_ReleaseHBA: start\n" ); if (thisCardInst != pCardInfo) { AGTIAPI_PRINTK( "agtiapi_ReleaseHBA: Wrong ag_card_info_t thisCardInst %p " "pCardInfo %p\n", thisCardInst, pCardInfo ); panic( "agtiapi_ReleaseHBA: Wrong ag_card_info_t thisCardInst %p pCardInfo " "%p\n", thisCardInst, pCardInfo ); return( EIO ); } AGTIAPI_PRINTK( "agtiapi_ReleaseHBA card %p\n", pCard ); pCard->flags |= AGTIAPI_SHUT_DOWN; // remove timer if (pCard->flags & AGTIAPI_TIMER_ON) { AG_SPIN_LOCK_IRQ( agtiapi_host_lock, flags ); callout_drain( &pCard->OS_timer ); callout_drain( &pCard->devRmTimer ); callout_drain(&pCard->IO_timer); AG_SPIN_UNLOCK_IRQ( agtiapi_host_lock, flags ); AGTIAPI_PRINTK( "agtiapi_ReleaseHBA: timer released\n" ); } #ifdef HIALEAH_ENCRYPTION //Release encryption table memory - Fix it //if(pCard->encrypt && (pCard->flags & AGTIAPI_INSTALLED)) //agtiapi_CleanupEncryption(pCard); #endif /* * Shutdown the channel so that chip gets frozen * and it does not do any more pci-bus accesses. */ if (pCard->flags & AGTIAPI_SYS_INTR_ON) { tiCOMSystemInterruptsActive( &pCard->tiRoot, FALSE ); pCard->flags &= ~AGTIAPI_SYS_INTR_ON; AGTIAPI_PRINTK( "agtiapi_ReleaseHBA: card interrupt off\n" ); } if (pCard->flags & AGTIAPI_INSTALLED) { tiCOMShutDown( &pCard->tiRoot ); AGTIAPI_PRINTK( "agtiapi_ReleaseHBA: low layers shutdown\n" ); } /* * first release IRQ, so that we do not get any more interrupts * from this host */ if (pCard->flags & AGTIAPI_IRQ_REQUESTED) { if (!agtiapi_intx_mode) { int i; for (i = 0; i< MAX_MSIX_NUM_VECTOR; i++) { if (pCard->irq[i] != agNULL && pCard->rscID[i] != 0) { bus_teardown_intr(dev, pCard->irq[i], pCard->intrcookie[i]); bus_release_resource( dev, SYS_RES_IRQ, pCard->rscID[i], pCard->irq[i] ); } } pci_release_msi(dev); } pCard->flags &= ~AGTIAPI_IRQ_REQUESTED; #ifdef AGTIAPI_DPC for (i = 0; i < MAX_MSIX_NUM_DPC; i++) tasklet_kill(&pCard->tasklet_dpc[i]); #endif AGTIAPI_PRINTK("agtiapi_ReleaseHBA: IRQ released\n"); } // release memory vs. alloc in agtiapi_alloc_ostimem; used in ostiAllocMemory if( pCard->osti_busaddr != 0 ) { bus_dmamap_unload( pCard->osti_dmat, pCard->osti_mapp ); } if( pCard->osti_mem != NULL ) { bus_dmamem_free( pCard->osti_dmat, pCard->osti_mem, pCard->osti_mapp ); } if( pCard->osti_dmat != NULL ) { bus_dma_tag_destroy( pCard->osti_dmat ); } /* unmap the mapped PCI memory */ /* calls bus_release_resource( ,SYS_RES_MEMORY, ..) */ agtiapi_ReleasePCIMem(thisCardInst); /* release all ccbs */ if (pCard->ccbTotal) { //calls bus_dmamap_destroy() for all pccbs agtiapi_ReleaseCCBs(pCard); AGTIAPI_PRINTK("agtiapi_ReleaseHBA: CCB released\n"); } #ifdef HIALEAH_ENCRYPTION /*release encryption resources - Fix it*/ if(pCard->encrypt) { /*Check that all IO's are completed */ if(atomic_read (&outstanding_encrypted_io_count) > 0) { printf("%s: WARNING: %d outstanding encrypted IOs !\n", __FUNCTION__, atomic_read(&outstanding_encrypted_io_count)); } //agtiapi_CleanupEncryptionPools(pCard); } #endif /* release device list */ if( pCard->pDevList ) { free((caddr_t)pCard->pDevList, M_PMC_MDVT); pCard->pDevList = NULL; AGTIAPI_PRINTK("agtiapi_ReleaseHBA: device list released\n"); } #ifdef LINUX_PERBI_SUPPORT // ## review use of PERBI AGTIAPI_PRINTK( "agtiapi_ReleaseHBA: WWN list %p \n", pCard->pWWNList ); if( pCard->pWWNList ) { free( (caddr_t)pCard->pWWNList, M_PMC_MTGT ); pCard->pWWNList = NULL; AGTIAPI_PRINTK("agtiapi_ReleaseHBA: WWN list released\n"); } if( pCard->pSLRList ) { free( (caddr_t)pCard->pSLRList, M_PMC_MSLR ); pCard->pSLRList = NULL; AGTIAPI_PRINTK("agtiapi_ReleaseHBA: SAS Local Remote list released\n"); } #endif if (pCard->pPortalData) { free((caddr_t)pCard->pPortalData, M_PMC_MPRT); pCard->pPortalData = NULL; AGTIAPI_PRINTK("agtiapi_ReleaseHBA: PortalData released\n"); } //calls contigfree() or free() agtiapi_MemFree(pCardInfo); AGTIAPI_PRINTK("agtiapi_ReleaseHBA: low level resource released\n"); #ifdef HOTPLUG_SUPPORT if (pCard->flags & AGTIAPI_PORT_INITIALIZED) { // agtiapi_FreeDevWorkList(pCard); AGTIAPI_PRINTK("agtiapi_ReleaseHBA: (HP dev) work resources released\n"); } #endif /* * TBD, scsi_unregister may release wrong host data structure * which cause NULL pointer shows up. */ if (pCard->flags & AGTIAPI_SCSI_REGISTERED) { pCard->flags &= ~AGTIAPI_SCSI_REGISTERED; #ifdef AGTIAPI_LOCAL_LOCK if (pCard->STLock) { //destroy mtx int maxLocks; maxLocks = pRscInfo->tiLoLevelResource.loLevelOption.numOfQueuesPerPort; for( i = 0; i < maxLocks; i++ ) { mtx_destroy(&pCard->STLock[i]); } free(pCard->STLock, M_PMC_MSTL); pCard->STLock = NULL; } #endif } ag_card_good--; /* reset agtiapi_1st_time if this is the only card */ if (!ag_card_good && !agtiapi_1st_time) { agtiapi_1st_time = 1; } /* for tiSgl_t memeory */ if (pCard->tisgl_busaddr != 0) { bus_dmamap_unload(pCard->tisgl_dmat, pCard->tisgl_map); } if (pCard->tisgl_mem != NULL) { bus_dmamem_free(pCard->tisgl_dmat, pCard->tisgl_mem, pCard->tisgl_map); } if (pCard->tisgl_dmat != NULL) { bus_dma_tag_destroy(pCard->tisgl_dmat); } if (pCard->buffer_dmat != agNULL) { bus_dma_tag_destroy(pCard->buffer_dmat); } if (pCard->sim != NULL) { mtx_lock(&thisCardInst->pmIOLock); xpt_setup_ccb(&csa.ccb_h, pCard->path, 5); csa.ccb_h.func_code = XPT_SASYNC_CB; csa.event_enable = 0; csa.callback = agtiapi_async; csa.callback_arg = pCard; xpt_action((union ccb *)&csa); xpt_free_path(pCard->path); // if (pCard->ccbTotal == 0) if (pCard->ccbTotal <= thisCard) { /* no link up so that simq has not been released. In order to remove cam, we call this. */ xpt_release_simq(pCard->sim, 1); } xpt_bus_deregister(cam_sim_path(pCard->sim)); cam_sim_free(pCard->sim, FALSE); mtx_unlock(&thisCardInst->pmIOLock); } if (pCard->devq != NULL) { cam_simq_free(pCard->devq); } //destroy mtx mtx_destroy( &thisCardInst->pmIOLock ); mtx_destroy( &pCard->sendLock ); mtx_destroy( &pCard->doneLock ); mtx_destroy( &pCard->sendSMPLock ); mtx_destroy( &pCard->doneSMPLock ); mtx_destroy( &pCard->ccbLock ); mtx_destroy( &pCard->devListLock ); mtx_destroy( &pCard->OS_timer_lock ); mtx_destroy( &pCard->devRmTimerLock ); mtx_destroy( &pCard->memLock ); mtx_destroy( &pCard->freezeLock ); destroy_dev( pCard->my_cdev ); memset((void *)pCardInfo, 0, sizeof(ag_card_info_t)); return 0; } // Called during system shutdown after sync static int agtiapi_shutdown( device_t dev ) { AGTIAPI_PRINTK( "agtiapi_shutdown\n" ); return( 0 ); } static int agtiapi_suspend( device_t dev ) // Device suspend routine. { AGTIAPI_PRINTK( "agtiapi_suspend\n" ); return( 0 ); } static int agtiapi_resume( device_t dev ) // Device resume routine. { AGTIAPI_PRINTK( "agtiapi_resume\n" ); return( 0 ); } static device_method_t agtiapi_methods[] = { // Device interface DEVMETHOD( device_probe, agtiapi_probe ), DEVMETHOD( device_attach, agtiapi_attach ), DEVMETHOD( device_detach, agtiapi_ReleaseHBA ), DEVMETHOD( device_shutdown, agtiapi_shutdown ), DEVMETHOD( device_suspend, agtiapi_suspend ), DEVMETHOD( device_resume, agtiapi_resume ), { 0, 0 } }; static devclass_t pmspcv_devclass; static driver_t pmspcv_driver = { "pmspcv", agtiapi_methods, sizeof( struct agtiapi_softc ) }; DRIVER_MODULE( pmspcv, pci, pmspcv_driver, pmspcv_devclass, 0, 0 ); MODULE_DEPEND( pmspcv, cam, 1, 1, 1 ); MODULE_DEPEND( pmspcv, pci, 1, 1, 1 ); #include #include #include #include Index: user/ngie/release-pkg-fix-tests/sys/dev/pms/freebsd/driver/ini/src/osapi.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/dev/pms/freebsd/driver/ini/src/osapi.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/dev/pms/freebsd/driver/ini/src/osapi.c (revision 299091) @@ -1,835 +1,835 @@ /******************************************************************************* *Copyright (c) 2014 PMC-Sierra, Inc. All rights reserved. * *Redistribution and use in source and binary forms, with or without modification, are permitted provided *that the following conditions are met: *1. Redistributions of source code must retain the above copyright notice, this list of conditions and the *following disclaimer. *2. Redistributions in binary form must reproduce the above copyright notice, *this list of conditions and the following disclaimer in the documentation and/or other materials provided *with the distribution. * *THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED *WARRANTIES,INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS *FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE *FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT *NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR *BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT *LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS *SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE * * $FreeBSD$ * *******************************************************************************/ /****************************************************************************** PMC-Sierra TISA Initiator Device Driver for Linux 2.x.x. Module Name: osapi.c Abstract: Linux iSCSI/FC Initiator driver module itsdk required OS functions Environment: Part of oslayer module, Kernel or loadable module ******************************************************************************* ostiInitiatorEvent() Purpose: TI layer call back to OSlayer to inform events Parameters: tiRoot_t *ptiRoot (IN) Pointer to HBA data structure tiDeviceHandle_t *ptiDevHandle (IN) Pointer to device handle tiIntrEvenType_t evenType (IN) Event type tiIntrEventStatus_t evetStatus (IN) Event status void *parm (IN) pointer to even specific data Return: Note: TBD, further event process required. ******************************************************************************/ void ostiInitiatorEvent( tiRoot_t *ptiRoot, tiPortalContext_t *ptiPortalContext, tiDeviceHandle_t *ptiDevHandle, tiIntrEventType_t eventType, U32 eventStatus, void *parm ) { ag_portal_data_t *pPortalData; ag_portal_info_t *pPortalInfo; struct agtiapi_softc *pCard = TIROOT_TO_CARD( ptiRoot ); ccb_t *pccb; ccb_t *pTMccb; ccb_t *ccbIO; #ifdef AGTIAPI_EVENT_LOG AGTIAPI_PRINTK("Initiator Event:\n"); AGTIAPI_PRINTK("DevHandle %p, eventType 0x%x, eventStatus 0x%x\n", ptiDevHandle, eventType, eventStatus); AGTIAPI_PRINTK("Parameter: %s\n", (char *)parm); #endif AGTIAPI_PRINTK("ostiInitiatorEvent: eventType 0x%x eventStatus 0x%x\n", eventType, eventStatus); switch (eventType) { case tiIntrEventTypeCnxError: if (eventStatus == tiCnxUp) { AGTIAPI_PRINTK("tiIntrEventTypeCnxError - tiCnxUp!\n"); } if (eventStatus == tiCnxDown) { AGTIAPI_PRINTK("tiIntrEventTypeCnxError - tiCnxDown!\n"); } break; case tiIntrEventTypeDiscovery: pPortalData = PORTAL_CONTEXT_TO_PORTALDATA(ptiPortalContext); pCard->flags |= AGTIAPI_CB_DONE; if (eventStatus == tiDiscOK) { AGTIAPI_PRINTK("eventStatus - tiDiscOK\n"); AGTIAPI_PRINTK("ostiInitiatorEvent: pcard %d eventStatus - tiDiscOK\n", pCard->cardNo ); PORTAL_STATUS(pPortalData) |= AGTIAPI_DISC_COMPLETE; #ifndef HOTPLUG_SUPPORT if (!(pCard->flags & AGTIAPI_INIT_TIME)) #else if (TRUE) #endif { agtiapi_GetDevHandle(pCard, &pPortalData->portalInfo, tiIntrEventTypeDiscovery, tiDiscOK); PORTAL_STATUS(pPortalData) |= (AGTIAPI_DISC_DONE | AGTIAPI_PORT_LINK_UP); } /* Trigger CheckIOTimeout */ callout_reset(&pCard->IO_timer, 20*hz, agtiapi_CheckIOTimeout, pCard); } else if (eventStatus == tiDiscFailed) { AGTIAPI_PRINTK("eventStatus - tiDiscFailed\n"); agtiapi_GetDevHandle(pCard, &pPortalData->portalInfo, tiIntrEventTypeDiscovery, tiDiscFailed); PORTAL_STATUS(pPortalData) &= ~AGTIAPI_DISC_DONE; } AGTIAPI_PRINTK("tiIntrEventTypeDiscovery - portal %p, status 0x%x\n", pPortalData, PORTAL_STATUS(pPortalData)); break; case tiIntrEventTypeDeviceChange: AGTIAPI_PRINTK("tiIntrEventTypeDeviceChange - portal %p es %d\n", ptiPortalContext->osData, eventStatus); pPortalData = PORTAL_CONTEXT_TO_PORTALDATA(ptiPortalContext); pPortalInfo = &pPortalData->portalInfo; #ifndef HOTPLUG_SUPPORT if (!(pCard->flags & AGTIAPI_INIT_TIME)) #else if (TRUE) #endif { agtiapi_GetDevHandle(pCard, pPortalInfo, tiIntrEventTypeDeviceChange, eventStatus); // agtiapi_StartIO(pCard); } break; case tiIntrEventTypeTransportRecovery: AGTIAPI_PRINTK("tiIntrEventTypeTransportRecovery!\n"); break; case tiIntrEventTypeTaskManagement: AGTIAPI_PRINTK("tiIntrEventTypeTaskManagement!\n"); pccb = (pccb_t)((tiIORequest_t *)parm)->osData; if (pccb->flags & TASK_TIMEOUT) { AGTIAPI_PRINTK("tiIntrEventTypeTaskManagement: TM timeout!\n"); agtiapi_FreeTMCCB(pCard, pccb); } else { pccb->flags |= AGTIAPI_CB_DONE; if (eventStatus == tiTMOK) { pccb->flags |= TASK_SUCCESS; AGTIAPI_PRINTK("tiIntrEventTypeTaskManagement: pTMccb %p flag %x \n", pccb, pccb->flags); /* Incase of TM_DEV_RESET, issue LocalAbort to abort pending IO */ if (pccb->flags & DEV_RESET) { AGTIAPI_PRINTK("tiIntrEventTypeTaskManagement: Target Reset\n"); ccbIO = pccb->pccbIO; AGTIAPI_PRINTK("tiIntrEventTypeTaskManagement: IO to be aborted locally %p flag %x \n", ccbIO, ccbIO->flags); if (ccbIO->startTime == 0) /* IO has been completed. No local abort */ { } else if (tiINIIOAbort(&pCard->tiRoot, &ccbIO->tiIORequest) != tiSuccess) { AGTIAPI_PRINTK("tiIntrEventTypeTaskManagement: Local Abort failed\n"); /* TODO: call Soft reset here */ } } else if (eventStatus == tiTMFailed) { ccbIO = pccb->pccbIO; if (ccbIO->startTime == 0) /* IO has been completed. */ { - AGTIAPI_PRINTK("tiIntrEventTypeTaskManagement: TM failed becasue IO has been completed! pTMccb %p flag %x \n", + AGTIAPI_PRINTK("tiIntrEventTypeTaskManagement: TM failed because IO has been completed! pTMccb %p flag %x \n", pccb, pccb->flags); } else { AGTIAPI_PRINTK("tiIntrEventTypeTaskManagement: TM failed! pTMccb %p flag %x \n", pccb, pccb->flags); /* TODO:*/ /* if TM_ABORT_TASK, call TM_TARGET_RESET */ /* if TM_TARGET_RESET, call Soft_Reset */ } } /* Free TM_DEV_RESET ccb */ agtiapi_FreeTMCCB(pCard, pccb); } } break; case tiIntrEventTypeLocalAbort: AGTIAPI_PRINTK("tiIntrEventTypeLocalAbort!\n"); pccb = (pccb_t)((tiIORequest_t *)parm)->osData; pccb->flags |= AGTIAPI_CB_DONE; if (eventStatus == tiAbortOK) { AGTIAPI_PRINTK("tiIntrEventTypeLocalAbort: taskTag pccb %p flag %x \n", pccb, pccb->flags); /* If this was LocalAbort for TM ABORT_TASK, issue TM_DEV_RESET */ if (pccb->flags & TASK_MANAGEMENT) { if ((pTMccb = agtiapi_GetCCB(pCard)) == NULL) { AGTIAPI_PRINTK("tiIntrEventTypeLocalAbort: TM resource unavailable!\n"); /* TODO: SoftReset here? */ } pTMccb->pmcsc = pCard; pTMccb->targetId = pccb->targetId; pTMccb->devHandle = pccb->devHandle; /* save pending io to issue local abort at Task mgmt CB */ pTMccb->pccbIO = pccb->pccbIO; pTMccb->flags &= ~(TASK_SUCCESS | ACTIVE); pTMccb->flags |= DEV_RESET; if (tiINITaskManagement(&pCard->tiRoot, pccb->devHandle, AG_TARGET_WARM_RESET, &pccb->tiSuperScsiRequest.scsiCmnd.lun, &pccb->tiIORequest, &pTMccb->tiIORequest) == tiSuccess) { AGTIAPI_PRINTK("tiIntrEventTypeLocalAbort: TM_TARGET_RESET request success ccb %p, pTMccb %p\n", pccb, pTMccb); pTMccb->startTime = ticks; } else { AGTIAPI_PRINTK("tiIntrEventTypeLocalAbort: TM_TARGET_RESET request failed ccb %p, pTMccb %p\n", pccb, pTMccb); agtiapi_FreeTMCCB(pCard, pTMccb); /* TODO: SoftReset here? */ } /* Free ABORT_TASK TM ccb */ agtiapi_FreeTMCCB(pCard, pccb); } } else if (eventStatus == tiAbortFailed) { /* TODO: */ /* If TM_ABORT_TASK fails, issue TM_DEV_RESET */ /* if TM_DEV_RESET fails, issue Soft_Reset */ AGTIAPI_PRINTK("tiIntrEventTypeLocalAbort: Abort Failed pccb %p\n", pccb); } break; default: AGTIAPI_PRINTK("tiIntrEventType default!\n"); break; } } /****************************************************************************** ostiInitiatorIOCompleted() Purpose: IO request completion call back Parameters: tiRoot_t *ptiRoot (IN) Pointer to the HBA tiRoot tiIORequest_t *ptiIORequest (IN) Pointer to the tiIORequest structure tiIOStatus_t IOStatus (IN) I/O complated status U32 statusDetail (IN) Additional information on status tiSenseData_t *pSensedata (IN) Sense data buffer pointer U32 context (IN) Interrupt dealing context Returns: Note: ******************************************************************************/ void ostiInitiatorIOCompleted(tiRoot_t *ptiRoot, tiIORequest_t *ptiIORequest, tiIOStatus_t IOStatus, U32 statusDetail, tiSenseData_t *pSenseData, U32 context ) { struct agtiapi_softc *pCard; ccb_t *pccb; pCard = TIROOT_TO_CARD(ptiRoot); pccb = (ccb_t *)ptiIORequest->osData; AGTIAPI_IO( "ostiInitiatorIOCompleted: start\n" ); if (IOStatus == tiIODifError) { return; } OSTI_OUT_ENTER(ptiRoot); pccb->ccbStatus = (U16)IOStatus; pccb->scsiStatus = statusDetail; if ((IOStatus == tiIOSuccess) && (statusDetail == SCSI_CHECK_CONDITION)) { if (pSenseData == (tiSenseData_t *)agNULL) { AGTIAPI_PRINTK( "ostiInitiatorIOCompleted: " "check condition without sense data!\n" ); } else { union ccb *ccb = pccb->ccb; struct ccb_scsiio *csio = &ccb->csio; int sense_len = 0; if (pccb->senseLen > pSenseData->senseLen) { csio->sense_resid = pccb->senseLen - pSenseData->senseLen; } else { csio->sense_resid = 0; } sense_len = MIN( pSenseData->senseLen, pccb->senseLen - csio->sense_resid ); bzero(&csio->sense_data, sizeof(csio->sense_data)); AGTIAPI_PRINTK("ostiInitiatorIOCompleted: check condition copying\n"); memcpy( (void *)pccb->pSenseData, pSenseData->senseData, sense_len ); agtiapi_hexdump( "ostiInitiatorIOCompleted check condition", (bit8 *)&csio->sense_data, sense_len ); } } if ((IOStatus == tiIOFailed) && (statusDetail == tiDetailAborted)) { AGTIAPI_PRINTK("ostiInitiatorIOCompleted - aborted ccb %p, flag %x\n", pccb, pccb->flags); /* indicate aborted IO completion */ pccb->startTime = 0; agtiapi_Done(pCard, pccb); } else { #ifdef AGTIAPI_SA /* * SAS no data command does not trigger interrupt. * Command is completed in tdlayer and IO completion is called directly. * The completed IO therefore is not post processed. * Flag is raised and TDTimer will check and process IO for SAS. * This is a temporary solution. - Eddie, 07-17-2006 */ pCard->flags |= AGTIAPI_FLAG_UP; #endif pccb->flags |= REQ_DONE; agtiapi_QueueCCB(pCard, &pCard->ccbDoneHead, &pCard->ccbDoneTail AG_CARD_LOCAL_LOCK(&pCard->doneLock), pccb); } OSTI_OUT_LEAVE(ptiRoot); return; } #ifdef HIALEAH_ENCRYPTION osGLOBAL void ostidisableEncryption(tiRoot_t *ptiRoot) { struct agtiapi_softc *pCard; pCard = TIROOT_TO_CARD(ptiRoot); pCard->encrypt=agFALSE; } #endif /* device Handle */ osGLOBAL //FORCEINLINE tiDeviceHandle_t* ostiGetDevHandleFromSasAddr( tiRoot_t *root, unsigned char *sas_addr ) { int i; unsigned long x; ag_portal_data_t *pPortal = NULL; tiDeviceHandle_t *devHandle = NULL; struct agtiapi_softc *pCard = TIROOT_TO_CARD(root); bit8 sas_addr_hi[4], sas_addr_lo[4]; for(i=0; i<4; i++) { sas_addr_hi[i] = sas_addr[3-i]; } for(i=0; i<4; i++) { sas_addr_lo[i] = sas_addr[7-i]; } /* Retrieve the handles for each portal */ for (x=0; x < pCard->portCount; x++) { pPortal = &pCard->pPortalData[x]; devHandle = tiINIGetExpDeviceHandleBySasAddress(&pCard->tiRoot, &pPortal->portalInfo.tiPortalContext, *(bit32*)sas_addr_hi, *(bit32*)sas_addr_lo, (bit32)1024/*gMaxTargets*/); if(devHandle != NULL) break; } return devHandle; return NULL; } /****************************************************************************** ostiInitiatorSMPCompleted() Purpose: IO request completion call back Parameters: tiRoot_t *ptiRoot (IN) Pointer to the HBA tiRoot tiIORequest_t *ptiSMPRequest (IN) Pointer to the SMP request structure tiIOStatus_t IOStatus (IN) I/O complated status U32 tiSMPInfoLen (IN) Number of bytes of response frame len tiFrameHandle (IN) Handle that referes to response frame U32 context (IN) Interrupt dealing context Returns: Note: ******************************************************************************/ void ostiInitiatorSMPCompleted(tiRoot_t *ptiRoot, tiIORequest_t *ptiSMPRequest, tiSMPStatus_t smpStatus, bit32 tiSMPInfoLen, void *tiFrameHandle, bit32 context) { struct agtiapi_softc *pCard; ccb_t *pccb; pCard = TIROOT_TO_CARD(ptiRoot); pccb = (ccb_t *)ptiSMPRequest->osData; AGTIAPI_PRINTK("ostiInitiatorSMPCompleted: start\n"); OSTI_OUT_ENTER(ptiRoot); pccb->ccbStatus = (U16)smpStatus; if(smpStatus != tiSMPSuccess) { AGTIAPI_PRINTK("ostiInitiatorSMPCompleted: SMP Error\n"); } else { union ccb *ccb = pccb->ccb; struct ccb_smpio *csmpio = &ccb->smpio; memcpy(csmpio->smp_response, tiFrameHandle, tiSMPInfoLen); csmpio->smp_response_len = tiSMPInfoLen; - agtiapi_hexdump("ostiInitiatorSMPCompleted: Responce Payload in CAM", (bit8 *)csmpio->smp_response, csmpio->smp_response_len); + agtiapi_hexdump("ostiInitiatorSMPCompleted: Response Payload in CAM", (bit8 *)csmpio->smp_response, csmpio->smp_response_len); } pccb->flags |= REQ_DONE; agtiapi_QueueCCB(pCard, &pCard->smpDoneHead, &pCard->smpDoneTail AG_CARD_LOCAL_LOCK(&pCard->doneSMPLock), pccb); AGTIAPI_PRINTK("ostiInitiatorSMPCompleted: Done\n"); OSTI_OUT_LEAVE(ptiRoot); return; } #ifdef FAST_IO_TEST void osti_FastIOCb(tiRoot_t *ptiRoot, void *arg, tiIOStatus_t IOStatus, U32 statusDetail) { ccb_t *pccb = (ccb_t*)arg; ag_card_t *pCard; static int callNum = 0; callNum++; BUG_ON(!pccb); if ((callNum % CMDS_PER_IO_DUP) != 0) { goto err; } pccb->ccbStatus = IOStatus; pccb->scsiStatus = statusDetail; /* pccb->pSenseData is copied already */ if (pccb->flags & AGTIAPI_ABORT) { AGTIAPI_PRINTK("agtiapi_SuperIOCb: aborted ccb %p, flag %x\n", pccb, pccb->flags); pccb->startTime = 0; /* indicate aborted IO completion */ BUG_ON(1); goto err; } pCard = TIROOT_TO_CARD(ptiRoot); pccb->flags |= REQ_DONE; agtiapi_QueueCCB(pCard, &pCard->ccbDoneHead, &pCard->ccbDoneTail AG_CARD_LOCAL_LOCK(&pCard->doneLock), pccb); err: return; } /* osti_FastIOCb */ #endif /****************************************************************************** ostiSingleThreadedEnter() Purpose: Critical region code excution protection. Parameters: tiRoot_t *ptiRoot (IN) Pointer to tiRoot data structure U32 queueId (IN) spinlock Id Returns: Note: Lock is held by oslayer. ******************************************************************************/ void ostiSingleThreadedEnter(tiRoot_t *ptiRoot, U32 queueId) { struct agtiapi_softc *pCard = TIROOT_TO_CARD(ptiRoot); mtx_lock( &pCard->STLock[queueId] ); // review: need irq save? ## } /****************************************************************************** ostiSingleThreadedLeave() Purpose: Restore multi-threading environment. Parameters: tiRoot_t *ptiRoot (IN) Pointer to the tiRoot data structure U32 queueId (IN) spinlock Id Returns: Note: Lock is held by oslayer. ******************************************************************************/ void ostiSingleThreadedLeave(tiRoot_t *ptiRoot, U32 queueId) { struct agtiapi_softc *pCard = TIROOT_TO_CARD(ptiRoot); mtx_unlock( &pCard->STLock[queueId] ); // review: need irq restore? ## } osGLOBAL tiDeviceHandle_t* ostiMapToDevHandle(tiRoot_t *root, bit8 pathId, bit8 targetId, bit8 LUN ) { tiDeviceHandle_t *dev = NULL; struct agtiapi_softc *pCard; bit32 offset; pCard = TIROOT_TO_CARD(root); offset = pathId * pCard->tgtCount + targetId; if (offset > (pCard->tgtCount - 1) ) { dev = NULL; } else { dev = pCard->pDevList[offset].pDevHandle; } return dev; } #ifdef PERF_COUNT #ifdef AGTIAPI_LOCAL_LOCK #define OSTI_SPIN_LOCK(lock) spin_lock(lock) #define OSTI_SPIN_UNLOCK(lock) spin_unlock(lock) #else #define OSTI_SPIN_LOCK(lock) #define OSTI_SPIN_UNLOCK(lock) #endif void ostiEnter(tiRoot_t *ptiRoot, U32 layer, int io) { ag_card_t *pCard = ((ag_card_info_t*)ptiRoot->osData)->pCard; int ini = ((pCard->flags & AGTIAPI_INIT_TIME) == AGTIAPI_INIT_TIME); BUG_ON((io != 0 && io != 1) || (layer != 0 && layer != 1 && layer != 2)); if (!ini) { unsigned long long cycles = get_cycles(); OSTI_SPIN_LOCK(&pCard->latLock); BUG_ON(pCard->callLevel[io] >= sizeof(pCard->layer[0]) / sizeof(pCard->layer[0][0])); if (pCard->callLevel[io] > 0) { unsigned int prev_layer = pCard->layer[io][pCard->callLevel[io] - 1]; pCard->totalCycles[io][prev_layer] += cycles - pCard->enterCycles[io][prev_layer]; } pCard->enterCycles[io][layer] = cycles; pCard->layer[io][pCard->callLevel[io]] = layer; pCard->callLevel[io]++; OSTI_SPIN_UNLOCK(&pCard->latLock); } } void ostiLeave(tiRoot_t *ptiRoot, U32 layer, int io) { ag_card_t *pCard = ((ag_card_info_t*)ptiRoot->osData)->pCard; int ini = ((pCard->flags & AGTIAPI_INIT_TIME) == AGTIAPI_INIT_TIME); BUG_ON((io != 0 && io != 1) || (layer != 0 && layer != 1 && layer != 2)); if (!ini) { unsigned long long cycles = get_cycles(); OSTI_SPIN_LOCK(&pCard->latLock); pCard->callLevel[io]--; BUG_ON(pCard->callLevel[io] < 0); BUG_ON(pCard->layer[io][pCard->callLevel[io]] != layer); pCard->totalCycles[io][layer] += cycles - pCard->enterCycles[io][layer]; if (pCard->callLevel[io] > 0) pCard->enterCycles[io][pCard->layer[io][pCard->callLevel[io] - 1]] = cycles; OSTI_SPIN_UNLOCK(&pCard->latLock); } } #endif osGLOBAL FORCEINLINE bit8 ostiBitScanForward( tiRoot_t *root, bit32 *Index, bit32 Mask ) { return 1; } #ifdef REMOVED osGLOBAL sbit32 ostiAtomicIncrement( tiRoot_t *root, sbit32 volatile *Addend ) { return 1; } osGLOBAL sbit32 ostiAtomicDecrement( tiRoot_t *root, sbit32 volatile *Addend ) { return 1; } osGLOBAL sbit32 ostiAtomicBitClear( tiRoot_t *root, sbit32 volatile *Destination, sbit32 Value ) { return 0; } osGLOBAL sbit32 ostiAtomicBitSet( tiRoot_t *root, sbit32 volatile *Destination, sbit32 Value ) { return 0; /* set_bit(Value, (volatile unsigned long *)Destination); return 0; */ } osGLOBAL sbit32 ostiAtomicExchange( tiRoot_t *root, sbit32 volatile *Target, sbit32 Value ) { return 0; } #endif osGLOBAL FORCEINLINE sbit32 ostiInterlockedExchange( tiRoot_t *root, sbit32 volatile *Target, sbit32 Value ) { return 0; } osGLOBAL FORCEINLINE sbit32 ostiInterlockedIncrement( tiRoot_t *root, sbit32 volatile *Addend ) { return 0; } osGLOBAL FORCEINLINE sbit32 ostiInterlockedDecrement( tiRoot_t *root, sbit32 volatile *Addend ) { return 0; } osGLOBAL FORCEINLINE sbit32 ostiInterlockedAnd( tiRoot_t *root, sbit32 volatile *Destination, sbit32 Value ) { return 0; } osGLOBAL FORCEINLINE sbit32 ostiInterlockedOr( tiRoot_t *root, sbit32 volatile *Destination, sbit32 Value ) { return 0; } // this is just stub code to allow compile and use of the module ... // now that a call to this function has been added with windows specific // intentions. osGLOBAL bit32 ostiSetDeviceQueueDepth( tiRoot_t *tiRoot, tiIORequest_t *tiIORequest, bit32 QueueDepth ) { bit32 retVal = 0; struct agtiapi_softc *pCard = TIROOT_TO_CARD(tiRoot); ccb_t *pccb = (ccb_t *) tiIORequest->osData; tiDeviceHandle_t *tiDeviceHandle = pccb->devHandle; ag_device_t *pDevice = (ag_device_t *)tiDeviceHandle->osData; AGTIAPI_PRINTK( "ostiSetDeviceQueueDepth stub only: root%p, req%p, qdeep%d\n", tiRoot, tiIORequest, QueueDepth ); pDevice->qdepth = QueueDepth; return retVal; } // this is just stub code to allow compile and use of the module ... // now that a call to this function has been added with windows specific // intentions. osGLOBAL void ostiGetSenseKeyCount(tiRoot_t *root, bit32 fIsClear, void *SenseKeyCount, bit32 length ) { AGTIAPI_PRINTK( "ostiGetSenseKeyCount stub only: rt%p, fcl%d, kyCt%p, ln%d\n", root, fIsClear, SenseKeyCount, length ); } osGLOBAL void ostiGetSCSIStatusCount(tiRoot_t *root, bit32 fIsClear, void *ScsiStatusCount, bit32 length ) { AGTIAPI_PRINTK( "ostiGetSCSIStatusCount: stub only rt%p, fcl%d, kyCt%p, ln%d\n", root, fIsClear, ScsiStatusCount, length ); } osGLOBAL void ostiPCI_TRIGGER( tiRoot_t *tiRoot ) { ostiChipReadBit32Ext(tiRoot, 0, 0x5C); } osGLOBAL bit32 ostiNumOfLUNIOCTLreq( tiRoot_t *root, void *param1, void *param2, void **tiRequestBody, tiIORequest_t **tiIORequest ) { bit32 status = IOCTL_CALL_SUCCESS; pccb_t pccb; AGTIAPI_PRINTK("ostiNumOfLUNIOCTLreq: start\n"); struct agtiapi_softc *pCard = TIROOT_TO_CARD(root); /* get a ccb */ if ((pccb = agtiapi_GetCCB(pCard)) == NULL) { printf("ostiNumOfLUNIOCTLreq - GetCCB ERROR\n"); status = IOCTL_CALL_FAIL; //BUG_ON(1); } *tiIORequest = (tiIORequest_t*)&pccb->tiIORequest; *tiRequestBody = &pccb->tdIOReqBody; AGTIAPI_PRINTK("ostiNumOfLUNIOCTLreq:end\n"); return status; } Index: user/ngie/release-pkg-fix-tests/sys/dev/usb/usb_device.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/dev/usb/usb_device.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/dev/usb/usb_device.c (revision 299091) @@ -1,2888 +1,2890 @@ /* $FreeBSD$ */ /*- * Copyright (c) 2008 Hans Petter Selasky. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #ifdef USB_GLOBAL_INCLUDE_FILE #include USB_GLOBAL_INCLUDE_FILE #else #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #if USB_HAVE_UGEN #include #endif #include "usbdevs.h" #define USB_DEBUG_VAR usb_debug #include #include #include #include #include #include #include #include #include #include #include #if USB_HAVE_UGEN #include #include #endif #include #include #include #endif /* USB_GLOBAL_INCLUDE_FILE */ /* function prototypes */ static void usb_init_endpoint(struct usb_device *, uint8_t, struct usb_endpoint_descriptor *, struct usb_endpoint_ss_comp_descriptor *, struct usb_endpoint *); static void usb_unconfigure(struct usb_device *, uint8_t); static void usb_detach_device_sub(struct usb_device *, device_t *, char **, uint8_t); static uint8_t usb_probe_and_attach_sub(struct usb_device *, struct usb_attach_arg *); static void usb_init_attach_arg(struct usb_device *, struct usb_attach_arg *); static void usb_suspend_resume_sub(struct usb_device *, device_t, uint8_t); static usb_proc_callback_t usbd_clear_stall_proc; static usb_error_t usb_config_parse(struct usb_device *, uint8_t, uint8_t); static void usbd_set_device_strings(struct usb_device *); #if USB_HAVE_DEVCTL static void usb_notify_addq(const char *type, struct usb_device *); #endif #if USB_HAVE_UGEN static void usb_fifo_free_wrap(struct usb_device *, uint8_t, uint8_t); static void usb_cdev_create(struct usb_device *); static void usb_cdev_free(struct usb_device *); #endif /* This variable is global to allow easy access to it: */ #ifdef USB_TEMPLATE int usb_template = USB_TEMPLATE; #else int usb_template; #endif SYSCTL_INT(_hw_usb, OID_AUTO, template, CTLFLAG_RWTUN, &usb_template, 0, "Selected USB device side template"); /* English is default language */ static int usb_lang_id = 0x0009; static int usb_lang_mask = 0x00FF; SYSCTL_INT(_hw_usb, OID_AUTO, usb_lang_id, CTLFLAG_RWTUN, &usb_lang_id, 0, "Preferred USB language ID"); SYSCTL_INT(_hw_usb, OID_AUTO, usb_lang_mask, CTLFLAG_RWTUN, &usb_lang_mask, 0, "Preferred USB language mask"); static const char* statestr[USB_STATE_MAX] = { [USB_STATE_DETACHED] = "DETACHED", [USB_STATE_ATTACHED] = "ATTACHED", [USB_STATE_POWERED] = "POWERED", [USB_STATE_ADDRESSED] = "ADDRESSED", [USB_STATE_CONFIGURED] = "CONFIGURED", }; const char * usb_statestr(enum usb_dev_state state) { return ((state < USB_STATE_MAX) ? statestr[state] : "UNKNOWN"); } const char * usb_get_manufacturer(struct usb_device *udev) { return (udev->manufacturer ? udev->manufacturer : "Unknown"); } const char * usb_get_product(struct usb_device *udev) { return (udev->product ? udev->product : ""); } const char * usb_get_serial(struct usb_device *udev) { return (udev->serial ? udev->serial : ""); } /*------------------------------------------------------------------------* * usbd_get_ep_by_addr * * This function searches for an USB ep by endpoint address and * direction. * * Returns: * NULL: Failure * Else: Success *------------------------------------------------------------------------*/ struct usb_endpoint * usbd_get_ep_by_addr(struct usb_device *udev, uint8_t ea_val) { struct usb_endpoint *ep = udev->endpoints; struct usb_endpoint *ep_end = udev->endpoints + udev->endpoints_max; enum { EA_MASK = (UE_DIR_IN | UE_DIR_OUT | UE_ADDR), }; /* * According to the USB specification not all bits are used * for the endpoint address. Keep defined bits only: */ ea_val &= EA_MASK; /* * Iterate across all the USB endpoints searching for a match * based on the endpoint address: */ for (; ep != ep_end; ep++) { if (ep->edesc == NULL) { continue; } /* do the mask and check the value */ if ((ep->edesc->bEndpointAddress & EA_MASK) == ea_val) { goto found; } } /* * The default endpoint is always present and is checked separately: */ if ((udev->ctrl_ep.edesc != NULL) && ((udev->ctrl_ep.edesc->bEndpointAddress & EA_MASK) == ea_val)) { ep = &udev->ctrl_ep; goto found; } return (NULL); found: return (ep); } /*------------------------------------------------------------------------* * usbd_get_endpoint * * This function searches for an USB endpoint based on the information * given by the passed "struct usb_config" pointer. * * Return values: * NULL: No match. * Else: Pointer to "struct usb_endpoint". *------------------------------------------------------------------------*/ struct usb_endpoint * usbd_get_endpoint(struct usb_device *udev, uint8_t iface_index, const struct usb_config *setup) { struct usb_endpoint *ep = udev->endpoints; struct usb_endpoint *ep_end = udev->endpoints + udev->endpoints_max; uint8_t index = setup->ep_index; uint8_t ea_mask; uint8_t ea_val; uint8_t type_mask; uint8_t type_val; DPRINTFN(10, "udev=%p iface_index=%d address=0x%x " "type=0x%x dir=0x%x index=%d\n", udev, iface_index, setup->endpoint, setup->type, setup->direction, setup->ep_index); /* check USB mode */ if (setup->usb_mode != USB_MODE_DUAL && udev->flags.usb_mode != setup->usb_mode) { /* wrong mode - no endpoint */ return (NULL); } /* setup expected endpoint direction mask and value */ if (setup->direction == UE_DIR_RX) { ea_mask = (UE_DIR_IN | UE_DIR_OUT); ea_val = (udev->flags.usb_mode == USB_MODE_DEVICE) ? UE_DIR_OUT : UE_DIR_IN; } else if (setup->direction == UE_DIR_TX) { ea_mask = (UE_DIR_IN | UE_DIR_OUT); ea_val = (udev->flags.usb_mode == USB_MODE_DEVICE) ? UE_DIR_IN : UE_DIR_OUT; } else if (setup->direction == UE_DIR_ANY) { /* match any endpoint direction */ ea_mask = 0; ea_val = 0; } else { /* match the given endpoint direction */ ea_mask = (UE_DIR_IN | UE_DIR_OUT); ea_val = (setup->direction & (UE_DIR_IN | UE_DIR_OUT)); } /* setup expected endpoint address */ if (setup->endpoint == UE_ADDR_ANY) { /* match any endpoint address */ } else { /* match the given endpoint address */ ea_mask |= UE_ADDR; ea_val |= (setup->endpoint & UE_ADDR); } /* setup expected endpoint type */ if (setup->type == UE_BULK_INTR) { /* this will match BULK and INTERRUPT endpoints */ type_mask = 2; type_val = 2; } else if (setup->type == UE_TYPE_ANY) { /* match any endpoint type */ type_mask = 0; type_val = 0; } else { /* match the given endpoint type */ type_mask = UE_XFERTYPE; type_val = (setup->type & UE_XFERTYPE); } /* * Iterate across all the USB endpoints searching for a match * based on the endpoint address. Note that we are searching * the endpoints from the beginning of the "udev->endpoints" array. */ for (; ep != ep_end; ep++) { if ((ep->edesc == NULL) || (ep->iface_index != iface_index)) { continue; } /* do the masks and check the values */ if (((ep->edesc->bEndpointAddress & ea_mask) == ea_val) && ((ep->edesc->bmAttributes & type_mask) == type_val)) { if (!index--) { goto found; } } } /* * Match against default endpoint last, so that "any endpoint", "any * address" and "any direction" returns the first endpoint of the * interface. "iface_index" and "direction" is ignored: */ if ((udev->ctrl_ep.edesc != NULL) && ((udev->ctrl_ep.edesc->bEndpointAddress & ea_mask) == ea_val) && ((udev->ctrl_ep.edesc->bmAttributes & type_mask) == type_val) && (!index)) { ep = &udev->ctrl_ep; goto found; } return (NULL); found: return (ep); } /*------------------------------------------------------------------------* * usbd_interface_count * * This function stores the number of USB interfaces excluding * alternate settings, which the USB config descriptor reports into * the unsigned 8-bit integer pointed to by "count". * * Returns: * 0: Success * Else: Failure *------------------------------------------------------------------------*/ usb_error_t usbd_interface_count(struct usb_device *udev, uint8_t *count) { if (udev->cdesc == NULL) { *count = 0; return (USB_ERR_NOT_CONFIGURED); } *count = udev->ifaces_max; return (USB_ERR_NORMAL_COMPLETION); } /*------------------------------------------------------------------------* * usb_init_endpoint * * This function will initialise the USB endpoint structure pointed to by * the "endpoint" argument. The structure pointed to by "endpoint" must be * zeroed before calling this function. *------------------------------------------------------------------------*/ static void usb_init_endpoint(struct usb_device *udev, uint8_t iface_index, struct usb_endpoint_descriptor *edesc, struct usb_endpoint_ss_comp_descriptor *ecomp, struct usb_endpoint *ep) { const struct usb_bus_methods *methods; usb_stream_t x; methods = udev->bus->methods; (methods->endpoint_init) (udev, edesc, ep); /* initialise USB endpoint structure */ ep->edesc = edesc; ep->ecomp = ecomp; ep->iface_index = iface_index; /* setup USB stream queues */ for (x = 0; x != USB_MAX_EP_STREAMS; x++) { TAILQ_INIT(&ep->endpoint_q[x].head); ep->endpoint_q[x].command = &usbd_pipe_start; } /* the pipe is not supported by the hardware */ if (ep->methods == NULL) return; /* check for SUPER-speed streams mode endpoint */ if (udev->speed == USB_SPEED_SUPER && ecomp != NULL && (edesc->bmAttributes & UE_XFERTYPE) == UE_BULK && (UE_GET_BULK_STREAMS(ecomp->bmAttributes) != 0)) { usbd_set_endpoint_mode(udev, ep, USB_EP_MODE_STREAMS); } else { usbd_set_endpoint_mode(udev, ep, USB_EP_MODE_DEFAULT); } /* clear stall, if any */ if (methods->clear_stall != NULL) { USB_BUS_LOCK(udev->bus); (methods->clear_stall) (udev, ep); USB_BUS_UNLOCK(udev->bus); } } /*-----------------------------------------------------------------------* * usb_endpoint_foreach * * This function will iterate all the USB endpoints except the control * endpoint. This function is NULL safe. * * Return values: * NULL: End of USB endpoints * Else: Pointer to next USB endpoint *------------------------------------------------------------------------*/ struct usb_endpoint * usb_endpoint_foreach(struct usb_device *udev, struct usb_endpoint *ep) { struct usb_endpoint *ep_end; /* be NULL safe */ if (udev == NULL) return (NULL); ep_end = udev->endpoints + udev->endpoints_max; /* get next endpoint */ if (ep == NULL) ep = udev->endpoints; else ep++; /* find next allocated ep */ while (ep != ep_end) { if (ep->edesc != NULL) return (ep); ep++; } return (NULL); } /*------------------------------------------------------------------------* * usb_wait_pending_refs * * This function will wait for any USB references to go away before * returning. This function is used before freeing a USB device. *------------------------------------------------------------------------*/ static void usb_wait_pending_refs(struct usb_device *udev) { #if USB_HAVE_UGEN DPRINTF("Refcount = %d\n", (int)udev->refcount); mtx_lock(&usb_ref_lock); udev->refcount--; while (1) { /* wait for any pending references to go away */ if (udev->refcount == 0) { /* prevent further refs being taken, if any */ udev->refcount = USB_DEV_REF_MAX; break; } cv_wait(&udev->ref_cv, &usb_ref_lock); } mtx_unlock(&usb_ref_lock); #endif } /*------------------------------------------------------------------------* * usb_unconfigure * * This function will free all USB interfaces and USB endpoints belonging * to an USB device. * * Flag values, see "USB_UNCFG_FLAG_XXX". *------------------------------------------------------------------------*/ static void usb_unconfigure(struct usb_device *udev, uint8_t flag) { uint8_t do_unlock; /* Prevent re-enumeration */ do_unlock = usbd_enum_lock(udev); /* detach all interface drivers */ usb_detach_device(udev, USB_IFACE_INDEX_ANY, flag); #if USB_HAVE_UGEN /* free all FIFOs except control endpoint FIFOs */ usb_fifo_free_wrap(udev, USB_IFACE_INDEX_ANY, flag); /* * Free all cdev's, if any. */ usb_cdev_free(udev); #endif #if USB_HAVE_COMPAT_LINUX /* free Linux compat device, if any */ if (udev->linux_endpoint_start != NULL) { usb_linux_free_device_p(udev); udev->linux_endpoint_start = NULL; } #endif usb_config_parse(udev, USB_IFACE_INDEX_ANY, USB_CFG_FREE); /* free "cdesc" after "ifaces" and "endpoints", if any */ if (udev->cdesc != NULL) { if (udev->flags.usb_mode != USB_MODE_DEVICE) usbd_free_config_desc(udev, udev->cdesc); udev->cdesc = NULL; } /* set unconfigured state */ udev->curr_config_no = USB_UNCONFIG_NO; udev->curr_config_index = USB_UNCONFIG_INDEX; if (do_unlock) usbd_enum_unlock(udev); } /*------------------------------------------------------------------------* * usbd_set_config_index * * This function selects configuration by index, independent of the * actual configuration number. This function should not be used by * USB drivers. * * Returns: * 0: Success * Else: Failure *------------------------------------------------------------------------*/ usb_error_t usbd_set_config_index(struct usb_device *udev, uint8_t index) { struct usb_status ds; struct usb_config_descriptor *cdp; uint16_t power; uint16_t max_power; uint8_t selfpowered; uint8_t do_unlock; usb_error_t err; DPRINTFN(6, "udev=%p index=%d\n", udev, index); /* Prevent re-enumeration */ do_unlock = usbd_enum_lock(udev); usb_unconfigure(udev, 0); if (index == USB_UNCONFIG_INDEX) { /* * Leave unallocated when unconfiguring the * device. "usb_unconfigure()" will also reset * the current config number and index. */ err = usbd_req_set_config(udev, NULL, USB_UNCONFIG_NO); if (udev->state == USB_STATE_CONFIGURED) usb_set_device_state(udev, USB_STATE_ADDRESSED); goto done; } /* get the full config descriptor */ if (udev->flags.usb_mode == USB_MODE_DEVICE) { /* save some memory */ err = usbd_req_get_descriptor_ptr(udev, &cdp, (UDESC_CONFIG << 8) | index); } else { /* normal request */ err = usbd_req_get_config_desc_full(udev, NULL, &cdp, index); } if (err) { goto done; } /* set the new config descriptor */ udev->cdesc = cdp; /* Figure out if the device is self or bus powered. */ selfpowered = 0; if ((!udev->flags.uq_bus_powered) && (cdp->bmAttributes & UC_SELF_POWERED) && (udev->flags.usb_mode == USB_MODE_HOST)) { /* May be self powered. */ if (cdp->bmAttributes & UC_BUS_POWERED) { /* Must ask device. */ err = usbd_req_get_device_status(udev, NULL, &ds); if (err) { DPRINTFN(0, "could not read " "device status: %s\n", usbd_errstr(err)); } else if (UGETW(ds.wStatus) & UDS_SELF_POWERED) { selfpowered = 1; } DPRINTF("status=0x%04x \n", UGETW(ds.wStatus)); } else selfpowered = 1; } DPRINTF("udev=%p cdesc=%p (addr %d) cno=%d attr=0x%02x, " "selfpowered=%d, power=%d\n", udev, cdp, udev->address, cdp->bConfigurationValue, cdp->bmAttributes, selfpowered, cdp->bMaxPower * 2); /* Check if we have enough power. */ power = cdp->bMaxPower * 2; if (udev->parent_hub) { max_power = udev->parent_hub->hub->portpower; } else { max_power = USB_MAX_POWER; } if (power > max_power) { DPRINTFN(0, "power exceeded %d > %d\n", power, max_power); err = USB_ERR_NO_POWER; goto done; } /* Only update "self_powered" in USB Host Mode */ if (udev->flags.usb_mode == USB_MODE_HOST) { udev->flags.self_powered = selfpowered; } udev->power = power; udev->curr_config_no = cdp->bConfigurationValue; udev->curr_config_index = index; usb_set_device_state(udev, USB_STATE_CONFIGURED); /* Set the actual configuration value. */ err = usbd_req_set_config(udev, NULL, cdp->bConfigurationValue); if (err) { goto done; } err = usb_config_parse(udev, USB_IFACE_INDEX_ANY, USB_CFG_ALLOC); if (err) { goto done; } err = usb_config_parse(udev, USB_IFACE_INDEX_ANY, USB_CFG_INIT); if (err) { goto done; } #if USB_HAVE_UGEN /* create device nodes for each endpoint */ usb_cdev_create(udev); #endif done: DPRINTF("error=%s\n", usbd_errstr(err)); if (err) { usb_unconfigure(udev, 0); } if (do_unlock) usbd_enum_unlock(udev); return (err); } /*------------------------------------------------------------------------* * usb_config_parse * * This function will allocate and free USB interfaces and USB endpoints, * parse the USB configuration structure and initialise the USB endpoints * and interfaces. If "iface_index" is not equal to * "USB_IFACE_INDEX_ANY" then the "cmd" parameter is the * alternate_setting to be selected for the given interface. Else the * "cmd" parameter is defined by "USB_CFG_XXX". "iface_index" can be * "USB_IFACE_INDEX_ANY" or a valid USB interface index. This function * is typically called when setting the configuration or when setting * an alternate interface. * * Returns: * 0: Success * Else: Failure *------------------------------------------------------------------------*/ static usb_error_t usb_config_parse(struct usb_device *udev, uint8_t iface_index, uint8_t cmd) { struct usb_idesc_parse_state ips; struct usb_interface_descriptor *id; struct usb_endpoint_descriptor *ed; struct usb_interface *iface; struct usb_endpoint *ep; usb_error_t err; uint8_t ep_curr; uint8_t ep_max; uint8_t temp; uint8_t do_init; uint8_t alt_index; if (iface_index != USB_IFACE_INDEX_ANY) { /* parameter overload */ alt_index = cmd; cmd = USB_CFG_INIT; } else { /* not used */ alt_index = 0; } err = 0; DPRINTFN(5, "iface_index=%d cmd=%d\n", iface_index, cmd); if (cmd == USB_CFG_FREE) goto cleanup; if (cmd == USB_CFG_INIT) { sx_assert(&udev->enum_sx, SA_LOCKED); /* check for in-use endpoints */ ep = udev->endpoints; ep_max = udev->endpoints_max; while (ep_max--) { /* look for matching endpoints */ if ((iface_index == USB_IFACE_INDEX_ANY) || (iface_index == ep->iface_index)) { if (ep->refcount_alloc != 0) { /* * This typically indicates a * more serious error. */ err = USB_ERR_IN_USE; } else { /* reset endpoint */ memset(ep, 0, sizeof(*ep)); /* make sure we don't zero the endpoint again */ ep->iface_index = USB_IFACE_INDEX_ANY; } } ep++; } if (err) return (err); } memset(&ips, 0, sizeof(ips)); ep_curr = 0; ep_max = 0; while ((id = usb_idesc_foreach(udev->cdesc, &ips))) { iface = udev->ifaces + ips.iface_index; /* check for specific interface match */ if (cmd == USB_CFG_INIT) { if ((iface_index != USB_IFACE_INDEX_ANY) && (iface_index != ips.iface_index)) { /* wrong interface */ do_init = 0; } else if (alt_index != ips.iface_index_alt) { /* wrong alternate setting */ do_init = 0; } else { /* initialise interface */ do_init = 1; } } else do_init = 0; /* check for new interface */ if (ips.iface_index_alt == 0) { /* update current number of endpoints */ ep_curr = ep_max; } /* check for init */ if (do_init) { /* setup the USB interface structure */ iface->idesc = id; /* set alternate index */ iface->alt_index = alt_index; /* set default interface parent */ if (iface_index == USB_IFACE_INDEX_ANY) { iface->parent_iface_index = USB_IFACE_INDEX_ANY; } } DPRINTFN(5, "found idesc nendpt=%d\n", id->bNumEndpoints); ed = (struct usb_endpoint_descriptor *)id; temp = ep_curr; /* iterate all the endpoint descriptors */ while ((ed = usb_edesc_foreach(udev->cdesc, ed))) { /* check if endpoint limit has been reached */ if (temp >= USB_MAX_EP_UNITS) { DPRINTF("Endpoint limit reached\n"); break; } ep = udev->endpoints + temp; if (do_init) { void *ecomp; ecomp = usb_ed_comp_foreach(udev->cdesc, (void *)ed); if (ecomp != NULL) DPRINTFN(5, "Found endpoint companion descriptor\n"); usb_init_endpoint(udev, ips.iface_index, ed, ecomp, ep); } temp ++; /* find maximum number of endpoints */ if (ep_max < temp) ep_max = temp; } } /* NOTE: It is valid to have no interfaces and no endpoints! */ if (cmd == USB_CFG_ALLOC) { udev->ifaces_max = ips.iface_index; #if (USB_HAVE_FIXED_IFACE == 0) udev->ifaces = NULL; if (udev->ifaces_max != 0) { udev->ifaces = malloc(sizeof(*iface) * udev->ifaces_max, M_USB, M_WAITOK | M_ZERO); if (udev->ifaces == NULL) { err = USB_ERR_NOMEM; goto done; } } #endif #if (USB_HAVE_FIXED_ENDPOINT == 0) if (ep_max != 0) { udev->endpoints = malloc(sizeof(*ep) * ep_max, M_USB, M_WAITOK | M_ZERO); if (udev->endpoints == NULL) { err = USB_ERR_NOMEM; goto done; } } else { udev->endpoints = NULL; } #endif USB_BUS_LOCK(udev->bus); udev->endpoints_max = ep_max; /* reset any ongoing clear-stall */ udev->ep_curr = NULL; USB_BUS_UNLOCK(udev->bus); } #if (USB_HAVE_FIXED_IFACE == 0) || (USB_HAVE_FIXED_ENDPOINT == 0) done: #endif if (err) { if (cmd == USB_CFG_ALLOC) { cleanup: USB_BUS_LOCK(udev->bus); udev->endpoints_max = 0; /* reset any ongoing clear-stall */ udev->ep_curr = NULL; USB_BUS_UNLOCK(udev->bus); #if (USB_HAVE_FIXED_IFACE == 0) free(udev->ifaces, M_USB); udev->ifaces = NULL; #endif #if (USB_HAVE_FIXED_ENDPOINT == 0) free(udev->endpoints, M_USB); udev->endpoints = NULL; #endif udev->ifaces_max = 0; } } return (err); } /*------------------------------------------------------------------------* * usbd_set_alt_interface_index * * This function will select an alternate interface index for the * given interface index. The interface should not be in use when this * function is called. That means there should not be any open USB * transfers. Else an error is returned. If the alternate setting is * already set this function will simply return success. This function * is called in Host mode and Device mode! * * Returns: * 0: Success * Else: Failure *------------------------------------------------------------------------*/ usb_error_t usbd_set_alt_interface_index(struct usb_device *udev, uint8_t iface_index, uint8_t alt_index) { struct usb_interface *iface = usbd_get_iface(udev, iface_index); usb_error_t err; uint8_t do_unlock; /* Prevent re-enumeration */ do_unlock = usbd_enum_lock(udev); if (iface == NULL) { err = USB_ERR_INVAL; goto done; } if (iface->alt_index == alt_index) { /* * Optimise away duplicate setting of * alternate setting in USB Host Mode! */ err = 0; goto done; } #if USB_HAVE_UGEN /* * Free all generic FIFOs for this interface, except control * endpoint FIFOs: */ usb_fifo_free_wrap(udev, iface_index, 0); #endif err = usb_config_parse(udev, iface_index, alt_index); if (err) { goto done; } if (iface->alt_index != alt_index) { /* the alternate setting does not exist */ err = USB_ERR_INVAL; goto done; } err = usbd_req_set_alt_interface_no(udev, NULL, iface_index, iface->idesc->bAlternateSetting); done: if (do_unlock) usbd_enum_unlock(udev); return (err); } /*------------------------------------------------------------------------* * usbd_set_endpoint_stall * * This function is used to make a BULK or INTERRUPT endpoint send * STALL tokens in USB device mode. * * Returns: * 0: Success * Else: Failure *------------------------------------------------------------------------*/ usb_error_t usbd_set_endpoint_stall(struct usb_device *udev, struct usb_endpoint *ep, uint8_t do_stall) { struct usb_xfer *xfer; usb_stream_t x; uint8_t et; uint8_t was_stalled; if (ep == NULL) { /* nothing to do */ DPRINTF("Cannot find endpoint\n"); /* * Pretend that the clear or set stall request is * successful else some USB host stacks can do * strange things, especially when a control endpoint * stalls. */ return (0); } et = (ep->edesc->bmAttributes & UE_XFERTYPE); if ((et != UE_BULK) && (et != UE_INTERRUPT)) { /* * Should not stall control * nor isochronous endpoints. */ DPRINTF("Invalid endpoint\n"); return (0); } USB_BUS_LOCK(udev->bus); /* store current stall state */ was_stalled = ep->is_stalled; /* check for no change */ if (was_stalled && do_stall) { /* if the endpoint is already stalled do nothing */ USB_BUS_UNLOCK(udev->bus); DPRINTF("No change\n"); return (0); } /* set stalled state */ ep->is_stalled = 1; if (do_stall || (!was_stalled)) { if (!was_stalled) { for (x = 0; x != USB_MAX_EP_STREAMS; x++) { /* lookup the current USB transfer, if any */ xfer = ep->endpoint_q[x].curr; if (xfer != NULL) { /* * The "xfer_stall" method * will complete the USB * transfer like in case of a * timeout setting the error * code "USB_ERR_STALLED". */ (udev->bus->methods->xfer_stall) (xfer); } } } (udev->bus->methods->set_stall) (udev, ep, &do_stall); } if (!do_stall) { ep->toggle_next = 0; /* reset data toggle */ ep->is_stalled = 0; /* clear stalled state */ (udev->bus->methods->clear_stall) (udev, ep); /* start the current or next transfer, if any */ for (x = 0; x != USB_MAX_EP_STREAMS; x++) { usb_command_wrapper(&ep->endpoint_q[x], ep->endpoint_q[x].curr); } } USB_BUS_UNLOCK(udev->bus); return (0); } /*------------------------------------------------------------------------* * usb_reset_iface_endpoints - used in USB device side mode *------------------------------------------------------------------------*/ usb_error_t usb_reset_iface_endpoints(struct usb_device *udev, uint8_t iface_index) { struct usb_endpoint *ep; struct usb_endpoint *ep_end; ep = udev->endpoints; ep_end = udev->endpoints + udev->endpoints_max; for (; ep != ep_end; ep++) { if ((ep->edesc == NULL) || (ep->iface_index != iface_index)) { continue; } /* simulate a clear stall from the peer */ usbd_set_endpoint_stall(udev, ep, 0); } return (0); } /*------------------------------------------------------------------------* * usb_detach_device_sub * * This function will try to detach an USB device. If it fails a panic * will result. * * Flag values, see "USB_UNCFG_FLAG_XXX". *------------------------------------------------------------------------*/ static void usb_detach_device_sub(struct usb_device *udev, device_t *ppdev, char **ppnpinfo, uint8_t flag) { device_t dev; char *pnpinfo; int err; dev = *ppdev; if (dev) { /* * NOTE: It is important to clear "*ppdev" before deleting * the child due to some device methods being called late * during the delete process ! */ *ppdev = NULL; if (!rebooting) { device_printf(dev, "at %s, port %d, addr %d " "(disconnected)\n", device_get_nameunit(udev->parent_dev), udev->port_no, udev->address); } if (device_is_attached(dev)) { if (udev->flags.peer_suspended) { err = DEVICE_RESUME(dev); if (err) { device_printf(dev, "Resume failed\n"); } } if (device_detach(dev)) { goto error; } } if (device_delete_child(udev->parent_dev, dev)) { goto error; } } pnpinfo = *ppnpinfo; if (pnpinfo != NULL) { *ppnpinfo = NULL; free(pnpinfo, M_USBDEV); } return; error: /* Detach is not allowed to fail in the USB world */ panic("usb_detach_device_sub: A USB driver would not detach\n"); } /*------------------------------------------------------------------------* * usb_detach_device * * The following function will detach the matching interfaces. * This function is NULL safe. * * Flag values, see "USB_UNCFG_FLAG_XXX". *------------------------------------------------------------------------*/ void usb_detach_device(struct usb_device *udev, uint8_t iface_index, uint8_t flag) { struct usb_interface *iface; uint8_t i; if (udev == NULL) { /* nothing to do */ return; } DPRINTFN(4, "udev=%p\n", udev); sx_assert(&udev->enum_sx, SA_LOCKED); /* * First detach the child to give the child's detach routine a * chance to detach the sub-devices in the correct order. * Then delete the child using "device_delete_child()" which * will detach all sub-devices from the bottom and upwards! */ if (iface_index != USB_IFACE_INDEX_ANY) { i = iface_index; iface_index = i + 1; } else { i = 0; iface_index = USB_IFACE_MAX; } /* do the detach */ for (; i != iface_index; i++) { iface = usbd_get_iface(udev, i); if (iface == NULL) { /* looks like the end of the USB interfaces */ break; } usb_detach_device_sub(udev, &iface->subdev, &iface->pnpinfo, flag); } } /*------------------------------------------------------------------------* * usb_probe_and_attach_sub * * Returns: * 0: Success * Else: Failure *------------------------------------------------------------------------*/ static uint8_t usb_probe_and_attach_sub(struct usb_device *udev, struct usb_attach_arg *uaa) { struct usb_interface *iface; device_t dev; int err; iface = uaa->iface; if (iface->parent_iface_index != USB_IFACE_INDEX_ANY) { /* leave interface alone */ return (0); } dev = iface->subdev; if (dev) { /* clean up after module unload */ if (device_is_attached(dev)) { /* already a device there */ return (0); } /* clear "iface->subdev" as early as possible */ iface->subdev = NULL; if (device_delete_child(udev->parent_dev, dev)) { /* * Panic here, else one can get a double call * to device_detach(). USB devices should * never fail on detach! */ panic("device_delete_child() failed\n"); } } if (uaa->temp_dev == NULL) { /* create a new child */ uaa->temp_dev = device_add_child(udev->parent_dev, NULL, -1); if (uaa->temp_dev == NULL) { device_printf(udev->parent_dev, "Device creation failed\n"); return (1); /* failure */ } device_set_ivars(uaa->temp_dev, uaa); device_quiet(uaa->temp_dev); } /* * Set "subdev" before probe and attach so that "devd" gets * the information it needs. */ iface->subdev = uaa->temp_dev; if (device_probe_and_attach(iface->subdev) == 0) { /* * The USB attach arguments are only available during probe * and attach ! */ uaa->temp_dev = NULL; device_set_ivars(iface->subdev, NULL); if (udev->flags.peer_suspended) { err = DEVICE_SUSPEND(iface->subdev); if (err) device_printf(iface->subdev, "Suspend failed\n"); } return (0); /* success */ } else { /* No USB driver found */ iface->subdev = NULL; } return (1); /* failure */ } /*------------------------------------------------------------------------* * usbd_set_parent_iface * * Using this function will lock the alternate interface setting on an * interface. It is typically used for multi interface drivers. In USB * device side mode it is assumed that the alternate interfaces all * have the same endpoint descriptors. The default parent index value * is "USB_IFACE_INDEX_ANY". Then the alternate setting value is not * locked. *------------------------------------------------------------------------*/ void usbd_set_parent_iface(struct usb_device *udev, uint8_t iface_index, uint8_t parent_index) { struct usb_interface *iface; if (udev == NULL) { /* nothing to do */ return; } iface = usbd_get_iface(udev, iface_index); if (iface != NULL) iface->parent_iface_index = parent_index; } static void usb_init_attach_arg(struct usb_device *udev, struct usb_attach_arg *uaa) { memset(uaa, 0, sizeof(*uaa)); uaa->device = udev; uaa->usb_mode = udev->flags.usb_mode; uaa->port = udev->port_no; uaa->dev_state = UAA_DEV_READY; uaa->info.idVendor = UGETW(udev->ddesc.idVendor); uaa->info.idProduct = UGETW(udev->ddesc.idProduct); uaa->info.bcdDevice = UGETW(udev->ddesc.bcdDevice); uaa->info.bDeviceClass = udev->ddesc.bDeviceClass; uaa->info.bDeviceSubClass = udev->ddesc.bDeviceSubClass; uaa->info.bDeviceProtocol = udev->ddesc.bDeviceProtocol; uaa->info.bConfigIndex = udev->curr_config_index; uaa->info.bConfigNum = udev->curr_config_no; } /*------------------------------------------------------------------------* * usb_probe_and_attach * * This function is called from "uhub_explore_sub()", * "usb_handle_set_config()" and "usb_handle_request()". * * Returns: * 0: Success * Else: A control transfer failed *------------------------------------------------------------------------*/ usb_error_t usb_probe_and_attach(struct usb_device *udev, uint8_t iface_index) { struct usb_attach_arg uaa; struct usb_interface *iface; uint8_t i; uint8_t j; uint8_t do_unlock; if (udev == NULL) { DPRINTF("udev == NULL\n"); return (USB_ERR_INVAL); } /* Prevent re-enumeration */ do_unlock = usbd_enum_lock(udev); if (udev->curr_config_index == USB_UNCONFIG_INDEX) { /* do nothing - no configuration has been set */ goto done; } /* setup USB attach arguments */ usb_init_attach_arg(udev, &uaa); /* * If the whole USB device is targeted, invoke the USB event * handler(s): */ if (iface_index == USB_IFACE_INDEX_ANY) { if (usb_test_quirk(&uaa, UQ_MSC_DYMO_EJECT) != 0 && usb_dymo_eject(udev, 0) == 0) { /* success, mark the udev as disappearing */ uaa.dev_state = UAA_DEV_EJECTING; } EVENTHANDLER_INVOKE(usb_dev_configured, udev, &uaa); if (uaa.dev_state != UAA_DEV_READY) { /* leave device unconfigured */ usb_unconfigure(udev, 0); goto done; } } /* Check if only one interface should be probed: */ if (iface_index != USB_IFACE_INDEX_ANY) { i = iface_index; j = i + 1; } else { i = 0; j = USB_IFACE_MAX; } /* Do the probe and attach */ for (; i != j; i++) { iface = usbd_get_iface(udev, i); if (iface == NULL) { /* * Looks like the end of the USB * interfaces ! */ DPRINTFN(2, "end of interfaces " "at %u\n", i); break; } if (iface->idesc == NULL) { /* no interface descriptor */ continue; } uaa.iface = iface; uaa.info.bInterfaceClass = iface->idesc->bInterfaceClass; uaa.info.bInterfaceSubClass = iface->idesc->bInterfaceSubClass; uaa.info.bInterfaceProtocol = iface->idesc->bInterfaceProtocol; uaa.info.bIfaceIndex = i; uaa.info.bIfaceNum = iface->idesc->bInterfaceNumber; uaa.driver_info = 0; /* reset driver_info */ DPRINTFN(2, "iclass=%u/%u/%u iindex=%u/%u\n", uaa.info.bInterfaceClass, uaa.info.bInterfaceSubClass, uaa.info.bInterfaceProtocol, uaa.info.bIfaceIndex, uaa.info.bIfaceNum); usb_probe_and_attach_sub(udev, &uaa); /* * Remove the leftover child, if any, to enforce that * a new nomatch devd event is generated for the next * interface if no driver is found: */ if (uaa.temp_dev == NULL) continue; if (device_delete_child(udev->parent_dev, uaa.temp_dev)) DPRINTFN(0, "device delete child failed\n"); uaa.temp_dev = NULL; } done: if (do_unlock) usbd_enum_unlock(udev); return (0); } /*------------------------------------------------------------------------* * usb_suspend_resume_sub * * This function is called when the suspend or resume methods should * be executed on an USB device. *------------------------------------------------------------------------*/ static void usb_suspend_resume_sub(struct usb_device *udev, device_t dev, uint8_t do_suspend) { int err; if (dev == NULL) { return; } if (!device_is_attached(dev)) { return; } if (do_suspend) { err = DEVICE_SUSPEND(dev); } else { err = DEVICE_RESUME(dev); } if (err) { device_printf(dev, "%s failed\n", do_suspend ? "Suspend" : "Resume"); } } /*------------------------------------------------------------------------* * usb_suspend_resume * * The following function will suspend or resume the USB device. * * Returns: * 0: Success * Else: Failure *------------------------------------------------------------------------*/ usb_error_t usb_suspend_resume(struct usb_device *udev, uint8_t do_suspend) { struct usb_interface *iface; uint8_t i; if (udev == NULL) { /* nothing to do */ return (0); } DPRINTFN(4, "udev=%p do_suspend=%d\n", udev, do_suspend); sx_assert(&udev->sr_sx, SA_LOCKED); USB_BUS_LOCK(udev->bus); /* filter the suspend events */ if (udev->flags.peer_suspended == do_suspend) { USB_BUS_UNLOCK(udev->bus); /* nothing to do */ return (0); } udev->flags.peer_suspended = do_suspend; USB_BUS_UNLOCK(udev->bus); /* do the suspend or resume */ for (i = 0; i != USB_IFACE_MAX; i++) { iface = usbd_get_iface(udev, i); if (iface == NULL) { /* looks like the end of the USB interfaces */ break; } usb_suspend_resume_sub(udev, iface->subdev, do_suspend); } return (0); } /*------------------------------------------------------------------------* * usbd_clear_stall_proc * * This function performs generic USB clear stall operations. *------------------------------------------------------------------------*/ static void usbd_clear_stall_proc(struct usb_proc_msg *_pm) { struct usb_udev_msg *pm = (void *)_pm; struct usb_device *udev = pm->udev; /* Change lock */ USB_BUS_UNLOCK(udev->bus); mtx_lock(&udev->device_mtx); /* Start clear stall callback */ usbd_transfer_start(udev->ctrl_xfer[1]); /* Change lock */ mtx_unlock(&udev->device_mtx); USB_BUS_LOCK(udev->bus); } /*------------------------------------------------------------------------* * usb_alloc_device * * This function allocates a new USB device. This function is called * when a new device has been put in the powered state, but not yet in * the addressed state. Get initial descriptor, set the address, get * full descriptor and get strings. * * Return values: * 0: Failure * Else: Success *------------------------------------------------------------------------*/ struct usb_device * usb_alloc_device(device_t parent_dev, struct usb_bus *bus, struct usb_device *parent_hub, uint8_t depth, uint8_t port_index, uint8_t port_no, enum usb_dev_speed speed, enum usb_hc_mode mode) { struct usb_attach_arg uaa; struct usb_device *udev; struct usb_device *adev; struct usb_device *hub; uint8_t *scratch_ptr; usb_error_t err; uint8_t device_index; uint8_t config_index; uint8_t config_quirk; uint8_t set_config_failed; uint8_t do_unlock; DPRINTF("parent_dev=%p, bus=%p, parent_hub=%p, depth=%u, " "port_index=%u, port_no=%u, speed=%u, usb_mode=%u\n", parent_dev, bus, parent_hub, depth, port_index, port_no, speed, mode); /* * Find an unused device index. In USB Host mode this is the * same as the device address. * * Device index zero is not used and device index 1 should * always be the root hub. */ for (device_index = USB_ROOT_HUB_ADDR; (device_index != bus->devices_max) && (bus->devices[device_index] != NULL); device_index++) /* nop */; if (device_index == bus->devices_max) { device_printf(bus->bdev, "No free USB device index for new device\n"); return (NULL); } if (depth > 0x10) { device_printf(bus->bdev, "Invalid device depth\n"); return (NULL); } udev = malloc(sizeof(*udev), M_USB, M_WAITOK | M_ZERO); if (udev == NULL) { return (NULL); } /* initialise our SX-lock */ sx_init_flags(&udev->enum_sx, "USB config SX lock", SX_DUPOK); sx_init_flags(&udev->sr_sx, "USB suspend and resume SX lock", SX_NOWITNESS); cv_init(&udev->ctrlreq_cv, "WCTRL"); cv_init(&udev->ref_cv, "UGONE"); /* initialise our mutex */ mtx_init(&udev->device_mtx, "USB device mutex", NULL, MTX_DEF); /* initialise generic clear stall */ udev->cs_msg[0].hdr.pm_callback = &usbd_clear_stall_proc; udev->cs_msg[0].udev = udev; udev->cs_msg[1].hdr.pm_callback = &usbd_clear_stall_proc; udev->cs_msg[1].udev = udev; /* initialise some USB device fields */ udev->parent_hub = parent_hub; udev->parent_dev = parent_dev; udev->port_index = port_index; udev->port_no = port_no; udev->depth = depth; udev->bus = bus; udev->address = USB_START_ADDR; /* default value */ udev->plugtime = (usb_ticks_t)ticks; /* * We need to force the power mode to "on" because there are plenty * of USB devices out there that do not work very well with * automatic suspend and resume! */ udev->power_mode = usbd_filter_power_mode(udev, USB_POWER_MODE_ON); udev->pwr_save.last_xfer_time = ticks; /* we are not ready yet */ udev->refcount = 1; /* set up default endpoint descriptor */ udev->ctrl_ep_desc.bLength = sizeof(udev->ctrl_ep_desc); udev->ctrl_ep_desc.bDescriptorType = UDESC_ENDPOINT; udev->ctrl_ep_desc.bEndpointAddress = USB_CONTROL_ENDPOINT; udev->ctrl_ep_desc.bmAttributes = UE_CONTROL; udev->ctrl_ep_desc.wMaxPacketSize[0] = USB_MAX_IPACKET; udev->ctrl_ep_desc.wMaxPacketSize[1] = 0; udev->ctrl_ep_desc.bInterval = 0; /* set up default endpoint companion descriptor */ udev->ctrl_ep_comp_desc.bLength = sizeof(udev->ctrl_ep_comp_desc); udev->ctrl_ep_comp_desc.bDescriptorType = UDESC_ENDPOINT_SS_COMP; udev->ddesc.bMaxPacketSize = USB_MAX_IPACKET; udev->speed = speed; udev->flags.usb_mode = mode; /* search for our High Speed USB HUB, if any */ adev = udev; hub = udev->parent_hub; while (hub) { if (hub->speed == USB_SPEED_HIGH) { udev->hs_hub_addr = hub->address; udev->parent_hs_hub = hub; udev->hs_port_no = adev->port_no; break; } adev = hub; hub = hub->parent_hub; } /* init the default endpoint */ usb_init_endpoint(udev, 0, &udev->ctrl_ep_desc, &udev->ctrl_ep_comp_desc, &udev->ctrl_ep); /* set device index */ udev->device_index = device_index; #if USB_HAVE_UGEN /* Create ugen name */ snprintf(udev->ugen_name, sizeof(udev->ugen_name), USB_GENERIC_NAME "%u.%u", device_get_unit(bus->bdev), device_index); LIST_INIT(&udev->pd_list); /* Create the control endpoint device */ udev->ctrl_dev = usb_make_dev(udev, NULL, 0, 0, FREAD|FWRITE, UID_ROOT, GID_OPERATOR, 0600); /* Create a link from /dev/ugenX.X to the default endpoint */ if (udev->ctrl_dev != NULL) make_dev_alias(udev->ctrl_dev->cdev, "%s", udev->ugen_name); #endif /* Initialise device */ if (bus->methods->device_init != NULL) { err = (bus->methods->device_init) (udev); if (err != 0) { DPRINTFN(0, "device init %d failed " "(%s, ignored)\n", device_index, usbd_errstr(err)); goto done; } } /* set powered device state after device init is complete */ usb_set_device_state(udev, USB_STATE_POWERED); if (udev->flags.usb_mode == USB_MODE_HOST) { err = usbd_req_set_address(udev, NULL, device_index); /* * This is the new USB device address from now on, if * the set address request didn't set it already. */ if (udev->address == USB_START_ADDR) udev->address = device_index; /* * We ignore any set-address errors, hence there are * buggy USB devices out there that actually receive * the SETUP PID, but manage to set the address before * the STATUS stage is ACK'ed. If the device responds * to the subsequent get-descriptor at the new * address, then we know that the set-address command * was successful. */ if (err) { DPRINTFN(0, "set address %d failed " "(%s, ignored)\n", udev->address, usbd_errstr(err)); } } else { /* We are not self powered */ udev->flags.self_powered = 0; /* Set unconfigured state */ udev->curr_config_no = USB_UNCONFIG_NO; udev->curr_config_index = USB_UNCONFIG_INDEX; /* Setup USB descriptors */ err = (usb_temp_setup_by_index_p) (udev, usb_template); if (err) { DPRINTFN(0, "setting up USB template failed maybe the USB " "template module has not been loaded\n"); goto done; } } usb_set_device_state(udev, USB_STATE_ADDRESSED); /* setup the device descriptor and the initial "wMaxPacketSize" */ err = usbd_setup_device_desc(udev, NULL); if (err != 0) { /* try to enumerate two more times */ err = usbd_req_re_enumerate(udev, NULL); if (err != 0) { err = usbd_req_re_enumerate(udev, NULL); if (err != 0) { goto done; } } } /* * Setup temporary USB attach args so that we can figure out some * basic quirks for this device. */ usb_init_attach_arg(udev, &uaa); if (usb_test_quirk(&uaa, UQ_BUS_POWERED)) { udev->flags.uq_bus_powered = 1; } if (usb_test_quirk(&uaa, UQ_NO_STRINGS)) { udev->flags.no_strings = 1; } /* * Workaround for buggy USB devices. * * It appears that some string-less USB chips will crash and * disappear if any attempts are made to read any string * descriptors. * * Try to detect such chips by checking the strings in the USB * device descriptor. If no strings are present there we * simply disable all USB strings. */ /* Protect scratch area */ do_unlock = usbd_enum_lock(udev); scratch_ptr = udev->scratch.data; - if (udev->ddesc.iManufacturer || + if (udev->flags.no_strings) { + err = USB_ERR_INVAL; + } else if (udev->ddesc.iManufacturer || udev->ddesc.iProduct || udev->ddesc.iSerialNumber) { /* read out the language ID string */ err = usbd_req_get_string_desc(udev, NULL, (char *)scratch_ptr, 4, 0, USB_LANGUAGE_TABLE); } else { err = USB_ERR_INVAL; } if (err || (scratch_ptr[0] < 4)) { udev->flags.no_strings = 1; } else { uint16_t langid; uint16_t pref; uint16_t mask; uint8_t x; /* load preferred value and mask */ pref = usb_lang_id; mask = usb_lang_mask; /* align length correctly */ scratch_ptr[0] &= ~1U; /* fix compiler warning */ langid = 0; /* search for preferred language */ for (x = 2; (x < scratch_ptr[0]); x += 2) { langid = UGETW(scratch_ptr + x); if ((langid & mask) == pref) break; } if (x >= scratch_ptr[0]) { /* pick the first language as the default */ DPRINTFN(1, "Using first language\n"); langid = UGETW(scratch_ptr + 2); } DPRINTFN(1, "Language selected: 0x%04x\n", langid); udev->langid = langid; } if (do_unlock) usbd_enum_unlock(udev); /* assume 100mA bus powered for now. Changed when configured. */ udev->power = USB_MIN_POWER; /* fetch the vendor and product strings from the device */ usbd_set_device_strings(udev); if (udev->flags.usb_mode == USB_MODE_DEVICE) { /* USB device mode setup is complete */ err = 0; goto config_done; } /* * Most USB devices should attach to config index 0 by * default */ if (usb_test_quirk(&uaa, UQ_CFG_INDEX_0)) { config_index = 0; config_quirk = 1; } else if (usb_test_quirk(&uaa, UQ_CFG_INDEX_1)) { config_index = 1; config_quirk = 1; } else if (usb_test_quirk(&uaa, UQ_CFG_INDEX_2)) { config_index = 2; config_quirk = 1; } else if (usb_test_quirk(&uaa, UQ_CFG_INDEX_3)) { config_index = 3; config_quirk = 1; } else if (usb_test_quirk(&uaa, UQ_CFG_INDEX_4)) { config_index = 4; config_quirk = 1; } else { config_index = 0; config_quirk = 0; } set_config_failed = 0; repeat_set_config: DPRINTF("setting config %u\n", config_index); /* get the USB device configured */ err = usbd_set_config_index(udev, config_index); if (err) { if (udev->ddesc.bNumConfigurations != 0) { if (!set_config_failed) { set_config_failed = 1; /* XXX try to re-enumerate the device */ err = usbd_req_re_enumerate(udev, NULL); if (err == 0) goto repeat_set_config; } DPRINTFN(0, "Failure selecting configuration index %u:" "%s, port %u, addr %u (ignored)\n", config_index, usbd_errstr(err), udev->port_no, udev->address); } /* * Some USB devices do not have any configurations. Ignore any * set config failures! */ err = 0; goto config_done; } if (!config_quirk && config_index + 1 < udev->ddesc.bNumConfigurations) { if ((udev->cdesc->bNumInterface < 2) && usbd_get_no_descriptors(udev->cdesc, UDESC_ENDPOINT) == 0) { DPRINTFN(0, "Found no endpoints, trying next config\n"); config_index++; goto repeat_set_config; } #if USB_HAVE_MSCTEST if (config_index == 0) { /* * Try to figure out if we have an * auto-install disk there: */ if (usb_iface_is_cdrom(udev, 0)) { DPRINTFN(0, "Found possible auto-install " "disk (trying next config)\n"); config_index++; goto repeat_set_config; } } #endif } #if USB_HAVE_MSCTEST if (set_config_failed == 0 && config_index == 0 && usb_test_quirk(&uaa, UQ_MSC_NO_SYNC_CACHE) == 0 && usb_test_quirk(&uaa, UQ_MSC_NO_GETMAXLUN) == 0) { /* * Try to figure out if there are any MSC quirks we * should apply automatically: */ err = usb_msc_auto_quirk(udev, 0); if (err != 0) { set_config_failed = 1; goto repeat_set_config; } } #endif config_done: DPRINTF("new dev (addr %d), udev=%p, parent_hub=%p\n", udev->address, udev, udev->parent_hub); /* register our device - we are ready */ usb_bus_port_set_device(bus, parent_hub ? parent_hub->hub->ports + port_index : NULL, udev, device_index); #if USB_HAVE_UGEN /* Symlink the ugen device name */ udev->ugen_symlink = usb_alloc_symlink(udev->ugen_name); /* Announce device */ printf("%s: <%s> at %s\n", udev->ugen_name, usb_get_manufacturer(udev), device_get_nameunit(udev->bus->bdev)); #endif #if USB_HAVE_DEVCTL usb_notify_addq("ATTACH", udev); #endif done: if (err) { /* * Free USB device and all subdevices, if any. */ usb_free_device(udev, 0); udev = NULL; } return (udev); } #if USB_HAVE_UGEN struct usb_fs_privdata * usb_make_dev(struct usb_device *udev, const char *devname, int ep, int fi, int rwmode, uid_t uid, gid_t gid, int mode) { struct usb_fs_privdata* pd; struct make_dev_args args; char buffer[32]; /* Store information to locate ourselves again later */ pd = malloc(sizeof(struct usb_fs_privdata), M_USBDEV, M_WAITOK | M_ZERO); pd->bus_index = device_get_unit(udev->bus->bdev); pd->dev_index = udev->device_index; pd->ep_addr = ep; pd->fifo_index = fi; pd->mode = rwmode; /* Now, create the device itself */ if (devname == NULL) { devname = buffer; snprintf(buffer, sizeof(buffer), USB_DEVICE_DIR "/%u.%u.%u", pd->bus_index, pd->dev_index, pd->ep_addr); } /* Setup arguments for make_dev_s() */ make_dev_args_init(&args); args.mda_devsw = &usb_devsw; args.mda_uid = uid; args.mda_gid = gid; args.mda_mode = mode; args.mda_si_drv1 = pd; if (make_dev_s(&args, &pd->cdev, "%s", devname) != 0) { DPRINTFN(0, "Failed to create device %s\n", devname); free(pd, M_USBDEV); return (NULL); } return (pd); } void usb_destroy_dev_sync(struct usb_fs_privdata *pd) { DPRINTFN(1, "Destroying device at ugen%d.%d\n", pd->bus_index, pd->dev_index); /* * Destroy character device synchronously. After this * all system calls are returned. Can block. */ destroy_dev(pd->cdev); free(pd, M_USBDEV); } void usb_destroy_dev(struct usb_fs_privdata *pd) { struct usb_bus *bus; if (pd == NULL) return; mtx_lock(&usb_ref_lock); bus = devclass_get_softc(usb_devclass_ptr, pd->bus_index); mtx_unlock(&usb_ref_lock); if (bus == NULL) { usb_destroy_dev_sync(pd); return; } /* make sure we can re-use the device name */ delist_dev(pd->cdev); USB_BUS_LOCK(bus); LIST_INSERT_HEAD(&bus->pd_cleanup_list, pd, pd_next); /* get cleanup going */ usb_proc_msignal(USB_BUS_EXPLORE_PROC(bus), &bus->cleanup_msg[0], &bus->cleanup_msg[1]); USB_BUS_UNLOCK(bus); } static void usb_cdev_create(struct usb_device *udev) { struct usb_config_descriptor *cd; struct usb_endpoint_descriptor *ed; struct usb_descriptor *desc; struct usb_fs_privdata* pd; int inmode, outmode, inmask, outmask, mode; uint8_t ep; KASSERT(LIST_FIRST(&udev->pd_list) == NULL, ("stale cdev entries")); DPRINTFN(2, "Creating device nodes\n"); if (usbd_get_mode(udev) == USB_MODE_DEVICE) { inmode = FWRITE; outmode = FREAD; } else { /* USB_MODE_HOST */ inmode = FREAD; outmode = FWRITE; } inmask = 0; outmask = 0; desc = NULL; /* * Collect all used endpoint numbers instead of just * generating 16 static endpoints. */ cd = usbd_get_config_descriptor(udev); while ((desc = usb_desc_foreach(cd, desc))) { /* filter out all endpoint descriptors */ if ((desc->bDescriptorType == UDESC_ENDPOINT) && (desc->bLength >= sizeof(*ed))) { ed = (struct usb_endpoint_descriptor *)desc; /* update masks */ ep = ed->bEndpointAddress; if (UE_GET_DIR(ep) == UE_DIR_OUT) outmask |= 1 << UE_GET_ADDR(ep); else inmask |= 1 << UE_GET_ADDR(ep); } } /* Create all available endpoints except EP0 */ for (ep = 1; ep < 16; ep++) { mode = (inmask & (1 << ep)) ? inmode : 0; mode |= (outmask & (1 << ep)) ? outmode : 0; if (mode == 0) continue; /* no IN or OUT endpoint */ pd = usb_make_dev(udev, NULL, ep, 0, mode, UID_ROOT, GID_OPERATOR, 0600); if (pd != NULL) LIST_INSERT_HEAD(&udev->pd_list, pd, pd_next); } } static void usb_cdev_free(struct usb_device *udev) { struct usb_fs_privdata* pd; DPRINTFN(2, "Freeing device nodes\n"); while ((pd = LIST_FIRST(&udev->pd_list)) != NULL) { KASSERT(pd->cdev->si_drv1 == pd, ("privdata corrupt")); LIST_REMOVE(pd, pd_next); usb_destroy_dev(pd); } } #endif /*------------------------------------------------------------------------* * usb_free_device * * This function is NULL safe and will free an USB device and its * children devices, if any. * * Flag values: Reserved, set to zero. *------------------------------------------------------------------------*/ void usb_free_device(struct usb_device *udev, uint8_t flag) { struct usb_bus *bus; if (udev == NULL) return; /* already freed */ DPRINTFN(4, "udev=%p port=%d\n", udev, udev->port_no); bus = udev->bus; /* set DETACHED state to prevent any further references */ usb_set_device_state(udev, USB_STATE_DETACHED); #if USB_HAVE_DEVCTL usb_notify_addq("DETACH", udev); #endif #if USB_HAVE_UGEN if (!rebooting) { printf("%s: <%s> at %s (disconnected)\n", udev->ugen_name, usb_get_manufacturer(udev), device_get_nameunit(bus->bdev)); } /* Destroy UGEN symlink, if any */ if (udev->ugen_symlink) { usb_free_symlink(udev->ugen_symlink); udev->ugen_symlink = NULL; } usb_destroy_dev(udev->ctrl_dev); #endif if (udev->flags.usb_mode == USB_MODE_DEVICE) { /* stop receiving any control transfers (Device Side Mode) */ usbd_transfer_unsetup(udev->ctrl_xfer, USB_CTRL_XFER_MAX); } /* the following will get the device unconfigured in software */ usb_unconfigure(udev, USB_UNCFG_FLAG_FREE_EP0); /* final device unregister after all character devices are closed */ usb_bus_port_set_device(bus, udev->parent_hub ? udev->parent_hub->hub->ports + udev->port_index : NULL, NULL, USB_ROOT_HUB_ADDR); /* unsetup any leftover default USB transfers */ usbd_transfer_unsetup(udev->ctrl_xfer, USB_CTRL_XFER_MAX); /* template unsetup, if any */ (usb_temp_unsetup_p) (udev); /* * Make sure that our clear-stall messages are not queued * anywhere: */ USB_BUS_LOCK(udev->bus); usb_proc_mwait(USB_BUS_CS_PROC(udev->bus), &udev->cs_msg[0], &udev->cs_msg[1]); USB_BUS_UNLOCK(udev->bus); /* wait for all references to go away */ usb_wait_pending_refs(udev); sx_destroy(&udev->enum_sx); sx_destroy(&udev->sr_sx); cv_destroy(&udev->ctrlreq_cv); cv_destroy(&udev->ref_cv); mtx_destroy(&udev->device_mtx); #if USB_HAVE_UGEN KASSERT(LIST_FIRST(&udev->pd_list) == NULL, ("leaked cdev entries")); #endif /* Uninitialise device */ if (bus->methods->device_uninit != NULL) (bus->methods->device_uninit) (udev); /* free device */ free(udev->serial, M_USB); free(udev->manufacturer, M_USB); free(udev->product, M_USB); free(udev, M_USB); } /*------------------------------------------------------------------------* * usbd_get_iface * * This function is the safe way to get the USB interface structure * pointer by interface index. * * Return values: * NULL: Interface not present. * Else: Pointer to USB interface structure. *------------------------------------------------------------------------*/ struct usb_interface * usbd_get_iface(struct usb_device *udev, uint8_t iface_index) { struct usb_interface *iface = udev->ifaces + iface_index; if (iface_index >= udev->ifaces_max) return (NULL); return (iface); } /*------------------------------------------------------------------------* * usbd_find_descriptor * * This function will lookup the first descriptor that matches the * criteria given by the arguments "type" and "subtype". Descriptors * will only be searched within the interface having the index * "iface_index". If the "id" argument points to an USB descriptor, * it will be skipped before the search is started. This allows * searching for multiple descriptors using the same criteria. Else * the search is started after the interface descriptor. * * Return values: * NULL: End of descriptors * Else: A descriptor matching the criteria *------------------------------------------------------------------------*/ void * usbd_find_descriptor(struct usb_device *udev, void *id, uint8_t iface_index, uint8_t type, uint8_t type_mask, uint8_t subtype, uint8_t subtype_mask) { struct usb_descriptor *desc; struct usb_config_descriptor *cd; struct usb_interface *iface; cd = usbd_get_config_descriptor(udev); if (cd == NULL) { return (NULL); } if (id == NULL) { iface = usbd_get_iface(udev, iface_index); if (iface == NULL) { return (NULL); } id = usbd_get_interface_descriptor(iface); if (id == NULL) { return (NULL); } } desc = (void *)id; while ((desc = usb_desc_foreach(cd, desc))) { if (desc->bDescriptorType == UDESC_INTERFACE) { break; } if (((desc->bDescriptorType & type_mask) == type) && ((desc->bDescriptorSubtype & subtype_mask) == subtype)) { return (desc); } } return (NULL); } /*------------------------------------------------------------------------* * usb_devinfo * * This function will dump information from the device descriptor * belonging to the USB device pointed to by "udev", to the string * pointed to by "dst_ptr" having a maximum length of "dst_len" bytes * including the terminating zero. *------------------------------------------------------------------------*/ void usb_devinfo(struct usb_device *udev, char *dst_ptr, uint16_t dst_len) { struct usb_device_descriptor *udd = &udev->ddesc; uint16_t bcdDevice; uint16_t bcdUSB; bcdUSB = UGETW(udd->bcdUSB); bcdDevice = UGETW(udd->bcdDevice); if (udd->bDeviceClass != 0xFF) { snprintf(dst_ptr, dst_len, "%s %s, class %d/%d, rev %x.%02x/" "%x.%02x, addr %d", usb_get_manufacturer(udev), usb_get_product(udev), udd->bDeviceClass, udd->bDeviceSubClass, (bcdUSB >> 8), bcdUSB & 0xFF, (bcdDevice >> 8), bcdDevice & 0xFF, udev->address); } else { snprintf(dst_ptr, dst_len, "%s %s, rev %x.%02x/" "%x.%02x, addr %d", usb_get_manufacturer(udev), usb_get_product(udev), (bcdUSB >> 8), bcdUSB & 0xFF, (bcdDevice >> 8), bcdDevice & 0xFF, udev->address); } } #ifdef USB_VERBOSE /* * Descriptions of of known vendors and devices ("products"). */ struct usb_knowndev { uint16_t vendor; uint16_t product; uint32_t flags; const char *vendorname; const char *productname; }; #define USB_KNOWNDEV_NOPROD 0x01 /* match on vendor only */ #include "usbdevs.h" #include "usbdevs_data.h" #endif /* USB_VERBOSE */ static void usbd_set_device_strings(struct usb_device *udev) { struct usb_device_descriptor *udd = &udev->ddesc; #ifdef USB_VERBOSE const struct usb_knowndev *kdp; #endif char *temp_ptr; size_t temp_size; uint16_t vendor_id; uint16_t product_id; uint8_t do_unlock; /* Protect scratch area */ do_unlock = usbd_enum_lock(udev); temp_ptr = (char *)udev->scratch.data; temp_size = sizeof(udev->scratch.data); vendor_id = UGETW(udd->idVendor); product_id = UGETW(udd->idProduct); /* get serial number string */ usbd_req_get_string_any(udev, NULL, temp_ptr, temp_size, udev->ddesc.iSerialNumber); udev->serial = strdup(temp_ptr, M_USB); /* get manufacturer string */ usbd_req_get_string_any(udev, NULL, temp_ptr, temp_size, udev->ddesc.iManufacturer); usb_trim_spaces(temp_ptr); if (temp_ptr[0] != '\0') udev->manufacturer = strdup(temp_ptr, M_USB); /* get product string */ usbd_req_get_string_any(udev, NULL, temp_ptr, temp_size, udev->ddesc.iProduct); usb_trim_spaces(temp_ptr); if (temp_ptr[0] != '\0') udev->product = strdup(temp_ptr, M_USB); #ifdef USB_VERBOSE if (udev->manufacturer == NULL || udev->product == NULL) { for (kdp = usb_knowndevs; kdp->vendorname != NULL; kdp++) { if (kdp->vendor == vendor_id && (kdp->product == product_id || (kdp->flags & USB_KNOWNDEV_NOPROD) != 0)) break; } if (kdp->vendorname != NULL) { /* XXX should use pointer to knowndevs string */ if (udev->manufacturer == NULL) { udev->manufacturer = strdup(kdp->vendorname, M_USB); } if (udev->product == NULL && (kdp->flags & USB_KNOWNDEV_NOPROD) == 0) { udev->product = strdup(kdp->productname, M_USB); } } } #endif /* Provide default strings if none were found */ if (udev->manufacturer == NULL) { snprintf(temp_ptr, temp_size, "vendor 0x%04x", vendor_id); udev->manufacturer = strdup(temp_ptr, M_USB); } if (udev->product == NULL) { snprintf(temp_ptr, temp_size, "product 0x%04x", product_id); udev->product = strdup(temp_ptr, M_USB); } if (do_unlock) usbd_enum_unlock(udev); } /* * Returns: * See: USB_MODE_XXX */ enum usb_hc_mode usbd_get_mode(struct usb_device *udev) { return (udev->flags.usb_mode); } /* * Returns: * See: USB_SPEED_XXX */ enum usb_dev_speed usbd_get_speed(struct usb_device *udev) { return (udev->speed); } uint32_t usbd_get_isoc_fps(struct usb_device *udev) { ; /* indent fix */ switch (udev->speed) { case USB_SPEED_LOW: case USB_SPEED_FULL: return (1000); default: return (8000); } } struct usb_device_descriptor * usbd_get_device_descriptor(struct usb_device *udev) { if (udev == NULL) return (NULL); /* be NULL safe */ return (&udev->ddesc); } struct usb_config_descriptor * usbd_get_config_descriptor(struct usb_device *udev) { if (udev == NULL) return (NULL); /* be NULL safe */ return (udev->cdesc); } /*------------------------------------------------------------------------* * usb_test_quirk - test a device for a given quirk * * Return values: * 0: The USB device does not have the given quirk. * Else: The USB device has the given quirk. *------------------------------------------------------------------------*/ uint8_t usb_test_quirk(const struct usb_attach_arg *uaa, uint16_t quirk) { uint8_t found; uint8_t x; if (quirk == UQ_NONE) return (0); /* search the automatic per device quirks first */ for (x = 0; x != USB_MAX_AUTO_QUIRK; x++) { if (uaa->device->autoQuirk[x] == quirk) return (1); } /* search global quirk table, if any */ found = (usb_test_quirk_p) (&uaa->info, quirk); return (found); } struct usb_interface_descriptor * usbd_get_interface_descriptor(struct usb_interface *iface) { if (iface == NULL) return (NULL); /* be NULL safe */ return (iface->idesc); } uint8_t usbd_get_interface_altindex(struct usb_interface *iface) { return (iface->alt_index); } uint8_t usbd_get_bus_index(struct usb_device *udev) { return ((uint8_t)device_get_unit(udev->bus->bdev)); } uint8_t usbd_get_device_index(struct usb_device *udev) { return (udev->device_index); } #if USB_HAVE_DEVCTL static void usb_notify_addq(const char *type, struct usb_device *udev) { struct usb_interface *iface; struct sbuf *sb; int i; /* announce the device */ sb = sbuf_new_auto(); sbuf_printf(sb, #if USB_HAVE_UGEN "ugen=%s " "cdev=%s " #endif "vendor=0x%04x " "product=0x%04x " "devclass=0x%02x " "devsubclass=0x%02x " "sernum=\"%s\" " "release=0x%04x " "mode=%s " "port=%u " #if USB_HAVE_UGEN "parent=%s" #endif "", #if USB_HAVE_UGEN udev->ugen_name, udev->ugen_name, #endif UGETW(udev->ddesc.idVendor), UGETW(udev->ddesc.idProduct), udev->ddesc.bDeviceClass, udev->ddesc.bDeviceSubClass, usb_get_serial(udev), UGETW(udev->ddesc.bcdDevice), (udev->flags.usb_mode == USB_MODE_HOST) ? "host" : "device", udev->port_no #if USB_HAVE_UGEN , udev->parent_hub != NULL ? udev->parent_hub->ugen_name : device_get_nameunit(device_get_parent(udev->bus->bdev)) #endif ); sbuf_finish(sb); devctl_notify("USB", "DEVICE", type, sbuf_data(sb)); sbuf_delete(sb); /* announce each interface */ for (i = 0; i < USB_IFACE_MAX; i++) { iface = usbd_get_iface(udev, i); if (iface == NULL) break; /* end of interfaces */ if (iface->idesc == NULL) continue; /* no interface descriptor */ sb = sbuf_new_auto(); sbuf_printf(sb, #if USB_HAVE_UGEN "ugen=%s " "cdev=%s " #endif "vendor=0x%04x " "product=0x%04x " "devclass=0x%02x " "devsubclass=0x%02x " "sernum=\"%s\" " "release=0x%04x " "mode=%s " "interface=%d " "endpoints=%d " "intclass=0x%02x " "intsubclass=0x%02x " "intprotocol=0x%02x", #if USB_HAVE_UGEN udev->ugen_name, udev->ugen_name, #endif UGETW(udev->ddesc.idVendor), UGETW(udev->ddesc.idProduct), udev->ddesc.bDeviceClass, udev->ddesc.bDeviceSubClass, usb_get_serial(udev), UGETW(udev->ddesc.bcdDevice), (udev->flags.usb_mode == USB_MODE_HOST) ? "host" : "device", iface->idesc->bInterfaceNumber, iface->idesc->bNumEndpoints, iface->idesc->bInterfaceClass, iface->idesc->bInterfaceSubClass, iface->idesc->bInterfaceProtocol); sbuf_finish(sb); devctl_notify("USB", "INTERFACE", type, sbuf_data(sb)); sbuf_delete(sb); } } #endif #if USB_HAVE_UGEN /*------------------------------------------------------------------------* * usb_fifo_free_wrap * * This function will free the FIFOs. * * Description of "flag" argument: If the USB_UNCFG_FLAG_FREE_EP0 flag * is set and "iface_index" is set to "USB_IFACE_INDEX_ANY", we free * all FIFOs. If the USB_UNCFG_FLAG_FREE_EP0 flag is not set and * "iface_index" is set to "USB_IFACE_INDEX_ANY", we free all non * control endpoint FIFOs. If "iface_index" is not set to * "USB_IFACE_INDEX_ANY" the flag has no effect. *------------------------------------------------------------------------*/ static void usb_fifo_free_wrap(struct usb_device *udev, uint8_t iface_index, uint8_t flag) { struct usb_fifo *f; uint16_t i; /* * Free any USB FIFOs on the given interface: */ for (i = 0; i != USB_FIFO_MAX; i++) { f = udev->fifo[i]; if (f == NULL) { continue; } /* Check if the interface index matches */ if (iface_index == f->iface_index) { if (f->methods != &usb_ugen_methods) { /* * Don't free any non-generic FIFOs in * this case. */ continue; } if ((f->dev_ep_index == 0) && (f->fs_xfer == NULL)) { /* no need to free this FIFO */ continue; } } else if (iface_index == USB_IFACE_INDEX_ANY) { if ((f->methods == &usb_ugen_methods) && (f->dev_ep_index == 0) && (!(flag & USB_UNCFG_FLAG_FREE_EP0)) && (f->fs_xfer == NULL)) { /* no need to free this FIFO */ continue; } } else { /* no need to free this FIFO */ continue; } /* free this FIFO */ usb_fifo_free(f); } } #endif /*------------------------------------------------------------------------* * usb_peer_can_wakeup * * Return values: * 0: Peer cannot do resume signalling. * Else: Peer can do resume signalling. *------------------------------------------------------------------------*/ uint8_t usb_peer_can_wakeup(struct usb_device *udev) { const struct usb_config_descriptor *cdp; cdp = udev->cdesc; if ((cdp != NULL) && (udev->flags.usb_mode == USB_MODE_HOST)) { return (cdp->bmAttributes & UC_REMOTE_WAKEUP); } return (0); /* not supported */ } void usb_set_device_state(struct usb_device *udev, enum usb_dev_state state) { KASSERT(state < USB_STATE_MAX, ("invalid udev state")); DPRINTF("udev %p state %s -> %s\n", udev, usb_statestr(udev->state), usb_statestr(state)); #if USB_HAVE_UGEN mtx_lock(&usb_ref_lock); #endif udev->state = state; #if USB_HAVE_UGEN mtx_unlock(&usb_ref_lock); #endif if (udev->bus->methods->device_state_change != NULL) (udev->bus->methods->device_state_change) (udev); } enum usb_dev_state usb_get_device_state(struct usb_device *udev) { if (udev == NULL) return (USB_STATE_DETACHED); return (udev->state); } uint8_t usbd_device_attached(struct usb_device *udev) { return (udev->state > USB_STATE_DETACHED); } /* * The following function locks enumerating the given USB device. If * the lock is already grabbed this function returns zero. Else a * non-zero value is returned. */ uint8_t usbd_enum_lock(struct usb_device *udev) { if (sx_xlocked(&udev->enum_sx)) return (0); sx_xlock(&udev->enum_sx); sx_xlock(&udev->sr_sx); /* * NEWBUS LOCK NOTE: We should check if any parent SX locks * are locked before locking Giant. Else the lock can be * locked multiple times. */ mtx_lock(&Giant); return (1); } /* The following function unlocks enumerating the given USB device. */ void usbd_enum_unlock(struct usb_device *udev) { mtx_unlock(&Giant); sx_xunlock(&udev->enum_sx); sx_xunlock(&udev->sr_sx); } /* The following function locks suspend and resume. */ void usbd_sr_lock(struct usb_device *udev) { sx_xlock(&udev->sr_sx); /* * NEWBUS LOCK NOTE: We should check if any parent SX locks * are locked before locking Giant. Else the lock can be * locked multiple times. */ mtx_lock(&Giant); } /* The following function unlocks suspend and resume. */ void usbd_sr_unlock(struct usb_device *udev) { mtx_unlock(&Giant); sx_xunlock(&udev->sr_sx); } /* * The following function checks the enumerating lock for the given * USB device. */ uint8_t usbd_enum_is_locked(struct usb_device *udev) { return (sx_xlocked(&udev->enum_sx)); } /* * The following function is used to set the per-interface specific * plug and play information. The string referred to by the pnpinfo * argument can safely be freed after calling this function. The * pnpinfo of an interface will be reset at device detach or when * passing a NULL argument to this function. This function * returns zero on success, else a USB_ERR_XXX failure code. */ usb_error_t usbd_set_pnpinfo(struct usb_device *udev, uint8_t iface_index, const char *pnpinfo) { struct usb_interface *iface; iface = usbd_get_iface(udev, iface_index); if (iface == NULL) return (USB_ERR_INVAL); if (iface->pnpinfo != NULL) { free(iface->pnpinfo, M_USBDEV); iface->pnpinfo = NULL; } if (pnpinfo == NULL || pnpinfo[0] == 0) return (0); /* success */ iface->pnpinfo = strdup(pnpinfo, M_USBDEV); if (iface->pnpinfo == NULL) return (USB_ERR_NOMEM); return (0); /* success */ } usb_error_t usbd_add_dynamic_quirk(struct usb_device *udev, uint16_t quirk) { uint8_t x; for (x = 0; x != USB_MAX_AUTO_QUIRK; x++) { if (udev->autoQuirk[x] == 0 || udev->autoQuirk[x] == quirk) { udev->autoQuirk[x] = quirk; return (0); /* success */ } } return (USB_ERR_NOMEM); } /* * The following function is used to select the endpoint mode. It * should not be called outside enumeration context. */ usb_error_t usbd_set_endpoint_mode(struct usb_device *udev, struct usb_endpoint *ep, uint8_t ep_mode) { usb_error_t error; uint8_t do_unlock; /* Prevent re-enumeration */ do_unlock = usbd_enum_lock(udev); if (udev->bus->methods->set_endpoint_mode != NULL) { error = (udev->bus->methods->set_endpoint_mode) ( udev, ep, ep_mode); } else if (ep_mode != USB_EP_MODE_DEFAULT) { error = USB_ERR_INVAL; } else { error = 0; } /* only set new mode regardless of error */ ep->ep_mode = ep_mode; if (do_unlock) usbd_enum_unlock(udev); return (error); } uint8_t usbd_get_endpoint_mode(struct usb_device *udev, struct usb_endpoint *ep) { return (ep->ep_mode); } Index: user/ngie/release-pkg-fix-tests/sys/dev/wi/if_wi.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/dev/wi/if_wi.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/dev/wi/if_wi.c (revision 299091) @@ -1,2054 +1,2054 @@ /*- * Copyright (c) 1997, 1998, 1999 * Bill Paul . All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. All advertising materials mentioning features or use of this software * must display the following acknowledgement: * This product includes software developed by Bill Paul. * 4. Neither the name of the author nor the names of any co-contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY Bill Paul AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL Bill Paul OR THE VOICES IN HIS HEAD * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF * THE POSSIBILITY OF SUCH DAMAGE. */ /* * Lucent WaveLAN/IEEE 802.11 PCMCIA driver. * * Original FreeBSD driver written by Bill Paul * Electrical Engineering Department * Columbia University, New York City */ /* * The WaveLAN/IEEE adapter is the second generation of the WaveLAN * from Lucent. Unlike the older cards, the new ones are programmed * entirely via a firmware-driven controller called the Hermes. * Unfortunately, Lucent will not release the Hermes programming manual * without an NDA (if at all). What they do release is an API library * called the HCF (Hardware Control Functions) which is supposed to * do the device-specific operations of a device driver for you. The * publicly available version of the HCF library (the 'HCF Light') is * a) extremely gross, b) lacks certain features, particularly support * for 802.11 frames, and c) is contaminated by the GNU Public License. * * This driver does not use the HCF or HCF Light at all. Instead, it * programs the Hermes controller directly, using information gleaned * from the HCF Light code and corresponding documentation. * * This driver supports the ISA, PCMCIA and PCI versions of the Lucent * WaveLan cards (based on the Hermes chipset), as well as the newer * Prism 2 chipsets with firmware from Intersil and Symbol. */ #include __FBSDID("$FreeBSD$"); #include "opt_wlan.h" #define WI_HERMES_STATS_WAR /* Work around stats counter bug. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include static struct ieee80211vap *wi_vap_create(struct ieee80211com *, const char [IFNAMSIZ], int, enum ieee80211_opmode, int, const uint8_t [IEEE80211_ADDR_LEN], const uint8_t [IEEE80211_ADDR_LEN]); static void wi_vap_delete(struct ieee80211vap *vap); static int wi_transmit(struct ieee80211com *, struct mbuf *); static void wi_start(struct wi_softc *); static int wi_start_tx(struct wi_softc *, struct wi_frame *, struct mbuf *); static int wi_raw_xmit(struct ieee80211_node *, struct mbuf *, const struct ieee80211_bpf_params *); static int wi_newstate_sta(struct ieee80211vap *, enum ieee80211_state, int); static int wi_newstate_hostap(struct ieee80211vap *, enum ieee80211_state, int); static void wi_recv_mgmt(struct ieee80211_node *ni, struct mbuf *m, int subtype, const struct ieee80211_rx_stats *rxs, int rssi, int nf); static int wi_reset(struct wi_softc *); static void wi_watchdog(void *); static void wi_parent(struct ieee80211com *); static void wi_media_status(struct ifnet *, struct ifmediareq *); static void wi_rx_intr(struct wi_softc *); static void wi_tx_intr(struct wi_softc *); static void wi_tx_ex_intr(struct wi_softc *); static void wi_info_intr(struct wi_softc *); static int wi_write_txrate(struct wi_softc *, struct ieee80211vap *); static int wi_write_wep(struct wi_softc *, struct ieee80211vap *); static int wi_write_multi(struct wi_softc *); static void wi_update_mcast(struct ieee80211com *); static void wi_update_promisc(struct ieee80211com *); static int wi_alloc_fid(struct wi_softc *, int, int *); static void wi_read_nicid(struct wi_softc *); static int wi_write_ssid(struct wi_softc *, int, u_int8_t *, int); static int wi_cmd(struct wi_softc *, int, int, int, int); static int wi_seek_bap(struct wi_softc *, int, int); static int wi_read_bap(struct wi_softc *, int, int, void *, int); static int wi_write_bap(struct wi_softc *, int, int, const void *, int); static int wi_mwrite_bap(struct wi_softc *, int, int, struct mbuf *, int); static int wi_read_rid(struct wi_softc *, int, void *, int *); static int wi_write_rid(struct wi_softc *, int, const void *, int); static int wi_write_appie(struct wi_softc *, int, const struct ieee80211_appie *); static void wi_scan_start(struct ieee80211com *); static void wi_scan_end(struct ieee80211com *); static void wi_set_channel(struct ieee80211com *); static __inline int wi_write_val(struct wi_softc *sc, int rid, u_int16_t val) { val = htole16(val); return wi_write_rid(sc, rid, &val, sizeof(val)); } static SYSCTL_NODE(_hw, OID_AUTO, wi, CTLFLAG_RD, 0, "Wireless driver parameters"); static struct timeval lasttxerror; /* time of last tx error msg */ static int curtxeps; /* current tx error msgs/sec */ static int wi_txerate = 0; /* tx error rate: max msgs/sec */ SYSCTL_INT(_hw_wi, OID_AUTO, txerate, CTLFLAG_RW, &wi_txerate, 0, "max tx error msgs/sec; 0 to disable msgs"); #define WI_DEBUG #ifdef WI_DEBUG static int wi_debug = 0; SYSCTL_INT(_hw_wi, OID_AUTO, debug, CTLFLAG_RW, &wi_debug, 0, "control debugging printfs"); #define DPRINTF(X) if (wi_debug) printf X #else #define DPRINTF(X) #endif #define WI_INTRS (WI_EV_RX | WI_EV_ALLOC | WI_EV_INFO) struct wi_card_ident wi_card_ident[] = { /* CARD_ID CARD_NAME FIRM_TYPE */ { WI_NIC_LUCENT_ID, WI_NIC_LUCENT_STR, WI_LUCENT }, { WI_NIC_SONY_ID, WI_NIC_SONY_STR, WI_LUCENT }, { WI_NIC_LUCENT_EMB_ID, WI_NIC_LUCENT_EMB_STR, WI_LUCENT }, { WI_NIC_EVB2_ID, WI_NIC_EVB2_STR, WI_INTERSIL }, { WI_NIC_HWB3763_ID, WI_NIC_HWB3763_STR, WI_INTERSIL }, { WI_NIC_HWB3163_ID, WI_NIC_HWB3163_STR, WI_INTERSIL }, { WI_NIC_HWB3163B_ID, WI_NIC_HWB3163B_STR, WI_INTERSIL }, { WI_NIC_EVB3_ID, WI_NIC_EVB3_STR, WI_INTERSIL }, { WI_NIC_HWB1153_ID, WI_NIC_HWB1153_STR, WI_INTERSIL }, { WI_NIC_P2_SST_ID, WI_NIC_P2_SST_STR, WI_INTERSIL }, { WI_NIC_EVB2_SST_ID, WI_NIC_EVB2_SST_STR, WI_INTERSIL }, { WI_NIC_3842_EVA_ID, WI_NIC_3842_EVA_STR, WI_INTERSIL }, { WI_NIC_3842_PCMCIA_AMD_ID, WI_NIC_3842_PCMCIA_STR, WI_INTERSIL }, { WI_NIC_3842_PCMCIA_SST_ID, WI_NIC_3842_PCMCIA_STR, WI_INTERSIL }, { WI_NIC_3842_PCMCIA_ATL_ID, WI_NIC_3842_PCMCIA_STR, WI_INTERSIL }, { WI_NIC_3842_PCMCIA_ATS_ID, WI_NIC_3842_PCMCIA_STR, WI_INTERSIL }, { WI_NIC_3842_MINI_AMD_ID, WI_NIC_3842_MINI_STR, WI_INTERSIL }, { WI_NIC_3842_MINI_SST_ID, WI_NIC_3842_MINI_STR, WI_INTERSIL }, { WI_NIC_3842_MINI_ATL_ID, WI_NIC_3842_MINI_STR, WI_INTERSIL }, { WI_NIC_3842_MINI_ATS_ID, WI_NIC_3842_MINI_STR, WI_INTERSIL }, { WI_NIC_3842_PCI_AMD_ID, WI_NIC_3842_PCI_STR, WI_INTERSIL }, { WI_NIC_3842_PCI_SST_ID, WI_NIC_3842_PCI_STR, WI_INTERSIL }, { WI_NIC_3842_PCI_ATS_ID, WI_NIC_3842_PCI_STR, WI_INTERSIL }, { WI_NIC_3842_PCI_ATL_ID, WI_NIC_3842_PCI_STR, WI_INTERSIL }, { WI_NIC_P3_PCMCIA_AMD_ID, WI_NIC_P3_PCMCIA_STR, WI_INTERSIL }, { WI_NIC_P3_PCMCIA_SST_ID, WI_NIC_P3_PCMCIA_STR, WI_INTERSIL }, { WI_NIC_P3_PCMCIA_ATL_ID, WI_NIC_P3_PCMCIA_STR, WI_INTERSIL }, { WI_NIC_P3_PCMCIA_ATS_ID, WI_NIC_P3_PCMCIA_STR, WI_INTERSIL }, { WI_NIC_P3_MINI_AMD_ID, WI_NIC_P3_MINI_STR, WI_INTERSIL }, { WI_NIC_P3_MINI_SST_ID, WI_NIC_P3_MINI_STR, WI_INTERSIL }, { WI_NIC_P3_MINI_ATL_ID, WI_NIC_P3_MINI_STR, WI_INTERSIL }, { WI_NIC_P3_MINI_ATS_ID, WI_NIC_P3_MINI_STR, WI_INTERSIL }, { 0, NULL, 0 }, }; static char *wi_firmware_names[] = { "none", "Hermes", "Intersil", "Symbol" }; devclass_t wi_devclass; int wi_attach(device_t dev) { struct wi_softc *sc = device_get_softc(dev); struct ieee80211com *ic = &sc->sc_ic; int i, nrates, buflen; u_int16_t val; u_int8_t ratebuf[2 + IEEE80211_RATE_SIZE]; struct ieee80211_rateset *rs; struct sysctl_ctx_list *sctx; struct sysctl_oid *soid; static const u_int8_t empty_macaddr[IEEE80211_ADDR_LEN] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; int error; sc->sc_firmware_type = WI_NOTYPE; sc->wi_cmd_count = 500; /* Reset the NIC. */ if (wi_reset(sc) != 0) { wi_free(dev); return ENXIO; /* XXX */ } /* Read NIC identification */ wi_read_nicid(sc); switch (sc->sc_firmware_type) { case WI_LUCENT: if (sc->sc_sta_firmware_ver < 60006) goto reject; break; case WI_INTERSIL: if (sc->sc_sta_firmware_ver < 800) goto reject; break; default: reject: device_printf(dev, "Sorry, this card is not supported " "(type %d, firmware ver %d)\n", sc->sc_firmware_type, sc->sc_sta_firmware_ver); wi_free(dev); return EOPNOTSUPP; } /* Export info about the device via sysctl */ sctx = device_get_sysctl_ctx(dev); soid = device_get_sysctl_tree(dev); SYSCTL_ADD_STRING(sctx, SYSCTL_CHILDREN(soid), OID_AUTO, "firmware_type", CTLFLAG_RD, wi_firmware_names[sc->sc_firmware_type], 0, "Firmware type string"); SYSCTL_ADD_INT(sctx, SYSCTL_CHILDREN(soid), OID_AUTO, "sta_version", CTLFLAG_RD, &sc->sc_sta_firmware_ver, 0, "Station Firmware version"); if (sc->sc_firmware_type == WI_INTERSIL) SYSCTL_ADD_INT(sctx, SYSCTL_CHILDREN(soid), OID_AUTO, "pri_version", CTLFLAG_RD, &sc->sc_pri_firmware_ver, 0, "Primary Firmware version"); SYSCTL_ADD_UINT(sctx, SYSCTL_CHILDREN(soid), OID_AUTO, "nic_id", CTLFLAG_RD, &sc->sc_nic_id, 0, "NIC id"); SYSCTL_ADD_STRING(sctx, SYSCTL_CHILDREN(soid), OID_AUTO, "nic_name", CTLFLAG_RD, sc->sc_nic_name, 0, "NIC name"); mtx_init(&sc->sc_mtx, device_get_nameunit(dev), MTX_NETWORK_LOCK, MTX_DEF | MTX_RECURSE); callout_init_mtx(&sc->sc_watchdog, &sc->sc_mtx, 0); mbufq_init(&sc->sc_snd, ifqmaxlen); /* * Read the station address. * And do it twice. I've seen PRISM-based cards that return * an error when trying to read it the first time, which causes * the probe to fail. */ buflen = IEEE80211_ADDR_LEN; error = wi_read_rid(sc, WI_RID_MAC_NODE, &ic->ic_macaddr, &buflen); if (error != 0) { buflen = IEEE80211_ADDR_LEN; error = wi_read_rid(sc, WI_RID_MAC_NODE, &ic->ic_macaddr, &buflen); } if (error || IEEE80211_ADDR_EQ(&ic->ic_macaddr, empty_macaddr)) { if (error != 0) device_printf(dev, "mac read failed %d\n", error); else { device_printf(dev, "mac read failed (all zeros)\n"); error = ENXIO; } wi_free(dev); return (error); } ic->ic_softc = sc; ic->ic_name = device_get_nameunit(dev); ic->ic_phytype = IEEE80211_T_DS; ic->ic_opmode = IEEE80211_M_STA; ic->ic_caps = IEEE80211_C_STA | IEEE80211_C_PMGT | IEEE80211_C_MONITOR ; /* * Query the card for available channels and setup the * channel table. We assume these are all 11b channels. */ buflen = sizeof(val); if (wi_read_rid(sc, WI_RID_CHANNEL_LIST, &val, &buflen) != 0) - val = htole16(0x1fff); /* assume 1-11 */ + val = htole16(0x1fff); /* assume 1-13 */ KASSERT(val != 0, ("wi_attach: no available channels listed!")); val <<= 1; /* shift for base 1 indices */ for (i = 1; i < 16; i++) { struct ieee80211_channel *c; if (!isset((u_int8_t*)&val, i)) continue; c = &ic->ic_channels[ic->ic_nchans++]; c->ic_freq = ieee80211_ieee2mhz(i, IEEE80211_CHAN_B); c->ic_flags = IEEE80211_CHAN_B; c->ic_ieee = i; /* XXX txpowers? */ } /* * Set flags based on firmware version. */ switch (sc->sc_firmware_type) { case WI_LUCENT: sc->sc_ntxbuf = 1; ic->ic_caps |= IEEE80211_C_IBSS; sc->sc_ibss_port = WI_PORTTYPE_BSS; sc->sc_monitor_port = WI_PORTTYPE_ADHOC; sc->sc_min_rssi = WI_LUCENT_MIN_RSSI; sc->sc_max_rssi = WI_LUCENT_MAX_RSSI; sc->sc_dbm_offset = WI_LUCENT_DBM_OFFSET; break; case WI_INTERSIL: sc->sc_ntxbuf = WI_NTXBUF; sc->sc_flags |= WI_FLAGS_HAS_FRAGTHR | WI_FLAGS_HAS_ROAMING; /* * Old firmware are slow, so give peace a chance. */ if (sc->sc_sta_firmware_ver < 10000) sc->wi_cmd_count = 5000; if (sc->sc_sta_firmware_ver > 10101) sc->sc_flags |= WI_FLAGS_HAS_DBMADJUST; ic->ic_caps |= IEEE80211_C_IBSS; /* * version 0.8.3 and newer are the only ones that are known * to currently work. Earlier versions can be made to work, * at least according to the Linux driver but we require * monitor mode so this is irrelevant. */ ic->ic_caps |= IEEE80211_C_HOSTAP; if (sc->sc_sta_firmware_ver >= 10603) sc->sc_flags |= WI_FLAGS_HAS_ENHSECURITY; if (sc->sc_sta_firmware_ver >= 10700) { /* * 1.7.0+ have the necessary support for sta mode WPA. */ sc->sc_flags |= WI_FLAGS_HAS_WPASUPPORT; ic->ic_caps |= IEEE80211_C_WPA; } sc->sc_ibss_port = WI_PORTTYPE_IBSS; sc->sc_monitor_port = WI_PORTTYPE_APSILENT; sc->sc_min_rssi = WI_PRISM_MIN_RSSI; sc->sc_max_rssi = WI_PRISM_MAX_RSSI; sc->sc_dbm_offset = WI_PRISM_DBM_OFFSET; break; } /* * Find out if we support WEP on this card. */ buflen = sizeof(val); if (wi_read_rid(sc, WI_RID_WEP_AVAIL, &val, &buflen) == 0 && val != htole16(0)) ic->ic_cryptocaps |= IEEE80211_CRYPTO_WEP; /* Find supported rates. */ buflen = sizeof(ratebuf); rs = &ic->ic_sup_rates[IEEE80211_MODE_11B]; if (wi_read_rid(sc, WI_RID_DATA_RATES, ratebuf, &buflen) == 0) { nrates = le16toh(*(u_int16_t *)ratebuf); if (nrates > IEEE80211_RATE_MAXSIZE) nrates = IEEE80211_RATE_MAXSIZE; rs->rs_nrates = 0; for (i = 0; i < nrates; i++) if (ratebuf[2+i]) rs->rs_rates[rs->rs_nrates++] = ratebuf[2+i]; } else { /* XXX fallback on error? */ } buflen = sizeof(val); if ((sc->sc_flags & WI_FLAGS_HAS_DBMADJUST) && wi_read_rid(sc, WI_RID_DBM_ADJUST, &val, &buflen) == 0) { sc->sc_dbm_offset = le16toh(val); } sc->sc_portnum = WI_DEFAULT_PORT; ieee80211_ifattach(ic); ic->ic_raw_xmit = wi_raw_xmit; ic->ic_scan_start = wi_scan_start; ic->ic_scan_end = wi_scan_end; ic->ic_set_channel = wi_set_channel; ic->ic_vap_create = wi_vap_create; ic->ic_vap_delete = wi_vap_delete; ic->ic_update_mcast = wi_update_mcast; ic->ic_update_promisc = wi_update_promisc; ic->ic_transmit = wi_transmit; ic->ic_parent = wi_parent; ieee80211_radiotap_attach(ic, &sc->sc_tx_th.wt_ihdr, sizeof(sc->sc_tx_th), WI_TX_RADIOTAP_PRESENT, &sc->sc_rx_th.wr_ihdr, sizeof(sc->sc_rx_th), WI_RX_RADIOTAP_PRESENT); if (bootverbose) ieee80211_announce(ic); error = bus_setup_intr(dev, sc->irq, INTR_TYPE_NET | INTR_MPSAFE, NULL, wi_intr, sc, &sc->wi_intrhand); if (error) { device_printf(dev, "bus_setup_intr() failed! (%d)\n", error); ieee80211_ifdetach(ic); wi_free(dev); return error; } return (0); } int wi_detach(device_t dev) { struct wi_softc *sc = device_get_softc(dev); struct ieee80211com *ic = &sc->sc_ic; WI_LOCK(sc); /* check if device was removed */ sc->wi_gone |= !bus_child_present(dev); wi_stop(sc, 0); WI_UNLOCK(sc); ieee80211_ifdetach(ic); bus_teardown_intr(dev, sc->irq, sc->wi_intrhand); wi_free(dev); mbufq_drain(&sc->sc_snd); mtx_destroy(&sc->sc_mtx); return (0); } static struct ieee80211vap * wi_vap_create(struct ieee80211com *ic, const char name[IFNAMSIZ], int unit, enum ieee80211_opmode opmode, int flags, const uint8_t bssid[IEEE80211_ADDR_LEN], const uint8_t mac[IEEE80211_ADDR_LEN]) { struct wi_softc *sc = ic->ic_softc; struct wi_vap *wvp; struct ieee80211vap *vap; if (!TAILQ_EMPTY(&ic->ic_vaps)) /* only one at a time */ return NULL; wvp = malloc(sizeof(struct wi_vap), M_80211_VAP, M_WAITOK | M_ZERO); vap = &wvp->wv_vap; ieee80211_vap_setup(ic, vap, name, unit, opmode, flags, bssid); vap->iv_max_aid = WI_MAX_AID; switch (opmode) { case IEEE80211_M_STA: sc->sc_porttype = WI_PORTTYPE_BSS; wvp->wv_newstate = vap->iv_newstate; vap->iv_newstate = wi_newstate_sta; /* need to filter mgt frames to avoid confusing state machine */ wvp->wv_recv_mgmt = vap->iv_recv_mgmt; vap->iv_recv_mgmt = wi_recv_mgmt; break; case IEEE80211_M_IBSS: sc->sc_porttype = sc->sc_ibss_port; wvp->wv_newstate = vap->iv_newstate; vap->iv_newstate = wi_newstate_sta; break; case IEEE80211_M_AHDEMO: sc->sc_porttype = WI_PORTTYPE_ADHOC; break; case IEEE80211_M_HOSTAP: sc->sc_porttype = WI_PORTTYPE_HOSTAP; wvp->wv_newstate = vap->iv_newstate; vap->iv_newstate = wi_newstate_hostap; break; case IEEE80211_M_MONITOR: sc->sc_porttype = sc->sc_monitor_port; break; default: break; } /* complete setup */ ieee80211_vap_attach(vap, ieee80211_media_change, wi_media_status, mac); ic->ic_opmode = opmode; return vap; } static void wi_vap_delete(struct ieee80211vap *vap) { struct wi_vap *wvp = WI_VAP(vap); ieee80211_vap_detach(vap); free(wvp, M_80211_VAP); } int wi_shutdown(device_t dev) { struct wi_softc *sc = device_get_softc(dev); WI_LOCK(sc); wi_stop(sc, 1); WI_UNLOCK(sc); return (0); } void wi_intr(void *arg) { struct wi_softc *sc = arg; u_int16_t status; WI_LOCK(sc); if (sc->wi_gone || !sc->sc_enabled || (sc->sc_flags & WI_FLAGS_RUNNING) == 0) { CSR_WRITE_2(sc, WI_INT_EN, 0); CSR_WRITE_2(sc, WI_EVENT_ACK, 0xFFFF); WI_UNLOCK(sc); return; } /* Disable interrupts. */ CSR_WRITE_2(sc, WI_INT_EN, 0); status = CSR_READ_2(sc, WI_EVENT_STAT); if (status & WI_EV_RX) wi_rx_intr(sc); if (status & WI_EV_ALLOC) wi_tx_intr(sc); if (status & WI_EV_TX_EXC) wi_tx_ex_intr(sc); if (status & WI_EV_INFO) wi_info_intr(sc); if (mbufq_first(&sc->sc_snd) != NULL) wi_start(sc); /* Re-enable interrupts. */ CSR_WRITE_2(sc, WI_INT_EN, WI_INTRS); WI_UNLOCK(sc); return; } static void wi_enable(struct wi_softc *sc) { /* Enable interrupts */ CSR_WRITE_2(sc, WI_INT_EN, WI_INTRS); /* enable port */ wi_cmd(sc, WI_CMD_ENABLE | sc->sc_portnum, 0, 0, 0); sc->sc_enabled = 1; } static int wi_setup_locked(struct wi_softc *sc, int porttype, int mode, const uint8_t mac[IEEE80211_ADDR_LEN]) { int i; wi_reset(sc); wi_write_val(sc, WI_RID_PORTTYPE, porttype); wi_write_val(sc, WI_RID_CREATE_IBSS, mode); wi_write_val(sc, WI_RID_MAX_DATALEN, 2304); /* XXX IEEE80211_BPF_NOACK wants 0 */ wi_write_val(sc, WI_RID_ALT_RETRY_CNT, 2); if (sc->sc_flags & WI_FLAGS_HAS_ROAMING) wi_write_val(sc, WI_RID_ROAMING_MODE, 3); /* NB: disabled */ wi_write_rid(sc, WI_RID_MAC_NODE, mac, IEEE80211_ADDR_LEN); /* Allocate fids for the card */ sc->sc_buflen = IEEE80211_MAX_LEN + sizeof(struct wi_frame); for (i = 0; i < sc->sc_ntxbuf; i++) { int error = wi_alloc_fid(sc, sc->sc_buflen, &sc->sc_txd[i].d_fid); if (error) { device_printf(sc->sc_dev, "tx buffer allocation failed (error %u)\n", error); return error; } sc->sc_txd[i].d_len = 0; } sc->sc_txcur = sc->sc_txnext = 0; return 0; } void wi_init(struct wi_softc *sc) { int wasenabled; WI_LOCK_ASSERT(sc); wasenabled = sc->sc_enabled; if (wasenabled) wi_stop(sc, 1); if (wi_setup_locked(sc, sc->sc_porttype, 3, sc->sc_ic.ic_macaddr) != 0) { device_printf(sc->sc_dev, "interface not running\n"); wi_stop(sc, 1); return; } sc->sc_flags |= WI_FLAGS_RUNNING; callout_reset(&sc->sc_watchdog, hz, wi_watchdog, sc); wi_enable(sc); /* Enable desired port */ } void wi_stop(struct wi_softc *sc, int disable) { WI_LOCK_ASSERT(sc); if (sc->sc_enabled && !sc->wi_gone) { CSR_WRITE_2(sc, WI_INT_EN, 0); wi_cmd(sc, WI_CMD_DISABLE | sc->sc_portnum, 0, 0, 0); if (disable) sc->sc_enabled = 0; } else if (sc->wi_gone && disable) /* gone --> not enabled */ sc->sc_enabled = 0; callout_stop(&sc->sc_watchdog); sc->sc_tx_timer = 0; sc->sc_false_syns = 0; sc->sc_flags &= ~WI_FLAGS_RUNNING; } static void wi_set_channel(struct ieee80211com *ic) { struct wi_softc *sc = ic->ic_softc; DPRINTF(("%s: channel %d, %sscanning\n", __func__, ieee80211_chan2ieee(ic, ic->ic_curchan), ic->ic_flags & IEEE80211_F_SCAN ? "" : "!")); WI_LOCK(sc); wi_write_val(sc, WI_RID_OWN_CHNL, ieee80211_chan2ieee(ic, ic->ic_curchan)); WI_UNLOCK(sc); } static void wi_scan_start(struct ieee80211com *ic) { struct wi_softc *sc = ic->ic_softc; struct ieee80211_scan_state *ss = ic->ic_scan; DPRINTF(("%s\n", __func__)); WI_LOCK(sc); /* * Switch device to monitor mode. */ wi_write_val(sc, WI_RID_PORTTYPE, sc->sc_monitor_port); if (sc->sc_firmware_type == WI_INTERSIL) { wi_cmd(sc, WI_CMD_DISABLE | WI_PORT0, 0, 0, 0); wi_cmd(sc, WI_CMD_ENABLE | WI_PORT0, 0, 0, 0); } /* force full dwell time to compensate for firmware overhead */ ss->ss_mindwell = ss->ss_maxdwell = msecs_to_ticks(400); WI_UNLOCK(sc); } static void wi_scan_end(struct ieee80211com *ic) { struct wi_softc *sc = ic->ic_softc; DPRINTF(("%s: restore port type %d\n", __func__, sc->sc_porttype)); WI_LOCK(sc); wi_write_val(sc, WI_RID_PORTTYPE, sc->sc_porttype); if (sc->sc_firmware_type == WI_INTERSIL) { wi_cmd(sc, WI_CMD_DISABLE | WI_PORT0, 0, 0, 0); wi_cmd(sc, WI_CMD_ENABLE | WI_PORT0, 0, 0, 0); } WI_UNLOCK(sc); } static void wi_recv_mgmt(struct ieee80211_node *ni, struct mbuf *m, int subtype, const struct ieee80211_rx_stats *rxs, int rssi, int nf) { struct ieee80211vap *vap = ni->ni_vap; switch (subtype) { case IEEE80211_FC0_SUBTYPE_AUTH: case IEEE80211_FC0_SUBTYPE_ASSOC_RESP: case IEEE80211_FC0_SUBTYPE_REASSOC_RESP: /* NB: filter frames that trigger state changes */ return; } WI_VAP(vap)->wv_recv_mgmt(ni, m, subtype, rxs, rssi, nf); } static int wi_newstate_sta(struct ieee80211vap *vap, enum ieee80211_state nstate, int arg) { struct ieee80211com *ic = vap->iv_ic; struct ieee80211_node *bss; struct wi_softc *sc = ic->ic_softc; DPRINTF(("%s: %s -> %s\n", __func__, ieee80211_state_name[vap->iv_state], ieee80211_state_name[nstate])); if (nstate == IEEE80211_S_AUTH) { WI_LOCK(sc); wi_setup_locked(sc, WI_PORTTYPE_BSS, 3, vap->iv_myaddr); if (vap->iv_flags & IEEE80211_F_PMGTON) { wi_write_val(sc, WI_RID_MAX_SLEEP, ic->ic_lintval); wi_write_val(sc, WI_RID_PM_ENABLED, 1); } wi_write_val(sc, WI_RID_RTS_THRESH, vap->iv_rtsthreshold); if (sc->sc_flags & WI_FLAGS_HAS_FRAGTHR) wi_write_val(sc, WI_RID_FRAG_THRESH, vap->iv_fragthreshold); wi_write_txrate(sc, vap); bss = vap->iv_bss; wi_write_ssid(sc, WI_RID_DESIRED_SSID, bss->ni_essid, bss->ni_esslen); wi_write_val(sc, WI_RID_OWN_CHNL, ieee80211_chan2ieee(ic, bss->ni_chan)); /* Configure WEP. */ if (ic->ic_cryptocaps & IEEE80211_CRYPTO_WEP) wi_write_wep(sc, vap); else sc->sc_encryption = 0; if ((sc->sc_flags & WI_FLAGS_HAS_WPASUPPORT) && (vap->iv_flags & IEEE80211_F_WPA)) { wi_write_val(sc, WI_RID_WPA_HANDLING, 1); if (vap->iv_appie_wpa != NULL) wi_write_appie(sc, WI_RID_WPA_DATA, vap->iv_appie_wpa); } wi_enable(sc); /* enable port */ /* Lucent firmware does not support the JOIN RID. */ if (sc->sc_firmware_type == WI_INTERSIL) { struct wi_joinreq join; memset(&join, 0, sizeof(join)); IEEE80211_ADDR_COPY(&join.wi_bssid, bss->ni_bssid); join.wi_chan = htole16( ieee80211_chan2ieee(ic, bss->ni_chan)); wi_write_rid(sc, WI_RID_JOIN_REQ, &join, sizeof(join)); } WI_UNLOCK(sc); /* * NB: don't go through 802.11 layer, it'll send auth frame; * instead we drive the state machine from the link status * notification we get on association. */ vap->iv_state = nstate; return (0); } return WI_VAP(vap)->wv_newstate(vap, nstate, arg); } static int wi_newstate_hostap(struct ieee80211vap *vap, enum ieee80211_state nstate, int arg) { struct ieee80211com *ic = vap->iv_ic; struct ieee80211_node *bss; struct wi_softc *sc = ic->ic_softc; int error; DPRINTF(("%s: %s -> %s\n", __func__, ieee80211_state_name[vap->iv_state], ieee80211_state_name[nstate])); error = WI_VAP(vap)->wv_newstate(vap, nstate, arg); if (error == 0 && nstate == IEEE80211_S_RUN) { WI_LOCK(sc); wi_setup_locked(sc, WI_PORTTYPE_HOSTAP, 0, vap->iv_myaddr); bss = vap->iv_bss; wi_write_ssid(sc, WI_RID_OWN_SSID, bss->ni_essid, bss->ni_esslen); wi_write_val(sc, WI_RID_OWN_CHNL, ieee80211_chan2ieee(ic, bss->ni_chan)); wi_write_val(sc, WI_RID_BASIC_RATE, 0x3); wi_write_val(sc, WI_RID_SUPPORT_RATE, 0xf); wi_write_txrate(sc, vap); wi_write_val(sc, WI_RID_OWN_BEACON_INT, bss->ni_intval); wi_write_val(sc, WI_RID_DTIM_PERIOD, vap->iv_dtim_period); wi_write_val(sc, WI_RID_RTS_THRESH, vap->iv_rtsthreshold); if (sc->sc_flags & WI_FLAGS_HAS_FRAGTHR) wi_write_val(sc, WI_RID_FRAG_THRESH, vap->iv_fragthreshold); if ((sc->sc_flags & WI_FLAGS_HAS_ENHSECURITY) && (vap->iv_flags & IEEE80211_F_HIDESSID)) { /* * bit 0 means hide SSID in beacons, * bit 1 means don't respond to bcast probe req */ wi_write_val(sc, WI_RID_ENH_SECURITY, 0x3); } if ((sc->sc_flags & WI_FLAGS_HAS_WPASUPPORT) && (vap->iv_flags & IEEE80211_F_WPA) && vap->iv_appie_wpa != NULL) wi_write_appie(sc, WI_RID_WPA_DATA, vap->iv_appie_wpa); wi_write_val(sc, WI_RID_PROMISC, 0); /* Configure WEP. */ if (ic->ic_cryptocaps & IEEE80211_CRYPTO_WEP) wi_write_wep(sc, vap); else sc->sc_encryption = 0; wi_enable(sc); /* enable port */ WI_UNLOCK(sc); } return error; } static int wi_transmit(struct ieee80211com *ic, struct mbuf *m) { struct wi_softc *sc = ic->ic_softc; int error; WI_LOCK(sc); if ((sc->sc_flags & WI_FLAGS_RUNNING) == 0) { WI_UNLOCK(sc); return (ENXIO); } error = mbufq_enqueue(&sc->sc_snd, m); if (error) { WI_UNLOCK(sc); return (error); } wi_start(sc); WI_UNLOCK(sc); return (0); } static void wi_start(struct wi_softc *sc) { struct ieee80211_node *ni; struct ieee80211_frame *wh; struct mbuf *m0; struct ieee80211_key *k; struct wi_frame frmhdr; const struct llc *llc; int cur; WI_LOCK_ASSERT(sc); if (sc->wi_gone) return; memset(&frmhdr, 0, sizeof(frmhdr)); cur = sc->sc_txnext; while (sc->sc_txd[cur].d_len == 0 && (m0 = mbufq_dequeue(&sc->sc_snd)) != NULL) { ni = (struct ieee80211_node *) m0->m_pkthdr.rcvif; /* reconstruct 802.3 header */ wh = mtod(m0, struct ieee80211_frame *); switch (wh->i_fc[1]) { case IEEE80211_FC1_DIR_TODS: IEEE80211_ADDR_COPY(frmhdr.wi_ehdr.ether_shost, wh->i_addr2); IEEE80211_ADDR_COPY(frmhdr.wi_ehdr.ether_dhost, wh->i_addr3); break; case IEEE80211_FC1_DIR_NODS: IEEE80211_ADDR_COPY(frmhdr.wi_ehdr.ether_shost, wh->i_addr2); IEEE80211_ADDR_COPY(frmhdr.wi_ehdr.ether_dhost, wh->i_addr1); break; case IEEE80211_FC1_DIR_FROMDS: IEEE80211_ADDR_COPY(frmhdr.wi_ehdr.ether_shost, wh->i_addr3); IEEE80211_ADDR_COPY(frmhdr.wi_ehdr.ether_dhost, wh->i_addr1); break; } llc = (const struct llc *)( mtod(m0, const uint8_t *) + ieee80211_hdrsize(wh)); frmhdr.wi_ehdr.ether_type = llc->llc_snap.ether_type; frmhdr.wi_tx_ctl = htole16(WI_ENC_TX_802_11|WI_TXCNTL_TX_EX); if (wh->i_fc[1] & IEEE80211_FC1_PROTECTED) { k = ieee80211_crypto_encap(ni, m0); if (k == NULL) { ieee80211_free_node(ni); m_freem(m0); continue; } frmhdr.wi_tx_ctl |= htole16(WI_TXCNTL_NOCRYPT); } if (ieee80211_radiotap_active_vap(ni->ni_vap)) { sc->sc_tx_th.wt_rate = ni->ni_txrate; ieee80211_radiotap_tx(ni->ni_vap, m0); } m_copydata(m0, 0, sizeof(struct ieee80211_frame), (caddr_t)&frmhdr.wi_whdr); m_adj(m0, sizeof(struct ieee80211_frame)); frmhdr.wi_dat_len = htole16(m0->m_pkthdr.len); ieee80211_free_node(ni); if (wi_start_tx(sc, &frmhdr, m0)) continue; sc->sc_txnext = cur = (cur + 1) % sc->sc_ntxbuf; } } static int wi_start_tx(struct wi_softc *sc, struct wi_frame *frmhdr, struct mbuf *m0) { int cur = sc->sc_txnext; int fid, off, error; fid = sc->sc_txd[cur].d_fid; off = sizeof(*frmhdr); error = wi_write_bap(sc, fid, 0, frmhdr, sizeof(*frmhdr)) != 0 || wi_mwrite_bap(sc, fid, off, m0, m0->m_pkthdr.len) != 0; m_freem(m0); if (error) { counter_u64_add(sc->sc_ic.ic_oerrors, 1); return -1; } sc->sc_txd[cur].d_len = off; if (sc->sc_txcur == cur) { if (wi_cmd(sc, WI_CMD_TX | WI_RECLAIM, fid, 0, 0)) { device_printf(sc->sc_dev, "xmit failed\n"); sc->sc_txd[cur].d_len = 0; return -1; } sc->sc_tx_timer = 5; } return 0; } static int wi_raw_xmit(struct ieee80211_node *ni, struct mbuf *m0, const struct ieee80211_bpf_params *params) { struct ieee80211com *ic = ni->ni_ic; struct ieee80211vap *vap = ni->ni_vap; struct wi_softc *sc = ic->ic_softc; struct ieee80211_key *k; struct ieee80211_frame *wh; struct wi_frame frmhdr; int cur; int rc = 0; WI_LOCK(sc); if (sc->wi_gone) { rc = ENETDOWN; goto out; } memset(&frmhdr, 0, sizeof(frmhdr)); cur = sc->sc_txnext; if (sc->sc_txd[cur].d_len != 0) { rc = ENOBUFS; goto out; } m0->m_pkthdr.rcvif = NULL; m_copydata(m0, 4, ETHER_ADDR_LEN * 2, (caddr_t)&frmhdr.wi_ehdr); frmhdr.wi_ehdr.ether_type = 0; wh = mtod(m0, struct ieee80211_frame *); frmhdr.wi_tx_ctl = htole16(WI_ENC_TX_802_11|WI_TXCNTL_TX_EX); if (params && (params->ibp_flags & IEEE80211_BPF_NOACK)) frmhdr.wi_tx_ctl |= htole16(WI_TXCNTL_ALTRTRY); if ((wh->i_fc[1] & IEEE80211_FC1_PROTECTED) && (!params || (params && (params->ibp_flags & IEEE80211_BPF_CRYPTO)))) { k = ieee80211_crypto_encap(ni, m0); if (k == NULL) { rc = ENOMEM; goto out; } frmhdr.wi_tx_ctl |= htole16(WI_TXCNTL_NOCRYPT); } if (ieee80211_radiotap_active_vap(vap)) { sc->sc_tx_th.wt_rate = ni->ni_txrate; ieee80211_radiotap_tx(vap, m0); } m_copydata(m0, 0, sizeof(struct ieee80211_frame), (caddr_t)&frmhdr.wi_whdr); m_adj(m0, sizeof(struct ieee80211_frame)); frmhdr.wi_dat_len = htole16(m0->m_pkthdr.len); if (wi_start_tx(sc, &frmhdr, m0) < 0) { m0 = NULL; rc = EIO; goto out; } m0 = NULL; ieee80211_free_node(ni); sc->sc_txnext = cur = (cur + 1) % sc->sc_ntxbuf; out: WI_UNLOCK(sc); if (m0 != NULL) m_freem(m0); return rc; } static int wi_reset(struct wi_softc *sc) { #define WI_INIT_TRIES 3 int i, error = 0; for (i = 0; i < WI_INIT_TRIES; i++) { error = wi_cmd(sc, WI_CMD_INI, 0, 0, 0); if (error == 0) break; DELAY(WI_DELAY * 1000); } sc->sc_reset = 1; if (i == WI_INIT_TRIES) { device_printf(sc->sc_dev, "reset failed\n"); return error; } CSR_WRITE_2(sc, WI_INT_EN, 0); CSR_WRITE_2(sc, WI_EVENT_ACK, 0xFFFF); /* Calibrate timer. */ wi_write_val(sc, WI_RID_TICK_TIME, 8); return 0; #undef WI_INIT_TRIES } static void wi_watchdog(void *arg) { struct wi_softc *sc = arg; WI_LOCK_ASSERT(sc); if (!sc->sc_enabled) return; if (sc->sc_tx_timer && --sc->sc_tx_timer == 0) { device_printf(sc->sc_dev, "device timeout\n"); counter_u64_add(sc->sc_ic.ic_oerrors, 1); wi_init(sc); return; } callout_reset(&sc->sc_watchdog, hz, wi_watchdog, sc); } static void wi_parent(struct ieee80211com *ic) { struct wi_softc *sc = ic->ic_softc; int startall = 0; WI_LOCK(sc); /* * Can't do promisc and hostap at the same time. If all that's * changing is the promisc flag, try to short-circuit a call to * wi_init() by just setting PROMISC in the hardware. */ if (ic->ic_nrunning > 0) { if (ic->ic_opmode != IEEE80211_M_HOSTAP && sc->sc_flags & WI_FLAGS_RUNNING) { if (ic->ic_promisc > 0 && (sc->sc_flags & WI_FLAGS_PROMISC) == 0) { wi_write_val(sc, WI_RID_PROMISC, 1); sc->sc_flags |= WI_FLAGS_PROMISC; } else if (ic->ic_promisc == 0 && (sc->sc_flags & WI_FLAGS_PROMISC) != 0) { wi_write_val(sc, WI_RID_PROMISC, 0); sc->sc_flags &= ~WI_FLAGS_PROMISC; } else { wi_init(sc); startall = 1; } } else { wi_init(sc); startall = 1; } } else if (sc->sc_flags & WI_FLAGS_RUNNING) { wi_stop(sc, 1); sc->wi_gone = 0; } WI_UNLOCK(sc); if (startall) ieee80211_start_all(ic); } static void wi_media_status(struct ifnet *ifp, struct ifmediareq *imr) { struct ieee80211vap *vap = ifp->if_softc; struct ieee80211com *ic = vap->iv_ic; struct wi_softc *sc = ic->ic_softc; u_int16_t val; int rate, len; len = sizeof(val); if (sc->sc_enabled && wi_read_rid(sc, WI_RID_CUR_TX_RATE, &val, &len) == 0 && len == sizeof(val)) { /* convert to 802.11 rate */ val = le16toh(val); rate = val * 2; if (sc->sc_firmware_type == WI_LUCENT) { if (rate == 10) rate = 11; /* 5.5Mbps */ } else { if (rate == 4*2) rate = 11; /* 5.5Mbps */ else if (rate == 8*2) rate = 22; /* 11Mbps */ } vap->iv_bss->ni_txrate = rate; } ieee80211_media_status(ifp, imr); } static void wi_sync_bssid(struct wi_softc *sc, u_int8_t new_bssid[IEEE80211_ADDR_LEN]) { struct ieee80211com *ic = &sc->sc_ic; struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps); struct ieee80211_node *ni = vap->iv_bss; if (IEEE80211_ADDR_EQ(new_bssid, ni->ni_bssid)) return; DPRINTF(("wi_sync_bssid: bssid %s -> ", ether_sprintf(ni->ni_bssid))); DPRINTF(("%s ?\n", ether_sprintf(new_bssid))); /* In promiscuous mode, the BSSID field is not a reliable * indicator of the firmware's BSSID. Damp spurious * change-of-BSSID indications. */ if (ic->ic_promisc > 0 && !ppsratecheck(&sc->sc_last_syn, &sc->sc_false_syns, WI_MAX_FALSE_SYNS)) return; sc->sc_false_syns = MAX(0, sc->sc_false_syns - 1); #if 0 /* * XXX hack; we should create a new node with the new bssid * and replace the existing ic_bss with it but since we don't * process management frames to collect state we cheat by * reusing the existing node as we know wi_newstate will be * called and it will overwrite the node state. */ ieee80211_sta_join(ic, ieee80211_ref_node(ni)); #endif } static __noinline void wi_rx_intr(struct wi_softc *sc) { struct ieee80211com *ic = &sc->sc_ic; struct wi_frame frmhdr; struct mbuf *m; struct ieee80211_frame *wh; struct ieee80211_node *ni; int fid, len, off; u_int8_t dir; u_int16_t status; int8_t rssi, nf; fid = CSR_READ_2(sc, WI_RX_FID); /* First read in the frame header */ if (wi_read_bap(sc, fid, 0, &frmhdr, sizeof(frmhdr))) { CSR_WRITE_2(sc, WI_EVENT_ACK, WI_EV_RX); counter_u64_add(ic->ic_ierrors, 1); DPRINTF(("wi_rx_intr: read fid %x failed\n", fid)); return; } /* * Drop undecryptable or packets with receive errors here */ status = le16toh(frmhdr.wi_status); if (status & WI_STAT_ERRSTAT) { CSR_WRITE_2(sc, WI_EVENT_ACK, WI_EV_RX); counter_u64_add(ic->ic_ierrors, 1); DPRINTF(("wi_rx_intr: fid %x error status %x\n", fid, status)); return; } len = le16toh(frmhdr.wi_dat_len); off = ALIGN(sizeof(struct ieee80211_frame)); /* * Sometimes the PRISM2.x returns bogusly large frames. Except * in monitor mode, just throw them away. */ if (off + len > MCLBYTES) { if (ic->ic_opmode != IEEE80211_M_MONITOR) { CSR_WRITE_2(sc, WI_EVENT_ACK, WI_EV_RX); counter_u64_add(ic->ic_ierrors, 1); DPRINTF(("wi_rx_intr: oversized packet\n")); return; } else len = 0; } if (off + len > MHLEN) m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); else m = m_gethdr(M_NOWAIT, MT_DATA); if (m == NULL) { CSR_WRITE_2(sc, WI_EVENT_ACK, WI_EV_RX); counter_u64_add(ic->ic_ierrors, 1); DPRINTF(("wi_rx_intr: MGET failed\n")); return; } m->m_data += off - sizeof(struct ieee80211_frame); memcpy(m->m_data, &frmhdr.wi_whdr, sizeof(struct ieee80211_frame)); wi_read_bap(sc, fid, sizeof(frmhdr), m->m_data + sizeof(struct ieee80211_frame), len); m->m_pkthdr.len = m->m_len = sizeof(struct ieee80211_frame) + len; CSR_WRITE_2(sc, WI_EVENT_ACK, WI_EV_RX); rssi = frmhdr.wi_rx_signal; nf = frmhdr.wi_rx_silence; if (ieee80211_radiotap_active(ic)) { struct wi_rx_radiotap_header *tap = &sc->sc_rx_th; uint32_t rstamp; rstamp = (le16toh(frmhdr.wi_rx_tstamp0) << 16) | le16toh(frmhdr.wi_rx_tstamp1); tap->wr_tsf = htole64((uint64_t)rstamp); /* XXX replace divide by table */ tap->wr_rate = frmhdr.wi_rx_rate / 5; tap->wr_flags = 0; if (frmhdr.wi_status & WI_STAT_PCF) tap->wr_flags |= IEEE80211_RADIOTAP_F_CFP; if (m->m_flags & M_WEP) tap->wr_flags |= IEEE80211_RADIOTAP_F_WEP; tap->wr_antsignal = rssi; tap->wr_antnoise = nf; } /* synchronize driver's BSSID with firmware's BSSID */ wh = mtod(m, struct ieee80211_frame *); dir = wh->i_fc[1] & IEEE80211_FC1_DIR_MASK; if (ic->ic_opmode == IEEE80211_M_IBSS && dir == IEEE80211_FC1_DIR_NODS) wi_sync_bssid(sc, wh->i_addr3); WI_UNLOCK(sc); ni = ieee80211_find_rxnode(ic, mtod(m, struct ieee80211_frame_min *)); if (ni != NULL) { (void) ieee80211_input(ni, m, rssi, nf); ieee80211_free_node(ni); } else (void) ieee80211_input_all(ic, m, rssi, nf); WI_LOCK(sc); } static __noinline void wi_tx_ex_intr(struct wi_softc *sc) { struct wi_frame frmhdr; int fid; fid = CSR_READ_2(sc, WI_TX_CMP_FID); /* Read in the frame header */ if (wi_read_bap(sc, fid, 0, &frmhdr, sizeof(frmhdr)) == 0) { u_int16_t status = le16toh(frmhdr.wi_status); /* * Spontaneous station disconnects appear as xmit * errors. Don't announce them and/or count them * as an output error. */ if ((status & WI_TXSTAT_DISCONNECT) == 0) { if (ppsratecheck(&lasttxerror, &curtxeps, wi_txerate)) { device_printf(sc->sc_dev, "tx failed"); if (status & WI_TXSTAT_RET_ERR) printf(", retry limit exceeded"); if (status & WI_TXSTAT_AGED_ERR) printf(", max transmit lifetime exceeded"); if (status & WI_TXSTAT_DISCONNECT) printf(", port disconnected"); if (status & WI_TXSTAT_FORM_ERR) printf(", invalid format (data len %u src %6D)", le16toh(frmhdr.wi_dat_len), frmhdr.wi_ehdr.ether_shost, ":"); if (status & ~0xf) printf(", status=0x%x", status); printf("\n"); } counter_u64_add(sc->sc_ic.ic_oerrors, 1); } else DPRINTF(("port disconnected\n")); } else DPRINTF(("wi_tx_ex_intr: read fid %x failed\n", fid)); CSR_WRITE_2(sc, WI_EVENT_ACK, WI_EV_TX_EXC); } static __noinline void wi_tx_intr(struct wi_softc *sc) { int fid, cur; if (sc->wi_gone) return; fid = CSR_READ_2(sc, WI_ALLOC_FID); CSR_WRITE_2(sc, WI_EVENT_ACK, WI_EV_ALLOC); cur = sc->sc_txcur; if (sc->sc_txd[cur].d_fid != fid) { device_printf(sc->sc_dev, "bad alloc %x != %x, cur %d nxt %d\n", fid, sc->sc_txd[cur].d_fid, cur, sc->sc_txnext); return; } sc->sc_tx_timer = 0; sc->sc_txd[cur].d_len = 0; sc->sc_txcur = cur = (cur + 1) % sc->sc_ntxbuf; if (sc->sc_txd[cur].d_len != 0) { if (wi_cmd(sc, WI_CMD_TX | WI_RECLAIM, sc->sc_txd[cur].d_fid, 0, 0)) { device_printf(sc->sc_dev, "xmit failed\n"); sc->sc_txd[cur].d_len = 0; } else { sc->sc_tx_timer = 5; } } } static __noinline void wi_info_intr(struct wi_softc *sc) { struct ieee80211com *ic = &sc->sc_ic; struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps); int i, fid, len, off; u_int16_t ltbuf[2]; u_int16_t stat; u_int32_t *ptr; fid = CSR_READ_2(sc, WI_INFO_FID); wi_read_bap(sc, fid, 0, ltbuf, sizeof(ltbuf)); switch (le16toh(ltbuf[1])) { case WI_INFO_LINK_STAT: wi_read_bap(sc, fid, sizeof(ltbuf), &stat, sizeof(stat)); DPRINTF(("wi_info_intr: LINK_STAT 0x%x\n", le16toh(stat))); if (vap == NULL) goto finish; switch (le16toh(stat)) { case WI_INFO_LINK_STAT_CONNECTED: if (vap->iv_state == IEEE80211_S_RUN && vap->iv_opmode != IEEE80211_M_IBSS) break; /* fall thru... */ case WI_INFO_LINK_STAT_AP_CHG: IEEE80211_LOCK(ic); vap->iv_bss->ni_associd = 1 | 0xc000; /* NB: anything will do */ ieee80211_new_state(vap, IEEE80211_S_RUN, 0); IEEE80211_UNLOCK(ic); break; case WI_INFO_LINK_STAT_AP_INR: break; case WI_INFO_LINK_STAT_DISCONNECTED: /* we dropped off the net; e.g. due to deauth/disassoc */ IEEE80211_LOCK(ic); vap->iv_bss->ni_associd = 0; vap->iv_stats.is_rx_deauth++; ieee80211_new_state(vap, IEEE80211_S_SCAN, 0); IEEE80211_UNLOCK(ic); break; case WI_INFO_LINK_STAT_AP_OOR: /* XXX does this need to be per-vap? */ ieee80211_beacon_miss(ic); break; case WI_INFO_LINK_STAT_ASSOC_FAILED: if (vap->iv_opmode == IEEE80211_M_STA) ieee80211_new_state(vap, IEEE80211_S_SCAN, IEEE80211_SCAN_FAIL_TIMEOUT); break; } break; case WI_INFO_COUNTERS: /* some card versions have a larger stats structure */ len = min(le16toh(ltbuf[0]) - 1, sizeof(sc->sc_stats) / 4); ptr = (u_int32_t *)&sc->sc_stats; off = sizeof(ltbuf); for (i = 0; i < len; i++, off += 2, ptr++) { wi_read_bap(sc, fid, off, &stat, sizeof(stat)); #ifdef WI_HERMES_STATS_WAR if (stat & 0xf000) stat = ~stat; #endif *ptr += stat; } break; default: DPRINTF(("wi_info_intr: got fid %x type %x len %d\n", fid, le16toh(ltbuf[1]), le16toh(ltbuf[0]))); break; } finish: CSR_WRITE_2(sc, WI_EVENT_ACK, WI_EV_INFO); } static int wi_write_multi(struct wi_softc *sc) { struct ieee80211com *ic = &sc->sc_ic; struct ieee80211vap *vap; struct wi_mcast mlist; int n; if (ic->ic_allmulti > 0 || ic->ic_promisc > 0) { allmulti: memset(&mlist, 0, sizeof(mlist)); return wi_write_rid(sc, WI_RID_MCAST_LIST, &mlist, sizeof(mlist)); } n = 0; TAILQ_FOREACH(vap, &ic->ic_vaps, iv_next) { struct ifnet *ifp; struct ifmultiaddr *ifma; ifp = vap->iv_ifp; if_maddr_rlock(ifp); TAILQ_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) { if (ifma->ifma_addr->sa_family != AF_LINK) continue; if (n >= 16) goto allmulti; IEEE80211_ADDR_COPY(&mlist.wi_mcast[n], (LLADDR((struct sockaddr_dl *)ifma->ifma_addr))); n++; } if_maddr_runlock(ifp); } return wi_write_rid(sc, WI_RID_MCAST_LIST, &mlist, IEEE80211_ADDR_LEN * n); } static void wi_update_mcast(struct ieee80211com *ic) { wi_write_multi(ic->ic_softc); } static void wi_update_promisc(struct ieee80211com *ic) { struct wi_softc *sc = ic->ic_softc; WI_LOCK(sc); /* XXX handle WEP special case handling? */ wi_write_val(sc, WI_RID_PROMISC, (ic->ic_opmode == IEEE80211_M_MONITOR || (ic->ic_promisc > 0))); WI_UNLOCK(sc); } static void wi_read_nicid(struct wi_softc *sc) { struct wi_card_ident *id; char *p; int len; u_int16_t ver[4]; /* getting chip identity */ memset(ver, 0, sizeof(ver)); len = sizeof(ver); wi_read_rid(sc, WI_RID_CARD_ID, ver, &len); sc->sc_firmware_type = WI_NOTYPE; sc->sc_nic_id = le16toh(ver[0]); for (id = wi_card_ident; id->card_name != NULL; id++) { if (sc->sc_nic_id == id->card_id) { sc->sc_nic_name = id->card_name; sc->sc_firmware_type = id->firm_type; break; } } if (sc->sc_firmware_type == WI_NOTYPE) { if (sc->sc_nic_id & 0x8000) { sc->sc_firmware_type = WI_INTERSIL; sc->sc_nic_name = "Unknown Prism chip"; } else { sc->sc_firmware_type = WI_LUCENT; sc->sc_nic_name = "Unknown Lucent chip"; } } if (bootverbose) device_printf(sc->sc_dev, "using %s\n", sc->sc_nic_name); /* get primary firmware version (Only Prism chips) */ if (sc->sc_firmware_type != WI_LUCENT) { memset(ver, 0, sizeof(ver)); len = sizeof(ver); wi_read_rid(sc, WI_RID_PRI_IDENTITY, ver, &len); sc->sc_pri_firmware_ver = le16toh(ver[2]) * 10000 + le16toh(ver[3]) * 100 + le16toh(ver[1]); } /* get station firmware version */ memset(ver, 0, sizeof(ver)); len = sizeof(ver); wi_read_rid(sc, WI_RID_STA_IDENTITY, ver, &len); sc->sc_sta_firmware_ver = le16toh(ver[2]) * 10000 + le16toh(ver[3]) * 100 + le16toh(ver[1]); if (sc->sc_firmware_type == WI_INTERSIL && (sc->sc_sta_firmware_ver == 10102 || sc->sc_sta_firmware_ver == 20102)) { char ident[12]; memset(ident, 0, sizeof(ident)); len = sizeof(ident); /* value should be the format like "V2.00-11" */ if (wi_read_rid(sc, WI_RID_SYMBOL_IDENTITY, ident, &len) == 0 && *(p = (char *)ident) >= 'A' && p[2] == '.' && p[5] == '-' && p[8] == '\0') { sc->sc_firmware_type = WI_SYMBOL; sc->sc_sta_firmware_ver = (p[1] - '0') * 10000 + (p[3] - '0') * 1000 + (p[4] - '0') * 100 + (p[6] - '0') * 10 + (p[7] - '0'); } } if (bootverbose) { device_printf(sc->sc_dev, "%s Firmware: ", wi_firmware_names[sc->sc_firmware_type]); if (sc->sc_firmware_type != WI_LUCENT) /* XXX */ printf("Primary (%u.%u.%u), ", sc->sc_pri_firmware_ver / 10000, (sc->sc_pri_firmware_ver % 10000) / 100, sc->sc_pri_firmware_ver % 100); printf("Station (%u.%u.%u)\n", sc->sc_sta_firmware_ver / 10000, (sc->sc_sta_firmware_ver % 10000) / 100, sc->sc_sta_firmware_ver % 100); } } static int wi_write_ssid(struct wi_softc *sc, int rid, u_int8_t *buf, int buflen) { struct wi_ssid ssid; if (buflen > IEEE80211_NWID_LEN) return ENOBUFS; memset(&ssid, 0, sizeof(ssid)); ssid.wi_len = htole16(buflen); memcpy(ssid.wi_ssid, buf, buflen); return wi_write_rid(sc, rid, &ssid, sizeof(ssid)); } static int wi_write_txrate(struct wi_softc *sc, struct ieee80211vap *vap) { static const uint16_t lucent_rates[12] = { [ 0] = 3, /* auto */ [ 1] = 1, /* 1Mb/s */ [ 2] = 2, /* 2Mb/s */ [ 5] = 4, /* 5.5Mb/s */ [11] = 5 /* 11Mb/s */ }; static const uint16_t intersil_rates[12] = { [ 0] = 0xf, /* auto */ [ 1] = 0, /* 1Mb/s */ [ 2] = 1, /* 2Mb/s */ [ 5] = 2, /* 5.5Mb/s */ [11] = 3, /* 11Mb/s */ }; const uint16_t *rates = sc->sc_firmware_type == WI_LUCENT ? lucent_rates : intersil_rates; struct ieee80211com *ic = vap->iv_ic; const struct ieee80211_txparam *tp; tp = &vap->iv_txparms[ieee80211_chan2mode(ic->ic_bsschan)]; return wi_write_val(sc, WI_RID_TX_RATE, (tp->ucastrate == IEEE80211_FIXED_RATE_NONE ? rates[0] : rates[tp->ucastrate / 2])); } static int wi_write_wep(struct wi_softc *sc, struct ieee80211vap *vap) { int error = 0; int i, keylen; u_int16_t val; struct wi_key wkey[IEEE80211_WEP_NKID]; switch (sc->sc_firmware_type) { case WI_LUCENT: val = (vap->iv_flags & IEEE80211_F_PRIVACY) ? 1 : 0; error = wi_write_val(sc, WI_RID_ENCRYPTION, val); if (error) break; if ((vap->iv_flags & IEEE80211_F_PRIVACY) == 0) break; error = wi_write_val(sc, WI_RID_TX_CRYPT_KEY, vap->iv_def_txkey); if (error) break; memset(wkey, 0, sizeof(wkey)); for (i = 0; i < IEEE80211_WEP_NKID; i++) { keylen = vap->iv_nw_keys[i].wk_keylen; wkey[i].wi_keylen = htole16(keylen); memcpy(wkey[i].wi_keydat, vap->iv_nw_keys[i].wk_key, keylen); } error = wi_write_rid(sc, WI_RID_DEFLT_CRYPT_KEYS, wkey, sizeof(wkey)); sc->sc_encryption = 0; break; case WI_INTERSIL: val = HOST_ENCRYPT | HOST_DECRYPT; if (vap->iv_flags & IEEE80211_F_PRIVACY) { /* * ONLY HWB3163 EVAL-CARD Firmware version * less than 0.8 variant2 * * If promiscuous mode disable, Prism2 chip * does not work with WEP . * It is under investigation for details. * (ichiro@netbsd.org) */ if (sc->sc_sta_firmware_ver < 802 ) { /* firm ver < 0.8 variant 2 */ wi_write_val(sc, WI_RID_PROMISC, 1); } wi_write_val(sc, WI_RID_CNFAUTHMODE, vap->iv_bss->ni_authmode); val |= PRIVACY_INVOKED; } else { wi_write_val(sc, WI_RID_CNFAUTHMODE, IEEE80211_AUTH_OPEN); } error = wi_write_val(sc, WI_RID_P2_ENCRYPTION, val); if (error) break; sc->sc_encryption = val; if ((val & PRIVACY_INVOKED) == 0) break; error = wi_write_val(sc, WI_RID_P2_TX_CRYPT_KEY, vap->iv_def_txkey); break; } return error; } static int wi_cmd(struct wi_softc *sc, int cmd, int val0, int val1, int val2) { int i, s = 0; if (sc->wi_gone) return (ENODEV); /* wait for the busy bit to clear */ for (i = sc->wi_cmd_count; i > 0; i--) { /* 500ms */ if (!(CSR_READ_2(sc, WI_COMMAND) & WI_CMD_BUSY)) break; DELAY(1*1000); /* 1ms */ } if (i == 0) { device_printf(sc->sc_dev, "%s: busy bit won't clear, cmd 0x%x\n", __func__, cmd); sc->wi_gone = 1; return(ETIMEDOUT); } CSR_WRITE_2(sc, WI_PARAM0, val0); CSR_WRITE_2(sc, WI_PARAM1, val1); CSR_WRITE_2(sc, WI_PARAM2, val2); CSR_WRITE_2(sc, WI_COMMAND, cmd); if (cmd == WI_CMD_INI) { /* XXX: should sleep here. */ DELAY(100*1000); /* 100ms delay for init */ } for (i = 0; i < WI_TIMEOUT; i++) { /* * Wait for 'command complete' bit to be * set in the event status register. */ s = CSR_READ_2(sc, WI_EVENT_STAT); if (s & WI_EV_CMD) { /* Ack the event and read result code. */ s = CSR_READ_2(sc, WI_STATUS); CSR_WRITE_2(sc, WI_EVENT_ACK, WI_EV_CMD); if (s & WI_STAT_CMD_RESULT) { return(EIO); } break; } DELAY(WI_DELAY); } if (i == WI_TIMEOUT) { device_printf(sc->sc_dev, "%s: timeout on cmd 0x%04x; " "event status 0x%04x\n", __func__, cmd, s); if (s == 0xffff) sc->wi_gone = 1; return(ETIMEDOUT); } return (0); } static int wi_seek_bap(struct wi_softc *sc, int id, int off) { int i, status; CSR_WRITE_2(sc, WI_SEL0, id); CSR_WRITE_2(sc, WI_OFF0, off); for (i = 0; ; i++) { status = CSR_READ_2(sc, WI_OFF0); if ((status & WI_OFF_BUSY) == 0) break; if (i == WI_TIMEOUT) { device_printf(sc->sc_dev, "%s: timeout, id %x off %x\n", __func__, id, off); sc->sc_bap_off = WI_OFF_ERR; /* invalidate */ if (status == 0xffff) sc->wi_gone = 1; return ETIMEDOUT; } DELAY(1); } if (status & WI_OFF_ERR) { device_printf(sc->sc_dev, "%s: error, id %x off %x\n", __func__, id, off); sc->sc_bap_off = WI_OFF_ERR; /* invalidate */ return EIO; } sc->sc_bap_id = id; sc->sc_bap_off = off; return 0; } static int wi_read_bap(struct wi_softc *sc, int id, int off, void *buf, int buflen) { int error, cnt; if (buflen == 0) return 0; if (id != sc->sc_bap_id || off != sc->sc_bap_off) { if ((error = wi_seek_bap(sc, id, off)) != 0) return error; } cnt = (buflen + 1) / 2; CSR_READ_MULTI_STREAM_2(sc, WI_DATA0, (u_int16_t *)buf, cnt); sc->sc_bap_off += cnt * 2; return 0; } static int wi_write_bap(struct wi_softc *sc, int id, int off, const void *buf, int buflen) { int error, cnt; if (buflen == 0) return 0; if (id != sc->sc_bap_id || off != sc->sc_bap_off) { if ((error = wi_seek_bap(sc, id, off)) != 0) return error; } cnt = (buflen + 1) / 2; CSR_WRITE_MULTI_STREAM_2(sc, WI_DATA0, (const uint16_t *)buf, cnt); sc->sc_bap_off += cnt * 2; return 0; } static int wi_mwrite_bap(struct wi_softc *sc, int id, int off, struct mbuf *m0, int totlen) { int error, len; struct mbuf *m; for (m = m0; m != NULL && totlen > 0; m = m->m_next) { if (m->m_len == 0) continue; len = min(m->m_len, totlen); if (((u_long)m->m_data) % 2 != 0 || len % 2 != 0) { m_copydata(m, 0, totlen, (caddr_t)&sc->sc_txbuf); return wi_write_bap(sc, id, off, (caddr_t)&sc->sc_txbuf, totlen); } if ((error = wi_write_bap(sc, id, off, m->m_data, len)) != 0) return error; off += m->m_len; totlen -= len; } return 0; } static int wi_alloc_fid(struct wi_softc *sc, int len, int *idp) { int i; if (wi_cmd(sc, WI_CMD_ALLOC_MEM, len, 0, 0)) { device_printf(sc->sc_dev, "%s: failed to allocate %d bytes on NIC\n", __func__, len); return ENOMEM; } for (i = 0; i < WI_TIMEOUT; i++) { if (CSR_READ_2(sc, WI_EVENT_STAT) & WI_EV_ALLOC) break; DELAY(1); } if (i == WI_TIMEOUT) { device_printf(sc->sc_dev, "%s: timeout in alloc\n", __func__); return ETIMEDOUT; } *idp = CSR_READ_2(sc, WI_ALLOC_FID); CSR_WRITE_2(sc, WI_EVENT_ACK, WI_EV_ALLOC); return 0; } static int wi_read_rid(struct wi_softc *sc, int rid, void *buf, int *buflenp) { int error, len; u_int16_t ltbuf[2]; /* Tell the NIC to enter record read mode. */ error = wi_cmd(sc, WI_CMD_ACCESS | WI_ACCESS_READ, rid, 0, 0); if (error) return error; error = wi_read_bap(sc, rid, 0, ltbuf, sizeof(ltbuf)); if (error) return error; if (le16toh(ltbuf[1]) != rid) { device_printf(sc->sc_dev, "record read mismatch, rid=%x, got=%x\n", rid, le16toh(ltbuf[1])); return EIO; } len = (le16toh(ltbuf[0]) - 1) * 2; /* already got rid */ if (*buflenp < len) { device_printf(sc->sc_dev, "record buffer is too small, " "rid=%x, size=%d, len=%d\n", rid, *buflenp, len); return ENOSPC; } *buflenp = len; return wi_read_bap(sc, rid, sizeof(ltbuf), buf, len); } static int wi_write_rid(struct wi_softc *sc, int rid, const void *buf, int buflen) { int error; u_int16_t ltbuf[2]; ltbuf[0] = htole16((buflen + 1) / 2 + 1); /* includes rid */ ltbuf[1] = htole16(rid); error = wi_write_bap(sc, rid, 0, ltbuf, sizeof(ltbuf)); if (error) { device_printf(sc->sc_dev, "%s: bap0 write failure, rid 0x%x\n", __func__, rid); return error; } error = wi_write_bap(sc, rid, sizeof(ltbuf), buf, buflen); if (error) { device_printf(sc->sc_dev, "%s: bap1 write failure, rid 0x%x\n", __func__, rid); return error; } return wi_cmd(sc, WI_CMD_ACCESS | WI_ACCESS_WRITE, rid, 0, 0); } static int wi_write_appie(struct wi_softc *sc, int rid, const struct ieee80211_appie *ie) { /* NB: 42 bytes is probably ok to have on the stack */ char buf[sizeof(uint16_t) + 40]; if (ie->ie_len > 40) return EINVAL; /* NB: firmware requires 16-bit ie length before ie data */ *(uint16_t *) buf = htole16(ie->ie_len); memcpy(buf + sizeof(uint16_t), ie->ie_data, ie->ie_len); return wi_write_rid(sc, rid, buf, ie->ie_len + sizeof(uint16_t)); } int wi_alloc(device_t dev, int rid) { struct wi_softc *sc = device_get_softc(dev); if (sc->wi_bus_type != WI_BUS_PCI_NATIVE) { sc->iobase_rid = rid; sc->iobase = bus_alloc_resource_anywhere(dev, SYS_RES_IOPORT, &sc->iobase_rid, (1 << 6), rman_make_alignment_flags(1 << 6) | RF_ACTIVE); if (sc->iobase == NULL) { device_printf(dev, "No I/O space?!\n"); return ENXIO; } sc->wi_io_addr = rman_get_start(sc->iobase); sc->wi_btag = rman_get_bustag(sc->iobase); sc->wi_bhandle = rman_get_bushandle(sc->iobase); } else { sc->mem_rid = rid; sc->mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &sc->mem_rid, RF_ACTIVE); if (sc->mem == NULL) { device_printf(dev, "No Mem space on prism2.5?\n"); return ENXIO; } sc->wi_btag = rman_get_bustag(sc->mem); sc->wi_bhandle = rman_get_bushandle(sc->mem); } sc->irq_rid = 0; sc->irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->irq_rid, RF_ACTIVE | ((sc->wi_bus_type == WI_BUS_PCCARD) ? 0 : RF_SHAREABLE)); if (sc->irq == NULL) { wi_free(dev); device_printf(dev, "No irq?!\n"); return ENXIO; } sc->sc_dev = dev; sc->sc_unit = device_get_unit(dev); return 0; } void wi_free(device_t dev) { struct wi_softc *sc = device_get_softc(dev); if (sc->iobase != NULL) { bus_release_resource(dev, SYS_RES_IOPORT, sc->iobase_rid, sc->iobase); sc->iobase = NULL; } if (sc->irq != NULL) { bus_release_resource(dev, SYS_RES_IRQ, sc->irq_rid, sc->irq); sc->irq = NULL; } if (sc->mem != NULL) { bus_release_resource(dev, SYS_RES_MEMORY, sc->mem_rid, sc->mem); sc->mem = NULL; } } Index: user/ngie/release-pkg-fix-tests/sys/dev/xen/blkback/blkback.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/dev/xen/blkback/blkback.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/dev/xen/blkback/blkback.c (revision 299091) @@ -1,3891 +1,3887 @@ /*- * Copyright (c) 2009-2012 Spectra Logic Corporation * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions, and the following disclaimer, * without modification. * 2. Redistributions in binary form must reproduce at minimum a disclaimer * substantially similar to the "NO WARRANTY" disclaimer below * ("Disclaimer") and any redistribution must be conditioned upon * including a substantially similar Disclaimer requirement for further * binary redistribution. * * NO WARRANTY * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGES. * * Authors: Justin T. Gibbs (Spectra Logic Corporation) * Ken Merry (Spectra Logic Corporation) */ #include __FBSDID("$FreeBSD$"); /** * \file blkback.c * * \brief Device driver supporting the vending of block storage from * a FreeBSD domain to other domains. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /*--------------------------- Compile-time Tunables --------------------------*/ /** * The maximum number of shared memory ring pages we will allow in a * negotiated block-front/back communication channel. Allow enough * ring space for all requests to be XBB_MAX_REQUEST_SIZE'd. */ #define XBB_MAX_RING_PAGES 32 /** * The maximum number of outstanding request blocks (request headers plus * additional segment blocks) we will allow in a negotiated block-front/back * communication channel. */ #define XBB_MAX_REQUESTS \ __CONST_RING_SIZE(blkif, PAGE_SIZE * XBB_MAX_RING_PAGES) /** * \brief Define to force all I/O to be performed on memory owned by the * backend device, with a copy-in/out to the remote domain's memory. * * \note This option is currently required when this driver's domain is * operating in HVM mode on a system using an IOMMU. * * This driver uses Xen's grant table API to gain access to the memory of * the remote domains it serves. When our domain is operating in PV mode, * the grant table mechanism directly updates our domain's page table entries * to point to the physical pages of the remote domain. This scheme guarantees * that blkback and the backing devices it uses can safely perform DMA * operations to satisfy requests. In HVM mode, Xen may use a HW IOMMU to * insure that our domain cannot DMA to pages owned by another domain. As * of Xen 4.0, IOMMU mappings for HVM guests are not updated via the grant * table API. For this reason, in HVM mode, we must bounce all requests into * memory that is mapped into our domain at domain startup and thus has * valid IOMMU mappings. */ #define XBB_USE_BOUNCE_BUFFERS /** * \brief Define to enable rudimentary request logging to the console. */ #undef XBB_DEBUG /*---------------------------------- Macros ----------------------------------*/ /** * Custom malloc type for all driver allocations. */ static MALLOC_DEFINE(M_XENBLOCKBACK, "xbbd", "Xen Block Back Driver Data"); #ifdef XBB_DEBUG #define DPRINTF(fmt, args...) \ printf("xbb(%s:%d): " fmt, __FUNCTION__, __LINE__, ##args) #else #define DPRINTF(fmt, args...) do {} while(0) #endif /** * The maximum mapped region size per request we will allow in a negotiated * block-front/back communication channel. */ #define XBB_MAX_REQUEST_SIZE \ MIN(MAXPHYS, BLKIF_MAX_SEGMENTS_PER_REQUEST * PAGE_SIZE) /** * The maximum number of segments (within a request header and accompanying * segment blocks) per request we will allow in a negotiated block-front/back * communication channel. */ #define XBB_MAX_SEGMENTS_PER_REQUEST \ (MIN(UIO_MAXIOV, \ MIN(BLKIF_MAX_SEGMENTS_PER_REQUEST, \ (XBB_MAX_REQUEST_SIZE / PAGE_SIZE) + 1))) /** * The maximum number of ring pages that we can allow per request list. * We limit this to the maximum number of segments per request, because * that is already a reasonable number of segments to aggregate. This * number should never be smaller than XBB_MAX_SEGMENTS_PER_REQUEST, * because that would leave situations where we can't dispatch even one * large request. */ #define XBB_MAX_SEGMENTS_PER_REQLIST XBB_MAX_SEGMENTS_PER_REQUEST /*--------------------------- Forward Declarations ---------------------------*/ struct xbb_softc; struct xbb_xen_req; static void xbb_attach_failed(struct xbb_softc *xbb, int err, const char *fmt, ...) __attribute__((format(printf, 3, 4))); static int xbb_shutdown(struct xbb_softc *xbb); static int xbb_detach(device_t dev); /*------------------------------ Data Structures -----------------------------*/ STAILQ_HEAD(xbb_xen_req_list, xbb_xen_req); typedef enum { XBB_REQLIST_NONE = 0x00, XBB_REQLIST_MAPPED = 0x01 } xbb_reqlist_flags; struct xbb_xen_reqlist { /** * Back reference to the parent block back instance for this * request. Used during bio_done handling. */ struct xbb_softc *xbb; /** * BLKIF_OP code for this request. */ int operation; /** * Set to BLKIF_RSP_* to indicate request status. * * This field allows an error status to be recorded even if the * delivery of this status must be deferred. Deferred reporting * is necessary, for example, when an error is detected during * completion processing of one bio when other bios for this * request are still outstanding. */ int status; /** * Number of 512 byte sectors not transferred. */ int residual_512b_sectors; /** * Starting sector number of the first request in the list. */ off_t starting_sector_number; /** * If we're going to coalesce, the next contiguous sector would be * this one. */ off_t next_contig_sector; /** * Number of child requests in the list. */ int num_children; /** * Number of I/O requests still pending on the backend. */ int pendcnt; /** * Total number of segments for requests in the list. */ int nr_segments; /** * Flags for this particular request list. */ xbb_reqlist_flags flags; /** * Kernel virtual address space reserved for this request * list structure and used to map the remote domain's pages for * this I/O, into our domain's address space. */ uint8_t *kva; /** * Base, psuedo-physical address, corresponding to the start * of this request's kva region. */ uint64_t gnt_base; #ifdef XBB_USE_BOUNCE_BUFFERS /** * Pre-allocated domain local memory used to proxy remote * domain memory during I/O operations. */ uint8_t *bounce; #endif /** * Array of grant handles (one per page) used to map this request. */ grant_handle_t *gnt_handles; /** * Device statistics request ordering type (ordered or simple). */ devstat_tag_type ds_tag_type; /** * Device statistics request type (read, write, no_data). */ devstat_trans_flags ds_trans_type; /** * The start time for this request. */ struct bintime ds_t0; /** * Linked list of contiguous requests with the same operation type. */ struct xbb_xen_req_list contig_req_list; /** * Linked list links used to aggregate idle requests in the * request list free pool (xbb->reqlist_free_stailq) and pending * requests waiting for execution (xbb->reqlist_pending_stailq). */ STAILQ_ENTRY(xbb_xen_reqlist) links; }; STAILQ_HEAD(xbb_xen_reqlist_list, xbb_xen_reqlist); /** * \brief Object tracking an in-flight I/O from a Xen VBD consumer. */ struct xbb_xen_req { /** * Linked list links used to aggregate requests into a reqlist * and to store them in the request free pool. */ STAILQ_ENTRY(xbb_xen_req) links; /** * The remote domain's identifier for this I/O request. */ uint64_t id; /** * The number of pages currently mapped for this request. */ int nr_pages; /** * The number of 512 byte sectors comprising this requests. */ int nr_512b_sectors; /** * BLKIF_OP code for this request. */ int operation; /** * Storage used for non-native ring requests. */ blkif_request_t ring_req_storage; /** * Pointer to the Xen request in the ring. */ blkif_request_t *ring_req; /** * Consumer index for this request. */ RING_IDX req_ring_idx; /** * The start time for this request. */ struct bintime ds_t0; /** * Pointer back to our parent request list. */ struct xbb_xen_reqlist *reqlist; }; SLIST_HEAD(xbb_xen_req_slist, xbb_xen_req); /** * \brief Configuration data for the shared memory request ring * used to communicate with the front-end client of this * this driver. */ struct xbb_ring_config { /** KVA address where ring memory is mapped. */ vm_offset_t va; /** The pseudo-physical address where ring memory is mapped.*/ uint64_t gnt_addr; /** * Grant table handles, one per-ring page, returned by the * hyperpervisor upon mapping of the ring and required to * unmap it when a connection is torn down. */ grant_handle_t handle[XBB_MAX_RING_PAGES]; /** * The device bus address returned by the hypervisor when * mapping the ring and required to unmap it when a connection * is torn down. */ uint64_t bus_addr[XBB_MAX_RING_PAGES]; /** The number of ring pages mapped for the current connection. */ u_int ring_pages; /** * The grant references, one per-ring page, supplied by the * front-end, allowing us to reference the ring pages in the * front-end's domain and to map these pages into our own domain. */ grant_ref_t ring_ref[XBB_MAX_RING_PAGES]; /** The interrupt driven even channel used to signal ring events. */ evtchn_port_t evtchn; }; /** * Per-instance connection state flags. */ typedef enum { /** * The front-end requested a read-only mount of the * back-end device/file. */ XBBF_READ_ONLY = 0x01, /** Communication with the front-end has been established. */ XBBF_RING_CONNECTED = 0x02, /** * Front-end requests exist in the ring and are waiting for * xbb_xen_req objects to free up. */ XBBF_RESOURCE_SHORTAGE = 0x04, /** Connection teardown in progress. */ XBBF_SHUTDOWN = 0x08, /** A thread is already performing shutdown processing. */ XBBF_IN_SHUTDOWN = 0x10 } xbb_flag_t; /** Backend device type. */ typedef enum { /** Backend type unknown. */ XBB_TYPE_NONE = 0x00, /** * Backend type disk (access via cdev switch * strategy routine). */ XBB_TYPE_DISK = 0x01, /** Backend type file (access vnode operations.). */ XBB_TYPE_FILE = 0x02 } xbb_type; /** * \brief Structure used to memoize information about a per-request * scatter-gather list. * * The chief benefit of using this data structure is it avoids having * to reparse the possibly discontiguous S/G list in the original * request. Due to the way that the mapping of the memory backing an * I/O transaction is handled by Xen, a second pass is unavoidable. * At least this way the second walk is a simple array traversal. * * \note A single Scatter/Gather element in the block interface covers * at most 1 machine page. In this context a sector (blkif * nomenclature, not what I'd choose) is a 512b aligned unit * of mapping within the machine page referenced by an S/G * element. */ struct xbb_sg { /** The number of 512b data chunks mapped in this S/G element. */ int16_t nsect; /** * The index (0 based) of the first 512b data chunk mapped * in this S/G element. */ uint8_t first_sect; /** * The index (0 based) of the last 512b data chunk mapped * in this S/G element. */ uint8_t last_sect; }; /** * Character device backend specific configuration data. */ struct xbb_dev_data { /** Cdev used for device backend access. */ struct cdev *cdev; /** Cdev switch used for device backend access. */ struct cdevsw *csw; /** Used to hold a reference on opened cdev backend devices. */ int dev_ref; }; /** * File backend specific configuration data. */ struct xbb_file_data { /** Credentials to use for vnode backed (file based) I/O. */ struct ucred *cred; /** * \brief Array of io vectors used to process file based I/O. * * Only a single file based request is outstanding per-xbb instance, * so we only need one of these. */ struct iovec xiovecs[XBB_MAX_SEGMENTS_PER_REQLIST]; #ifdef XBB_USE_BOUNCE_BUFFERS /** * \brief Array of io vectors used to handle bouncing of file reads. * * Vnode operations are free to modify uio data during their * exectuion. In the case of a read with bounce buffering active, * we need some of the data from the original uio in order to * bounce-out the read data. This array serves as the temporary * storage for this saved data. */ struct iovec saved_xiovecs[XBB_MAX_SEGMENTS_PER_REQLIST]; /** * \brief Array of memoized bounce buffer kva offsets used * in the file based backend. * * Due to the way that the mapping of the memory backing an * I/O transaction is handled by Xen, a second pass through * the request sg elements is unavoidable. We memoize the computed * bounce address here to reduce the cost of the second walk. */ void *xiovecs_vaddr[XBB_MAX_SEGMENTS_PER_REQLIST]; #endif /* XBB_USE_BOUNCE_BUFFERS */ }; /** * Collection of backend type specific data. */ union xbb_backend_data { struct xbb_dev_data dev; struct xbb_file_data file; }; /** * Function signature of backend specific I/O handlers. */ typedef int (*xbb_dispatch_t)(struct xbb_softc *xbb, struct xbb_xen_reqlist *reqlist, int operation, int flags); /** * Per-instance configuration data. */ struct xbb_softc { /** * Task-queue used to process I/O requests. */ struct taskqueue *io_taskqueue; /** * Single "run the request queue" task enqueued * on io_taskqueue. */ struct task io_task; /** Device type for this instance. */ xbb_type device_type; /** NewBus device corresponding to this instance. */ device_t dev; /** Backend specific dispatch routine for this instance. */ xbb_dispatch_t dispatch_io; /** The number of requests outstanding on the backend device/file. */ int active_request_count; /** Free pool of request tracking structures. */ struct xbb_xen_req_list request_free_stailq; /** Array, sized at connection time, of request tracking structures. */ struct xbb_xen_req *requests; /** Free pool of request list structures. */ struct xbb_xen_reqlist_list reqlist_free_stailq; /** List of pending request lists awaiting execution. */ struct xbb_xen_reqlist_list reqlist_pending_stailq; /** Array, sized at connection time, of request list structures. */ struct xbb_xen_reqlist *request_lists; /** * Global pool of kva used for mapping remote domain ring * and I/O transaction data. */ vm_offset_t kva; /** Psuedo-physical address corresponding to kva. */ uint64_t gnt_base_addr; /** The size of the global kva pool. */ int kva_size; /** The size of the KVA area used for request lists. */ int reqlist_kva_size; /** The number of pages of KVA used for request lists */ int reqlist_kva_pages; /** Bitmap of free KVA pages */ bitstr_t *kva_free; /** * \brief Cached value of the front-end's domain id. * * This value is used at once for each mapped page in * a transaction. We cache it to avoid incuring the * cost of an ivar access every time this is needed. */ domid_t otherend_id; /** * \brief The blkif protocol abi in effect. * * There are situations where the back and front ends can * have a different, native abi (e.g. intel x86_64 and * 32bit x86 domains on the same machine). The back-end * always accommodates the front-end's native abi. That * value is pulled from the XenStore and recorded here. */ int abi; /** * \brief The maximum number of requests and request lists allowed * to be in flight at a time. * * This value is negotiated via the XenStore. */ u_int max_requests; /** * \brief The maximum number of segments (1 page per segment) * that can be mapped by a request. * * This value is negotiated via the XenStore. */ u_int max_request_segments; /** * \brief Maximum number of segments per request list. * * This value is derived from and will generally be larger than * max_request_segments. */ u_int max_reqlist_segments; /** * The maximum size of any request to this back-end * device. * * This value is negotiated via the XenStore. */ u_int max_request_size; /** * The maximum size of any request list. This is derived directly * from max_reqlist_segments. */ u_int max_reqlist_size; /** Various configuration and state bit flags. */ xbb_flag_t flags; /** Ring mapping and interrupt configuration data. */ struct xbb_ring_config ring_config; /** Runtime, cross-abi safe, structures for ring access. */ blkif_back_rings_t rings; /** IRQ mapping for the communication ring event channel. */ xen_intr_handle_t xen_intr_handle; /** * \brief Backend access mode flags (e.g. write, or read-only). * * This value is passed to us by the front-end via the XenStore. */ char *dev_mode; /** * \brief Backend device type (e.g. "disk", "cdrom", "floppy"). * * This value is passed to us by the front-end via the XenStore. * Currently unused. */ char *dev_type; /** * \brief Backend device/file identifier. * * This value is passed to us by the front-end via the XenStore. * We expect this to be a POSIX path indicating the file or * device to open. */ char *dev_name; /** * Vnode corresponding to the backend device node or file * we are acessing. */ struct vnode *vn; union xbb_backend_data backend; /** The native sector size of the backend. */ u_int sector_size; /** log2 of sector_size. */ u_int sector_size_shift; /** Size in bytes of the backend device or file. */ off_t media_size; /** * \brief media_size expressed in terms of the backend native * sector size. * * (e.g. xbb->media_size >> xbb->sector_size_shift). */ uint64_t media_num_sectors; /** * \brief Array of memoized scatter gather data computed during the * conversion of blkif ring requests to internal xbb_xen_req * structures. * * Ring processing is serialized so we only need one of these. */ struct xbb_sg xbb_sgs[XBB_MAX_SEGMENTS_PER_REQLIST]; /** * Temporary grant table map used in xbb_dispatch_io(). When * XBB_MAX_SEGMENTS_PER_REQLIST gets large, keeping this on the * stack could cause a stack overflow. */ struct gnttab_map_grant_ref maps[XBB_MAX_SEGMENTS_PER_REQLIST]; /** Mutex protecting per-instance data. */ struct mtx lock; /** * Resource representing allocated physical address space * associated with our per-instance kva region. */ struct resource *pseudo_phys_res; /** Resource id for allocated physical address space. */ int pseudo_phys_res_id; /** * I/O statistics from BlockBack dispatch down. These are * coalesced requests, and we start them right before execution. */ struct devstat *xbb_stats; /** * I/O statistics coming into BlockBack. These are the requests as * we get them from BlockFront. They are started as soon as we * receive a request, and completed when the I/O is complete. */ struct devstat *xbb_stats_in; /** Disable sending flush to the backend */ int disable_flush; /** Send a real flush for every N flush requests */ int flush_interval; /** Count of flush requests in the interval */ int flush_count; /** Don't coalesce requests if this is set */ int no_coalesce_reqs; /** Number of requests we have received */ uint64_t reqs_received; /** Number of requests we have completed*/ uint64_t reqs_completed; /** Number of requests we queued but not pushed*/ uint64_t reqs_queued_for_completion; /** Number of requests we completed with an error status*/ uint64_t reqs_completed_with_error; /** How many forced dispatches (i.e. without coalescing) have happened */ uint64_t forced_dispatch; /** How many normal dispatches have happened */ uint64_t normal_dispatch; /** How many total dispatches have happened */ uint64_t total_dispatch; /** How many times we have run out of KVA */ uint64_t kva_shortages; /** How many times we have run out of request structures */ uint64_t request_shortages; }; /*---------------------------- Request Processing ----------------------------*/ /** * Allocate an internal transaction tracking structure from the free pool. * * \param xbb Per-instance xbb configuration structure. * * \return On success, a pointer to the allocated xbb_xen_req structure. * Otherwise NULL. */ static inline struct xbb_xen_req * xbb_get_req(struct xbb_softc *xbb) { struct xbb_xen_req *req; req = NULL; mtx_assert(&xbb->lock, MA_OWNED); if ((req = STAILQ_FIRST(&xbb->request_free_stailq)) != NULL) { STAILQ_REMOVE_HEAD(&xbb->request_free_stailq, links); xbb->active_request_count++; } return (req); } /** * Return an allocated transaction tracking structure to the free pool. * * \param xbb Per-instance xbb configuration structure. * \param req The request structure to free. */ static inline void xbb_release_req(struct xbb_softc *xbb, struct xbb_xen_req *req) { mtx_assert(&xbb->lock, MA_OWNED); STAILQ_INSERT_HEAD(&xbb->request_free_stailq, req, links); xbb->active_request_count--; KASSERT(xbb->active_request_count >= 0, ("xbb_release_req: negative active count")); } /** * Return an xbb_xen_req_list of allocated xbb_xen_reqs to the free pool. * * \param xbb Per-instance xbb configuration structure. * \param req_list The list of requests to free. * \param nreqs The number of items in the list. */ static inline void xbb_release_reqs(struct xbb_softc *xbb, struct xbb_xen_req_list *req_list, int nreqs) { mtx_assert(&xbb->lock, MA_OWNED); STAILQ_CONCAT(&xbb->request_free_stailq, req_list); xbb->active_request_count -= nreqs; KASSERT(xbb->active_request_count >= 0, ("xbb_release_reqs: negative active count")); } /** * Given a page index and 512b sector offset within that page, * calculate an offset into a request's kva region. * * \param reqlist The request structure whose kva region will be accessed. * \param pagenr The page index used to compute the kva offset. * \param sector The 512b sector index used to compute the page relative * kva offset. * * \return The computed global KVA offset. */ static inline uint8_t * xbb_reqlist_vaddr(struct xbb_xen_reqlist *reqlist, int pagenr, int sector) { return (reqlist->kva + (PAGE_SIZE * pagenr) + (sector << 9)); } #ifdef XBB_USE_BOUNCE_BUFFERS /** * Given a page index and 512b sector offset within that page, * calculate an offset into a request's local bounce memory region. * * \param reqlist The request structure whose bounce region will be accessed. * \param pagenr The page index used to compute the bounce offset. * \param sector The 512b sector index used to compute the page relative * bounce offset. * * \return The computed global bounce buffer address. */ static inline uint8_t * xbb_reqlist_bounce_addr(struct xbb_xen_reqlist *reqlist, int pagenr, int sector) { return (reqlist->bounce + (PAGE_SIZE * pagenr) + (sector << 9)); } #endif /** * Given a page number and 512b sector offset within that page, * calculate an offset into the request's memory region that the * underlying backend device/file should use for I/O. * * \param reqlist The request structure whose I/O region will be accessed. * \param pagenr The page index used to compute the I/O offset. * \param sector The 512b sector index used to compute the page relative * I/O offset. * * \return The computed global I/O address. * * Depending on configuration, this will either be a local bounce buffer * or a pointer to the memory mapped in from the front-end domain for * this request. */ static inline uint8_t * xbb_reqlist_ioaddr(struct xbb_xen_reqlist *reqlist, int pagenr, int sector) { #ifdef XBB_USE_BOUNCE_BUFFERS return (xbb_reqlist_bounce_addr(reqlist, pagenr, sector)); #else return (xbb_reqlist_vaddr(reqlist, pagenr, sector)); #endif } /** * Given a page index and 512b sector offset within that page, calculate * an offset into the local psuedo-physical address space used to map a * front-end's request data into a request. * * \param reqlist The request list structure whose pseudo-physical region * will be accessed. * \param pagenr The page index used to compute the pseudo-physical offset. * \param sector The 512b sector index used to compute the page relative * pseudo-physical offset. * * \return The computed global pseudo-phsyical address. * * Depending on configuration, this will either be a local bounce buffer * or a pointer to the memory mapped in from the front-end domain for * this request. */ static inline uintptr_t xbb_get_gntaddr(struct xbb_xen_reqlist *reqlist, int pagenr, int sector) { struct xbb_softc *xbb; xbb = reqlist->xbb; return ((uintptr_t)(xbb->gnt_base_addr + (uintptr_t)(reqlist->kva - xbb->kva) + (PAGE_SIZE * pagenr) + (sector << 9))); } /** * Get Kernel Virtual Address space for mapping requests. * * \param xbb Per-instance xbb configuration structure. * \param nr_pages Number of pages needed. * \param check_only If set, check for free KVA but don't allocate it. * \param have_lock If set, xbb lock is already held. * * \return On success, a pointer to the allocated KVA region. Otherwise NULL. * * Note: This should be unnecessary once we have either chaining or * scatter/gather support for struct bio. At that point we'll be able to * put multiple addresses and lengths in one bio/bio chain and won't need * to map everything into one virtual segment. */ static uint8_t * xbb_get_kva(struct xbb_softc *xbb, int nr_pages) { - intptr_t first_clear; - intptr_t num_clear; + int first_clear; + int num_clear; uint8_t *free_kva; int i; KASSERT(nr_pages != 0, ("xbb_get_kva of zero length")); first_clear = 0; free_kva = NULL; mtx_lock(&xbb->lock); /* * Look for the first available page. If there are none, we're done. */ bit_ffc(xbb->kva_free, xbb->reqlist_kva_pages, &first_clear); if (first_clear == -1) goto bailout; /* * Starting at the first available page, look for consecutive free * pages that will satisfy the user's request. */ for (i = first_clear, num_clear = 0; i < xbb->reqlist_kva_pages; i++) { /* * If this is true, the page is used, so we have to reset * the number of clear pages and the first clear page * (since it pointed to a region with an insufficient number * of clear pages). */ if (bit_test(xbb->kva_free, i)) { num_clear = 0; first_clear = -1; continue; } if (first_clear == -1) first_clear = i; /* * If this is true, we've found a large enough free region * to satisfy the request. */ if (++num_clear == nr_pages) { bit_nset(xbb->kva_free, first_clear, first_clear + nr_pages - 1); free_kva = xbb->kva + - (uint8_t *)(first_clear * PAGE_SIZE); + (uint8_t *)((intptr_t)first_clear * PAGE_SIZE); KASSERT(free_kva >= (uint8_t *)xbb->kva && free_kva + (nr_pages * PAGE_SIZE) <= (uint8_t *)xbb->ring_config.va, ("Free KVA %p len %d out of range, " "kva = %#jx, ring VA = %#jx\n", free_kva, nr_pages * PAGE_SIZE, (uintmax_t)xbb->kva, (uintmax_t)xbb->ring_config.va)); break; } } bailout: if (free_kva == NULL) { xbb->flags |= XBBF_RESOURCE_SHORTAGE; xbb->kva_shortages++; } mtx_unlock(&xbb->lock); return (free_kva); } /** * Free allocated KVA. * * \param xbb Per-instance xbb configuration structure. * \param kva_ptr Pointer to allocated KVA region. * \param nr_pages Number of pages in the KVA region. */ static void xbb_free_kva(struct xbb_softc *xbb, uint8_t *kva_ptr, int nr_pages) { intptr_t start_page; mtx_assert(&xbb->lock, MA_OWNED); start_page = (intptr_t)(kva_ptr - xbb->kva) >> PAGE_SHIFT; bit_nclear(xbb->kva_free, start_page, start_page + nr_pages - 1); } /** * Unmap the front-end pages associated with this I/O request. * * \param req The request structure to unmap. */ static void xbb_unmap_reqlist(struct xbb_xen_reqlist *reqlist) { struct gnttab_unmap_grant_ref unmap[XBB_MAX_SEGMENTS_PER_REQLIST]; u_int i; u_int invcount; int error; invcount = 0; for (i = 0; i < reqlist->nr_segments; i++) { if (reqlist->gnt_handles[i] == GRANT_REF_INVALID) continue; unmap[invcount].host_addr = xbb_get_gntaddr(reqlist, i, 0); unmap[invcount].dev_bus_addr = 0; unmap[invcount].handle = reqlist->gnt_handles[i]; reqlist->gnt_handles[i] = GRANT_REF_INVALID; invcount++; } error = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, unmap, invcount); KASSERT(error == 0, ("Grant table operation failed")); } /** * Allocate an internal transaction tracking structure from the free pool. * * \param xbb Per-instance xbb configuration structure. * * \return On success, a pointer to the allocated xbb_xen_reqlist structure. * Otherwise NULL. */ static inline struct xbb_xen_reqlist * xbb_get_reqlist(struct xbb_softc *xbb) { struct xbb_xen_reqlist *reqlist; reqlist = NULL; mtx_assert(&xbb->lock, MA_OWNED); if ((reqlist = STAILQ_FIRST(&xbb->reqlist_free_stailq)) != NULL) { STAILQ_REMOVE_HEAD(&xbb->reqlist_free_stailq, links); reqlist->flags = XBB_REQLIST_NONE; reqlist->kva = NULL; reqlist->status = BLKIF_RSP_OKAY; reqlist->residual_512b_sectors = 0; reqlist->num_children = 0; reqlist->nr_segments = 0; STAILQ_INIT(&reqlist->contig_req_list); } return (reqlist); } /** * Return an allocated transaction tracking structure to the free pool. * * \param xbb Per-instance xbb configuration structure. * \param req The request list structure to free. * \param wakeup If set, wakeup the work thread if freeing this reqlist * during a resource shortage condition. */ static inline void xbb_release_reqlist(struct xbb_softc *xbb, struct xbb_xen_reqlist *reqlist, int wakeup) { mtx_assert(&xbb->lock, MA_OWNED); if (wakeup) { wakeup = xbb->flags & XBBF_RESOURCE_SHORTAGE; xbb->flags &= ~XBBF_RESOURCE_SHORTAGE; } if (reqlist->kva != NULL) xbb_free_kva(xbb, reqlist->kva, reqlist->nr_segments); xbb_release_reqs(xbb, &reqlist->contig_req_list, reqlist->num_children); STAILQ_INSERT_TAIL(&xbb->reqlist_free_stailq, reqlist, links); if ((xbb->flags & XBBF_SHUTDOWN) != 0) { /* * Shutdown is in progress. See if we can * progress further now that one more request * has completed and been returned to the * free pool. */ xbb_shutdown(xbb); } if (wakeup != 0) taskqueue_enqueue(xbb->io_taskqueue, &xbb->io_task); } /** * Request resources and do basic request setup. * * \param xbb Per-instance xbb configuration structure. * \param reqlist Pointer to reqlist pointer. * \param ring_req Pointer to a block ring request. * \param ring_index The ring index of this request. * * \return 0 for success, non-zero for failure. */ static int xbb_get_resources(struct xbb_softc *xbb, struct xbb_xen_reqlist **reqlist, blkif_request_t *ring_req, RING_IDX ring_idx) { struct xbb_xen_reqlist *nreqlist; struct xbb_xen_req *nreq; nreqlist = NULL; nreq = NULL; mtx_lock(&xbb->lock); /* * We don't allow new resources to be allocated if we're in the * process of shutting down. */ if ((xbb->flags & XBBF_SHUTDOWN) != 0) { mtx_unlock(&xbb->lock); return (1); } /* * Allocate a reqlist if the caller doesn't have one already. */ if (*reqlist == NULL) { nreqlist = xbb_get_reqlist(xbb); if (nreqlist == NULL) goto bailout_error; } /* We always allocate a request. */ nreq = xbb_get_req(xbb); if (nreq == NULL) goto bailout_error; mtx_unlock(&xbb->lock); if (*reqlist == NULL) { *reqlist = nreqlist; nreqlist->operation = ring_req->operation; nreqlist->starting_sector_number = ring_req->sector_number; STAILQ_INSERT_TAIL(&xbb->reqlist_pending_stailq, nreqlist, links); } nreq->reqlist = *reqlist; nreq->req_ring_idx = ring_idx; nreq->id = ring_req->id; nreq->operation = ring_req->operation; if (xbb->abi != BLKIF_PROTOCOL_NATIVE) { bcopy(ring_req, &nreq->ring_req_storage, sizeof(*ring_req)); nreq->ring_req = &nreq->ring_req_storage; } else { nreq->ring_req = ring_req; } binuptime(&nreq->ds_t0); devstat_start_transaction(xbb->xbb_stats_in, &nreq->ds_t0); STAILQ_INSERT_TAIL(&(*reqlist)->contig_req_list, nreq, links); (*reqlist)->num_children++; (*reqlist)->nr_segments += ring_req->nr_segments; return (0); bailout_error: /* * We're out of resources, so set the shortage flag. The next time * a request is released, we'll try waking up the work thread to * see if we can allocate more resources. */ xbb->flags |= XBBF_RESOURCE_SHORTAGE; xbb->request_shortages++; if (nreq != NULL) xbb_release_req(xbb, nreq); if (nreqlist != NULL) xbb_release_reqlist(xbb, nreqlist, /*wakeup*/ 0); mtx_unlock(&xbb->lock); return (1); } /** * Create and queue a response to a blkif request. * * \param xbb Per-instance xbb configuration structure. * \param req The request structure to which to respond. * \param status The status code to report. See BLKIF_RSP_* * in sys/xen/interface/io/blkif.h. */ static void xbb_queue_response(struct xbb_softc *xbb, struct xbb_xen_req *req, int status) { blkif_response_t *resp; /* * The mutex is required here, and should be held across this call * until after the subsequent call to xbb_push_responses(). This * is to guarantee that another context won't queue responses and * push them while we're active. * * That could lead to the other end being notified of responses * before the resources have been freed on this end. The other end * would then be able to queue additional I/O, and we may run out * of resources because we haven't freed them all yet. */ mtx_assert(&xbb->lock, MA_OWNED); /* * Place on the response ring for the relevant domain. * For now, only the spacing between entries is different * in the different ABIs, not the response entry layout. */ switch (xbb->abi) { case BLKIF_PROTOCOL_NATIVE: resp = RING_GET_RESPONSE(&xbb->rings.native, xbb->rings.native.rsp_prod_pvt); break; case BLKIF_PROTOCOL_X86_32: resp = (blkif_response_t *) RING_GET_RESPONSE(&xbb->rings.x86_32, xbb->rings.x86_32.rsp_prod_pvt); break; case BLKIF_PROTOCOL_X86_64: resp = (blkif_response_t *) RING_GET_RESPONSE(&xbb->rings.x86_64, xbb->rings.x86_64.rsp_prod_pvt); break; default: panic("Unexpected blkif protocol ABI."); } resp->id = req->id; resp->operation = req->operation; resp->status = status; if (status != BLKIF_RSP_OKAY) xbb->reqs_completed_with_error++; xbb->rings.common.rsp_prod_pvt++; xbb->reqs_queued_for_completion++; } /** * Send queued responses to blkif requests. * * \param xbb Per-instance xbb configuration structure. * \param run_taskqueue Flag that is set to 1 if the taskqueue * should be run, 0 if it does not need to be run. * \param notify Flag that is set to 1 if the other end should be * notified via irq, 0 if the other end should not be * notified. */ static void xbb_push_responses(struct xbb_softc *xbb, int *run_taskqueue, int *notify) { int more_to_do; /* * The mutex is required here. */ mtx_assert(&xbb->lock, MA_OWNED); more_to_do = 0; RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&xbb->rings.common, *notify); if (xbb->rings.common.rsp_prod_pvt == xbb->rings.common.req_cons) { /* * Tail check for pending requests. Allows frontend to avoid * notifications if requests are already in flight (lower * overheads and promotes batching). */ RING_FINAL_CHECK_FOR_REQUESTS(&xbb->rings.common, more_to_do); } else if (RING_HAS_UNCONSUMED_REQUESTS(&xbb->rings.common)) { more_to_do = 1; } xbb->reqs_completed += xbb->reqs_queued_for_completion; xbb->reqs_queued_for_completion = 0; *run_taskqueue = more_to_do; } /** * Complete a request list. * * \param xbb Per-instance xbb configuration structure. * \param reqlist Allocated internal request list structure. */ static void xbb_complete_reqlist(struct xbb_softc *xbb, struct xbb_xen_reqlist *reqlist) { struct xbb_xen_req *nreq; off_t sectors_sent; int notify, run_taskqueue; sectors_sent = 0; if (reqlist->flags & XBB_REQLIST_MAPPED) xbb_unmap_reqlist(reqlist); mtx_lock(&xbb->lock); /* * All I/O is done, send the response. A lock is not necessary * to protect the request list, because all requests have * completed. Therefore this is the only context accessing this * reqlist right now. However, in order to make sure that no one * else queues responses onto the queue or pushes them to the other * side while we're active, we need to hold the lock across the * calls to xbb_queue_response() and xbb_push_responses(). */ STAILQ_FOREACH(nreq, &reqlist->contig_req_list, links) { off_t cur_sectors_sent; /* Put this response on the ring, but don't push yet */ xbb_queue_response(xbb, nreq, reqlist->status); /* We don't report bytes sent if there is an error. */ if (reqlist->status == BLKIF_RSP_OKAY) cur_sectors_sent = nreq->nr_512b_sectors; else cur_sectors_sent = 0; sectors_sent += cur_sectors_sent; devstat_end_transaction(xbb->xbb_stats_in, /*bytes*/cur_sectors_sent << 9, reqlist->ds_tag_type, reqlist->ds_trans_type, /*now*/NULL, /*then*/&nreq->ds_t0); } /* * Take out any sectors not sent. If we wind up negative (which * might happen if an error is reported as well as a residual), just * report 0 sectors sent. */ sectors_sent -= reqlist->residual_512b_sectors; if (sectors_sent < 0) sectors_sent = 0; devstat_end_transaction(xbb->xbb_stats, /*bytes*/ sectors_sent << 9, reqlist->ds_tag_type, reqlist->ds_trans_type, /*now*/NULL, /*then*/&reqlist->ds_t0); xbb_release_reqlist(xbb, reqlist, /*wakeup*/ 1); xbb_push_responses(xbb, &run_taskqueue, ¬ify); mtx_unlock(&xbb->lock); if (run_taskqueue) taskqueue_enqueue(xbb->io_taskqueue, &xbb->io_task); if (notify) xen_intr_signal(xbb->xen_intr_handle); } /** * Completion handler for buffer I/O requests issued by the device * backend driver. * * \param bio The buffer I/O request on which to perform completion * processing. */ static void xbb_bio_done(struct bio *bio) { struct xbb_softc *xbb; struct xbb_xen_reqlist *reqlist; reqlist = bio->bio_caller1; xbb = reqlist->xbb; reqlist->residual_512b_sectors += bio->bio_resid >> 9; /* * This is a bit imprecise. With aggregated I/O a single * request list can contain multiple front-end requests and * a multiple bios may point to a single request. By carefully * walking the request list, we could map residuals and errors * back to the original front-end request, but the interface * isn't sufficiently rich for us to properly report the error. * So, we just treat the entire request list as having failed if an * error occurs on any part. And, if an error occurs, we treat * the amount of data transferred as 0. * * For residuals, we report it on the overall aggregated device, * but not on the individual requests, since we don't currently * do the work to determine which front-end request to which the * residual applies. */ if (bio->bio_error) { DPRINTF("BIO returned error %d for operation on device %s\n", bio->bio_error, xbb->dev_name); reqlist->status = BLKIF_RSP_ERROR; if (bio->bio_error == ENXIO && xenbus_get_state(xbb->dev) == XenbusStateConnected) { /* * Backend device has disappeared. Signal the * front-end that we (the device proxy) want to * go away. */ xenbus_set_state(xbb->dev, XenbusStateClosing); } } #ifdef XBB_USE_BOUNCE_BUFFERS if (bio->bio_cmd == BIO_READ) { vm_offset_t kva_offset; kva_offset = (vm_offset_t)bio->bio_data - (vm_offset_t)reqlist->bounce; memcpy((uint8_t *)reqlist->kva + kva_offset, bio->bio_data, bio->bio_bcount); } #endif /* XBB_USE_BOUNCE_BUFFERS */ /* * Decrement the pending count for the request list. When we're * done with the requests, send status back for all of them. */ if (atomic_fetchadd_int(&reqlist->pendcnt, -1) == 1) xbb_complete_reqlist(xbb, reqlist); g_destroy_bio(bio); } /** * Parse a blkif request into an internal request structure and send * it to the backend for processing. * * \param xbb Per-instance xbb configuration structure. * \param reqlist Allocated internal request list structure. * * \return On success, 0. For resource shortages, non-zero. * * This routine performs the backend common aspects of request parsing * including compiling an internal request structure, parsing the S/G * list and any secondary ring requests in which they may reside, and * the mapping of front-end I/O pages into our domain. */ static int xbb_dispatch_io(struct xbb_softc *xbb, struct xbb_xen_reqlist *reqlist) { struct xbb_sg *xbb_sg; struct gnttab_map_grant_ref *map; struct blkif_request_segment *sg; struct blkif_request_segment *last_block_sg; struct xbb_xen_req *nreq; u_int nseg; u_int seg_idx; u_int block_segs; int nr_sects; int total_sects; int operation; uint8_t bio_flags; int error; reqlist->ds_tag_type = DEVSTAT_TAG_SIMPLE; bio_flags = 0; total_sects = 0; nr_sects = 0; /* * First determine whether we have enough free KVA to satisfy this * request list. If not, tell xbb_run_queue() so it can go to * sleep until we have more KVA. */ reqlist->kva = NULL; if (reqlist->nr_segments != 0) { reqlist->kva = xbb_get_kva(xbb, reqlist->nr_segments); if (reqlist->kva == NULL) { /* * If we're out of KVA, return ENOMEM. */ return (ENOMEM); } } binuptime(&reqlist->ds_t0); devstat_start_transaction(xbb->xbb_stats, &reqlist->ds_t0); switch (reqlist->operation) { case BLKIF_OP_WRITE_BARRIER: bio_flags |= BIO_ORDERED; reqlist->ds_tag_type = DEVSTAT_TAG_ORDERED; /* FALLTHROUGH */ case BLKIF_OP_WRITE: operation = BIO_WRITE; reqlist->ds_trans_type = DEVSTAT_WRITE; if ((xbb->flags & XBBF_READ_ONLY) != 0) { DPRINTF("Attempt to write to read only device %s\n", xbb->dev_name); reqlist->status = BLKIF_RSP_ERROR; goto send_response; } break; case BLKIF_OP_READ: operation = BIO_READ; reqlist->ds_trans_type = DEVSTAT_READ; break; case BLKIF_OP_FLUSH_DISKCACHE: /* * If this is true, the user has requested that we disable * flush support. So we just complete the requests * successfully. */ if (xbb->disable_flush != 0) { goto send_response; } /* * The user has requested that we only send a real flush * for every N flush requests. So keep count, and either * complete the request immediately or queue it for the * backend. */ if (xbb->flush_interval != 0) { if (++(xbb->flush_count) < xbb->flush_interval) { goto send_response; } else xbb->flush_count = 0; } operation = BIO_FLUSH; reqlist->ds_tag_type = DEVSTAT_TAG_ORDERED; reqlist->ds_trans_type = DEVSTAT_NO_DATA; goto do_dispatch; /*NOTREACHED*/ default: DPRINTF("error: unknown block io operation [%d]\n", reqlist->operation); reqlist->status = BLKIF_RSP_ERROR; goto send_response; } reqlist->xbb = xbb; xbb_sg = xbb->xbb_sgs; map = xbb->maps; seg_idx = 0; STAILQ_FOREACH(nreq, &reqlist->contig_req_list, links) { blkif_request_t *ring_req; RING_IDX req_ring_idx; u_int req_seg_idx; ring_req = nreq->ring_req; req_ring_idx = nreq->req_ring_idx; nr_sects = 0; nseg = ring_req->nr_segments; nreq->nr_pages = nseg; nreq->nr_512b_sectors = 0; req_seg_idx = 0; sg = NULL; /* Check that number of segments is sane. */ if (__predict_false(nseg == 0) || __predict_false(nseg > xbb->max_request_segments)) { DPRINTF("Bad number of segments in request (%d)\n", nseg); reqlist->status = BLKIF_RSP_ERROR; goto send_response; } block_segs = nseg; sg = ring_req->seg; last_block_sg = sg + block_segs; while (sg < last_block_sg) { KASSERT(seg_idx < XBB_MAX_SEGMENTS_PER_REQLIST, ("seg_idx %d is too large, max " "segs %d\n", seg_idx, XBB_MAX_SEGMENTS_PER_REQLIST)); xbb_sg->first_sect = sg->first_sect; xbb_sg->last_sect = sg->last_sect; xbb_sg->nsect = (int8_t)(sg->last_sect - sg->first_sect + 1); if ((sg->last_sect >= (PAGE_SIZE >> 9)) || (xbb_sg->nsect <= 0)) { reqlist->status = BLKIF_RSP_ERROR; goto send_response; } nr_sects += xbb_sg->nsect; map->host_addr = xbb_get_gntaddr(reqlist, seg_idx, /*sector*/0); KASSERT(map->host_addr + PAGE_SIZE <= xbb->ring_config.gnt_addr, ("Host address %#jx len %d overlaps " "ring address %#jx\n", (uintmax_t)map->host_addr, PAGE_SIZE, (uintmax_t)xbb->ring_config.gnt_addr)); map->flags = GNTMAP_host_map; map->ref = sg->gref; map->dom = xbb->otherend_id; if (operation == BIO_WRITE) map->flags |= GNTMAP_readonly; sg++; map++; xbb_sg++; seg_idx++; req_seg_idx++; } /* Convert to the disk's sector size */ nreq->nr_512b_sectors = nr_sects; nr_sects = (nr_sects << 9) >> xbb->sector_size_shift; total_sects += nr_sects; if ((nreq->nr_512b_sectors & ((xbb->sector_size >> 9) - 1)) != 0) { device_printf(xbb->dev, "%s: I/O size (%d) is not " "a multiple of the backing store sector " "size (%d)\n", __func__, nreq->nr_512b_sectors << 9, xbb->sector_size); reqlist->status = BLKIF_RSP_ERROR; goto send_response; } } error = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, xbb->maps, reqlist->nr_segments); if (error != 0) panic("Grant table operation failed (%d)", error); reqlist->flags |= XBB_REQLIST_MAPPED; for (seg_idx = 0, map = xbb->maps; seg_idx < reqlist->nr_segments; seg_idx++, map++){ if (__predict_false(map->status != 0)) { DPRINTF("invalid buffer -- could not remap " "it (%d)\n", map->status); DPRINTF("Mapping(%d): Host Addr 0x%"PRIx64", flags " "0x%x ref 0x%x, dom %d\n", seg_idx, map->host_addr, map->flags, map->ref, map->dom); reqlist->status = BLKIF_RSP_ERROR; goto send_response; } reqlist->gnt_handles[seg_idx] = map->handle; } if (reqlist->starting_sector_number + total_sects > xbb->media_num_sectors) { DPRINTF("%s of [%" PRIu64 ",%" PRIu64 "] " "extends past end of device %s\n", operation == BIO_READ ? "read" : "write", reqlist->starting_sector_number, reqlist->starting_sector_number + total_sects, xbb->dev_name); reqlist->status = BLKIF_RSP_ERROR; goto send_response; } do_dispatch: error = xbb->dispatch_io(xbb, reqlist, operation, bio_flags); if (error != 0) { reqlist->status = BLKIF_RSP_ERROR; goto send_response; } return (0); send_response: xbb_complete_reqlist(xbb, reqlist); return (0); } static __inline int xbb_count_sects(blkif_request_t *ring_req) { int i; int cur_size = 0; for (i = 0; i < ring_req->nr_segments; i++) { int nsect; nsect = (int8_t)(ring_req->seg[i].last_sect - ring_req->seg[i].first_sect + 1); if (nsect <= 0) break; cur_size += nsect; } return (cur_size); } /** * Process incoming requests from the shared communication ring in response * to a signal on the ring's event channel. * * \param context Callback argument registerd during task initialization - * the xbb_softc for this instance. * \param pending The number of taskqueue_enqueue events that have * occurred since this handler was last run. */ static void xbb_run_queue(void *context, int pending) { struct xbb_softc *xbb; blkif_back_rings_t *rings; RING_IDX rp; uint64_t cur_sector; int cur_operation; struct xbb_xen_reqlist *reqlist; xbb = (struct xbb_softc *)context; rings = &xbb->rings; /* * Work gather and dispatch loop. Note that we have a bias here * towards gathering I/O sent by blockfront. We first gather up * everything in the ring, as long as we have resources. Then we * dispatch one request, and then attempt to gather up any * additional requests that have come in while we were dispatching * the request. * * This allows us to get a clearer picture (via devstat) of how * many requests blockfront is queueing to us at any given time. */ for (;;) { int retval; /* * Initialize reqlist to the last element in the pending * queue, if there is one. This allows us to add more * requests to that request list, if we have room. */ reqlist = STAILQ_LAST(&xbb->reqlist_pending_stailq, xbb_xen_reqlist, links); if (reqlist != NULL) { cur_sector = reqlist->next_contig_sector; cur_operation = reqlist->operation; } else { cur_operation = 0; cur_sector = 0; } /* * Cache req_prod to avoid accessing a cache line shared * with the frontend. */ rp = rings->common.sring->req_prod; /* Ensure we see queued requests up to 'rp'. */ rmb(); /** * Run so long as there is work to consume and the generation * of a response will not overflow the ring. * * @note There's a 1 to 1 relationship between requests and * responses, so an overflow should never occur. This * test is to protect our domain from digesting bogus * data. Shouldn't we log this? */ while (rings->common.req_cons != rp && RING_REQUEST_CONS_OVERFLOW(&rings->common, rings->common.req_cons) == 0){ blkif_request_t ring_req_storage; blkif_request_t *ring_req; int cur_size; switch (xbb->abi) { case BLKIF_PROTOCOL_NATIVE: ring_req = RING_GET_REQUEST(&xbb->rings.native, rings->common.req_cons); break; case BLKIF_PROTOCOL_X86_32: { struct blkif_x86_32_request *ring_req32; ring_req32 = RING_GET_REQUEST( &xbb->rings.x86_32, rings->common.req_cons); blkif_get_x86_32_req(&ring_req_storage, ring_req32); ring_req = &ring_req_storage; break; } case BLKIF_PROTOCOL_X86_64: { struct blkif_x86_64_request *ring_req64; ring_req64 =RING_GET_REQUEST(&xbb->rings.x86_64, rings->common.req_cons); blkif_get_x86_64_req(&ring_req_storage, ring_req64); ring_req = &ring_req_storage; break; } default: panic("Unexpected blkif protocol ABI."); /* NOTREACHED */ } /* * Check for situations that would require closing * off this I/O for further coalescing: * - Coalescing is turned off. * - Current I/O is out of sequence with the previous * I/O. * - Coalesced I/O would be too large. */ if ((reqlist != NULL) && ((xbb->no_coalesce_reqs != 0) || ((xbb->no_coalesce_reqs == 0) && ((ring_req->sector_number != cur_sector) || (ring_req->operation != cur_operation) || ((ring_req->nr_segments + reqlist->nr_segments) > xbb->max_reqlist_segments))))) { reqlist = NULL; } /* * Grab and check for all resources in one shot. * If we can't get all of the resources we need, * the shortage is noted and the thread will get * woken up when more resources are available. */ retval = xbb_get_resources(xbb, &reqlist, ring_req, xbb->rings.common.req_cons); if (retval != 0) { /* * Resource shortage has been recorded. * We'll be scheduled to run once a request * object frees up due to a completion. */ break; } /* * Signify that we can overwrite this request with * a response by incrementing our consumer index. * The response won't be generated until after * we've already consumed all necessary data out * of the version of the request in the ring buffer * (for native mode). We must update the consumer * index before issuing back-end I/O so there is * no possibility that it will complete and a * response be generated before we make room in * the queue for that response. */ xbb->rings.common.req_cons++; xbb->reqs_received++; cur_size = xbb_count_sects(ring_req); cur_sector = ring_req->sector_number + cur_size; reqlist->next_contig_sector = cur_sector; cur_operation = ring_req->operation; } /* Check for I/O to dispatch */ reqlist = STAILQ_FIRST(&xbb->reqlist_pending_stailq); if (reqlist == NULL) { /* * We're out of work to do, put the task queue to * sleep. */ break; } /* * Grab the first request off the queue and attempt * to dispatch it. */ STAILQ_REMOVE_HEAD(&xbb->reqlist_pending_stailq, links); retval = xbb_dispatch_io(xbb, reqlist); if (retval != 0) { /* * xbb_dispatch_io() returns non-zero only when * there is a resource shortage. If that's the * case, re-queue this request on the head of the * queue, and go to sleep until we have more * resources. */ STAILQ_INSERT_HEAD(&xbb->reqlist_pending_stailq, reqlist, links); break; } else { /* * If we still have anything on the queue after * removing the head entry, that is because we * met one of the criteria to create a new * request list (outlined above), and we'll call * that a forced dispatch for statistical purposes. * * Otherwise, if there is only one element on the * queue, we coalesced everything available on * the ring and we'll call that a normal dispatch. */ reqlist = STAILQ_FIRST(&xbb->reqlist_pending_stailq); if (reqlist != NULL) xbb->forced_dispatch++; else xbb->normal_dispatch++; xbb->total_dispatch++; } } } /** * Interrupt handler bound to the shared ring's event channel. * * \param arg Callback argument registerd during event channel * binding - the xbb_softc for this instance. */ static int xbb_filter(void *arg) { struct xbb_softc *xbb; /* Defer to taskqueue thread. */ xbb = (struct xbb_softc *)arg; taskqueue_enqueue(xbb->io_taskqueue, &xbb->io_task); return (FILTER_HANDLED); } SDT_PROVIDER_DEFINE(xbb); SDT_PROBE_DEFINE1(xbb, kernel, xbb_dispatch_dev, flush, "int"); SDT_PROBE_DEFINE3(xbb, kernel, xbb_dispatch_dev, read, "int", "uint64_t", "uint64_t"); SDT_PROBE_DEFINE3(xbb, kernel, xbb_dispatch_dev, write, "int", "uint64_t", "uint64_t"); /*----------------------------- Backend Handlers -----------------------------*/ /** * Backend handler for character device access. * * \param xbb Per-instance xbb configuration structure. * \param reqlist Allocated internal request list structure. * \param operation BIO_* I/O operation code. * \param bio_flags Additional bio_flag data to pass to any generated * bios (e.g. BIO_ORDERED).. * * \return 0 for success, errno codes for failure. */ static int xbb_dispatch_dev(struct xbb_softc *xbb, struct xbb_xen_reqlist *reqlist, int operation, int bio_flags) { struct xbb_dev_data *dev_data; struct bio *bios[XBB_MAX_SEGMENTS_PER_REQLIST]; off_t bio_offset; struct bio *bio; struct xbb_sg *xbb_sg; u_int nbio; u_int bio_idx; u_int nseg; u_int seg_idx; int error; dev_data = &xbb->backend.dev; bio_offset = (off_t)reqlist->starting_sector_number << xbb->sector_size_shift; error = 0; nbio = 0; bio_idx = 0; if (operation == BIO_FLUSH) { bio = g_new_bio(); if (__predict_false(bio == NULL)) { DPRINTF("Unable to allocate bio for BIO_FLUSH\n"); error = ENOMEM; return (error); } bio->bio_cmd = BIO_FLUSH; bio->bio_flags |= BIO_ORDERED; bio->bio_dev = dev_data->cdev; bio->bio_offset = 0; bio->bio_data = 0; bio->bio_done = xbb_bio_done; bio->bio_caller1 = reqlist; bio->bio_pblkno = 0; reqlist->pendcnt = 1; SDT_PROBE1(xbb, kernel, xbb_dispatch_dev, flush, device_get_unit(xbb->dev)); (*dev_data->csw->d_strategy)(bio); return (0); } xbb_sg = xbb->xbb_sgs; bio = NULL; nseg = reqlist->nr_segments; for (seg_idx = 0; seg_idx < nseg; seg_idx++, xbb_sg++) { /* * KVA will not be contiguous, so any additional * I/O will need to be represented in a new bio. */ if ((bio != NULL) && (xbb_sg->first_sect != 0)) { if ((bio->bio_length & (xbb->sector_size - 1)) != 0) { printf("%s: Discontiguous I/O request " "from domain %d ends on " "non-sector boundary\n", __func__, xbb->otherend_id); error = EINVAL; goto fail_free_bios; } bio = NULL; } if (bio == NULL) { /* * Make sure that the start of this bio is * aligned to a device sector. */ if ((bio_offset & (xbb->sector_size - 1)) != 0){ printf("%s: Misaligned I/O request " "from domain %d\n", __func__, xbb->otherend_id); error = EINVAL; goto fail_free_bios; } bio = bios[nbio++] = g_new_bio(); if (__predict_false(bio == NULL)) { error = ENOMEM; goto fail_free_bios; } bio->bio_cmd = operation; bio->bio_flags |= bio_flags; bio->bio_dev = dev_data->cdev; bio->bio_offset = bio_offset; bio->bio_data = xbb_reqlist_ioaddr(reqlist, seg_idx, xbb_sg->first_sect); bio->bio_done = xbb_bio_done; bio->bio_caller1 = reqlist; bio->bio_pblkno = bio_offset >> xbb->sector_size_shift; } bio->bio_length += xbb_sg->nsect << 9; bio->bio_bcount = bio->bio_length; bio_offset += xbb_sg->nsect << 9; if (xbb_sg->last_sect != (PAGE_SIZE - 512) >> 9) { if ((bio->bio_length & (xbb->sector_size - 1)) != 0) { printf("%s: Discontiguous I/O request " "from domain %d ends on " "non-sector boundary\n", __func__, xbb->otherend_id); error = EINVAL; goto fail_free_bios; } /* * KVA will not be contiguous, so any additional * I/O will need to be represented in a new bio. */ bio = NULL; } } reqlist->pendcnt = nbio; for (bio_idx = 0; bio_idx < nbio; bio_idx++) { #ifdef XBB_USE_BOUNCE_BUFFERS vm_offset_t kva_offset; kva_offset = (vm_offset_t)bios[bio_idx]->bio_data - (vm_offset_t)reqlist->bounce; if (operation == BIO_WRITE) { memcpy(bios[bio_idx]->bio_data, (uint8_t *)reqlist->kva + kva_offset, bios[bio_idx]->bio_bcount); } #endif if (operation == BIO_READ) { SDT_PROBE3(xbb, kernel, xbb_dispatch_dev, read, device_get_unit(xbb->dev), bios[bio_idx]->bio_offset, bios[bio_idx]->bio_length); } else if (operation == BIO_WRITE) { SDT_PROBE3(xbb, kernel, xbb_dispatch_dev, write, device_get_unit(xbb->dev), bios[bio_idx]->bio_offset, bios[bio_idx]->bio_length); } (*dev_data->csw->d_strategy)(bios[bio_idx]); } return (error); fail_free_bios: for (bio_idx = 0; bio_idx < (nbio-1); bio_idx++) g_destroy_bio(bios[bio_idx]); return (error); } SDT_PROBE_DEFINE1(xbb, kernel, xbb_dispatch_file, flush, "int"); SDT_PROBE_DEFINE3(xbb, kernel, xbb_dispatch_file, read, "int", "uint64_t", "uint64_t"); SDT_PROBE_DEFINE3(xbb, kernel, xbb_dispatch_file, write, "int", "uint64_t", "uint64_t"); /** * Backend handler for file access. * * \param xbb Per-instance xbb configuration structure. * \param reqlist Allocated internal request list. * \param operation BIO_* I/O operation code. * \param flags Additional bio_flag data to pass to any generated bios * (e.g. BIO_ORDERED).. * * \return 0 for success, errno codes for failure. */ static int xbb_dispatch_file(struct xbb_softc *xbb, struct xbb_xen_reqlist *reqlist, int operation, int flags) { struct xbb_file_data *file_data; u_int seg_idx; u_int nseg; struct uio xuio; struct xbb_sg *xbb_sg; struct iovec *xiovec; #ifdef XBB_USE_BOUNCE_BUFFERS void **p_vaddr; int saved_uio_iovcnt; #endif /* XBB_USE_BOUNCE_BUFFERS */ int error; file_data = &xbb->backend.file; error = 0; bzero(&xuio, sizeof(xuio)); switch (operation) { case BIO_READ: xuio.uio_rw = UIO_READ; break; case BIO_WRITE: xuio.uio_rw = UIO_WRITE; break; case BIO_FLUSH: { struct mount *mountpoint; SDT_PROBE1(xbb, kernel, xbb_dispatch_file, flush, device_get_unit(xbb->dev)); (void) vn_start_write(xbb->vn, &mountpoint, V_WAIT); vn_lock(xbb->vn, LK_EXCLUSIVE | LK_RETRY); error = VOP_FSYNC(xbb->vn, MNT_WAIT, curthread); VOP_UNLOCK(xbb->vn, 0); vn_finished_write(mountpoint); goto bailout_send_response; /* NOTREACHED */ } default: panic("invalid operation %d", operation); /* NOTREACHED */ } xuio.uio_offset = (vm_offset_t)reqlist->starting_sector_number << xbb->sector_size_shift; xuio.uio_segflg = UIO_SYSSPACE; xuio.uio_iov = file_data->xiovecs; xuio.uio_iovcnt = 0; xbb_sg = xbb->xbb_sgs; nseg = reqlist->nr_segments; for (xiovec = NULL, seg_idx = 0; seg_idx < nseg; seg_idx++, xbb_sg++) { /* * If the first sector is not 0, the KVA will * not be contiguous and we'll need to go on * to another segment. */ if (xbb_sg->first_sect != 0) xiovec = NULL; if (xiovec == NULL) { xiovec = &file_data->xiovecs[xuio.uio_iovcnt]; xiovec->iov_base = xbb_reqlist_ioaddr(reqlist, seg_idx, xbb_sg->first_sect); #ifdef XBB_USE_BOUNCE_BUFFERS /* * Store the address of the incoming * buffer at this particular offset * as well, so we can do the copy * later without having to do more * work to recalculate this address. */ p_vaddr = &file_data->xiovecs_vaddr[xuio.uio_iovcnt]; *p_vaddr = xbb_reqlist_vaddr(reqlist, seg_idx, xbb_sg->first_sect); #endif /* XBB_USE_BOUNCE_BUFFERS */ xiovec->iov_len = 0; xuio.uio_iovcnt++; } xiovec->iov_len += xbb_sg->nsect << 9; xuio.uio_resid += xbb_sg->nsect << 9; /* * If the last sector is not the full page * size count, the next segment will not be * contiguous in KVA and we need a new iovec. */ if (xbb_sg->last_sect != (PAGE_SIZE - 512) >> 9) xiovec = NULL; } xuio.uio_td = curthread; #ifdef XBB_USE_BOUNCE_BUFFERS saved_uio_iovcnt = xuio.uio_iovcnt; if (operation == BIO_WRITE) { /* Copy the write data to the local buffer. */ for (seg_idx = 0, p_vaddr = file_data->xiovecs_vaddr, xiovec = xuio.uio_iov; seg_idx < xuio.uio_iovcnt; seg_idx++, xiovec++, p_vaddr++) { memcpy(xiovec->iov_base, *p_vaddr, xiovec->iov_len); } } else { /* * We only need to save off the iovecs in the case of a * read, because the copy for the read happens after the * VOP_READ(). (The uio will get modified in that call * sequence.) */ memcpy(file_data->saved_xiovecs, xuio.uio_iov, xuio.uio_iovcnt * sizeof(xuio.uio_iov[0])); } #endif /* XBB_USE_BOUNCE_BUFFERS */ switch (operation) { case BIO_READ: SDT_PROBE3(xbb, kernel, xbb_dispatch_file, read, device_get_unit(xbb->dev), xuio.uio_offset, xuio.uio_resid); vn_lock(xbb->vn, LK_EXCLUSIVE | LK_RETRY); /* * UFS pays attention to IO_DIRECT for reads. If the * DIRECTIO option is configured into the kernel, it calls * ffs_rawread(). But that only works for single-segment * uios with user space addresses. In our case, with a * kernel uio, it still reads into the buffer cache, but it * will just try to release the buffer from the cache later * on in ffs_read(). * * ZFS does not pay attention to IO_DIRECT for reads. * * UFS does not pay attention to IO_SYNC for reads. * * ZFS pays attention to IO_SYNC (which translates into the * Solaris define FRSYNC for zfs_read()) for reads. It * attempts to sync the file before reading. * * So, to attempt to provide some barrier semantics in the * BIO_ORDERED case, set both IO_DIRECT and IO_SYNC. */ error = VOP_READ(xbb->vn, &xuio, (flags & BIO_ORDERED) ? (IO_DIRECT|IO_SYNC) : 0, file_data->cred); VOP_UNLOCK(xbb->vn, 0); break; case BIO_WRITE: { struct mount *mountpoint; SDT_PROBE3(xbb, kernel, xbb_dispatch_file, write, device_get_unit(xbb->dev), xuio.uio_offset, xuio.uio_resid); (void)vn_start_write(xbb->vn, &mountpoint, V_WAIT); vn_lock(xbb->vn, LK_EXCLUSIVE | LK_RETRY); /* * UFS pays attention to IO_DIRECT for writes. The write * is done asynchronously. (Normally the write would just * get put into cache. * * UFS pays attention to IO_SYNC for writes. It will * attempt to write the buffer out synchronously if that * flag is set. * * ZFS does not pay attention to IO_DIRECT for writes. * * ZFS pays attention to IO_SYNC (a.k.a. FSYNC or FRSYNC) * for writes. It will flush the transaction from the * cache before returning. * * So if we've got the BIO_ORDERED flag set, we want * IO_SYNC in either the UFS or ZFS case. */ error = VOP_WRITE(xbb->vn, &xuio, (flags & BIO_ORDERED) ? IO_SYNC : 0, file_data->cred); VOP_UNLOCK(xbb->vn, 0); vn_finished_write(mountpoint); break; } default: panic("invalid operation %d", operation); /* NOTREACHED */ } #ifdef XBB_USE_BOUNCE_BUFFERS /* We only need to copy here for read operations */ if (operation == BIO_READ) { for (seg_idx = 0, p_vaddr = file_data->xiovecs_vaddr, xiovec = file_data->saved_xiovecs; seg_idx < saved_uio_iovcnt; seg_idx++, xiovec++, p_vaddr++) { /* * Note that we have to use the copy of the * io vector we made above. uiomove() modifies * the uio and its referenced vector as uiomove * performs the copy, so we can't rely on any * state from the original uio. */ memcpy(*p_vaddr, xiovec->iov_base, xiovec->iov_len); } } #endif /* XBB_USE_BOUNCE_BUFFERS */ bailout_send_response: if (error != 0) reqlist->status = BLKIF_RSP_ERROR; xbb_complete_reqlist(xbb, reqlist); return (0); } /*--------------------------- Backend Configuration --------------------------*/ /** * Close and cleanup any backend device/file specific state for this * block back instance. * * \param xbb Per-instance xbb configuration structure. */ static void xbb_close_backend(struct xbb_softc *xbb) { DROP_GIANT(); DPRINTF("closing dev=%s\n", xbb->dev_name); if (xbb->vn) { int flags = FREAD; if ((xbb->flags & XBBF_READ_ONLY) == 0) flags |= FWRITE; switch (xbb->device_type) { case XBB_TYPE_DISK: if (xbb->backend.dev.csw) { dev_relthread(xbb->backend.dev.cdev, xbb->backend.dev.dev_ref); xbb->backend.dev.csw = NULL; xbb->backend.dev.cdev = NULL; } break; case XBB_TYPE_FILE: break; case XBB_TYPE_NONE: default: panic("Unexpected backend type."); break; } (void)vn_close(xbb->vn, flags, NOCRED, curthread); xbb->vn = NULL; switch (xbb->device_type) { case XBB_TYPE_DISK: break; case XBB_TYPE_FILE: if (xbb->backend.file.cred != NULL) { crfree(xbb->backend.file.cred); xbb->backend.file.cred = NULL; } break; case XBB_TYPE_NONE: default: panic("Unexpected backend type."); break; } } PICKUP_GIANT(); } /** * Open a character device to be used for backend I/O. * * \param xbb Per-instance xbb configuration structure. * * \return 0 for success, errno codes for failure. */ static int xbb_open_dev(struct xbb_softc *xbb) { struct vattr vattr; struct cdev *dev; struct cdevsw *devsw; int error; xbb->device_type = XBB_TYPE_DISK; xbb->dispatch_io = xbb_dispatch_dev; xbb->backend.dev.cdev = xbb->vn->v_rdev; xbb->backend.dev.csw = dev_refthread(xbb->backend.dev.cdev, &xbb->backend.dev.dev_ref); if (xbb->backend.dev.csw == NULL) panic("Unable to retrieve device switch"); error = VOP_GETATTR(xbb->vn, &vattr, NOCRED); if (error) { xenbus_dev_fatal(xbb->dev, error, "error getting " "vnode attributes for device %s", xbb->dev_name); return (error); } dev = xbb->vn->v_rdev; devsw = dev->si_devsw; if (!devsw->d_ioctl) { xenbus_dev_fatal(xbb->dev, ENODEV, "no d_ioctl for " "device %s!", xbb->dev_name); return (ENODEV); } error = devsw->d_ioctl(dev, DIOCGSECTORSIZE, (caddr_t)&xbb->sector_size, FREAD, curthread); if (error) { xenbus_dev_fatal(xbb->dev, error, "error calling ioctl DIOCGSECTORSIZE " "for device %s", xbb->dev_name); return (error); } error = devsw->d_ioctl(dev, DIOCGMEDIASIZE, (caddr_t)&xbb->media_size, FREAD, curthread); if (error) { xenbus_dev_fatal(xbb->dev, error, "error calling ioctl DIOCGMEDIASIZE " "for device %s", xbb->dev_name); return (error); } return (0); } /** * Open a file to be used for backend I/O. * * \param xbb Per-instance xbb configuration structure. * * \return 0 for success, errno codes for failure. */ static int xbb_open_file(struct xbb_softc *xbb) { struct xbb_file_data *file_data; struct vattr vattr; int error; file_data = &xbb->backend.file; xbb->device_type = XBB_TYPE_FILE; xbb->dispatch_io = xbb_dispatch_file; error = VOP_GETATTR(xbb->vn, &vattr, curthread->td_ucred); if (error != 0) { xenbus_dev_fatal(xbb->dev, error, "error calling VOP_GETATTR()" "for file %s", xbb->dev_name); return (error); } /* * Verify that we have the ability to upgrade to exclusive * access on this file so we can trap errors at open instead * of reporting them during first access. */ if (VOP_ISLOCKED(xbb->vn) != LK_EXCLUSIVE) { vn_lock(xbb->vn, LK_UPGRADE | LK_RETRY); if (xbb->vn->v_iflag & VI_DOOMED) { error = EBADF; xenbus_dev_fatal(xbb->dev, error, "error locking file %s", xbb->dev_name); return (error); } } file_data->cred = crhold(curthread->td_ucred); xbb->media_size = vattr.va_size; /* * XXX KDM vattr.va_blocksize may be larger than 512 bytes here. * With ZFS, it is 131072 bytes. Block sizes that large don't work * with disklabel and UFS on FreeBSD at least. Large block sizes * may not work with other OSes as well. So just export a sector * size of 512 bytes, which should work with any OS or * application. Since our backing is a file, any block size will * work fine for the backing store. */ #if 0 xbb->sector_size = vattr.va_blocksize; #endif xbb->sector_size = 512; /* * Sanity check. The media size has to be at least one * sector long. */ if (xbb->media_size < xbb->sector_size) { error = EINVAL; xenbus_dev_fatal(xbb->dev, error, "file %s size %ju < block size %u", xbb->dev_name, (uintmax_t)xbb->media_size, xbb->sector_size); } return (error); } /** * Open the backend provider for this connection. * * \param xbb Per-instance xbb configuration structure. * * \return 0 for success, errno codes for failure. */ static int xbb_open_backend(struct xbb_softc *xbb) { struct nameidata nd; int flags; int error; flags = FREAD; error = 0; DPRINTF("opening dev=%s\n", xbb->dev_name); if (rootvnode == NULL) { xenbus_dev_fatal(xbb->dev, ENOENT, "Root file system not mounted"); return (ENOENT); } if ((xbb->flags & XBBF_READ_ONLY) == 0) flags |= FWRITE; pwd_ensure_dirs(); again: NDINIT(&nd, LOOKUP, FOLLOW, UIO_SYSSPACE, xbb->dev_name, curthread); error = vn_open(&nd, &flags, 0, NULL); if (error) { /* * This is the only reasonable guess we can make as far as * path if the user doesn't give us a fully qualified path. * If they want to specify a file, they need to specify the * full path. */ if (xbb->dev_name[0] != '/') { char *dev_path = "/dev/"; char *dev_name; /* Try adding device path at beginning of name */ dev_name = malloc(strlen(xbb->dev_name) + strlen(dev_path) + 1, M_XENBLOCKBACK, M_NOWAIT); if (dev_name) { sprintf(dev_name, "%s%s", dev_path, xbb->dev_name); free(xbb->dev_name, M_XENBLOCKBACK); xbb->dev_name = dev_name; goto again; } } xenbus_dev_fatal(xbb->dev, error, "error opening device %s", xbb->dev_name); return (error); } NDFREE(&nd, NDF_ONLY_PNBUF); xbb->vn = nd.ni_vp; /* We only support disks and files. */ if (vn_isdisk(xbb->vn, &error)) { error = xbb_open_dev(xbb); } else if (xbb->vn->v_type == VREG) { error = xbb_open_file(xbb); } else { error = EINVAL; xenbus_dev_fatal(xbb->dev, error, "%s is not a disk " "or file", xbb->dev_name); } VOP_UNLOCK(xbb->vn, 0); if (error != 0) { xbb_close_backend(xbb); return (error); } xbb->sector_size_shift = fls(xbb->sector_size) - 1; xbb->media_num_sectors = xbb->media_size >> xbb->sector_size_shift; DPRINTF("opened %s=%s sector_size=%u media_size=%" PRId64 "\n", (xbb->device_type == XBB_TYPE_DISK) ? "dev" : "file", xbb->dev_name, xbb->sector_size, xbb->media_size); return (0); } /*------------------------ Inter-Domain Communication ------------------------*/ /** * Free dynamically allocated KVA or pseudo-physical address allocations. * * \param xbb Per-instance xbb configuration structure. */ static void xbb_free_communication_mem(struct xbb_softc *xbb) { if (xbb->kva != 0) { if (xbb->pseudo_phys_res != NULL) { xenmem_free(xbb->dev, xbb->pseudo_phys_res_id, xbb->pseudo_phys_res); xbb->pseudo_phys_res = NULL; } } xbb->kva = 0; xbb->gnt_base_addr = 0; if (xbb->kva_free != NULL) { free(xbb->kva_free, M_XENBLOCKBACK); xbb->kva_free = NULL; } } /** * Cleanup all inter-domain communication mechanisms. * * \param xbb Per-instance xbb configuration structure. */ static int xbb_disconnect(struct xbb_softc *xbb) { struct gnttab_unmap_grant_ref ops[XBB_MAX_RING_PAGES]; struct gnttab_unmap_grant_ref *op; u_int ring_idx; int error; DPRINTF("\n"); if ((xbb->flags & XBBF_RING_CONNECTED) == 0) return (0); xen_intr_unbind(&xbb->xen_intr_handle); mtx_unlock(&xbb->lock); taskqueue_drain(xbb->io_taskqueue, &xbb->io_task); mtx_lock(&xbb->lock); /* * No new interrupts can generate work, but we must wait * for all currently active requests to drain. */ if (xbb->active_request_count != 0) return (EAGAIN); for (ring_idx = 0, op = ops; ring_idx < xbb->ring_config.ring_pages; ring_idx++, op++) { op->host_addr = xbb->ring_config.gnt_addr + (ring_idx * PAGE_SIZE); op->dev_bus_addr = xbb->ring_config.bus_addr[ring_idx]; op->handle = xbb->ring_config.handle[ring_idx]; } error = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, ops, xbb->ring_config.ring_pages); if (error != 0) panic("Grant table op failed (%d)", error); xbb_free_communication_mem(xbb); if (xbb->requests != NULL) { free(xbb->requests, M_XENBLOCKBACK); xbb->requests = NULL; } if (xbb->request_lists != NULL) { struct xbb_xen_reqlist *reqlist; int i; /* There is one request list for ever allocated request. */ for (i = 0, reqlist = xbb->request_lists; i < xbb->max_requests; i++, reqlist++){ #ifdef XBB_USE_BOUNCE_BUFFERS if (reqlist->bounce != NULL) { free(reqlist->bounce, M_XENBLOCKBACK); reqlist->bounce = NULL; } #endif if (reqlist->gnt_handles != NULL) { free(reqlist->gnt_handles, M_XENBLOCKBACK); reqlist->gnt_handles = NULL; } } free(xbb->request_lists, M_XENBLOCKBACK); xbb->request_lists = NULL; } xbb->flags &= ~XBBF_RING_CONNECTED; return (0); } /** * Map shared memory ring into domain local address space, initialize * ring control structures, and bind an interrupt to the event channel * used to notify us of ring changes. * * \param xbb Per-instance xbb configuration structure. */ static int xbb_connect_ring(struct xbb_softc *xbb) { struct gnttab_map_grant_ref gnts[XBB_MAX_RING_PAGES]; struct gnttab_map_grant_ref *gnt; u_int ring_idx; int error; if ((xbb->flags & XBBF_RING_CONNECTED) != 0) return (0); /* * Kva for our ring is at the tail of the region of kva allocated * by xbb_alloc_communication_mem(). */ xbb->ring_config.va = xbb->kva + (xbb->kva_size - (xbb->ring_config.ring_pages * PAGE_SIZE)); xbb->ring_config.gnt_addr = xbb->gnt_base_addr + (xbb->kva_size - (xbb->ring_config.ring_pages * PAGE_SIZE)); for (ring_idx = 0, gnt = gnts; ring_idx < xbb->ring_config.ring_pages; ring_idx++, gnt++) { gnt->host_addr = xbb->ring_config.gnt_addr + (ring_idx * PAGE_SIZE); gnt->flags = GNTMAP_host_map; gnt->ref = xbb->ring_config.ring_ref[ring_idx]; gnt->dom = xbb->otherend_id; } error = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, gnts, xbb->ring_config.ring_pages); if (error) panic("blkback: Ring page grant table op failed (%d)", error); for (ring_idx = 0, gnt = gnts; ring_idx < xbb->ring_config.ring_pages; ring_idx++, gnt++) { if (gnt->status != 0) { xbb->ring_config.va = 0; xenbus_dev_fatal(xbb->dev, EACCES, "Ring shared page mapping failed. " "Status %d.", gnt->status); return (EACCES); } xbb->ring_config.handle[ring_idx] = gnt->handle; xbb->ring_config.bus_addr[ring_idx] = gnt->dev_bus_addr; } /* Initialize the ring based on ABI. */ switch (xbb->abi) { case BLKIF_PROTOCOL_NATIVE: { blkif_sring_t *sring; sring = (blkif_sring_t *)xbb->ring_config.va; BACK_RING_INIT(&xbb->rings.native, sring, xbb->ring_config.ring_pages * PAGE_SIZE); break; } case BLKIF_PROTOCOL_X86_32: { blkif_x86_32_sring_t *sring_x86_32; sring_x86_32 = (blkif_x86_32_sring_t *)xbb->ring_config.va; BACK_RING_INIT(&xbb->rings.x86_32, sring_x86_32, xbb->ring_config.ring_pages * PAGE_SIZE); break; } case BLKIF_PROTOCOL_X86_64: { blkif_x86_64_sring_t *sring_x86_64; sring_x86_64 = (blkif_x86_64_sring_t *)xbb->ring_config.va; BACK_RING_INIT(&xbb->rings.x86_64, sring_x86_64, xbb->ring_config.ring_pages * PAGE_SIZE); break; } default: panic("Unexpected blkif protocol ABI."); } xbb->flags |= XBBF_RING_CONNECTED; error = xen_intr_bind_remote_port(xbb->dev, xbb->otherend_id, xbb->ring_config.evtchn, xbb_filter, /*ithread_handler*/NULL, /*arg*/xbb, INTR_TYPE_BIO | INTR_MPSAFE, &xbb->xen_intr_handle); if (error) { (void)xbb_disconnect(xbb); xenbus_dev_fatal(xbb->dev, error, "binding event channel"); return (error); } DPRINTF("rings connected!\n"); return 0; } -/* Needed to make bit_alloc() macro work */ -#define calloc(count, size) malloc((count)*(size), M_XENBLOCKBACK, \ - M_NOWAIT|M_ZERO); - /** * Size KVA and pseudo-physical address allocations based on negotiated * values for the size and number of I/O requests, and the size of our * communication ring. * * \param xbb Per-instance xbb configuration structure. * * These address spaces are used to dynamically map pages in the * front-end's domain into our own. */ static int xbb_alloc_communication_mem(struct xbb_softc *xbb) { xbb->reqlist_kva_pages = xbb->max_requests * xbb->max_request_segments; xbb->reqlist_kva_size = xbb->reqlist_kva_pages * PAGE_SIZE; xbb->kva_size = xbb->reqlist_kva_size + (xbb->ring_config.ring_pages * PAGE_SIZE); - xbb->kva_free = bit_alloc(xbb->reqlist_kva_pages); + xbb->kva_free = bit_alloc(xbb->reqlist_kva_pages, M_XENBLOCKBACK, M_NOWAIT); if (xbb->kva_free == NULL) return (ENOMEM); DPRINTF("%s: kva_size = %d, reqlist_kva_size = %d\n", device_get_nameunit(xbb->dev), xbb->kva_size, xbb->reqlist_kva_size); /* * Reserve a range of pseudo physical memory that we can map * into kva. These pages will only be backed by machine * pages ("real memory") during the lifetime of front-end requests * via grant table operations. */ xbb->pseudo_phys_res_id = 0; xbb->pseudo_phys_res = xenmem_alloc(xbb->dev, &xbb->pseudo_phys_res_id, xbb->kva_size); if (xbb->pseudo_phys_res == NULL) { xbb->kva = 0; return (ENOMEM); } xbb->kva = (vm_offset_t)rman_get_virtual(xbb->pseudo_phys_res); xbb->gnt_base_addr = rman_get_start(xbb->pseudo_phys_res); DPRINTF("%s: kva: %#jx, gnt_base_addr: %#jx\n", device_get_nameunit(xbb->dev), (uintmax_t)xbb->kva, (uintmax_t)xbb->gnt_base_addr); return (0); } /** * Collect front-end information from the XenStore. * * \param xbb Per-instance xbb configuration structure. */ static int xbb_collect_frontend_info(struct xbb_softc *xbb) { char protocol_abi[64]; const char *otherend_path; int error; u_int ring_idx; u_int ring_page_order; size_t ring_size; otherend_path = xenbus_get_otherend_path(xbb->dev); /* * Protocol defaults valid even if all negotiation fails. */ xbb->ring_config.ring_pages = 1; xbb->max_request_segments = BLKIF_MAX_SEGMENTS_PER_REQUEST; xbb->max_request_size = xbb->max_request_segments * PAGE_SIZE; /* * Mandatory data (used in all versions of the protocol) first. */ error = xs_scanf(XST_NIL, otherend_path, "event-channel", NULL, "%" PRIu32, &xbb->ring_config.evtchn); if (error != 0) { xenbus_dev_fatal(xbb->dev, error, "Unable to retrieve event-channel information " "from frontend %s. Unable to connect.", xenbus_get_otherend_path(xbb->dev)); return (error); } /* * These fields are initialized to legacy protocol defaults * so we only need to fail if reading the updated value succeeds * and the new value is outside of its allowed range. * * \note xs_gather() returns on the first encountered error, so * we must use independent calls in order to guarantee * we don't miss information in a sparsly populated front-end * tree. * * \note xs_scanf() does not update variables for unmatched * fields. */ ring_page_order = 0; xbb->max_requests = 32; (void)xs_scanf(XST_NIL, otherend_path, "ring-page-order", NULL, "%u", &ring_page_order); xbb->ring_config.ring_pages = 1 << ring_page_order; ring_size = PAGE_SIZE * xbb->ring_config.ring_pages; xbb->max_requests = BLKIF_MAX_RING_REQUESTS(ring_size); if (xbb->ring_config.ring_pages > XBB_MAX_RING_PAGES) { xenbus_dev_fatal(xbb->dev, EINVAL, "Front-end specified ring-pages of %u " "exceeds backend limit of %u. " "Unable to connect.", xbb->ring_config.ring_pages, XBB_MAX_RING_PAGES); return (EINVAL); } if (xbb->ring_config.ring_pages == 1) { error = xs_gather(XST_NIL, otherend_path, "ring-ref", "%" PRIu32, &xbb->ring_config.ring_ref[0], NULL); if (error != 0) { xenbus_dev_fatal(xbb->dev, error, "Unable to retrieve ring information " "from frontend %s. Unable to " "connect.", xenbus_get_otherend_path(xbb->dev)); return (error); } } else { /* Multi-page ring format. */ for (ring_idx = 0; ring_idx < xbb->ring_config.ring_pages; ring_idx++) { char ring_ref_name[]= "ring_refXX"; snprintf(ring_ref_name, sizeof(ring_ref_name), "ring-ref%u", ring_idx); error = xs_scanf(XST_NIL, otherend_path, ring_ref_name, NULL, "%" PRIu32, &xbb->ring_config.ring_ref[ring_idx]); if (error != 0) { xenbus_dev_fatal(xbb->dev, error, "Failed to retriev grant " "reference for page %u of " "shared ring. Unable " "to connect.", ring_idx); return (error); } } } error = xs_gather(XST_NIL, otherend_path, "protocol", "%63s", protocol_abi, NULL); if (error != 0 || !strcmp(protocol_abi, XEN_IO_PROTO_ABI_NATIVE)) { /* * Assume native if the frontend has not * published ABI data or it has published and * matches our own ABI. */ xbb->abi = BLKIF_PROTOCOL_NATIVE; } else if (!strcmp(protocol_abi, XEN_IO_PROTO_ABI_X86_32)) { xbb->abi = BLKIF_PROTOCOL_X86_32; } else if (!strcmp(protocol_abi, XEN_IO_PROTO_ABI_X86_64)) { xbb->abi = BLKIF_PROTOCOL_X86_64; } else { xenbus_dev_fatal(xbb->dev, EINVAL, "Unknown protocol ABI (%s) published by " "frontend. Unable to connect.", protocol_abi); return (EINVAL); } return (0); } /** * Allocate per-request data structures given request size and number * information negotiated with the front-end. * * \param xbb Per-instance xbb configuration structure. */ static int xbb_alloc_requests(struct xbb_softc *xbb) { struct xbb_xen_req *req; struct xbb_xen_req *last_req; /* * Allocate request book keeping datastructures. */ xbb->requests = malloc(xbb->max_requests * sizeof(*xbb->requests), M_XENBLOCKBACK, M_NOWAIT|M_ZERO); if (xbb->requests == NULL) { xenbus_dev_fatal(xbb->dev, ENOMEM, "Unable to allocate request structures"); return (ENOMEM); } req = xbb->requests; last_req = &xbb->requests[xbb->max_requests - 1]; STAILQ_INIT(&xbb->request_free_stailq); while (req <= last_req) { STAILQ_INSERT_TAIL(&xbb->request_free_stailq, req, links); req++; } return (0); } static int xbb_alloc_request_lists(struct xbb_softc *xbb) { struct xbb_xen_reqlist *reqlist; int i; /* * If no requests can be merged, we need 1 request list per * in flight request. */ xbb->request_lists = malloc(xbb->max_requests * sizeof(*xbb->request_lists), M_XENBLOCKBACK, M_NOWAIT|M_ZERO); if (xbb->request_lists == NULL) { xenbus_dev_fatal(xbb->dev, ENOMEM, "Unable to allocate request list structures"); return (ENOMEM); } STAILQ_INIT(&xbb->reqlist_free_stailq); STAILQ_INIT(&xbb->reqlist_pending_stailq); for (i = 0; i < xbb->max_requests; i++) { int seg; reqlist = &xbb->request_lists[i]; reqlist->xbb = xbb; #ifdef XBB_USE_BOUNCE_BUFFERS reqlist->bounce = malloc(xbb->max_reqlist_size, M_XENBLOCKBACK, M_NOWAIT); if (reqlist->bounce == NULL) { xenbus_dev_fatal(xbb->dev, ENOMEM, "Unable to allocate request " "bounce buffers"); return (ENOMEM); } #endif /* XBB_USE_BOUNCE_BUFFERS */ reqlist->gnt_handles = malloc(xbb->max_reqlist_segments * sizeof(*reqlist->gnt_handles), M_XENBLOCKBACK, M_NOWAIT|M_ZERO); if (reqlist->gnt_handles == NULL) { xenbus_dev_fatal(xbb->dev, ENOMEM, "Unable to allocate request " "grant references"); return (ENOMEM); } for (seg = 0; seg < xbb->max_reqlist_segments; seg++) reqlist->gnt_handles[seg] = GRANT_REF_INVALID; STAILQ_INSERT_TAIL(&xbb->reqlist_free_stailq, reqlist, links); } return (0); } /** * Supply information about the physical device to the frontend * via XenBus. * * \param xbb Per-instance xbb configuration structure. */ static int xbb_publish_backend_info(struct xbb_softc *xbb) { struct xs_transaction xst; const char *our_path; const char *leaf; int error; our_path = xenbus_get_node(xbb->dev); while (1) { error = xs_transaction_start(&xst); if (error != 0) { xenbus_dev_fatal(xbb->dev, error, "Error publishing backend info " "(start transaction)"); return (error); } leaf = "sectors"; error = xs_printf(xst, our_path, leaf, "%"PRIu64, xbb->media_num_sectors); if (error != 0) break; /* XXX Support all VBD attributes here. */ leaf = "info"; error = xs_printf(xst, our_path, leaf, "%u", xbb->flags & XBBF_READ_ONLY ? VDISK_READONLY : 0); if (error != 0) break; leaf = "sector-size"; error = xs_printf(xst, our_path, leaf, "%u", xbb->sector_size); if (error != 0) break; error = xs_transaction_end(xst, 0); if (error == 0) { return (0); } else if (error != EAGAIN) { xenbus_dev_fatal(xbb->dev, error, "ending transaction"); return (error); } } xenbus_dev_fatal(xbb->dev, error, "writing %s/%s", our_path, leaf); xs_transaction_end(xst, 1); return (error); } /** * Connect to our blkfront peer now that it has completed publishing * its configuration into the XenStore. * * \param xbb Per-instance xbb configuration structure. */ static void xbb_connect(struct xbb_softc *xbb) { int error; if (xenbus_get_state(xbb->dev) == XenbusStateConnected) return; if (xbb_collect_frontend_info(xbb) != 0) return; xbb->flags &= ~XBBF_SHUTDOWN; /* * We limit the maximum number of reqlist segments to the maximum * number of segments in the ring, or our absolute maximum, * whichever is smaller. */ xbb->max_reqlist_segments = MIN(xbb->max_request_segments * xbb->max_requests, XBB_MAX_SEGMENTS_PER_REQLIST); /* * The maximum size is simply a function of the number of segments * we can handle. */ xbb->max_reqlist_size = xbb->max_reqlist_segments * PAGE_SIZE; /* Allocate resources whose size depends on front-end configuration. */ error = xbb_alloc_communication_mem(xbb); if (error != 0) { xenbus_dev_fatal(xbb->dev, error, "Unable to allocate communication memory"); return; } error = xbb_alloc_requests(xbb); if (error != 0) { /* Specific errors are reported by xbb_alloc_requests(). */ return; } error = xbb_alloc_request_lists(xbb); if (error != 0) { /* Specific errors are reported by xbb_alloc_request_lists(). */ return; } /* * Connect communication channel. */ error = xbb_connect_ring(xbb); if (error != 0) { /* Specific errors are reported by xbb_connect_ring(). */ return; } if (xbb_publish_backend_info(xbb) != 0) { /* * If we can't publish our data, we cannot participate * in this connection, and waiting for a front-end state * change will not help the situation. */ (void)xbb_disconnect(xbb); return; } /* Ready for I/O. */ xenbus_set_state(xbb->dev, XenbusStateConnected); } /*-------------------------- Device Teardown Support -------------------------*/ /** * Perform device shutdown functions. * * \param xbb Per-instance xbb configuration structure. * * Mark this instance as shutting down, wait for any active I/O on the * backend device/file to drain, disconnect from the front-end, and notify * any waiters (e.g. a thread invoking our detach method) that detach can * now proceed. */ static int xbb_shutdown(struct xbb_softc *xbb) { XenbusState frontState; int error; DPRINTF("\n"); /* * Due to the need to drop our mutex during some * xenbus operations, it is possible for two threads * to attempt to close out shutdown processing at * the same time. Tell the caller that hits this * race to try back later. */ if ((xbb->flags & XBBF_IN_SHUTDOWN) != 0) return (EAGAIN); xbb->flags |= XBBF_IN_SHUTDOWN; mtx_unlock(&xbb->lock); if (xenbus_get_state(xbb->dev) < XenbusStateClosing) xenbus_set_state(xbb->dev, XenbusStateClosing); frontState = xenbus_get_otherend_state(xbb->dev); mtx_lock(&xbb->lock); xbb->flags &= ~XBBF_IN_SHUTDOWN; /* The front can submit I/O until entering the closed state. */ if (frontState < XenbusStateClosed) return (EAGAIN); DPRINTF("\n"); /* Indicate shutdown is in progress. */ xbb->flags |= XBBF_SHUTDOWN; /* Disconnect from the front-end. */ error = xbb_disconnect(xbb); if (error != 0) { /* * Requests still outstanding. We'll be called again * once they complete. */ KASSERT(error == EAGAIN, ("%s: Unexpected xbb_disconnect() failure %d", __func__, error)); return (error); } DPRINTF("\n"); /* Indicate to xbb_detach() that is it safe to proceed. */ wakeup(xbb); return (0); } /** * Report an attach time error to the console and Xen, and cleanup * this instance by forcing immediate detach processing. * * \param xbb Per-instance xbb configuration structure. * \param err Errno describing the error. * \param fmt Printf style format and arguments */ static void xbb_attach_failed(struct xbb_softc *xbb, int err, const char *fmt, ...) { va_list ap; va_list ap_hotplug; va_start(ap, fmt); va_copy(ap_hotplug, ap); xs_vprintf(XST_NIL, xenbus_get_node(xbb->dev), "hotplug-error", fmt, ap_hotplug); va_end(ap_hotplug); xs_printf(XST_NIL, xenbus_get_node(xbb->dev), "hotplug-status", "error"); xenbus_dev_vfatal(xbb->dev, err, fmt, ap); va_end(ap); xs_printf(XST_NIL, xenbus_get_node(xbb->dev), "online", "0"); xbb_detach(xbb->dev); } /*---------------------------- NewBus Entrypoints ----------------------------*/ /** * Inspect a XenBus device and claim it if is of the appropriate type. * * \param dev NewBus device object representing a candidate XenBus device. * * \return 0 for success, errno codes for failure. */ static int xbb_probe(device_t dev) { if (!strcmp(xenbus_get_type(dev), "vbd")) { device_set_desc(dev, "Backend Virtual Block Device"); device_quiet(dev); return (0); } return (ENXIO); } /** * Setup sysctl variables to control various Block Back parameters. * * \param xbb Xen Block Back softc. * */ static void xbb_setup_sysctl(struct xbb_softc *xbb) { struct sysctl_ctx_list *sysctl_ctx = NULL; struct sysctl_oid *sysctl_tree = NULL; sysctl_ctx = device_get_sysctl_ctx(xbb->dev); if (sysctl_ctx == NULL) return; sysctl_tree = device_get_sysctl_tree(xbb->dev); if (sysctl_tree == NULL) return; SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO, "disable_flush", CTLFLAG_RW, &xbb->disable_flush, 0, "fake the flush command"); SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO, "flush_interval", CTLFLAG_RW, &xbb->flush_interval, 0, "send a real flush for N flush requests"); SYSCTL_ADD_INT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO, "no_coalesce_reqs", CTLFLAG_RW, &xbb->no_coalesce_reqs,0, "Don't coalesce contiguous requests"); SYSCTL_ADD_UQUAD(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO, "reqs_received", CTLFLAG_RW, &xbb->reqs_received, "how many I/O requests we have received"); SYSCTL_ADD_UQUAD(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO, "reqs_completed", CTLFLAG_RW, &xbb->reqs_completed, "how many I/O requests have been completed"); SYSCTL_ADD_UQUAD(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO, "reqs_queued_for_completion", CTLFLAG_RW, &xbb->reqs_queued_for_completion, "how many I/O requests queued but not yet pushed"); SYSCTL_ADD_UQUAD(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO, "reqs_completed_with_error", CTLFLAG_RW, &xbb->reqs_completed_with_error, "how many I/O requests completed with error status"); SYSCTL_ADD_UQUAD(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO, "forced_dispatch", CTLFLAG_RW, &xbb->forced_dispatch, "how many I/O dispatches were forced"); SYSCTL_ADD_UQUAD(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO, "normal_dispatch", CTLFLAG_RW, &xbb->normal_dispatch, "how many I/O dispatches were normal"); SYSCTL_ADD_UQUAD(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO, "total_dispatch", CTLFLAG_RW, &xbb->total_dispatch, "total number of I/O dispatches"); SYSCTL_ADD_UQUAD(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO, "kva_shortages", CTLFLAG_RW, &xbb->kva_shortages, "how many times we have run out of KVA"); SYSCTL_ADD_UQUAD(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO, "request_shortages", CTLFLAG_RW, &xbb->request_shortages, "how many times we have run out of requests"); SYSCTL_ADD_UINT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO, "max_requests", CTLFLAG_RD, &xbb->max_requests, 0, "maximum outstanding requests (negotiated)"); SYSCTL_ADD_UINT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO, "max_request_segments", CTLFLAG_RD, &xbb->max_request_segments, 0, "maximum number of pages per requests (negotiated)"); SYSCTL_ADD_UINT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO, "max_request_size", CTLFLAG_RD, &xbb->max_request_size, 0, "maximum size in bytes of a request (negotiated)"); SYSCTL_ADD_UINT(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), OID_AUTO, "ring_pages", CTLFLAG_RD, &xbb->ring_config.ring_pages, 0, "communication channel pages (negotiated)"); } /** * Attach to a XenBus device that has been claimed by our probe routine. * * \param dev NewBus device object representing this Xen Block Back instance. * * \return 0 for success, errno codes for failure. */ static int xbb_attach(device_t dev) { struct xbb_softc *xbb; int error; u_int max_ring_page_order; DPRINTF("Attaching to %s\n", xenbus_get_node(dev)); /* * Basic initialization. * After this block it is safe to call xbb_detach() * to clean up any allocated data for this instance. */ xbb = device_get_softc(dev); xbb->dev = dev; xbb->otherend_id = xenbus_get_otherend_id(dev); TASK_INIT(&xbb->io_task, /*priority*/0, xbb_run_queue, xbb); mtx_init(&xbb->lock, device_get_nameunit(dev), NULL, MTX_DEF); /* * Publish protocol capabilities for consumption by the * front-end. */ error = xs_printf(XST_NIL, xenbus_get_node(xbb->dev), "feature-barrier", "1"); if (error) { xbb_attach_failed(xbb, error, "writing %s/feature-barrier", xenbus_get_node(xbb->dev)); return (error); } error = xs_printf(XST_NIL, xenbus_get_node(xbb->dev), "feature-flush-cache", "1"); if (error) { xbb_attach_failed(xbb, error, "writing %s/feature-flush-cache", xenbus_get_node(xbb->dev)); return (error); } max_ring_page_order = flsl(XBB_MAX_RING_PAGES) - 1; error = xs_printf(XST_NIL, xenbus_get_node(xbb->dev), "max-ring-page-order", "%u", max_ring_page_order); if (error) { xbb_attach_failed(xbb, error, "writing %s/max-ring-page-order", xenbus_get_node(xbb->dev)); return (error); } /* Collect physical device information. */ error = xs_gather(XST_NIL, xenbus_get_otherend_path(xbb->dev), "device-type", NULL, &xbb->dev_type, NULL); if (error != 0) xbb->dev_type = NULL; error = xs_gather(XST_NIL, xenbus_get_node(dev), "mode", NULL, &xbb->dev_mode, "params", NULL, &xbb->dev_name, NULL); if (error != 0) { xbb_attach_failed(xbb, error, "reading backend fields at %s", xenbus_get_node(dev)); return (ENXIO); } /* Parse fopen style mode flags. */ if (strchr(xbb->dev_mode, 'w') == NULL) xbb->flags |= XBBF_READ_ONLY; /* * Verify the physical device is present and can support * the desired I/O mode. */ DROP_GIANT(); error = xbb_open_backend(xbb); PICKUP_GIANT(); if (error != 0) { xbb_attach_failed(xbb, error, "Unable to open %s", xbb->dev_name); return (ENXIO); } /* Use devstat(9) for recording statistics. */ xbb->xbb_stats = devstat_new_entry("xbb", device_get_unit(xbb->dev), xbb->sector_size, DEVSTAT_ALL_SUPPORTED, DEVSTAT_TYPE_DIRECT | DEVSTAT_TYPE_IF_OTHER, DEVSTAT_PRIORITY_OTHER); xbb->xbb_stats_in = devstat_new_entry("xbbi", device_get_unit(xbb->dev), xbb->sector_size, DEVSTAT_ALL_SUPPORTED, DEVSTAT_TYPE_DIRECT | DEVSTAT_TYPE_IF_OTHER, DEVSTAT_PRIORITY_OTHER); /* * Setup sysctl variables. */ xbb_setup_sysctl(xbb); /* * Create a taskqueue for doing work that must occur from a * thread context. */ xbb->io_taskqueue = taskqueue_create_fast(device_get_nameunit(dev), M_NOWAIT, taskqueue_thread_enqueue, /*contxt*/&xbb->io_taskqueue); if (xbb->io_taskqueue == NULL) { xbb_attach_failed(xbb, error, "Unable to create taskqueue"); return (ENOMEM); } taskqueue_start_threads(&xbb->io_taskqueue, /*num threads*/1, /*priority*/PWAIT, /*thread name*/ "%s taskq", device_get_nameunit(dev)); /* Update hot-plug status to satisfy xend. */ error = xs_printf(XST_NIL, xenbus_get_node(xbb->dev), "hotplug-status", "connected"); if (error) { xbb_attach_failed(xbb, error, "writing %s/hotplug-status", xenbus_get_node(xbb->dev)); return (error); } /* Tell the front end that we are ready to connect. */ xenbus_set_state(dev, XenbusStateInitWait); return (0); } /** * Detach from a block back device instance. * * \param dev NewBus device object representing this Xen Block Back instance. * * \return 0 for success, errno codes for failure. * * \note A block back device may be detached at any time in its life-cycle, * including part way through the attach process. For this reason, * initialization order and the initialization state checks in this * routine must be carefully coupled so that attach time failures * are gracefully handled. */ static int xbb_detach(device_t dev) { struct xbb_softc *xbb; DPRINTF("\n"); xbb = device_get_softc(dev); mtx_lock(&xbb->lock); while (xbb_shutdown(xbb) == EAGAIN) { msleep(xbb, &xbb->lock, /*wakeup prio unchanged*/0, "xbb_shutdown", 0); } mtx_unlock(&xbb->lock); DPRINTF("\n"); if (xbb->io_taskqueue != NULL) taskqueue_free(xbb->io_taskqueue); if (xbb->xbb_stats != NULL) devstat_remove_entry(xbb->xbb_stats); if (xbb->xbb_stats_in != NULL) devstat_remove_entry(xbb->xbb_stats_in); xbb_close_backend(xbb); if (xbb->dev_mode != NULL) { free(xbb->dev_mode, M_XENSTORE); xbb->dev_mode = NULL; } if (xbb->dev_type != NULL) { free(xbb->dev_type, M_XENSTORE); xbb->dev_type = NULL; } if (xbb->dev_name != NULL) { free(xbb->dev_name, M_XENSTORE); xbb->dev_name = NULL; } mtx_destroy(&xbb->lock); return (0); } /** * Prepare this block back device for suspension of this VM. * * \param dev NewBus device object representing this Xen Block Back instance. * * \return 0 for success, errno codes for failure. */ static int xbb_suspend(device_t dev) { #ifdef NOT_YET struct xbb_softc *sc = device_get_softc(dev); /* Prevent new requests being issued until we fix things up. */ mtx_lock(&sc->xb_io_lock); sc->connected = BLKIF_STATE_SUSPENDED; mtx_unlock(&sc->xb_io_lock); #endif return (0); } /** * Perform any processing required to recover from a suspended state. * * \param dev NewBus device object representing this Xen Block Back instance. * * \return 0 for success, errno codes for failure. */ static int xbb_resume(device_t dev) { return (0); } /** * Handle state changes expressed via the XenStore by our front-end peer. * * \param dev NewBus device object representing this Xen * Block Back instance. * \param frontend_state The new state of the front-end. * * \return 0 for success, errno codes for failure. */ static void xbb_frontend_changed(device_t dev, XenbusState frontend_state) { struct xbb_softc *xbb = device_get_softc(dev); DPRINTF("frontend_state=%s, xbb_state=%s\n", xenbus_strstate(frontend_state), xenbus_strstate(xenbus_get_state(xbb->dev))); switch (frontend_state) { case XenbusStateInitialising: break; case XenbusStateInitialised: case XenbusStateConnected: xbb_connect(xbb); break; case XenbusStateClosing: case XenbusStateClosed: mtx_lock(&xbb->lock); xbb_shutdown(xbb); mtx_unlock(&xbb->lock); if (frontend_state == XenbusStateClosed) xenbus_set_state(xbb->dev, XenbusStateClosed); break; default: xenbus_dev_fatal(xbb->dev, EINVAL, "saw state %d at frontend", frontend_state); break; } } /*---------------------------- NewBus Registration ---------------------------*/ static device_method_t xbb_methods[] = { /* Device interface */ DEVMETHOD(device_probe, xbb_probe), DEVMETHOD(device_attach, xbb_attach), DEVMETHOD(device_detach, xbb_detach), DEVMETHOD(device_shutdown, bus_generic_shutdown), DEVMETHOD(device_suspend, xbb_suspend), DEVMETHOD(device_resume, xbb_resume), /* Xenbus interface */ DEVMETHOD(xenbus_otherend_changed, xbb_frontend_changed), { 0, 0 } }; static driver_t xbb_driver = { "xbbd", xbb_methods, sizeof(struct xbb_softc), }; devclass_t xbb_devclass; DRIVER_MODULE(xbbd, xenbusb_back, xbb_driver, xbb_devclass, 0, 0); Index: user/ngie/release-pkg-fix-tests/sys/dev/xen/timer/timer.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/dev/xen/timer/timer.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/dev/xen/timer/timer.c (revision 299091) @@ -1,542 +1,547 @@ /*- * Copyright (c) 2009 Adrian Chadd * Copyright (c) 2012 Spectra Logic Corporation * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ /** * \file dev/xen/timer/timer.c * \brief A timer driver for the Xen hypervisor's PV clock. */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "clock_if.h" static devclass_t xentimer_devclass; #define NSEC_IN_SEC 1000000000ULL #define NSEC_IN_USEC 1000ULL /* 18446744073 = int(2^64 / NSEC_IN_SC) = 1 ns in 64-bit fractions */ #define FRAC_IN_NSEC 18446744073LL /* Xen timers may fire up to 100us off */ #define XENTIMER_MIN_PERIOD_IN_NSEC 100*NSEC_IN_USEC -#define XENCLOCK_RESOLUTION 1000001 /* ATRTC resolution + 1 */ + +/* + * The real resolution of the PV clock is 1ns, but the highest + * resolution that FreeBSD supports is 1us, so just use that. + */ +#define XENCLOCK_RESOLUTION 1 #define XENTIMER_QUALITY 950 struct xentimer_pcpu_data { uint64_t timer; uint64_t last_processed; void *irq_handle; }; DPCPU_DEFINE(struct xentimer_pcpu_data, xentimer_pcpu); DPCPU_DECLARE(struct vcpu_info *, vcpu_info); struct xentimer_softc { device_t dev; struct timecounter tc; struct eventtimer et; }; static void xentimer_identify(driver_t *driver, device_t parent) { if (!xen_domain()) return; /* Handle all Xen PV timers in one device instance. */ if (devclass_get_device(xentimer_devclass, 0)) return; BUS_ADD_CHILD(parent, 0, "xen_et", 0); } static int xentimer_probe(device_t dev) { KASSERT((xen_domain()), ("Trying to use Xen timer on bare metal")); /* * In order to attach, this driver requires the following: * - Vector callback support by the hypervisor, in order to deliver * timer interrupts to the correct CPU for CPUs other than 0. * - Access to the hypervisor shared info page, in order to look up * each VCPU's timer information and the Xen wallclock time. * - The hypervisor must say its PV clock is "safe" to use. * - The hypervisor must support VCPUOP hypercalls. * - The maximum number of CPUs supported by FreeBSD must not exceed * the number of VCPUs supported by the hypervisor. */ #define XTREQUIRES(condition, reason...) \ if (!(condition)) { \ device_printf(dev, ## reason); \ device_detach(dev); \ return (ENXIO); \ } if (xen_hvm_domain()) { XTREQUIRES(xen_vector_callback_enabled, "vector callbacks unavailable\n"); XTREQUIRES(xen_feature(XENFEAT_hvm_safe_pvclock), "HVM safe pvclock unavailable\n"); } XTREQUIRES(HYPERVISOR_shared_info != NULL, "shared info page unavailable\n"); XTREQUIRES(HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, 0, NULL) == 0, "VCPUOPs interface unavailable\n"); #undef XTREQUIRES device_set_desc(dev, "Xen PV Clock"); return (BUS_PROBE_NOWILDCARD); } /** * \brief Get the current time, in nanoseconds, since the hypervisor booted. * * \param vcpu vcpu_info structure to fetch the time from. * */ static uint64_t xen_fetch_vcpu_time(struct vcpu_info *vcpu) { struct pvclock_vcpu_time_info *time; time = (struct pvclock_vcpu_time_info *) &vcpu->time; return (pvclock_get_timecount(time)); } static uint32_t xentimer_get_timecount(struct timecounter *tc) { uint64_t vcpu_time; /* * We don't disable preemption here because the worst that can * happen is reading the vcpu_info area of a different CPU than * the one we are currently running on, but that would also * return a valid tc (and we avoid the overhead of * critical_{enter/exit} calls). */ vcpu_time = xen_fetch_vcpu_time(DPCPU_GET(vcpu_info)); return (vcpu_time & UINT32_MAX); } /** * \brief Fetch the hypervisor boot time, known as the "Xen wallclock". * * \param ts Timespec to store the current stable value. * \param version Pointer to store the corresponding wallclock version. * * \note This value is updated when Domain-0 shifts its clock to follow * clock drift, e.g. as detected by NTP. */ static void xen_fetch_wallclock(struct timespec *ts) { shared_info_t *src = HYPERVISOR_shared_info; struct pvclock_wall_clock *wc; wc = (struct pvclock_wall_clock *) &src->wc_version; pvclock_get_wallclock(wc, ts); } static void xen_fetch_uptime(struct timespec *ts) { uint64_t uptime; uptime = xen_fetch_vcpu_time(DPCPU_GET(vcpu_info)); ts->tv_sec = uptime / NSEC_IN_SEC; ts->tv_nsec = uptime % NSEC_IN_SEC; } static int xentimer_settime(device_t dev __unused, struct timespec *ts) { struct xen_platform_op settime; int ret; /* * Don't return EINVAL here; just silently fail if the domain isn't * privileged enough to set the TOD. */ if (!xen_initial_domain()) return (0); /* Set the native RTC. */ atrtc_set(ts); settime.cmd = XENPF_settime64; settime.u.settime64.mbz = 0; settime.u.settime64.secs = ts->tv_sec; settime.u.settime64.nsecs = ts->tv_nsec; settime.u.settime64.system_time = xen_fetch_vcpu_time(DPCPU_GET(vcpu_info)); ret = HYPERVISOR_platform_op(&settime); ret = ret != 0 ? xen_translate_error(ret) : 0; if (ret != 0 && bootverbose) device_printf(dev, "failed to set Xen PV clock: %d\n", ret); return (ret); } /** * \brief Return current time according to the Xen Hypervisor wallclock. * * \param dev Xentimer device. * \param ts Pointer to store the wallclock time. * * \note The Xen time structures document the hypervisor start time and the * uptime-since-hypervisor-start (in nsec.) They need to be combined * in order to calculate a TOD clock. */ static int xentimer_gettime(device_t dev, struct timespec *ts) { struct timespec u_ts; timespecclear(ts); xen_fetch_wallclock(ts); xen_fetch_uptime(&u_ts); timespecadd(ts, &u_ts); return (0); } /** * \brief Handle a timer interrupt for the Xen PV timer driver. * * \param arg Xen timer driver softc that is expecting the interrupt. */ static int xentimer_intr(void *arg) { struct xentimer_softc *sc = (struct xentimer_softc *)arg; struct xentimer_pcpu_data *pcpu = DPCPU_PTR(xentimer_pcpu); pcpu->last_processed = xen_fetch_vcpu_time(DPCPU_GET(vcpu_info)); if (pcpu->timer != 0 && sc->et.et_active) sc->et.et_event_cb(&sc->et, sc->et.et_arg); return (FILTER_HANDLED); } static int xentimer_vcpu_start_timer(int vcpu, uint64_t next_time) { struct vcpu_set_singleshot_timer single; single.timeout_abs_ns = next_time; /* Get an event anyway, even if the timeout is already expired */ single.flags = 0; return (HYPERVISOR_vcpu_op(VCPUOP_set_singleshot_timer, vcpu, &single)); } static int xentimer_vcpu_stop_timer(int vcpu) { return (HYPERVISOR_vcpu_op(VCPUOP_stop_singleshot_timer, vcpu, NULL)); } /** * \brief Set the next oneshot time for the current CPU. * * \param et Xen timer driver event timer to schedule on. * \param first Delta to the next time to schedule the interrupt for. * \param period Not used. * * \note See eventtimers(9) for more information. * \note * * \returns 0 */ static int xentimer_et_start(struct eventtimer *et, sbintime_t first, sbintime_t period) { int error; struct xentimer_softc *sc = et->et_priv; int cpu = PCPU_GET(vcpu_id); struct xentimer_pcpu_data *pcpu = DPCPU_PTR(xentimer_pcpu); struct vcpu_info *vcpu = DPCPU_GET(vcpu_info); uint64_t first_in_ns, next_time; #ifdef INVARIANTS struct thread *td = curthread; #endif KASSERT(td->td_critnest != 0, ("xentimer_et_start called without preemption disabled")); /* See sbttots() for this formula. */ first_in_ns = (((first >> 32) * NSEC_IN_SEC) + (((uint64_t)NSEC_IN_SEC * (uint32_t)first) >> 32)); next_time = xen_fetch_vcpu_time(vcpu) + first_in_ns; error = xentimer_vcpu_start_timer(cpu, next_time); if (error) panic("%s: Error %d setting singleshot timer to %"PRIu64"\n", device_get_nameunit(sc->dev), error, next_time); pcpu->timer = next_time; return (error); } /** * \brief Cancel the event timer's currently running timer, if any. */ static int xentimer_et_stop(struct eventtimer *et) { int cpu = PCPU_GET(vcpu_id); struct xentimer_pcpu_data *pcpu = DPCPU_PTR(xentimer_pcpu); pcpu->timer = 0; return (xentimer_vcpu_stop_timer(cpu)); } /** * \brief Attach a Xen PV timer driver instance. * * \param dev Bus device object to attach. * * \note * \returns EINVAL */ static int xentimer_attach(device_t dev) { struct xentimer_softc *sc = device_get_softc(dev); int error, i; sc->dev = dev; /* Bind an event channel to a VIRQ on each VCPU. */ CPU_FOREACH(i) { struct xentimer_pcpu_data *pcpu; pcpu = DPCPU_ID_PTR(i, xentimer_pcpu); error = HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, i, NULL); if (error) { device_printf(dev, "Error disabling Xen periodic timer " "on CPU %d\n", i); return (error); } error = xen_intr_bind_virq(dev, VIRQ_TIMER, i, xentimer_intr, NULL, sc, INTR_TYPE_CLK, &pcpu->irq_handle); if (error) { device_printf(dev, "Error %d binding VIRQ_TIMER " "to VCPU %d\n", error, i); return (error); } xen_intr_describe(pcpu->irq_handle, "c%d", i); } /* Register the event timer. */ sc->et.et_name = "XENTIMER"; sc->et.et_quality = XENTIMER_QUALITY; sc->et.et_flags = ET_FLAGS_ONESHOT | ET_FLAGS_PERCPU; sc->et.et_frequency = NSEC_IN_SEC; /* See tstosbt() for this formula */ sc->et.et_min_period = (XENTIMER_MIN_PERIOD_IN_NSEC * (((uint64_t)1 << 63) / 500000000) >> 32); sc->et.et_max_period = ((sbintime_t)4 << 32); sc->et.et_start = xentimer_et_start; sc->et.et_stop = xentimer_et_stop; sc->et.et_priv = sc; et_register(&sc->et); /* Register the timecounter. */ sc->tc.tc_name = "XENTIMER"; sc->tc.tc_quality = XENTIMER_QUALITY; sc->tc.tc_flags = TC_FLAGS_SUSPEND_SAFE; /* * The underlying resolution is in nanoseconds, since the timer info * scales TSC frequencies using a fraction that represents time in * terms of nanoseconds. */ sc->tc.tc_frequency = NSEC_IN_SEC; sc->tc.tc_counter_mask = ~0u; sc->tc.tc_get_timecount = xentimer_get_timecount; sc->tc.tc_priv = sc; tc_init(&sc->tc); /* Register the Hypervisor wall clock */ clock_register(dev, XENCLOCK_RESOLUTION); return (0); } static int xentimer_detach(device_t dev) { /* Implement Xen PV clock teardown - XXX see hpet_detach ? */ /* If possible: * 1. need to deregister timecounter * 2. need to deregister event timer * 3. need to deregister virtual IRQ event channels */ return (EBUSY); } static void xentimer_percpu_resume(void *arg) { device_t dev = (device_t) arg; struct xentimer_softc *sc = device_get_softc(dev); xentimer_et_start(&sc->et, sc->et.et_min_period, 0); } static int xentimer_resume(device_t dev) { int error; int i; /* Disable the periodic timer */ CPU_FOREACH(i) { error = HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, i, NULL); if (error != 0) { device_printf(dev, "Error disabling Xen periodic timer on CPU %d\n", i); return (error); } } /* Reset the last uptime value */ pvclock_resume(); /* Kick the timers on all CPUs */ smp_rendezvous(NULL, xentimer_percpu_resume, NULL, dev); if (bootverbose) device_printf(dev, "resumed operation after suspension\n"); return (0); } static int xentimer_suspend(device_t dev) { return (0); } /* * Xen early clock init */ void xen_clock_init(void) { } /* * Xen PV DELAY function * * When running on PVH mode we don't have an emulated i8524, so * make use of the Xen time info in order to code a simple DELAY * function that can be used during early boot. */ void xen_delay(int n) { struct vcpu_info *vcpu = &HYPERVISOR_shared_info->vcpu_info[0]; uint64_t end_ns; uint64_t current; end_ns = xen_fetch_vcpu_time(vcpu); end_ns += n * NSEC_IN_USEC; for (;;) { current = xen_fetch_vcpu_time(vcpu); if (current >= end_ns) break; } } static device_method_t xentimer_methods[] = { DEVMETHOD(device_identify, xentimer_identify), DEVMETHOD(device_probe, xentimer_probe), DEVMETHOD(device_attach, xentimer_attach), DEVMETHOD(device_detach, xentimer_detach), DEVMETHOD(device_suspend, xentimer_suspend), DEVMETHOD(device_resume, xentimer_resume), /* clock interface */ DEVMETHOD(clock_gettime, xentimer_gettime), DEVMETHOD(clock_settime, xentimer_settime), DEVMETHOD_END }; static driver_t xentimer_driver = { "xen_et", xentimer_methods, sizeof(struct xentimer_softc), }; DRIVER_MODULE(xentimer, xenpv, xentimer_driver, xentimer_devclass, 0, 0); MODULE_DEPEND(xentimer, xenpv, 1, 1, 1); Index: user/ngie/release-pkg-fix-tests/sys/kern/subr_rtc.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/kern/subr_rtc.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/kern/subr_rtc.c (revision 299091) @@ -1,178 +1,178 @@ /*- * Copyright (c) 1988 University of Utah. * Copyright (c) 1982, 1990, 1993 * The Regents of the University of California. * Copyright (c) 2011 The FreeBSD Foundation * All rights reserved. * * This code is derived from software contributed to Berkeley by * the Systems Programming Group of the University of Utah Computer * Science Department. * * Portions of this software were developed by Julien Ridoux at the University * of Melbourne under sponsorship from the FreeBSD Foundation. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: Utah $Hdr: clock.c 1.18 91/01/21$ * from: @(#)clock.c 8.2 (Berkeley) 1/12/94 * from: NetBSD: clock_subr.c,v 1.6 2001/07/07 17:04:02 thorpej Exp * and * from: src/sys/i386/isa/clock.c,v 1.176 2001/09/04 */ /* * Helpers for time-of-day clocks. This is useful for architectures that need * support multiple models of such clocks, and generally serves to make the * code more machine-independent. * If the clock in question can also be used as a time counter, the driver * needs to initiate this. * This code is not yet used by all architectures. */ #include __FBSDID("$FreeBSD$"); #include "opt_ffclock.h" #include #include #include #include #include #include #ifdef FFCLOCK #include #endif #include #include "clock_if.h" static device_t clock_dev = NULL; static long clock_res; static struct timespec clock_adj; /* XXX: should be kern. now, it's no longer machdep. */ static int disable_rtc_set; SYSCTL_INT(_machdep, OID_AUTO, disable_rtc_set, CTLFLAG_RW, &disable_rtc_set, 0, "Disallow adjusting time-of-day clock"); void clock_register(device_t dev, long res) /* res has units of microseconds */ { if (clock_dev != NULL) { - if (clock_res > res) { + if (clock_res <= res) { if (bootverbose) device_printf(dev, "not installed as " "time-of-day clock: clock %s has higher " "resolution\n", device_get_name(clock_dev)); return; } if (bootverbose) device_printf(clock_dev, "removed as " "time-of-day clock: clock %s has higher " "resolution\n", device_get_name(dev)); } clock_dev = dev; clock_res = res; clock_adj.tv_sec = res / 2 / 1000000; clock_adj.tv_nsec = res / 2 % 1000000 * 1000; if (bootverbose) device_printf(dev, "registered as a time-of-day clock " "(resolution %ldus, adjustment %jd.%09jds)\n", res, (intmax_t)clock_adj.tv_sec, (intmax_t)clock_adj.tv_nsec); } /* * inittodr and settodr derived from the i386 versions written * by Christoph Robitschko , reintroduced and * updated by Chris Stenton 8/10/94 */ /* * Initialize the time of day register, based on the time base which is, e.g. * from a filesystem. */ void inittodr(time_t base) { struct timespec ts; int error; if (clock_dev == NULL) { printf("warning: no time-of-day clock registered, system time " "will not be set accurately\n"); goto wrong_time; } /* XXX: We should poll all registered RTCs in case of failure */ error = CLOCK_GETTIME(clock_dev, &ts); if (error != 0 && error != EINVAL) { printf("warning: clock_gettime failed (%d), the system time " "will not be set accurately\n", error); goto wrong_time; } if (error == EINVAL || ts.tv_sec < 0) { printf("Invalid time in real time clock.\n" "Check and reset the date immediately!\n"); goto wrong_time; } ts.tv_sec += utc_offset(); timespecadd(&ts, &clock_adj); tc_setclock(&ts); #ifdef FFCLOCK ffclock_reset_clock(&ts); #endif return; wrong_time: if (base > 0) { ts.tv_sec = base; ts.tv_nsec = 0; tc_setclock(&ts); } } /* * Write system time back to RTC */ void resettodr(void) { struct timespec ts; int error; if (disable_rtc_set || clock_dev == NULL) return; getnanotime(&ts); timespecadd(&ts, &clock_adj); ts.tv_sec -= utc_offset(); /* XXX: We should really set all registered RTCs */ if ((error = CLOCK_SETTIME(clock_dev, &ts)) != 0) printf("warning: clock_settime failed (%d), time-of-day clock " "not adjusted to system time\n", error); } Index: user/ngie/release-pkg-fix-tests/sys/kern/subr_unit.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/kern/subr_unit.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/kern/subr_unit.c (revision 299091) @@ -1,1060 +1,1061 @@ /*- * Copyright (c) 2004 Poul-Henning Kamp * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ * * * Unit number allocation functions. * * These functions implement a mixed run-length/bitmap management of unit * number spaces in a very memory efficient manner. * * Allocation policy is always lowest free number first. * * A return value of -1 signals that no more unit numbers are available. * * There is no cost associated with the range of unitnumbers, so unless * the resource really is finite, specify INT_MAX to new_unrhdr() and * forget about checking the return value. * * If a mutex is not provided when the unit number space is created, a * default global mutex is used. The advantage to passing a mutex in, is * that the alloc_unrl() function can be called with the mutex already * held (it will not be released by alloc_unrl()). * * The allocation function alloc_unr{l}() never sleeps (but it may block on * the mutex of course). * * Freeing a unit number may require allocating memory, and can therefore * sleep so the free_unr() function does not come in a pre-locked variant. * * A userland test program is included. * * Memory usage is a very complex function of the exact allocation * pattern, but always very compact: * * For the very typical case where a single unbroken run of unit * numbers are allocated 44 bytes are used on i386. * * For a unit number space of 1000 units and the random pattern * in the usermode test program included, the worst case usage * was 252 bytes on i386 for 500 allocated and 500 free units. * * For a unit number space of 10000 units and the random pattern * in the usermode test program included, the worst case usage * was 798 bytes on i386 for 5000 allocated and 5000 free units. * * The worst case is where every other unit number is allocated and * the rest are free. In that case 44 + N/4 bytes are used where * N is the number of the highest unit allocated. */ +#include #include #include #ifdef _KERNEL #include -#include #include #include #include #include #include #include /* * In theory it would be smarter to allocate the individual blocks * with the zone allocator, but at this time the expectation is that * there will typically not even be enough allocations to fill a single * page, so we stick with malloc for now. */ static MALLOC_DEFINE(M_UNIT, "Unitno", "Unit number allocation"); #define Malloc(foo) malloc(foo, M_UNIT, M_WAITOK | M_ZERO) #define Free(foo) free(foo, M_UNIT) static struct mtx unitmtx; MTX_SYSINIT(unit, &unitmtx, "unit# allocation", MTX_DEF); #else /* ...USERLAND */ #include #include #include #include #include #include #include #include #define KASSERT(cond, arg) \ do { \ if (!(cond)) { \ printf arg; \ abort(); \ } \ } while (0) static int no_alloc; #define Malloc(foo) _Malloc(foo, __LINE__) static void * _Malloc(size_t foo, int line) { KASSERT(no_alloc == 0, ("malloc in wrong place() line %d", line)); return (calloc(foo, 1)); } #define Free(foo) free(foo) struct unrhdr; struct mtx { int state; } unitmtx; static void mtx_lock(struct mtx *mp) { KASSERT(mp->state == 0, ("mutex already locked")); mp->state = 1; } static void mtx_unlock(struct mtx *mp) { KASSERT(mp->state == 1, ("mutex not locked")); mp->state = 0; } #define MA_OWNED 9 static void mtx_assert(struct mtx *mp, int flag) { if (flag == MA_OWNED) { KASSERT(mp->state == 1, ("mtx_assert(MA_OWNED) not true")); } } #define CTASSERT(foo) #define WITNESS_WARN(flags, lock, fmt, ...) (void)0 #endif /* USERLAND */ /* * This is our basic building block. * * It can be used in three different ways depending on the value of the ptr * element: * If ptr is NULL, it represents a run of free items. * If ptr points to the unrhdr it represents a run of allocated items. - * Otherwise it points to an bitstring of allocated items. + * Otherwise it points to a bitstring of allocated items. * * For runs the len field is the length of the run. * For bitmaps the len field represents the number of allocated items. * * The bitmap is the same size as struct unr to optimize memory management. */ struct unr { TAILQ_ENTRY(unr) list; u_int len; void *ptr; }; struct unrb { - u_char busy; - bitstr_t map[sizeof(struct unr) - 1]; + bitstr_t map[sizeof(struct unr) / sizeof(bitstr_t)]; }; -CTASSERT(sizeof(struct unr) == sizeof(struct unrb)); +CTASSERT((sizeof(struct unr) % sizeof(bitstr_t)) == 0); -/* Number of bits in the bitmap */ -#define NBITS ((int)sizeof(((struct unrb *)NULL)->map) * 8) +/* Number of bits we can store in the bitmap */ +#define NBITS (8 * sizeof(((struct unrb*)NULL)->map)) +/* Is the unrb empty in at least the first len bits? */ +static inline bool +ub_empty(struct unrb *ub, int len) { + int first_set; + + bit_ffs(ub->map, len, &first_set); + return (first_set == -1); +} + +/* Is the unrb full? That is, is the number of set elements equal to len? */ +static inline bool +ub_full(struct unrb *ub, int len) +{ + int first_clear; + + bit_ffc(ub->map, len, &first_clear); + return (first_clear == -1); +} + + #if defined(DIAGNOSTIC) || !defined(_KERNEL) /* * Consistency check function. * * Checks the internal consistency as well as we can. * * Called at all boundaries of this API. */ static void check_unrhdr(struct unrhdr *uh, int line) { struct unr *up; struct unrb *ub; u_int x, y, z, w; y = uh->first; z = 0; TAILQ_FOREACH(up, &uh->head, list) { z++; if (up->ptr != uh && up->ptr != NULL) { ub = up->ptr; KASSERT (up->len <= NBITS, - ("UNR inconsistency: len %u max %d (line %d)\n", + ("UNR inconsistency: len %u max %zd (line %d)\n", up->len, NBITS, line)); z++; w = 0; for (x = 0; x < up->len; x++) if (bit_test(ub->map, x)) w++; - KASSERT (w == ub->busy, - ("UNR inconsistency: busy %u found %u (line %d)\n", - ub->busy, w, line)); y += w; } else if (up->ptr != NULL) y += up->len; } KASSERT (y == uh->busy, ("UNR inconsistency: items %u found %u (line %d)\n", uh->busy, y, line)); KASSERT (z == uh->alloc, ("UNR inconsistency: chunks %u found %u (line %d)\n", uh->alloc, z, line)); } #else static __inline void -check_unrhdr(struct unrhdr *uh, int line) +check_unrhdr(struct unrhdr *uh __unused, int line __unused) { } #endif /* * Userland memory management. Just use calloc and keep track of how * many elements we have allocated for check_unrhdr(). */ static __inline void * new_unr(struct unrhdr *uh, void **p1, void **p2) { void *p; uh->alloc++; KASSERT(*p1 != NULL || *p2 != NULL, ("Out of cached memory")); if (*p1 != NULL) { p = *p1; *p1 = NULL; return (p); } else { p = *p2; *p2 = NULL; return (p); } } static __inline void delete_unr(struct unrhdr *uh, void *ptr) { struct unr *up; uh->alloc--; up = ptr; TAILQ_INSERT_TAIL(&uh->ppfree, up, list); } void clean_unrhdrl(struct unrhdr *uh) { struct unr *up; mtx_assert(uh->mtx, MA_OWNED); while ((up = TAILQ_FIRST(&uh->ppfree)) != NULL) { TAILQ_REMOVE(&uh->ppfree, up, list); mtx_unlock(uh->mtx); Free(up); mtx_lock(uh->mtx); } } void clean_unrhdr(struct unrhdr *uh) { mtx_lock(uh->mtx); clean_unrhdrl(uh); mtx_unlock(uh->mtx); } void init_unrhdr(struct unrhdr *uh, int low, int high, struct mtx *mutex) { KASSERT(low >= 0 && low <= high, ("UNR: use error: new_unrhdr(%d, %d)", low, high)); if (mutex != NULL) uh->mtx = mutex; else uh->mtx = &unitmtx; TAILQ_INIT(&uh->head); TAILQ_INIT(&uh->ppfree); uh->low = low; uh->high = high; uh->first = 0; uh->last = 1 + (high - low); check_unrhdr(uh, __LINE__); } /* * Allocate a new unrheader set. * * Highest and lowest valid values given as parameters. */ struct unrhdr * new_unrhdr(int low, int high, struct mtx *mutex) { struct unrhdr *uh; uh = Malloc(sizeof *uh); init_unrhdr(uh, low, high, mutex); return (uh); } void delete_unrhdr(struct unrhdr *uh) { check_unrhdr(uh, __LINE__); KASSERT(uh->busy == 0, ("unrhdr has %u allocations", uh->busy)); KASSERT(uh->alloc == 0, ("UNR memory leak in delete_unrhdr")); KASSERT(TAILQ_FIRST(&uh->ppfree) == NULL, ("unrhdr has postponed item for free")); Free(uh); } static __inline int is_bitmap(struct unrhdr *uh, struct unr *up) { return (up->ptr != uh && up->ptr != NULL); } /* * Look for sequence of items which can be combined into a bitmap, if * multiple are present, take the one which saves most memory. * * Return (1) if a sequence was found to indicate that another call * might be able to do more. Return (0) if we found no suitable sequence. * * NB: called from alloc_unr(), no new memory allocation allowed. */ static int optimize_unr(struct unrhdr *uh) { struct unr *up, *uf, *us; struct unrb *ub, *ubf; u_int a, l, ba; /* * Look for the run of items (if any) which when collapsed into * a bitmap would save most memory. */ us = NULL; ba = 0; TAILQ_FOREACH(uf, &uh->head, list) { if (uf->len >= NBITS) continue; a = 1; if (is_bitmap(uh, uf)) a++; l = uf->len; up = uf; while (1) { up = TAILQ_NEXT(up, list); if (up == NULL) break; if ((up->len + l) > NBITS) break; a++; if (is_bitmap(uh, up)) a++; l += up->len; } if (a > ba) { ba = a; us = uf; } } if (ba < 3) return (0); /* * If the first element is not a bitmap, make it one. * Trying to do so without allocating more memory complicates things * a bit */ if (!is_bitmap(uh, us)) { uf = TAILQ_NEXT(us, list); TAILQ_REMOVE(&uh->head, us, list); a = us->len; l = us->ptr == uh ? 1 : 0; ub = (void *)us; - ub->busy = 0; - if (l) { + bit_nclear(ub->map, 0, NBITS - 1); + if (l) bit_nset(ub->map, 0, a); - ub->busy += a; - } else { - bit_nclear(ub->map, 0, a); - } if (!is_bitmap(uh, uf)) { - if (uf->ptr == NULL) { + if (uf->ptr == NULL) bit_nclear(ub->map, a, a + uf->len - 1); - } else { + else bit_nset(ub->map, a, a + uf->len - 1); - ub->busy += uf->len; - } uf->ptr = ub; uf->len += a; us = uf; } else { ubf = uf->ptr; for (l = 0; l < uf->len; l++, a++) { - if (bit_test(ubf->map, l)) { + if (bit_test(ubf->map, l)) bit_set(ub->map, a); - ub->busy++; - } else { + else bit_clear(ub->map, a); - } } uf->len = a; delete_unr(uh, uf->ptr); uf->ptr = ub; us = uf; } } ub = us->ptr; while (1) { uf = TAILQ_NEXT(us, list); if (uf == NULL) return (1); if (uf->len + us->len > NBITS) return (1); if (uf->ptr == NULL) { bit_nclear(ub->map, us->len, us->len + uf->len - 1); us->len += uf->len; TAILQ_REMOVE(&uh->head, uf, list); delete_unr(uh, uf); } else if (uf->ptr == uh) { bit_nset(ub->map, us->len, us->len + uf->len - 1); - ub->busy += uf->len; us->len += uf->len; TAILQ_REMOVE(&uh->head, uf, list); delete_unr(uh, uf); } else { ubf = uf->ptr; for (l = 0; l < uf->len; l++, us->len++) { - if (bit_test(ubf->map, l)) { + if (bit_test(ubf->map, l)) bit_set(ub->map, us->len); - ub->busy++; - } else { + else bit_clear(ub->map, us->len); - } } TAILQ_REMOVE(&uh->head, uf, list); delete_unr(uh, ubf); delete_unr(uh, uf); } } } /* * See if a given unr should be collapsed with a neighbor. * * NB: called from alloc_unr(), no new memory allocation allowed. */ static void collapse_unr(struct unrhdr *uh, struct unr *up) { struct unr *upp; struct unrb *ub; /* If bitmap is all set or clear, change it to runlength */ if (is_bitmap(uh, up)) { ub = up->ptr; - if (ub->busy == up->len) { + if (ub_full(ub, up->len)) { delete_unr(uh, up->ptr); up->ptr = uh; - } else if (ub->busy == 0) { + } else if (ub_empty(ub, up->len)) { delete_unr(uh, up->ptr); up->ptr = NULL; } } /* If nothing left in runlength, delete it */ if (up->len == 0) { upp = TAILQ_PREV(up, unrhd, list); if (upp == NULL) upp = TAILQ_NEXT(up, list); TAILQ_REMOVE(&uh->head, up, list); delete_unr(uh, up); up = upp; } /* If we have "hot-spot" still, merge with neighbor if possible */ if (up != NULL) { upp = TAILQ_PREV(up, unrhd, list); if (upp != NULL && up->ptr == upp->ptr) { up->len += upp->len; TAILQ_REMOVE(&uh->head, upp, list); delete_unr(uh, upp); } upp = TAILQ_NEXT(up, list); if (upp != NULL && up->ptr == upp->ptr) { up->len += upp->len; TAILQ_REMOVE(&uh->head, upp, list); delete_unr(uh, upp); } } /* Merge into ->first if possible */ upp = TAILQ_FIRST(&uh->head); if (upp != NULL && upp->ptr == uh) { uh->first += upp->len; TAILQ_REMOVE(&uh->head, upp, list); delete_unr(uh, upp); if (up == upp) up = NULL; } /* Merge into ->last if possible */ upp = TAILQ_LAST(&uh->head, unrhd); if (upp != NULL && upp->ptr == NULL) { uh->last += upp->len; TAILQ_REMOVE(&uh->head, upp, list); delete_unr(uh, upp); if (up == upp) up = NULL; } /* Try to make bitmaps */ while (optimize_unr(uh)) continue; } /* * Allocate a free unr. */ int alloc_unrl(struct unrhdr *uh) { struct unr *up; struct unrb *ub; u_int x; int y; mtx_assert(uh->mtx, MA_OWNED); check_unrhdr(uh, __LINE__); x = uh->low + uh->first; up = TAILQ_FIRST(&uh->head); /* * If we have an ideal split, just adjust the first+last */ if (up == NULL && uh->last > 0) { uh->first++; uh->last--; uh->busy++; return (x); } /* * We can always allocate from the first list element, so if we have * nothing on the list, we must have run out of unit numbers. */ if (up == NULL) return (-1); KASSERT(up->ptr != uh, ("UNR first element is allocated")); if (up->ptr == NULL) { /* free run */ uh->first++; up->len--; } else { /* bitmap */ ub = up->ptr; - KASSERT(ub->busy < up->len, ("UNR bitmap confusion")); bit_ffc(ub->map, up->len, &y); KASSERT(y != -1, ("UNR corruption: No clear bit in bitmap.")); bit_set(ub->map, y); - ub->busy++; x += y; } uh->busy++; collapse_unr(uh, up); return (x); } int alloc_unr(struct unrhdr *uh) { int i; mtx_lock(uh->mtx); i = alloc_unrl(uh); clean_unrhdrl(uh); mtx_unlock(uh->mtx); return (i); } static int alloc_unr_specificl(struct unrhdr *uh, u_int item, void **p1, void **p2) { struct unr *up, *upn; struct unrb *ub; u_int i, last, tl; mtx_assert(uh->mtx, MA_OWNED); if (item < uh->low + uh->first || item > uh->high) return (-1); up = TAILQ_FIRST(&uh->head); /* Ideal split. */ if (up == NULL && item - uh->low == uh->first) { uh->first++; uh->last--; uh->busy++; check_unrhdr(uh, __LINE__); return (item); } i = item - uh->low - uh->first; if (up == NULL) { up = new_unr(uh, p1, p2); up->ptr = NULL; up->len = i; TAILQ_INSERT_TAIL(&uh->head, up, list); up = new_unr(uh, p1, p2); up->ptr = uh; up->len = 1; TAILQ_INSERT_TAIL(&uh->head, up, list); uh->last = uh->high - uh->low - i; uh->busy++; check_unrhdr(uh, __LINE__); return (item); } else { /* Find the item which contains the unit we want to allocate. */ TAILQ_FOREACH(up, &uh->head, list) { if (up->len > i) break; i -= up->len; } } if (up == NULL) { if (i > 0) { up = new_unr(uh, p1, p2); up->ptr = NULL; up->len = i; TAILQ_INSERT_TAIL(&uh->head, up, list); } up = new_unr(uh, p1, p2); up->ptr = uh; up->len = 1; TAILQ_INSERT_TAIL(&uh->head, up, list); goto done; } if (is_bitmap(uh, up)) { ub = up->ptr; if (bit_test(ub->map, i) == 0) { bit_set(ub->map, i); - ub->busy++; goto done; } else return (-1); } else if (up->ptr == uh) return (-1); KASSERT(up->ptr == NULL, ("alloc_unr_specificl: up->ptr != NULL (up=%p)", up)); /* Split off the tail end, if any. */ tl = up->len - (1 + i); if (tl > 0) { upn = new_unr(uh, p1, p2); upn->ptr = NULL; upn->len = tl; TAILQ_INSERT_AFTER(&uh->head, up, upn, list); } /* Split off head end, if any */ if (i > 0) { upn = new_unr(uh, p1, p2); upn->len = i; upn->ptr = NULL; TAILQ_INSERT_BEFORE(up, upn, list); } up->len = 1; up->ptr = uh; done: last = uh->high - uh->low - (item - uh->low); if (uh->last > last) uh->last = last; uh->busy++; collapse_unr(uh, up); check_unrhdr(uh, __LINE__); return (item); } int alloc_unr_specific(struct unrhdr *uh, u_int item) { void *p1, *p2; int i; WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "alloc_unr_specific"); p1 = Malloc(sizeof(struct unr)); p2 = Malloc(sizeof(struct unr)); mtx_lock(uh->mtx); i = alloc_unr_specificl(uh, item, &p1, &p2); mtx_unlock(uh->mtx); if (p1 != NULL) Free(p1); if (p2 != NULL) Free(p2); return (i); } /* * Free a unr. * * If we can save unrs by using a bitmap, do so. */ static void free_unrl(struct unrhdr *uh, u_int item, void **p1, void **p2) { struct unr *up, *upp, *upn; struct unrb *ub; u_int pl; KASSERT(item >= uh->low && item <= uh->high, ("UNR: free_unr(%u) out of range [%u...%u]", item, uh->low, uh->high)); check_unrhdr(uh, __LINE__); item -= uh->low; upp = TAILQ_FIRST(&uh->head); /* * Freeing in the ideal split case */ if (item + 1 == uh->first && upp == NULL) { uh->last++; uh->first--; uh->busy--; check_unrhdr(uh, __LINE__); return; } /* * Freeing in the ->first section. Create a run starting at the * freed item. The code below will subdivide it. */ if (item < uh->first) { up = new_unr(uh, p1, p2); up->ptr = uh; up->len = uh->first - item; TAILQ_INSERT_HEAD(&uh->head, up, list); uh->first -= up->len; } item -= uh->first; /* Find the item which contains the unit we want to free */ TAILQ_FOREACH(up, &uh->head, list) { if (up->len > item) break; item -= up->len; } /* Handle bitmap items */ if (is_bitmap(uh, up)) { ub = up->ptr; KASSERT(bit_test(ub->map, item) != 0, ("UNR: Freeing free item %d (bitmap)\n", item)); bit_clear(ub->map, item); uh->busy--; - ub->busy--; collapse_unr(uh, up); return; } KASSERT(up->ptr == uh, ("UNR Freeing free item %d (run))\n", item)); /* Just this one left, reap it */ if (up->len == 1) { up->ptr = NULL; uh->busy--; collapse_unr(uh, up); return; } /* Check if we can shift the item into the previous 'free' run */ upp = TAILQ_PREV(up, unrhd, list); if (item == 0 && upp != NULL && upp->ptr == NULL) { upp->len++; up->len--; uh->busy--; collapse_unr(uh, up); return; } /* Check if we can shift the item to the next 'free' run */ upn = TAILQ_NEXT(up, list); if (item == up->len - 1 && upn != NULL && upn->ptr == NULL) { upn->len++; up->len--; uh->busy--; collapse_unr(uh, up); return; } /* Split off the tail end, if any. */ pl = up->len - (1 + item); if (pl > 0) { upp = new_unr(uh, p1, p2); upp->ptr = uh; upp->len = pl; TAILQ_INSERT_AFTER(&uh->head, up, upp, list); } /* Split off head end, if any */ if (item > 0) { upp = new_unr(uh, p1, p2); upp->len = item; upp->ptr = uh; TAILQ_INSERT_BEFORE(up, upp, list); } up->len = 1; up->ptr = NULL; uh->busy--; collapse_unr(uh, up); } void free_unr(struct unrhdr *uh, u_int item) { void *p1, *p2; WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, "free_unr"); p1 = Malloc(sizeof(struct unr)); p2 = Malloc(sizeof(struct unr)); mtx_lock(uh->mtx); free_unrl(uh, item, &p1, &p2); clean_unrhdrl(uh); mtx_unlock(uh->mtx); if (p1 != NULL) Free(p1); if (p2 != NULL) Free(p2); } #ifndef _KERNEL /* USERLAND test driver */ /* * Simple stochastic test driver for the above functions. The code resides * here so that it can access static functions and structures. */ static bool verbose; #define VPRINTF(...) {if (verbose) printf(__VA_ARGS__);} static void print_unr(struct unrhdr *uh, struct unr *up) { u_int x; struct unrb *ub; printf(" %p len = %5u ", up, up->len); if (up->ptr == NULL) printf("free\n"); else if (up->ptr == uh) printf("alloc\n"); else { ub = up->ptr; - printf("bitmap(%d) [", ub->busy); + printf("bitmap ["); for (x = 0; x < up->len; x++) { if (bit_test(ub->map, x)) printf("#"); else printf(" "); } printf("]\n"); } } static void print_unrhdr(struct unrhdr *uh) { struct unr *up; u_int x; printf( "%p low = %u high = %u first = %u last = %u busy %u chunks = %u\n", uh, uh->low, uh->high, uh->first, uh->last, uh->busy, uh->alloc); x = uh->low + uh->first; TAILQ_FOREACH(up, &uh->head, list) { printf(" from = %5u", x); print_unr(uh, up); if (up->ptr == NULL || up->ptr == uh) x += up->len; else x += NBITS; } } static void test_alloc_unr(struct unrhdr *uh, u_int i, char a[]) { int j; if (a[i]) { VPRINTF("F %u\n", i); free_unr(uh, i); a[i] = 0; } else { no_alloc = 1; j = alloc_unr(uh); if (j != -1) { a[j] = 1; VPRINTF("A %d\n", j); } no_alloc = 0; } } static void test_alloc_unr_specific(struct unrhdr *uh, u_int i, char a[]) { int j; j = alloc_unr_specific(uh, i); if (j == -1) { VPRINTF("F %u\n", i); a[i] = 0; free_unr(uh, i); } else { a[i] = 1; VPRINTF("A %d\n", j); } } static void usage(char** argv) { printf("%s [-h] [-r REPETITIONS] [-v]\n", argv[0]); } int main(int argc, char **argv) { struct unrhdr *uh; char *a; long count = 10000; /* Number of unrs to test */ long reps = 1; int ch; u_int i, x, m, j; verbose = false; while ((ch = getopt(argc, argv, "hr:v")) != -1) { switch (ch) { case 'r': errno = 0; reps = strtol(optarg, NULL, 0); if (errno == ERANGE || errno == EINVAL) { usage(argv); exit(2); } break; case 'v': verbose = true; break; case 'h': default: usage(argv); exit(2); } } setbuf(stdout, NULL); uh = new_unrhdr(0, count - 1, NULL); print_unrhdr(uh); a = calloc(count, sizeof(char)); if (a == NULL) err(1, "calloc failed"); srandomdev(); printf("sizeof(struct unr) %zu\n", sizeof(struct unr)); printf("sizeof(struct unrb) %zu\n", sizeof(struct unrb)); printf("sizeof(struct unrhdr) %zu\n", sizeof(struct unrhdr)); - printf("NBITS %d\n", NBITS); + printf("NBITS %lu\n", NBITS); x = 1; for (m = 0; m < count * reps; m++) { j = random(); i = (j >> 1) % count; #if 0 if (a[i] && (j & 1)) continue; #endif if ((random() & 1) != 0) test_alloc_unr(uh, i, a); else test_alloc_unr_specific(uh, i, a); if (verbose) print_unrhdr(uh); check_unrhdr(uh, __LINE__); } for (i = 0; i < count; i++) { if (a[i]) { if (verbose) { printf("C %u\n", i); print_unrhdr(uh); } free_unr(uh, i); } } print_unrhdr(uh); delete_unrhdr(uh); free(a); return (0); } #endif Index: user/ngie/release-pkg-fix-tests/sys/net/flowtable.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/net/flowtable.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/net/flowtable.c (revision 299091) @@ -1,1194 +1,1189 @@ /*- * Copyright (c) 2014 Gleb Smirnoff * Copyright (c) 2008-2010, BitGravity Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * * 2. Neither the name of the BitGravity Corporation nor the names of its * contributors may be used to endorse or promote products derived from * this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include "opt_route.h" #include "opt_mpath.h" #include "opt_ddb.h" #include "opt_inet.h" #include "opt_inet6.h" #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #ifdef INET6 #include #endif #ifdef FLOWTABLE_HASH_ALL #include #include #include #endif #include #ifdef FLOWTABLE_HASH_ALL #define KEY_PORTS (sizeof(uint16_t) * 2) #define KEY_ADDRS 2 #else #define KEY_PORTS 0 #define KEY_ADDRS 1 #endif #ifdef INET6 #define KEY_ADDR_LEN sizeof(struct in6_addr) #else #define KEY_ADDR_LEN sizeof(struct in_addr) #endif #define KEYLEN ((KEY_ADDR_LEN * KEY_ADDRS + KEY_PORTS) / sizeof(uint32_t)) struct flentry { uint32_t f_hash; /* hash flowing forward */ uint32_t f_key[KEYLEN]; /* address(es and ports) */ uint32_t f_uptime; /* uptime at last access */ uint16_t f_fibnum; /* fib index */ #ifdef FLOWTABLE_HASH_ALL uint8_t f_proto; /* protocol */ uint8_t f_flags; /* stale? */ #define FL_STALE 1 #endif SLIST_ENTRY(flentry) f_next; /* pointer to collision entry */ struct rtentry *f_rt; /* rtentry for flow */ struct llentry *f_lle; /* llentry for flow */ }; #undef KEYLEN SLIST_HEAD(flist, flentry); /* Make sure we can use pcpu_zone_ptr for struct flist. */ CTASSERT(sizeof(struct flist) == sizeof(void *)); struct flowtable { counter_u64_t *ft_stat; int ft_size; /* * ft_table is a malloc(9)ed array of pointers. Pointers point to * memory from UMA_ZONE_PCPU zone. * ft_masks is per-cpu pointer itself. Each instance points * to a malloc(9)ed bitset, that is private to corresponding CPU. */ struct flist **ft_table; bitstr_t **ft_masks; bitstr_t *ft_tmpmask; }; #define FLOWSTAT_ADD(ft, name, v) \ counter_u64_add((ft)->ft_stat[offsetof(struct flowtable_stat, name) / sizeof(uint64_t)], (v)) #define FLOWSTAT_INC(ft, name) FLOWSTAT_ADD(ft, name, 1) static struct proc *flowcleanerproc; static uint32_t flow_hashjitter; static struct cv flowclean_f_cv; static struct cv flowclean_c_cv; static struct mtx flowclean_lock; static uint32_t flowclean_cycles; /* * TODO: * - add sysctls to resize && flush flow tables * - Add per flowtable sysctls for statistics and configuring timeouts * - add saturation counter to rtentry to support per-packet load-balancing * add flag to indicate round-robin flow, add list lookup from head for flows * - add sysctl / device node / syscall to support exporting and importing * of flows with flag to indicate that a flow was imported so should * not be considered for auto-cleaning * - support explicit connection state (currently only ad-hoc for DSR) * - idetach() cleanup for options VIMAGE builds. */ #ifdef INET static VNET_DEFINE(struct flowtable, ip4_ft); #define V_ip4_ft VNET(ip4_ft) #endif #ifdef INET6 static VNET_DEFINE(struct flowtable, ip6_ft); #define V_ip6_ft VNET(ip6_ft) #endif static uma_zone_t flow_zone; static VNET_DEFINE(int, flowtable_enable) = 1; #define V_flowtable_enable VNET(flowtable_enable) static SYSCTL_NODE(_net, OID_AUTO, flowtable, CTLFLAG_RD, NULL, "flowtable"); SYSCTL_INT(_net_flowtable, OID_AUTO, enable, CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(flowtable_enable), 0, "enable flowtable caching."); SYSCTL_UMA_MAX(_net_flowtable, OID_AUTO, maxflows, CTLFLAG_RW, &flow_zone, "Maximum number of flows allowed"); static MALLOC_DEFINE(M_FTABLE, "flowtable", "flowtable hashes and bitstrings"); static struct flentry * flowtable_lookup_common(struct flowtable *, uint32_t *, int, uint32_t); #ifdef INET static struct flentry * flowtable_lookup_ipv4(struct mbuf *m, struct route *ro) { struct flentry *fle; struct sockaddr_in *sin; struct ip *ip; uint32_t fibnum; #ifdef FLOWTABLE_HASH_ALL uint32_t key[3]; int iphlen; uint16_t sport, dport; uint8_t proto; #endif ip = mtod(m, struct ip *); if (ip->ip_src.s_addr == ip->ip_dst.s_addr || (ntohl(ip->ip_dst.s_addr) >> IN_CLASSA_NSHIFT) == IN_LOOPBACKNET || (ntohl(ip->ip_src.s_addr) >> IN_CLASSA_NSHIFT) == IN_LOOPBACKNET) return (NULL); fibnum = M_GETFIB(m); #ifdef FLOWTABLE_HASH_ALL iphlen = ip->ip_hl << 2; proto = ip->ip_p; switch (proto) { case IPPROTO_TCP: { struct tcphdr *th; th = (struct tcphdr *)((char *)ip + iphlen); sport = th->th_sport; dport = th->th_dport; if (th->th_flags & (TH_RST|TH_FIN)) fibnum |= (FL_STALE << 24); break; } case IPPROTO_UDP: { struct udphdr *uh; uh = (struct udphdr *)((char *)ip + iphlen); sport = uh->uh_sport; dport = uh->uh_dport; break; } case IPPROTO_SCTP: { struct sctphdr *sh; sh = (struct sctphdr *)((char *)ip + iphlen); sport = sh->src_port; dport = sh->dest_port; /* XXXGL: handle stale? */ break; } default: sport = dport = 0; break; } key[0] = ip->ip_dst.s_addr; key[1] = ip->ip_src.s_addr; key[2] = (dport << 16) | sport; fibnum |= proto << 16; fle = flowtable_lookup_common(&V_ip4_ft, key, 3 * sizeof(uint32_t), fibnum); #else /* !FLOWTABLE_HASH_ALL */ fle = flowtable_lookup_common(&V_ip4_ft, (uint32_t *)&ip->ip_dst, sizeof(struct in_addr), fibnum); #endif /* FLOWTABLE_HASH_ALL */ if (fle == NULL) return (NULL); sin = (struct sockaddr_in *)&ro->ro_dst; sin->sin_family = AF_INET; sin->sin_len = sizeof(*sin); sin->sin_addr = ip->ip_dst; return (fle); } #endif /* INET */ #ifdef INET6 /* * PULLUP_TO(len, p, T) makes sure that len + sizeof(T) is contiguous, * then it sets p to point at the offset "len" in the mbuf. WARNING: the * pointer might become stale after other pullups (but we never use it * this way). */ #define PULLUP_TO(_len, p, T) \ do { \ int x = (_len) + sizeof(T); \ if ((m)->m_len < x) \ return (NULL); \ p = (mtod(m, char *) + (_len)); \ } while (0) #define TCP(p) ((struct tcphdr *)(p)) #define SCTP(p) ((struct sctphdr *)(p)) #define UDP(p) ((struct udphdr *)(p)) static struct flentry * flowtable_lookup_ipv6(struct mbuf *m, struct route *ro) { struct flentry *fle; struct sockaddr_in6 *sin6; struct ip6_hdr *ip6; uint32_t fibnum; #ifdef FLOWTABLE_HASH_ALL uint32_t key[9]; void *ulp; int hlen; uint16_t sport, dport; u_short offset; uint8_t proto; #else uint32_t key[4]; #endif ip6 = mtod(m, struct ip6_hdr *); if (in6_localaddr(&ip6->ip6_dst)) return (NULL); fibnum = M_GETFIB(m); #ifdef FLOWTABLE_HASH_ALL hlen = sizeof(struct ip6_hdr); proto = ip6->ip6_nxt; offset = sport = dport = 0; ulp = NULL; while (ulp == NULL) { switch (proto) { case IPPROTO_ICMPV6: case IPPROTO_OSPFIGP: case IPPROTO_PIM: case IPPROTO_CARP: case IPPROTO_ESP: case IPPROTO_NONE: ulp = ip6; break; case IPPROTO_TCP: PULLUP_TO(hlen, ulp, struct tcphdr); dport = TCP(ulp)->th_dport; sport = TCP(ulp)->th_sport; if (TCP(ulp)->th_flags & (TH_RST|TH_FIN)) fibnum |= (FL_STALE << 24); break; case IPPROTO_SCTP: PULLUP_TO(hlen, ulp, struct sctphdr); dport = SCTP(ulp)->src_port; sport = SCTP(ulp)->dest_port; /* XXXGL: handle stale? */ break; case IPPROTO_UDP: PULLUP_TO(hlen, ulp, struct udphdr); dport = UDP(ulp)->uh_dport; sport = UDP(ulp)->uh_sport; break; case IPPROTO_HOPOPTS: /* RFC 2460 */ PULLUP_TO(hlen, ulp, struct ip6_hbh); hlen += (((struct ip6_hbh *)ulp)->ip6h_len + 1) << 3; proto = ((struct ip6_hbh *)ulp)->ip6h_nxt; ulp = NULL; break; case IPPROTO_ROUTING: /* RFC 2460 */ PULLUP_TO(hlen, ulp, struct ip6_rthdr); hlen += (((struct ip6_rthdr *)ulp)->ip6r_len + 1) << 3; proto = ((struct ip6_rthdr *)ulp)->ip6r_nxt; ulp = NULL; break; case IPPROTO_FRAGMENT: /* RFC 2460 */ PULLUP_TO(hlen, ulp, struct ip6_frag); hlen += sizeof (struct ip6_frag); proto = ((struct ip6_frag *)ulp)->ip6f_nxt; offset = ((struct ip6_frag *)ulp)->ip6f_offlg & IP6F_OFF_MASK; ulp = NULL; break; case IPPROTO_DSTOPTS: /* RFC 2460 */ PULLUP_TO(hlen, ulp, struct ip6_hbh); hlen += (((struct ip6_hbh *)ulp)->ip6h_len + 1) << 3; proto = ((struct ip6_hbh *)ulp)->ip6h_nxt; ulp = NULL; break; case IPPROTO_AH: /* RFC 2402 */ PULLUP_TO(hlen, ulp, struct ip6_ext); hlen += (((struct ip6_ext *)ulp)->ip6e_len + 2) << 2; proto = ((struct ip6_ext *)ulp)->ip6e_nxt; ulp = NULL; break; default: PULLUP_TO(hlen, ulp, struct ip6_ext); break; } } bcopy(&ip6->ip6_dst, &key[0], sizeof(struct in6_addr)); bcopy(&ip6->ip6_src, &key[4], sizeof(struct in6_addr)); key[8] = (dport << 16) | sport; fibnum |= proto << 16; fle = flowtable_lookup_common(&V_ip6_ft, key, 9 * sizeof(uint32_t), fibnum); #else /* !FLOWTABLE_HASH_ALL */ bcopy(&ip6->ip6_dst, &key[0], sizeof(struct in6_addr)); fle = flowtable_lookup_common(&V_ip6_ft, key, sizeof(struct in6_addr), fibnum); #endif /* FLOWTABLE_HASH_ALL */ if (fle == NULL) return (NULL); sin6 = (struct sockaddr_in6 *)&ro->ro_dst; sin6->sin6_family = AF_INET6; sin6->sin6_len = sizeof(*sin6); bcopy(&ip6->ip6_dst, &sin6->sin6_addr, sizeof(struct in6_addr)); return (fle); } #endif /* INET6 */ static bitstr_t * flowtable_mask(struct flowtable *ft) { /* * flowtable_free_stale() calls w/o critical section, but * with sched_bind(). Since pointer is stable throughout * ft lifetime, it is safe, otherwise... * * CRITICAL_ASSERT(curthread); */ return (*(bitstr_t **)zpcpu_get(ft->ft_masks)); } static struct flist * flowtable_list(struct flowtable *ft, uint32_t hash) { CRITICAL_ASSERT(curthread); return (zpcpu_get(ft->ft_table[hash % ft->ft_size])); } static int flow_stale(struct flowtable *ft, struct flentry *fle, int maxidle) { if (((fle->f_rt->rt_flags & RTF_UP) == 0) || (fle->f_rt->rt_ifp == NULL) || !RT_LINK_IS_UP(fle->f_rt->rt_ifp) || (fle->f_lle->la_flags & LLE_VALID) == 0) return (1); if (time_uptime - fle->f_uptime > maxidle) return (1); #ifdef FLOWTABLE_HASH_ALL if (fle->f_flags & FL_STALE) return (1); #endif return (0); } static int flow_full(void) { int count, max; count = uma_zone_get_cur(flow_zone); max = uma_zone_get_max(flow_zone); return (count > (max - (max >> 3))); } static int flow_matches(struct flentry *fle, uint32_t *key, int keylen, uint32_t fibnum) { #ifdef FLOWTABLE_HASH_ALL uint8_t proto; proto = (fibnum >> 16) & 0xff; fibnum &= 0xffff; #endif CRITICAL_ASSERT(curthread); /* Microoptimization for IPv4: don't use bcmp(). */ if (((keylen == sizeof(uint32_t) && (fle->f_key[0] == key[0])) || (bcmp(fle->f_key, key, keylen) == 0)) && fibnum == fle->f_fibnum && #ifdef FLOWTABLE_HASH_ALL proto == fle->f_proto && #endif (fle->f_rt->rt_flags & RTF_UP) && fle->f_rt->rt_ifp != NULL && (fle->f_lle->la_flags & LLE_VALID)) return (1); return (0); } static struct flentry * flowtable_insert(struct flowtable *ft, uint32_t hash, uint32_t *key, int keylen, uint32_t fibnum0) { #ifdef INET6 struct route_in6 sro6; #endif #ifdef INET struct route sro; #endif struct route *ro = NULL; struct rtentry *rt; struct lltable *lt = NULL; struct llentry *lle; struct sockaddr_storage *l3addr; struct ifnet *ifp; struct flist *flist; struct flentry *fle, *iter; bitstr_t *mask; uint16_t fibnum = fibnum0; #ifdef FLOWTABLE_HASH_ALL uint8_t proto; proto = (fibnum0 >> 16) & 0xff; fibnum = fibnum0 & 0xffff; #endif /* * This bit of code ends up locking the * same route 3 times (just like ip_output + ether_output) * - at lookup * - in rt_check when called by arpresolve * - dropping the refcount for the rtentry * * This could be consolidated to one if we wrote a variant * of arpresolve with an rt_check variant that expected to * receive the route locked */ #ifdef INET if (ft == &V_ip4_ft) { struct sockaddr_in *sin; ro = &sro; bzero(&sro.ro_dst, sizeof(sro.ro_dst)); sin = (struct sockaddr_in *)&sro.ro_dst; sin->sin_family = AF_INET; sin->sin_len = sizeof(*sin); sin->sin_addr.s_addr = key[0]; } #endif #ifdef INET6 if (ft == &V_ip6_ft) { struct sockaddr_in6 *sin6; ro = (struct route *)&sro6; sin6 = &sro6.ro_dst; bzero(sin6, sizeof(*sin6)); sin6->sin6_family = AF_INET6; sin6->sin6_len = sizeof(*sin6); bcopy(key, &sin6->sin6_addr, sizeof(struct in6_addr)); } #endif ro->ro_rt = NULL; #ifdef RADIX_MPATH rtalloc_mpath_fib(ro, hash, fibnum); #else rtalloc_ign_fib(ro, 0, fibnum); #endif if (ro->ro_rt == NULL) return (NULL); rt = ro->ro_rt; ifp = rt->rt_ifp; if (ifp->if_flags & (IFF_POINTOPOINT | IFF_LOOPBACK)) { RTFREE(rt); return (NULL); } #ifdef INET if (ft == &V_ip4_ft) lt = LLTABLE(ifp); #endif #ifdef INET6 if (ft == &V_ip6_ft) lt = LLTABLE6(ifp); #endif if (rt->rt_flags & RTF_GATEWAY) l3addr = (struct sockaddr_storage *)rt->rt_gateway; else l3addr = (struct sockaddr_storage *)&ro->ro_dst; lle = llentry_alloc(ifp, lt, l3addr); if (lle == NULL) { RTFREE(rt); return (NULL); } /* Don't insert the entry if the ARP hasn't yet finished resolving. */ if ((lle->la_flags & LLE_VALID) == 0) { RTFREE(rt); LLE_FREE(lle); FLOWSTAT_INC(ft, ft_fail_lle_invalid); return (NULL); } fle = uma_zalloc(flow_zone, M_NOWAIT | M_ZERO); if (fle == NULL) { RTFREE(rt); LLE_FREE(lle); return (NULL); } fle->f_hash = hash; bcopy(key, &fle->f_key, keylen); fle->f_rt = rt; fle->f_lle = lle; fle->f_fibnum = fibnum; fle->f_uptime = time_uptime; #ifdef FLOWTABLE_HASH_ALL fle->f_proto = proto; fle->f_flags = fibnum0 >> 24; #endif critical_enter(); mask = flowtable_mask(ft); flist = flowtable_list(ft, hash); if (SLIST_EMPTY(flist)) { bit_set(mask, (hash % ft->ft_size)); SLIST_INSERT_HEAD(flist, fle, f_next); goto skip; } /* * find end of list and make sure that we were not * preempted by another thread handling this flow */ SLIST_FOREACH(iter, flist, f_next) { KASSERT(iter->f_hash % ft->ft_size == hash % ft->ft_size, ("%s: wrong hash", __func__)); if (flow_matches(iter, key, keylen, fibnum)) { /* * We probably migrated to an other CPU after * lookup in flowtable_lookup_common() failed. * It appeared that this CPU already has flow * entry. */ iter->f_uptime = time_uptime; #ifdef FLOWTABLE_HASH_ALL iter->f_flags |= fibnum >> 24; #endif critical_exit(); FLOWSTAT_INC(ft, ft_collisions); uma_zfree(flow_zone, fle); return (iter); } } SLIST_INSERT_HEAD(flist, fle, f_next); skip: critical_exit(); FLOWSTAT_INC(ft, ft_inserts); return (fle); } int flowtable_lookup(sa_family_t sa, struct mbuf *m, struct route *ro) { struct flentry *fle; struct llentry *lle; if (V_flowtable_enable == 0) return (ENXIO); switch (sa) { #ifdef INET case AF_INET: fle = flowtable_lookup_ipv4(m, ro); break; #endif #ifdef INET6 case AF_INET6: fle = flowtable_lookup_ipv6(m, ro); break; #endif default: panic("%s: sa %d", __func__, sa); } if (fle == NULL) return (EHOSTUNREACH); if (M_HASHTYPE_GET(m) == M_HASHTYPE_NONE) { M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE); m->m_pkthdr.flowid = fle->f_hash; } ro->ro_rt = fle->f_rt; ro->ro_flags |= RT_NORTREF; lle = fle->f_lle; if (lle != NULL && (lle->la_flags & LLE_VALID)) { ro->ro_prepend = lle->r_linkdata; ro->ro_plen = lle->r_hdrlen; ro->ro_flags |= RT_MAY_LOOP; if (lle->la_flags & LLE_IFADDR) ro->ro_flags |= RT_L2_ME; } return (0); } static struct flentry * flowtable_lookup_common(struct flowtable *ft, uint32_t *key, int keylen, uint32_t fibnum) { struct flist *flist; struct flentry *fle; uint32_t hash; FLOWSTAT_INC(ft, ft_lookups); hash = jenkins_hash32(key, keylen / sizeof(uint32_t), flow_hashjitter); critical_enter(); flist = flowtable_list(ft, hash); SLIST_FOREACH(fle, flist, f_next) { KASSERT(fle->f_hash % ft->ft_size == hash % ft->ft_size, ("%s: wrong hash", __func__)); if (flow_matches(fle, key, keylen, fibnum)) { fle->f_uptime = time_uptime; #ifdef FLOWTABLE_HASH_ALL fle->f_flags |= fibnum >> 24; #endif critical_exit(); FLOWSTAT_INC(ft, ft_hits); return (fle); } } critical_exit(); FLOWSTAT_INC(ft, ft_misses); return (flowtable_insert(ft, hash, key, keylen, fibnum)); } -/* - * used by the bit_alloc macro - */ -#define calloc(count, size) malloc((count)*(size), M_FTABLE, M_WAITOK | M_ZERO) static void flowtable_alloc(struct flowtable *ft) { ft->ft_table = malloc(ft->ft_size * sizeof(struct flist), M_FTABLE, M_WAITOK); for (int i = 0; i < ft->ft_size; i++) ft->ft_table[i] = uma_zalloc(pcpu_zone_ptr, M_WAITOK | M_ZERO); ft->ft_masks = uma_zalloc(pcpu_zone_ptr, M_WAITOK); for (int i = 0; i < mp_ncpus; i++) { bitstr_t **b; b = zpcpu_get_cpu(ft->ft_masks, i); - *b = bit_alloc(ft->ft_size); + *b = bit_alloc(ft->ft_size, M_FTABLE, M_WAITOK); } - ft->ft_tmpmask = bit_alloc(ft->ft_size); + ft->ft_tmpmask = bit_alloc(ft->ft_size, M_FTABLE, M_WAITOK); } -#undef calloc static void flowtable_free_stale(struct flowtable *ft, struct rtentry *rt, int maxidle) { struct flist *flist, freelist; struct flentry *fle, *fle1, *fleprev; bitstr_t *mask, *tmpmask; int curbit, tmpsize; SLIST_INIT(&freelist); mask = flowtable_mask(ft); tmpmask = ft->ft_tmpmask; tmpsize = ft->ft_size; memcpy(tmpmask, mask, ft->ft_size/8); curbit = 0; fleprev = NULL; /* pacify gcc */ /* * XXX Note to self, bit_ffs operates at the byte level * and thus adds gratuitous overhead */ bit_ffs(tmpmask, ft->ft_size, &curbit); while (curbit != -1) { if (curbit >= ft->ft_size || curbit < -1) { log(LOG_ALERT, "warning: bad curbit value %d \n", curbit); break; } FLOWSTAT_INC(ft, ft_free_checks); critical_enter(); flist = flowtable_list(ft, curbit); #ifdef DIAGNOSTIC if (SLIST_EMPTY(flist) && curbit > 0) { log(LOG_ALERT, "warning bit=%d set, but no fle found\n", curbit); } #endif SLIST_FOREACH_SAFE(fle, flist, f_next, fle1) { if (rt != NULL && fle->f_rt != rt) { fleprev = fle; continue; } if (!flow_stale(ft, fle, maxidle)) { fleprev = fle; continue; } if (fle == SLIST_FIRST(flist)) SLIST_REMOVE_HEAD(flist, f_next); else SLIST_REMOVE_AFTER(fleprev, f_next); SLIST_INSERT_HEAD(&freelist, fle, f_next); } if (SLIST_EMPTY(flist)) bit_clear(mask, curbit); critical_exit(); bit_clear(tmpmask, curbit); bit_ffs(tmpmask, tmpsize, &curbit); } SLIST_FOREACH_SAFE(fle, &freelist, f_next, fle1) { FLOWSTAT_INC(ft, ft_frees); if (fle->f_rt != NULL) RTFREE(fle->f_rt); if (fle->f_lle != NULL) LLE_FREE(fle->f_lle); uma_zfree(flow_zone, fle); } } static void flowtable_clean_vnet(struct flowtable *ft, struct rtentry *rt, int maxidle) { int i; CPU_FOREACH(i) { if (smp_started == 1) { thread_lock(curthread); sched_bind(curthread, i); thread_unlock(curthread); } flowtable_free_stale(ft, rt, maxidle); if (smp_started == 1) { thread_lock(curthread); sched_unbind(curthread); thread_unlock(curthread); } } } void flowtable_route_flush(sa_family_t sa, struct rtentry *rt) { struct flowtable *ft; switch (sa) { #ifdef INET case AF_INET: ft = &V_ip4_ft; break; #endif #ifdef INET6 case AF_INET6: ft = &V_ip6_ft; break; #endif default: panic("%s: sa %d", __func__, sa); } flowtable_clean_vnet(ft, rt, 0); } static void flowtable_cleaner(void) { VNET_ITERATOR_DECL(vnet_iter); struct thread *td; if (bootverbose) log(LOG_INFO, "flowtable cleaner started\n"); td = curthread; while (1) { uint32_t flowclean_freq, maxidle; /* * The maximum idle time, as well as frequency are arbitrary. */ if (flow_full()) maxidle = 5; else maxidle = 30; VNET_LIST_RLOCK(); VNET_FOREACH(vnet_iter) { CURVNET_SET(vnet_iter); #ifdef INET flowtable_clean_vnet(&V_ip4_ft, NULL, maxidle); #endif #ifdef INET6 flowtable_clean_vnet(&V_ip6_ft, NULL, maxidle); #endif CURVNET_RESTORE(); } VNET_LIST_RUNLOCK(); if (flow_full()) flowclean_freq = 4*hz; else flowclean_freq = 20*hz; mtx_lock(&flowclean_lock); thread_lock(td); sched_prio(td, PPAUSE); thread_unlock(td); flowclean_cycles++; cv_broadcast(&flowclean_f_cv); cv_timedwait(&flowclean_c_cv, &flowclean_lock, flowclean_freq); mtx_unlock(&flowclean_lock); } } static void flowtable_flush(void *unused __unused) { uint64_t start; mtx_lock(&flowclean_lock); start = flowclean_cycles; while (start == flowclean_cycles) { cv_broadcast(&flowclean_c_cv); cv_wait(&flowclean_f_cv, &flowclean_lock); } mtx_unlock(&flowclean_lock); } static struct kproc_desc flow_kp = { "flowcleaner", flowtable_cleaner, &flowcleanerproc }; SYSINIT(flowcleaner, SI_SUB_KTHREAD_IDLE, SI_ORDER_ANY, kproc_start, &flow_kp); static int flowtable_get_size(char *name) { int size; if (TUNABLE_INT_FETCH(name, &size)) { if (size < 256) size = 256; if (!powerof2(size)) { printf("%s must be power of 2\n", name); size = 2048; } } else { /* * round up to the next power of 2 */ size = 1 << fls((1024 + maxusers * 64) - 1); } return (size); } static void flowtable_init(const void *unused __unused) { flow_hashjitter = arc4random(); flow_zone = uma_zcreate("flows", sizeof(struct flentry), NULL, NULL, NULL, NULL, (64-1), UMA_ZONE_MAXBUCKET); uma_zone_set_max(flow_zone, 1024 + maxusers * 64 * mp_ncpus); cv_init(&flowclean_c_cv, "c_flowcleanwait"); cv_init(&flowclean_f_cv, "f_flowcleanwait"); mtx_init(&flowclean_lock, "flowclean lock", NULL, MTX_DEF); EVENTHANDLER_REGISTER(ifnet_departure_event, flowtable_flush, NULL, EVENTHANDLER_PRI_ANY); } SYSINIT(flowtable_init, SI_SUB_PROTO_BEGIN, SI_ORDER_FIRST, flowtable_init, NULL); #ifdef INET static SYSCTL_NODE(_net_flowtable, OID_AUTO, ip4, CTLFLAG_RD, NULL, "Flowtable for IPv4"); static VNET_PCPUSTAT_DEFINE(struct flowtable_stat, ip4_ftstat); VNET_PCPUSTAT_SYSINIT(ip4_ftstat); VNET_PCPUSTAT_SYSUNINIT(ip4_ftstat); SYSCTL_VNET_PCPUSTAT(_net_flowtable_ip4, OID_AUTO, stat, struct flowtable_stat, ip4_ftstat, "Flowtable statistics for IPv4 " "(struct flowtable_stat, net/flowtable.h)"); static void flowtable_init_vnet_v4(const void *unused __unused) { V_ip4_ft.ft_size = flowtable_get_size("net.flowtable.ip4.size"); V_ip4_ft.ft_stat = VNET(ip4_ftstat); flowtable_alloc(&V_ip4_ft); } VNET_SYSINIT(ft_vnet_v4, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_ANY, flowtable_init_vnet_v4, NULL); #endif /* INET */ #ifdef INET6 static SYSCTL_NODE(_net_flowtable, OID_AUTO, ip6, CTLFLAG_RD, NULL, "Flowtable for IPv6"); static VNET_PCPUSTAT_DEFINE(struct flowtable_stat, ip6_ftstat); VNET_PCPUSTAT_SYSINIT(ip6_ftstat); VNET_PCPUSTAT_SYSUNINIT(ip6_ftstat); SYSCTL_VNET_PCPUSTAT(_net_flowtable_ip6, OID_AUTO, stat, struct flowtable_stat, ip6_ftstat, "Flowtable statistics for IPv6 " "(struct flowtable_stat, net/flowtable.h)"); static void flowtable_init_vnet_v6(const void *unused __unused) { V_ip6_ft.ft_size = flowtable_get_size("net.flowtable.ip6.size"); V_ip6_ft.ft_stat = VNET(ip6_ftstat); flowtable_alloc(&V_ip6_ft); } VNET_SYSINIT(flowtable_init_vnet_v6, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_ANY, flowtable_init_vnet_v6, NULL); #endif /* INET6 */ #ifdef DDB static bitstr_t * flowtable_mask_pcpu(struct flowtable *ft, int cpuid) { return (zpcpu_get_cpu(*ft->ft_masks, cpuid)); } static struct flist * flowtable_list_pcpu(struct flowtable *ft, uint32_t hash, int cpuid) { return (zpcpu_get_cpu(&ft->ft_table[hash % ft->ft_size], cpuid)); } static void flow_show(struct flowtable *ft, struct flentry *fle) { int idle_time; int rt_valid, ifp_valid; volatile struct rtentry *rt; struct ifnet *ifp = NULL; uint32_t *hashkey = fle->f_key; idle_time = (int)(time_uptime - fle->f_uptime); rt = fle->f_rt; rt_valid = rt != NULL; if (rt_valid) ifp = rt->rt_ifp; ifp_valid = ifp != NULL; #ifdef INET if (ft == &V_ip4_ft) { char daddr[4*sizeof "123"]; #ifdef FLOWTABLE_HASH_ALL char saddr[4*sizeof "123"]; uint16_t sport, dport; #endif inet_ntoa_r(*(struct in_addr *) &hashkey[0], daddr); #ifdef FLOWTABLE_HASH_ALL inet_ntoa_r(*(struct in_addr *) &hashkey[1], saddr); dport = ntohs((uint16_t)(hashkey[2] >> 16)); sport = ntohs((uint16_t)(hashkey[2] & 0xffff)); db_printf("%s:%d->%s:%d", saddr, sport, daddr, dport); #else db_printf("%s ", daddr); #endif } #endif /* INET */ #ifdef INET6 if (ft == &V_ip6_ft) { #ifdef FLOWTABLE_HASH_ALL db_printf("\n\tkey=%08x:%08x:%08x%08x:%08x:%08x%08x:%08x:%08x", hashkey[0], hashkey[1], hashkey[2], hashkey[3], hashkey[4], hashkey[5], hashkey[6], hashkey[7], hashkey[8]); #else db_printf("\n\tkey=%08x:%08x:%08x ", hashkey[0], hashkey[1], hashkey[2]); #endif } #endif /* INET6 */ db_printf("hash=%08x idle_time=%03d" "\n\tfibnum=%02d rt=%p", fle->f_hash, idle_time, fle->f_fibnum, fle->f_rt); #ifdef FLOWTABLE_HASH_ALL if (fle->f_flags & FL_STALE) db_printf(" FL_STALE "); #endif if (rt_valid) { if (rt->rt_flags & RTF_UP) db_printf(" RTF_UP "); } if (ifp_valid) { if (ifp->if_flags & IFF_LOOPBACK) db_printf(" IFF_LOOPBACK "); if (ifp->if_flags & IFF_UP) db_printf(" IFF_UP "); if (ifp->if_flags & IFF_POINTOPOINT) db_printf(" IFF_POINTOPOINT "); } db_printf("\n"); } static void flowtable_show(struct flowtable *ft, int cpuid) { int curbit = 0; bitstr_t *mask, *tmpmask; if (cpuid != -1) db_printf("cpu: %d\n", cpuid); mask = flowtable_mask_pcpu(ft, cpuid); tmpmask = ft->ft_tmpmask; memcpy(tmpmask, mask, ft->ft_size/8); /* * XXX Note to self, bit_ffs operates at the byte level * and thus adds gratuitous overhead */ bit_ffs(tmpmask, ft->ft_size, &curbit); while (curbit != -1) { struct flist *flist; struct flentry *fle; if (curbit >= ft->ft_size || curbit < -1) { db_printf("warning: bad curbit value %d \n", curbit); break; } flist = flowtable_list_pcpu(ft, curbit, cpuid); SLIST_FOREACH(fle, flist, f_next) flow_show(ft, fle); bit_clear(tmpmask, curbit); bit_ffs(tmpmask, ft->ft_size, &curbit); } } static void flowtable_show_vnet(struct flowtable *ft) { int i; CPU_FOREACH(i) flowtable_show(ft, i); } DB_SHOW_COMMAND(flowtables, db_show_flowtables) { VNET_ITERATOR_DECL(vnet_iter); VNET_FOREACH(vnet_iter) { CURVNET_SET(vnet_iter); #ifdef VIMAGE db_printf("vnet %p\n", vnet_iter); #endif #ifdef INET printf("IPv4:\n"); flowtable_show_vnet(&V_ip4_ft); #endif #ifdef INET6 printf("IPv6:\n"); flowtable_show_vnet(&V_ip6_ft); #endif CURVNET_RESTORE(); } } #endif Index: user/ngie/release-pkg-fix-tests/sys/net80211/ieee80211_ht.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/net80211/ieee80211_ht.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/net80211/ieee80211_ht.c (revision 299091) @@ -1,2984 +1,2992 @@ /*- * Copyright (c) 2007-2008 Sam Leffler, Errno Consulting * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include #ifdef __FreeBSD__ __FBSDID("$FreeBSD$"); #endif /* * IEEE 802.11n protocol support. */ #include "opt_inet.h" #include "opt_wlan.h" #include #include #include #include #include #include #include #include #include #include #include #include #include /* define here, used throughout file */ #define MS(_v, _f) (((_v) & _f) >> _f##_S) #define SM(_v, _f) (((_v) << _f##_S) & _f) const struct ieee80211_mcs_rates ieee80211_htrates[IEEE80211_HTRATE_MAXSIZE] = { { 13, 14, 27, 30 }, /* MCS 0 */ { 26, 29, 54, 60 }, /* MCS 1 */ { 39, 43, 81, 90 }, /* MCS 2 */ { 52, 58, 108, 120 }, /* MCS 3 */ { 78, 87, 162, 180 }, /* MCS 4 */ { 104, 116, 216, 240 }, /* MCS 5 */ { 117, 130, 243, 270 }, /* MCS 6 */ { 130, 144, 270, 300 }, /* MCS 7 */ { 26, 29, 54, 60 }, /* MCS 8 */ { 52, 58, 108, 120 }, /* MCS 9 */ { 78, 87, 162, 180 }, /* MCS 10 */ { 104, 116, 216, 240 }, /* MCS 11 */ { 156, 173, 324, 360 }, /* MCS 12 */ { 208, 231, 432, 480 }, /* MCS 13 */ { 234, 260, 486, 540 }, /* MCS 14 */ { 260, 289, 540, 600 }, /* MCS 15 */ { 39, 43, 81, 90 }, /* MCS 16 */ { 78, 87, 162, 180 }, /* MCS 17 */ { 117, 130, 243, 270 }, /* MCS 18 */ { 156, 173, 324, 360 }, /* MCS 19 */ { 234, 260, 486, 540 }, /* MCS 20 */ { 312, 347, 648, 720 }, /* MCS 21 */ { 351, 390, 729, 810 }, /* MCS 22 */ { 390, 433, 810, 900 }, /* MCS 23 */ { 52, 58, 108, 120 }, /* MCS 24 */ { 104, 116, 216, 240 }, /* MCS 25 */ { 156, 173, 324, 360 }, /* MCS 26 */ { 208, 231, 432, 480 }, /* MCS 27 */ { 312, 347, 648, 720 }, /* MCS 28 */ { 416, 462, 864, 960 }, /* MCS 29 */ { 468, 520, 972, 1080 }, /* MCS 30 */ { 520, 578, 1080, 1200 }, /* MCS 31 */ { 0, 0, 12, 13 }, /* MCS 32 */ { 78, 87, 162, 180 }, /* MCS 33 */ { 104, 116, 216, 240 }, /* MCS 34 */ { 130, 144, 270, 300 }, /* MCS 35 */ { 117, 130, 243, 270 }, /* MCS 36 */ { 156, 173, 324, 360 }, /* MCS 37 */ { 195, 217, 405, 450 }, /* MCS 38 */ { 104, 116, 216, 240 }, /* MCS 39 */ { 130, 144, 270, 300 }, /* MCS 40 */ { 130, 144, 270, 300 }, /* MCS 41 */ { 156, 173, 324, 360 }, /* MCS 42 */ { 182, 202, 378, 420 }, /* MCS 43 */ { 182, 202, 378, 420 }, /* MCS 44 */ { 208, 231, 432, 480 }, /* MCS 45 */ { 156, 173, 324, 360 }, /* MCS 46 */ { 195, 217, 405, 450 }, /* MCS 47 */ { 195, 217, 405, 450 }, /* MCS 48 */ { 234, 260, 486, 540 }, /* MCS 49 */ { 273, 303, 567, 630 }, /* MCS 50 */ { 273, 303, 567, 630 }, /* MCS 51 */ { 312, 347, 648, 720 }, /* MCS 52 */ { 130, 144, 270, 300 }, /* MCS 53 */ { 156, 173, 324, 360 }, /* MCS 54 */ { 182, 202, 378, 420 }, /* MCS 55 */ { 156, 173, 324, 360 }, /* MCS 56 */ { 182, 202, 378, 420 }, /* MCS 57 */ { 208, 231, 432, 480 }, /* MCS 58 */ { 234, 260, 486, 540 }, /* MCS 59 */ { 208, 231, 432, 480 }, /* MCS 60 */ { 234, 260, 486, 540 }, /* MCS 61 */ { 260, 289, 540, 600 }, /* MCS 62 */ { 260, 289, 540, 600 }, /* MCS 63 */ { 286, 318, 594, 660 }, /* MCS 64 */ { 195, 217, 405, 450 }, /* MCS 65 */ { 234, 260, 486, 540 }, /* MCS 66 */ { 273, 303, 567, 630 }, /* MCS 67 */ { 234, 260, 486, 540 }, /* MCS 68 */ { 273, 303, 567, 630 }, /* MCS 69 */ { 312, 347, 648, 720 }, /* MCS 70 */ { 351, 390, 729, 810 }, /* MCS 71 */ { 312, 347, 648, 720 }, /* MCS 72 */ { 351, 390, 729, 810 }, /* MCS 73 */ { 390, 433, 810, 900 }, /* MCS 74 */ { 390, 433, 810, 900 }, /* MCS 75 */ { 429, 477, 891, 990 }, /* MCS 76 */ }; static int ieee80211_ampdu_age = -1; /* threshold for ampdu reorder q (ms) */ SYSCTL_PROC(_net_wlan, OID_AUTO, ampdu_age, CTLTYPE_INT | CTLFLAG_RW, &ieee80211_ampdu_age, 0, ieee80211_sysctl_msecs_ticks, "I", "AMPDU max reorder age (ms)"); static int ieee80211_recv_bar_ena = 1; SYSCTL_INT(_net_wlan, OID_AUTO, recv_bar, CTLFLAG_RW, &ieee80211_recv_bar_ena, 0, "BAR frame processing (ena/dis)"); static int ieee80211_addba_timeout = -1;/* timeout for ADDBA response */ SYSCTL_PROC(_net_wlan, OID_AUTO, addba_timeout, CTLTYPE_INT | CTLFLAG_RW, &ieee80211_addba_timeout, 0, ieee80211_sysctl_msecs_ticks, "I", "ADDBA request timeout (ms)"); static int ieee80211_addba_backoff = -1;/* backoff after max ADDBA requests */ SYSCTL_PROC(_net_wlan, OID_AUTO, addba_backoff, CTLTYPE_INT | CTLFLAG_RW, &ieee80211_addba_backoff, 0, ieee80211_sysctl_msecs_ticks, "I", "ADDBA request backoff (ms)"); static int ieee80211_addba_maxtries = 3;/* max ADDBA requests before backoff */ SYSCTL_INT(_net_wlan, OID_AUTO, addba_maxtries, CTLFLAG_RW, &ieee80211_addba_maxtries, 0, "max ADDBA requests sent before backoff"); static int ieee80211_bar_timeout = -1; /* timeout waiting for BAR response */ static int ieee80211_bar_maxtries = 50;/* max BAR requests before DELBA */ static ieee80211_recv_action_func ht_recv_action_ba_addba_request; static ieee80211_recv_action_func ht_recv_action_ba_addba_response; static ieee80211_recv_action_func ht_recv_action_ba_delba; static ieee80211_recv_action_func ht_recv_action_ht_mimopwrsave; static ieee80211_recv_action_func ht_recv_action_ht_txchwidth; static ieee80211_send_action_func ht_send_action_ba_addba; static ieee80211_send_action_func ht_send_action_ba_delba; static ieee80211_send_action_func ht_send_action_ht_txchwidth; static void ieee80211_ht_init(void) { /* * Setup HT parameters that depends on the clock frequency. */ ieee80211_ampdu_age = msecs_to_ticks(500); ieee80211_addba_timeout = msecs_to_ticks(250); ieee80211_addba_backoff = msecs_to_ticks(10*1000); ieee80211_bar_timeout = msecs_to_ticks(250); /* * Register action frame handlers. */ ieee80211_recv_action_register(IEEE80211_ACTION_CAT_BA, IEEE80211_ACTION_BA_ADDBA_REQUEST, ht_recv_action_ba_addba_request); ieee80211_recv_action_register(IEEE80211_ACTION_CAT_BA, IEEE80211_ACTION_BA_ADDBA_RESPONSE, ht_recv_action_ba_addba_response); ieee80211_recv_action_register(IEEE80211_ACTION_CAT_BA, IEEE80211_ACTION_BA_DELBA, ht_recv_action_ba_delba); ieee80211_recv_action_register(IEEE80211_ACTION_CAT_HT, IEEE80211_ACTION_HT_MIMOPWRSAVE, ht_recv_action_ht_mimopwrsave); ieee80211_recv_action_register(IEEE80211_ACTION_CAT_HT, IEEE80211_ACTION_HT_TXCHWIDTH, ht_recv_action_ht_txchwidth); ieee80211_send_action_register(IEEE80211_ACTION_CAT_BA, IEEE80211_ACTION_BA_ADDBA_REQUEST, ht_send_action_ba_addba); ieee80211_send_action_register(IEEE80211_ACTION_CAT_BA, IEEE80211_ACTION_BA_ADDBA_RESPONSE, ht_send_action_ba_addba); ieee80211_send_action_register(IEEE80211_ACTION_CAT_BA, IEEE80211_ACTION_BA_DELBA, ht_send_action_ba_delba); ieee80211_send_action_register(IEEE80211_ACTION_CAT_HT, IEEE80211_ACTION_HT_TXCHWIDTH, ht_send_action_ht_txchwidth); } SYSINIT(wlan_ht, SI_SUB_DRIVERS, SI_ORDER_FIRST, ieee80211_ht_init, NULL); static int ieee80211_ampdu_enable(struct ieee80211_node *ni, struct ieee80211_tx_ampdu *tap); static int ieee80211_addba_request(struct ieee80211_node *ni, struct ieee80211_tx_ampdu *tap, int dialogtoken, int baparamset, int batimeout); static int ieee80211_addba_response(struct ieee80211_node *ni, struct ieee80211_tx_ampdu *tap, int code, int baparamset, int batimeout); static void ieee80211_addba_stop(struct ieee80211_node *ni, struct ieee80211_tx_ampdu *tap); static void null_addba_response_timeout(struct ieee80211_node *ni, struct ieee80211_tx_ampdu *tap); static void ieee80211_bar_response(struct ieee80211_node *ni, struct ieee80211_tx_ampdu *tap, int status); static void ampdu_tx_stop(struct ieee80211_tx_ampdu *tap); static void bar_stop_timer(struct ieee80211_tx_ampdu *tap); static int ampdu_rx_start(struct ieee80211_node *, struct ieee80211_rx_ampdu *, int baparamset, int batimeout, int baseqctl); static void ampdu_rx_stop(struct ieee80211_node *, struct ieee80211_rx_ampdu *); void ieee80211_ht_attach(struct ieee80211com *ic) { /* setup default aggregation policy */ ic->ic_recv_action = ieee80211_recv_action; ic->ic_send_action = ieee80211_send_action; ic->ic_ampdu_enable = ieee80211_ampdu_enable; ic->ic_addba_request = ieee80211_addba_request; ic->ic_addba_response = ieee80211_addba_response; ic->ic_addba_response_timeout = null_addba_response_timeout; ic->ic_addba_stop = ieee80211_addba_stop; ic->ic_bar_response = ieee80211_bar_response; ic->ic_ampdu_rx_start = ampdu_rx_start; ic->ic_ampdu_rx_stop = ampdu_rx_stop; ic->ic_htprotmode = IEEE80211_PROT_RTSCTS; ic->ic_curhtprotmode = IEEE80211_HTINFO_OPMODE_PURE; } void ieee80211_ht_detach(struct ieee80211com *ic) { } void ieee80211_ht_vattach(struct ieee80211vap *vap) { /* driver can override defaults */ vap->iv_ampdu_rxmax = IEEE80211_HTCAP_MAXRXAMPDU_8K; vap->iv_ampdu_density = IEEE80211_HTCAP_MPDUDENSITY_NA; vap->iv_ampdu_limit = vap->iv_ampdu_rxmax; vap->iv_amsdu_limit = vap->iv_htcaps & IEEE80211_HTCAP_MAXAMSDU; /* tx aggregation traffic thresholds */ vap->iv_ampdu_mintraffic[WME_AC_BK] = 128; vap->iv_ampdu_mintraffic[WME_AC_BE] = 64; vap->iv_ampdu_mintraffic[WME_AC_VO] = 32; vap->iv_ampdu_mintraffic[WME_AC_VI] = 32; if (vap->iv_htcaps & IEEE80211_HTC_HT) { /* * Device is HT capable; enable all HT-related * facilities by default. * XXX these choices may be too aggressive. */ vap->iv_flags_ht |= IEEE80211_FHT_HT | IEEE80211_FHT_HTCOMPAT ; if (vap->iv_htcaps & IEEE80211_HTCAP_SHORTGI20) vap->iv_flags_ht |= IEEE80211_FHT_SHORTGI20; /* XXX infer from channel list? */ if (vap->iv_htcaps & IEEE80211_HTCAP_CHWIDTH40) { vap->iv_flags_ht |= IEEE80211_FHT_USEHT40; if (vap->iv_htcaps & IEEE80211_HTCAP_SHORTGI40) vap->iv_flags_ht |= IEEE80211_FHT_SHORTGI40; } /* enable RIFS if capable */ if (vap->iv_htcaps & IEEE80211_HTC_RIFS) vap->iv_flags_ht |= IEEE80211_FHT_RIFS; /* NB: A-MPDU and A-MSDU rx are mandated, these are tx only */ vap->iv_flags_ht |= IEEE80211_FHT_AMPDU_RX; if (vap->iv_htcaps & IEEE80211_HTC_AMPDU) vap->iv_flags_ht |= IEEE80211_FHT_AMPDU_TX; vap->iv_flags_ht |= IEEE80211_FHT_AMSDU_RX; if (vap->iv_htcaps & IEEE80211_HTC_AMSDU) vap->iv_flags_ht |= IEEE80211_FHT_AMSDU_TX; if (vap->iv_htcaps & IEEE80211_HTCAP_TXSTBC) vap->iv_flags_ht |= IEEE80211_FHT_STBC_TX; if (vap->iv_htcaps & IEEE80211_HTCAP_RXSTBC) vap->iv_flags_ht |= IEEE80211_FHT_STBC_RX; } /* NB: disable default legacy WDS, too many issues right now */ if (vap->iv_flags_ext & IEEE80211_FEXT_WDSLEGACY) vap->iv_flags_ht &= ~IEEE80211_FHT_HT; } void ieee80211_ht_vdetach(struct ieee80211vap *vap) { } static int ht_getrate(struct ieee80211com *ic, int index, enum ieee80211_phymode mode, int ratetype) { int mword, rate; mword = ieee80211_rate2media(ic, index | IEEE80211_RATE_MCS, mode); if (IFM_SUBTYPE(mword) != IFM_IEEE80211_MCS) return (0); switch (ratetype) { case 0: rate = ieee80211_htrates[index].ht20_rate_800ns; break; case 1: rate = ieee80211_htrates[index].ht20_rate_400ns; break; case 2: rate = ieee80211_htrates[index].ht40_rate_800ns; break; default: rate = ieee80211_htrates[index].ht40_rate_400ns; break; } return (rate); } static struct printranges { int minmcs; int maxmcs; int txstream; int ratetype; int htcapflags; } ranges[] = { { 0, 7, 1, 0, 0 }, { 8, 15, 2, 0, 0 }, { 16, 23, 3, 0, 0 }, { 24, 31, 4, 0, 0 }, { 32, 0, 1, 2, IEEE80211_HTC_TXMCS32 }, { 33, 38, 2, 0, IEEE80211_HTC_TXUNEQUAL }, { 39, 52, 3, 0, IEEE80211_HTC_TXUNEQUAL }, { 53, 76, 4, 0, IEEE80211_HTC_TXUNEQUAL }, { 0, 0, 0, 0, 0 }, }; static void ht_rateprint(struct ieee80211com *ic, enum ieee80211_phymode mode, int ratetype) { int minrate, maxrate; struct printranges *range; for (range = ranges; range->txstream != 0; range++) { if (ic->ic_txstream < range->txstream) continue; if (range->htcapflags && (ic->ic_htcaps & range->htcapflags) == 0) continue; if (ratetype < range->ratetype) continue; minrate = ht_getrate(ic, range->minmcs, mode, ratetype); maxrate = ht_getrate(ic, range->maxmcs, mode, ratetype); if (range->maxmcs) { ic_printf(ic, "MCS %d-%d: %d%sMbps - %d%sMbps\n", range->minmcs, range->maxmcs, minrate/2, ((minrate & 0x1) != 0 ? ".5" : ""), maxrate/2, ((maxrate & 0x1) != 0 ? ".5" : "")); } else { ic_printf(ic, "MCS %d: %d%sMbps\n", range->minmcs, minrate/2, ((minrate & 0x1) != 0 ? ".5" : "")); } } } static void ht_announce(struct ieee80211com *ic, enum ieee80211_phymode mode) { const char *modestr = ieee80211_phymode_name[mode]; ic_printf(ic, "%s MCS 20MHz\n", modestr); ht_rateprint(ic, mode, 0); if (ic->ic_htcaps & IEEE80211_HTCAP_SHORTGI20) { ic_printf(ic, "%s MCS 20MHz SGI\n", modestr); ht_rateprint(ic, mode, 1); } if (ic->ic_htcaps & IEEE80211_HTCAP_CHWIDTH40) { ic_printf(ic, "%s MCS 40MHz:\n", modestr); ht_rateprint(ic, mode, 2); } if ((ic->ic_htcaps & IEEE80211_HTCAP_CHWIDTH40) && (ic->ic_htcaps & IEEE80211_HTCAP_SHORTGI40)) { ic_printf(ic, "%s MCS 40MHz SGI:\n", modestr); ht_rateprint(ic, mode, 3); } } void ieee80211_ht_announce(struct ieee80211com *ic) { if (isset(ic->ic_modecaps, IEEE80211_MODE_11NA) || isset(ic->ic_modecaps, IEEE80211_MODE_11NG)) ic_printf(ic, "%dT%dR\n", ic->ic_txstream, ic->ic_rxstream); if (isset(ic->ic_modecaps, IEEE80211_MODE_11NA)) ht_announce(ic, IEEE80211_MODE_11NA); if (isset(ic->ic_modecaps, IEEE80211_MODE_11NG)) ht_announce(ic, IEEE80211_MODE_11NG); } static struct ieee80211_htrateset htrateset; const struct ieee80211_htrateset * ieee80211_get_suphtrates(struct ieee80211com *ic, const struct ieee80211_channel *c) { #define ADDRATE(x) do { \ htrateset.rs_rates[htrateset.rs_nrates] = x; \ htrateset.rs_nrates++; \ } while (0) int i; memset(&htrateset, 0, sizeof(struct ieee80211_htrateset)); for (i = 0; i < ic->ic_txstream * 8; i++) ADDRATE(i); if ((ic->ic_htcaps & IEEE80211_HTCAP_CHWIDTH40) && (ic->ic_htcaps & IEEE80211_HTC_TXMCS32)) ADDRATE(32); if (ic->ic_htcaps & IEEE80211_HTC_TXUNEQUAL) { if (ic->ic_txstream >= 2) { for (i = 33; i <= 38; i++) ADDRATE(i); } if (ic->ic_txstream >= 3) { for (i = 39; i <= 52; i++) ADDRATE(i); } if (ic->ic_txstream == 4) { for (i = 53; i <= 76; i++) ADDRATE(i); } } return &htrateset; #undef ADDRATE } /* * Receive processing. */ /* * Decap the encapsulated A-MSDU frames and dispatch all but * the last for delivery. The last frame is returned for * delivery via the normal path. */ struct mbuf * ieee80211_decap_amsdu(struct ieee80211_node *ni, struct mbuf *m) { struct ieee80211vap *vap = ni->ni_vap; int framelen; struct mbuf *n; /* discard 802.3 header inserted by ieee80211_decap */ m_adj(m, sizeof(struct ether_header)); vap->iv_stats.is_amsdu_decap++; for (;;) { /* * Decap the first frame, bust it apart from the * remainder and deliver. We leave the last frame * delivery to the caller (for consistency with other * code paths, could also do it here). */ m = ieee80211_decap1(m, &framelen); if (m == NULL) { IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_ANY, ni->ni_macaddr, "a-msdu", "%s", "decap failed"); vap->iv_stats.is_amsdu_tooshort++; return NULL; } if (m->m_pkthdr.len == framelen) break; n = m_split(m, framelen, M_NOWAIT); if (n == NULL) { IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_ANY, ni->ni_macaddr, "a-msdu", "%s", "unable to split encapsulated frames"); vap->iv_stats.is_amsdu_split++; m_freem(m); /* NB: must reclaim */ return NULL; } vap->iv_deliver_data(vap, ni, m); /* * Remove frame contents; each intermediate frame * is required to be aligned to a 4-byte boundary. */ m = n; m_adj(m, roundup2(framelen, 4) - framelen); /* padding */ } return m; /* last delivered by caller */ } /* * Purge all frames in the A-MPDU re-order queue. */ static void ampdu_rx_purge(struct ieee80211_rx_ampdu *rap) { struct mbuf *m; int i; for (i = 0; i < rap->rxa_wnd; i++) { m = rap->rxa_m[i]; if (m != NULL) { rap->rxa_m[i] = NULL; rap->rxa_qbytes -= m->m_pkthdr.len; m_freem(m); if (--rap->rxa_qframes == 0) break; } } KASSERT(rap->rxa_qbytes == 0 && rap->rxa_qframes == 0, ("lost %u data, %u frames on ampdu rx q", rap->rxa_qbytes, rap->rxa_qframes)); } /* * Start A-MPDU rx/re-order processing for the specified TID. */ static int ampdu_rx_start(struct ieee80211_node *ni, struct ieee80211_rx_ampdu *rap, int baparamset, int batimeout, int baseqctl) { int bufsiz = MS(baparamset, IEEE80211_BAPS_BUFSIZ); if (rap->rxa_flags & IEEE80211_AGGR_RUNNING) { /* * AMPDU previously setup and not terminated with a DELBA, * flush the reorder q's in case anything remains. */ ampdu_rx_purge(rap); } memset(rap, 0, sizeof(*rap)); rap->rxa_wnd = (bufsiz == 0) ? IEEE80211_AGGR_BAWMAX : min(bufsiz, IEEE80211_AGGR_BAWMAX); rap->rxa_start = MS(baseqctl, IEEE80211_BASEQ_START); rap->rxa_flags |= IEEE80211_AGGR_RUNNING | IEEE80211_AGGR_XCHGPEND; return 0; } /* * Public function; manually setup the RX ampdu state. */ int ieee80211_ampdu_rx_start_ext(struct ieee80211_node *ni, int tid, int seq, int baw) { struct ieee80211_rx_ampdu *rap; /* XXX TODO: sanity check tid, seq, baw */ rap = &ni->ni_rx_ampdu[tid]; if (rap->rxa_flags & IEEE80211_AGGR_RUNNING) { /* * AMPDU previously setup and not terminated with a DELBA, * flush the reorder q's in case anything remains. */ ampdu_rx_purge(rap); } memset(rap, 0, sizeof(*rap)); rap->rxa_wnd = (baw== 0) ? IEEE80211_AGGR_BAWMAX : min(baw, IEEE80211_AGGR_BAWMAX); rap->rxa_start = seq; rap->rxa_flags |= IEEE80211_AGGR_RUNNING | IEEE80211_AGGR_XCHGPEND; IEEE80211_NOTE(ni->ni_vap, IEEE80211_MSG_11N, ni, "%s: tid=%d, start=%d, wnd=%d, flags=0x%08x\n", __func__, tid, seq, rap->rxa_wnd, rap->rxa_flags); return 0; } /* * Stop A-MPDU rx processing for the specified TID. */ static void ampdu_rx_stop(struct ieee80211_node *ni, struct ieee80211_rx_ampdu *rap) { ampdu_rx_purge(rap); rap->rxa_flags &= ~(IEEE80211_AGGR_RUNNING | IEEE80211_AGGR_XCHGPEND); } /* * Dispatch a frame from the A-MPDU reorder queue. The * frame is fed back into ieee80211_input marked with an * M_AMPDU_MPDU flag so it doesn't come back to us (it also * permits ieee80211_input to optimize re-processing). */ static __inline void ampdu_dispatch(struct ieee80211_node *ni, struct mbuf *m) { m->m_flags |= M_AMPDU_MPDU; /* bypass normal processing */ /* NB: rssi and noise are ignored w/ M_AMPDU_MPDU set */ (void) ieee80211_input(ni, m, 0, 0); } /* * Dispatch as many frames as possible from the re-order queue. * Frames will always be "at the front"; we process all frames * up to the first empty slot in the window. On completion we * cleanup state if there are still pending frames in the current * BA window. We assume the frame at slot 0 is already handled * by the caller; we always start at slot 1. */ static void ampdu_rx_dispatch(struct ieee80211_rx_ampdu *rap, struct ieee80211_node *ni) { struct ieee80211vap *vap = ni->ni_vap; struct mbuf *m; int i; /* flush run of frames */ for (i = 1; i < rap->rxa_wnd; i++) { m = rap->rxa_m[i]; if (m == NULL) break; rap->rxa_m[i] = NULL; rap->rxa_qbytes -= m->m_pkthdr.len; rap->rxa_qframes--; ampdu_dispatch(ni, m); } /* * If frames remain, copy the mbuf pointers down so * they correspond to the offsets in the new window. */ if (rap->rxa_qframes != 0) { int n = rap->rxa_qframes, j; for (j = i+1; j < rap->rxa_wnd; j++) { if (rap->rxa_m[j] != NULL) { rap->rxa_m[j-i] = rap->rxa_m[j]; rap->rxa_m[j] = NULL; if (--n == 0) break; } } KASSERT(n == 0, ("lost %d frames", n)); vap->iv_stats.is_ampdu_rx_copy += rap->rxa_qframes; } /* * Adjust the start of the BA window to * reflect the frames just dispatched. */ rap->rxa_start = IEEE80211_SEQ_ADD(rap->rxa_start, i); vap->iv_stats.is_ampdu_rx_oor += i; } /* * Dispatch all frames in the A-MPDU re-order queue. */ static void ampdu_rx_flush(struct ieee80211_node *ni, struct ieee80211_rx_ampdu *rap) { struct ieee80211vap *vap = ni->ni_vap; struct mbuf *m; int i; for (i = 0; i < rap->rxa_wnd; i++) { m = rap->rxa_m[i]; if (m == NULL) continue; rap->rxa_m[i] = NULL; rap->rxa_qbytes -= m->m_pkthdr.len; rap->rxa_qframes--; vap->iv_stats.is_ampdu_rx_oor++; ampdu_dispatch(ni, m); if (rap->rxa_qframes == 0) break; } } /* * Dispatch all frames in the A-MPDU re-order queue * preceding the specified sequence number. This logic * handles window moves due to a received MSDU or BAR. */ static void ampdu_rx_flush_upto(struct ieee80211_node *ni, struct ieee80211_rx_ampdu *rap, ieee80211_seq winstart) { struct ieee80211vap *vap = ni->ni_vap; struct mbuf *m; ieee80211_seq seqno; int i; /* * Flush any complete MSDU's with a sequence number lower * than winstart. Gaps may exist. Note that we may actually * dispatch frames past winstart if a run continues; this is * an optimization that avoids having to do a separate pass * to dispatch frames after moving the BA window start. */ seqno = rap->rxa_start; for (i = 0; i < rap->rxa_wnd; i++) { m = rap->rxa_m[i]; if (m != NULL) { rap->rxa_m[i] = NULL; rap->rxa_qbytes -= m->m_pkthdr.len; rap->rxa_qframes--; vap->iv_stats.is_ampdu_rx_oor++; ampdu_dispatch(ni, m); } else { if (!IEEE80211_SEQ_BA_BEFORE(seqno, winstart)) break; } seqno = IEEE80211_SEQ_INC(seqno); } /* * If frames remain, copy the mbuf pointers down so * they correspond to the offsets in the new window. */ if (rap->rxa_qframes != 0) { int n = rap->rxa_qframes, j; /* NB: this loop assumes i > 0 and/or rxa_m[0] is NULL */ KASSERT(rap->rxa_m[0] == NULL, ("%s: BA window slot 0 occupied", __func__)); for (j = i+1; j < rap->rxa_wnd; j++) { if (rap->rxa_m[j] != NULL) { rap->rxa_m[j-i] = rap->rxa_m[j]; rap->rxa_m[j] = NULL; if (--n == 0) break; } } KASSERT(n == 0, ("%s: lost %d frames, qframes %d off %d " "BA win <%d:%d> winstart %d", __func__, n, rap->rxa_qframes, i, rap->rxa_start, IEEE80211_SEQ_ADD(rap->rxa_start, rap->rxa_wnd-1), winstart)); vap->iv_stats.is_ampdu_rx_copy += rap->rxa_qframes; } /* * Move the start of the BA window; we use the * sequence number of the last MSDU that was * passed up the stack+1 or winstart if stopped on * a gap in the reorder buffer. */ rap->rxa_start = seqno; } /* * Process a received QoS data frame for an HT station. Handle * A-MPDU reordering: if this frame is received out of order * and falls within the BA window hold onto it. Otherwise if * this frame completes a run, flush any pending frames. We * return 1 if the frame is consumed. A 0 is returned if * the frame should be processed normally by the caller. */ int ieee80211_ampdu_reorder(struct ieee80211_node *ni, struct mbuf *m) { #define IEEE80211_FC0_QOSDATA \ (IEEE80211_FC0_TYPE_DATA|IEEE80211_FC0_SUBTYPE_QOS|IEEE80211_FC0_VERSION_0) #define PROCESS 0 /* caller should process frame */ #define CONSUMED 1 /* frame consumed, caller does nothing */ struct ieee80211vap *vap = ni->ni_vap; struct ieee80211_qosframe *wh; struct ieee80211_rx_ampdu *rap; ieee80211_seq rxseq; uint8_t tid; int off; KASSERT((m->m_flags & (M_AMPDU | M_AMPDU_MPDU)) == M_AMPDU, ("!a-mpdu or already re-ordered, flags 0x%x", m->m_flags)); KASSERT(ni->ni_flags & IEEE80211_NODE_HT, ("not an HT sta")); /* NB: m_len known to be sufficient */ wh = mtod(m, struct ieee80211_qosframe *); if (wh->i_fc[0] != IEEE80211_FC0_QOSDATA) { /* * Not QoS data, shouldn't get here but just * return it to the caller for processing. */ return PROCESS; } if (IEEE80211_IS_DSTODS(wh)) tid = ((struct ieee80211_qosframe_addr4 *)wh)->i_qos[0]; else tid = wh->i_qos[0]; tid &= IEEE80211_QOS_TID; rap = &ni->ni_rx_ampdu[tid]; if ((rap->rxa_flags & IEEE80211_AGGR_XCHGPEND) == 0) { /* * No ADDBA request yet, don't touch. */ return PROCESS; } rxseq = le16toh(*(uint16_t *)wh->i_seq); if ((rxseq & IEEE80211_SEQ_FRAG_MASK) != 0) { /* * Fragments are not allowed; toss. */ IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_INPUT | IEEE80211_MSG_11N, ni->ni_macaddr, "A-MPDU", "fragment, rxseq 0x%x tid %u%s", rxseq, tid, wh->i_fc[1] & IEEE80211_FC1_RETRY ? " (retransmit)" : ""); vap->iv_stats.is_ampdu_rx_drop++; IEEE80211_NODE_STAT(ni, rx_drop); m_freem(m); return CONSUMED; } rxseq >>= IEEE80211_SEQ_SEQ_SHIFT; rap->rxa_nframes++; again: if (rxseq == rap->rxa_start) { /* * First frame in window. */ if (rap->rxa_qframes != 0) { /* * Dispatch as many packets as we can. */ KASSERT(rap->rxa_m[0] == NULL, ("unexpected dup")); ampdu_dispatch(ni, m); ampdu_rx_dispatch(rap, ni); return CONSUMED; } else { /* * In order; advance window and notify * caller to dispatch directly. */ rap->rxa_start = IEEE80211_SEQ_INC(rxseq); return PROCESS; } } /* * Frame is out of order; store if in the BA window. */ /* calculate offset in BA window */ off = IEEE80211_SEQ_SUB(rxseq, rap->rxa_start); if (off < rap->rxa_wnd) { /* * Common case (hopefully): in the BA window. * Sec 9.10.7.6.2 a) (p.137) */ /* * Check for frames sitting too long in the reorder queue. * This should only ever happen if frames are not delivered * without the sender otherwise notifying us (e.g. with a * BAR to move the window). Typically this happens because * of vendor bugs that cause the sequence number to jump. * When this happens we get a gap in the reorder queue that * leaves frame sitting on the queue until they get pushed * out due to window moves. When the vendor does not send * BAR this move only happens due to explicit packet sends * * NB: we only track the time of the oldest frame in the * reorder q; this means that if we flush we might push * frames that still "new"; if this happens then subsequent * frames will result in BA window moves which cost something * but is still better than a big throughput dip. */ if (rap->rxa_qframes != 0) { /* XXX honor batimeout? */ if (ticks - rap->rxa_age > ieee80211_ampdu_age) { /* * Too long since we received the first * frame; flush the reorder buffer. */ if (rap->rxa_qframes != 0) { vap->iv_stats.is_ampdu_rx_age += rap->rxa_qframes; ampdu_rx_flush(ni, rap); } rap->rxa_start = IEEE80211_SEQ_INC(rxseq); return PROCESS; } } else { /* * First frame, start aging timer. */ rap->rxa_age = ticks; } /* save packet */ if (rap->rxa_m[off] == NULL) { rap->rxa_m[off] = m; rap->rxa_qframes++; rap->rxa_qbytes += m->m_pkthdr.len; vap->iv_stats.is_ampdu_rx_reorder++; } else { IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_INPUT | IEEE80211_MSG_11N, ni->ni_macaddr, "a-mpdu duplicate", "seqno %u tid %u BA win <%u:%u>", rxseq, tid, rap->rxa_start, IEEE80211_SEQ_ADD(rap->rxa_start, rap->rxa_wnd-1)); vap->iv_stats.is_rx_dup++; IEEE80211_NODE_STAT(ni, rx_dup); m_freem(m); } return CONSUMED; } if (off < IEEE80211_SEQ_BA_RANGE) { /* * Outside the BA window, but within range; * flush the reorder q and move the window. * Sec 9.10.7.6.2 b) (p.138) */ IEEE80211_NOTE(vap, IEEE80211_MSG_11N, ni, "move BA win <%u:%u> (%u frames) rxseq %u tid %u", rap->rxa_start, IEEE80211_SEQ_ADD(rap->rxa_start, rap->rxa_wnd-1), rap->rxa_qframes, rxseq, tid); vap->iv_stats.is_ampdu_rx_move++; /* * The spec says to flush frames up to but not including: * WinStart_B = rxseq - rap->rxa_wnd + 1 * Then insert the frame or notify the caller to process * it immediately. We can safely do this by just starting * over again because we know the frame will now be within * the BA window. */ /* NB: rxa_wnd known to be >0 */ ampdu_rx_flush_upto(ni, rap, IEEE80211_SEQ_SUB(rxseq, rap->rxa_wnd-1)); goto again; } else { /* * Outside the BA window and out of range; toss. * Sec 9.10.7.6.2 c) (p.138) */ IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_INPUT | IEEE80211_MSG_11N, ni->ni_macaddr, "MPDU", "BA win <%u:%u> (%u frames) rxseq %u tid %u%s", rap->rxa_start, IEEE80211_SEQ_ADD(rap->rxa_start, rap->rxa_wnd-1), rap->rxa_qframes, rxseq, tid, wh->i_fc[1] & IEEE80211_FC1_RETRY ? " (retransmit)" : ""); vap->iv_stats.is_ampdu_rx_drop++; IEEE80211_NODE_STAT(ni, rx_drop); m_freem(m); return CONSUMED; } #undef CONSUMED #undef PROCESS #undef IEEE80211_FC0_QOSDATA } /* * Process a BAR ctl frame. Dispatch all frames up to * the sequence number of the frame. If this frame is * out of range it's discarded. */ void ieee80211_recv_bar(struct ieee80211_node *ni, struct mbuf *m0) { struct ieee80211vap *vap = ni->ni_vap; struct ieee80211_frame_bar *wh; struct ieee80211_rx_ampdu *rap; ieee80211_seq rxseq; int tid, off; if (!ieee80211_recv_bar_ena) { #if 0 IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_11N, ni->ni_macaddr, "BAR", "%s", "processing disabled"); #endif vap->iv_stats.is_ampdu_bar_bad++; return; } wh = mtod(m0, struct ieee80211_frame_bar *); /* XXX check basic BAR */ tid = MS(le16toh(wh->i_ctl), IEEE80211_BAR_TID); rap = &ni->ni_rx_ampdu[tid]; if ((rap->rxa_flags & IEEE80211_AGGR_XCHGPEND) == 0) { /* * No ADDBA request yet, don't touch. */ IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_INPUT | IEEE80211_MSG_11N, ni->ni_macaddr, "BAR", "no BA stream, tid %u", tid); vap->iv_stats.is_ampdu_bar_bad++; return; } vap->iv_stats.is_ampdu_bar_rx++; rxseq = le16toh(wh->i_seq) >> IEEE80211_SEQ_SEQ_SHIFT; if (rxseq == rap->rxa_start) return; /* calculate offset in BA window */ off = IEEE80211_SEQ_SUB(rxseq, rap->rxa_start); if (off < IEEE80211_SEQ_BA_RANGE) { /* * Flush the reorder q up to rxseq and move the window. * Sec 9.10.7.6.3 a) (p.138) */ IEEE80211_NOTE(vap, IEEE80211_MSG_11N, ni, "BAR moves BA win <%u:%u> (%u frames) rxseq %u tid %u", rap->rxa_start, IEEE80211_SEQ_ADD(rap->rxa_start, rap->rxa_wnd-1), rap->rxa_qframes, rxseq, tid); vap->iv_stats.is_ampdu_bar_move++; ampdu_rx_flush_upto(ni, rap, rxseq); if (off >= rap->rxa_wnd) { /* * BAR specifies a window start to the right of BA * window; we must move it explicitly since * ampdu_rx_flush_upto will not. */ rap->rxa_start = rxseq; } } else { /* * Out of range; toss. * Sec 9.10.7.6.3 b) (p.138) */ IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_INPUT | IEEE80211_MSG_11N, ni->ni_macaddr, "BAR", "BA win <%u:%u> (%u frames) rxseq %u tid %u%s", rap->rxa_start, IEEE80211_SEQ_ADD(rap->rxa_start, rap->rxa_wnd-1), rap->rxa_qframes, rxseq, tid, wh->i_fc[1] & IEEE80211_FC1_RETRY ? " (retransmit)" : ""); vap->iv_stats.is_ampdu_bar_oow++; IEEE80211_NODE_STAT(ni, rx_drop); } } /* * Setup HT-specific state in a node. Called only * when HT use is negotiated so we don't do extra * work for temporary and/or legacy sta's. */ void ieee80211_ht_node_init(struct ieee80211_node *ni) { struct ieee80211_tx_ampdu *tap; int tid; IEEE80211_NOTE(ni->ni_vap, IEEE80211_MSG_11N, ni, "%s: called (%p)", __func__, ni); if (ni->ni_flags & IEEE80211_NODE_HT) { /* * Clean AMPDU state on re-associate. This handles the case * where a station leaves w/o notifying us and then returns * before node is reaped for inactivity. */ IEEE80211_NOTE(ni->ni_vap, IEEE80211_MSG_11N, ni, "%s: calling cleanup (%p)", __func__, ni); ieee80211_ht_node_cleanup(ni); } for (tid = 0; tid < WME_NUM_TID; tid++) { tap = &ni->ni_tx_ampdu[tid]; tap->txa_tid = tid; tap->txa_ni = ni; ieee80211_txampdu_init_pps(tap); /* NB: further initialization deferred */ } ni->ni_flags |= IEEE80211_NODE_HT | IEEE80211_NODE_AMPDU; } /* * Cleanup HT-specific state in a node. Called only * when HT use has been marked. */ void ieee80211_ht_node_cleanup(struct ieee80211_node *ni) { struct ieee80211com *ic = ni->ni_ic; int i; IEEE80211_NOTE(ni->ni_vap, IEEE80211_MSG_11N, ni, "%s: called (%p)", __func__, ni); KASSERT(ni->ni_flags & IEEE80211_NODE_HT, ("not an HT node")); /* XXX optimize this */ for (i = 0; i < WME_NUM_TID; i++) { struct ieee80211_tx_ampdu *tap = &ni->ni_tx_ampdu[i]; if (tap->txa_flags & IEEE80211_AGGR_SETUP) ampdu_tx_stop(tap); } for (i = 0; i < WME_NUM_TID; i++) ic->ic_ampdu_rx_stop(ni, &ni->ni_rx_ampdu[i]); ni->ni_htcap = 0; ni->ni_flags &= ~IEEE80211_NODE_HT_ALL; } /* * Age out HT resources for a station. */ void ieee80211_ht_node_age(struct ieee80211_node *ni) { struct ieee80211vap *vap = ni->ni_vap; uint8_t tid; KASSERT(ni->ni_flags & IEEE80211_NODE_HT, ("not an HT sta")); for (tid = 0; tid < WME_NUM_TID; tid++) { struct ieee80211_rx_ampdu *rap; rap = &ni->ni_rx_ampdu[tid]; if ((rap->rxa_flags & IEEE80211_AGGR_XCHGPEND) == 0) continue; if (rap->rxa_qframes == 0) continue; /* * Check for frames sitting too long in the reorder queue. * See above for more details on what's happening here. */ /* XXX honor batimeout? */ if (ticks - rap->rxa_age > ieee80211_ampdu_age) { /* * Too long since we received the first * frame; flush the reorder buffer. */ vap->iv_stats.is_ampdu_rx_age += rap->rxa_qframes; ampdu_rx_flush(ni, rap); } } } static struct ieee80211_channel * findhtchan(struct ieee80211com *ic, struct ieee80211_channel *c, int htflags) { return ieee80211_find_channel(ic, c->ic_freq, (c->ic_flags &~ IEEE80211_CHAN_HT) | htflags); } /* * Adjust a channel to be HT/non-HT according to the vap's configuration. */ struct ieee80211_channel * ieee80211_ht_adjust_channel(struct ieee80211com *ic, struct ieee80211_channel *chan, int flags) { struct ieee80211_channel *c; if (flags & IEEE80211_FHT_HT) { /* promote to HT if possible */ if (flags & IEEE80211_FHT_USEHT40) { if (!IEEE80211_IS_CHAN_HT40(chan)) { /* NB: arbitrarily pick ht40+ over ht40- */ c = findhtchan(ic, chan, IEEE80211_CHAN_HT40U); if (c == NULL) c = findhtchan(ic, chan, IEEE80211_CHAN_HT40D); if (c == NULL) c = findhtchan(ic, chan, IEEE80211_CHAN_HT20); if (c != NULL) chan = c; } } else if (!IEEE80211_IS_CHAN_HT20(chan)) { c = findhtchan(ic, chan, IEEE80211_CHAN_HT20); if (c != NULL) chan = c; } } else if (IEEE80211_IS_CHAN_HT(chan)) { /* demote to legacy, HT use is disabled */ c = ieee80211_find_channel(ic, chan->ic_freq, chan->ic_flags &~ IEEE80211_CHAN_HT); if (c != NULL) chan = c; } return chan; } /* * Setup HT-specific state for a legacy WDS peer. */ void ieee80211_ht_wds_init(struct ieee80211_node *ni) { struct ieee80211vap *vap = ni->ni_vap; struct ieee80211_tx_ampdu *tap; int tid; KASSERT(vap->iv_flags_ht & IEEE80211_FHT_HT, ("no HT requested")); /* XXX check scan cache in case peer has an ap and we have info */ /* * If setup with a legacy channel; locate an HT channel. * Otherwise if the inherited channel (from a companion * AP) is suitable use it so we use the same location * for the extension channel). */ ni->ni_chan = ieee80211_ht_adjust_channel(ni->ni_ic, ni->ni_chan, ieee80211_htchanflags(ni->ni_chan)); ni->ni_htcap = 0; if (vap->iv_flags_ht & IEEE80211_FHT_SHORTGI20) ni->ni_htcap |= IEEE80211_HTCAP_SHORTGI20; if (IEEE80211_IS_CHAN_HT40(ni->ni_chan)) { ni->ni_htcap |= IEEE80211_HTCAP_CHWIDTH40; ni->ni_chw = 40; if (IEEE80211_IS_CHAN_HT40U(ni->ni_chan)) ni->ni_ht2ndchan = IEEE80211_HTINFO_2NDCHAN_ABOVE; else if (IEEE80211_IS_CHAN_HT40D(ni->ni_chan)) ni->ni_ht2ndchan = IEEE80211_HTINFO_2NDCHAN_BELOW; if (vap->iv_flags_ht & IEEE80211_FHT_SHORTGI40) ni->ni_htcap |= IEEE80211_HTCAP_SHORTGI40; } else { ni->ni_chw = 20; ni->ni_ht2ndchan = IEEE80211_HTINFO_2NDCHAN_NONE; } ni->ni_htctlchan = ni->ni_chan->ic_ieee; if (vap->iv_flags_ht & IEEE80211_FHT_RIFS) ni->ni_flags |= IEEE80211_NODE_RIFS; /* XXX does it make sense to enable SMPS? */ ni->ni_htopmode = 0; /* XXX need protection state */ ni->ni_htstbc = 0; /* XXX need info */ for (tid = 0; tid < WME_NUM_TID; tid++) { tap = &ni->ni_tx_ampdu[tid]; tap->txa_tid = tid; ieee80211_txampdu_init_pps(tap); } /* NB: AMPDU tx/rx governed by IEEE80211_FHT_AMPDU_{TX,RX} */ ni->ni_flags |= IEEE80211_NODE_HT | IEEE80211_NODE_AMPDU; } /* * Notify hostap vaps of a change in the HTINFO ie. */ static void htinfo_notify(struct ieee80211com *ic) { struct ieee80211vap *vap; int first = 1; IEEE80211_LOCK_ASSERT(ic); TAILQ_FOREACH(vap, &ic->ic_vaps, iv_next) { if (vap->iv_opmode != IEEE80211_M_HOSTAP) continue; if (vap->iv_state != IEEE80211_S_RUN || !IEEE80211_IS_CHAN_HT(vap->iv_bss->ni_chan)) continue; if (first) { IEEE80211_NOTE(vap, IEEE80211_MSG_ASSOC | IEEE80211_MSG_11N, vap->iv_bss, "HT bss occupancy change: %d sta, %d ht, " "%d ht40%s, HT protmode now 0x%x" , ic->ic_sta_assoc , ic->ic_ht_sta_assoc , ic->ic_ht40_sta_assoc , (ic->ic_flags_ht & IEEE80211_FHT_NONHT_PR) ? ", non-HT sta present" : "" , ic->ic_curhtprotmode); first = 0; } ieee80211_beacon_notify(vap, IEEE80211_BEACON_HTINFO); } } /* * Calculate HT protection mode from current * state and handle updates. */ static void htinfo_update(struct ieee80211com *ic) { uint8_t protmode; if (ic->ic_sta_assoc != ic->ic_ht_sta_assoc) { protmode = IEEE80211_HTINFO_OPMODE_MIXED | IEEE80211_HTINFO_NONHT_PRESENT; } else if (ic->ic_flags_ht & IEEE80211_FHT_NONHT_PR) { protmode = IEEE80211_HTINFO_OPMODE_PROTOPT | IEEE80211_HTINFO_NONHT_PRESENT; } else if (ic->ic_bsschan != IEEE80211_CHAN_ANYC && IEEE80211_IS_CHAN_HT40(ic->ic_bsschan) && ic->ic_sta_assoc != ic->ic_ht40_sta_assoc) { protmode = IEEE80211_HTINFO_OPMODE_HT20PR; } else { protmode = IEEE80211_HTINFO_OPMODE_PURE; } if (protmode != ic->ic_curhtprotmode) { ic->ic_curhtprotmode = protmode; htinfo_notify(ic); } } /* * Handle an HT station joining a BSS. */ void ieee80211_ht_node_join(struct ieee80211_node *ni) { struct ieee80211com *ic = ni->ni_ic; IEEE80211_LOCK_ASSERT(ic); if (ni->ni_flags & IEEE80211_NODE_HT) { ic->ic_ht_sta_assoc++; if (ni->ni_chw == 40) ic->ic_ht40_sta_assoc++; } htinfo_update(ic); } /* * Handle an HT station leaving a BSS. */ void ieee80211_ht_node_leave(struct ieee80211_node *ni) { struct ieee80211com *ic = ni->ni_ic; IEEE80211_LOCK_ASSERT(ic); if (ni->ni_flags & IEEE80211_NODE_HT) { ic->ic_ht_sta_assoc--; if (ni->ni_chw == 40) ic->ic_ht40_sta_assoc--; } htinfo_update(ic); } /* * Public version of htinfo_update; used for processing * beacon frames from overlapping bss. * * Caller can specify either IEEE80211_HTINFO_OPMODE_MIXED * (on receipt of a beacon that advertises MIXED) or * IEEE80211_HTINFO_OPMODE_PROTOPT (on receipt of a beacon * from an overlapping legacy bss). We treat MIXED with * a higher precedence than PROTOPT (i.e. we will not change * change PROTOPT -> MIXED; only MIXED -> PROTOPT). This * corresponds to how we handle things in htinfo_update. */ void ieee80211_htprot_update(struct ieee80211com *ic, int protmode) { #define OPMODE(x) SM(x, IEEE80211_HTINFO_OPMODE) IEEE80211_LOCK(ic); /* track non-HT station presence */ KASSERT(protmode & IEEE80211_HTINFO_NONHT_PRESENT, ("protmode 0x%x", protmode)); ic->ic_flags_ht |= IEEE80211_FHT_NONHT_PR; ic->ic_lastnonht = ticks; if (protmode != ic->ic_curhtprotmode && (OPMODE(ic->ic_curhtprotmode) != IEEE80211_HTINFO_OPMODE_MIXED || OPMODE(protmode) == IEEE80211_HTINFO_OPMODE_PROTOPT)) { /* push beacon update */ ic->ic_curhtprotmode = protmode; htinfo_notify(ic); } IEEE80211_UNLOCK(ic); #undef OPMODE } /* * Time out presence of an overlapping bss with non-HT * stations. When operating in hostap mode we listen for * beacons from other stations and if we identify a non-HT * station is present we update the opmode field of the * HTINFO ie. To identify when all non-HT stations are * gone we time out this condition. */ void ieee80211_ht_timeout(struct ieee80211com *ic) { IEEE80211_LOCK_ASSERT(ic); if ((ic->ic_flags_ht & IEEE80211_FHT_NONHT_PR) && ieee80211_time_after(ticks, ic->ic_lastnonht + IEEE80211_NONHT_PRESENT_AGE)) { #if 0 IEEE80211_NOTE(vap, IEEE80211_MSG_11N, ni, "%s", "time out non-HT STA present on channel"); #endif ic->ic_flags_ht &= ~IEEE80211_FHT_NONHT_PR; htinfo_update(ic); } } /* * Process an 802.11n HT capabilities ie. */ void ieee80211_parse_htcap(struct ieee80211_node *ni, const uint8_t *ie) { if (ie[0] == IEEE80211_ELEMID_VENDOR) { /* * Station used Vendor OUI ie to associate; * mark the node so when we respond we'll use * the Vendor OUI's and not the standard ie's. */ ni->ni_flags |= IEEE80211_NODE_HTCOMPAT; ie += 4; } else ni->ni_flags &= ~IEEE80211_NODE_HTCOMPAT; ni->ni_htcap = le16dec(ie + __offsetof(struct ieee80211_ie_htcap, hc_cap)); ni->ni_htparam = ie[__offsetof(struct ieee80211_ie_htcap, hc_param)]; } static void htinfo_parse(struct ieee80211_node *ni, const struct ieee80211_ie_htinfo *htinfo) { uint16_t w; ni->ni_htctlchan = htinfo->hi_ctrlchannel; ni->ni_ht2ndchan = SM(htinfo->hi_byte1, IEEE80211_HTINFO_2NDCHAN); w = le16dec(&htinfo->hi_byte2); ni->ni_htopmode = SM(w, IEEE80211_HTINFO_OPMODE); w = le16dec(&htinfo->hi_byte45); ni->ni_htstbc = SM(w, IEEE80211_HTINFO_BASIC_STBCMCS); } /* * Parse an 802.11n HT info ie and save useful information * to the node state. Note this does not effect any state * changes such as for channel width change. */ void ieee80211_parse_htinfo(struct ieee80211_node *ni, const uint8_t *ie) { if (ie[0] == IEEE80211_ELEMID_VENDOR) ie += 4; htinfo_parse(ni, (const struct ieee80211_ie_htinfo *) ie); } /* * Handle 11n channel switch. Use the received HT ie's to * identify the right channel to use. If we cannot locate it * in the channel table then fallback to legacy operation. * Note that we use this information to identify the node's * channel only; the caller is responsible for insuring any * required channel change is done (e.g. in sta mode when * parsing the contents of a beacon frame). */ static int htinfo_update_chw(struct ieee80211_node *ni, int htflags) { struct ieee80211com *ic = ni->ni_ic; struct ieee80211_channel *c; int chanflags; int ret = 0; chanflags = (ni->ni_chan->ic_flags &~ IEEE80211_CHAN_HT) | htflags; if (chanflags != ni->ni_chan->ic_flags) { /* XXX not right for ht40- */ c = ieee80211_find_channel(ic, ni->ni_chan->ic_freq, chanflags); if (c == NULL && (htflags & IEEE80211_CHAN_HT40)) { /* * No HT40 channel entry in our table; fall back * to HT20 operation. This should not happen. */ c = findhtchan(ic, ni->ni_chan, IEEE80211_CHAN_HT20); #if 0 IEEE80211_NOTE(ni->ni_vap, IEEE80211_MSG_ASSOC | IEEE80211_MSG_11N, ni, "no HT40 channel (freq %u), falling back to HT20", ni->ni_chan->ic_freq); #endif /* XXX stat */ } if (c != NULL && c != ni->ni_chan) { IEEE80211_NOTE(ni->ni_vap, IEEE80211_MSG_ASSOC | IEEE80211_MSG_11N, ni, "switch station to HT%d channel %u/0x%x", IEEE80211_IS_CHAN_HT40(c) ? 40 : 20, c->ic_freq, c->ic_flags); ni->ni_chan = c; ret = 1; } /* NB: caller responsible for forcing any channel change */ } /* update node's tx channel width */ ni->ni_chw = IEEE80211_IS_CHAN_HT40(ni->ni_chan)? 40 : 20; return (ret); } /* * Update 11n MIMO PS state according to received htcap. */ static __inline int htcap_update_mimo_ps(struct ieee80211_node *ni) { uint16_t oflags = ni->ni_flags; switch (ni->ni_htcap & IEEE80211_HTCAP_SMPS) { case IEEE80211_HTCAP_SMPS_DYNAMIC: ni->ni_flags |= IEEE80211_NODE_MIMO_PS; ni->ni_flags |= IEEE80211_NODE_MIMO_RTS; break; case IEEE80211_HTCAP_SMPS_ENA: ni->ni_flags |= IEEE80211_NODE_MIMO_PS; ni->ni_flags &= ~IEEE80211_NODE_MIMO_RTS; break; case IEEE80211_HTCAP_SMPS_OFF: default: /* disable on rx of reserved value */ ni->ni_flags &= ~IEEE80211_NODE_MIMO_PS; ni->ni_flags &= ~IEEE80211_NODE_MIMO_RTS; break; } return (oflags ^ ni->ni_flags); } /* * Update short GI state according to received htcap * and local settings. */ static __inline void htcap_update_shortgi(struct ieee80211_node *ni) { struct ieee80211vap *vap = ni->ni_vap; ni->ni_flags &= ~(IEEE80211_NODE_SGI20|IEEE80211_NODE_SGI40); if ((ni->ni_htcap & IEEE80211_HTCAP_SHORTGI20) && (vap->iv_flags_ht & IEEE80211_FHT_SHORTGI20)) ni->ni_flags |= IEEE80211_NODE_SGI20; if ((ni->ni_htcap & IEEE80211_HTCAP_SHORTGI40) && (vap->iv_flags_ht & IEEE80211_FHT_SHORTGI40)) ni->ni_flags |= IEEE80211_NODE_SGI40; } /* * Parse and update HT-related state extracted from * the HT cap and info ie's. */ int ieee80211_ht_updateparams(struct ieee80211_node *ni, const uint8_t *htcapie, const uint8_t *htinfoie) { struct ieee80211vap *vap = ni->ni_vap; const struct ieee80211_ie_htinfo *htinfo; int htflags; int ret = 0; ieee80211_parse_htcap(ni, htcapie); if (vap->iv_htcaps & IEEE80211_HTCAP_SMPS) htcap_update_mimo_ps(ni); htcap_update_shortgi(ni); if (htinfoie[0] == IEEE80211_ELEMID_VENDOR) htinfoie += 4; htinfo = (const struct ieee80211_ie_htinfo *) htinfoie; htinfo_parse(ni, htinfo); htflags = (vap->iv_flags_ht & IEEE80211_FHT_HT) ? IEEE80211_CHAN_HT20 : 0; /* NB: honor operating mode constraint */ if ((htinfo->hi_byte1 & IEEE80211_HTINFO_TXWIDTH_2040) && (vap->iv_flags_ht & IEEE80211_FHT_USEHT40)) { if (ni->ni_ht2ndchan == IEEE80211_HTINFO_2NDCHAN_ABOVE) htflags = IEEE80211_CHAN_HT40U; else if (ni->ni_ht2ndchan == IEEE80211_HTINFO_2NDCHAN_BELOW) htflags = IEEE80211_CHAN_HT40D; } if (htinfo_update_chw(ni, htflags)) ret = 1; if ((htinfo->hi_byte1 & IEEE80211_HTINFO_RIFSMODE_PERM) && (vap->iv_flags_ht & IEEE80211_FHT_RIFS)) ni->ni_flags |= IEEE80211_NODE_RIFS; else ni->ni_flags &= ~IEEE80211_NODE_RIFS; return (ret); } /* * Parse and update HT-related state extracted from the HT cap ie * for a station joining an HT BSS. */ void ieee80211_ht_updatehtcap(struct ieee80211_node *ni, const uint8_t *htcapie) { struct ieee80211vap *vap = ni->ni_vap; int htflags; ieee80211_parse_htcap(ni, htcapie); if (vap->iv_htcaps & IEEE80211_HTCAP_SMPS) htcap_update_mimo_ps(ni); htcap_update_shortgi(ni); /* NB: honor operating mode constraint */ /* XXX 40 MHz intolerant */ htflags = (vap->iv_flags_ht & IEEE80211_FHT_HT) ? IEEE80211_CHAN_HT20 : 0; if ((ni->ni_htcap & IEEE80211_HTCAP_CHWIDTH40) && (vap->iv_flags_ht & IEEE80211_FHT_USEHT40)) { if (IEEE80211_IS_CHAN_HT40U(vap->iv_bss->ni_chan)) htflags = IEEE80211_CHAN_HT40U; else if (IEEE80211_IS_CHAN_HT40D(vap->iv_bss->ni_chan)) htflags = IEEE80211_CHAN_HT40D; } (void) htinfo_update_chw(ni, htflags); } /* * Install received HT rate set by parsing the HT cap ie. */ int ieee80211_setup_htrates(struct ieee80211_node *ni, const uint8_t *ie, int flags) { struct ieee80211com *ic = ni->ni_ic; struct ieee80211vap *vap = ni->ni_vap; const struct ieee80211_ie_htcap *htcap; struct ieee80211_htrateset *rs; int i, maxequalmcs, maxunequalmcs; maxequalmcs = ic->ic_txstream * 8 - 1; if (ic->ic_htcaps & IEEE80211_HTC_TXUNEQUAL) { if (ic->ic_txstream >= 2) maxunequalmcs = 38; if (ic->ic_txstream >= 3) maxunequalmcs = 52; if (ic->ic_txstream >= 4) maxunequalmcs = 76; } else maxunequalmcs = 0; rs = &ni->ni_htrates; memset(rs, 0, sizeof(*rs)); if (ie != NULL) { if (ie[0] == IEEE80211_ELEMID_VENDOR) ie += 4; htcap = (const struct ieee80211_ie_htcap *) ie; for (i = 0; i < IEEE80211_HTRATE_MAXSIZE; i++) { if (isclr(htcap->hc_mcsset, i)) continue; if (rs->rs_nrates == IEEE80211_HTRATE_MAXSIZE) { IEEE80211_NOTE(vap, IEEE80211_MSG_XRATE | IEEE80211_MSG_11N, ni, "WARNING, HT rate set too large; only " "using %u rates", IEEE80211_HTRATE_MAXSIZE); vap->iv_stats.is_rx_rstoobig++; break; } if (i <= 31 && i > maxequalmcs) continue; if (i == 32 && (ic->ic_htcaps & IEEE80211_HTC_TXMCS32) == 0) continue; if (i > 32 && i > maxunequalmcs) continue; rs->rs_rates[rs->rs_nrates++] = i; } } return ieee80211_fix_rate(ni, (struct ieee80211_rateset *) rs, flags); } /* * Mark rates in a node's HT rate set as basic according * to the information in the supplied HT info ie. */ void ieee80211_setup_basic_htrates(struct ieee80211_node *ni, const uint8_t *ie) { const struct ieee80211_ie_htinfo *htinfo; struct ieee80211_htrateset *rs; int i, j; if (ie[0] == IEEE80211_ELEMID_VENDOR) ie += 4; htinfo = (const struct ieee80211_ie_htinfo *) ie; rs = &ni->ni_htrates; if (rs->rs_nrates == 0) { IEEE80211_NOTE(ni->ni_vap, IEEE80211_MSG_XRATE | IEEE80211_MSG_11N, ni, "%s", "WARNING, empty HT rate set"); return; } for (i = 0; i < IEEE80211_HTRATE_MAXSIZE; i++) { if (isclr(htinfo->hi_basicmcsset, i)) continue; for (j = 0; j < rs->rs_nrates; j++) if ((rs->rs_rates[j] & IEEE80211_RATE_VAL) == i) rs->rs_rates[j] |= IEEE80211_RATE_BASIC; } } static void ampdu_tx_setup(struct ieee80211_tx_ampdu *tap) { callout_init(&tap->txa_timer, 1); tap->txa_flags |= IEEE80211_AGGR_SETUP; tap->txa_lastsample = ticks; } static void ampdu_tx_stop(struct ieee80211_tx_ampdu *tap) { struct ieee80211_node *ni = tap->txa_ni; struct ieee80211com *ic = ni->ni_ic; IEEE80211_NOTE(tap->txa_ni->ni_vap, IEEE80211_MSG_11N, tap->txa_ni, "%s: called", __func__); KASSERT(tap->txa_flags & IEEE80211_AGGR_SETUP, ("txa_flags 0x%x tid %d ac %d", tap->txa_flags, tap->txa_tid, TID_TO_WME_AC(tap->txa_tid))); /* * Stop BA stream if setup so driver has a chance * to reclaim any resources it might have allocated. */ ic->ic_addba_stop(ni, tap); /* * Stop any pending BAR transmit. */ bar_stop_timer(tap); /* * Reset packet estimate. */ ieee80211_txampdu_init_pps(tap); /* NB: clearing NAK means we may re-send ADDBA */ tap->txa_flags &= ~(IEEE80211_AGGR_SETUP | IEEE80211_AGGR_NAK); } /* * ADDBA response timeout. * * If software aggregation and per-TID queue management was done here, * that queue would be unpaused after the ADDBA timeout occurs. */ static void addba_timeout(void *arg) { struct ieee80211_tx_ampdu *tap = arg; struct ieee80211_node *ni = tap->txa_ni; struct ieee80211com *ic = ni->ni_ic; /* XXX ? */ tap->txa_flags &= ~IEEE80211_AGGR_XCHGPEND; tap->txa_attempts++; ic->ic_addba_response_timeout(ni, tap); } static void addba_start_timeout(struct ieee80211_tx_ampdu *tap) { /* XXX use CALLOUT_PENDING instead? */ callout_reset(&tap->txa_timer, ieee80211_addba_timeout, addba_timeout, tap); tap->txa_flags |= IEEE80211_AGGR_XCHGPEND; tap->txa_nextrequest = ticks + ieee80211_addba_timeout; } static void addba_stop_timeout(struct ieee80211_tx_ampdu *tap) { /* XXX use CALLOUT_PENDING instead? */ if (tap->txa_flags & IEEE80211_AGGR_XCHGPEND) { callout_stop(&tap->txa_timer); tap->txa_flags &= ~IEEE80211_AGGR_XCHGPEND; } } static void null_addba_response_timeout(struct ieee80211_node *ni, struct ieee80211_tx_ampdu *tap) { } /* * Default method for requesting A-MPDU tx aggregation. * We setup the specified state block and start a timer * to wait for an ADDBA response frame. */ static int ieee80211_addba_request(struct ieee80211_node *ni, struct ieee80211_tx_ampdu *tap, int dialogtoken, int baparamset, int batimeout) { int bufsiz; /* XXX locking */ tap->txa_token = dialogtoken; tap->txa_flags |= IEEE80211_AGGR_IMMEDIATE; bufsiz = MS(baparamset, IEEE80211_BAPS_BUFSIZ); tap->txa_wnd = (bufsiz == 0) ? IEEE80211_AGGR_BAWMAX : min(bufsiz, IEEE80211_AGGR_BAWMAX); addba_start_timeout(tap); return 1; } /* * Called by drivers that wish to request an ADDBA session be * setup. This brings it up and starts the request timer. */ int ieee80211_ampdu_tx_request_ext(struct ieee80211_node *ni, int tid) { struct ieee80211_tx_ampdu *tap; if (tid < 0 || tid > 15) return (0); tap = &ni->ni_tx_ampdu[tid]; /* XXX locking */ if ((tap->txa_flags & IEEE80211_AGGR_SETUP) == 0) { /* do deferred setup of state */ ampdu_tx_setup(tap); } /* XXX hack for not doing proper locking */ tap->txa_flags &= ~IEEE80211_AGGR_NAK; addba_start_timeout(tap); return (1); } /* * Called by drivers that have marked a session as active. */ int ieee80211_ampdu_tx_request_active_ext(struct ieee80211_node *ni, int tid, int status) { struct ieee80211_tx_ampdu *tap; if (tid < 0 || tid > 15) return (0); tap = &ni->ni_tx_ampdu[tid]; /* XXX locking */ addba_stop_timeout(tap); if (status == 1) { tap->txa_flags |= IEEE80211_AGGR_RUNNING; tap->txa_attempts = 0; } else { /* mark tid so we don't try again */ tap->txa_flags |= IEEE80211_AGGR_NAK; } return (1); } /* * Default method for processing an A-MPDU tx aggregation * response. We shutdown any pending timer and update the * state block according to the reply. */ static int ieee80211_addba_response(struct ieee80211_node *ni, struct ieee80211_tx_ampdu *tap, int status, int baparamset, int batimeout) { int bufsiz, tid; /* XXX locking */ addba_stop_timeout(tap); if (status == IEEE80211_STATUS_SUCCESS) { bufsiz = MS(baparamset, IEEE80211_BAPS_BUFSIZ); /* XXX override our request? */ tap->txa_wnd = (bufsiz == 0) ? IEEE80211_AGGR_BAWMAX : min(bufsiz, IEEE80211_AGGR_BAWMAX); /* XXX AC/TID */ tid = MS(baparamset, IEEE80211_BAPS_TID); tap->txa_flags |= IEEE80211_AGGR_RUNNING; tap->txa_attempts = 0; } else { /* mark tid so we don't try again */ tap->txa_flags |= IEEE80211_AGGR_NAK; } return 1; } /* * Default method for stopping A-MPDU tx aggregation. * Any timer is cleared and we drain any pending frames. */ static void ieee80211_addba_stop(struct ieee80211_node *ni, struct ieee80211_tx_ampdu *tap) { /* XXX locking */ addba_stop_timeout(tap); if (tap->txa_flags & IEEE80211_AGGR_RUNNING) { /* XXX clear aggregation queue */ tap->txa_flags &= ~IEEE80211_AGGR_RUNNING; } tap->txa_attempts = 0; } /* * Process a received action frame using the default aggregation * policy. We intercept ADDBA-related frames and use them to * update our aggregation state. All other frames are passed up * for processing by ieee80211_recv_action. */ static int ht_recv_action_ba_addba_request(struct ieee80211_node *ni, const struct ieee80211_frame *wh, const uint8_t *frm, const uint8_t *efrm) { struct ieee80211com *ic = ni->ni_ic; struct ieee80211vap *vap = ni->ni_vap; struct ieee80211_rx_ampdu *rap; uint8_t dialogtoken; uint16_t baparamset, batimeout, baseqctl; uint16_t args[5]; int tid; dialogtoken = frm[2]; baparamset = le16dec(frm+3); batimeout = le16dec(frm+5); baseqctl = le16dec(frm+7); tid = MS(baparamset, IEEE80211_BAPS_TID); IEEE80211_NOTE(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_11N, ni, "recv ADDBA request: dialogtoken %u baparamset 0x%x " "(tid %d bufsiz %d) batimeout %d baseqctl %d:%d", dialogtoken, baparamset, tid, MS(baparamset, IEEE80211_BAPS_BUFSIZ), batimeout, MS(baseqctl, IEEE80211_BASEQ_START), MS(baseqctl, IEEE80211_BASEQ_FRAG)); rap = &ni->ni_rx_ampdu[tid]; /* Send ADDBA response */ args[0] = dialogtoken; /* * NB: We ack only if the sta associated with HT and * the ap is configured to do AMPDU rx (the latter * violates the 11n spec and is mostly for testing). */ if ((ni->ni_flags & IEEE80211_NODE_AMPDU_RX) && (vap->iv_flags_ht & IEEE80211_FHT_AMPDU_RX)) { /* XXX handle ampdu_rx_start failure */ ic->ic_ampdu_rx_start(ni, rap, baparamset, batimeout, baseqctl); args[1] = IEEE80211_STATUS_SUCCESS; } else { IEEE80211_NOTE(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_11N, ni, "reject ADDBA request: %s", ni->ni_flags & IEEE80211_NODE_AMPDU_RX ? "administratively disabled" : "not negotiated for station"); vap->iv_stats.is_addba_reject++; args[1] = IEEE80211_STATUS_UNSPECIFIED; } /* XXX honor rap flags? */ args[2] = IEEE80211_BAPS_POLICY_IMMEDIATE | SM(tid, IEEE80211_BAPS_TID) | SM(rap->rxa_wnd, IEEE80211_BAPS_BUFSIZ) ; args[3] = 0; args[4] = 0; ic->ic_send_action(ni, IEEE80211_ACTION_CAT_BA, IEEE80211_ACTION_BA_ADDBA_RESPONSE, args); return 0; } static int ht_recv_action_ba_addba_response(struct ieee80211_node *ni, const struct ieee80211_frame *wh, const uint8_t *frm, const uint8_t *efrm) { struct ieee80211com *ic = ni->ni_ic; struct ieee80211vap *vap = ni->ni_vap; struct ieee80211_tx_ampdu *tap; uint8_t dialogtoken, policy; uint16_t baparamset, batimeout, code; int tid, bufsiz; dialogtoken = frm[2]; code = le16dec(frm+3); baparamset = le16dec(frm+5); tid = MS(baparamset, IEEE80211_BAPS_TID); bufsiz = MS(baparamset, IEEE80211_BAPS_BUFSIZ); policy = MS(baparamset, IEEE80211_BAPS_POLICY); batimeout = le16dec(frm+7); tap = &ni->ni_tx_ampdu[tid]; if ((tap->txa_flags & IEEE80211_AGGR_XCHGPEND) == 0) { IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_11N, ni->ni_macaddr, "ADDBA response", "no pending ADDBA, tid %d dialogtoken %u " "code %d", tid, dialogtoken, code); vap->iv_stats.is_addba_norequest++; return 0; } if (dialogtoken != tap->txa_token) { IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_11N, ni->ni_macaddr, "ADDBA response", "dialogtoken mismatch: waiting for %d, " "received %d, tid %d code %d", tap->txa_token, dialogtoken, tid, code); vap->iv_stats.is_addba_badtoken++; return 0; } /* NB: assumes IEEE80211_AGGR_IMMEDIATE is 1 */ if (policy != (tap->txa_flags & IEEE80211_AGGR_IMMEDIATE)) { IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_11N, ni->ni_macaddr, "ADDBA response", "policy mismatch: expecting %s, " "received %s, tid %d code %d", tap->txa_flags & IEEE80211_AGGR_IMMEDIATE, policy, tid, code); vap->iv_stats.is_addba_badpolicy++; return 0; } #if 0 /* XXX we take MIN in ieee80211_addba_response */ if (bufsiz > IEEE80211_AGGR_BAWMAX) { IEEE80211_DISCARD_MAC(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_11N, ni->ni_macaddr, "ADDBA response", "BA window too large: max %d, " "received %d, tid %d code %d", bufsiz, IEEE80211_AGGR_BAWMAX, tid, code); vap->iv_stats.is_addba_badbawinsize++; return 0; } #endif IEEE80211_NOTE(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_11N, ni, "recv ADDBA response: dialogtoken %u code %d " "baparamset 0x%x (tid %d bufsiz %d) batimeout %d", dialogtoken, code, baparamset, tid, bufsiz, batimeout); ic->ic_addba_response(ni, tap, code, baparamset, batimeout); return 0; } static int ht_recv_action_ba_delba(struct ieee80211_node *ni, const struct ieee80211_frame *wh, const uint8_t *frm, const uint8_t *efrm) { struct ieee80211com *ic = ni->ni_ic; struct ieee80211_rx_ampdu *rap; struct ieee80211_tx_ampdu *tap; uint16_t baparamset, code; int tid; baparamset = le16dec(frm+2); code = le16dec(frm+4); tid = MS(baparamset, IEEE80211_DELBAPS_TID); IEEE80211_NOTE(ni->ni_vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_11N, ni, "recv DELBA: baparamset 0x%x (tid %d initiator %d) " "code %d", baparamset, tid, MS(baparamset, IEEE80211_DELBAPS_INIT), code); if ((baparamset & IEEE80211_DELBAPS_INIT) == 0) { tap = &ni->ni_tx_ampdu[tid]; ic->ic_addba_stop(ni, tap); } else { rap = &ni->ni_rx_ampdu[tid]; ic->ic_ampdu_rx_stop(ni, rap); } return 0; } static int ht_recv_action_ht_txchwidth(struct ieee80211_node *ni, const struct ieee80211_frame *wh, const uint8_t *frm, const uint8_t *efrm) { int chw; chw = (frm[2] == IEEE80211_A_HT_TXCHWIDTH_2040) ? 40 : 20; IEEE80211_NOTE(ni->ni_vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_11N, ni, "%s: HT txchwidth, width %d%s", __func__, chw, ni->ni_chw != chw ? "*" : ""); if (chw != ni->ni_chw) { ni->ni_chw = chw; /* XXX notify on change */ } return 0; } static int ht_recv_action_ht_mimopwrsave(struct ieee80211_node *ni, const struct ieee80211_frame *wh, const uint8_t *frm, const uint8_t *efrm) { const struct ieee80211_action_ht_mimopowersave *mps = (const struct ieee80211_action_ht_mimopowersave *) frm; /* XXX check iv_htcaps */ if (mps->am_control & IEEE80211_A_HT_MIMOPWRSAVE_ENA) ni->ni_flags |= IEEE80211_NODE_MIMO_PS; else ni->ni_flags &= ~IEEE80211_NODE_MIMO_PS; if (mps->am_control & IEEE80211_A_HT_MIMOPWRSAVE_MODE) ni->ni_flags |= IEEE80211_NODE_MIMO_RTS; else ni->ni_flags &= ~IEEE80211_NODE_MIMO_RTS; /* XXX notify on change */ IEEE80211_NOTE(ni->ni_vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_11N, ni, "%s: HT MIMO PS (%s%s)", __func__, (ni->ni_flags & IEEE80211_NODE_MIMO_PS) ? "on" : "off", (ni->ni_flags & IEEE80211_NODE_MIMO_RTS) ? "+rts" : "" ); return 0; } /* * Transmit processing. */ /* * Check if A-MPDU should be requested/enabled for a stream. * We require a traffic rate above a per-AC threshold and we * also handle backoff from previous failed attempts. * * Drivers may override this method to bring in information * such as link state conditions in making the decision. */ static int ieee80211_ampdu_enable(struct ieee80211_node *ni, struct ieee80211_tx_ampdu *tap) { struct ieee80211vap *vap = ni->ni_vap; if (tap->txa_avgpps < vap->iv_ampdu_mintraffic[TID_TO_WME_AC(tap->txa_tid)]) return 0; /* XXX check rssi? */ if (tap->txa_attempts >= ieee80211_addba_maxtries && ieee80211_time_after(ticks, tap->txa_nextrequest)) { /* * Don't retry too often; txa_nextrequest is set * to the minimum interval we'll retry after * ieee80211_addba_maxtries failed attempts are made. */ return 0; } IEEE80211_NOTE(vap, IEEE80211_MSG_11N, ni, "enable AMPDU on tid %d (%s), avgpps %d pkts %d attempt %d", tap->txa_tid, ieee80211_wme_acnames[TID_TO_WME_AC(tap->txa_tid)], tap->txa_avgpps, tap->txa_pkts, tap->txa_attempts); return 1; } /* * Request A-MPDU tx aggregation. Setup local state and * issue an ADDBA request. BA use will only happen after * the other end replies with ADDBA response. */ int ieee80211_ampdu_request(struct ieee80211_node *ni, struct ieee80211_tx_ampdu *tap) { struct ieee80211com *ic = ni->ni_ic; uint16_t args[5]; int tid, dialogtoken; static int tokens = 0; /* XXX */ /* XXX locking */ if ((tap->txa_flags & IEEE80211_AGGR_SETUP) == 0) { /* do deferred setup of state */ ampdu_tx_setup(tap); } /* XXX hack for not doing proper locking */ tap->txa_flags &= ~IEEE80211_AGGR_NAK; dialogtoken = (tokens+1) % 63; /* XXX */ tid = tap->txa_tid; tap->txa_start = ni->ni_txseqs[tid]; args[0] = dialogtoken; args[1] = 0; /* NB: status code not used */ args[2] = IEEE80211_BAPS_POLICY_IMMEDIATE | SM(tid, IEEE80211_BAPS_TID) | SM(IEEE80211_AGGR_BAWMAX, IEEE80211_BAPS_BUFSIZ) ; args[3] = 0; /* batimeout */ /* NB: do first so there's no race against reply */ if (!ic->ic_addba_request(ni, tap, dialogtoken, args[2], args[3])) { /* unable to setup state, don't make request */ IEEE80211_NOTE(ni->ni_vap, IEEE80211_MSG_11N, ni, "%s: could not setup BA stream for TID %d AC %d", __func__, tap->txa_tid, TID_TO_WME_AC(tap->txa_tid)); /* defer next try so we don't slam the driver with requests */ tap->txa_attempts = ieee80211_addba_maxtries; /* NB: check in case driver wants to override */ if (tap->txa_nextrequest <= ticks) tap->txa_nextrequest = ticks + ieee80211_addba_backoff; return 0; } tokens = dialogtoken; /* allocate token */ /* NB: after calling ic_addba_request so driver can set txa_start */ args[4] = SM(tap->txa_start, IEEE80211_BASEQ_START) | SM(0, IEEE80211_BASEQ_FRAG) ; return ic->ic_send_action(ni, IEEE80211_ACTION_CAT_BA, IEEE80211_ACTION_BA_ADDBA_REQUEST, args); } /* * Terminate an AMPDU tx stream. State is reclaimed * and the peer notified with a DelBA Action frame. */ void ieee80211_ampdu_stop(struct ieee80211_node *ni, struct ieee80211_tx_ampdu *tap, int reason) { struct ieee80211com *ic = ni->ni_ic; struct ieee80211vap *vap = ni->ni_vap; uint16_t args[4]; /* XXX locking */ tap->txa_flags &= ~IEEE80211_AGGR_BARPEND; if (IEEE80211_AMPDU_RUNNING(tap)) { IEEE80211_NOTE(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_11N, ni, "%s: stop BA stream for TID %d (reason: %d (%s))", __func__, tap->txa_tid, reason, ieee80211_reason_to_string(reason)); vap->iv_stats.is_ampdu_stop++; ic->ic_addba_stop(ni, tap); args[0] = tap->txa_tid; args[1] = IEEE80211_DELBAPS_INIT; args[2] = reason; /* XXX reason code */ ic->ic_send_action(ni, IEEE80211_ACTION_CAT_BA, IEEE80211_ACTION_BA_DELBA, args); } else { IEEE80211_NOTE(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_11N, ni, "%s: BA stream for TID %d not running " "(reason: %d (%s))", __func__, tap->txa_tid, reason, ieee80211_reason_to_string(reason)); vap->iv_stats.is_ampdu_stop_failed++; } } /* XXX */ static void bar_start_timer(struct ieee80211_tx_ampdu *tap); static void bar_timeout(void *arg) { struct ieee80211_tx_ampdu *tap = arg; struct ieee80211_node *ni = tap->txa_ni; KASSERT((tap->txa_flags & IEEE80211_AGGR_XCHGPEND) == 0, ("bar/addba collision, flags 0x%x", tap->txa_flags)); IEEE80211_NOTE(ni->ni_vap, IEEE80211_MSG_11N, ni, "%s: tid %u flags 0x%x attempts %d", __func__, tap->txa_tid, tap->txa_flags, tap->txa_attempts); /* guard against race with bar_tx_complete */ if ((tap->txa_flags & IEEE80211_AGGR_BARPEND) == 0) return; /* XXX ? */ if (tap->txa_attempts >= ieee80211_bar_maxtries) { struct ieee80211com *ic = ni->ni_ic; ni->ni_vap->iv_stats.is_ampdu_bar_tx_fail++; /* * If (at least) the last BAR TX timeout was due to * an ieee80211_send_bar() failures, then we need * to make sure we notify the driver that a BAR * TX did occur and fail. This gives the driver * a chance to undo any queue pause that may * have occurred. */ ic->ic_bar_response(ni, tap, 1); ieee80211_ampdu_stop(ni, tap, IEEE80211_REASON_TIMEOUT); } else { ni->ni_vap->iv_stats.is_ampdu_bar_tx_retry++; if (ieee80211_send_bar(ni, tap, tap->txa_seqpending) != 0) { IEEE80211_NOTE(ni->ni_vap, IEEE80211_MSG_11N, ni, "%s: failed to TX, starting timer\n", __func__); /* * If ieee80211_send_bar() fails here, the * timer may have stopped and/or the pending * flag may be clear. Because of this, * fake the BARPEND and reset the timer. * A retransmission attempt will then occur * during the next timeout. */ /* XXX locking */ tap->txa_flags |= IEEE80211_AGGR_BARPEND; bar_start_timer(tap); } } } static void bar_start_timer(struct ieee80211_tx_ampdu *tap) { IEEE80211_NOTE(tap->txa_ni->ni_vap, IEEE80211_MSG_11N, tap->txa_ni, "%s: called", __func__); callout_reset(&tap->txa_timer, ieee80211_bar_timeout, bar_timeout, tap); } static void bar_stop_timer(struct ieee80211_tx_ampdu *tap) { IEEE80211_NOTE(tap->txa_ni->ni_vap, IEEE80211_MSG_11N, tap->txa_ni, "%s: called", __func__); callout_stop(&tap->txa_timer); } static void bar_tx_complete(struct ieee80211_node *ni, void *arg, int status) { struct ieee80211_tx_ampdu *tap = arg; IEEE80211_NOTE(ni->ni_vap, IEEE80211_MSG_11N, ni, "%s: tid %u flags 0x%x pending %d status %d", __func__, tap->txa_tid, tap->txa_flags, callout_pending(&tap->txa_timer), status); ni->ni_vap->iv_stats.is_ampdu_bar_tx++; /* XXX locking */ if ((tap->txa_flags & IEEE80211_AGGR_BARPEND) && callout_pending(&tap->txa_timer)) { struct ieee80211com *ic = ni->ni_ic; if (status == 0) /* ACK'd */ bar_stop_timer(tap); ic->ic_bar_response(ni, tap, status); /* NB: just let timer expire so we pace requests */ } } static void ieee80211_bar_response(struct ieee80211_node *ni, struct ieee80211_tx_ampdu *tap, int status) { IEEE80211_NOTE(tap->txa_ni->ni_vap, IEEE80211_MSG_11N, tap->txa_ni, "%s: called", __func__); if (status == 0) { /* got ACK */ IEEE80211_NOTE(ni->ni_vap, IEEE80211_MSG_11N, ni, "BAR moves BA win <%u:%u> (%u frames) txseq %u tid %u", tap->txa_start, IEEE80211_SEQ_ADD(tap->txa_start, tap->txa_wnd-1), tap->txa_qframes, tap->txa_seqpending, tap->txa_tid); /* NB: timer already stopped in bar_tx_complete */ tap->txa_start = tap->txa_seqpending; tap->txa_flags &= ~IEEE80211_AGGR_BARPEND; } } /* * Transmit a BAR frame to the specified node. The * BAR contents are drawn from the supplied aggregation * state associated with the node. * * NB: we only handle immediate ACK w/ compressed bitmap. */ int ieee80211_send_bar(struct ieee80211_node *ni, struct ieee80211_tx_ampdu *tap, ieee80211_seq seq) { #define senderr(_x, _v) do { vap->iv_stats._v++; ret = _x; goto bad; } while (0) struct ieee80211vap *vap = ni->ni_vap; struct ieee80211com *ic = ni->ni_ic; struct ieee80211_frame_bar *bar; struct mbuf *m; uint16_t barctl, barseqctl; uint8_t *frm; int tid, ret; IEEE80211_NOTE(tap->txa_ni->ni_vap, IEEE80211_MSG_11N, tap->txa_ni, "%s: called", __func__); if ((tap->txa_flags & IEEE80211_AGGR_RUNNING) == 0) { /* no ADDBA response, should not happen */ /* XXX stat+msg */ return EINVAL; } /* XXX locking */ bar_stop_timer(tap); ieee80211_ref_node(ni); m = ieee80211_getmgtframe(&frm, ic->ic_headroom, sizeof(*bar)); if (m == NULL) senderr(ENOMEM, is_tx_nobuf); if (!ieee80211_add_callback(m, bar_tx_complete, tap)) { m_freem(m); senderr(ENOMEM, is_tx_nobuf); /* XXX */ /* NOTREACHED */ } bar = mtod(m, struct ieee80211_frame_bar *); bar->i_fc[0] = IEEE80211_FC0_VERSION_0 | IEEE80211_FC0_TYPE_CTL | IEEE80211_FC0_SUBTYPE_BAR; bar->i_fc[1] = 0; IEEE80211_ADDR_COPY(bar->i_ra, ni->ni_macaddr); IEEE80211_ADDR_COPY(bar->i_ta, vap->iv_myaddr); tid = tap->txa_tid; barctl = (tap->txa_flags & IEEE80211_AGGR_IMMEDIATE ? 0 : IEEE80211_BAR_NOACK) | IEEE80211_BAR_COMP | SM(tid, IEEE80211_BAR_TID) ; barseqctl = SM(seq, IEEE80211_BAR_SEQ_START); /* NB: known to have proper alignment */ bar->i_ctl = htole16(barctl); bar->i_seq = htole16(barseqctl); m->m_pkthdr.len = m->m_len = sizeof(struct ieee80211_frame_bar); M_WME_SETAC(m, WME_AC_VO); IEEE80211_NODE_STAT(ni, tx_mgmt); /* XXX tx_ctl? */ /* XXX locking */ /* init/bump attempts counter */ if ((tap->txa_flags & IEEE80211_AGGR_BARPEND) == 0) tap->txa_attempts = 1; else tap->txa_attempts++; tap->txa_seqpending = seq; tap->txa_flags |= IEEE80211_AGGR_BARPEND; IEEE80211_NOTE(vap, IEEE80211_MSG_DEBUG | IEEE80211_MSG_11N, ni, "send BAR: tid %u ctl 0x%x start %u (attempt %d)", tid, barctl, seq, tap->txa_attempts); /* * ic_raw_xmit will free the node reference * regardless of queue/TX success or failure. */ IEEE80211_TX_LOCK(ic); ret = ieee80211_raw_output(vap, ni, m, NULL); IEEE80211_TX_UNLOCK(ic); if (ret != 0) { IEEE80211_NOTE(vap, IEEE80211_MSG_DEBUG | IEEE80211_MSG_11N, ni, "send BAR: failed: (ret = %d)\n", ret); /* xmit failed, clear state flag */ tap->txa_flags &= ~IEEE80211_AGGR_BARPEND; vap->iv_stats.is_ampdu_bar_tx_fail++; return ret; } /* XXX hack against tx complete happening before timer is started */ if (tap->txa_flags & IEEE80211_AGGR_BARPEND) bar_start_timer(tap); return 0; bad: IEEE80211_NOTE(tap->txa_ni->ni_vap, IEEE80211_MSG_11N, tap->txa_ni, "%s: bad! ret=%d", __func__, ret); vap->iv_stats.is_ampdu_bar_tx_fail++; ieee80211_free_node(ni); return ret; #undef senderr } static int ht_action_output(struct ieee80211_node *ni, struct mbuf *m) { struct ieee80211_bpf_params params; memset(¶ms, 0, sizeof(params)); params.ibp_pri = WME_AC_VO; params.ibp_rate0 = ni->ni_txparms->mgmtrate; /* NB: we know all frames are unicast */ params.ibp_try0 = ni->ni_txparms->maxretry; params.ibp_power = ni->ni_txpower; return ieee80211_mgmt_output(ni, m, IEEE80211_FC0_SUBTYPE_ACTION, ¶ms); } #define ADDSHORT(frm, v) do { \ frm[0] = (v) & 0xff; \ frm[1] = (v) >> 8; \ frm += 2; \ } while (0) /* * Send an action management frame. The arguments are stuff * into a frame without inspection; the caller is assumed to * prepare them carefully (e.g. based on the aggregation state). */ static int ht_send_action_ba_addba(struct ieee80211_node *ni, int category, int action, void *arg0) { struct ieee80211vap *vap = ni->ni_vap; struct ieee80211com *ic = ni->ni_ic; uint16_t *args = arg0; struct mbuf *m; uint8_t *frm; IEEE80211_NOTE(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_11N, ni, "send ADDBA %s: dialogtoken %d status %d " "baparamset 0x%x (tid %d) batimeout 0x%x baseqctl 0x%x", (action == IEEE80211_ACTION_BA_ADDBA_REQUEST) ? "request" : "response", args[0], args[1], args[2], MS(args[2], IEEE80211_BAPS_TID), args[3], args[4]); IEEE80211_DPRINTF(vap, IEEE80211_MSG_NODE, "ieee80211_ref_node (%s:%u) %p<%s> refcnt %d\n", __func__, __LINE__, ni, ether_sprintf(ni->ni_macaddr), ieee80211_node_refcnt(ni)+1); ieee80211_ref_node(ni); m = ieee80211_getmgtframe(&frm, ic->ic_headroom + sizeof(struct ieee80211_frame), sizeof(uint16_t) /* action+category */ /* XXX may action payload */ + sizeof(struct ieee80211_action_ba_addbaresponse) ); if (m != NULL) { *frm++ = category; *frm++ = action; *frm++ = args[0]; /* dialog token */ if (action == IEEE80211_ACTION_BA_ADDBA_RESPONSE) ADDSHORT(frm, args[1]); /* status code */ ADDSHORT(frm, args[2]); /* baparamset */ ADDSHORT(frm, args[3]); /* batimeout */ if (action == IEEE80211_ACTION_BA_ADDBA_REQUEST) ADDSHORT(frm, args[4]); /* baseqctl */ m->m_pkthdr.len = m->m_len = frm - mtod(m, uint8_t *); return ht_action_output(ni, m); } else { vap->iv_stats.is_tx_nobuf++; ieee80211_free_node(ni); return ENOMEM; } } static int ht_send_action_ba_delba(struct ieee80211_node *ni, int category, int action, void *arg0) { struct ieee80211vap *vap = ni->ni_vap; struct ieee80211com *ic = ni->ni_ic; uint16_t *args = arg0; struct mbuf *m; uint16_t baparamset; uint8_t *frm; baparamset = SM(args[0], IEEE80211_DELBAPS_TID) | args[1] ; IEEE80211_NOTE(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_11N, ni, "send DELBA action: tid %d, initiator %d reason %d (%s)", args[0], args[1], args[2], ieee80211_reason_to_string(args[2])); IEEE80211_DPRINTF(vap, IEEE80211_MSG_NODE, "ieee80211_ref_node (%s:%u) %p<%s> refcnt %d\n", __func__, __LINE__, ni, ether_sprintf(ni->ni_macaddr), ieee80211_node_refcnt(ni)+1); ieee80211_ref_node(ni); m = ieee80211_getmgtframe(&frm, ic->ic_headroom + sizeof(struct ieee80211_frame), sizeof(uint16_t) /* action+category */ /* XXX may action payload */ + sizeof(struct ieee80211_action_ba_addbaresponse) ); if (m != NULL) { *frm++ = category; *frm++ = action; ADDSHORT(frm, baparamset); ADDSHORT(frm, args[2]); /* reason code */ m->m_pkthdr.len = m->m_len = frm - mtod(m, uint8_t *); return ht_action_output(ni, m); } else { vap->iv_stats.is_tx_nobuf++; ieee80211_free_node(ni); return ENOMEM; } } static int ht_send_action_ht_txchwidth(struct ieee80211_node *ni, int category, int action, void *arg0) { struct ieee80211vap *vap = ni->ni_vap; struct ieee80211com *ic = ni->ni_ic; struct mbuf *m; uint8_t *frm; IEEE80211_NOTE(vap, IEEE80211_MSG_ACTION | IEEE80211_MSG_11N, ni, "send HT txchwidth: width %d", IEEE80211_IS_CHAN_HT40(ni->ni_chan) ? 40 : 20); IEEE80211_DPRINTF(vap, IEEE80211_MSG_NODE, "ieee80211_ref_node (%s:%u) %p<%s> refcnt %d\n", __func__, __LINE__, ni, ether_sprintf(ni->ni_macaddr), ieee80211_node_refcnt(ni)+1); ieee80211_ref_node(ni); m = ieee80211_getmgtframe(&frm, ic->ic_headroom + sizeof(struct ieee80211_frame), sizeof(uint16_t) /* action+category */ /* XXX may action payload */ + sizeof(struct ieee80211_action_ba_addbaresponse) ); if (m != NULL) { *frm++ = category; *frm++ = action; *frm++ = IEEE80211_IS_CHAN_HT40(ni->ni_chan) ? IEEE80211_A_HT_TXCHWIDTH_2040 : IEEE80211_A_HT_TXCHWIDTH_20; m->m_pkthdr.len = m->m_len = frm - mtod(m, uint8_t *); return ht_action_output(ni, m); } else { vap->iv_stats.is_tx_nobuf++; ieee80211_free_node(ni); return ENOMEM; } } #undef ADDSHORT /* * Construct the MCS bit mask for inclusion in an HT capabilities * information element. */ static void ieee80211_set_mcsset(struct ieee80211com *ic, uint8_t *frm) { int i; uint8_t txparams; KASSERT((ic->ic_rxstream > 0 && ic->ic_rxstream <= 4), ("ic_rxstream %d out of range", ic->ic_rxstream)); KASSERT((ic->ic_txstream > 0 && ic->ic_txstream <= 4), ("ic_txstream %d out of range", ic->ic_txstream)); for (i = 0; i < ic->ic_rxstream * 8; i++) setbit(frm, i); if ((ic->ic_htcaps & IEEE80211_HTCAP_CHWIDTH40) && (ic->ic_htcaps & IEEE80211_HTC_RXMCS32)) setbit(frm, 32); if (ic->ic_htcaps & IEEE80211_HTC_RXUNEQUAL) { if (ic->ic_rxstream >= 2) { for (i = 33; i <= 38; i++) setbit(frm, i); } if (ic->ic_rxstream >= 3) { for (i = 39; i <= 52; i++) setbit(frm, i); } if (ic->ic_txstream >= 4) { for (i = 53; i <= 76; i++) setbit(frm, i); } } if (ic->ic_rxstream != ic->ic_txstream) { txparams = 0x1; /* TX MCS set defined */ txparams |= 0x2; /* TX RX MCS not equal */ txparams |= (ic->ic_txstream - 1) << 2; /* num TX streams */ if (ic->ic_htcaps & IEEE80211_HTC_TXUNEQUAL) txparams |= 0x16; /* TX unequal modulation sup */ } else txparams = 0; frm[12] = txparams; } /* * Add body of an HTCAP information element. */ static uint8_t * ieee80211_add_htcap_body(uint8_t *frm, struct ieee80211_node *ni) { #define ADDSHORT(frm, v) do { \ frm[0] = (v) & 0xff; \ frm[1] = (v) >> 8; \ frm += 2; \ } while (0) struct ieee80211com *ic = ni->ni_ic; struct ieee80211vap *vap = ni->ni_vap; uint16_t caps, extcaps; int rxmax, density; /* HT capabilities */ caps = vap->iv_htcaps & 0xffff; /* * Note channel width depends on whether we are operating as * a sta or not. When operating as a sta we are generating * a request based on our desired configuration. Otherwise * we are operational and the channel attributes identify * how we've been setup (which might be different if a fixed * channel is specified). */ if (vap->iv_opmode == IEEE80211_M_STA) { /* override 20/40 use based on config */ if (vap->iv_flags_ht & IEEE80211_FHT_USEHT40) caps |= IEEE80211_HTCAP_CHWIDTH40; else caps &= ~IEEE80211_HTCAP_CHWIDTH40; /* Start by using the advertised settings */ rxmax = MS(ni->ni_htparam, IEEE80211_HTCAP_MAXRXAMPDU); density = MS(ni->ni_htparam, IEEE80211_HTCAP_MPDUDENSITY); + IEEE80211_DPRINTF(vap, IEEE80211_MSG_11N, + "%s: advertised rxmax=%d, density=%d, vap rxmax=%d, density=%d\n", + __func__, + rxmax, + density, + vap->iv_ampdu_rxmax, + vap->iv_ampdu_density); + /* Cap at VAP rxmax */ if (rxmax > vap->iv_ampdu_rxmax) rxmax = vap->iv_ampdu_rxmax; /* * If the VAP ampdu density value greater, use that. * * (Larger density value == larger minimum gap between A-MPDU * subframes.) */ if (vap->iv_ampdu_density > density) density = vap->iv_ampdu_density; /* * NB: Hardware might support HT40 on some but not all * channels. We can't determine this earlier because only * after association the channel is upgraded to HT based * on the negotiated capabilities. */ if (ni->ni_chan != IEEE80211_CHAN_ANYC && findhtchan(ic, ni->ni_chan, IEEE80211_CHAN_HT40U) == NULL && findhtchan(ic, ni->ni_chan, IEEE80211_CHAN_HT40D) == NULL) caps &= ~IEEE80211_HTCAP_CHWIDTH40; } else { /* override 20/40 use based on current channel */ if (IEEE80211_IS_CHAN_HT40(ni->ni_chan)) caps |= IEEE80211_HTCAP_CHWIDTH40; else caps &= ~IEEE80211_HTCAP_CHWIDTH40; /* XXX TODO should it start by using advertised settings? */ rxmax = vap->iv_ampdu_rxmax; density = vap->iv_ampdu_density; } /* adjust short GI based on channel and config */ if ((vap->iv_flags_ht & IEEE80211_FHT_SHORTGI20) == 0) caps &= ~IEEE80211_HTCAP_SHORTGI20; if ((vap->iv_flags_ht & IEEE80211_FHT_SHORTGI40) == 0 || (caps & IEEE80211_HTCAP_CHWIDTH40) == 0) caps &= ~IEEE80211_HTCAP_SHORTGI40; /* adjust STBC based on receive capabilities */ if ((vap->iv_flags_ht & IEEE80211_FHT_STBC_RX) == 0) caps &= ~IEEE80211_HTCAP_RXSTBC; /* XXX TODO: adjust LDPC based on receive capabilities */ ADDSHORT(frm, caps); /* HT parameters */ *frm = SM(rxmax, IEEE80211_HTCAP_MAXRXAMPDU) | SM(density, IEEE80211_HTCAP_MPDUDENSITY) ; frm++; /* pre-zero remainder of ie */ memset(frm, 0, sizeof(struct ieee80211_ie_htcap) - __offsetof(struct ieee80211_ie_htcap, hc_mcsset)); /* supported MCS set */ /* * XXX: For sta mode the rate set should be restricted based * on the AP's capabilities, but ni_htrates isn't setup when * we're called to form an AssocReq frame so for now we're * restricted to the device capabilities. */ ieee80211_set_mcsset(ni->ni_ic, frm); frm += __offsetof(struct ieee80211_ie_htcap, hc_extcap) - __offsetof(struct ieee80211_ie_htcap, hc_mcsset); /* HT extended capabilities */ extcaps = vap->iv_htextcaps & 0xffff; ADDSHORT(frm, extcaps); frm += sizeof(struct ieee80211_ie_htcap) - __offsetof(struct ieee80211_ie_htcap, hc_txbf); return frm; #undef ADDSHORT } /* * Add 802.11n HT capabilities information element */ uint8_t * ieee80211_add_htcap(uint8_t *frm, struct ieee80211_node *ni) { frm[0] = IEEE80211_ELEMID_HTCAP; frm[1] = sizeof(struct ieee80211_ie_htcap) - 2; return ieee80211_add_htcap_body(frm + 2, ni); } /* * Add Broadcom OUI wrapped standard HTCAP ie; this is * used for compatibility w/ pre-draft implementations. */ uint8_t * ieee80211_add_htcap_vendor(uint8_t *frm, struct ieee80211_node *ni) { frm[0] = IEEE80211_ELEMID_VENDOR; frm[1] = 4 + sizeof(struct ieee80211_ie_htcap) - 2; frm[2] = (BCM_OUI >> 0) & 0xff; frm[3] = (BCM_OUI >> 8) & 0xff; frm[4] = (BCM_OUI >> 16) & 0xff; frm[5] = BCM_OUI_HTCAP; return ieee80211_add_htcap_body(frm + 6, ni); } /* * Construct the MCS bit mask of basic rates * for inclusion in an HT information element. */ static void ieee80211_set_basic_htrates(uint8_t *frm, const struct ieee80211_htrateset *rs) { int i; for (i = 0; i < rs->rs_nrates; i++) { int r = rs->rs_rates[i] & IEEE80211_RATE_VAL; if ((rs->rs_rates[i] & IEEE80211_RATE_BASIC) && r < IEEE80211_HTRATE_MAXSIZE) { /* NB: this assumes a particular implementation */ setbit(frm, r); } } } /* * Update the HTINFO ie for a beacon frame. */ void ieee80211_ht_update_beacon(struct ieee80211vap *vap, struct ieee80211_beacon_offsets *bo) { #define PROTMODE (IEEE80211_HTINFO_OPMODE|IEEE80211_HTINFO_NONHT_PRESENT) struct ieee80211_node *ni; const struct ieee80211_channel *bsschan; struct ieee80211com *ic = vap->iv_ic; struct ieee80211_ie_htinfo *ht = (struct ieee80211_ie_htinfo *) bo->bo_htinfo; ni = ieee80211_ref_node(vap->iv_bss); bsschan = ni->ni_chan; /* XXX only update on channel change */ ht->hi_ctrlchannel = ieee80211_chan2ieee(ic, bsschan); if (vap->iv_flags_ht & IEEE80211_FHT_RIFS) ht->hi_byte1 = IEEE80211_HTINFO_RIFSMODE_PERM; else ht->hi_byte1 = IEEE80211_HTINFO_RIFSMODE_PROH; if (IEEE80211_IS_CHAN_HT40U(bsschan)) ht->hi_byte1 |= IEEE80211_HTINFO_2NDCHAN_ABOVE; else if (IEEE80211_IS_CHAN_HT40D(bsschan)) ht->hi_byte1 |= IEEE80211_HTINFO_2NDCHAN_BELOW; else ht->hi_byte1 |= IEEE80211_HTINFO_2NDCHAN_NONE; if (IEEE80211_IS_CHAN_HT40(bsschan)) ht->hi_byte1 |= IEEE80211_HTINFO_TXWIDTH_2040; /* protection mode */ ht->hi_byte2 = (ht->hi_byte2 &~ PROTMODE) | ic->ic_curhtprotmode; ieee80211_free_node(ni); /* XXX propagate to vendor ie's */ #undef PROTMODE } /* * Add body of an HTINFO information element. * * NB: We don't use struct ieee80211_ie_htinfo because we can * be called to fillin both a standard ie and a compat ie that * has a vendor OUI at the front. */ static uint8_t * ieee80211_add_htinfo_body(uint8_t *frm, struct ieee80211_node *ni) { struct ieee80211vap *vap = ni->ni_vap; struct ieee80211com *ic = ni->ni_ic; /* pre-zero remainder of ie */ memset(frm, 0, sizeof(struct ieee80211_ie_htinfo) - 2); /* primary/control channel center */ *frm++ = ieee80211_chan2ieee(ic, ni->ni_chan); if (vap->iv_flags_ht & IEEE80211_FHT_RIFS) frm[0] = IEEE80211_HTINFO_RIFSMODE_PERM; else frm[0] = IEEE80211_HTINFO_RIFSMODE_PROH; if (IEEE80211_IS_CHAN_HT40U(ni->ni_chan)) frm[0] |= IEEE80211_HTINFO_2NDCHAN_ABOVE; else if (IEEE80211_IS_CHAN_HT40D(ni->ni_chan)) frm[0] |= IEEE80211_HTINFO_2NDCHAN_BELOW; else frm[0] |= IEEE80211_HTINFO_2NDCHAN_NONE; if (IEEE80211_IS_CHAN_HT40(ni->ni_chan)) frm[0] |= IEEE80211_HTINFO_TXWIDTH_2040; frm[1] = ic->ic_curhtprotmode; frm += 5; /* basic MCS set */ ieee80211_set_basic_htrates(frm, &ni->ni_htrates); frm += sizeof(struct ieee80211_ie_htinfo) - __offsetof(struct ieee80211_ie_htinfo, hi_basicmcsset); return frm; } /* * Add 802.11n HT information information element. */ uint8_t * ieee80211_add_htinfo(uint8_t *frm, struct ieee80211_node *ni) { frm[0] = IEEE80211_ELEMID_HTINFO; frm[1] = sizeof(struct ieee80211_ie_htinfo) - 2; return ieee80211_add_htinfo_body(frm + 2, ni); } /* * Add Broadcom OUI wrapped standard HTINFO ie; this is * used for compatibility w/ pre-draft implementations. */ uint8_t * ieee80211_add_htinfo_vendor(uint8_t *frm, struct ieee80211_node *ni) { frm[0] = IEEE80211_ELEMID_VENDOR; frm[1] = 4 + sizeof(struct ieee80211_ie_htinfo) - 2; frm[2] = (BCM_OUI >> 0) & 0xff; frm[3] = (BCM_OUI >> 8) & 0xff; frm[4] = (BCM_OUI >> 16) & 0xff; frm[5] = BCM_OUI_HTINFO; return ieee80211_add_htinfo_body(frm + 6, ni); } Index: user/ngie/release-pkg-fix-tests/sys/net80211/ieee80211_var.h =================================================================== --- user/ngie/release-pkg-fix-tests/sys/net80211/ieee80211_var.h (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/net80211/ieee80211_var.h (revision 299091) @@ -1,1012 +1,1013 @@ /*- * Copyright (c) 2001 Atsushi Onoe * Copyright (c) 2002-2009 Sam Leffler, Errno Consulting * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * $FreeBSD$ */ #ifndef _NET80211_IEEE80211_VAR_H_ #define _NET80211_IEEE80211_VAR_H_ /* * Definitions for IEEE 802.11 drivers. */ /* NB: portability glue must go first */ #if defined(__NetBSD__) #include #elif defined(__FreeBSD__) #include #elif defined(__linux__) #include #else #error "No support for your operating system!" #endif #include #include #include #include #include #include /* for ieee80211_stats */ #include #include #include #include #include #include #define IEEE80211_TXPOWER_MAX 100 /* .5 dBm (XXX units?) */ #define IEEE80211_TXPOWER_MIN 0 /* kill radio */ #define IEEE80211_DTIM_DEFAULT 1 /* default DTIM period */ #define IEEE80211_BINTVAL_DEFAULT 100 /* default beacon interval (TU's) */ #define IEEE80211_BMISS_MAX 2 /* maximum consecutive bmiss allowed */ #define IEEE80211_HWBMISS_DEFAULT 7 /* h/w bmiss threshold (beacons) */ #define IEEE80211_BGSCAN_INTVAL_MIN 15 /* min bg scan intvl (secs) */ #define IEEE80211_BGSCAN_INTVAL_DEFAULT (5*60) /* default bg scan intvl */ #define IEEE80211_BGSCAN_IDLE_MIN 100 /* min idle time (ms) */ #define IEEE80211_BGSCAN_IDLE_DEFAULT 250 /* default idle time (ms) */ #define IEEE80211_SCAN_VALID_MIN 10 /* min scan valid time (secs) */ #define IEEE80211_SCAN_VALID_DEFAULT 60 /* default scan valid time */ #define IEEE80211_PS_SLEEP 0x1 /* STA is in power saving mode */ #define IEEE80211_PS_MAX_QUEUE 50 /* maximum saved packets */ #define IEEE80211_FIXED_RATE_NONE 0xff #define IEEE80211_TXMAX_DEFAULT 6 /* default ucast max retries */ #define IEEE80211_RTS_DEFAULT IEEE80211_RTS_MAX #define IEEE80211_FRAG_DEFAULT IEEE80211_FRAG_MAX #define IEEE80211_MS_TO_TU(x) (((x) * 1000) / 1024) #define IEEE80211_TU_TO_MS(x) (((x) * 1024) / 1000) /* XXX TODO: cap this at 1, in case hz is not 1000 */ #define IEEE80211_TU_TO_TICKS(x)(((uint64_t)(x) * 1024 * hz) / (1000 * 1000)) /* * 802.11 control state is split into a common portion that maps * 1-1 to a physical device and one or more "Virtual AP's" (VAP) * that are bound to an ieee80211com instance and share a single * underlying device. Each VAP has a corresponding OS device * entity through which traffic flows and that applications use * for issuing ioctls, etc. */ /* * Data common to one or more virtual AP's. State shared by * the underlying device and the net80211 layer is exposed here; * e.g. device-specific callbacks. */ struct ieee80211vap; typedef void (*ieee80211vap_attach)(struct ieee80211vap *); struct ieee80211_appie { uint16_t ie_len; /* size of ie_data */ uint8_t ie_data[]; /* user-specified IE's */ }; struct ieee80211_tdma_param; struct ieee80211_rate_table; struct ieee80211_tx_ampdu; struct ieee80211_rx_ampdu; struct ieee80211_superg; struct ieee80211_frame; struct ieee80211com { void *ic_softc; /* driver softc */ const char *ic_name; /* usually device name */ ieee80211_com_lock_t ic_comlock; /* state update lock */ ieee80211_tx_lock_t ic_txlock; /* ic/vap TX lock */ LIST_ENTRY(ieee80211com) ic_next; /* on global list */ TAILQ_HEAD(, ieee80211vap) ic_vaps; /* list of vap instances */ int ic_headroom; /* driver tx headroom needs */ enum ieee80211_phytype ic_phytype; /* XXX wrong for multi-mode */ enum ieee80211_opmode ic_opmode; /* operation mode */ struct callout ic_inact; /* inactivity processing */ struct taskqueue *ic_tq; /* deferred state thread */ struct task ic_parent_task; /* deferred parent processing */ struct task ic_promisc_task;/* deferred promisc update */ struct task ic_mcast_task; /* deferred mcast update */ struct task ic_chan_task; /* deferred channel change */ struct task ic_bmiss_task; /* deferred beacon miss hndlr */ struct task ic_chw_task; /* deferred HT CHW update */ struct task ic_wme_task; /* deferred WME update */ struct task ic_restart_task; /* deferred device restart */ counter_u64_t ic_ierrors; /* input errors */ counter_u64_t ic_oerrors; /* output errors */ uint32_t ic_flags; /* state flags */ uint32_t ic_flags_ext; /* extended state flags */ uint32_t ic_flags_ht; /* HT state flags */ uint32_t ic_flags_ven; /* vendor state flags */ uint32_t ic_caps; /* capabilities */ uint32_t ic_htcaps; /* HT capabilities */ uint32_t ic_htextcaps; /* HT extended capabilities */ uint32_t ic_cryptocaps; /* crypto capabilities */ - uint8_t ic_modecaps[2]; /* set of mode capabilities */ + /* set of mode capabilities */ + uint8_t ic_modecaps[IEEE80211_MODE_BYTES]; uint8_t ic_promisc; /* vap's needing promisc mode */ uint8_t ic_allmulti; /* vap's needing all multicast*/ uint8_t ic_nrunning; /* vap's marked running */ uint8_t ic_curmode; /* current mode */ uint8_t ic_macaddr[IEEE80211_ADDR_LEN]; uint16_t ic_bintval; /* beacon interval */ uint16_t ic_lintval; /* listen interval */ uint16_t ic_holdover; /* PM hold over duration */ uint16_t ic_txpowlimit; /* global tx power limit */ struct ieee80211_rateset ic_sup_rates[IEEE80211_MODE_MAX]; /* * Channel state: * * ic_channels is the set of available channels for the device; * it is setup by the driver * ic_nchans is the number of valid entries in ic_channels * ic_chan_avail is a bit vector of these channels used to check * whether a channel is available w/o searching the channel table. * ic_chan_active is a (potentially) constrained subset of * ic_chan_avail that reflects any mode setting or user-specified * limit on the set of channels to use/scan * ic_curchan is the current channel the device is set to; it may * be different from ic_bsschan when we are off-channel scanning * or otherwise doing background work * ic_bsschan is the channel selected for operation; it may * be undefined (IEEE80211_CHAN_ANYC) * ic_prevchan is a cached ``previous channel'' used to optimize * lookups when switching back+forth between two channels * (e.g. for dynamic turbo) */ int ic_nchans; /* # entries in ic_channels */ struct ieee80211_channel ic_channels[IEEE80211_CHAN_MAX]; uint8_t ic_chan_avail[IEEE80211_CHAN_BYTES]; uint8_t ic_chan_active[IEEE80211_CHAN_BYTES]; uint8_t ic_chan_scan[IEEE80211_CHAN_BYTES]; struct ieee80211_channel *ic_curchan; /* current channel */ const struct ieee80211_rate_table *ic_rt; /* table for ic_curchan */ struct ieee80211_channel *ic_bsschan; /* bss channel */ struct ieee80211_channel *ic_prevchan; /* previous channel */ struct ieee80211_regdomain ic_regdomain;/* regulatory data */ struct ieee80211_appie *ic_countryie; /* calculated country ie */ struct ieee80211_channel *ic_countryie_chan; /* 802.11h/DFS state */ struct ieee80211_channel *ic_csa_newchan;/* channel for doing CSA */ short ic_csa_mode; /* mode for doing CSA */ short ic_csa_count; /* count for doing CSA */ struct ieee80211_dfs_state ic_dfs; /* DFS state */ struct ieee80211_scan_state *ic_scan; /* scan state */ struct ieee80211_scan_methods *ic_scan_methods; /* scan methods */ int ic_lastdata; /* time of last data frame */ int ic_lastscan; /* time last scan completed */ /* NB: this is the union of all vap stations/neighbors */ int ic_max_keyix; /* max h/w key index */ struct ieee80211_node_table ic_sta; /* stations/neighbors */ struct ieee80211_ageq ic_stageq; /* frame staging queue */ uint32_t ic_hash_key; /* random key for mac hash */ /* XXX multi-bss: split out common/vap parts */ struct ieee80211_wme_state ic_wme; /* WME/WMM state */ /* XXX multi-bss: can per-vap be done/make sense? */ enum ieee80211_protmode ic_protmode; /* 802.11g protection mode */ uint16_t ic_nonerpsta; /* # non-ERP stations */ uint16_t ic_longslotsta; /* # long slot time stations */ uint16_t ic_sta_assoc; /* stations associated */ uint16_t ic_ht_sta_assoc;/* HT stations associated */ uint16_t ic_ht40_sta_assoc;/* HT40 stations associated */ uint8_t ic_curhtprotmode;/* HTINFO bss state */ enum ieee80211_protmode ic_htprotmode; /* HT protection mode */ int ic_lastnonerp; /* last time non-ERP sta noted*/ int ic_lastnonht; /* last time non-HT sta noted */ uint8_t ic_rxstream; /* # RX streams */ uint8_t ic_txstream; /* # TX streams */ /* optional state for Atheros SuperG protocol extensions */ struct ieee80211_superg *ic_superg; /* radiotap handling */ struct ieee80211_radiotap_header *ic_th;/* tx radiotap headers */ void *ic_txchan; /* channel state in ic_th */ struct ieee80211_radiotap_header *ic_rh;/* rx radiotap headers */ void *ic_rxchan; /* channel state in ic_rh */ int ic_montaps; /* active monitor mode taps */ /* virtual ap create/delete */ struct ieee80211vap* (*ic_vap_create)(struct ieee80211com *, const char [IFNAMSIZ], int, enum ieee80211_opmode, int, const uint8_t [IEEE80211_ADDR_LEN], const uint8_t [IEEE80211_ADDR_LEN]); void (*ic_vap_delete)(struct ieee80211vap *); /* device specific ioctls */ int (*ic_ioctl)(struct ieee80211com *, u_long, void *); /* start/stop device */ void (*ic_parent)(struct ieee80211com *); /* operating mode attachment */ ieee80211vap_attach ic_vattach[IEEE80211_OPMODE_MAX]; /* return hardware/radio capabilities */ void (*ic_getradiocaps)(struct ieee80211com *, int, int *, struct ieee80211_channel []); /* check and/or prepare regdomain state change */ int (*ic_setregdomain)(struct ieee80211com *, struct ieee80211_regdomain *, int, struct ieee80211_channel []); int (*ic_set_quiet)(struct ieee80211_node *, u_int8_t *quiet_elm); /* regular transmit */ int (*ic_transmit)(struct ieee80211com *, struct mbuf *); /* send/recv 802.11 management frame */ int (*ic_send_mgmt)(struct ieee80211_node *, int, int); /* send raw 802.11 frame */ int (*ic_raw_xmit)(struct ieee80211_node *, struct mbuf *, const struct ieee80211_bpf_params *); /* update device state for 802.11 slot time change */ void (*ic_updateslot)(struct ieee80211com *); /* handle multicast state changes */ void (*ic_update_mcast)(struct ieee80211com *); /* handle promiscuous mode changes */ void (*ic_update_promisc)(struct ieee80211com *); /* new station association callback/notification */ void (*ic_newassoc)(struct ieee80211_node *, int); /* TDMA update notification */ void (*ic_tdma_update)(struct ieee80211_node *, const struct ieee80211_tdma_param *, int); /* node state management */ struct ieee80211_node* (*ic_node_alloc)(struct ieee80211vap *, const uint8_t [IEEE80211_ADDR_LEN]); void (*ic_node_free)(struct ieee80211_node *); void (*ic_node_cleanup)(struct ieee80211_node *); void (*ic_node_age)(struct ieee80211_node *); void (*ic_node_drain)(struct ieee80211_node *); int8_t (*ic_node_getrssi)(const struct ieee80211_node*); void (*ic_node_getsignal)(const struct ieee80211_node*, int8_t *, int8_t *); void (*ic_node_getmimoinfo)( const struct ieee80211_node*, struct ieee80211_mimo_info *); /* scanning support */ void (*ic_scan_start)(struct ieee80211com *); void (*ic_scan_end)(struct ieee80211com *); void (*ic_set_channel)(struct ieee80211com *); void (*ic_scan_curchan)(struct ieee80211_scan_state *, unsigned long); void (*ic_scan_mindwell)(struct ieee80211_scan_state *); /* * 802.11n ADDBA support. A simple/generic implementation * of A-MPDU tx aggregation is provided; the driver may * override these methods to provide their own support. * A-MPDU rx re-ordering happens automatically if the * driver passes out-of-order frames to ieee80211_input * from an assocated HT station. */ int (*ic_recv_action)(struct ieee80211_node *, const struct ieee80211_frame *, const uint8_t *frm, const uint8_t *efrm); int (*ic_send_action)(struct ieee80211_node *, int category, int action, void *); /* check if A-MPDU should be enabled this station+ac */ int (*ic_ampdu_enable)(struct ieee80211_node *, struct ieee80211_tx_ampdu *); /* start/stop doing A-MPDU tx aggregation for a station */ int (*ic_addba_request)(struct ieee80211_node *, struct ieee80211_tx_ampdu *, int dialogtoken, int baparamset, int batimeout); int (*ic_addba_response)(struct ieee80211_node *, struct ieee80211_tx_ampdu *, int status, int baparamset, int batimeout); void (*ic_addba_stop)(struct ieee80211_node *, struct ieee80211_tx_ampdu *); void (*ic_addba_response_timeout)(struct ieee80211_node *, struct ieee80211_tx_ampdu *); /* BAR response received */ void (*ic_bar_response)(struct ieee80211_node *, struct ieee80211_tx_ampdu *, int status); /* start/stop doing A-MPDU rx processing for a station */ int (*ic_ampdu_rx_start)(struct ieee80211_node *, struct ieee80211_rx_ampdu *, int baparamset, int batimeout, int baseqctl); void (*ic_ampdu_rx_stop)(struct ieee80211_node *, struct ieee80211_rx_ampdu *); /* The channel width has changed (20<->2040) */ void (*ic_update_chw)(struct ieee80211com *); uint64_t ic_spare[7]; }; struct ieee80211_aclator; struct ieee80211_tdma_state; struct ieee80211_mesh_state; struct ieee80211_hwmp_state; struct ieee80211vap { struct ifmedia iv_media; /* interface media config */ struct ifnet *iv_ifp; /* associated device */ struct bpf_if *iv_rawbpf; /* packet filter structure */ struct sysctl_ctx_list *iv_sysctl; /* dynamic sysctl context */ struct sysctl_oid *iv_oid; /* net.wlan.X sysctl oid */ TAILQ_ENTRY(ieee80211vap) iv_next; /* list of vap instances */ struct ieee80211com *iv_ic; /* back ptr to common state */ /* MAC address: ifp or ic */ uint8_t iv_myaddr[IEEE80211_ADDR_LEN]; uint32_t iv_debug; /* debug msg flags */ struct ieee80211_stats iv_stats; /* statistics */ uint32_t iv_flags; /* state flags */ uint32_t iv_flags_ext; /* extended state flags */ uint32_t iv_flags_ht; /* HT state flags */ uint32_t iv_flags_ven; /* vendor state flags */ uint32_t iv_ifflags; /* ifnet flags */ uint32_t iv_caps; /* capabilities */ uint32_t iv_htcaps; /* HT capabilities */ uint32_t iv_htextcaps; /* HT extended capabilities */ enum ieee80211_opmode iv_opmode; /* operation mode */ enum ieee80211_state iv_state; /* state machine state */ enum ieee80211_state iv_nstate; /* pending state */ int iv_nstate_arg; /* pending state arg */ struct task iv_nstate_task; /* deferred state processing */ struct task iv_swbmiss_task;/* deferred iv_bmiss call */ struct callout iv_mgtsend; /* mgmt frame response timer */ /* inactivity timer settings */ int iv_inact_init; /* setting for new station */ int iv_inact_auth; /* auth but not assoc setting */ int iv_inact_run; /* authorized setting */ int iv_inact_probe; /* inactive probe time */ int iv_des_nssid; /* # desired ssids */ struct ieee80211_scan_ssid iv_des_ssid[1];/* desired ssid table */ uint8_t iv_des_bssid[IEEE80211_ADDR_LEN]; struct ieee80211_channel *iv_des_chan; /* desired channel */ uint16_t iv_des_mode; /* desired mode */ int iv_nicknamelen; /* XXX junk */ uint8_t iv_nickname[IEEE80211_NWID_LEN]; u_int iv_bgscanidle; /* bg scan idle threshold */ u_int iv_bgscanintvl; /* bg scan min interval */ u_int iv_scanvalid; /* scan cache valid threshold */ u_int iv_scanreq_duration; u_int iv_scanreq_mindwell; u_int iv_scanreq_maxdwell; uint16_t iv_scanreq_flags;/* held scan request params */ uint8_t iv_scanreq_nssid; struct ieee80211_scan_ssid iv_scanreq_ssid[IEEE80211_SCAN_MAX_SSID]; /* sta-mode roaming state */ enum ieee80211_roamingmode iv_roaming; /* roaming mode */ struct ieee80211_roamparam iv_roamparms[IEEE80211_MODE_MAX]; uint8_t iv_bmissthreshold; uint8_t iv_bmiss_count; /* current beacon miss count */ int iv_bmiss_max; /* max bmiss before scan */ uint16_t iv_swbmiss_count;/* beacons in last period */ uint16_t iv_swbmiss_period;/* s/w bmiss period */ struct callout iv_swbmiss; /* s/w beacon miss timer */ int iv_ampdu_rxmax; /* A-MPDU rx limit (bytes) */ int iv_ampdu_density;/* A-MPDU density */ int iv_ampdu_limit; /* A-MPDU tx limit (bytes) */ int iv_amsdu_limit; /* A-MSDU tx limit (bytes) */ u_int iv_ampdu_mintraffic[WME_NUM_AC]; struct ieee80211_beacon_offsets iv_bcn_off; uint32_t *iv_aid_bitmap; /* association id map */ uint16_t iv_max_aid; uint16_t iv_sta_assoc; /* stations associated */ uint16_t iv_ps_sta; /* stations in power save */ uint16_t iv_ps_pending; /* ps sta's w/ pending frames */ uint16_t iv_txseq; /* mcast xmit seq# space */ uint16_t iv_tim_len; /* ic_tim_bitmap size (bytes) */ uint8_t *iv_tim_bitmap; /* power-save stations w/ data*/ uint8_t iv_dtim_period; /* DTIM period */ uint8_t iv_dtim_count; /* DTIM count from last bcn */ /* set/unset aid pwrsav state */ uint8_t iv_quiet; /* Quiet Element */ uint8_t iv_quiet_count; /* constant count for Quiet Element */ uint8_t iv_quiet_count_value; /* variable count for Quiet Element */ uint8_t iv_quiet_period; /* period for Quiet Element */ uint16_t iv_quiet_duration; /* duration for Quiet Element */ uint16_t iv_quiet_offset; /* offset for Quiet Element */ int iv_csa_count; /* count for doing CSA */ struct ieee80211_node *iv_bss; /* information for this node */ struct ieee80211_txparam iv_txparms[IEEE80211_MODE_MAX]; uint16_t iv_rtsthreshold; uint16_t iv_fragthreshold; int iv_inact_timer; /* inactivity timer wait */ /* application-specified IE's to attach to mgt frames */ struct ieee80211_appie *iv_appie_beacon; struct ieee80211_appie *iv_appie_probereq; struct ieee80211_appie *iv_appie_proberesp; struct ieee80211_appie *iv_appie_assocreq; struct ieee80211_appie *iv_appie_assocresp; struct ieee80211_appie *iv_appie_wpa; uint8_t *iv_wpa_ie; uint8_t *iv_rsn_ie; uint16_t iv_max_keyix; /* max h/w key index */ ieee80211_keyix iv_def_txkey; /* default/group tx key index */ struct ieee80211_key iv_nw_keys[IEEE80211_WEP_NKID]; int (*iv_key_alloc)(struct ieee80211vap *, struct ieee80211_key *, ieee80211_keyix *, ieee80211_keyix *); int (*iv_key_delete)(struct ieee80211vap *, const struct ieee80211_key *); int (*iv_key_set)(struct ieee80211vap *, const struct ieee80211_key *); void (*iv_key_update_begin)(struct ieee80211vap *); void (*iv_key_update_end)(struct ieee80211vap *); const struct ieee80211_authenticator *iv_auth; /* authenticator glue */ void *iv_ec; /* private auth state */ const struct ieee80211_aclator *iv_acl; /* acl glue */ void *iv_as; /* private aclator state */ const struct ieee80211_ratectl *iv_rate; void *iv_rs; /* private ratectl state */ struct ieee80211_tdma_state *iv_tdma; /* tdma state */ struct ieee80211_mesh_state *iv_mesh; /* MBSS state */ struct ieee80211_hwmp_state *iv_hwmp; /* HWMP state */ /* operate-mode detach hook */ void (*iv_opdetach)(struct ieee80211vap *); /* receive processing */ int (*iv_input)(struct ieee80211_node *, struct mbuf *, const struct ieee80211_rx_stats *, int, int); void (*iv_recv_mgmt)(struct ieee80211_node *, struct mbuf *, int, const struct ieee80211_rx_stats *, int, int); void (*iv_recv_ctl)(struct ieee80211_node *, struct mbuf *, int); void (*iv_deliver_data)(struct ieee80211vap *, struct ieee80211_node *, struct mbuf *); #if 0 /* send processing */ int (*iv_send_mgmt)(struct ieee80211_node *, int, int); #endif /* beacon miss processing */ void (*iv_bmiss)(struct ieee80211vap *); /* reset device state after 802.11 parameter/state change */ int (*iv_reset)(struct ieee80211vap *, u_long); /* [schedule] beacon frame update */ void (*iv_update_beacon)(struct ieee80211vap *, int); /* power save handling */ void (*iv_update_ps)(struct ieee80211vap *, int); int (*iv_set_tim)(struct ieee80211_node *, int); void (*iv_node_ps)(struct ieee80211_node *, int); void (*iv_sta_ps)(struct ieee80211vap *, int); void (*iv_recv_pspoll)(struct ieee80211_node *, struct mbuf *); /* state machine processing */ int (*iv_newstate)(struct ieee80211vap *, enum ieee80211_state, int); /* 802.3 output method for raw frame xmit */ int (*iv_output)(struct ifnet *, struct mbuf *, const struct sockaddr *, struct route *); uint64_t iv_spare[6]; }; MALLOC_DECLARE(M_80211_VAP); #define IEEE80211_ADDR_EQ(a1,a2) (memcmp(a1,a2,IEEE80211_ADDR_LEN) == 0) #define IEEE80211_ADDR_COPY(dst,src) memcpy(dst,src,IEEE80211_ADDR_LEN) /* ic_flags/iv_flags */ #define IEEE80211_F_TURBOP 0x00000001 /* CONF: ATH Turbo enabled*/ #define IEEE80211_F_COMP 0x00000002 /* CONF: ATH comp enabled */ #define IEEE80211_F_FF 0x00000004 /* CONF: ATH FF enabled */ #define IEEE80211_F_BURST 0x00000008 /* CONF: bursting enabled */ /* NB: this is intentionally setup to be IEEE80211_CAPINFO_PRIVACY */ #define IEEE80211_F_PRIVACY 0x00000010 /* CONF: privacy enabled */ #define IEEE80211_F_PUREG 0x00000020 /* CONF: 11g w/o 11b sta's */ #define IEEE80211_F_SCAN 0x00000080 /* STATUS: scanning */ #define IEEE80211_F_ASCAN 0x00000100 /* STATUS: active scan */ #define IEEE80211_F_SIBSS 0x00000200 /* STATUS: start IBSS */ /* NB: this is intentionally setup to be IEEE80211_CAPINFO_SHORT_SLOTTIME */ #define IEEE80211_F_SHSLOT 0x00000400 /* STATUS: use short slot time*/ #define IEEE80211_F_PMGTON 0x00000800 /* CONF: Power mgmt enable */ #define IEEE80211_F_DESBSSID 0x00001000 /* CONF: des_bssid is set */ #define IEEE80211_F_WME 0x00002000 /* CONF: enable WME use */ #define IEEE80211_F_BGSCAN 0x00004000 /* CONF: bg scan enabled (???)*/ #define IEEE80211_F_SWRETRY 0x00008000 /* CONF: sw tx retry enabled */ #define IEEE80211_F_TXPOW_FIXED 0x00010000 /* TX Power: fixed rate */ #define IEEE80211_F_IBSSON 0x00020000 /* CONF: IBSS creation enable */ #define IEEE80211_F_SHPREAMBLE 0x00040000 /* STATUS: use short preamble */ #define IEEE80211_F_DATAPAD 0x00080000 /* CONF: do alignment pad */ #define IEEE80211_F_USEPROT 0x00100000 /* STATUS: protection enabled */ #define IEEE80211_F_USEBARKER 0x00200000 /* STATUS: use barker preamble*/ #define IEEE80211_F_CSAPENDING 0x00400000 /* STATUS: chan switch pending*/ #define IEEE80211_F_WPA1 0x00800000 /* CONF: WPA enabled */ #define IEEE80211_F_WPA2 0x01000000 /* CONF: WPA2 enabled */ #define IEEE80211_F_WPA 0x01800000 /* CONF: WPA/WPA2 enabled */ #define IEEE80211_F_DROPUNENC 0x02000000 /* CONF: drop unencrypted */ #define IEEE80211_F_COUNTERM 0x04000000 /* CONF: TKIP countermeasures */ #define IEEE80211_F_HIDESSID 0x08000000 /* CONF: hide SSID in beacon */ #define IEEE80211_F_NOBRIDGE 0x10000000 /* CONF: dis. internal bridge */ #define IEEE80211_F_PCF 0x20000000 /* CONF: PCF enabled */ #define IEEE80211_F_DOTH 0x40000000 /* CONF: 11h enabled */ #define IEEE80211_F_DWDS 0x80000000 /* CONF: Dynamic WDS enabled */ #define IEEE80211_F_BITS \ "\20\1TURBOP\2COMP\3FF\4BURST\5PRIVACY\6PUREG\10SCAN\11ASCAN\12SIBSS" \ "\13SHSLOT\14PMGTON\15DESBSSID\16WME\17BGSCAN\20SWRETRY\21TXPOW_FIXED" \ "\22IBSSON\23SHPREAMBLE\24DATAPAD\25USEPROT\26USERBARKER\27CSAPENDING" \ "\30WPA1\31WPA2\32DROPUNENC\33COUNTERM\34HIDESSID\35NOBRIDG\36PCF" \ "\37DOTH\40DWDS" /* Atheros protocol-specific flags */ #define IEEE80211_F_ATHEROS \ (IEEE80211_F_FF | IEEE80211_F_COMP | IEEE80211_F_TURBOP) /* Check if an Atheros capability was negotiated for use */ #define IEEE80211_ATH_CAP(vap, ni, bit) \ ((vap)->iv_flags & (ni)->ni_ath_flags & (bit)) /* ic_flags_ext/iv_flags_ext */ #define IEEE80211_FEXT_INACT 0x00000002 /* CONF: sta inact handling */ #define IEEE80211_FEXT_SCANWAIT 0x00000004 /* STATUS: awaiting scan */ /* 0x00000006 reserved */ #define IEEE80211_FEXT_BGSCAN 0x00000008 /* STATUS: complete bgscan */ #define IEEE80211_FEXT_WPS 0x00000010 /* CONF: WPS enabled */ #define IEEE80211_FEXT_TSN 0x00000020 /* CONF: TSN enabled */ #define IEEE80211_FEXT_SCANREQ 0x00000040 /* STATUS: scan req params */ #define IEEE80211_FEXT_RESUME 0x00000080 /* STATUS: start on resume */ #define IEEE80211_FEXT_4ADDR 0x00000100 /* CONF: apply 4-addr encap */ #define IEEE80211_FEXT_NONERP_PR 0x00000200 /* STATUS: non-ERP sta present*/ #define IEEE80211_FEXT_SWBMISS 0x00000400 /* CONF: do bmiss in s/w */ #define IEEE80211_FEXT_DFS 0x00000800 /* CONF: DFS enabled */ #define IEEE80211_FEXT_DOTD 0x00001000 /* CONF: 11d enabled */ #define IEEE80211_FEXT_STATEWAIT 0x00002000 /* STATUS: awaiting state chg */ #define IEEE80211_FEXT_REINIT 0x00004000 /* STATUS: INIT state first */ #define IEEE80211_FEXT_BPF 0x00008000 /* STATUS: BPF tap present */ /* NB: immutable: should be set only when creating a vap */ #define IEEE80211_FEXT_WDSLEGACY 0x00010000 /* CONF: legacy WDS operation */ #define IEEE80211_FEXT_PROBECHAN 0x00020000 /* CONF: probe passive channel*/ #define IEEE80211_FEXT_UNIQMAC 0x00040000 /* CONF: user or computed mac */ #define IEEE80211_FEXT_BITS \ "\20\2INACT\3SCANWAIT\4BGSCAN\5WPS\6TSN\7SCANREQ\10RESUME" \ "\0114ADDR\12NONEPR_PR\13SWBMISS\14DFS\15DOTD\16STATEWAIT\17REINIT" \ "\20BPF\21WDSLEGACY\22PROBECHAN\23UNIQMAC" /* ic_flags_ht/iv_flags_ht */ #define IEEE80211_FHT_NONHT_PR 0x00000001 /* STATUS: non-HT sta present */ #define IEEE80211_FHT_GF 0x00040000 /* CONF: Greenfield enabled */ #define IEEE80211_FHT_HT 0x00080000 /* CONF: HT supported */ #define IEEE80211_FHT_AMPDU_TX 0x00100000 /* CONF: A-MPDU tx supported */ #define IEEE80211_FHT_AMPDU_RX 0x00200000 /* CONF: A-MPDU rx supported */ #define IEEE80211_FHT_AMSDU_TX 0x00400000 /* CONF: A-MSDU tx supported */ #define IEEE80211_FHT_AMSDU_RX 0x00800000 /* CONF: A-MSDU rx supported */ #define IEEE80211_FHT_USEHT40 0x01000000 /* CONF: 20/40 use enabled */ #define IEEE80211_FHT_PUREN 0x02000000 /* CONF: 11n w/o legacy sta's */ #define IEEE80211_FHT_SHORTGI20 0x04000000 /* CONF: short GI in HT20 */ #define IEEE80211_FHT_SHORTGI40 0x08000000 /* CONF: short GI in HT40 */ #define IEEE80211_FHT_HTCOMPAT 0x10000000 /* CONF: HT vendor OUI's */ #define IEEE80211_FHT_RIFS 0x20000000 /* CONF: RIFS enabled */ #define IEEE80211_FHT_STBC_TX 0x40000000 /* CONF: STBC tx enabled */ #define IEEE80211_FHT_STBC_RX 0x80000000 /* CONF: STBC rx enabled */ #define IEEE80211_FHT_BITS \ "\20\1NONHT_PR" \ "\23GF\24HT\25AMPDU_TX\26AMPDU_TX" \ "\27AMSDU_TX\30AMSDU_RX\31USEHT40\32PUREN\33SHORTGI20\34SHORTGI40" \ "\35HTCOMPAT\36RIFS\37STBC_TX\40STBC_RX" #define IEEE80211_FVEN_BITS "\20" /* ic_caps/iv_caps: device driver capabilities */ /* 0x2e available */ #define IEEE80211_C_STA 0x00000001 /* CAPABILITY: STA available */ #define IEEE80211_C_8023ENCAP 0x00000002 /* CAPABILITY: 802.3 encap */ #define IEEE80211_C_FF 0x00000040 /* CAPABILITY: ATH FF avail */ #define IEEE80211_C_TURBOP 0x00000080 /* CAPABILITY: ATH Turbo avail*/ #define IEEE80211_C_IBSS 0x00000100 /* CAPABILITY: IBSS available */ #define IEEE80211_C_PMGT 0x00000200 /* CAPABILITY: Power mgmt */ #define IEEE80211_C_HOSTAP 0x00000400 /* CAPABILITY: HOSTAP avail */ #define IEEE80211_C_AHDEMO 0x00000800 /* CAPABILITY: Old Adhoc Demo */ #define IEEE80211_C_SWRETRY 0x00001000 /* CAPABILITY: sw tx retry */ #define IEEE80211_C_TXPMGT 0x00002000 /* CAPABILITY: tx power mgmt */ #define IEEE80211_C_SHSLOT 0x00004000 /* CAPABILITY: short slottime */ #define IEEE80211_C_SHPREAMBLE 0x00008000 /* CAPABILITY: short preamble */ #define IEEE80211_C_MONITOR 0x00010000 /* CAPABILITY: monitor mode */ #define IEEE80211_C_DFS 0x00020000 /* CAPABILITY: DFS/radar avail*/ #define IEEE80211_C_MBSS 0x00040000 /* CAPABILITY: MBSS available */ #define IEEE80211_C_SWSLEEP 0x00080000 /* CAPABILITY: do sleep here */ #define IEEE80211_C_SWAMSDUTX 0x00100000 /* CAPABILITY: software A-MSDU TX */ /* 0x7c0000 available */ #define IEEE80211_C_WPA1 0x00800000 /* CAPABILITY: WPA1 avail */ #define IEEE80211_C_WPA2 0x01000000 /* CAPABILITY: WPA2 avail */ #define IEEE80211_C_WPA 0x01800000 /* CAPABILITY: WPA1+WPA2 avail*/ #define IEEE80211_C_BURST 0x02000000 /* CAPABILITY: frame bursting */ #define IEEE80211_C_WME 0x04000000 /* CAPABILITY: WME avail */ #define IEEE80211_C_WDS 0x08000000 /* CAPABILITY: 4-addr support */ /* 0x10000000 reserved */ #define IEEE80211_C_BGSCAN 0x20000000 /* CAPABILITY: bg scanning */ #define IEEE80211_C_TXFRAG 0x40000000 /* CAPABILITY: tx fragments */ #define IEEE80211_C_TDMA 0x80000000 /* CAPABILITY: TDMA avail */ /* XXX protection/barker? */ #define IEEE80211_C_OPMODE \ (IEEE80211_C_STA | IEEE80211_C_IBSS | IEEE80211_C_HOSTAP | \ IEEE80211_C_AHDEMO | IEEE80211_C_MONITOR | IEEE80211_C_WDS | \ IEEE80211_C_TDMA | IEEE80211_C_MBSS) #define IEEE80211_C_BITS \ "\20\1STA\002803ENCAP\7FF\10TURBOP\11IBSS\12PMGT" \ "\13HOSTAP\14AHDEMO\15SWRETRY\16TXPMGT\17SHSLOT\20SHPREAMBLE" \ "\21MONITOR\22DFS\23MBSS\30WPA1\31WPA2\32BURST\33WME\34WDS\36BGSCAN" \ "\37TXFRAG\40TDMA" /* * ic_htcaps/iv_htcaps: HT-specific device/driver capabilities * * NB: the low 16-bits are the 802.11 definitions, the upper * 16-bits are used to define s/w/driver capabilities. */ #define IEEE80211_HTC_AMPDU 0x00010000 /* CAPABILITY: A-MPDU tx */ #define IEEE80211_HTC_AMSDU 0x00020000 /* CAPABILITY: A-MSDU tx */ /* NB: HT40 is implied by IEEE80211_HTCAP_CHWIDTH40 */ #define IEEE80211_HTC_HT 0x00040000 /* CAPABILITY: HT operation */ #define IEEE80211_HTC_SMPS 0x00080000 /* CAPABILITY: MIMO power save*/ #define IEEE80211_HTC_RIFS 0x00100000 /* CAPABILITY: RIFS support */ #define IEEE80211_HTC_RXUNEQUAL 0x00200000 /* CAPABILITY: RX unequal MCS */ #define IEEE80211_HTC_RXMCS32 0x00400000 /* CAPABILITY: MCS32 support */ #define IEEE80211_HTC_TXUNEQUAL 0x00800000 /* CAPABILITY: TX unequal MCS */ #define IEEE80211_HTC_TXMCS32 0x01000000 /* CAPABILITY: MCS32 suport */ #define IEEE80211_C_HTCAP_BITS \ "\20\1LDPC\2CHWIDTH40\5GREENFIELD\6SHORTGI20\7SHORTGI40\10TXSTBC" \ "\21AMPDU\22AMSDU\23HT\24SMPS\25RIFS" int ic_printf(struct ieee80211com *, const char *, ...) __printflike(2, 3); void ieee80211_ifattach(struct ieee80211com *); void ieee80211_ifdetach(struct ieee80211com *); int ieee80211_vap_setup(struct ieee80211com *, struct ieee80211vap *, const char name[IFNAMSIZ], int unit, enum ieee80211_opmode opmode, int flags, const uint8_t bssid[IEEE80211_ADDR_LEN]); int ieee80211_vap_attach(struct ieee80211vap *, ifm_change_cb_t, ifm_stat_cb_t, const uint8_t macaddr[IEEE80211_ADDR_LEN]); void ieee80211_vap_detach(struct ieee80211vap *); const struct ieee80211_rateset *ieee80211_get_suprates(struct ieee80211com *ic, const struct ieee80211_channel *); void ieee80211_announce(struct ieee80211com *); void ieee80211_announce_channels(struct ieee80211com *); void ieee80211_drain(struct ieee80211com *); void ieee80211_chan_init(struct ieee80211com *); struct ieee80211com *ieee80211_find_vap(const uint8_t mac[IEEE80211_ADDR_LEN]); struct ieee80211com *ieee80211_find_com(const char *name); int ieee80211_media_change(struct ifnet *); void ieee80211_media_status(struct ifnet *, struct ifmediareq *); int ieee80211_ioctl(struct ifnet *, u_long, caddr_t); int ieee80211_rate2media(struct ieee80211com *, int, enum ieee80211_phymode); int ieee80211_media2rate(int); int ieee80211_mhz2ieee(u_int, u_int); int ieee80211_chan2ieee(struct ieee80211com *, const struct ieee80211_channel *); u_int ieee80211_ieee2mhz(u_int, u_int); int ieee80211_add_channel(struct ieee80211_channel[], int, int *, uint8_t, uint16_t, int8_t, uint32_t, const uint8_t[]); int ieee80211_add_channel_ht40(struct ieee80211_channel[], int, int *, uint8_t, int8_t, uint32_t); int ieee80211_add_channel_list_2ghz(struct ieee80211_channel[], int, int *, const uint8_t[], int, const uint8_t[], int); int ieee80211_add_channel_list_5ghz(struct ieee80211_channel[], int, int *, const uint8_t[], int, const uint8_t[], int); struct ieee80211_channel *ieee80211_find_channel(struct ieee80211com *, int freq, int flags); struct ieee80211_channel *ieee80211_find_channel_byieee(struct ieee80211com *, int ieee, int flags); struct ieee80211_channel *ieee80211_lookup_channel_rxstatus(struct ieee80211vap *, const struct ieee80211_rx_stats *); int ieee80211_setmode(struct ieee80211com *, enum ieee80211_phymode); enum ieee80211_phymode ieee80211_chan2mode(const struct ieee80211_channel *); uint32_t ieee80211_mac_hash(const struct ieee80211com *, const uint8_t addr[IEEE80211_ADDR_LEN]); char ieee80211_channel_type_char(const struct ieee80211_channel *c); void ieee80211_radiotap_attach(struct ieee80211com *, struct ieee80211_radiotap_header *th, int tlen, uint32_t tx_radiotap, struct ieee80211_radiotap_header *rh, int rlen, uint32_t rx_radiotap); void ieee80211_radiotap_attachv(struct ieee80211com *, struct ieee80211_radiotap_header *th, int tlen, int n_tx_v, uint32_t tx_radiotap, struct ieee80211_radiotap_header *rh, int rlen, int n_rx_v, uint32_t rx_radiotap); void ieee80211_radiotap_detach(struct ieee80211com *); void ieee80211_radiotap_vattach(struct ieee80211vap *); void ieee80211_radiotap_vdetach(struct ieee80211vap *); void ieee80211_radiotap_chan_change(struct ieee80211com *); void ieee80211_radiotap_tx(struct ieee80211vap *, struct mbuf *); void ieee80211_radiotap_rx(struct ieee80211vap *, struct mbuf *); void ieee80211_radiotap_rx_all(struct ieee80211com *, struct mbuf *); static __inline int ieee80211_radiotap_active(const struct ieee80211com *ic) { return (ic->ic_flags_ext & IEEE80211_FEXT_BPF) != 0; } static __inline int ieee80211_radiotap_active_vap(const struct ieee80211vap *vap) { return (vap->iv_flags_ext & IEEE80211_FEXT_BPF) || vap->iv_ic->ic_montaps != 0; } /* * Enqueue a task on the state thread. */ static __inline void ieee80211_runtask(struct ieee80211com *ic, struct task *task) { taskqueue_enqueue(ic->ic_tq, task); } /* * Wait for a queued task to complete. */ static __inline void ieee80211_draintask(struct ieee80211com *ic, struct task *task) { taskqueue_drain(ic->ic_tq, task); } /* * Key update synchronization methods. XXX should not be visible. */ static __inline void ieee80211_key_update_begin(struct ieee80211vap *vap) { vap->iv_key_update_begin(vap); } static __inline void ieee80211_key_update_end(struct ieee80211vap *vap) { vap->iv_key_update_end(vap); } /* * XXX these need to be here for IEEE80211_F_DATAPAD */ /* * Return the space occupied by the 802.11 header and any * padding required by the driver. This works for a * management or data frame. */ static __inline int ieee80211_hdrspace(struct ieee80211com *ic, const void *data) { int size = ieee80211_hdrsize(data); if (ic->ic_flags & IEEE80211_F_DATAPAD) size = roundup(size, sizeof(uint32_t)); return size; } /* * Like ieee80211_hdrspace, but handles any type of frame. */ static __inline int ieee80211_anyhdrspace(struct ieee80211com *ic, const void *data) { int size = ieee80211_anyhdrsize(data); if (ic->ic_flags & IEEE80211_F_DATAPAD) size = roundup(size, sizeof(uint32_t)); return size; } /* * Notify a vap that beacon state has been updated. */ static __inline void ieee80211_beacon_notify(struct ieee80211vap *vap, int what) { if (vap->iv_state == IEEE80211_S_RUN) vap->iv_update_beacon(vap, what); } /* * Calculate HT channel promotion flags for a channel. * XXX belongs in ieee80211_ht.h but needs IEEE80211_FHT_* */ static __inline int ieee80211_htchanflags(const struct ieee80211_channel *c) { return IEEE80211_IS_CHAN_HT40(c) ? IEEE80211_FHT_HT | IEEE80211_FHT_USEHT40 : IEEE80211_IS_CHAN_HT(c) ? IEEE80211_FHT_HT : 0; } /* * Fetch the current TX power (cap) for the given node. * * This includes the node and ic/vap TX power limit as needed, * but it doesn't take into account any per-rate limit. */ static __inline uint16_t ieee80211_get_node_txpower(struct ieee80211_node *ni) { struct ieee80211com *ic = ni->ni_ic; uint16_t txpower; txpower = ni->ni_txpower; txpower = MIN(txpower, ic->ic_txpowlimit); if (ic->ic_curchan != NULL) { txpower = MIN(txpower, 2 * ic->ic_curchan->ic_maxregpower); txpower = MIN(txpower, ic->ic_curchan->ic_maxpower); } return (txpower); } /* * Debugging facilities compiled in when IEEE80211_DEBUG is defined. * * The intent is that any problem in the net80211 layer can be * diagnosed by inspecting the statistics (dumped by the wlanstats * program) and/or the msgs generated by net80211. Messages are * broken into functional classes and can be controlled with the * wlandebug program. Certain of these msg groups are for facilities * that are no longer part of net80211 (e.g. IEEE80211_MSG_DOT1XSM). */ #define IEEE80211_MSG_11N 0x80000000 /* 11n mode debug */ #define IEEE80211_MSG_DEBUG 0x40000000 /* IFF_DEBUG equivalent */ #define IEEE80211_MSG_DUMPPKTS 0x20000000 /* IFF_LINK2 equivalant */ #define IEEE80211_MSG_CRYPTO 0x10000000 /* crypto work */ #define IEEE80211_MSG_INPUT 0x08000000 /* input handling */ #define IEEE80211_MSG_XRATE 0x04000000 /* rate set handling */ #define IEEE80211_MSG_ELEMID 0x02000000 /* element id parsing */ #define IEEE80211_MSG_NODE 0x01000000 /* node handling */ #define IEEE80211_MSG_ASSOC 0x00800000 /* association handling */ #define IEEE80211_MSG_AUTH 0x00400000 /* authentication handling */ #define IEEE80211_MSG_SCAN 0x00200000 /* scanning */ #define IEEE80211_MSG_OUTPUT 0x00100000 /* output handling */ #define IEEE80211_MSG_STATE 0x00080000 /* state machine */ #define IEEE80211_MSG_POWER 0x00040000 /* power save handling */ #define IEEE80211_MSG_HWMP 0x00020000 /* hybrid mesh protocol */ #define IEEE80211_MSG_DOT1XSM 0x00010000 /* 802.1x state machine */ #define IEEE80211_MSG_RADIUS 0x00008000 /* 802.1x radius client */ #define IEEE80211_MSG_RADDUMP 0x00004000 /* dump 802.1x radius packets */ #define IEEE80211_MSG_MESH 0x00002000 /* mesh networking */ #define IEEE80211_MSG_WPA 0x00001000 /* WPA/RSN protocol */ #define IEEE80211_MSG_ACL 0x00000800 /* ACL handling */ #define IEEE80211_MSG_WME 0x00000400 /* WME protocol */ #define IEEE80211_MSG_SUPERG 0x00000200 /* Atheros SuperG protocol */ #define IEEE80211_MSG_DOTH 0x00000100 /* 802.11h support */ #define IEEE80211_MSG_INACT 0x00000080 /* inactivity handling */ #define IEEE80211_MSG_ROAM 0x00000040 /* sta-mode roaming */ #define IEEE80211_MSG_RATECTL 0x00000020 /* tx rate control */ #define IEEE80211_MSG_ACTION 0x00000010 /* action frame handling */ #define IEEE80211_MSG_WDS 0x00000008 /* WDS handling */ #define IEEE80211_MSG_IOCTL 0x00000004 /* ioctl handling */ #define IEEE80211_MSG_TDMA 0x00000002 /* TDMA handling */ #define IEEE80211_MSG_ANY 0xffffffff /* anything */ #define IEEE80211_MSG_BITS \ "\20\2TDMA\3IOCTL\4WDS\5ACTION\6RATECTL\7ROAM\10INACT\11DOTH\12SUPERG" \ "\13WME\14ACL\15WPA\16RADKEYS\17RADDUMP\20RADIUS\21DOT1XSM\22HWMP" \ "\23POWER\24STATE\25OUTPUT\26SCAN\27AUTH\30ASSOC\31NODE\32ELEMID" \ "\33XRATE\34INPUT\35CRYPTO\36DUPMPKTS\37DEBUG\04011N" #ifdef IEEE80211_DEBUG #define ieee80211_msg(_vap, _m) ((_vap)->iv_debug & (_m)) #define IEEE80211_DPRINTF(_vap, _m, _fmt, ...) do { \ if (ieee80211_msg(_vap, _m)) \ ieee80211_note(_vap, _fmt, __VA_ARGS__); \ } while (0) #define IEEE80211_NOTE(_vap, _m, _ni, _fmt, ...) do { \ if (ieee80211_msg(_vap, _m)) \ ieee80211_note_mac(_vap, (_ni)->ni_macaddr, _fmt, __VA_ARGS__);\ } while (0) #define IEEE80211_NOTE_MAC(_vap, _m, _mac, _fmt, ...) do { \ if (ieee80211_msg(_vap, _m)) \ ieee80211_note_mac(_vap, _mac, _fmt, __VA_ARGS__); \ } while (0) #define IEEE80211_NOTE_FRAME(_vap, _m, _wh, _fmt, ...) do { \ if (ieee80211_msg(_vap, _m)) \ ieee80211_note_frame(_vap, _wh, _fmt, __VA_ARGS__); \ } while (0) void ieee80211_note(const struct ieee80211vap *, const char *, ...); void ieee80211_note_mac(const struct ieee80211vap *, const uint8_t mac[IEEE80211_ADDR_LEN], const char *, ...); void ieee80211_note_frame(const struct ieee80211vap *, const struct ieee80211_frame *, const char *, ...); #define ieee80211_msg_debug(_vap) \ ((_vap)->iv_debug & IEEE80211_MSG_DEBUG) #define ieee80211_msg_dumppkts(_vap) \ ((_vap)->iv_debug & IEEE80211_MSG_DUMPPKTS) #define ieee80211_msg_input(_vap) \ ((_vap)->iv_debug & IEEE80211_MSG_INPUT) #define ieee80211_msg_radius(_vap) \ ((_vap)->iv_debug & IEEE80211_MSG_RADIUS) #define ieee80211_msg_dumpradius(_vap) \ ((_vap)->iv_debug & IEEE80211_MSG_RADDUMP) #define ieee80211_msg_dumpradkeys(_vap) \ ((_vap)->iv_debug & IEEE80211_MSG_RADKEYS) #define ieee80211_msg_scan(_vap) \ ((_vap)->iv_debug & IEEE80211_MSG_SCAN) #define ieee80211_msg_assoc(_vap) \ ((_vap)->iv_debug & IEEE80211_MSG_ASSOC) /* * Emit a debug message about discarding a frame or information * element. One format is for extracting the mac address from * the frame header; the other is for when a header is not * available or otherwise appropriate. */ #define IEEE80211_DISCARD(_vap, _m, _wh, _type, _fmt, ...) do { \ if ((_vap)->iv_debug & (_m)) \ ieee80211_discard_frame(_vap, _wh, _type, _fmt, __VA_ARGS__);\ } while (0) #define IEEE80211_DISCARD_IE(_vap, _m, _wh, _type, _fmt, ...) do { \ if ((_vap)->iv_debug & (_m)) \ ieee80211_discard_ie(_vap, _wh, _type, _fmt, __VA_ARGS__);\ } while (0) #define IEEE80211_DISCARD_MAC(_vap, _m, _mac, _type, _fmt, ...) do { \ if ((_vap)->iv_debug & (_m)) \ ieee80211_discard_mac(_vap, _mac, _type, _fmt, __VA_ARGS__);\ } while (0) void ieee80211_discard_frame(const struct ieee80211vap *, const struct ieee80211_frame *, const char *type, const char *fmt, ...); void ieee80211_discard_ie(const struct ieee80211vap *, const struct ieee80211_frame *, const char *type, const char *fmt, ...); void ieee80211_discard_mac(const struct ieee80211vap *, const uint8_t mac[IEEE80211_ADDR_LEN], const char *type, const char *fmt, ...); #else #define IEEE80211_DPRINTF(_vap, _m, _fmt, ...) #define IEEE80211_NOTE(_vap, _m, _ni, _fmt, ...) #define IEEE80211_NOTE_FRAME(_vap, _m, _wh, _fmt, ...) #define IEEE80211_NOTE_MAC(_vap, _m, _mac, _fmt, ...) #define ieee80211_msg_dumppkts(_vap) 0 #define ieee80211_msg(_vap, _m) 0 #define IEEE80211_DISCARD(_vap, _m, _wh, _type, _fmt, ...) #define IEEE80211_DISCARD_IE(_vap, _m, _wh, _type, _fmt, ...) #define IEEE80211_DISCARD_MAC(_vap, _m, _mac, _type, _fmt, ...) #endif #endif /* _NET80211_IEEE80211_VAR_H_ */ Index: user/ngie/release-pkg-fix-tests/sys/sparc64/sbus/sbus.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/sparc64/sbus/sbus.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/sparc64/sbus/sbus.c (revision 299091) @@ -1,936 +1,936 @@ /*- * Copyright (c) 1999-2002 Eduardo Horvath * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: NetBSD: sbus.c,v 1.50 2002/06/20 18:26:24 eeh Exp */ /*- * Copyright (c) 2002 by Thomas Moestl . * Copyright (c) 2005 Marius Strobl * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include __FBSDID("$FreeBSD$"); /* * SBus support. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include struct sbus_devinfo { int sdi_burstsz; int sdi_clockfreq; int sdi_slot; struct ofw_bus_devinfo sdi_obdinfo; struct resource_list sdi_rl; }; /* Range descriptor, allocated for each sc_range. */ struct sbus_rd { bus_addr_t rd_poffset; bus_addr_t rd_pend; int rd_slot; bus_addr_t rd_coffset; bus_addr_t rd_cend; struct rman rd_rman; bus_space_handle_t rd_bushandle; struct resource *rd_res; }; struct sbus_softc { device_t sc_dev; bus_dma_tag_t sc_cdmatag; int sc_clockfreq; /* clock frequency (in Hz) */ int sc_nrange; struct sbus_rd *sc_rd; int sc_burst; /* burst transfer sizes supp. */ struct resource *sc_sysio_res; int sc_ign; /* IGN for this sysio */ struct iommu_state sc_is; /* IOMMU state (iommuvar.h) */ struct resource *sc_ot_ires; void *sc_ot_ihand; struct resource *sc_pf_ires; void *sc_pf_ihand; }; #define SYSIO_READ8(sc, off) \ bus_read_8((sc)->sc_sysio_res, (off)) #define SYSIO_WRITE8(sc, off, v) \ bus_write_8((sc)->sc_sysio_res, (off), (v)) static device_probe_t sbus_probe; static device_attach_t sbus_attach; static bus_print_child_t sbus_print_child; static bus_probe_nomatch_t sbus_probe_nomatch; static bus_read_ivar_t sbus_read_ivar; static bus_get_resource_list_t sbus_get_resource_list; static bus_setup_intr_t sbus_setup_intr; static bus_alloc_resource_t sbus_alloc_resource; static bus_activate_resource_t sbus_activate_resource; static bus_adjust_resource_t sbus_adjust_resource; static bus_release_resource_t sbus_release_resource; static bus_get_dma_tag_t sbus_get_dma_tag; static ofw_bus_get_devinfo_t sbus_get_devinfo; static int sbus_inlist(const char *, const char *const *); static struct sbus_devinfo * sbus_setup_dinfo(device_t, struct sbus_softc *, phandle_t); static void sbus_destroy_dinfo(struct sbus_devinfo *); static void sbus_intr_enable(void *); static void sbus_intr_disable(void *); static void sbus_intr_assign(void *); static void sbus_intr_clear(void *); static int sbus_find_intrmap(struct sbus_softc *, u_int, bus_addr_t *, bus_addr_t *); static driver_intr_t sbus_overtemp; static driver_intr_t sbus_pwrfail; static int sbus_print_res(struct sbus_devinfo *); static device_method_t sbus_methods[] = { /* Device interface */ DEVMETHOD(device_probe, sbus_probe), DEVMETHOD(device_attach, sbus_attach), DEVMETHOD(device_shutdown, bus_generic_shutdown), DEVMETHOD(device_suspend, bus_generic_suspend), DEVMETHOD(device_resume, bus_generic_resume), /* Bus interface */ DEVMETHOD(bus_print_child, sbus_print_child), DEVMETHOD(bus_probe_nomatch, sbus_probe_nomatch), DEVMETHOD(bus_read_ivar, sbus_read_ivar), DEVMETHOD(bus_alloc_resource, sbus_alloc_resource), DEVMETHOD(bus_activate_resource, sbus_activate_resource), DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource), DEVMETHOD(bus_adjust_resource, sbus_adjust_resource), DEVMETHOD(bus_release_resource, sbus_release_resource), DEVMETHOD(bus_setup_intr, sbus_setup_intr), DEVMETHOD(bus_teardown_intr, bus_generic_teardown_intr), DEVMETHOD(bus_get_resource, bus_generic_rl_get_resource), DEVMETHOD(bus_get_resource_list, sbus_get_resource_list), DEVMETHOD(bus_child_pnpinfo_str, ofw_bus_gen_child_pnpinfo_str), DEVMETHOD(bus_get_dma_tag, sbus_get_dma_tag), /* ofw_bus interface */ DEVMETHOD(ofw_bus_get_devinfo, sbus_get_devinfo), DEVMETHOD(ofw_bus_get_compat, ofw_bus_gen_get_compat), DEVMETHOD(ofw_bus_get_model, ofw_bus_gen_get_model), DEVMETHOD(ofw_bus_get_name, ofw_bus_gen_get_name), DEVMETHOD(ofw_bus_get_node, ofw_bus_gen_get_node), DEVMETHOD(ofw_bus_get_type, ofw_bus_gen_get_type), DEVMETHOD_END }; static driver_t sbus_driver = { "sbus", sbus_methods, sizeof(struct sbus_softc), }; static devclass_t sbus_devclass; EARLY_DRIVER_MODULE(sbus, nexus, sbus_driver, sbus_devclass, NULL, NULL, BUS_PASS_BUS); MODULE_DEPEND(sbus, nexus, 1, 1, 1); MODULE_VERSION(sbus, 1); #define OFW_SBUS_TYPE "sbus" #define OFW_SBUS_NAME "sbus" static const struct intr_controller sbus_ic = { sbus_intr_enable, sbus_intr_disable, sbus_intr_assign, sbus_intr_clear }; struct sbus_icarg { struct sbus_softc *sica_sc; bus_addr_t sica_map; bus_addr_t sica_clr; }; static const char *const sbus_order_first[] = { "auxio", "dma", NULL }; static int sbus_inlist(const char *name, const char *const *list) { int i; if (name == NULL) return (0); for (i = 0; list[i] != NULL; i++) { if (strcmp(name, list[i]) == 0) return (1); } return (0); } static int sbus_probe(device_t dev) { const char *t; t = ofw_bus_get_type(dev); if (((t == NULL || strcmp(t, OFW_SBUS_TYPE) != 0)) && strcmp(ofw_bus_get_name(dev), OFW_SBUS_NAME) != 0) return (ENXIO); device_set_desc(dev, "U2S UPA-SBus bridge"); return (0); } static int sbus_attach(device_t dev) { struct sbus_softc *sc; struct sbus_devinfo *sdi; struct sbus_icarg *sica; struct sbus_ranges *range; struct resource *res; struct resource_list *rl; device_t cdev; bus_addr_t intrclr, intrmap, phys; bus_size_t size; u_long vec; phandle_t child, node; uint32_t prop; int i, j; sc = device_get_softc(dev); sc->sc_dev = dev; node = ofw_bus_get_node(dev); i = 0; sc->sc_sysio_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &i, RF_ACTIVE); if (sc->sc_sysio_res == NULL) panic("%s: cannot allocate device memory", __func__); if (OF_getprop(node, "interrupts", &prop, sizeof(prop)) == -1) panic("%s: cannot get IGN", __func__); sc->sc_ign = INTIGN(prop); /* * Record clock frequency for synchronous SCSI. * IS THIS THE CORRECT DEFAULT?? */ if (OF_getprop(node, "clock-frequency", &prop, sizeof(prop)) == -1) prop = 25000000; sc->sc_clockfreq = prop; prop /= 1000; device_printf(dev, "clock %d.%03d MHz\n", prop / 1000, prop % 1000); /* * Collect address translations from the OBP. */ if ((sc->sc_nrange = OF_getprop_alloc(node, "ranges", sizeof(*range), (void **)&range)) == -1) { panic("%s: error getting ranges property", __func__); } sc->sc_rd = malloc(sizeof(*sc->sc_rd) * sc->sc_nrange, M_DEVBUF, M_NOWAIT | M_ZERO); if (sc->sc_rd == NULL) panic("%s: cannot allocate rmans", __func__); /* * Preallocate all space that the SBus bridge decodes, so that nothing * else gets in the way; set up rmans etc. */ rl = BUS_GET_RESOURCE_LIST(device_get_parent(dev), dev); for (i = 0; i < sc->sc_nrange; i++) { phys = range[i].poffset | ((bus_addr_t)range[i].pspace << 32); size = range[i].size; sc->sc_rd[i].rd_slot = range[i].cspace; sc->sc_rd[i].rd_coffset = range[i].coffset; sc->sc_rd[i].rd_cend = sc->sc_rd[i].rd_coffset + size; j = resource_list_add_next(rl, SYS_RES_MEMORY, phys, phys + size - 1, size); if ((res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &j, RF_ACTIVE)) == NULL) panic("%s: cannot allocate decoded range", __func__); sc->sc_rd[i].rd_bushandle = rman_get_bushandle(res); sc->sc_rd[i].rd_rman.rm_type = RMAN_ARRAY; sc->sc_rd[i].rd_rman.rm_descr = "SBus Device Memory"; if (rman_init(&sc->sc_rd[i].rd_rman) != 0 || rman_manage_region(&sc->sc_rd[i].rd_rman, 0, size) != 0) panic("%s: failed to set up memory rman", __func__); sc->sc_rd[i].rd_poffset = phys; sc->sc_rd[i].rd_pend = phys + size; sc->sc_rd[i].rd_res = res; } free(range, M_OFWPROP); /* * Get the SBus burst transfer size if burst transfers are supported. */ if (OF_getprop(node, "up-burst-sizes", &sc->sc_burst, sizeof(sc->sc_burst)) == -1 || sc->sc_burst == 0) sc->sc_burst = (SBUS_BURST64_DEF << SBUS_BURST64_SHIFT) | SBUS_BURST_DEF; - /* initalise the IOMMU */ + /* initialise the IOMMU */ /* punch in our copies */ sc->sc_is.is_pmaxaddr = IOMMU_MAXADDR(SBUS_IOMMU_BITS); sc->sc_is.is_bustag = rman_get_bustag(sc->sc_sysio_res); sc->sc_is.is_bushandle = rman_get_bushandle(sc->sc_sysio_res); sc->sc_is.is_iommu = SBR_IOMMU; sc->sc_is.is_dtag = SBR_IOMMU_TLB_TAG_DIAG; sc->sc_is.is_ddram = SBR_IOMMU_TLB_DATA_DIAG; sc->sc_is.is_dqueue = SBR_IOMMU_QUEUE_DIAG; sc->sc_is.is_dva = SBR_IOMMU_SVADIAG; sc->sc_is.is_dtcmp = 0; sc->sc_is.is_sb[0] = SBR_STRBUF; sc->sc_is.is_sb[1] = 0; /* * Note: the SBus IOMMU ignores the high bits of an address, so a NULL * DMA pointer will be translated by the first page of the IOTSB. * To detect bugs we'll allocate and ignore the first entry. */ iommu_init(device_get_nameunit(dev), &sc->sc_is, 3, -1, 1); /* Create the DMA tag. */ if (bus_dma_tag_create(bus_get_dma_tag(dev), 8, 0, sc->sc_is.is_pmaxaddr, ~0, NULL, NULL, sc->sc_is.is_pmaxaddr, 0xff, 0xffffffff, 0, NULL, NULL, &sc->sc_cdmatag) != 0) panic("%s: bus_dma_tag_create failed", __func__); /* Customize the tag. */ sc->sc_cdmatag->dt_cookie = &sc->sc_is; sc->sc_cdmatag->dt_mt = &iommu_dma_methods; /* * Hunt through all the interrupt mapping regs and register our * interrupt controller for the corresponding interrupt vectors. * We do this early in order to be able to catch stray interrupts. */ for (i = 0; i <= SBUS_MAX_INO; i++) { if (sbus_find_intrmap(sc, i, &intrmap, &intrclr) == 0) continue; sica = malloc(sizeof(*sica), M_DEVBUF, M_NOWAIT); if (sica == NULL) panic("%s: could not allocate interrupt controller " "argument", __func__); sica->sica_sc = sc; sica->sica_map = intrmap; sica->sica_clr = intrclr; #ifdef SBUS_DEBUG device_printf(dev, "intr map (INO %d, %s) %#lx: %#lx, clr: %#lx\n", i, (i & INTMAP_OBIO_MASK) == 0 ? "SBus slot" : "OBIO", (u_long)intrmap, (u_long)SYSIO_READ8(sc, intrmap), (u_long)intrclr); #endif j = intr_controller_register(INTMAP_VEC(sc->sc_ign, i), &sbus_ic, sica); if (j != 0) device_printf(dev, "could not register interrupt " "controller for INO %d (%d)\n", i, j); } /* Enable the over-temperature and power-fail interrupts. */ i = 4; sc->sc_ot_ires = bus_alloc_resource_any(dev, SYS_RES_IRQ, &i, RF_ACTIVE); if (sc->sc_ot_ires == NULL || INTIGN(vec = rman_get_start(sc->sc_ot_ires)) != sc->sc_ign || INTVEC(SYSIO_READ8(sc, SBR_THERM_INT_MAP)) != vec || intr_vectors[vec].iv_ic != &sbus_ic || bus_setup_intr(dev, sc->sc_ot_ires, INTR_TYPE_MISC | INTR_BRIDGE, NULL, sbus_overtemp, sc, &sc->sc_ot_ihand) != 0) panic("%s: failed to set up temperature interrupt", __func__); i = 3; sc->sc_pf_ires = bus_alloc_resource_any(dev, SYS_RES_IRQ, &i, RF_ACTIVE); if (sc->sc_pf_ires == NULL || INTIGN(vec = rman_get_start(sc->sc_pf_ires)) != sc->sc_ign || INTVEC(SYSIO_READ8(sc, SBR_POWER_INT_MAP)) != vec || intr_vectors[vec].iv_ic != &sbus_ic || bus_setup_intr(dev, sc->sc_pf_ires, INTR_TYPE_MISC | INTR_BRIDGE, NULL, sbus_pwrfail, sc, &sc->sc_pf_ihand) != 0) panic("%s: failed to set up power fail interrupt", __func__); /* Initialize the counter-timer. */ sparc64_counter_init(device_get_nameunit(dev), rman_get_bustag(sc->sc_sysio_res), rman_get_bushandle(sc->sc_sysio_res), SBR_TC0); /* * Loop through ROM children, fixing any relative addresses * and then configuring each device. */ for (child = OF_child(node); child != 0; child = OF_peer(child)) { if ((sdi = sbus_setup_dinfo(dev, sc, child)) == NULL) continue; /* * For devices where there are variants that are actually * split into two SBus devices (as opposed to the first * half of the device being a SBus device and the second * half hanging off of the first one) like 'auxio' and * 'SUNW,fdtwo' or 'dma' and 'esp' probe the SBus device * which is a prerequisite to the driver attaching to the * second one with a lower order. Saves us from dealing * with different probe orders in the respective device * drivers which generally is more hackish. */ cdev = device_add_child_ordered(dev, (OF_child(child) == 0 && sbus_inlist(sdi->sdi_obdinfo.obd_name, sbus_order_first)) ? SBUS_ORDER_FIRST : SBUS_ORDER_NORMAL, NULL, -1); if (cdev == NULL) { device_printf(dev, "<%s>: device_add_child_ordered failed\n", sdi->sdi_obdinfo.obd_name); sbus_destroy_dinfo(sdi); continue; } device_set_ivars(cdev, sdi); } return (bus_generic_attach(dev)); } static struct sbus_devinfo * sbus_setup_dinfo(device_t dev, struct sbus_softc *sc, phandle_t node) { struct sbus_devinfo *sdi; struct sbus_regs *reg; u_int32_t base, iv, *intr; int i, nreg, nintr, slot, rslot; sdi = malloc(sizeof(*sdi), M_DEVBUF, M_ZERO | M_WAITOK); if (ofw_bus_gen_setup_devinfo(&sdi->sdi_obdinfo, node) != 0) { free(sdi, M_DEVBUF); return (NULL); } resource_list_init(&sdi->sdi_rl); slot = -1; nreg = OF_getprop_alloc(node, "reg", sizeof(*reg), (void **)®); if (nreg == -1) { if (sdi->sdi_obdinfo.obd_type == NULL || strcmp(sdi->sdi_obdinfo.obd_type, "hierarchical") != 0) { device_printf(dev, "<%s>: incomplete\n", sdi->sdi_obdinfo.obd_name); goto fail; } } else { for (i = 0; i < nreg; i++) { base = reg[i].sbr_offset; if (SBUS_ABS(base)) { rslot = SBUS_ABS_TO_SLOT(base); base = SBUS_ABS_TO_OFFSET(base); } else rslot = reg[i].sbr_slot; if (slot != -1 && slot != rslot) { device_printf(dev, "<%s>: multiple slots\n", sdi->sdi_obdinfo.obd_name); free(reg, M_OFWPROP); goto fail; } slot = rslot; resource_list_add(&sdi->sdi_rl, SYS_RES_MEMORY, i, base, base + reg[i].sbr_size, reg[i].sbr_size); } free(reg, M_OFWPROP); } sdi->sdi_slot = slot; /* * The `interrupts' property contains the SBus interrupt level. */ nintr = OF_getprop_alloc(node, "interrupts", sizeof(*intr), (void **)&intr); if (nintr != -1) { for (i = 0; i < nintr; i++) { iv = intr[i]; /* * SBus card devices need the slot number encoded into * the vector as this is generally not done. */ if ((iv & INTMAP_OBIO_MASK) == 0) iv |= slot << 3; iv = INTMAP_VEC(sc->sc_ign, iv); resource_list_add(&sdi->sdi_rl, SYS_RES_IRQ, i, iv, iv, 1); } free(intr, M_OFWPROP); } if (OF_getprop(node, "burst-sizes", &sdi->sdi_burstsz, sizeof(sdi->sdi_burstsz)) == -1) sdi->sdi_burstsz = sc->sc_burst; else sdi->sdi_burstsz &= sc->sc_burst; if (OF_getprop(node, "clock-frequency", &sdi->sdi_clockfreq, sizeof(sdi->sdi_clockfreq)) == -1) sdi->sdi_clockfreq = sc->sc_clockfreq; return (sdi); fail: sbus_destroy_dinfo(sdi); return (NULL); } static void sbus_destroy_dinfo(struct sbus_devinfo *dinfo) { resource_list_free(&dinfo->sdi_rl); ofw_bus_gen_destroy_devinfo(&dinfo->sdi_obdinfo); free(dinfo, M_DEVBUF); } static int sbus_print_child(device_t dev, device_t child) { int rv; rv = bus_print_child_header(dev, child); rv += sbus_print_res(device_get_ivars(child)); rv += bus_print_child_footer(dev, child); return (rv); } static void sbus_probe_nomatch(device_t dev, device_t child) { const char *type; device_printf(dev, "<%s>", ofw_bus_get_name(child)); sbus_print_res(device_get_ivars(child)); type = ofw_bus_get_type(child); printf(" type %s (no driver attached)\n", type != NULL ? type : "unknown"); } static int sbus_read_ivar(device_t dev, device_t child, int which, uintptr_t *result) { struct sbus_softc *sc; struct sbus_devinfo *dinfo; sc = device_get_softc(dev); if ((dinfo = device_get_ivars(child)) == NULL) return (ENOENT); switch (which) { case SBUS_IVAR_BURSTSZ: *result = dinfo->sdi_burstsz; break; case SBUS_IVAR_CLOCKFREQ: *result = dinfo->sdi_clockfreq; break; case SBUS_IVAR_IGN: *result = sc->sc_ign; break; case SBUS_IVAR_SLOT: *result = dinfo->sdi_slot; break; default: return (ENOENT); } return (0); } static struct resource_list * sbus_get_resource_list(device_t dev, device_t child) { struct sbus_devinfo *sdi; sdi = device_get_ivars(child); return (&sdi->sdi_rl); } static void sbus_intr_enable(void *arg) { struct intr_vector *iv = arg; struct sbus_icarg *sica = iv->iv_icarg; SYSIO_WRITE8(sica->sica_sc, sica->sica_map, INTMAP_ENABLE(iv->iv_vec, iv->iv_mid)); } static void sbus_intr_disable(void *arg) { struct intr_vector *iv = arg; struct sbus_icarg *sica = iv->iv_icarg; SYSIO_WRITE8(sica->sica_sc, sica->sica_map, iv->iv_vec); } static void sbus_intr_assign(void *arg) { struct intr_vector *iv = arg; struct sbus_icarg *sica = iv->iv_icarg; SYSIO_WRITE8(sica->sica_sc, sica->sica_map, INTMAP_TID( SYSIO_READ8(sica->sica_sc, sica->sica_map), iv->iv_mid)); } static void sbus_intr_clear(void *arg) { struct intr_vector *iv = arg; struct sbus_icarg *sica = iv->iv_icarg; SYSIO_WRITE8(sica->sica_sc, sica->sica_clr, INTCLR_IDLE); } static int sbus_find_intrmap(struct sbus_softc *sc, u_int ino, bus_addr_t *intrmapptr, bus_addr_t *intrclrptr) { bus_addr_t intrclr, intrmap; int i; if (ino > SBUS_MAX_INO) { device_printf(sc->sc_dev, "out of range INO %d requested\n", ino); return (0); } if ((ino & INTMAP_OBIO_MASK) == 0) { intrmap = SBR_SLOT0_INT_MAP + INTSLOT(ino) * 8; intrclr = SBR_SLOT0_INT_CLR + (INTSLOT(ino) * 8 * 8) + (INTPRI(ino) * 8); } else { intrclr = 0; for (i = 0, intrmap = SBR_SCSI_INT_MAP; intrmap <= SBR_RESERVED_INT_MAP; intrmap += 8, i++) { if (INTVEC(SYSIO_READ8(sc, intrmap)) == INTMAP_VEC(sc->sc_ign, ino)) { intrclr = SBR_SCSI_INT_CLR + i * 8; break; } } if (intrclr == 0) return (0); } if (intrmapptr != NULL) *intrmapptr = intrmap; if (intrclrptr != NULL) *intrclrptr = intrclr; return (1); } static int sbus_setup_intr(device_t dev, device_t child, struct resource *ires, int flags, driver_filter_t *filt, driver_intr_t *intr, void *arg, void **cookiep) { struct sbus_softc *sc; u_long vec; sc = device_get_softc(dev); /* * Make sure the vector is fully specified and we registered * our interrupt controller for it. */ vec = rman_get_start(ires); if (INTIGN(vec) != sc->sc_ign || intr_vectors[vec].iv_ic != &sbus_ic) { device_printf(dev, "invalid interrupt vector 0x%lx\n", vec); return (EINVAL); } return (bus_generic_setup_intr(dev, child, ires, flags, filt, intr, arg, cookiep)); } static struct resource * sbus_alloc_resource(device_t bus, device_t child, int type, int *rid, rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) { struct sbus_softc *sc; struct rman *rm; struct resource *rv; struct resource_list *rl; struct resource_list_entry *rle; device_t schild; bus_addr_t toffs; bus_size_t tend; int i, slot; int isdefault, passthrough; isdefault = RMAN_IS_DEFAULT_RANGE(start, end); passthrough = (device_get_parent(child) != bus); rle = NULL; sc = device_get_softc(bus); rl = BUS_GET_RESOURCE_LIST(bus, child); switch (type) { case SYS_RES_IRQ: return (resource_list_alloc(rl, bus, child, type, rid, start, end, count, flags)); case SYS_RES_MEMORY: if (!passthrough) { rle = resource_list_find(rl, type, *rid); if (rle == NULL) return (NULL); if (rle->res != NULL) panic("%s: resource entry is busy", __func__); if (isdefault) { start = rle->start; count = ulmax(count, rle->count); end = ulmax(rle->end, start + count - 1); } } rm = NULL; schild = child; while (device_get_parent(schild) != bus) schild = device_get_parent(schild); slot = sbus_get_slot(schild); for (i = 0; i < sc->sc_nrange; i++) { if (sc->sc_rd[i].rd_slot != slot || start < sc->sc_rd[i].rd_coffset || start > sc->sc_rd[i].rd_cend) continue; /* Disallow cross-range allocations. */ if (end > sc->sc_rd[i].rd_cend) return (NULL); /* We've found the connection to the parent bus */ toffs = start - sc->sc_rd[i].rd_coffset; tend = end - sc->sc_rd[i].rd_coffset; rm = &sc->sc_rd[i].rd_rman; break; } if (rm == NULL) return (NULL); rv = rman_reserve_resource(rm, toffs, tend, count, flags & ~RF_ACTIVE, child); if (rv == NULL) return (NULL); rman_set_rid(rv, *rid); if ((flags & RF_ACTIVE) != 0 && bus_activate_resource(child, type, *rid, rv)) { rman_release_resource(rv); return (NULL); } if (!passthrough) rle->res = rv; return (rv); default: return (NULL); } } static int sbus_activate_resource(device_t bus, device_t child, int type, int rid, struct resource *r) { struct sbus_softc *sc; struct bus_space_tag *tag; int i; switch (type) { case SYS_RES_IRQ: return (bus_generic_activate_resource(bus, child, type, rid, r)); case SYS_RES_MEMORY: sc = device_get_softc(bus); for (i = 0; i < sc->sc_nrange; i++) { if (rman_is_region_manager(r, &sc->sc_rd[i].rd_rman) != 0) { tag = sparc64_alloc_bus_tag(r, SBUS_BUS_SPACE); if (tag == NULL) return (ENOMEM); rman_set_bustag(r, tag); rman_set_bushandle(r, sc->sc_rd[i].rd_bushandle + rman_get_start(r)); return (rman_activate_resource(r)); } } /* FALLTHROUGH */ default: return (EINVAL); } } static int sbus_adjust_resource(device_t bus, device_t child, int type, struct resource *r, rman_res_t start, rman_res_t end) { struct sbus_softc *sc; int i; if (type == SYS_RES_MEMORY) { sc = device_get_softc(bus); for (i = 0; i < sc->sc_nrange; i++) if (rman_is_region_manager(r, &sc->sc_rd[i].rd_rman) != 0) return (rman_adjust_resource(r, start, end)); return (EINVAL); } return (bus_generic_adjust_resource(bus, child, type, r, start, end)); } static int sbus_release_resource(device_t bus, device_t child, int type, int rid, struct resource *r) { struct resource_list *rl; struct resource_list_entry *rle; int error, passthrough; passthrough = (device_get_parent(child) != bus); rl = BUS_GET_RESOURCE_LIST(bus, child); if (type == SYS_RES_MEMORY) { if ((rman_get_flags(r) & RF_ACTIVE) != 0) { error = bus_deactivate_resource(child, type, rid, r); if (error) return (error); } error = rman_release_resource(r); if (error != 0) return (error); if (!passthrough) { rle = resource_list_find(rl, type, rid); KASSERT(rle != NULL, ("%s: resource entry not found!", __func__)); KASSERT(rle->res != NULL, ("%s: resource entry is not busy", __func__)); rle->res = NULL; } return (0); } return (resource_list_release(rl, bus, child, type, rid, r)); } static bus_dma_tag_t sbus_get_dma_tag(device_t bus, device_t child) { struct sbus_softc *sc; sc = device_get_softc(bus); return (sc->sc_cdmatag); } static const struct ofw_bus_devinfo * sbus_get_devinfo(device_t bus, device_t child) { struct sbus_devinfo *sdi; sdi = device_get_ivars(child); return (&sdi->sdi_obdinfo); } /* * Handle an overtemp situation. * * SPARCs have temperature sensors which generate interrupts * if the machine's temperature exceeds a certain threshold. * This handles the interrupt and powers off the machine. * The same needs to be done to PCI controller drivers. */ static void sbus_overtemp(void *arg __unused) { static int shutdown; /* As the interrupt is cleared we may be called multiple times. */ if (shutdown != 0) return; shutdown++; printf("DANGER: OVER TEMPERATURE detected\nShutting down NOW.\n"); shutdown_nice(RB_POWEROFF); } /* Try to shut down in time in case of power failure. */ static void sbus_pwrfail(void *arg __unused) { static int shutdown; /* As the interrupt is cleared we may be called multiple times. */ if (shutdown != 0) return; shutdown++; printf("Power failure detected\nShutting down NOW.\n"); shutdown_nice(RB_POWEROFF); } static int sbus_print_res(struct sbus_devinfo *sdi) { int rv; rv = 0; rv += resource_list_print_type(&sdi->sdi_rl, "mem", SYS_RES_MEMORY, "%#jx"); rv += resource_list_print_type(&sdi->sdi_rl, "irq", SYS_RES_IRQ, "%jd"); return (rv); } Index: user/ngie/release-pkg-fix-tests/sys/sparc64/sparc64/intr_machdep.c =================================================================== --- user/ngie/release-pkg-fix-tests/sys/sparc64/sparc64/intr_machdep.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/sparc64/sparc64/intr_machdep.c (revision 299091) @@ -1,557 +1,557 @@ /*- * Copyright (c) 1991 The Regents of the University of California. * All rights reserved. * * This code is derived from software contributed to Berkeley by * William Jolitz. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ /*- * Copyright (c) 2001 Jake Burkholder. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * from: @(#)isa.c 7.2 (Berkeley) 5/13/91 * form: src/sys/i386/isa/intr_machdep.c,v 1.57 2001/07/20 */ #include __FBSDID("$FreeBSD$"); #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define MAX_STRAY_LOG 5 CTASSERT((1 << IV_SHIFT) == sizeof(struct intr_vector)); ih_func_t *intr_handlers[PIL_MAX]; uint16_t pil_countp[PIL_MAX]; static uint16_t pil_stray_count[PIL_MAX]; struct intr_vector intr_vectors[IV_MAX]; uint16_t intr_countp[IV_MAX]; static uint16_t intr_stray_count[IV_MAX]; static const char *const pil_names[] = { "stray", "low", /* PIL_LOW */ "preempt", /* PIL_PREEMPT */ "ithrd", /* PIL_ITHREAD */ "rndzvs", /* PIL_RENDEZVOUS */ "ast", /* PIL_AST */ "hardclock", /* PIL_HARDCLOCK */ "stray", "stray", "stray", "stray", "filter", /* PIL_FILTER */ "bridge", /* PIL_BRIDGE */ "stop", /* PIL_STOP */ "tick", /* PIL_TICK */ }; /* protect the intr_vectors table */ static struct sx intr_table_lock; /* protect intrcnt_index */ static struct mtx intrcnt_lock; #ifdef SMP static int assign_cpu; static void intr_assign_next_cpu(struct intr_vector *iv); static void intr_shuffle_irqs(void *arg __unused); #endif static int intr_assign_cpu(void *arg, int cpu); static void intr_execute_handlers(void *); static void intr_stray_level(struct trapframe *); static void intr_stray_vector(void *); static int intrcnt_setname(const char *, int); static void intrcnt_updatename(int, const char *, int); static void intrcnt_updatename(int vec, const char *name, int ispil) { static int intrcnt_index, stray_pil_index, stray_vec_index; int name_index; mtx_lock_spin(&intrcnt_lock); if (intrnames[0] == '\0') { /* for bitbucket */ if (bootverbose) printf("initalizing intr_countp\n"); intrcnt_setname("???", intrcnt_index++); stray_vec_index = intrcnt_index++; intrcnt_setname("stray", stray_vec_index); for (name_index = 0; name_index < IV_MAX; name_index++) intr_countp[name_index] = stray_vec_index; stray_pil_index = intrcnt_index++; intrcnt_setname("pil", stray_pil_index); for (name_index = 0; name_index < PIL_MAX; name_index++) pil_countp[name_index] = stray_pil_index; } if (name == NULL) name = "???"; if (!ispil && intr_countp[vec] != stray_vec_index) name_index = intr_countp[vec]; else if (ispil && pil_countp[vec] != stray_pil_index) name_index = pil_countp[vec]; else name_index = intrcnt_index++; if (intrcnt_setname(name, name_index)) name_index = 0; if (!ispil) intr_countp[vec] = name_index; else pil_countp[vec] = name_index; mtx_unlock_spin(&intrcnt_lock); } static int intrcnt_setname(const char *name, int index) { if ((MAXCOMLEN + 1) * index >= sintrnames) return (E2BIG); snprintf(intrnames + (MAXCOMLEN + 1) * index, MAXCOMLEN + 1, "%-*s", MAXCOMLEN, name); return (0); } void intr_setup(int pri, ih_func_t *ihf, int vec, iv_func_t *ivf, void *iva) { char pilname[MAXCOMLEN + 1]; register_t s; s = intr_disable(); if (vec != -1) { intr_vectors[vec].iv_func = ivf; intr_vectors[vec].iv_arg = iva; intr_vectors[vec].iv_pri = pri; intr_vectors[vec].iv_vec = vec; } intr_handlers[pri] = ihf; intr_restore(s); snprintf(pilname, MAXCOMLEN + 1, "pil%d: %s", pri, pil_names[pri]); intrcnt_updatename(pri, pilname, 1); } static void intr_stray_level(struct trapframe *tf) { uint64_t level; level = tf->tf_level; if (pil_stray_count[level] < MAX_STRAY_LOG) { printf("stray level interrupt %ld\n", level); pil_stray_count[level]++; if (pil_stray_count[level] >= MAX_STRAY_LOG) printf("got %d stray level interrupt %ld's: not " "logging anymore\n", MAX_STRAY_LOG, level); } } static void intr_stray_vector(void *cookie) { struct intr_vector *iv; u_int vec; iv = cookie; vec = iv->iv_vec; if (intr_stray_count[vec] < MAX_STRAY_LOG) { printf("stray vector interrupt %d\n", vec); intr_stray_count[vec]++; if (intr_stray_count[vec] >= MAX_STRAY_LOG) printf("got %d stray vector interrupt %d's: not " "logging anymore\n", MAX_STRAY_LOG, vec); } } void intr_init1() { int i; /* Mark all interrupts as being stray. */ for (i = 0; i < PIL_MAX; i++) intr_handlers[i] = intr_stray_level; for (i = 0; i < IV_MAX; i++) { intr_vectors[i].iv_func = intr_stray_vector; intr_vectors[i].iv_arg = &intr_vectors[i]; intr_vectors[i].iv_pri = PIL_LOW; intr_vectors[i].iv_vec = i; intr_vectors[i].iv_refcnt = 0; } intr_handlers[PIL_LOW] = intr_fast; } void intr_init2() { sx_init(&intr_table_lock, "intr sources"); mtx_init(&intrcnt_lock, "intrcnt", NULL, MTX_SPIN); } static int intr_assign_cpu(void *arg, int cpu) { #ifdef SMP struct pcpu *pc; struct intr_vector *iv; /* * Don't do anything during early boot. We will pick up the * assignment once the APs are started. */ if (assign_cpu && cpu != NOCPU) { pc = pcpu_find(cpu); if (pc == NULL) return (EINVAL); iv = arg; sx_xlock(&intr_table_lock); iv->iv_mid = pc->pc_mid; iv->iv_ic->ic_assign(iv); sx_xunlock(&intr_table_lock); } return (0); #else return (EOPNOTSUPP); #endif } static void intr_execute_handlers(void *cookie) { struct intr_vector *iv; iv = cookie; if (__predict_false(intr_event_handle(iv->iv_event, NULL) != 0)) intr_stray_vector(iv); } int intr_controller_register(int vec, const struct intr_controller *ic, void *icarg) { struct intr_event *ie; struct intr_vector *iv; int error; if (vec < 0 || vec >= IV_MAX) return (EINVAL); sx_xlock(&intr_table_lock); iv = &intr_vectors[vec]; ie = iv->iv_event; sx_xunlock(&intr_table_lock); if (ie != NULL) return (EEXIST); error = intr_event_create(&ie, iv, 0, vec, NULL, ic->ic_clear, ic->ic_clear, intr_assign_cpu, "vec%d:", vec); if (error != 0) return (error); sx_xlock(&intr_table_lock); if (iv->iv_event != NULL) { sx_xunlock(&intr_table_lock); intr_event_destroy(ie); return (EEXIST); } iv->iv_ic = ic; iv->iv_icarg = icarg; iv->iv_event = ie; iv->iv_mid = PCPU_GET(mid); sx_xunlock(&intr_table_lock); return (0); } int inthand_add(const char *name, int vec, driver_filter_t *filt, driver_intr_t *handler, void *arg, int flags, void **cookiep) { const struct intr_controller *ic; struct intr_event *ie; struct intr_handler *ih; struct intr_vector *iv; int error, filter; if (vec < 0 || vec >= IV_MAX) return (EINVAL); /* * INTR_BRIDGE filters/handlers are special purpose only, allowing * them to be shared just would complicate things unnecessarily. */ if ((flags & INTR_BRIDGE) != 0 && (flags & INTR_EXCL) == 0) return (EINVAL); sx_xlock(&intr_table_lock); iv = &intr_vectors[vec]; ic = iv->iv_ic; ie = iv->iv_event; sx_xunlock(&intr_table_lock); if (ic == NULL || ie == NULL) return (EINVAL); error = intr_event_add_handler(ie, name, filt, handler, arg, intr_priority(flags), flags, cookiep); if (error != 0) return (error); sx_xlock(&intr_table_lock); /* Disable the interrupt while we fiddle with it. */ ic->ic_disable(iv); iv->iv_refcnt++; if (iv->iv_refcnt == 1) intr_setup((flags & INTR_BRIDGE) != 0 ? PIL_BRIDGE : filt != NULL ? PIL_FILTER : PIL_ITHREAD, intr_fast, vec, intr_execute_handlers, iv); else if (filt != NULL) { /* * Check if we need to upgrade from PIL_ITHREAD to PIL_FILTER. * Given that apart from the on-board SCCs and UARTs shared - * interrupts are rather uncommon on sparc64 this sould be + * interrupts are rather uncommon on sparc64 this should be * pretty rare in practice. */ filter = 0; TAILQ_FOREACH(ih, &ie->ie_handlers, ih_next) { if (ih->ih_filter != NULL && ih->ih_filter != filt) { filter = 1; break; } } if (filter == 0) intr_setup(PIL_FILTER, intr_fast, vec, intr_execute_handlers, iv); } intr_stray_count[vec] = 0; intrcnt_updatename(vec, ie->ie_fullname, 0); #ifdef SMP if (assign_cpu) intr_assign_next_cpu(iv); #endif ic->ic_enable(iv); /* Ensure the interrupt is cleared, it might have triggered before. */ if (ic->ic_clear != NULL) ic->ic_clear(iv); sx_xunlock(&intr_table_lock); return (0); } int inthand_remove(int vec, void *cookie) { struct intr_vector *iv; int error; if (vec < 0 || vec >= IV_MAX) return (EINVAL); error = intr_event_remove_handler(cookie); if (error == 0) { /* * XXX: maybe this should be done regardless of whether * intr_event_remove_handler() succeeded? */ sx_xlock(&intr_table_lock); iv = &intr_vectors[vec]; iv->iv_refcnt--; if (iv->iv_refcnt == 0) { /* * Don't disable the interrupt for now, so that * stray interrupts get detected... */ intr_setup(PIL_LOW, intr_fast, vec, intr_stray_vector, iv); } sx_xunlock(&intr_table_lock); } return (error); } /* Add a description to an active interrupt handler. */ int intr_describe(int vec, void *ih, const char *descr) { struct intr_vector *iv; int error; if (vec < 0 || vec >= IV_MAX) return (EINVAL); sx_xlock(&intr_table_lock); iv = &intr_vectors[vec]; if (iv == NULL) { sx_xunlock(&intr_table_lock); return (EINVAL); } error = intr_event_describe_handler(iv->iv_event, ih, descr); if (error) { sx_xunlock(&intr_table_lock); return (error); } intrcnt_updatename(vec, iv->iv_event->ie_fullname, 0); sx_xunlock(&intr_table_lock); return (error); } #ifdef SMP /* * Support for balancing interrupt sources across CPUs. For now we just * allocate CPUs round-robin. */ static cpuset_t intr_cpus = CPUSET_T_INITIALIZER(0x1); static int current_cpu; static void intr_assign_next_cpu(struct intr_vector *iv) { struct pcpu *pc; sx_assert(&intr_table_lock, SA_XLOCKED); /* * Assign this source to a CPU in a round-robin fashion. */ pc = pcpu_find(current_cpu); if (pc == NULL) return; iv->iv_mid = pc->pc_mid; iv->iv_ic->ic_assign(iv); do { current_cpu++; if (current_cpu > mp_maxid) current_cpu = 0; } while (!CPU_ISSET(current_cpu, &intr_cpus)); } /* Attempt to bind the specified IRQ to the specified CPU. */ int intr_bind(int vec, u_char cpu) { struct intr_vector *iv; int error; if (vec < 0 || vec >= IV_MAX) return (EINVAL); sx_xlock(&intr_table_lock); iv = &intr_vectors[vec]; if (iv == NULL) { sx_xunlock(&intr_table_lock); return (EINVAL); } error = intr_event_bind(iv->iv_event, cpu); sx_xunlock(&intr_table_lock); return (error); } /* * Add a CPU to our mask of valid CPUs that can be destinations of * interrupts. */ void intr_add_cpu(u_int cpu) { if (cpu >= MAXCPU) panic("%s: Invalid CPU ID", __func__); if (bootverbose) printf("INTR: Adding CPU %d as a target\n", cpu); CPU_SET(cpu, &intr_cpus); } /* * Distribute all the interrupt sources among the available CPUs once the * APs have been launched. */ static void intr_shuffle_irqs(void *arg __unused) { struct pcpu *pc; struct intr_vector *iv; int i; /* Don't bother on UP. */ if (mp_ncpus == 1) return; sx_xlock(&intr_table_lock); assign_cpu = 1; for (i = 0; i < IV_MAX; i++) { iv = &intr_vectors[i]; if (iv != NULL && iv->iv_refcnt > 0) { /* * If this event is already bound to a CPU, * then assign the source to that CPU instead * of picking one via round-robin. */ if (iv->iv_event->ie_cpu != NOCPU && (pc = pcpu_find(iv->iv_event->ie_cpu)) != NULL) { iv->iv_mid = pc->pc_mid; iv->iv_ic->ic_assign(iv); } else intr_assign_next_cpu(iv); } } sx_xunlock(&intr_table_lock); } SYSINIT(intr_shuffle_irqs, SI_SUB_SMP, SI_ORDER_SECOND, intr_shuffle_irqs, NULL); #endif Index: user/ngie/release-pkg-fix-tests/sys/sys/bitstring.h =================================================================== --- user/ngie/release-pkg-fix-tests/sys/sys/bitstring.h (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/sys/bitstring.h (revision 299091) @@ -1,146 +1,259 @@ /*- * Copyright (c) 1989, 1993 * The Regents of the University of California. All rights reserved. * * This code is derived from software contributed to Berkeley by * Paul Vixie. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * + * Copyright (c) 2014 Spectra Logic Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions, and the following disclaimer, + * without modification. + * 2. Redistributions in binary form must reproduce at minimum a disclaimer + * substantially similar to the "NO WARRANTY" disclaimer below + * ("Disclaimer") and any redistribution must be conditioned upon + * including a substantially similar Disclaimer requirement for further + * binary redistribution. + * + * NO WARRANTY + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGES. + * * $FreeBSD$ */ - #ifndef _SYS_BITSTRING_H_ #define _SYS_BITSTRING_H_ -typedef unsigned char bitstr_t; +#ifdef _KERNEL +#include +#include +#endif -/* internal macros */ - /* byte of the bitstring bit is in */ -#define _bit_byte(bit) \ - ((bit) >> 3) +typedef unsigned long bitstr_t; - /* mask for the bit within its byte */ -#define _bit_mask(bit) \ - (1 << ((bit)&0x7)) +/*---------------------- Private Implementation Details ----------------------*/ +#define _BITSTR_MASK (~0UL) +#define _BITSTR_BITS (sizeof(bitstr_t) * 8) -/* external macros */ - /* bytes in a bitstring of nbits bits */ -#define bitstr_size(nbits) \ - (((nbits) + 7) >> 3) +/* bitstr_t in bit string containing the bit. */ +static inline int +_bit_idx(int _bit) +{ + return (_bit / _BITSTR_BITS); +} - /* allocate a bitstring */ -#define bit_alloc(nbits) \ - (bitstr_t *)calloc((size_t)bitstr_size(nbits), sizeof(bitstr_t)) +/* bit number within bitstr_t at _bit_idx(_bit). */ +static inline int +_bit_offset(int _bit) +{ + return (_bit % _BITSTR_BITS); +} - /* allocate a bitstring on the stack */ +/* Mask for the bit within its long. */ +static inline bitstr_t +_bit_mask(int _bit) +{ + return (1UL << _bit_offset(_bit)); +} + +static inline bitstr_t +_bit_make_mask(int _start, int _stop) +{ + return ((_BITSTR_MASK << _bit_offset(_start)) & + (_BITSTR_MASK >> (_BITSTR_BITS - _bit_offset(_stop) - 1))); +} + +/*----------------------------- Public Interface -----------------------------*/ +/* Number of bytes consumed by a bit string of nbits bits */ +#define bitstr_size(_nbits) \ + (((_nbits) + _BITSTR_BITS - 1) / 8) + +/* Allocate a bit string initialized with no bits set. */ +#ifdef _KERNEL +static inline bitstr_t * +bit_alloc(int _nbits, struct malloc_type *type, int flags) +{ + return ((bitstr_t *)malloc(bitstr_size(_nbits), type, flags | M_ZERO)); +} +#else +static inline bitstr_t * +bit_alloc(int _nbits) +{ + return ((bitstr_t *)calloc(bitstr_size(_nbits), 1)); +} +#endif + +/* Allocate a bit string on the stack with no bits set. */ #define bit_decl(name, nbits) \ - ((name)[bitstr_size(nbits)]) + ((name)[bitstr_size(nbits) / sizeof(bitstr_t)]) - /* is bit N of bitstring name set? */ -#define bit_test(name, bit) \ - ((name)[_bit_byte(bit)] & _bit_mask(bit)) +/* Is bit N of bit string set? */ +static inline int +bit_test(const bitstr_t *_bitstr, int _bit) +{ + return ((_bitstr[_bit_idx(_bit)] & _bit_mask(_bit)) != 0); +} - /* set bit N of bitstring name */ -#define bit_set(name, bit) \ - ((name)[_bit_byte(bit)] |= _bit_mask(bit)) +/* Set bit N of bit string. */ +static inline void +bit_set(bitstr_t *_bitstr, int _bit) +{ + _bitstr[_bit_idx(_bit)] |= _bit_mask(_bit); +} - /* clear bit N of bitstring name */ -#define bit_clear(name, bit) \ - ((name)[_bit_byte(bit)] &= ~_bit_mask(bit)) +/* clear bit N of bit string name */ +static inline void +bit_clear(bitstr_t *_bitstr, int _bit) +{ + _bitstr[_bit_idx(_bit)] &= ~_bit_mask(_bit); +} - /* clear bits start ... stop in bitstring */ -#define bit_nclear(name, start, stop) do { \ - register bitstr_t *_name = (name); \ - register int _start = (start), _stop = (stop); \ - register int _startbyte = _bit_byte(_start); \ - register int _stopbyte = _bit_byte(_stop); \ - if (_startbyte == _stopbyte) { \ - _name[_startbyte] &= ((0xff >> (8 - (_start&0x7))) | \ - (0xff << ((_stop&0x7) + 1))); \ - } else { \ - _name[_startbyte] &= 0xff >> (8 - (_start&0x7)); \ - while (++_startbyte < _stopbyte) \ - _name[_startbyte] = 0; \ - _name[_stopbyte] &= 0xff << ((_stop&0x7) + 1); \ - } \ -} while (0) +/* Set bits start ... stop inclusive in bit string. */ +static inline void +bit_nset(bitstr_t *_bitstr, int _start, int _stop) +{ + bitstr_t *_stopbitstr; - /* set bits start ... stop in bitstring */ -#define bit_nset(name, start, stop) do { \ - register bitstr_t *_name = (name); \ - register int _start = (start), _stop = (stop); \ - register int _startbyte = _bit_byte(_start); \ - register int _stopbyte = _bit_byte(_stop); \ - if (_startbyte == _stopbyte) { \ - _name[_startbyte] |= ((0xff << (_start&0x7)) & \ - (0xff >> (7 - (_stop&0x7)))); \ - } else { \ - _name[_startbyte] |= 0xff << ((_start)&0x7); \ - while (++_startbyte < _stopbyte) \ - _name[_startbyte] = 0xff; \ - _name[_stopbyte] |= 0xff >> (7 - (_stop&0x7)); \ - } \ -} while (0) + _stopbitstr = _bitstr + _bit_idx(_stop); + _bitstr += _bit_idx(_start); - /* find first bit clear in name */ -#define bit_ffc(name, nbits, value) do { \ - register bitstr_t *_name = (name); \ - register int _byte, _nbits = (nbits); \ - register int _stopbyte = _bit_byte(_nbits - 1), _value = -1; \ - if (_nbits > 0) \ - for (_byte = 0; _byte <= _stopbyte; ++_byte) \ - if (_name[_byte] != 0xff) { \ - bitstr_t _lb; \ - _value = _byte << 3; \ - for (_lb = _name[_byte]; (_lb&0x1); \ - ++_value, _lb >>= 1); \ - break; \ - } \ - if (_value >= nbits) \ - _value = -1; \ - *(value) = _value; \ -} while (0) + if (_bitstr == _stopbitstr) { + *_bitstr |= _bit_make_mask(_start, _stop); + } else { + *_bitstr |= _bit_make_mask(_start, _BITSTR_BITS - 1); + while (++_bitstr < _stopbitstr) + *_bitstr = _BITSTR_MASK; + *_stopbitstr |= _bit_make_mask(0, _stop); + } +} - /* find first bit set in name */ -#define bit_ffs(name, nbits, value) do { \ - register bitstr_t *_name = (name); \ - register int _byte, _nbits = (nbits); \ - register int _stopbyte = _bit_byte(_nbits - 1), _value = -1; \ - if (_nbits > 0) \ - for (_byte = 0; _byte <= _stopbyte; ++_byte) \ - if (_name[_byte]) { \ - bitstr_t _lb; \ - _value = _byte << 3; \ - for (_lb = _name[_byte]; !(_lb&0x1); \ - ++_value, _lb >>= 1); \ - break; \ - } \ - if (_value >= nbits) \ - _value = -1; \ - *(value) = _value; \ -} while (0) +/* Clear bits start ... stop inclusive in bit string. */ +static inline void +bit_nclear(bitstr_t *_bitstr, int _start, int _stop) +{ + bitstr_t *_stopbitstr; -#endif /* !_SYS_BITSTRING_H_ */ + _stopbitstr = _bitstr + _bit_idx(_stop); + _bitstr += _bit_idx(_start); + + if (_bitstr == _stopbitstr) { + *_bitstr &= ~_bit_make_mask(_start, _stop); + } else { + *_bitstr &= ~_bit_make_mask(_start, _BITSTR_BITS - 1); + while (++_bitstr < _stopbitstr) + *_bitstr = 0; + *_stopbitstr &= ~_bit_make_mask(0, _stop); + } +} + +/* Find the first bit set in bit string at or after bit start. */ +static inline void +bit_ffs_at(bitstr_t *_bitstr, int _start, int _nbits, int *_result) +{ + bitstr_t *_curbitstr; + bitstr_t *_stopbitstr; + bitstr_t _test; + int _value, _offset; + + if (_nbits > 0) { + _curbitstr = _bitstr + _bit_idx(_start); + _stopbitstr = _bitstr + _bit_idx(_nbits - 1); + + _test = *_curbitstr; + if (_bit_offset(_start) != 0) + _test &= _bit_make_mask(_start, _BITSTR_BITS - 1); + while (_test == 0 && _curbitstr < _stopbitstr) + _test = *(++_curbitstr); + + _offset = ffsl(_test); + _value = ((_curbitstr - _bitstr) * _BITSTR_BITS) + _offset - 1; + if (_offset == 0 || _value >= _nbits) + _value = -1; + } else { + _value = -1; + } + *_result = _value; +} + +/* Find the first bit clear in bit string at or after bit start. */ +static inline void +bit_ffc_at(bitstr_t *_bitstr, int _start, int _nbits, int *_result) +{ + bitstr_t *_curbitstr; + bitstr_t *_stopbitstr; + bitstr_t _test; + int _value, _offset; + + if (_nbits > 0) { + _curbitstr = _bitstr + _bit_idx(_start); + _stopbitstr = _bitstr + _bit_idx(_nbits - 1); + + _test = *_curbitstr; + if (_bit_offset(_start) != 0) + _test |= _bit_make_mask(0, _start - 1); + while (_test == _BITSTR_MASK && _curbitstr < _stopbitstr) + _test = *(++_curbitstr); + + _offset = ffsl(~_test); + _value = ((_curbitstr - _bitstr) * _BITSTR_BITS) + _offset - 1; + if (_offset == 0 || _value >= _nbits) + _value = -1; + } else { + _value = -1; + } + *_result = _value; +} + +/* Find the first bit set in bit string. */ +static inline void +bit_ffs(bitstr_t *_bitstr, int _nbits, int *_result) +{ + bit_ffs_at(_bitstr, /*start*/0, _nbits, _result); +} + +/* Find the first bit clear in bit string. */ +static inline void +bit_ffc(bitstr_t *_bitstr, int _nbits, int *_result) +{ + bit_ffc_at(_bitstr, /*start*/0, _nbits, _result); +} + +#endif /* _SYS_BITSTRING_H_ */ Index: user/ngie/release-pkg-fix-tests/sys/sys/param.h =================================================================== --- user/ngie/release-pkg-fix-tests/sys/sys/param.h (revision 299090) +++ user/ngie/release-pkg-fix-tests/sys/sys/param.h (revision 299091) @@ -1,363 +1,363 @@ /*- * Copyright (c) 1982, 1986, 1989, 1993 * The Regents of the University of California. All rights reserved. * (c) UNIX System Laboratories, Inc. * All or some portions of this file are derived from material licensed * to the University of California by American Telephone and Telegraph * Co. or Unix System Laboratories, Inc. and are reproduced herein with * the permission of UNIX System Laboratories, Inc. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 4. Neither the name of the University nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * @(#)param.h 8.3 (Berkeley) 4/4/95 * $FreeBSD$ */ #ifndef _SYS_PARAM_H_ #define _SYS_PARAM_H_ #include #define BSD 199506 /* System version (year & month). */ #define BSD4_3 1 #define BSD4_4 1 /* * __FreeBSD_version numbers are documented in the Porter's Handbook. * If you bump the version for any reason, you should update the documentation * there. * Currently this lives here in the doc/ repository: * * head/en_US.ISO8859-1/books/porters-handbook/versions/chapter.xml * * scheme is: Rxx * 'R' is in the range 0 to 4 if this is a release branch or * x.0-CURRENT before RELENG_*_0 is created, otherwise 'R' is * in the range 5 to 9. */ #undef __FreeBSD_version -#define __FreeBSD_version 1100106 /* Master, propagated to newvers */ +#define __FreeBSD_version 1100107 /* Master, propagated to newvers */ /* * __FreeBSD_kernel__ indicates that this system uses the kernel of FreeBSD, * which by definition is always true on FreeBSD. This macro is also defined * on other systems that use the kernel of FreeBSD, such as GNU/kFreeBSD. * * It is tempting to use this macro in userland code when we want to enable * kernel-specific routines, and in fact it's fine to do this in code that * is part of FreeBSD itself. However, be aware that as presence of this * macro is still not widespread (e.g. older FreeBSD versions, 3rd party * compilers, etc), it is STRONGLY DISCOURAGED to check for this macro in * external applications without also checking for __FreeBSD__ as an * alternative. */ #undef __FreeBSD_kernel__ #define __FreeBSD_kernel__ #ifdef _KERNEL #define P_OSREL_SIGWAIT 700000 #define P_OSREL_SIGSEGV 700004 #define P_OSREL_MAP_ANON 800104 #define P_OSREL_MAP_FSTRICT 1100036 #define P_OSREL_SHUTDOWN_ENOTCONN 1100077 #define P_OSREL_MAJOR(x) ((x) / 100000) #endif #ifndef LOCORE #include #endif /* * Machine-independent constants (some used in following include files). * Redefined constants are from POSIX 1003.1 limits file. * * MAXCOMLEN should be >= sizeof(ac_comm) (see ) */ #include #define MAXCOMLEN 19 /* max command name remembered */ #define MAXINTERP PATH_MAX /* max interpreter file name length */ #define MAXLOGNAME 33 /* max login name length (incl. NUL) */ #define MAXUPRC CHILD_MAX /* max simultaneous processes */ #define NCARGS ARG_MAX /* max bytes for an exec function */ #define NGROUPS (NGROUPS_MAX+1) /* max number groups */ #define NOFILE OPEN_MAX /* max open files per process */ #define NOGROUP 65535 /* marker for empty group set member */ #define MAXHOSTNAMELEN 256 /* max hostname size */ #define SPECNAMELEN 63 /* max length of devicename */ /* More types and definitions used throughout the kernel. */ #ifdef _KERNEL #include #include #ifndef LOCORE #include #include #endif #ifndef FALSE #define FALSE 0 #endif #ifndef TRUE #define TRUE 1 #endif #endif #ifndef _KERNEL /* Signals. */ #include #endif /* Machine type dependent parameters. */ #include #ifndef _KERNEL #include #endif #ifndef DEV_BSHIFT #define DEV_BSHIFT 9 /* log2(DEV_BSIZE) */ #endif #define DEV_BSIZE (1<>PAGE_SHIFT) #endif /* * btodb() is messy and perhaps slow because `bytes' may be an off_t. We * want to shift an unsigned type to avoid sign extension and we don't * want to widen `bytes' unnecessarily. Assume that the result fits in * a daddr_t. */ #ifndef btodb #define btodb(bytes) /* calculates (bytes / DEV_BSIZE) */ \ (sizeof (bytes) > sizeof(long) \ ? (daddr_t)((unsigned long long)(bytes) >> DEV_BSHIFT) \ : (daddr_t)((unsigned long)(bytes) >> DEV_BSHIFT)) #endif #ifndef dbtob #define dbtob(db) /* calculates (db * DEV_BSIZE) */ \ ((off_t)(db) << DEV_BSHIFT) #endif #define PRIMASK 0x0ff #define PCATCH 0x100 /* OR'd with pri for tsleep to check signals */ #define PDROP 0x200 /* OR'd with pri to stop re-entry of interlock mutex */ #define NZERO 0 /* default "nice" */ #define NBBY 8 /* number of bits in a byte */ #define NBPW sizeof(int) /* number of bytes per word (integer) */ #define CMASK 022 /* default file mask: S_IWGRP|S_IWOTH */ #define NODEV (dev_t)(-1) /* non-existent device */ /* * File system parameters and macros. * * MAXBSIZE - Filesystems are made out of blocks of at most MAXBSIZE bytes * per block. MAXBSIZE may be made larger without effecting * any existing filesystems as long as it does not exceed MAXPHYS, * and may be made smaller at the risk of not being able to use * filesystems which require a block size exceeding MAXBSIZE. * * MAXBCACHEBUF - Maximum size of a buffer in the buffer cache. This must * be >= MAXBSIZE and can be set differently for different * architectures by defining it in . * Making this larger allows NFS to do larger reads/writes. * * BKVASIZE - Nominal buffer space per buffer, in bytes. BKVASIZE is the * minimum KVM memory reservation the kernel is willing to make. * Filesystems can of course request smaller chunks. Actual * backing memory uses a chunk size of a page (PAGE_SIZE). * The default value here can be overridden on a per-architecture * basis by defining it in . This should * probably be done to increase its value, when MAXBCACHEBUF is * defined as a larger value in . * * If you make BKVASIZE too small you risk seriously fragmenting * the buffer KVM map which may slow things down a bit. If you * make it too big the kernel will not be able to optimally use * the KVM memory reserved for the buffer cache and will wind * up with too-few buffers. * * The default is 16384, roughly 2x the block size used by a * normal UFS filesystem. */ #define MAXBSIZE 65536 /* must be power of 2 */ #ifndef MAXBCACHEBUF #define MAXBCACHEBUF MAXBSIZE /* must be a power of 2 >= MAXBSIZE */ #endif #ifndef BKVASIZE #define BKVASIZE 16384 /* must be power of 2 */ #endif #define BKVAMASK (BKVASIZE-1) /* * MAXPATHLEN defines the longest permissible path length after expanding * symbolic links. It is used to allocate a temporary buffer from the buffer * pool in which to do the name expansion, hence should be a power of two, * and must be less than or equal to MAXBSIZE. MAXSYMLINKS defines the * maximum number of symbolic links that may be expanded in a path name. * It should be set high enough to allow all legitimate uses, but halt * infinite loops reasonably quickly. */ #define MAXPATHLEN PATH_MAX #define MAXSYMLINKS 32 /* Bit map related macros. */ #define setbit(a,i) (((unsigned char *)(a))[(i)/NBBY] |= 1<<((i)%NBBY)) #define clrbit(a,i) (((unsigned char *)(a))[(i)/NBBY] &= ~(1<<((i)%NBBY))) #define isset(a,i) \ (((const unsigned char *)(a))[(i)/NBBY] & (1<<((i)%NBBY))) #define isclr(a,i) \ ((((const unsigned char *)(a))[(i)/NBBY] & (1<<((i)%NBBY))) == 0) /* Macros for counting and rounding. */ #ifndef howmany #define howmany(x, y) (((x)+((y)-1))/(y)) #endif #define nitems(x) (sizeof((x)) / sizeof((x)[0])) #define rounddown(x, y) (((x)/(y))*(y)) #define rounddown2(x, y) ((x)&(~((y)-1))) /* if y is power of two */ #define roundup(x, y) ((((x)+((y)-1))/(y))*(y)) /* to any y */ #define roundup2(x, y) (((x)+((y)-1))&(~((y)-1))) /* if y is powers of two */ #define powerof2(x) ((((x)-1)&(x))==0) /* Macros for min/max. */ #define MIN(a,b) (((a)<(b))?(a):(b)) #define MAX(a,b) (((a)>(b))?(a):(b)) #ifdef _KERNEL /* * Basic byte order function prototypes for non-inline functions. */ #ifndef LOCORE #ifndef _BYTEORDER_PROTOTYPED #define _BYTEORDER_PROTOTYPED __BEGIN_DECLS __uint32_t htonl(__uint32_t); __uint16_t htons(__uint16_t); __uint32_t ntohl(__uint32_t); __uint16_t ntohs(__uint16_t); __END_DECLS #endif #endif #ifndef lint #ifndef _BYTEORDER_FUNC_DEFINED #define _BYTEORDER_FUNC_DEFINED #define htonl(x) __htonl(x) #define htons(x) __htons(x) #define ntohl(x) __ntohl(x) #define ntohs(x) __ntohs(x) #endif /* !_BYTEORDER_FUNC_DEFINED */ #endif /* lint */ #endif /* _KERNEL */ /* * Scale factor for scaled integers used to count %cpu time and load avgs. * * The number of CPU `tick's that map to a unique `%age' can be expressed * by the formula (1 / (2 ^ (FSHIFT - 11))). The maximum load average that * can be calculated (assuming 32 bits) can be closely approximated using * the formula (2 ^ (2 * (16 - FSHIFT))) for (FSHIFT < 15). * * For the scheduler to maintain a 1:1 mapping of CPU `tick' to `%age', * FSHIFT must be at least 11; this gives us a maximum load avg of ~1024. */ #define FSHIFT 11 /* bits to right of fixed binary point */ #define FSCALE (1<> (PAGE_SHIFT - DEV_BSHIFT)) #define ctodb(db) /* calculates pages to devblks */ \ ((db) << (PAGE_SHIFT - DEV_BSHIFT)) /* * Old spelling of __containerof(). */ #define member2struct(s, m, x) \ ((struct s *)(void *)((char *)(x) - offsetof(struct s, m))) /* * Access a variable length array that has been declared as a fixed * length array. */ #define __PAST_END(array, offset) (((__typeof__(*(array)) *)(array))[offset]) #endif /* _SYS_PARAM_H_ */ Index: user/ngie/release-pkg-fix-tests/tests/sys/Makefile =================================================================== --- user/ngie/release-pkg-fix-tests/tests/sys/Makefile (revision 299090) +++ user/ngie/release-pkg-fix-tests/tests/sys/Makefile (revision 299091) @@ -1,23 +1,24 @@ # $FreeBSD$ TESTSDIR= ${TESTSBASE}/sys TESTS_SUBDIRS+= acl TESTS_SUBDIRS+= aio TESTS_SUBDIRS+= fifo TESTS_SUBDIRS+= file TESTS_SUBDIRS+= geom TESTS_SUBDIRS+= kern TESTS_SUBDIRS+= kqueue TESTS_SUBDIRS+= mac TESTS_SUBDIRS+= mqueue TESTS_SUBDIRS+= netinet TESTS_SUBDIRS+= opencrypto TESTS_SUBDIRS+= posixshm +TESTS_SUBDIRS+= sys TESTS_SUBDIRS+= vfs TESTS_SUBDIRS+= vm # Items not integrated into kyua runs by default SUBDIR+= pjdfstest .include Index: user/ngie/release-pkg-fix-tests/tests/sys/sys/Makefile =================================================================== --- user/ngie/release-pkg-fix-tests/tests/sys/sys/Makefile (nonexistent) +++ user/ngie/release-pkg-fix-tests/tests/sys/sys/Makefile (revision 299091) @@ -0,0 +1,13 @@ +# $FreeBSD$ + +PACKAGE= tests +FILESGROUPS= TESTS +TESTSPACKAGE= ${PACKAGE} + +TESTSDIR= ${TESTSBASE}/sys/sys + +ATF_TESTS_C= bitstring_test + +WARNS?= 5 + +.include Property changes on: user/ngie/release-pkg-fix-tests/tests/sys/sys/Makefile ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: user/ngie/release-pkg-fix-tests/tests/sys/sys/bitstring_test.c =================================================================== --- user/ngie/release-pkg-fix-tests/tests/sys/sys/bitstring_test.c (nonexistent) +++ user/ngie/release-pkg-fix-tests/tests/sys/sys/bitstring_test.c (revision 299091) @@ -0,0 +1,359 @@ +/*- + * Copyright (c) 2014 Spectra Logic Corporation + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions, and the following disclaimer, + * without modification. + * 2. Redistributions in binary form must reproduce at minimum a disclaimer + * substantially similar to the "NO WARRANTY" disclaimer below + * ("Disclaimer") and any redistribution must be conditioned upon + * including a substantially similar Disclaimer requirement for further + * binary redistribution. + * + * NO WARRANTY + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGES. + * + * $FreeBSD$ + */ +#include + +#include +#include + +#include + +typedef void (testfunc_t)(bitstr_t *bstr, int nbits, const char *memloc); + +static void +bitstring_run_stack_test(testfunc_t *test, int nbits) +{ + bitstr_t bit_decl(bitstr, nbits); + + test(bitstr, nbits, "stack"); +} + +static void +bitstring_run_heap_test(testfunc_t *test, int nbits) +{ + bitstr_t *bitstr = bit_alloc(nbits); + + test(bitstr, nbits, "heap"); +} + +static void +bitstring_test_runner(testfunc_t *test) +{ + const int bitstr_sizes[] = { + 0, + 1, + _BITSTR_BITS - 1, + _BITSTR_BITS, + _BITSTR_BITS + 1, + 2 * _BITSTR_BITS - 1, + 2 * _BITSTR_BITS, + 1023, + 1024 + }; + + for (unsigned long i = 0; i < nitems(bitstr_sizes); i++) { + bitstring_run_stack_test(test, bitstr_sizes[i]); + bitstring_run_heap_test(test, bitstr_sizes[i]); + } +} + +#define BITSTRING_TC_DEFINE(name) \ +ATF_TC_WITHOUT_HEAD(name); \ +static testfunc_t name ## _test; \ + \ +ATF_TC_BODY(name, tc) \ +{ \ + bitstring_test_runner(name ## _test); \ +} \ + \ +static void \ +name ## _test(bitstr_t *bitstr, int nbits, const char *memloc) + +#define BITSTRING_TC_ADD(tp, name) \ +do { \ + ATF_TP_ADD_TC(tp, name); \ +} while (0) + +ATF_TC_WITHOUT_HEAD(bitstr_in_struct); +ATF_TC_BODY(bitstr_in_struct, tc) +{ + struct bitstr_containing_struct { + bitstr_t bit_decl(bitstr, 8); + } test_struct; + + bit_nclear(test_struct.bitstr, 0, 8); +} + +BITSTRING_TC_DEFINE(bit_set) +/* bitstr_t *bitstr, int nbits, const char *memloc */ +{ + memset(bitstr, 0, bitstr_size(nbits)); + + for (int i = 0; i < nbits; i++) { + bit_set(bitstr, i); + + for (int j = 0; j < nbits; j++) { + ATF_REQUIRE_MSG(bit_test(bitstr, j) == (j == i) ? 1 : 0, + "bit_set_%d_%s: Failed on bit %d", + nbits, memloc, i); + } + + bit_clear(bitstr, i); + } +} + +BITSTRING_TC_DEFINE(bit_clear) +/* bitstr_t *bitstr, int nbits, const char *memloc */ +{ + int i, j; + + memset(bitstr, 0xFF, bitstr_size(nbits)); + for (i = 0; i < nbits; i++) { + bit_clear(bitstr, i); + + for (j = 0; j < nbits; j++) { + ATF_REQUIRE_MSG(bit_test(bitstr, j) == (j == i) ? 0 : 1, + "bit_clear_%d_%s: Failed on bit %d", + nbits, memloc, i); + } + + bit_set(bitstr, i); + } +} + +BITSTRING_TC_DEFINE(bit_ffs) +/* bitstr_t *bitstr, int nbits, const char *memloc */ +{ + int i; + int found_set_bit; + + memset(bitstr, 0, bitstr_size(nbits)); + bit_ffs(bitstr, nbits, &found_set_bit); + ATF_REQUIRE_MSG(found_set_bit == -1, + "bit_ffs_%d_%s: Failed all clear bits.", nbits, memloc); + + for (i = 0; i < nbits; i++) { + memset(bitstr, 0xFF, bitstr_size(nbits)); + if (i > 0) + bit_nclear(bitstr, 0, i - 1); + + bit_ffs(bitstr, nbits, &found_set_bit); + ATF_REQUIRE_MSG(found_set_bit == i, + "bit_ffs_%d_%s: Failed on bit %d, Result %d", + nbits, memloc, i, found_set_bit); + } +} + +BITSTRING_TC_DEFINE(bit_ffc) +/* bitstr_t *bitstr, int nbits, const char *memloc */ +{ + int i; + int found_clear_bit; + + memset(bitstr, 0xFF, bitstr_size(nbits)); + bit_ffc(bitstr, nbits, &found_clear_bit); + ATF_REQUIRE_MSG(found_clear_bit == -1, + "bit_ffc_%d_%s: Failed all set bits.", nbits, memloc); + + for (i = 0; i < nbits; i++) { + memset(bitstr, 0, bitstr_size(nbits)); + if (i > 0) + bit_nset(bitstr, 0, i - 1); + + bit_ffc(bitstr, nbits, &found_clear_bit); + ATF_REQUIRE_MSG(found_clear_bit == i, + "bit_ffc_%d_%s: Failed on bit %d, Result %d", + nbits, memloc, i, found_clear_bit); + } +} + +BITSTRING_TC_DEFINE(bit_ffs_at) +/* bitstr_t *bitstr, int nbits, const char *memloc */ +{ + int i; + int found_set_bit; + + memset(bitstr, 0xFF, bitstr_size(nbits)); + for (i = 0; i < nbits; i++) { + bit_ffs_at(bitstr, i, nbits, &found_set_bit); + ATF_REQUIRE_MSG(found_set_bit == i, + "bit_ffs_at_%d_%s: Failed on bit %d, Result %d", + nbits, memloc, i, found_set_bit); + } + + memset(bitstr, 0, bitstr_size(nbits)); + for (i = 0; i < nbits; i++) { + bit_ffs_at(bitstr, i, nbits, &found_set_bit); + ATF_REQUIRE_MSG(found_set_bit == -1, + "bit_ffs_at_%d_%s: Failed on bit %d, Result %d", + nbits, memloc, i, found_set_bit); + } + + memset(bitstr, 0x55, bitstr_size(nbits)); + for (i = 0; i < nbits; i++) { + bit_ffs_at(bitstr, i, nbits, &found_set_bit); + if (i == nbits - 1 && (nbits & 1) == 0) { + ATF_REQUIRE_MSG(found_set_bit == -1, + "bit_ffs_at_%d_%s: Failed on bit %d, Result %d", + nbits, memloc, i, found_set_bit); + } else { + ATF_REQUIRE_MSG(found_set_bit == i + (i & 1), + "bit_ffs_at_%d_%s: Failed on bit %d, Result %d", + nbits, memloc, i, found_set_bit); + } + } + + memset(bitstr, 0xAA, bitstr_size(nbits)); + for (i = 0; i < nbits; i++) { + bit_ffs_at(bitstr, i, nbits, &found_set_bit); + if (i == nbits - 1 && (nbits & 1) != 0) { + ATF_REQUIRE_MSG(found_set_bit == -1, + "bit_ffs_at_%d_%s: Failed on bit %d, Result %d", + nbits, memloc, i, found_set_bit); + } else { + ATF_REQUIRE_MSG( + found_set_bit == i + ((i & 1) ? 0 : 1), + "bit_ffs_at_%d_%s: Failed on bit %d, Result %d", + nbits, memloc, i, found_set_bit); + } + } +} + +BITSTRING_TC_DEFINE(bit_ffc_at) +/* bitstr_t *bitstr, int nbits, const char *memloc */ +{ + int i, found_clear_bit; + + memset(bitstr, 0, bitstr_size(nbits)); + for (i = 0; i < nbits; i++) { + bit_ffc_at(bitstr, i, nbits, &found_clear_bit); + ATF_REQUIRE_MSG(found_clear_bit == i, + "bit_ffc_at_%d_%s: Failed on bit %d, Result %d", + nbits, memloc, i, found_clear_bit); + } + + memset(bitstr, 0xFF, bitstr_size(nbits)); + for (i = 0; i < nbits; i++) { + bit_ffc_at(bitstr, i, nbits, &found_clear_bit); + ATF_REQUIRE_MSG(found_clear_bit == -1, + "bit_ffc_at_%d_%s: Failed on bit %d, Result %d", + nbits, memloc, i, found_clear_bit); + } + + memset(bitstr, 0x55, bitstr_size(nbits)); + for (i = 0; i < nbits; i++) { + bit_ffc_at(bitstr, i, nbits, &found_clear_bit); + if (i == nbits - 1 && (nbits & 1) != 0) { + ATF_REQUIRE_MSG(found_clear_bit == -1, + "bit_ffc_at_%d_%s: Failed on bit %d, Result %d", + nbits, memloc, i, found_clear_bit); + } else { + ATF_REQUIRE_MSG( + found_clear_bit == i + ((i & 1) ? 0 : 1), + "bit_ffc_at_%d_%s: Failed on bit %d, Result %d", + nbits, memloc, i, found_clear_bit); + } + } + + memset(bitstr, 0xAA, bitstr_size(nbits)); + for (i = 0; i < nbits; i++) { + bit_ffc_at(bitstr, i, nbits, &found_clear_bit); + if (i == nbits - 1 && (nbits & 1) == 0) { + ATF_REQUIRE_MSG(found_clear_bit == -1, + "bit_ffc_at_%d_%s: Failed on bit %d, Result %d", + nbits, memloc, i, found_clear_bit); + } else { + ATF_REQUIRE_MSG(found_clear_bit == i + (i & 1), + "bit_ffc_at_%d_%s: Failed on bit %d, Result %d", + nbits, memloc, i, found_clear_bit); + } + } +} + +BITSTRING_TC_DEFINE(bit_nclear) +/* bitstr_t *bitstr, int nbits, const char *memloc */ +{ + int i, j; + int found_set_bit; + int found_clear_bit; + + for (i = 0; i < nbits; i++) { + for (j = i; j < nbits; j++) { + memset(bitstr, 0xFF, bitstr_size(nbits)); + bit_nclear(bitstr, i, j); + + bit_ffc(bitstr, nbits, &found_clear_bit); + ATF_REQUIRE_MSG( + found_clear_bit == i, + "bit_nclear_%d_%d_%d%s: Failed with result %d", + nbits, i, j, memloc, found_clear_bit); + + bit_ffs_at(bitstr, i, nbits, &found_set_bit); + ATF_REQUIRE_MSG( + (j + 1 < nbits) ? found_set_bit == j + 1 : -1, + "bit_nset_%d_%d_%d%s: Failed with result %d", + nbits, i, j, memloc, found_set_bit); + } + } +} + +BITSTRING_TC_DEFINE(bit_nset) +/* bitstr_t *bitstr, int nbits, const char *memloc */ +{ + int i, j; + int found_set_bit; + int found_clear_bit; + + for (i = 0; i < nbits; i++) { + for (j = i; j < nbits; j++) { + memset(bitstr, 0, bitstr_size(nbits)); + bit_nset(bitstr, i, j); + + bit_ffs(bitstr, nbits, &found_set_bit); + ATF_REQUIRE_MSG( + found_set_bit == i, + "bit_nset_%d_%d_%d%s: Failed with result %d", + nbits, i, j, memloc, found_set_bit); + + bit_ffc_at(bitstr, i, nbits, &found_clear_bit); + ATF_REQUIRE_MSG( + (j + 1 < nbits) ? found_clear_bit == j + 1 : -1, + "bit_nset_%d_%d_%d%s: Failed with result %d", + nbits, i, j, memloc, found_clear_bit); + } + } +} + +ATF_TP_ADD_TCS(tp) +{ + + ATF_TP_ADD_TC(tp, bitstr_in_struct); + BITSTRING_TC_ADD(tp, bit_set); + BITSTRING_TC_ADD(tp, bit_clear); + BITSTRING_TC_ADD(tp, bit_ffs); + BITSTRING_TC_ADD(tp, bit_ffc); + BITSTRING_TC_ADD(tp, bit_ffs_at); + BITSTRING_TC_ADD(tp, bit_ffc_at); + BITSTRING_TC_ADD(tp, bit_nclear); + BITSTRING_TC_ADD(tp, bit_nset); + + return (atf_no_error()); +} Property changes on: user/ngie/release-pkg-fix-tests/tests/sys/sys/bitstring_test.c ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Added: svn:keywords ## -0,0 +1 ## +FreeBSD=%H \ No newline at end of property Added: svn:mime-type ## -0,0 +1 ## +text/plain \ No newline at end of property Index: user/ngie/release-pkg-fix-tests/tools/tools/locale/etc/charmaps.xml =================================================================== --- user/ngie/release-pkg-fix-tests/tools/tools/locale/etc/charmaps.xml (revision 299090) +++ user/ngie/release-pkg-fix-tests/tools/tools/locale/etc/charmaps.xml (revision 299091) @@ -1,743 +1,743 @@ "; ""; ""; ""; ""; ""; ""; ""; ""; ""; ""; "" ]]> "; ""; ""; ""; ""; ""; ""; "

"; ""; ""; "

"; "

" ]]> "; ""; ""; ""; ""; ""; ""; ""; ""; ""; ""; "" ]]> "; ""; ""; ""; ""; ""; ""; ""; ""; ""; ""; "" ]]> "; ""; ""; ""; ""; ""; "

"; "

"; ""; "

"; "

"; "" ]]> "; ""; ""; ""; ""; ""; ""; ""; ""; ""; ""; "" ]]> "; ""; ""; ""; ""; ""; "

"; "

"; ""; "

"; "

"; "" ]]> "; ""; ""; ""; ""; ""; ""; ""; ""; ""; ""; "" ]]> "; ""; ""; ""; ""; ""; ""; ""; ""; ""; ""; "" ]]> Index: user/ngie/release-pkg-fix-tests/usr.sbin/bluetooth/hccontrol/le.c =================================================================== --- user/ngie/release-pkg-fix-tests/usr.sbin/bluetooth/hccontrol/le.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/usr.sbin/bluetooth/hccontrol/le.c (revision 299091) @@ -1,356 +1,356 @@ /* * le.c * * Copyright (c) 2015 Takanori Watanabe * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $Id: hccontrol.c,v 1.5 2003/09/05 00:38:24 max Exp $ * $FreeBSD$ */ #include #include #include -#include #include #include +#include #include #include #include #include #include #include #include #include #include #define L2CAP_SOCKET_CHECKED #include #include "hccontrol.h" static int le_set_scan_param(int s, int argc, char *argv[]); static int le_set_scan_enable(int s, int argc, char *argv[]); static int parse_param(int argc, char *argv[], char *buf, int *len); static int le_set_scan_response(int s, int argc, char *argv[]); static int le_read_supported_status(int s, int argc, char *argv[]); static int le_read_local_supported_features(int s, int argc ,char *argv[]); static int set_le_event_mask(int s, uint64_t mask); static int set_event_mask(int s, uint64_t mask); static int le_enable(int s, int argc, char *argv[]); static int le_set_scan_param(int s, int argc, char *argv[]) { int type; int interval; int window; int adrtype; int policy; int e, n; ng_hci_le_set_scan_parameters_cp cp; ng_hci_le_set_scan_parameters_rp rp; if (argc != 5) return USAGE; if (strcmp(argv[0], "active") == 0) type = 1; else if (strcmp(argv[0], "passive") == 0) type = 0; else return USAGE; interval = (int)(atof(argv[1])/0.625); interval = (interval < 4)? 4: interval; window = (int)(atof(argv[2])/0.625); window = (window < 4) ? 4 : interval; if (strcmp(argv[3], "public") == 0) adrtype = 0; else if (strcmp(argv[3], "random") == 0) adrtype = 1; else return USAGE; if (strcmp(argv[4], "all") == 0) policy = 0; else if (strcmp(argv[4], "whitelist") == 0) policy = 1; else return USAGE; cp.le_scan_type = type; cp.le_scan_interval = interval; cp.own_address_type = adrtype; cp.le_scan_window = window; cp.scanning_filter_policy = policy; n = sizeof(rp); e = hci_request(s, NG_HCI_OPCODE(NG_HCI_OGF_LE, NG_HCI_OCF_LE_SET_SCAN_PARAMETERS), (void *)&cp, sizeof(cp), (void *)&rp, &n); return 0; } static int le_set_scan_enable(int s, int argc, char *argv[]) { ng_hci_le_set_scan_enable_cp cp; ng_hci_le_set_scan_enable_rp rp; int e, n, enable = 0; if (argc != 1) return USAGE; if (strcmp(argv[0], "enable") == 0) enable = 1; else if (strcmp(argv[0], "disable") != 0) return USAGE; n = sizeof(rp); cp.le_scan_enable = enable; cp.filter_duplicates = 0; e = hci_request(s, NG_HCI_OPCODE(NG_HCI_OGF_LE, NG_HCI_OCF_LE_SET_SCAN_ENABLE), (void *)&cp, sizeof(cp), (void *)&rp, &n); if (e != 0 || rp.status != 0) return ERROR; return OK; } static int parse_param(int argc, char *argv[], char *buf, int *len) { char *buflast = buf + (*len); char *curbuf = buf; char *token,*lenpos; int ch; int datalen; uint16_t value; optreset = 1; optind = 0; while ((ch = getopt(argc, argv , "n:f:u:")) != -1) { switch(ch){ case 'n': datalen = strlen(optarg); if ((curbuf + datalen + 2) >= buflast) goto done; curbuf[0] = datalen + 1; curbuf[1] = 8; curbuf += 2; memcpy(curbuf, optarg, datalen); curbuf += datalen; break; case 'f': if (curbuf+3 > buflast) goto done; curbuf[0] = 2; curbuf[1] = 1; curbuf[2] = atoi(optarg); curbuf += 3; break; case 'u': lenpos = buf; if ((buf+2) >= buflast) goto done; curbuf[1] = 2; *lenpos = 1; curbuf += 2; while ((token = strsep(&optarg, ",")) != NULL) { value = strtol(token, NULL, 16); if ((curbuf+2) >= buflast) break; curbuf[0] = value &0xff; curbuf[1] = (value>>8)&0xff; curbuf += 2; } } } done: *len = curbuf - buf; return OK; } static int le_set_scan_response(int s, int argc, char *argv[]) { ng_hci_le_set_scan_response_data_cp cp; ng_hci_le_set_scan_response_data_rp rp; int n; int e; int len; char buf[NG_HCI_ADVERTISING_DATA_SIZE]; len = sizeof(buf); parse_param(argc, argv, buf, &len); memset(cp.scan_response_data, 0, sizeof(cp.scan_response_data)); cp.scan_response_data_length = len; memcpy(cp.scan_response_data, buf, len); n = sizeof(rp); e = hci_request(s, NG_HCI_OPCODE(NG_HCI_OGF_LE, NG_HCI_OCF_LE_SET_SCAN_RESPONSE_DATA), (void *)&cp, sizeof(cp), (void *)&rp, &n); printf("SET SCAN RESPONSE %d %d %d\n", e, rp.status, n); return OK; } static int le_read_local_supported_features(int s, int argc ,char *argv[]) { ng_hci_le_read_local_supported_features_rp rp; int e; int n = sizeof(rp); e = hci_simple_request(s, NG_HCI_OPCODE(NG_HCI_OGF_LE, NG_HCI_OCF_LE_READ_LOCAL_SUPPORTED_FEATURES), (void *)&rp, &n); printf("LOCAL SUPPORTED: %d %d %jx\n", e, rp.status, (uintmax_t) rp.le_features); return 0; } static int le_read_supported_status(int s, int argc, char *argv[]) { ng_hci_le_read_supported_status_rp rp; int e; int n = sizeof(rp); e = hci_simple_request(s, NG_HCI_OPCODE( NG_HCI_OGF_LE, NG_HCI_OCF_LE_READ_SUPPORTED_STATUS), (void *)&rp, &n); printf("LE_STATUS: %d %d %jx\n", e, rp.status, (uintmax_t)rp.le_status); return 0; } static int set_le_event_mask(int s, uint64_t mask) { ng_hci_le_set_event_mask_cp semc; ng_hci_le_set_event_mask_rp rp; int i, n ,e; n = sizeof(rp); for (i=0; i < NG_HCI_LE_EVENT_MASK_SIZE; i++) { semc.event_mask[i] = mask&0xff; mask >>= 8; } e = hci_request(s, NG_HCI_OPCODE(NG_HCI_OGF_LE, NG_HCI_OCF_LE_SET_EVENT_MASK), (void *)&semc, sizeof(semc), (void *)&rp, &n); return 0; } static int set_event_mask(int s, uint64_t mask) { ng_hci_set_event_mask_cp semc; ng_hci_set_event_mask_rp rp; int i, n, e; n = sizeof(rp); for (i=0; i < NG_HCI_EVENT_MASK_SIZE; i++) { semc.event_mask[i] = mask&0xff; mask >>= 8; } e = hci_request(s, NG_HCI_OPCODE(NG_HCI_OGF_HC_BASEBAND, NG_HCI_OCF_SET_EVENT_MASK), (void *)&semc, sizeof(semc), (void *)&rp, &n); return 0; } static int le_enable(int s, int argc, char *argv[]) { if (argc != 1) return USAGE; if (strcasecmp(argv[0], "enable") == 0) { set_event_mask(s, NG_HCI_EVENT_MASK_DEFAULT | NG_HCI_EVENT_MASK_LE); set_le_event_mask(s, NG_HCI_LE_EVENT_MASK_ALL); } else if (strcasecmp(argv[0], "disble") == 0) set_event_mask(s, NG_HCI_EVENT_MASK_DEFAULT); else return USAGE; return OK; } struct hci_command le_commands[] = { { "le_enable", "le_enable [enable|disable] \n" "Enable LE event ", &le_enable, }, { "le_read_local_supported_features", "le_read_local_supported_features\n" "read local supported features mask", &le_read_local_supported_features, }, { "le_read_supported_status", "le_read_supported_status\n" "read supported status" , &le_read_supported_status, }, { "le_set_scan_response", "le_set_scan_response -n $name -f $flag -u $uuid16,$uuid16 \n" "set LE scan response data" , &le_set_scan_response, }, { "le_set_scan_enable", "le_set_scan_enable [enable|disable] \n" "enable or disable LE device scan", &le_set_scan_enable }, { "le_set_scan_param", "le_set_scan_param [active|passive] interval(ms) window(ms) [public|random] [all|whitelist] \n" "set LE device scan parameter", &le_set_scan_param }, }; Index: user/ngie/release-pkg-fix-tests/usr.sbin/extattr/Makefile =================================================================== --- user/ngie/release-pkg-fix-tests/usr.sbin/extattr/Makefile (revision 299090) +++ user/ngie/release-pkg-fix-tests/usr.sbin/extattr/Makefile (revision 299091) @@ -1,20 +1,22 @@ # $FreeBSD$ PROG= rmextattr MAN= rmextattr.8 +LIBADD= sbuf + LINKS+= ${BINDIR}/rmextattr ${BINDIR}/getextattr LINKS+= ${BINDIR}/rmextattr ${BINDIR}/setextattr LINKS+= ${BINDIR}/rmextattr ${BINDIR}/lsextattr MLINKS+= rmextattr.8 setextattr.8 MLINKS+= rmextattr.8 getextattr.8 MLINKS+= rmextattr.8 lsextattr.8 .include .if ${MK_TESTS} != "no" SUBDIR+= tests .endif .include Index: user/ngie/release-pkg-fix-tests/usr.sbin/extattr/rmextattr.8 =================================================================== --- user/ngie/release-pkg-fix-tests/usr.sbin/extattr/rmextattr.8 (revision 299090) +++ user/ngie/release-pkg-fix-tests/usr.sbin/extattr/rmextattr.8 (revision 299091) @@ -1,135 +1,142 @@ .\"- .\" Copyright (c) 2000, 2001 Robert N. M. Watson .\" Copyright (c) 2002 Networks Associates Technology, Inc. .\" All rights reserved. .\" .\" This software was developed for the FreeBSD Project by Poul-Henning .\" Kamp and Network Associates Laboratories, the Security Research Division .\" of Network Associates, Inc. under DARPA/SPAWAR contract N66001-01-C-8035 .\" ("CBOSS"), as part of the DARPA CHATS research program .\" .\" Redistribution and use in source and binary forms, with or without .\" modification, are permitted provided that the following conditions .\" are met: .\" 1. Redistributions of source code must retain the above copyright .\" notice, this list of conditions and the following disclaimer. .\" 2. Redistributions in binary form must reproduce the above copyright .\" notice, this list of conditions and the following disclaimer in the .\" documentation and/or other materials provided with the distribution. .\" .\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND .\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE .\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE .\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE .\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL .\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS .\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) .\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT .\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF .\" SUCH DAMAGE. .\" .\" $FreeBSD$ .\" -.Dd August 30, 2000 +.Dd April 27, 2016 .Dt RMEXTATTR 8 .Os .Sh NAME .Nm getextattr , .Nm lsextattr , .Nm rmextattr , .Nm setextattr .Nd manipulate extended attributes .Sh SYNOPSIS .Nm getextattr .Op Fl fhqsx .Ar attrnamespace .Ar attrname .Ar filename ... .Nm lsextattr .Op Fl fhq .Ar attrnamespace .Ar filename ... .Nm rmextattr .Op Fl fhq .Ar attrnamespace .Ar attrname .Ar filename ... .Nm setextattr .Op Fl fhnq .Ar attrnamespace .Ar attrname .Ar attrvalue .Ar filename ... +.Nm setextattr +.Fl i +.Op Fl fhnq +.Ar attrnamespace +.Ar attrname +.Ar filename ... .Sh DESCRIPTION These utilities are user tools to manipulate the named extended attributes on files and directories. The .Ar attrnamespace argument should be the namespace of the attribute to retrieve: legal values are .Cm user and .Cm system . The .Ar attrname argument should be the name of the attribute, .Ar filename the name of the target file or directory, .Ar attrvalue a string to store in the attribute. .Pp The following options are available: .Bl -tag -width indent .It Fl f (Force.) Ignore errors on individual filenames and continue with the remaining arguments. .It Fl h (No follow.) If the file is a symbolic link, perform the operation on the link itself rather than the file that the link points to. +.It Fl i +(From stdin.) +Read attribute data from stdin instead of as an argument. .It Fl n .Dv ( NUL Ns -terminate.) .Dv NUL Ns -terminate the extent content written out. .It Fl q (Quiet.) Do not print out the pathname and suppress error messages. +When given twice, do not print a trailing newline. .It Fl s (Stringify.) Escape nonprinting characters and put quotes around the output. .It Fl x (Hex.) Print the output in hexadecimal. .El .Sh EXAMPLES .Bd -literal setextattr system md5 `md5 -q /boot/kernel/kernel` /boot/kernel/kernel +md5 -q /boot/kernel/kernel | setextattr -i system md5 /boot/kernel/kernel getextattr system md5 /boot/kernel/kernel lsextattr system /boot/kernel/kernel rmextattr system md5 /boot/kernel/kernel .Ed .Sh SEE ALSO .Xr extattr 2 , .Xr extattr 3 , .Xr extattrctl 8 , .Xr extattr 9 .Sh HISTORY Extended attribute support was developed as part of the .Tn TrustedBSD Project, and introduced in .Fx 5.0 . It was developed to support security extensions requiring additional labels to be associated with each file or directory. .Sh AUTHORS .An Robert N M Watson .An Poul-Henning Kamp -.Sh BUGS -The -.Nm setextattr -utility can only be used to set attributes to strings. Index: user/ngie/release-pkg-fix-tests/usr.sbin/extattr/rmextattr.c =================================================================== --- user/ngie/release-pkg-fix-tests/usr.sbin/extattr/rmextattr.c (revision 299090) +++ user/ngie/release-pkg-fix-tests/usr.sbin/extattr/rmextattr.c (revision 299091) @@ -1,296 +1,315 @@ /*- * Copyright (c) 2002, 2003 Networks Associates Technology, Inc. * Copyright (c) 2002 Poul-Henning Kamp. * Copyright (c) 1999, 2000, 2001, 2002 Robert N. M. Watson * All rights reserved. * * This software was developed for the FreeBSD Project by Poul-Henning * Kamp and Network Associates Laboratories, the Security Research Division * of Network Associates, Inc. under DARPA/SPAWAR contract N66001-01-C-8035 * ("CBOSS"), as part of the DARPA CHATS research program * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The names of the authors may not be used to endorse or promote * products derived from this software without specific prior written * permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * * $FreeBSD$ */ #include +#include #include #include #include #include #include #include #include #include #include #include #include static enum { EADUNNO, EAGET, EASET, EARM, EALS } what = EADUNNO; static void __dead2 usage(void) { switch (what) { case EAGET: fprintf(stderr, "usage: getextattr [-fhqsx] attrnamespace"); fprintf(stderr, " attrname filename ...\n"); exit(-1); case EASET: fprintf(stderr, "usage: setextattr [-fhnq] attrnamespace"); fprintf(stderr, " attrname attrvalue filename ...\n"); + fprintf(stderr, " or setextattr -i [-fhnq] attrnamespace"); + fprintf(stderr, " attrname filename ...\n"); exit(-1); case EARM: fprintf(stderr, "usage: rmextattr [-fhq] attrnamespace"); fprintf(stderr, " attrname filename ...\n"); exit(-1); case EALS: fprintf(stderr, "usage: lsextattr [-fhq] attrnamespace"); fprintf(stderr, " filename ...\n"); exit(-1); case EADUNNO: default: fprintf(stderr, "usage: (getextattr|lsextattr|rmextattr"); fprintf(stderr, "|setextattr)\n"); exit (-1); } } static void mkbuf(char **buf, int *oldlen, int newlen) { if (*oldlen >= newlen) return; if (*buf != NULL) free(*buf); *buf = malloc(newlen); if (*buf == NULL) err(1, "malloc"); *oldlen = newlen; return; } int main(int argc, char *argv[]) { - char *buf, *visbuf, *p; +#define STDIN_BUF_SZ 1024 + char stdin_data[STDIN_BUF_SZ]; + char *p; const char *options, *attrname; size_t len; ssize_t ret; - int buflen, visbuflen, ch, error, i, arg_counter, attrnamespace, - minargc; + int ch, error, i, arg_counter, attrnamespace, minargc; + char *visbuf = NULL; + int visbuflen = 0; + char *buf = NULL; + int buflen = 0; + struct sbuf *attrvalue = NULL; int flag_force = 0; int flag_nofollow = 0; int flag_null = 0; - int flag_quiet = 0; + int count_quiet = 0; + int flag_from_stdin = 0; int flag_string = 0; int flag_hex = 0; - visbuflen = buflen = 0; - visbuf = buf = NULL; - p = basename(argv[0]); if (p == NULL) p = argv[0]; if (!strcmp(p, "getextattr")) { what = EAGET; options = "fhqsx"; minargc = 3; } else if (!strcmp(p, "setextattr")) { what = EASET; - options = "fhnq"; - minargc = 4; + options = "fhinq"; + minargc = 3; } else if (!strcmp(p, "rmextattr")) { what = EARM; options = "fhq"; minargc = 3; } else if (!strcmp(p, "lsextattr")) { what = EALS; options = "fhq"; minargc = 2; } else { usage(); } while ((ch = getopt(argc, argv, options)) != -1) { switch (ch) { case 'f': flag_force = 1; break; case 'h': flag_nofollow = 1; break; + case 'i': + flag_from_stdin = 1; + break; case 'n': flag_null = 1; break; case 'q': - flag_quiet = 1; + count_quiet += 1; break; case 's': flag_string = 1; break; case 'x': flag_hex = 1; break; case '?': default: usage(); } } argc -= optind; argv += optind; + if (what == EASET && flag_from_stdin == 0) + minargc++; + if (argc < minargc) usage(); error = extattr_string_to_namespace(argv[0], &attrnamespace); if (error) err(-1, "%s", argv[0]); argc--; argv++; if (what != EALS) { attrname = argv[0]; argc--; argv++; } else attrname = NULL; if (what == EASET) { - mkbuf(&buf, &buflen, strlen(argv[0]) + 1); - strcpy(buf, argv[0]); - argc--; argv++; + attrvalue = sbuf_new_auto(); + if (flag_from_stdin) { + while ((error = read(0, stdin_data, STDIN_BUF_SZ)) > 0) + sbuf_bcat(attrvalue, stdin_data, error); + } else { + sbuf_cpy(attrvalue, argv[0]); + argc--; argv++; + } + sbuf_finish(attrvalue); } for (arg_counter = 0; arg_counter < argc; arg_counter++) { switch (what) { case EARM: if (flag_nofollow) error = extattr_delete_link(argv[arg_counter], attrnamespace, attrname); else error = extattr_delete_file(argv[arg_counter], attrnamespace, attrname); if (error >= 0) continue; break; case EASET: - len = strlen(buf) + flag_null; + len = sbuf_len(attrvalue) + flag_null; if (flag_nofollow) ret = extattr_set_link(argv[arg_counter], - attrnamespace, attrname, buf, len); + attrnamespace, attrname, + sbuf_data(attrvalue), len); else ret = extattr_set_file(argv[arg_counter], - attrnamespace, attrname, buf, len); + attrnamespace, attrname, + sbuf_data(attrvalue), len); if (ret >= 0) { - if ((size_t)ret != len && !flag_quiet) { + if ((size_t)ret != len && !count_quiet) { warnx("Set %zd bytes of %zu for %s", ret, len, attrname); } continue; } break; case EALS: if (flag_nofollow) ret = extattr_list_link(argv[arg_counter], attrnamespace, NULL, 0); else ret = extattr_list_file(argv[arg_counter], attrnamespace, NULL, 0); if (ret < 0) break; mkbuf(&buf, &buflen, ret); if (flag_nofollow) ret = extattr_list_link(argv[arg_counter], attrnamespace, buf, buflen); else ret = extattr_list_file(argv[arg_counter], attrnamespace, buf, buflen); if (ret < 0) break; - if (!flag_quiet) + if (!count_quiet) printf("%s\t", argv[arg_counter]); for (i = 0; i < ret; i += ch + 1) { /* The attribute name length is unsigned. */ ch = (unsigned char)buf[i]; printf("%s%*.*s", i ? "\t" : "", ch, ch, buf + i + 1); } - if (!flag_quiet || ret > 0) + if (!count_quiet || ret > 0) printf("\n"); continue; case EAGET: if (flag_nofollow) ret = extattr_get_link(argv[arg_counter], attrnamespace, attrname, NULL, 0); else ret = extattr_get_file(argv[arg_counter], attrnamespace, attrname, NULL, 0); if (ret < 0) break; mkbuf(&buf, &buflen, ret); if (flag_nofollow) ret = extattr_get_link(argv[arg_counter], attrnamespace, attrname, buf, buflen); else ret = extattr_get_file(argv[arg_counter], attrnamespace, attrname, buf, buflen); if (ret < 0) break; - if (!flag_quiet) + if (!count_quiet) printf("%s\t", argv[arg_counter]); if (flag_string) { mkbuf(&visbuf, &visbuflen, ret * 4 + 1); strvisx(visbuf, buf, ret, VIS_SAFE | VIS_WHITE); - printf("\"%s\"\n", visbuf); - continue; + printf("\"%s\"", visbuf); } else if (flag_hex) { for (i = 0; i < ret; i++) printf("%s%02x", i ? " " : "", - buf[i]); - printf("\n"); - continue; + (unsigned char)buf[i]); } else { fwrite(buf, ret, 1, stdout); - printf("\n"); - continue; } + if (count_quiet < 2) + printf("\n"); + continue; default: break; } - if (!flag_quiet) + if (!count_quiet) warn("%s: failed", argv[arg_counter]); if (flag_force) continue; return(1); } return (0); } Index: user/ngie/release-pkg-fix-tests/usr.sbin/extattr/tests/extattr_test.sh =================================================================== --- user/ngie/release-pkg-fix-tests/usr.sbin/extattr/tests/extattr_test.sh (revision 299090) +++ user/ngie/release-pkg-fix-tests/usr.sbin/extattr/tests/extattr_test.sh (revision 299091) @@ -1,290 +1,335 @@ # # Copyright (c) 2016 Spectra Logic Corp # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # $FreeBSD$ atf_test_case bad_namespace bad_namespace_head() { atf_set "descr" "Can't set attributes for nonexistent namespaces" } bad_namespace_body() { touch foo atf_check -s not-exit:0 -e match:"Invalid argument" \ setextattr badnamespace myattr X foo atf_check -s not-exit:0 -e match:"Invalid argument" \ lsextattr -q badnamespace foo } atf_test_case hex hex_head() { atf_set "descr" "Set and get attribute values in hexadecimal" } hex_body() { touch foo - atf_check -s exit:0 -o empty setextattr user myattr1 XYZ foo + atf_check -s exit:0 -o empty setextattr user myattr XYZ foo atf_check -s exit:0 -o inline:"58 59 5a\n" \ - getextattr -qx user myattr1 foo + getextattr -qx user myattr foo } +atf_test_case hex_nonascii +hex_nonascii_head() { + atf_set "descr" "Get binary attribute values in hexadecimal" +} +hex_nonascii_body() { + touch foo + BINSTUFF=`echo $'\x20\x30\x40\x55\x66\x70\x81\xa2\xb3\xee\xff'` + atf_check -s exit:0 -o empty setextattr user myattr "$BINSTUFF" foo + getextattr user myattr foo + atf_check -s exit:0 -o inline:"20 30 40 55 66 70 81 a2 b3 ee ff\n" \ + getextattr -qx user myattr foo +} + atf_test_case long_name long_name_head() { atf_set "descr" "A maximum length attribute name" } long_name_body() { # https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=208965 atf_expect_fail "BUG 208965 extattr(2) doesn't allow maxlen attr names" ATTRNAME=`jot -b X -s "" 255 0` touch foo atf_check -s exit:0 -o empty setextattr user $ATTRNAME myvalue foo atf_check -s exit:0 -o inline:"${ATTRNAME}\n" lsextattr -q user foo atf_check -s exit:0 -o inline:"myvalue\n" \ getextattr -q user ${ATTRNAME} foo atf_check -s exit:0 -o empty rmextattr user ${ATTRNAME} foo atf_check -s exit:0 -o empty lsextattr -q user foo } +atf_test_case loud +loud_head() { + atf_set "descr" "Loud (non -q) output for each command" +} +loud_body() { + touch foo + # setextattr(8) and friends print hard tabs. Use printf to convert + # them to spaces before checking the output. + atf_check -s exit:0 -o empty setextattr user myattr myvalue foo + atf_check -s exit:0 -o inline:"foo myattr" \ + printf "%s %s" $(lsextattr user foo) + atf_check -s exit:0 -o inline:"foo myvalue" \ + printf "%s %s" $(getextattr user myattr foo) + atf_check -s exit:0 -o empty rmextattr user myattr foo + atf_check -s exit:0 -o inline:"foo" printf %s $(lsextattr user foo) +} + atf_test_case noattrs noattrs_head() { atf_set "descr" "A file with no extended attributes" } noattrs_body() { touch foo atf_check -s exit:0 -o empty lsextattr -q user foo } atf_test_case nonexistent_file nonexistent_file_head() { atf_set "descr" "A file that does not exist" } nonexistent_file_body() { atf_check -s exit:1 -e match:"No such file or directory" \ lsextattr user foo atf_check -s exit:1 -e match:"No such file or directory" \ setextattr user myattr myvalue foo atf_check -s exit:1 -e match:"No such file or directory" \ getextattr user myattr foo atf_check -s exit:1 -e match:"No such file or directory" \ rmextattr user myattr foo } atf_test_case null null_head() { atf_set "descr" "NUL-terminate an attribute value" } null_body() { touch foo atf_check -s exit:0 -o empty setextattr -n user myattr myvalue foo atf_check -s exit:0 -o inline:"myvalue\0\n" getextattr -q user myattr foo } atf_test_case one_user_attr one_user_attr_head() { atf_set "descr" "A file with one extended attribute" } one_user_attr_body() { touch foo atf_check -s exit:0 -o empty setextattr user myattr myvalue foo atf_check -s exit:0 -o inline:"myattr\n" lsextattr -q user foo atf_check -s exit:0 -o inline:"myvalue\n" getextattr -q user myattr foo atf_check -s exit:0 -o empty rmextattr user myattr foo atf_check -s exit:0 -o empty lsextattr -q user foo } atf_test_case one_system_attr one_system_attr_head() { atf_set "descr" "A file with one extended attribute" atf_set "require.user" "root" } one_system_attr_body() { touch foo atf_check -s exit:0 -o empty setextattr system myattr myvalue foo atf_check -s exit:0 -o inline:"myattr\n" lsextattr -q system foo atf_check -s exit:0 -o inline:"myvalue\n" getextattr -q system myattr foo atf_check -s exit:0 -o empty rmextattr system myattr foo atf_check -s exit:0 -o empty lsextattr -q system foo } +atf_test_case stdin +stdin_head() { + atf_set "descr" "Set attribute value from stdin" +} +stdin_body() { + dd if=/dev/random of=infile bs=1k count=8 + touch foo + setextattr -i user myattr foo < infile || atf_fail "setextattr failed" + atf_check -s exit:0 -o inline:"myattr\n" lsextattr -q user foo + getextattr -qq user myattr foo > outfile || atf_fail "getextattr failed" + atf_check -s exit:0 cmp -s infile outfile +} + atf_test_case stringify stringify_head() { atf_set "descr" "Stringify the output of getextattr" } stringify_body() { touch foo atf_check -s exit:0 -o empty setextattr user myattr "my value" foo atf_check -s exit:0 -o inline:"\"my\\\040value\"\n" \ getextattr -qs user myattr foo } atf_test_case symlink symlink_head() { atf_set "descr" "A symlink to an ordinary file" } symlink_body() { touch foo ln -s foo foolink atf_check -s exit:0 -o empty setextattr user myattr myvalue foolink atf_check -s exit:0 -o inline:"myvalue\n" \ getextattr -q user myattr foolink atf_check -s exit:0 -o inline:"myvalue\n" getextattr -q user myattr foo } atf_test_case symlink_nofollow symlink_nofollow_head() { atf_set "descr" "Operating directly on a symlink" } symlink_nofollow_body() { touch foo ln -s foo foolink # Check that with -h we can operate directly on the link atf_check -s exit:0 -o empty setextattr -h user myattr myvalue foolink atf_check -s exit:0 -o inline:"myvalue\n" \ getextattr -qh user myattr foolink atf_check -s exit:1 -e match:"Attribute not found" \ getextattr user myattr foolink atf_check -s exit:1 -e match:"Attribute not found" \ getextattr user myattr foo # Check that with -h we cannot operate on the destination file atf_check -s exit:0 -o empty setextattr user otherattr othervalue foo atf_check -s exit:1 getextattr -qh user otherattr foolink } atf_test_case system_and_user_attrs system_and_user_attrs_head() { atf_set "descr" "A file with both system and user extended attributes" atf_set "require.user" "root" } system_and_user_attrs_body() { touch foo atf_check -s exit:0 -o empty setextattr user userattr userval foo atf_check -s exit:0 -o empty setextattr system sysattr sysval foo atf_check -s exit:0 -o inline:"userattr\n" lsextattr -q user foo atf_check -s exit:0 -o inline:"sysattr\n" lsextattr -q system foo atf_check -s exit:0 -o inline:"userval\n" getextattr -q user userattr foo atf_check -s exit:0 -o inline:"sysval\n" getextattr -q system sysattr foo atf_check -s exit:0 -o empty rmextattr user userattr foo atf_check -s exit:0 -o empty rmextattr system sysattr foo atf_check -s exit:0 -o empty lsextattr -q user foo atf_check -s exit:0 -o empty lsextattr -q system foo } atf_test_case two_files two_files_head() { atf_set "descr" "Manipulate two files" } two_files_body() { touch foo bar atf_check -s exit:0 -o empty setextattr user myattr myvalue foo bar atf_check -s exit:0 -o inline:"foo\tmyattr\nbar\tmyattr\n" \ lsextattr user foo bar atf_check -s exit:0 \ -o inline:"foo\tmyvalue\nbar\tmyvalue\n" \ getextattr user myattr foo bar atf_check -s exit:0 -o empty rmextattr user myattr foo bar atf_check -s exit:0 -o empty lsextattr -q user foo bar } atf_test_case two_files_force two_files_force_head() { atf_set "descr" "Manipulate two files. The first does not exist" } two_files_force_body() { touch bar atf_check -s exit:1 -e match:"No such file or directory" \ setextattr user myattr myvalue foo bar atf_check -s exit:0 -e ignore setextattr -f user myattr myvalue foo bar atf_check -s exit:1 -e match:"No such file or directory" \ lsextattr user foo bar atf_check -s exit:0 -e ignore -o inline:"bar\tmyattr\n" \ lsextattr -f user foo bar atf_check -s exit:1 -e match:"No such file or directory" \ getextattr user myattr foo bar atf_check -s exit:0 -e ignore \ -o inline:"bar\tmyvalue\n" \ getextattr -f user myattr foo bar atf_check -s exit:1 -e match:"No such file or directory" \ rmextattr user myattr foo bar atf_check -s exit:0 -e ignore \ rmextattr -f user myattr foo bar atf_check -s exit:0 -o empty lsextattr -q user bar } atf_test_case two_user_attrs two_user_attrs_head() { atf_set "descr" "A file with two extended attributes" } two_user_attrs_body() { touch foo atf_check -s exit:0 -o empty setextattr user myattr1 myvalue1 foo atf_check -s exit:0 -o empty setextattr user myattr2 myvalue2 foo # lsextattr could return the attributes in any order, so we must be # careful how we compare them. raw_output=`lsextattr -q user foo` || atf_fail "lsextattr failed" tabless_output=`printf "%s %s" ${raw_output}` if [ "myattr1 myattr2" != "${tabless_output}" -a \ "myattr2 myattr1" != "${tabless_output}" ]; then atf_fail "lsextattr printed ${tabless_output}" fi atf_check -s exit:0 -o inline:"myvalue1\n" getextattr -q user myattr1 foo atf_check -s exit:0 -o inline:"myvalue2\n" getextattr -q user myattr2 foo atf_check -s exit:0 -o empty rmextattr user myattr2 foo atf_check -s exit:0 -o empty rmextattr user myattr1 foo atf_check -s exit:0 -o empty lsextattr -q user foo } atf_test_case unprivileged_user_cannot_set_system_attr unprivileged_user_cannot_set_system_attr_head() { atf_set "descr" "Unprivileged users can't set system attributes" atf_set "require.user" "unprivileged" } unprivileged_user_cannot_set_system_attr_body() { touch foo atf_check -s exit:1 -e match:"Operation not permitted" \ setextattr system myattr myvalue foo } atf_init_test_cases() { - # TODO: add test cases for verbose output (without -q) atf_add_test_case bad_namespace atf_add_test_case hex + atf_add_test_case hex_nonascii atf_add_test_case long_name + atf_add_test_case loud atf_add_test_case noattrs atf_add_test_case nonexistent_file atf_add_test_case null atf_add_test_case symlink_nofollow atf_add_test_case one_user_attr atf_add_test_case one_system_attr + atf_add_test_case stdin atf_add_test_case stringify atf_add_test_case symlink atf_add_test_case symlink_nofollow atf_add_test_case system_and_user_attrs atf_add_test_case two_files atf_add_test_case two_files_force atf_add_test_case two_user_attrs atf_add_test_case unprivileged_user_cannot_set_system_attr } Index: user/ngie/release-pkg-fix-tests =================================================================== --- user/ngie/release-pkg-fix-tests (revision 299090) +++ user/ngie/release-pkg-fix-tests (revision 299091) Property changes on: user/ngie/release-pkg-fix-tests ___________________________________________________________________ Modified: svn:mergeinfo ## -0,0 +0,1 ## Merged /head:r299055-299090